data.py

# Imports

from datasets import load_dataset
from huggingface_hub import login

from settings import *


# Load Data

NUMERIC_PROFILE = ["Laser Cutting",	"Wood Working",	"Wood CNC",	"Metal Machining",	"Metal CNC",	"3D Printer",	"Welding",	"Electronics"]

login(HF_TOKEN)

def load_data_from_huggingface():
    """
    Loads data from HuggingFace.
    """
    # Staff (People)
    ds_staff = load_dataset(REPO_ID_TECHSPARK_STAFF)
    staff_df = ds_staff["train"].to_pandas()

    # Courses
    ds_courses = load_dataset(REPO_ID_TECHSPARK_COURSES)
    courses_df = ds_courses["train"].to_pandas()

    # Tools
    ds_tools = load_dataset(REPO_ID_TECHSPARK_TOOLS)
    tools_df = ds_tools["train"].to_pandas()

    # Map Nodes
    ds_nodes = load_dataset(REPO_ID_TECHSPARK_MAP_NODES)
    nodes_df = ds_nodes["train"].to_pandas()

    # Map Edges
    ds_edges = load_dataset(REPO_ID_TECHSPARK_MAP_EDGES)
    edges_df = ds_edges["train"].to_pandas()

    return staff_df, courses_df, tools_df, nodes_df, edges_df

staff_df, courses_df, tools_df, nodes_df, edges_df = load_data_from_huggingface()