utils/loaders.py

import pandas as pd
import os
import streamlit as st
from datasets import load_dataset
from dotenv import load_dotenv

load_dotenv()
HF_TOKEN = os.getenv("HF_TOKEN")
REPO_NAME = os.getenv("DATA_REPO")
DATA_FILES = os.getenv("GEMINI_DATA_FILES")


@st.cache_data
def load_data():
    try:
        data = pd.read_csv("data/user-evaluation/merged.csv")
        return data
    except Exception as e:

        print("data not found, loading from huggingface dataset")

        dataset = load_dataset(REPO_NAME, token=True, data_files=DATA_FILES, revision="main")
        dataset.set_format(type='pandas')  ## converting it into pandas
        df = dataset["train"][:]
        return df[:60]


def load_html(file_name):
    with open(file_name, 'r') as file:
        return file.read()