# uses the yaml file generated by explore_ds.py

import yaml
import csv

# Define file paths
yaml_file_path = 'arxiv_papers_for_llm.yaml'  # Path to your input YAML file
csv_file_path = 'arxiv_papers_for_llm.csv'  # Path to your output CSV file

# Load the YAML data
with open(yaml_file_path, 'r', encoding='utf-8') as yaml_file:
    data = yaml.safe_load(yaml_file)

# Add paper_id to each entry if it's not already present
for idx, entry in enumerate(data, start=1):
    if 'paper_id' not in entry:  # Only add paper_id if it's not already in the entry
        entry['paper_id'] = idx

# Write data to a CSV file
with open(csv_file_path, 'w', newline='', encoding='utf-8') as csv_file:
    fieldnames = list(data[0].keys())
    if 'paper_id' not in fieldnames:
        fieldnames = ['paper_id'] + fieldnames
    
    writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
    writer.writeheader()  
    writer.writerows(data) 

print(f"Data has been successfully converted to {csv_file_path}")

