davanstrien HF Staff
Card: Edge/d1 context, credits and ideas, aligned tables; curate.py [tool.hf-jobs] header; commented datatrove example (#3)
d5c845e Download examples/datatrove_curation.py from davanstrien/ultrabert-380m-curation: direct link, hf CLI and curl.
- Browser
- Download file 10.3 kB
-
https://huggingface.co/davanstrien/ultrabert-380m-curation/resolve/main/examples/datatrove_curation.py
- Command line
-
hf download hf://davanstrien/ultrabert-380m-curation/examples/datatrove_curation.py
-
curl -L -o datatrove_curation.py https://huggingface.co/davanstrien/ultrabert-380m-curation/resolve/main/examples/datatrove_curation.py
10.3 kB
| """Curate a web dataset with davanstrien/ultrabert-380m-curation and datatrove on Hugging Face Jobs. | |
| Reads parquet files from a Hub dataset, asks four curation questions about every page (topic, format, educational | |
| value, reasoning depth), writes every page with its answers, and writes a filtered subset (educational value | |
| "useful for education" or higher, and not spam, a listing, a product page or incomplete). Scoring uses the packed GPU | |
| path from the model repo (fast_inference.py). Pages are cut at 3,000 characters for the model only; the written rows | |
| keep the full text, and no rows are dropped from the scored output. | |
| How it runs: | |
| - You run this script on your machine. It is the coordinator: it builds the pipeline and launches one GPU Job per | |
| task with JobsPipelineExecutor, then waits for them. | |
| - Each Job installs the same dependencies, receives the pipeline (serialised with dill) and runs it on its share of | |
| the input files. Only the pipeline objects travel to the Job, not this module, so CurationScorer imports what it | |
| needs inside run(). | |
| - Each Job writes parquet files to a storage bucket: OUT/scored (every page) and OUT/subset (pages that pass the | |
| rule). datatrove keeps its progress in OUT/logs, so rerunning the same command skips finished tasks. | |
| Use the same Python and datatrove versions on both sides: | |
| OUT="hf://buckets/YOUR_NAME/YOUR_BUCKET/run1" | |
| uv run --python 3.12 --with "datatrove[io,processing]==0.10.1" --with "huggingface_hub>=1.32" \\ | |
| datatrove_curation.py --out "$OUT" --limit 2000 | |
| Example (FineWeb CC-MAIN-2025-26, --glob "000_0000*.parquet" --limit 170000 --tasks 6): 1,020,000 pages in about | |
| 19 minutes, about $5; 63,788 pages in the subset. | |
| """ | |
| import argparse | |
| from datatrove.executor.jobs import JobsPipelineExecutor | |
| from datatrove.pipeline.base import PipelineStep | |
| from datatrove.pipeline.filters import LambdaFilter | |
| from datatrove.pipeline.readers import ParquetReader | |
| from datatrove.pipeline.writers import ParquetWriter | |
| MODEL = "davanstrien/ultrabert-380m-curation" | |
| class CurationScorer(PipelineStep): | |
| """Adds the answers to each document's metadata; ParquetWriter later writes them under `metadata`.""" | |
| name = "curation scorer" | |
| type = "🏷 - TAGGER" | |
| def __init__(self, model_id=MODEL, chunk_docs=2048): | |
| super().__init__() | |
| self.model_id = model_id | |
| # Documents per scoring call. The fast path packs many documents into each 64K-token row, so it needs a | |
| # few thousand at a time to keep the GPU busy. | |
| self.chunk_docs = chunk_docs | |
| def load_fast_scorer(self): | |
| """Import FastScorer from fast_inference.py in the model repo (a single file, not a pip package).""" | |
| import importlib.util | |
| import sys | |
| from huggingface_hub import hf_hub_download | |
| path = hf_hub_download(self.model_id, "fast_inference.py") | |
| spec = importlib.util.spec_from_file_location("fast_inference", path) | |
| module = importlib.util.module_from_spec(spec) | |
| sys.modules["fast_inference"] = module # registered by name, as a normal import would be | |
| spec.loader.exec_module(module) | |
| return module.FastScorer(self.model_id) # loads, fuses and compiles the model (about a minute) | |
| def run(self, data, rank=0, world_size=1): | |
| # This runs on the Job: `data` is this task's stream of documents, and the step yields them on to the next | |
| # step (the writer) with the answers added. | |
| questions = { # name -> (prompt, options), as in training | |
| "topic": ("Classify the topic of this web page.", [ | |
| "Adult Content", "Art & Design", "Crime & Law", "Education & Jobs", "Electronics & Hardare", | |
| "Entertainment", "Fashion & Beauty", "Finance & Business", "Food & Dining", "Games", "Health", | |
| "History & Geography", "Home & Hobbies", "Industrial", "Literature", "Politics", "Religion", | |
| "Science, Math & Technology", "Social Life", "Software", "Software Development", "Sports & Fitness", | |
| "Transportation", "Travel & Tourism"]), | |
| "format": ("Classify the format of this web page.", [ | |
| "Academic Writing", "Content Listing", "Creative Writing", "Customer Support Page", | |
| "Discussion Forum / Comment Section", "FAQs", "Incomplete Content", "Knowledge Article", | |
| "Legal Notices", "Listicle", "News Article", "Nonfiction Writing", "Organizational About Page", | |
| "Organizational Announcement", "Personal About Page", "Personal Blog", "Product Page", "Q&A Forum", | |
| "Spam / Ads", "Structured Data", "Technical Writing", "Transcript / Interview", | |
| "Tutorial / How-To Guide", "User Reviews"]), | |
| "edu": ("Classify the educational value of this web page.", [ # ordered, lowest first | |
| "not educational", "minimal educational value", "some educational value", "useful for education", | |
| "highly educational", "outstanding educational value"]), | |
| "reasoning": ("Classify the depth of reasoning of this web page.", [ | |
| "No Reasoning", "Basic Reasoning", "Intermediate Reasoning", "Advanced Reasoning", "Indeterminate"]), | |
| } | |
| # The topic label is spelled as in the WebOrganizer training labels ("Hardare"); the output uses the correct spelling. | |
| rename = {"Electronics & Hardare": "Electronics & Hardware"} | |
| # The subset rule: educational value level 3 ("useful for education") or higher, and none of these formats. | |
| drop_formats = {"Spam / Ads", "Content Listing", "Product Page", "Incomplete Content"} | |
| scorer = self.load_fast_scorer() | |
| def score_chunk(docs): | |
| texts = [d.text[:3000] for d in docs] # the training web text was cut at 3,000 characters | |
| # One request per (question, document), all scored in one packed run. Results come back in request | |
| # order: all documents for the first question, then all documents for the second, and so on. | |
| requests = [(t, labels, prompt) for prompt, labels in questions.values() for t in texts] | |
| results = scorer.score_requests(requests) # one {option: probability} dict per request | |
| for qi, (name, (_, labels)) in enumerate(questions.items()): | |
| for di, d in enumerate(docs): | |
| probs = results[qi * len(docs) + di] | |
| top = max(probs, key=probs.get) if probs else "" | |
| d.metadata[name] = rename.get(top, top) # "" only if labels were truncated (never at 3,000 chars) | |
| d.metadata[f"{name}_p"] = probs[top] if probs else 0.0 | |
| if name == "edu": | |
| d.metadata["edu_level"] = labels.index(top) if probs else -1 | |
| d.metadata["edu_expected"] = sum(i * probs[x] for i, x in enumerate(labels)) if probs else -1.0 | |
| for d in docs: | |
| # ParquetWriter infers one schema per file, so source columns that are sometimes missing or nested | |
| # are written as strings. | |
| for k in list(d.metadata): | |
| if k not in questions and not k.startswith(tuple(questions)): | |
| v = d.metadata[k] | |
| if v is None or not isinstance(v, (str, int, float, bool)): | |
| d.metadata[k] = "" if v is None else str(v) | |
| d.metadata["cut_at_3000_chars"] = len(d.text) > 3000 | |
| d.metadata["in_subset"] = d.metadata["edu_level"] >= 3 and d.metadata["format"] not in drop_formats | |
| # Collect documents into chunks, score each chunk, and pass the documents on. | |
| buf = [] | |
| for doc in data: | |
| buf.append(doc) | |
| if len(buf) >= self.chunk_docs: | |
| with self.track_time(unit="chunk"): | |
| score_chunk(buf) | |
| self.stat_update("docs_scored", value=len(buf)) | |
| yield from buf | |
| buf = [] | |
| if buf: | |
| score_chunk(buf) | |
| self.stat_update("docs_scored", value=len(buf)) | |
| yield from buf | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--out", required=True, help="output prefix, e.g. hf://buckets/<you>/<bucket>/run1") | |
| ap.add_argument("--source", default="hf://datasets/HuggingFaceFW/fineweb/data/CC-MAIN-2025-26") | |
| ap.add_argument("--glob", default="*.parquet") | |
| ap.add_argument("--text-key", default="text") | |
| ap.add_argument("--id-key", default="id") | |
| ap.add_argument("--limit", type=int, default=-1, help="pages per task; -1 = all") | |
| ap.add_argument("--tasks", type=int, default=1, help="parallel GPU Jobs; input files are split across them") | |
| ap.add_argument("--flavor", default="rtx-pro-6000") | |
| ap.add_argument("--timeout", default="2h") | |
| args = ap.parse_args() | |
| JobsPipelineExecutor( | |
| job_name="curation-demo", | |
| # Steps run in order on each Job. The first writer writes every page; the filter then drops pages outside | |
| # the subset, and the second writer writes the rest. | |
| pipeline=[ | |
| ParquetReader(args.source, glob_pattern=args.glob, limit=args.limit, text_key=args.text_key, | |
| id_key=args.id_key), | |
| CurationScorer(), | |
| ParquetWriter(f"{args.out}/scored", max_file_size=64 * 2**20), | |
| LambdaFilter(lambda doc: doc.metadata["in_subset"]), | |
| ParquetWriter(f"{args.out}/subset", max_file_size=64 * 2**20), | |
| ], | |
| tasks=args.tasks, # input files are divided between tasks | |
| workers=args.tasks, # Jobs running at the same time (here: all of them) | |
| flavor=args.flavor, | |
| python="3.12", # must match the coordinator's Python: the pipeline is serialised with dill | |
| dependencies=["datatrove[io,processing]==0.10.1", "huggingface_hub>=1.32", "gliclass>=0.1.21", "torch", | |
| "transformers>=5.20", "kernels>=0.17.2,<0.18", "numpy"], | |
| logging_dir=f"{args.out}/logs", | |
| timeout=args.timeout, | |
| max_retries=0, # a retry reruns the whole task on the GPU; rerun the script to resume incomplete tasks | |
| ).run() | |
| if __name__ == "__main__": | |
| main() | |