GPU Processing Guide

1
from nemo_curator.stages.text.classifiers import QualityClassifier
2
from nemo_curator.pipeline import Pipeline
3
from nemo_curator.tasks import DocumentBatch
4
import pandas as pd
5
6
# Create sample data
7
data = pd.DataFrame({
8
    "text": ["This is high quality text.", "Poor quality text here."]
9
})
10
batch = DocumentBatch(data=data, task_id="test_task", dataset_name="test_dataset")
11
12
# Set up GPU-accelerated classifier
13
classifier = QualityClassifier(
14
    model_inference_batch_size=256,
15
    autocast=True  # Enable mixed precision for faster inference
16
)
17
18
# Create and run pipeline
19
pipeline = Pipeline(name="test_pipeline")
20
pipeline.add_stage(classifier)
21
result = pipeline.run(initial_tasks=[batch])
22
23
print(result)

1
from nemo_curator.stages.deduplication.fuzzy.workflow import FuzzyDeduplicationWorkflow
2
3
# Set up GPU-accelerated fuzzy deduplication
4
workflow = FuzzyDeduplicationWorkflow(
5
    input_path="/path/to/input/data",
6
    cache_path="/path/to/cache",
7
    output_path="/path/to/output",
8
    text_field="text",
9
    # GPU-accelerated MinHash parameters
10
    char_ngrams=24,
11
    num_bands=20,
12
    minhashes_per_band=13,
13
    use_64_bit_hash=False
14
)
15
16
# Run deduplication workflow
17
workflow.run()