Content Processing & Cleaning

1
from nemo_curator.core.client import RayClient
2
from nemo_curator.pipeline import Pipeline
3
from nemo_curator.stages.text.io.reader import JsonlReader
4
from nemo_curator.stages.text.io.writer import JsonlWriter
5
from nemo_curator.stages.text.modifiers import UnicodeReformatter, UrlRemover, NewlineNormalizer
6
from nemo_curator.stages.text.modules import Modify
7
8
# Initialize Ray client
9
ray_client = RayClient()
10
ray_client.start()
11
12
# Create a comprehensive cleaning pipeline
13
processing_pipeline = Pipeline(
14
    name="content_processing_pipeline",
15
    description="Comprehensive text cleaning and processing"
16
)
17
18
# Load dataset
19
reader = JsonlReader(file_paths="input_data/")
20
processing_pipeline.add_stage(reader)
21
22
# Fix Unicode encoding issues
23
processing_pipeline.add_stage(
24
    Modify(modifier_fn=UnicodeReformatter(), input_fields="text")
25
)
26
27
# Standardize newlines
28
processing_pipeline.add_stage(
29
    Modify(modifier_fn=NewlineNormalizer(), input_fields="text")
30
)
31
32
# Remove URLs
33
processing_pipeline.add_stage(
34
    Modify(modifier_fn=UrlRemover(), input_fields="text")
35
)
36
37
# Save the processed dataset
38
writer = JsonlWriter(path="processed_output/")
39
processing_pipeline.add_stage(writer)
40
41
# Execute pipeline
42
results = processing_pipeline.run()
43
44
# Stop Ray client
45
ray_client.stop()