Adding Document IDs

1
from nemo_curator.stages.text.modules import AddId
2
3
# Initialize pipeline, read stage, etc.
4
5
# Add to your pipeline
6
pipeline.add_stage(AddId(id_field="doc_id"))

1
# Customize ID generation
2
pipeline.add_stage(AddId(
3
    id_field="document_id",        # Field name for IDs
4
    id_prefix="corpus_v2",         # Optional prefix
5
    overwrite=True                 # Overwrite existing IDs
6
))

1
from nemo_curator.core.client import RayClient
2
from nemo_curator.pipeline import Pipeline
3
from nemo_curator.stages.text.io.reader import JsonlReader
4
from nemo_curator.stages.text.modules import AddId
5
from nemo_curator.stages.text.io.writer import JsonlWriter
6
7
# Initialize Ray client
8
ray_client = RayClient()
9
ray_client.start()
10
11
# Create pipeline
12
pipeline = Pipeline(name="add_ids")
13
14
# Add stages
15
pipeline.add_stage(JsonlReader(file_paths="input/"))
16
pipeline.add_stage(AddId(id_field="doc_id", id_prefix="v1"))
17
pipeline.add_stage(JsonlWriter("output/"))
18
19
# Run pipeline
20
result = pipeline.run()
21
22
# Stop Ray client
23
ray_client.stop()

1
from nemo_curator.core.client import RayClient
2
from nemo_curator.pipeline import Pipeline
3
from nemo_curator.stages.deduplication.id_generator import create_id_generator_actor
4
from nemo_curator.stages.text.io.reader import JsonlReader
5
6
# Initialize Ray client
7
ray_client = RayClient()
8
ray_client.start()
9
10
pipeline = Pipeline(name="id_generator_example")
11
12
# Create ID generator
13
create_id_generator_actor()
14
15
# Reader generates IDs automatically
16
reader = JsonlReader(
17
    file_paths="data/",
18
    _generate_ids=True  # Adds '_curator_dedup_id' field
19
)
20
pipeline.add_stage(reader)
21
22
# Run pipeline
23
results = pipeline.run()
24
25
# Stop Ray client
26
ray_client.stop()
27
28
# Examine the first 5 rows of the first DocumentBatch
29
print(results[0].data.head())

1
# This raises ValueError if 'doc_id' already exists
2
AddId(id_field="doc_id", overwrite=False)
3
4
# This overwrites existing field with warning
5
AddId(id_field="doc_id", overwrite=True)