Generate Multilingual Q&A Data

1
import os
2
from nemo_curator.core.client import RayClient
3
from nemo_curator.models.client.openai_client import AsyncOpenAIClient
4
from nemo_curator.models.client.llm_client import GenerationConfig
5
from nemo_curator.pipeline import Pipeline
6
from nemo_curator.stages.synthetic.qa_multilingual_synthetic import QAMultilingualSyntheticStage
7
from nemo_curator.stages.text.io.writer.jsonl import JsonlWriter
8
9
# Initialize Ray
10
client = RayClient(include_dashboard=False)
11
client.start()
12
13
# Create LLM client
14
llm_client = AsyncOpenAIClient(
15
    api_key=os.environ["NVIDIA_API_KEY"],
16
    base_url="https://integrate.api.nvidia.com/v1",
17
    max_concurrent_requests=5,
18
)
19
20
# Create pipeline
21
pipeline = Pipeline(name="multilingual_qa")
22
23
# Add synthetic generation stage
24
pipeline.add_stage(
25
    QAMultilingualSyntheticStage(
26
        prompt="Generate a Q&A pair about science in {language}.",
27
        languages=["English", "French", "German", "Spanish"],
28
        client=llm_client,
29
        model_name="meta/llama-3.3-70b-instruct",
30
        num_samples=50,
31
        generation_config=GenerationConfig(temperature=0.9),
32
    )
33
)
34
35
# Write output
36
pipeline.add_stage(JsonlWriter(path="./synthetic_qa/"))
37
38
# Run pipeline
39
results = pipeline.run()
40
41
client.stop()

1
from nemo_curator.models.client.openai_client import AsyncOpenAIClient
2
from nemo_curator.models.client.llm_client import GenerationConfig
3
4
llm_client = AsyncOpenAIClient(
5
    api_key=os.environ["NVIDIA_API_KEY"],
6
    base_url="https://integrate.api.nvidia.com/v1",
7
    max_concurrent_requests=5,  # Adjust based on rate limits
8
    max_retries=3,              # Retry on transient failures
9
    base_delay=1.0,             # Backoff delay in seconds
10
)
11
12
# Configure generation parameters
13
generation_config = GenerationConfig(
14
    temperature=0.9,   # Higher for more diverse outputs (default: 0.0)
15
    top_p=0.95,
16
    max_tokens=2048,
17
    seed=None,         # Set to None for non-deterministic (default: 0 for reproducibility)
18
)

1
# Simple Q&A prompt
2
prompt = "Generate a Q&A pair about science in {language}."
3
4
# Structured prompt with language prefixes
5
prompt = """
6
Generate a short question and a short answer in the general science domain in {language}.
7
Begin with the language name using the 2-letter code in square brackets,
8
for example, [EN] for English, [FR] for French, [DE] for German.
9
"""

1
from nemo_curator.pipeline import Pipeline
2
from nemo_curator.stages.synthetic.qa_multilingual_synthetic import QAMultilingualSyntheticStage
3
4
pipeline = Pipeline(
5
    name="multilingual_qa_generation",
6
    description="Generate synthetic Q&A pairs in multiple languages",
7
)
8
9
pipeline.add_stage(
10
    QAMultilingualSyntheticStage(
11
        prompt=prompt,
12
        languages=["English", "French", "German", "Spanish", "Italian"],
13
        client=llm_client,
14
        model_name="meta/llama-3.3-70b-instruct",
15
        num_samples=100,
16
        generation_config=generation_config,
17
    )
18
)

1
from nemo_curator.stages.text.filters.doc_filter import DocumentFilter
2
from nemo_curator.stages.text.modules.score_filter import ScoreFilter
3
4
class BeginsWithLanguageFilter(DocumentFilter):
5
    """Filter documents based on language prefix codes."""
6
7
    def __init__(self, languages: list[str]):
8
        super().__init__()
9
        self._name = "begins_with_language_filter"
10
        self.languages = languages
11
12
    def score_document(self, text: str) -> float:
13
        if not self.languages:
14
            return 1.0
15
        return 1.0 if text.startswith(tuple(self.languages)) else 0.0
16
17
    def keep_document(self, score: float) -> bool:
18
        return score == 1.0
19
20
# Add filter to keep only English outputs
21
pipeline.add_stage(
22
    ScoreFilter(
23
        BeginsWithLanguageFilter(languages=["[EN]"]),
24
        text_field="text",
25
    ),
26
)

1
from nemo_curator.stages.text.io.writer.jsonl import JsonlWriter
2
from nemo_curator.stages.text.io.writer.parquet import ParquetWriter
3
4
# JSONL output
5
pipeline.add_stage(JsonlWriter(path="./output/synthetic_qa/"))
6
7
# Or Parquet output
8
# pipeline.add_stage(ParquetWriter(path="./output/synthetic_qa/"))

1
from nemo_curator.core.client import RayClient
2
3
# Initialize Ray
4
client = RayClient(include_dashboard=False)
5
client.start()
6
7
# Execute pipeline
8
print(pipeline.describe())
9
results = pipeline.run()
10
11
# Print results summary
12
if results:
13
    for result in results:
14
        if hasattr(result, "data") and result.data:
15
            for file_path in result.data:
16
                print(f"Generated: {file_path}")
17
18
client.stop()