Load FLEURS Dataset

1
from nemo_curator.stages.audio.datasets.fleurs.create_initial_manifest import CreateInitialManifestFleursStage
2
from nemo_curator.pipeline import Pipeline
3
4
# Create FLEURS loading stage
5
fleurs_stage = CreateInitialManifestFleursStage(
6
    lang="hy_am",              # Armenian language
7
    split="dev",               # Development split
8
    raw_data_dir="/path/to/audio/data"
9
)
10
11
# Add to pipeline
12
pipeline = Pipeline(name="fleurs_loading")
13
pipeline.add_stage(fleurs_stage.with_(batch_size=4))
14
15
# Execute
16
pipeline.run()

1
# Common language examples
2
languages = [
3
    "en_us",    # English (US)
4
    "es_419",   # Spanish (Latin America) 
5
    "fr_fr",    # French (France)
6
    "de_de",    # German (Germany)
7
    "zh_cn",    # Chinese (Simplified)
8
    "ja_jp",    # Japanese
9
    "ko_kr",    # Korean
10
    "hy_am",    # Armenian
11
    "ar_eg",    # Arabic (Egypt)
12
]

1
splits = ["train", "dev", "test"]
2
3
# Load training data for multiple languages
4
for lang in ["en_us", "es_419", "fr_fr"]:
5
    stage = CreateInitialManifestFleursStage(
6
        lang=lang,
7
        split="train", 
8
        raw_data_dir=f"/data/fleurs/{lang}"
9
    )

1
CreateInitialManifestFleursStage(
2
    lang="en_us",                    # Language code
3
    split="train",                   # Data split
4
    raw_data_dir="/data/fleurs",     # Storage directory
5
    filepath_key="audio_filepath",   # Key for audio file paths
6
    text_key="text",                 # Key for transcription text
7
)

1
# Configure batch size for processing
2
fleurs_stage.with_(batch_size=8)  # Process 8 files per batch