Get Started with Audio Curation

1
from nemo_curator.pipeline import Pipeline
2
from nemo_curator.stages.audio.datasets.fleurs.create_initial_manifest import CreateInitialManifestFleursStage
3
from nemo_curator.stages.audio.inference.asr_nemo import InferenceAsrNemoStage
4
from nemo_curator.stages.audio.metrics.get_wer import GetPairwiseWerStage
5
from nemo_curator.stages.audio.common import GetAudioDurationStage, PreserveByValueStage
6
from nemo_curator.stages.resources import Resources
7
8
# Create audio curation pipeline
9
pipeline = Pipeline(name="audio_curation", description="FLEURS audio curation with ASR and WER filtering")
10
11
# 1. Load FLEURS dataset (Armenian development set)
12
pipeline.add_stage(
13
    CreateInitialManifestFleursStage(
14
        lang="hy_am",
15
        split="dev", 
16
        raw_data_dir="~/nemo_curator/audio_data"
17
    ).with_(batch_size=4)
18
)
19
20
# 2. Perform ASR inference using NeMo model
21
pipeline.add_stage(
22
    InferenceAsrNemoStage(
23
        model_name="nvidia/stt_hy_fastconformer_hybrid_large_pc"
24
    ).with_(resources=Resources(gpus=1.0))
25
)
26
27
# 3. Calculate Word Error Rate (WER)
28
pipeline.add_stage(
29
    GetPairwiseWerStage(
30
        text_key="text",
31
        pred_text_key="pred_text", 
32
        wer_key="wer"
33
    )
34
)
35
36
# 4. Calculate audio duration
37
pipeline.add_stage(
38
    GetAudioDurationStage(
39
        audio_filepath_key="audio_filepath",
40
        duration_key="duration"
41
    )
42
)
43
44
# 5. Filter by WER threshold (keep samples with WER <= 75%)
45
pipeline.add_stage(
46
    PreserveByValueStage(
47
        input_value_key="wer",
48
        target_value=75.0,
49
        operator="le"  # less than or equal
50
    )
51
)
52
53
# Execute the pipeline
54
pipeline.run()