Duration Filtering

1
from nemo_curator.stages.audio.common import GetAudioDurationStage, PreserveByValueStage
2
3
# Calculate duration for each audio file
4
duration_stage = GetAudioDurationStage(
5
    audio_filepath_key="audio_filepath",
6
    duration_key="duration"
7
)
8
9
# Filter for optimal duration range (1-15 seconds)
10
min_duration_filter = PreserveByValueStage(
11
    input_value_key="duration",
12
    target_value=1.0,
13
    operator="ge"  # greater than or equal
14
)
15
16
max_duration_filter = PreserveByValueStage(
17
    input_value_key="duration", 
18
    target_value=15.0,
19
    operator="le"  # less than or equal
20
)
21
22
# Add to pipeline
23
pipeline.add_stage(duration_stage)
24
pipeline.add_stage(min_duration_filter) 
25
pipeline.add_stage(max_duration_filter)

1
# Duration ranges for different applications
2
duration_configs = {
3
    "asr_training": {
4
        "min_duration": 1.0,
5
        "max_duration": 20.0,
6
        "optimal_range": (2.0, 10.0)
7
    },
8
    
9
    "voice_cloning": {
10
        "min_duration": 3.0,
11
        "max_duration": 10.0, 
12
        "optimal_range": (4.0, 8.0)
13
    },
14
    
15
    "speech_synthesis": {
16
        "min_duration": 2.0,
17
        "max_duration": 15.0,
18
        "optimal_range": (3.0, 12.0)
19
    },
20
    
21
    "keyword_spotting": {
22
        "min_duration": 0.5,
23
        "max_duration": 3.0,
24
        "optimal_range": (1.0, 2.0)
25
    }
26
}
27
28
def create_use_case_duration_filter(use_case: str) -> list[PreserveByValueStage]:
29
    """Create duration filters for specific use case."""
30
    
31
    config = duration_configs.get(use_case, duration_configs["asr_training"])
32
    
33
    return [
34
        PreserveByValueStage(
35
            input_value_key="duration",
36
            target_value=config["min_duration"],
37
            operator="ge"
38
        ),
39
        PreserveByValueStage(
40
            input_value_key="duration",
41
            target_value=config["max_duration"],
42
            operator="le"
43
        )
44
    ]

1
from nemo_curator.stages.audio.common import PreserveByValueStage
2
from nemo_curator.pipeline import Pipeline
3
4
# Example: Filter by speech rate if you have word_rate field in your data
5
pipeline = Pipeline(name="speech_rate_filtering")
6
7
# Filter by speech rate (1.5-5 words per second)
8
pipeline.add_stage(
9
    PreserveByValueStage(
10
        input_value_key="word_rate",  # Assumes this field exists in your data
11
        target_value=1.5,
12
        operator="ge"
13
    )
14
)
15
16
pipeline.add_stage(
17
    PreserveByValueStage(
18
        input_value_key="word_rate",
19
        target_value=5.0,
20
        operator="le"
21
    )
22
)

1
from nemo_curator.stages.audio.common import PreserveByValueStage
2
3
# Filter by word rate (assumes word_rate field exists in your data)
4
word_rate_min_filter = PreserveByValueStage(
5
    input_value_key="word_rate",
6
    target_value=1.5,
7
    operator="ge"
8
)
9
10
word_rate_max_filter = PreserveByValueStage(
11
    input_value_key="word_rate",
12
    target_value=5.0,
13
    operator="le"
14
)
15
16
# Filter by character rate (assumes char_rate field exists in your data)
17
char_rate_min_filter = PreserveByValueStage(
18
    input_value_key="char_rate",
19
    target_value=8.0,
20
    operator="ge"
21
)
22
23
char_rate_max_filter = PreserveByValueStage(
24
    input_value_key="char_rate",
25
    target_value=30.0,
26
    operator="le"
27
)

1
# Check retention rates before applying filters
2
retention_rate = filtered_count / original_count
3
if retention_rate < 0.5:  # Less than 50% retained
4
    print("Warning: Very aggressive filtering - consider relaxing thresholds")

1
from nemo_curator.pipeline import Pipeline
2
from nemo_curator.stages.audio.common import GetAudioDurationStage, PreserveByValueStage
3
from nemo_curator.stages.audio.datasets.fleurs.create_initial_manifest import CreateInitialManifestFleursStage
4
from nemo_curator.stages.audio.inference.asr_nemo import InferenceAsrNemoStage
5
from nemo_curator.stages.audio.metrics.get_wer import GetPairwiseWerStage
6
from nemo_curator.stages.audio.io.convert import AudioToDocumentStage
7
from nemo_curator.stages.resources import Resources
8
9
def create_audio_pipeline(raw_data_dir: str, wer_threshold: float = 75.0) -> Pipeline:
10
    """Real working pipeline from NeMo Curator tutorials."""
11
    
12
    pipeline = Pipeline(name="audio_inference", description="Inference audio and filter by WER threshold.")
13
    
14
    # Load FLEURS dataset
15
    pipeline.add_stage(
16
        CreateInitialManifestFleursStage(
17
            lang="hy_am",
18
            split="dev", 
19
            raw_data_dir=raw_data_dir,
20
        ).with_(batch_size=4)
21
    )
22
    
23
    # ASR inference
24
    pipeline.add_stage(
25
        InferenceAsrNemoStage(
26
            model_name="nvidia/stt_hy_fastconformer_hybrid_large_pc"
27
        ).with_(resources=Resources(gpus=1.0))
28
    )
29
    
30
    # Calculate WER
31
    pipeline.add_stage(
32
        GetPairwiseWerStage(
33
            text_key="text", 
34
            pred_text_key="pred_text", 
35
            wer_key="wer"
36
        )
37
    )
38
    
39
    # Calculate duration
40
    pipeline.add_stage(
41
        GetAudioDurationStage(
42
            audio_filepath_key="audio_filepath", 
43
            duration_key="duration"
44
        )
45
    )
46
    
47
    # Filter by WER threshold
48
    pipeline.add_stage(
49
        PreserveByValueStage(
50
            input_value_key="wer", 
51
            target_value=wer_threshold, 
52
            operator="le"
53
        )
54
    )
55
    
56
    # Convert to document format
57
    pipeline.add_stage(AudioToDocumentStage().with_(batch_size=1))
58
    
59
    return pipeline