`classifiers.quality`#

Module Contents#

Classes#

`QualityClassifier`
`QualityModel`
`QualityModelConfig`

Data#

QUALITY_IDENTIFIER

API#

classifiers.quality.QUALITY_IDENTIFIER#: ‘nvidia/quality-classifier-deberta’

class classifiers.quality.QualityClassifier( filter_by: list[str] | None = None, batch_size: int = 256, text_field: str = 'text', pred_column: str = 'quality_pred', prob_column: str = 'quality_prob', max_chars: int = 6000, device_type: str = 'cuda', autocast: bool = True, max_mem_gb: int | None = None, )#

Bases: nemo_curator.classifiers.base.DistributedDataClassifier

Initialization

class classifiers.quality.QualityModel( config: classifiers.quality.QualityModelConfig, autocast: bool = False, max_mem_gb: int | None = None, )#

Bases: crossfit.backend.torch.hf.model.HFModel