modules.splitter

Module Contents

Classes

DocumentSplitter

API

class modules.splitter.DocumentSplitter(

separator: str,

text_field: str = 'text',

segment_id_field: str = 'segment_id',

)

Bases: nemo_curator.modules.base.BaseModule

Initialization

call(

dataset: nemo_curator.datasets.DocumentDataset,

) → nemo_curator.datasets.DocumentDataset