nemo_curator.stages.interleaved.io.readers.base

View as Markdown

Module Contents

Classes

NameDescription
BaseInterleavedReaderBase contract for interleaved readers.

API

class nemo_curator.stages.interleaved.io.readers.base.BaseInterleavedReader(
read_kwargs: dict[str, typing.Any] = dict(),
name: str = 'base_interleaved_reader'
)
Dataclass

Bases: ProcessingStage[FileGroupTask, InterleavedBatch]

Base contract for interleaved readers.

name
str = 'base_interleaved_reader'
read_kwargs
dict[str, Any] = field(default_factory=dict)
nemo_curator.stages.interleaved.io.readers.base.BaseInterleavedReader.inputs() -> tuple[list[str], list[str]]
nemo_curator.stages.interleaved.io.readers.base.BaseInterleavedReader.outputs() -> tuple[list[str], list[str]]