nemo_curator.stages.interleaved.lance.reader

Lance reader for row-wise interleaved multimodal datasets.

Module Contents

Classes

Name	Description
`InterleavedLanceReader`	Partition and read a Lance dataset as row-wise interleaved batches.
`InterleavedLanceReaderStage`	Read Lance fragments into validated `InterleavedBatch` objects.

Functions

Name	Description
`_validate_positive_optional`	-

Data

_GROUP_COLUMN

API

class nemo_curator.stages.interleaved.lance.reader.InterleavedLanceReader(
    path: str,
    fragments_per_partition: int = 1,
    fields: list[str] | None = None,
    max_batch_bytes: int | None = None,
    max_batch_rows: int | None = None,
    read_kwargs: dict[str, typing.Any] | None = None,
    include_lance_metadata: bool = True,
    fragment_ids: list[int] | None = None,
    name: str = 'interleaved_lance_reader'
)

Dataclass

Bases: CompositeStage[EmptyTask, InterleavedBatch]

Partition and read a Lance dataset as row-wise interleaved batches.

fields

list[str] | None = None

fragment_ids

list[int] | None = None

fragments_per_partition

int = 1

include_lance_metadata

bool = True

max_batch_bytes

int | None = None

max_batch_rows

int | None = None

name

str = 'interleaved_lance_reader'

path

str

read_kwargs

dict[str, Any] | None = None

nemo_curator.stages.interleaved.lance.reader.InterleavedLanceReader.__post_init__() -> None

nemo_curator.stages.interleaved.lance.reader.InterleavedLanceReader.decompose() -> list[nemo_curator.stages.base.ProcessingStage]

class nemo_curator.stages.interleaved.lance.reader.InterleavedLanceReaderStage(
    fields: list[str] | None = None,
    read_kwargs: dict[str, typing.Any] = dict(),
    name: str = 'interleaved_lance_reader',
    _generate_ids: bool = False,
    _assign_ids: bool = False,
    allow_empty: bool = True,
    include_lance_metadata: bool = True,
    max_batch_bytes: int | None = None,
    max_batch_rows: int | None = None
)

Dataclass

Bases: LanceReaderStage

Read Lance fragments into validated InterleavedBatch objects.

max_batch_bytes

int | None = None

max_batch_rows

int | None = None

name

str = 'interleaved_lance_reader'

nemo_curator.stages.interleaved.lance.reader.InterleavedLanceReaderStage.__post_init__() -> None

nemo_curator.stages.interleaved.lance.reader.InterleavedLanceReaderStage.process(
    task: nemo_curator.tasks.LanceReadTask
) -> nemo_curator.tasks.InterleavedBatch | list[nemo_curator.tasks.InterleavedBatch]

nemo_curator.stages.interleaved.lance.reader._validate_positive_optional(
    name: str,
    value: int | None
) -> None

nemo_curator.stages.interleaved.lance.reader._GROUP_COLUMN = 'sample_id'