nemo_curator.tasks.lance

Module Contents

Classes

Name	Description
`LanceReadTask`	Task containing Lance fragment ids assigned to one read partition.

Data

FragmentIds

API

class nemo_curator.tasks.lance.LanceReadTask(
    dataset_name: str,
    data: nemo_curator.tasks.lance.FragmentIds = list(),
    _stage_perf: list[nemo_curator.utils.performance_utils.StagePerfStats] = list(),
    _metadata: dict[str, typing.Any] = dict(),
    path: str,
    version: int
)

Dataclass

Bases: Task[FragmentIds]

Task containing Lance fragment ids assigned to one read partition.

Parameters:

path

str

Path or URI of the Lance dataset to read.

version

int

Lance dataset version to read.

data

FragmentIdsDefaults to list()

Lance fragment ids to read.

data

FragmentIds = field(default_factory=list)

num_items

int

path

str = field(kw_only=True)

version

int = field(kw_only=True)

nemo_curator.tasks.lance.LanceReadTask.get_deterministic_id() -> str

nemo_curator.tasks.lance.LanceReadTask.validate() -> bool

nemo_curator.tasks.lance.FragmentIds: TypeAlias = list[int]