nemo_automodel.components.datasets.zephon.config

Typed options for shared Zephon source construction and runtime delivery.

Module Contents

Classes

Name	Description
`TokenEstimationSettings`	Existing Zephon TokenEstimation fields accepted at the recipe boundary.
`ZephonCacheOptions`	Downloaded source cache settings.
`ZephonCheckpointOptions`	Canonical lane layout and all-rank checkpoint aggregation.
`ZephonOptions`	Grouped settings nested under `dataset.zephon`.
`ZephonPackingOptions`	Optional online packing; seq_len bounds shifted model inputs.
`ZephonParquetRGCacheOptions`	Decoded Parquet cache overrides; omitted fields retain Zephon defaults.
`ZephonPrefetchOptions`	Cache-warmup operator settings; None fields keep Zephon’s defaults.
`ZephonRuntimeOptions`	Pipeline execution and queue capacities; None uses Zephon’s automatic sizing.
`ZephonSource`	A named local/cloud dataset descriptor using Zephon’s URI grammar.
`ZephonSourceOptions`	Source traversal, token-aware allocation and end-of-data policy.
`ZephonTrainingOptions`	Consumer attention policy and Zephon boundary-target supervision.
`ZephonWorkerOptions`	Per-stage operator parallelism.

API

class nemo_automodel.components.datasets.zephon.config.TokenEstimationSettings

Bases: typing.TypedDict

Existing Zephon TokenEstimation fields accepted at the recipe boundary.

calibration_samples

int

calibration_shards_max

int

calibration_shards_min

int

fallback_tokens_per_byte

float

primer

Literal['measure'] | dict[str, float] | float

class nemo_automodel.components.datasets.zephon.config.ZephonCacheOptions(
    cache_dir: str | None = None,
    cache_limit_bytes: int | str | None = None,
    parquet_rg_cache: nemo_automodel.components.datasets.zephon.config.ZephonParquetRGCacheOptions | None = None
)

Dataclass

Downloaded source cache settings.

cache_dir

str | None = None

cache_limit_bytes

int | str | None = None

parquet_rg_cache

ZephonParquetRGCacheOptions | None = None

Separate decoded-cache budget; source shards use cache_limit_bytes.

nemo_automodel.components.datasets.zephon.config.ZephonCacheOptions.__post_init__() -> None

nemo_automodel.components.datasets.zephon.config.ZephonCacheOptions.build() -> zephon.io.StoreOptions

Configure source and decoded caches through Zephon’s existing I/O API.

class nemo_automodel.components.datasets.zephon.config.ZephonCheckpointOptions(
    canonical_replicas: int | None = None,
    aggregate_dir: str | None = None,
    aggregate_store: str | None = None,
    run_id: str | None = None
)

Dataclass

Canonical lane layout and all-rank checkpoint aggregation.

aggregate_dir

str | None = None

aggregate_store

str | None = None

canonical_replicas

int | None = None

run_id

str | None = None

nemo_automodel.components.datasets.zephon.config.ZephonCheckpointOptions.__post_init__() -> None

class nemo_automodel.components.datasets.zephon.config.ZephonOptions(
    sources: tuple[nemo_automodel.components.datasets.zephon.config.ZephonSource, ...] = (),
    source: nemo_automodel.components.datasets.zephon.config.ZephonSourceOptions = ZephonSourceOptions(),
    packing: nemo_automodel.components.datasets.zephon.config.ZephonPackingOptions = ZephonPackingOptions(),
    training: nemo_automodel.components.datasets.zephon.config.ZephonTrainingOptions = ZephonTrainingOptions(),
    workers: nemo_automodel.components.datasets.zephon.config.ZephonWorkerOptions = ZephonWorkerOptions(),
    runtime: nemo_automodel.components.datasets.zephon.config.ZephonRuntimeOptions = ZephonRuntimeOptions(),
    cache: nemo_automodel.components.datasets.zephon.config.ZephonCacheOptions = ZephonCacheOptions(),
    checkpoint: nemo_automodel.components.datasets.zephon.config.ZephonCheckpointOptions = ZephonCheckpointOptions()
)

Dataclass

Grouped settings nested under dataset.zephon.

Dataset adapters select encoding. These groups configure the shared source, packing and delivery backbone for both text and image pipelines. Concrete values are integration defaults. Field comments document automatic choices and values delegated to Zephon; these remain declarative until build time.

cache

ZephonCacheOptions = field(default_factory=ZephonCacheOptions)

checkpoint

ZephonCheckpointOptions = field(default_factory=ZephonCheckpointOptions)

packing

ZephonPackingOptions = field(default_factory=ZephonPackingOptions)

runtime

ZephonRuntimeOptions = field(default_factory=ZephonRuntimeOptions)

source

ZephonSourceOptions = field(default_factory=ZephonSourceOptions)

sources

tuple[ZephonSource, ...] = ()

training

ZephonTrainingOptions = field(default_factory=ZephonTrainingOptions)

workers

ZephonWorkerOptions = field(default_factory=ZephonWorkerOptions)

nemo_automodel.components.datasets.zephon.config.ZephonOptions.__post_init__() -> None

class nemo_automodel.components.datasets.zephon.config.ZephonPackingOptions(
    seq_len: int | None = None,
    pack_algorithm: typing.Literal['wrap', 'first_fit', 'best_fit', 'best_fit_wrap'] | None = 'wrap',
    pack_num_bins: int | None = None,
    max_num_documents: int | None = None,
    max_num_documents_scope: typing.Literal['batch', 'bin'] = 'batch',
    drop_oversized: bool | None = None,
    pad_token_id: int | None = None,
    shuffle_after_pack: bool | None = None,
    shuffle_buffer_size: int | None = None
)

Dataclass

Optional online packing; seq_len bounds shifted model inputs.

Omitted lengths use the model’s input-token limit. pack_algorithm=None keeps supplied documents separate and lets to_training pad shorter inputs.

drop_oversized

bool | None = None

max_num_documents

int | None = None

max_num_documents_scope

Literal['batch', 'bin'] = 'batch'

pack_algorithm

Literal['wrap', 'first_fit', 'best_fit', 'best_fit_wrap'] | None = 'wrap'

pack_num_bins

int | None = None

pad_token_id

int | None = None

seq_len

int | None = None

shuffle_after_pack

bool | None = None

shuffle_buffer_size

int | None = None

nemo_automodel.components.datasets.zephon.config.ZephonPackingOptions.__post_init__() -> None

class nemo_automodel.components.datasets.zephon.config.ZephonParquetRGCacheOptions(
    enabled: bool | None = None,
    root: str | None = None,
    limit_bytes: int | str | None = None,
    min_free_bytes: int | str | None = None
)

Dataclass

Decoded Parquet cache overrides; omitted fields retain Zephon defaults.

enabled

bool | None = None

limit_bytes

int | str | None = None

min_free_bytes

int | str | None = None

root

str | None = None

nemo_automodel.components.datasets.zephon.config.ZephonParquetRGCacheOptions.build() -> zephon.io.ParquetRGCacheOptions

Construct Zephon’s cache settings without replacing automatic defaults.

class nemo_automodel.components.datasets.zephon.config.ZephonPrefetchOptions(
    buffer_size: int | None = None,
    parallelism: int | None = None
)

Dataclass

Cache-warmup operator settings; None fields keep Zephon’s defaults.

buffer_size

int | None = None

Number of samples to look ahead; None uses Zephon’s operator default.

parallelism

int | None = None

Concurrent cache-warmup workers; None uses Zephon’s operator default.

nemo_automodel.components.datasets.zephon.config.ZephonPrefetchOptions.__post_init__() -> None

class nemo_automodel.components.datasets.zephon.config.ZephonRuntimeOptions(
    runner: typing.Literal['process', 'threads', 'inline'] = 'process',
    mtp_mode: bool = True,
    prefetch_batches: int | None = None,
    default_stage_prefetch: int | None = None,
    op_queue_capacity: int | None = None,
    mtp_buffer: int | None = None,
    mtp_prefetch: int | None = None
)

Dataclass

Pipeline execution and queue capacities; None uses Zephon’s automatic sizing.

default_stage_prefetch

int | None = None

Prefetch depth between stages; zero disables the optional stage prefetch buffer.

mtp_buffer

int | None = None

Positive capacity of the MTP engine-to-consumer IPC queue.

mtp_mode

bool = True

mtp_prefetch

int | None = None

Consumer-side MTP prefetch depth; zero disables the background receiver thread.

op_queue_capacity

int | None = None

Positive queue capacity between operators within a stage.

prefetch_batches

int | None = None

Pipeline-output prefetch depth in batches; zero disables this prefetch buffer.

runner

Literal['process', 'threads', 'inline'] = 'process'

nemo_automodel.components.datasets.zephon.config.ZephonRuntimeOptions.__post_init__() -> None

class nemo_automodel.components.datasets.zephon.config.ZephonSource(
    name: str,
    path: str,
    fmt: str | None = None,
    weight: float = 1.0
)

Dataclass

A named local/cloud dataset descriptor using Zephon’s URI grammar.

fmt

str | None = None

name

str

path

str

weight

float = 1.0

nemo_automodel.components.datasets.zephon.config.ZephonSource.__post_init__() -> None

class nemo_automodel.components.datasets.zephon.config.ZephonSourceOptions(
    chunk_size: int = 16384,
    shuffle_shards: bool = True,
    shuffle_within_shard: bool = True,
    shuffle_block_size: int | str | None = 'auto',
    exhausted_policy: str | dict[str, str] | None = None,
    reshuffle_on_repeat: bool | dict[str, bool] = True,
    max_repeats: int | dict[str, int | None] | None = None,
    stop_after_passes: int | None = None,
    token_estimation: bool | nemo_automodel.components.datasets.zephon.config.TokenEstimationSettings | typing.Literal['auto'] | None = 'auto',
    prefetch: nemo_automodel.components.datasets.zephon.config.ZephonPrefetchOptions | None = None
)

Dataclass

Source traversal, token-aware allocation and end-of-data policy.

chunk_size

int = 16384

exhausted_policy

str | dict[str, str] | None = None

max_repeats

int | dict[str, int | None] | None = None

prefetch

ZephonPrefetchOptions | None = None

reshuffle_on_repeat

bool | dict[str, bool] = True

shuffle_block_size

int | str | None = 'auto'

shuffle_shards

bool = True

shuffle_within_shard

bool = True

stop_after_passes

int | None = None

token_estimation

bool | TokenEstimationSettings | Literal['auto'] | None = 'auto'

nemo_automodel.components.datasets.zephon.config.ZephonSourceOptions.__post_init__() -> None

class nemo_automodel.components.datasets.zephon.config.ZephonTrainingOptions(
    allow_cross_document_attention: bool = False,
    mask_segment_starts: bool = False
)

Dataclass

Consumer attention policy and Zephon boundary-target supervision.

allow_cross_document_attention

bool = False

mask_segment_starts

bool = False

class nemo_automodel.components.datasets.zephon.config.ZephonWorkerOptions(
    tokenize_parallelism: int | None = 3,
    map_parallelism: int | None = None,
    pack_parallelism: int | None = 1,
    fetch_parallelism: int | None = 2,
    shuffle_parallelism: int | None = None
)

Dataclass

Per-stage operator parallelism.

Concrete values are integration tuning defaults. None uses each Zephon operator’s default, including the image encoder’s advertised traits.

fetch_parallelism

int | None = 2

map_parallelism

int | None = None

pack_parallelism

int | None = 1

shuffle_parallelism

int | None = None

tokenize_parallelism

int | None = 3

nemo_automodel.components.datasets.zephon.config.ZephonWorkerOptions.__post_init__() -> None