nemo_curator.utils.retry_manifest

Module Contents

Classes

Name	Description
`CompletionManifest`	Durable proof that work identified by stable fields completed successfully.
`CompletionManifestRecord`	One durable completion manifest read from disk.

Functions

Name	Description
`_mapping_digest`	-
`_safe_token`	-
`read_completion_manifests`	Read completed identities for one manifest namespace.

Data

METADATA_DIRNAME

API

class nemo_curator.utils.retry_manifest.CompletionManifest(
    checkpoint_path: str | pathlib.Path,
    namespace: str,
    identity: collections.abc.Mapping[str, object],
    metadata: collections.abc.Mapping[str, object] | None = None,
    completion_dirname: str | None = None,
    enabled: bool = True,
    flatten_identity: bool = True,
    flatten_metadata: bool = False
)

Durable proof that work identified by stable fields completed successfully.

checkpoint_path

= Path(checkpoint_path)

completion_dirname

filename_prefix

str

identity

= dict(identity)

manifest_dir

Path

manifest_file

Path | None = None

metadata

= dict(metadata or {})

nemo_curator.utils.retry_manifest.CompletionManifest.__enter__() -> nemo_curator.utils.retry_manifest.CompletionManifest

nemo_curator.utils.retry_manifest.CompletionManifest.__exit__(
    _exc_type: type[BaseException] | None,
    exc: BaseException | None,
    _: object
) -> bool

nemo_curator.utils.retry_manifest.CompletionManifest._payload(
    extra: collections.abc.Mapping[str, object] | None = None
) -> dict[str, object]

nemo_curator.utils.retry_manifest.CompletionManifest.mark_completed(
    extra: collections.abc.Mapping[str, object] | None = None
) -> pathlib.Path | None

Atomically record successful completion.

class nemo_curator.utils.retry_manifest.CompletionManifestRecord(
    path: pathlib.Path,
    payload: dict[str, object]
)

Dataclass

One durable completion manifest read from disk.

path

Path

payload

dict[str, object]

nemo_curator.utils.retry_manifest._mapping_digest(
    mapping: collections.abc.Mapping[str, object]
) -> str

nemo_curator.utils.retry_manifest._safe_token(
    value: object
) -> str

nemo_curator.utils.retry_manifest.read_completion_manifests(
    checkpoint_path: str | pathlib.Path,
    namespace: str,
    completion_dirname: str | None = None
) -> list[nemo_curator.utils.retry_manifest.CompletionManifestRecord]

Read completed identities for one manifest namespace.

nemo_curator.utils.retry_manifest.METADATA_DIRNAME = '.nemo_curator_metadata'