nat.data_models.evaluate

Classes

JobEvictionPolicy	Policy for evicting old jobs when max_jobs is exceeded.
EvalCustomScriptConfig
JobManagementConfig
EvalOutputConfig
EvalGeneralConfig
EvalConfig

Module Contents

class JobEvictionPolicy

Bases: str, enum.Enum

Policy for evicting old jobs when max_jobs is exceeded.

Initialize self. See help(type(self)) for accurate signature.

TIME_CREATED = 'time_created'

TIME_MODIFIED = 'time_modified'

class EvalCustomScriptConfig(/, **data: Any)

Bases: pydantic.BaseModel

script: pathlib.Path

kwargs: dict[str, str]

class JobManagementConfig(/, **data: Any)

Bases: pydantic.BaseModel

append_job_id_to_output_dir: bool = False

max_jobs: int = 0

eviction_policy: JobEvictionPolicy

class EvalOutputConfig(/, **data: Any)

Bases: pydantic.BaseModel

dir: pathlib.Path

remote_dir: str | None = None

custom_scripts: dict[str, EvalCustomScriptConfig]

s3: nat.data_models.dataset_handler.EvalS3Config | None = None

cleanup: bool = True

job_management: JobManagementConfig

workflow_output_step_filter: list[nat.data_models.intermediate_step.IntermediateStepType] | None = None

class EvalGeneralConfig(/, **data: Any)

Bases: pydantic.BaseModel

max_concurrency: int = 8

workflow_alias: str | None = None

output_dir: pathlib.Path

output: EvalOutputConfig | None = None

dataset: nat.data_models.dataset_handler.EvalDatasetConfig | None = None

profiler: nat.data_models.profiler.ProfilerConfig | None = None

classmethod override_output_dir(values)

class EvalConfig(/, **data: Any)

Bases: pydantic.BaseModel

general: EvalGeneralConfig

evaluators: dict[str, nat.data_models.evaluator.EvaluatorBaseConfig]

classmethod rebuild_annotations()