aiq.data_models.evaluate

Classes

EvalCustomScriptConfig
EvalOutputConfig
EvalGeneralConfig
EvalConfig

Module Contents

class EvalCustomScriptConfig(/, **data: Any)

Bases: pydantic.BaseModel

script: pathlib.Path

kwargs: dict[str, str]

class EvalOutputConfig(/, **data: Any)

Bases: pydantic.BaseModel

dir: pathlib.Path

remote_dir: str | None = None

custom_scripts: dict[str, EvalCustomScriptConfig]

s3: aiq.data_models.dataset_handler.EvalS3Config | None = None

cleanup: bool = True

workflow_output_step_filter: list[aiq.data_models.intermediate_step.IntermediateStepType] | None = None

class EvalGeneralConfig(/, **data: Any)

Bases: pydantic.BaseModel

max_concurrency: int = 8

output_dir: pathlib.Path

output: EvalOutputConfig | None = None

dataset: aiq.data_models.dataset_handler.EvalDatasetConfig | None = None

profiler: aiq.data_models.profiler.ProfilerConfig | None = None

classmethod override_output_dir(values)

class EvalConfig(/, **data: Any)

Bases: pydantic.BaseModel

general: EvalGeneralConfig

evaluators: dict[str, aiq.data_models.evaluator.EvaluatorBaseConfig]

classmethod rebuild_annotations()