nat.plugins.eval.runtime.builder#

Attributes#

logger

Classes#

`ConfiguredEvaluator`
`ConfiguredDatasetLoader`
`WorkflowEvalBuilder`

Module Contents#

logger#

class ConfiguredEvaluator#

config: nat.data_models.evaluator.EvaluatorBaseConfig#

instance: nat.builder.evaluator.EvaluatorInfo#

class ConfiguredDatasetLoader#

config: nat.data_models.dataset_handler.EvalDatasetBaseConfig#

instance: nat.builder.dataset_loader.DatasetLoaderInfo#

class WorkflowEvalBuilder( general_config: nat.data_models.config.GeneralConfig | None = None, eval_general_config: nat.data_models.evaluate_config.EvalGeneralConfig | None = None, registry: nat.cli.type_registry.TypeRegistry | None = None, )#

Bases: nat.builder.workflow_builder.WorkflowEvalBuilderBase

eval_general_config = None#

_evaluators: dict[str, ConfiguredEvaluator]#

_dataset_loaders: dict[str, ConfiguredDatasetLoader]#

async add_evaluator( name: str, config: nat.data_models.evaluator.EvaluatorBaseConfig, )#

get_evaluator( evaluator_name: str, ) → nat.builder.evaluator.EvaluatorInfo#

get_evaluator_config( evaluator_name: str, ) → nat.data_models.evaluator.EvaluatorBaseConfig#

async add_dataset_loader( name: str, config: nat.data_models.dataset_handler.EvalDatasetBaseConfig, )#

get_dataset_loader( dataset_loader_name: str, ) → nat.builder.dataset_loader.DatasetLoaderInfo#

get_dataset_loader_config( dataset_loader_name: str, ) → nat.data_models.dataset_handler.EvalDatasetBaseConfig#

get_max_concurrency() → int#

get_output_dir() → pathlib.Path#

async get_all_tools( wrapper_type: nat.builder.framework_enum.LLMFrameworkEnum | str, )#

_log_build_failure_evaluator( failing_evaluator_name: str, completed_evaluators: list[str], remaining_evaluators: list[str], original_error: Exception, ) → None#

Log comprehensive evaluator build failure information.

Args:: failing_evaluator_name (str): The name of the evaluator that failed to build completed_evaluators (list[str]): List of evaluator names that were successfully built remaining_evaluators (list[str]): List of evaluator names still to be built original_error (Exception): The original exception that caused the failure

async populate_builder( config: nat.data_models.config.Config, skip_workflow: bool = False, )#

classmethod from_config(config: nat.data_models.config.Config)#

Async: