nat.eval.evaluator.evaluator_model#
Classes#
Module Contents#
- class EvalInputItem(/, **data: Any)#
Bases:
pydantic.BaseModel- id: Any#
- input_obj: Any#
- expected_output_obj: Any#
- output_obj: Any = None#
- expected_trajectory: list[nat.data_models.intermediate_step.IntermediateStep] = []#
- trajectory: list[nat.data_models.intermediate_step.IntermediateStep] = []#
- full_dataset_entry: Any#
- copy_with_updates(**updates) EvalInputItem#
Copy EvalInputItem with optional field updates.
- class EvalInput(/, **data: Any)#
Bases:
pydantic.BaseModel- eval_input_items: list[EvalInputItem]#
- class EvalOutputItem(/, **data: Any)#
Bases:
pydantic.BaseModel- id: Any#
- score: Any#
- reasoning: Any#
- class EvalOutput(/, **data: Any)#
Bases:
pydantic.BaseModel- average_score: Any#
- eval_output_items: list[EvalOutputItem]#