nat.plugins.langchain.eval.tunable_rag_evaluator#

Attributes#

logger

Classes#

`TunableRagEvaluatorConfig`	Configuration for tunable RAG evaluator.
`TunableRagEvaluator`	Tunable RAG evaluator with customizable judge prompt.

Functions#

`evaluation_prompt`(→ str)	Generate a prompt for the judge LLM.
`runnable_with_retries`(original_fn[, ...])	Wrap a runnable with retry controls.
`register_tunable_rag_evaluator`(config, builder)	Register tunable RAG evaluator.

Module Contents#

logger#

class TunableRagEvaluatorConfig#

Bases: nat.data_models.evaluator.EvaluatorBaseConfig

Configuration for tunable RAG evaluator.

llm_name: nat.data_models.component_ref.LLMRef = None#

llm_retry_control_params: dict | None = None#

judge_llm_prompt: str = None#

default_scoring: bool = None#

default_score_weights: dict = None#

enable_atif_evaluator: bool = None#

evaluation_prompt( judge_llm_prompt: str, question: str, answer_description: str, generated_answer: str, format_instructions: str, default_scoring: bool, ) → str#: Generate a prompt for the judge LLM.

runnable_with_retries( original_fn: collections.abc.Callable, llm_retry_control_params: dict | None = None, )#: Wrap a runnable with retry controls.

class TunableRagEvaluator( llm: langchain_core.language_models.BaseChatModel, judge_llm_prompt: str, llm_retry_control_params: dict | None, max_concurrency: int, default_scoring: bool, default_score_weights: dict, )#

Bases: nat.plugins.eval.evaluator.base_evaluator.BaseEvaluator

Tunable RAG evaluator with customizable judge prompt.

llm#

judge_llm_prompt#

llm_retry_control_params#

default_scoring#

default_score_weights#

async _evaluate_item_core( item_id, question: str, answer_description: str, generated_answer: str, ) → nat.plugins.eval.data_models.evaluator_io.EvalOutputItem#

async evaluate_item( item: nat.data_models.evaluator.EvalInputItem, ) → nat.plugins.eval.data_models.evaluator_io.EvalOutputItem#: Each evaluator must implement this for item-level evaluation

static _content_part_to_text(part: nat.atif.ATIFContentPart) → str#

classmethod _message_to_text( message: str | list[nat.atif.ATIFContentPart] | None, ) → str#

classmethod _trajectory_to_user_input(trajectory: nat.atif.ATIFTrajectory) → str#

async evaluate_atif_item( sample: nat.plugins.eval.evaluator.atif_evaluator.AtifEvalSample, ) → nat.plugins.eval.data_models.evaluator_io.EvalOutputItem#

async evaluate_atif_fn( atif_samples: nat.plugins.eval.evaluator.atif_evaluator.AtifEvalSampleList, ) → nat.plugins.eval.data_models.evaluator_io.EvalOutput#

async register_tunable_rag_evaluator( config: TunableRagEvaluatorConfig, builder: nat.builder.builder.EvalBuilder, )#: Register tunable RAG evaluator.