Agent Evaluation Quickstart

1
from nemo_evaluator_sdk.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult
2
3
4
class KeywordMatchMetric:
5
    """Score 1.0 when the agent's answer contains the expected keyword, else 0.0."""
6
7
    @property
8
    def type(self) -> str:
9
        return "keyword_match"
10
11
    def output_spec(self) -> list[MetricOutputSpec]:
12
        return [MetricOutputSpec.continuous_score("score")]
13
14
    async def compute_scores(self, input: MetricInput) -> MetricResult:
15
        expected = str(input.row.data.get("reference", {}).get("expected", "")).lower()
16
        answer = (input.candidate.output_text or "").lower()
17
        return MetricResult(outputs=[MetricOutput(name="score", value=1.0 if expected and expected in answer else 0.0)])

1
from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalTask
2
3
4
def make_tasks() -> list[AgentEvalTask]:
5
    return [
6
        AgentEvalTask(
7
            id="capital-france",
8
            intent="Answer the geography question.",
9
            inputs={"instruction": "What is the capital of France?"},
10
            reference={"expected": "Paris"},
11
            metrics=[KeywordMatchMetric()],
12
        ),
13
        AgentEvalTask(
14
            id="capital-japan",
15
            intent="Answer the geography question.",
16
            inputs={"instruction": "What is the capital of Japan?"},
17
            reference={"expected": "Tokyo"},
18
            metrics=[KeywordMatchMetric()],
19
        ),
20
    ]

1
async def my_agent(task: AgentEvalTask) -> str:
2
    # Stand-in agent — replace with a real model or deployed agent later.
3
    answers = {
4
        "capital-france": "The capital of France is Paris.",
5
        "capital-japan": "The capital of Japan is Tokyo.",
6
    }
7
    return answers[task.id]

1
from nemo_evaluator_sdk.agent_eval.evaluator import AgentEvaluator
2
from nemo_evaluator_sdk.agent_eval.runtimes.callable_runtime import CallableAgentTaskRunner
3
from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig
4
5
result = await AgentEvaluator().run(
6
    tasks=make_tasks(),
7
    target=CallableAgentTaskRunner(my_agent),
8
    config=AgentEvalRunConfig(output_dir="./agent-eval-run", parallelism=2),
9
)
10
11
for aggregate in result.summary.scores.scores:
12
    print(f"{aggregate.name}: {aggregate.mean}")

1
import asyncio
2
3
from nemo_evaluator_sdk.agent_eval.evaluator import AgentEvaluator
4
from nemo_evaluator_sdk.agent_eval.runtimes.callable_runtime import CallableAgentTaskRunner
5
from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask
6
from nemo_evaluator_sdk.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult
7
8
9
class KeywordMatchMetric:
10
    """Score 1.0 when the agent's answer contains the expected keyword, else 0.0."""
11
12
    @property
13
    def type(self) -> str:
14
        return "keyword_match"
15
16
    def output_spec(self) -> list[MetricOutputSpec]:
17
        return [MetricOutputSpec.continuous_score("score")]
18
19
    async def compute_scores(self, input: MetricInput) -> MetricResult:
20
        expected = str(input.row.data.get("reference", {}).get("expected", "")).lower()
21
        answer = (input.candidate.output_text or "").lower()
22
        return MetricResult(outputs=[MetricOutput(name="score", value=1.0 if expected and expected in answer else 0.0)])
23
24
25
def make_tasks() -> list[AgentEvalTask]:
26
    return [
27
        AgentEvalTask(
28
            id="capital-france",
29
            intent="Answer the geography question.",
30
            inputs={"instruction": "What is the capital of France?"},
31
            reference={"expected": "Paris"},
32
            metrics=[KeywordMatchMetric()],
33
        ),
34
        AgentEvalTask(
35
            id="capital-japan",
36
            intent="Answer the geography question.",
37
            inputs={"instruction": "What is the capital of Japan?"},
38
            reference={"expected": "Tokyo"},
39
            metrics=[KeywordMatchMetric()],
40
        ),
41
    ]
42
43
44
async def my_agent(task: AgentEvalTask) -> str:
45
    # Stand-in agent — replace with a real model or deployed agent later.
46
    answers = {
47
        "capital-france": "The capital of France is Paris.",
48
        "capital-japan": "The capital of Japan is Tokyo.",
49
    }
50
    return answers[task.id]
51
52
53
async def main() -> None:
54
    result = await AgentEvaluator().run(
55
        tasks=make_tasks(),
56
        target=CallableAgentTaskRunner(my_agent),
57
        config=AgentEvalRunConfig(output_dir="./agent-eval-run", parallelism=2),
58
    )
59
    for aggregate in result.summary.scores.scores:
60
        print(f"{aggregate.name}: {aggregate.mean}")
61
62
63
asyncio.run(main())