Evaluate with a NeMo Fabric Harness

1
config = {
2
    "schema_version": "fabric.agent/v1alpha1",
3
    "metadata": {"name": "eval-fabric"},
4
    "harness": {
5
        "adapter_id": "nvidia.fabric.codex",
6
        "resolution": "preinstalled",
7
        "settings": {"sandbox": "workspace-write"},
8
    },
9
    "runtime": {
10
        "mode": "oneshot",
11
        "transport": "cli",
12
        "input_schema": "text",
13
        "output_schema": "message",
14
        "timeout_seconds": 180,
15
    },
16
    "environment": {"provider": "local"},
17
    "models": {"default": {"provider": "openai", "model": "<provider-model>"}},
18
    "telemetry": {"enabled": False},
19
}

1
from pathlib import Path
2
3
from nemo_evaluator_sdk.agent_eval.evaluator import AgentEvaluator
4
from nemo_evaluator_sdk.agent_eval.runtimes.fabric.runtime import FabricAgentRuntime
5
from nemo_evaluator_sdk import StringCheckMetric
6
from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask
7
8
tasks = [
9
    AgentEvalTask(
10
        id="reply-ok",
11
        intent="Reply with a fixed token.",
12
        inputs={"instruction": "Reply with the word OK."},
13
        metrics=[
14
            StringCheckMetric(
15
                operation="contains",
16
                left_template="{{sample.output_text}}",
17
                right_template="OK",
18
            )
19
        ],
20
    )
21
]
22
23
runtime = FabricAgentRuntime(
24
    config=config,
25
    work_root="fabric",
26
    capture_trajectory=True,
27
)
28
29
result = AgentEvaluator().run_sync(
30
    tasks=tasks,
31
    target=runtime,
32
    config=AgentEvalRunConfig(work_dir=Path("out"), parallelism=1),
33
)

1
task = AgentEvalTask(
2
    id="read-seed",
3
    intent="Read a seeded file and echo its contents.",
4
    inputs={
5
        "instruction": "Read notes.txt in your workspace and reply with exactly its contents.",
6
        "files": {"notes.txt": "MARKER-7f3a"},
7
    },
8
    metrics=[
9
        StringCheckMetric(
10
            operation="contains",
11
            left_template="{{sample.output_text}}",
12
            right_template="MARKER-7f3a",
13
        )
14
    ],
15
)

1
trial = result.trials[0]
2
print(trial.status)  # AgentEvalTrialStatus.COMPLETED
3
assert trial.output is not None
4
print(trial.output.output_text)  # MARKER-7f3a
5
6
for score in result.summary.scores.scores:
7
    print(score.name)
8
# string-check.string-check
9
# string-check.string-check.pass@1

1
assert trial.evidence is not None
2
trace = trial.evidence.descriptors["trace"]
3
# trace.format == "atif"; trace.ref is a path to the trajectory JSON, which has a "steps" key
4
5
workspace = trial.evidence.descriptors["workspace"]
6
# workspace.ref is a directory — what the agent left behind, including any seeded files

1
from nemo_evaluator.jobs.agent_spec import FabricRunnerTarget
2
3
target = FabricRunnerTarget(config=config, model="<provider>/<model>", capture_trajectory=True)

1
from nemo_evaluator_sdk.agent_eval.runtimes.fabric.container_runtime import FabricContainerRuntime
2
from nemo_evaluator_sdk.agent_eval.runtimes.sandbox.providers.docker import DockerSandboxProvider
3
4
runtime = FabricContainerRuntime(config=config, provider=DockerSandboxProvider())