Bring Your Own Metric

1
import os
2
3
from nemo_evaluator.sdk import Evaluator
4
from nemo_platform import NeMoPlatform
5
6
7
from nemo_evaluator_sdk import (
8
    NemoAgentToolkitRemoteMetric,
9
    RemoteMetric,
10
)
11
sdk = NeMoPlatform(
12
    base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"),
13
    workspace="default",
14
)
15
evaluator: Evaluator = sdk.evaluator  # this object is an Evaluator resource

1
from nemo_evaluator_sdk import JSONScoreParser, RemoteScore, RemoteMetric
2
3
metric = RemoteMetric(
4
    url="https://my-evaluation-server.test/evaluate",
5
    body={
6
        "reference": "{{item.reference}}",
7
        "response": "{{item.output}}",
8
    },
9
    scores=[
10
        RemoteScore(
11
            name="accuracy",
12
            parser=JSONScoreParser(json_path="$.result.accuracy"),
13
            description="Measures response accuracy against the reference",
14
            minimum=0.0,
15
            maximum=1.0,
16
        )
17
    ],
18
    timeout_seconds=30.0,
19
    max_retries=3,
20
)
21
22
23
result = evaluator.run(
24
    metric=metric,
25
    dataset=[
26
        {"reference": "The capital is Paris", "output": "Paris is the capital"},
27
        {"reference": "2", "output": "2"},
28
    ],
29
)
30
31
for score in result.aggregate_scores.scores:
32
    print(f"{score.name}: mean={score.mean}, count={score.count}")

1
from nemo_evaluator_sdk import NemoAgentToolkitRemoteMetric
2
3
metric = NemoAgentToolkitRemoteMetric(
4
    url="http://localhost:8001/evaluate_item",
5
    evaluator_name="similarity_eval",
6
    timeout_seconds=30.0,
7
    max_retries=3,
8
)
9
10
11
result = evaluator.run(
12
    metric=metric,
13
    dataset=[
14
        {
15
            "id": "item_1",
16
            "input_obj": "What is the capital of France?",
17
            "expected_output_obj": "The capital of France is Paris.",
18
            "output_obj": "Paris is the capital of France.",
19
            "trajectory": [],
20
            "expected_trajectory": [],
21
            "full_dataset_entry": {},
22
        }
23
    ],
24
)
25
26
for score in result.aggregate_scores.scores:
27
    print(f"{score.name}: mean={score.mean}")

1
from nemo_evaluator_sdk import JSONScoreParser, RemoteScore, SecretRef, RemoteMetric
2
3
metric = RemoteMetric(
4
    url="https://my-authenticated-endpoint.test/evaluate",
5
    body={"input": "{{item.input}}"},
6
    scores=[RemoteScore(name="score", parser=JSONScoreParser(json_path="$.score"))],
7
    api_key_secret=SecretRef(root="my-remote-api-key"),
8
)
9
10
result = evaluator.run(metric=metric, dataset=[{"input": "test"}])

1
from fastapi import FastAPI
2
from pydantic import BaseModel
3
4
5
app = FastAPI()
6
7
8
class MetricRequest(BaseModel):
9
    reference: str
10
    response: str
11
12
13
class MetricResponse(BaseModel):
14
    result: dict[str, float]
15
16
17
@app.post("/evaluate")
18
async def evaluate(request: MetricRequest) -> MetricResponse:
19
    accuracy = 1.0 if request.reference == request.response else 0.0
20
    return MetricResponse(result={"accuracy": accuracy})

1
from nemo_evaluator_sdk import JSONScoreParser, RemoteScore
2
3
RemoteScore(
4
    name="accuracy",
5
    parser=JSONScoreParser(json_path="$.result.accuracy"),
6
    description="Measures response accuracy against reference",
7
    minimum=0.0,
8
    maximum=1.0,
9
)