1
import os
2
import uuid
3
4
from nemo_evaluator.sdk import Evaluator
5
from nemo_platform import ConflictError, NeMoPlatform
6
7
# Workspace for this tutorial.
8
WORKSPACE = "llm-as-judge-tutorial"
9
10
# Create an SDK instance to interact with the NeMo Platform.
11
client = NeMoPlatform(
12
    base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"),
13
    workspace=WORKSPACE,
14
)
15
evaluator: Evaluator = client.evaluator  # this object is an Evaluator resource
16
17
# Create the workspace
18
try:
19
    client.workspaces.create(name=WORKSPACE)
20
    print(f"Workspace '{WORKSPACE}' created")
21
except ConflictError:
22
    print(f"Workspace '{WORKSPACE}' already exists, continuing...")
23
24
# Verify evaluator plugin connectivity.
25
print(evaluator.plugin_status())

1
# Export NVIDIA_API_KEY before running this tutorial.
2
# For quick local testing only:
3
# os.environ["NVIDIA_API_KEY"] = "<your-nvidia-api-key>"
4
NVIDIA_BUILD_API_KEY_ENV = "NVIDIA_API_KEY"
5
6
nvidia_api_key = os.getenv(NVIDIA_BUILD_API_KEY_ENV)
7
if not nvidia_api_key:
8
    raise ValueError(f"{NVIDIA_BUILD_API_KEY_ENV} is not set")
9
10
try:
11
    nvidia_api_key_secret = client.secrets.create(
12
        name="nvidia-api-key",
13
        workspace=WORKSPACE,
14
        value=nvidia_api_key,
15
    )
16
    print("Created secret: nvidia-api-key")
17
except ConflictError:
18
    print("Secret 'nvidia-api-key' already exists, retrieving...")
19
    nvidia_api_key_secret = client.secrets.retrieve(
20
        name="nvidia-api-key",
21
        workspace=WORKSPACE,
22
    )
23
    print("Retrieved existing secret: nvidia-api-key")
24
25
print(
26
    f"NVIDIA_API_KEY secret reference: {nvidia_api_key_secret.workspace}/{nvidia_api_key_secret.name}"
27
)

1
from nemo_evaluator_sdk import RunConfig, LLMJudgeMetric
2
from nemo_evaluator_sdk.values import (
3
    FilesetRef,
4
    InferenceParams,
5
    JSONScoreParser,
6
    Model,
7
    RangeScore,
8
)
9
10
JUDGE_MODEL_URL = "https://integrate.api.nvidia.com/v1/chat/completions"
11
JUDGE_MODEL_NAME = "nvidia/nemotron-3-nano-30b-a3b"
12
13
# Local evaluator.run() resolves this value from os.environ["NVIDIA_API_KEY"].
14
LOCAL_JUDGE_MODEL = Model(
15
    url=JUDGE_MODEL_URL,
16
    name=JUDGE_MODEL_NAME,
17
    api_key_secret=NVIDIA_BUILD_API_KEY_ENV,
18
)
19
20
# Remote evaluator.submit() resolves this value from the platform secret.
21
REMOTE_JUDGE_MODEL = Model(
22
    url=JUDGE_MODEL_URL,
23
    name=JUDGE_MODEL_NAME,
24
    api_key_secret=nvidia_api_key_secret.name,
25
)

1
from nemo_platform.types.files import HuggingfaceStorageConfigParam
2
3
DATASET_NAME = "helpsteer2-eval"
4
DATASET_SPLIT_PATH = "validation.jsonl.gz"
5
6
try:
7
    fileset = client.files.filesets.create(
8
        name=DATASET_NAME,
9
        description="NVIDIA HelpSteer2 dataset for quality evaluation",
10
        storage=HuggingfaceStorageConfigParam(
11
            type="huggingface",
12
            repo_id="nvidia/HelpSteer2",
13
            repo_type="dataset",
14
        ),
15
        metadata={
16
            "dataset": {
17
                "schema": {
18
                    "type": "object",
19
                    "properties": {
20
                        "prompt": {"type": "string"},
21
                        "response": {"type": "string"},
22
                        "helpfulness": {"type": "number"},
23
                        "correctness": {"type": "number"},
24
                        "coherence": {"type": "number"},
25
                        "complexity": {"type": "number"},
26
                        "verbosity": {"type": "number"},
27
                    },
28
                    "required": ["prompt", "response"],
29
                    "additionalProperties": True,
30
                }
31
            },
32
        },
33
    )
34
    print(f"Registered HelpSteer2 fileset: {fileset.workspace}/{fileset.name}")
35
except ConflictError:
36
    fileset = client.files.filesets.retrieve(name=DATASET_NAME)
37
    print(f"{fileset.workspace}/{fileset.name} dataset already registered")

1
dataset_ref = FilesetRef(root=f"{fileset.workspace}/{fileset.name}").with_fragment(
2
    DATASET_SPLIT_PATH
3
)
4
5
print(f"Using dataset reference: {dataset_ref.root}")

1
def create_helpfulness_metric(prompt_template: str, judge_model: Model) -> LLMJudgeMetric:
2
    """Create a helpfulness metric with the given system prompt."""
3
    score = RangeScore(
4
        name="helpfulness",
5
        description="How well does the response help the user?",
6
        minimum=0,
7
        maximum=4,
8
        parser=JSONScoreParser(json_path="helpfulness"),
9
    )
10
11
    return LLMJudgeMetric(
12
        model=judge_model,
13
        scores=[score],
14
        prompt_template={
15
            "messages": [
16
                {"role": "system", "content": prompt_template},
17
                {
18
                    "role": "user",
19
                    "content": (
20
                        "User prompt: {{item.prompt}}\n\n"
21
                        "Assistant response: {{item.response}}\n\n"
22
                        "Rate this response."
23
                    ),
24
                },
25
            ],
26
        },
27
        inference=InferenceParams(temperature=0.0, max_tokens=32768),
28
        ignore_request_failure=True,
29
    )
30
31
32
# Version 1: A basic, minimal prompt.
33
PROMPT_V1 = """You are an evaluator. Rate the response's helpfulness from 0-4.
34
Respond with JSON only: {"helpfulness": <0-4>}"""
35
36
metric_v1_local = create_helpfulness_metric(PROMPT_V1, LOCAL_JUDGE_MODEL)
37
metric_v1_remote = create_helpfulness_metric(PROMPT_V1, REMOTE_JUDGE_MODEL)

1
quick_test_result = evaluator.run(
2
    metric=metric_v1_local,
3
    dataset=[
4
        {
5
            "prompt": "What is the capital of France?",
6
            "response": "The capital of France is Paris. It serves as France's political, economic, and cultural center.",
7
        },
8
        {
9
            "prompt": "How do I make scrambled eggs?",
10
            "response": "Eggs.",
11
        },
12
    ],
13
    config=RunConfig(parallelism=1),
14
)
15
16
17
def score_value(row_score, score_name: str) -> float | None:
18
    """Return one named score from an evaluator row result."""
19
    for metric_result in row_score.metrics.values():
20
        scores = getattr(metric_result, "scores", metric_result)
21
        for score in scores:
22
            if score.name == score_name:
23
                try:
24
                    return float(score.value)
25
                except (TypeError, ValueError):
26
                    return None
27
    return None
28
29
30
print("Quick test results:")
31
for row in quick_test_result.row_scores:
32
    helpfulness = score_value(row, "helpfulness")
33
    print(f" Row {row.row_index}: helpfulness = {helpfulness}")

1
sample_config = RunConfig(
2
    parallelism=1,
3
    limit_samples=5,
4
)
5
6
7
def wait_for_job(label: str, job):
8
    """Wait for a remote evaluator job and download its result."""
9
    try:
10
        status = job.get_job_status()
11
        print(f"{label} initial status: {status.status}")
12
13
        job.wait_until_done()
14
        result = job.get_result()
15
    except Exception:
16
        try:
17
            print(f"{label} failure status: {job.get_job_status()}")
18
        except Exception as status_error:
19
            print(f"{label} status retrieval failed: {status_error!r}")
20
        raise
21
22
    print(f"{label} job finished successfully!")
23
    print(result.aggregate_scores.scores)
24
    return result
25
26
27
job_v1 = evaluator.submit(
28
    metric=metric_v1_remote,
29
    dataset=dataset_ref,
30
    config=sample_config,
31
)
32
print(f"Job submitted: {job_v1.name}")

1
result_v1 = wait_for_job("Prompt V1", job_v1)

1
import numpy as np
2
3
4
def extract_dimension_scores(result, dimension: str):
5
    """Return judge scores, human scores, and failure count for one dimension."""
6
    judge_scores = []
7
    human_scores = []
8
    failed_rows = 0
9
10
    for row in result.row_scores:
11
        judge_score = score_value(row, dimension)
12
        human_score = row.item.get(dimension)
13
14
        if judge_score is None or human_score is None:
15
            failed_rows += 1
16
            continue
17
18
        judge_scores.append(judge_score)
19
        human_scores.append(float(human_score))
20
21
    print(f"Total skipped rows: {failed_rows}")
22
    return np.array(judge_scores), np.array(human_scores)
23
24
25
judge_scores, human_scores = extract_dimension_scores(result_v1, "helpfulness")
26
print(f"Evaluated: {len(judge_scores)} samples")

1
from scipy import stats
2
3
4
def correlation_summary(human_scores, judge_scores):
5
    """Return Pearson, Spearman, and MAE for valid score arrays."""
6
    if len(human_scores) < 2 or len(judge_scores) < 2:
7
        return np.nan, np.nan, np.nan
8
9
    pearson, _ = stats.pearsonr(human_scores, judge_scores)
10
    spearman, _ = stats.spearmanr(human_scores, judge_scores)
11
    mae = abs(human_scores - judge_scores).mean()
12
    return pearson, spearman, mae
13
14
15
pearson_v1, spearman_v1, mae_v1 = correlation_summary(human_scores, judge_scores)
16
17
print("\n=== Prompt V1 Results ===")
18
print(f"Pearson r: {pearson_v1:.3f} (>0.6 is good)")
19
print(f"Spearman rho: {spearman_v1:.3f} (>0.6 is good)")
20
print(f"MAE: {mae_v1:.2f} (<0.5 is good)")

1
# Version 2: Prompt aligned with HelpSteer2 annotation guidelines.
2
PROMPT_V2 = """You are evaluating the HELPFULNESS of an AI assistant's response.
3
4
Helpfulness measures the overall utility of the response in addressing the user's needs.
5
Rate on a 0-4 integer scale:
6
7
0 - The response fails to address the user's request, provides irrelevant information, or could cause harm.
8
1 - The response partially addresses the request but has significant gaps, errors, or misunderstandings.
9
2 - The response addresses the core request adequately but may lack detail, clarity, or completeness.
10
3 - The response fully addresses the request with appropriate detail and is genuinely useful.
11
4 - The response excellently addresses the request, providing comprehensive and well-structured information that fully satisfies the user's needs.
12
13
Focus on whether the response helps the user accomplish their goal, not on style or verbosity.
14
A shorter response that directly solves the problem can score higher than a longer one that misses the point.
15
16
Respond with JSON only: {"helpfulness": <0-4>}"""
17
18
metric_v2_local = create_helpfulness_metric(PROMPT_V2, LOCAL_JUDGE_MODEL)
19
metric_v2_remote = create_helpfulness_metric(PROMPT_V2, REMOTE_JUDGE_MODEL)

1
job_v2 = evaluator.submit(
2
    metric=metric_v2_remote,
3
    dataset=dataset_ref,
4
    config=sample_config,
5
)
6
print(f"Job submitted: {job_v2.name}")
7
8
result_v2 = wait_for_job("Prompt V2", job_v2)
9
10
judge_scores_v2, human_scores_v2 = extract_dimension_scores(result_v2, "helpfulness")
11
print(f"Evaluated: {len(judge_scores_v2)} samples")

1
# Calculate metrics for V2.
2
pearson_v2, spearman_v2, mae_v2 = correlation_summary(human_scores_v2, judge_scores_v2)
3
4
# Side-by-side comparison.
5
print("\n=== Prompt Comparison ===")
6
print(f"{'Metric':<12} {'Pearson':<12} {'Spearman':<12} {'MAE':<8}")
7
print("-" * 44)
8
print(f"{'Prompt V1':<12} {pearson_v1:<12.3f} {spearman_v1:<12.3f} {mae_v1:<8.2f}")
9
print(f"{'Prompt V2':<12} {pearson_v2:<12.3f} {spearman_v2:<12.3f} {mae_v2:<8.2f}")
10
11
if pearson_v2 > pearson_v1:
12
    best_prompt = PROMPT_V2
13
    print("\nPrompt V2 shows better correlation with human judgments!")
14
else:
15
    best_prompt = PROMPT_V1
16
    print("\nPrompt V1 performs better; simpler prompts can work well.")

1
import matplotlib.pyplot as plt
2
3
fig, axes = plt.subplots(1, 3, figsize=(14, 4))
4
5
# Human scores distribution.
6
axes[0].hist(
7
    human_scores,
8
    bins=range(6),
9
    align="left",
10
    alpha=0.7,
11
    color="green",
12
    edgecolor="black",
13
)
14
axes[0].set_xlabel("Score")
15
axes[0].set_ylabel("Count")
16
axes[0].set_title("Human Annotations")
17
axes[0].set_xticks(range(5))
18
19
# Judge V1 distribution.
20
axes[1].hist(
21
    judge_scores,
22
    bins=range(6),
23
    align="left",
24
    alpha=0.7,
25
    color="blue",
26
    edgecolor="black",
27
)
28
axes[1].set_xlabel("Score")
29
axes[1].set_title(f"Judge V1 (r={pearson_v1:.2f})")
30
axes[1].set_xticks(range(5))
31
32
# Judge V2 distribution.
33
axes[2].hist(
34
    judge_scores_v2,
35
    bins=range(6),
36
    align="left",
37
    alpha=0.7,
38
    color="orange",
39
    edgecolor="black",
40
)
41
axes[2].set_xlabel("Score")
42
axes[2].set_title(f"Judge V2 (r={pearson_v2:.2f})")
43
axes[2].set_xticks(range(5))
44
45
plt.suptitle("Helpfulness Score Distributions")
46
plt.tight_layout()
47
plt.savefig("score_distributions.png", dpi=150)
48
plt.show()
49
50
print("Saved visualization to score_distributions.png")

1
import pandas as pd
2
3
for name, scores in [
4
    ("Human", human_scores),
5
    ("Judge V1", judge_scores),
6
    ("Judge V2", judge_scores_v2),
7
]:
8
    p25 = pd.Series(scores).quantile(0.25)
9
    p50 = pd.Series(scores).quantile(0.50)
10
    p75 = pd.Series(scores).quantile(0.75)
11
    p90 = pd.Series(scores).quantile(0.90)
12
13
    print(f"\n{name} percentiles:")
14
    print(
15
        f" 25th: {p25:.1f} | 50th (median): {p50:.1f} | 75th: {p75:.1f} | 90th: {p90:.1f}"
16
    )

1
from nemo_platform import NotFoundError
2
3
# Delete remote evaluation jobs. Local evaluator.run() results are in-memory
4
# objects and do not create platform jobs.
5
for job_name in [job_v1.name, job_v2.name]:
6
    try:
7
        client.jobs.delete(name=job_name, workspace=WORKSPACE)
8
    except Exception:
9
        pass
10
11
# Delete fileset and secret.
12
try:
13
    client.files.filesets.delete(name=DATASET_NAME, workspace=WORKSPACE)
14
except NotFoundError:
15
    pass
16
17
try:
18
    client.secrets.delete(name=nvidia_api_key_secret.name, workspace=WORKSPACE)
19
except NotFoundError:
20
    pass
21
22
# Now delete the workspace.
23
client.workspaces.delete(name=WORKSPACE)
24
print("Cleanup complete!")

1
client.workspaces.delete(name=WORKSPACE)

1
import os
2
3
assert os.environ["NVIDIA_API_KEY"]

1
LOCAL_JUDGE_MODEL = Model(
2
    url=JUDGE_MODEL_URL,
3
    name=JUDGE_MODEL_NAME,
4
    api_key_secret="NVIDIA_API_KEY",
5
)

1
REMOTE_JUDGE_MODEL = Model(
2
    url=JUDGE_MODEL_URL,
3
    name=JUDGE_MODEL_NAME,
4
    api_key_secret=nvidia_api_key_secret.name,
5
)

1
status = job_v1.get_job_status()
2
print(f"Status: {status.status}")
3
print(f"Details: {status.status_details}")

1
client.secrets.create(name="hf-token", value="hf_your_token_here")

1
client.files.filesets.create(
2
    name="my-dataset",
3
    storage=HuggingfaceStorageConfigParam(
4
        type="huggingface",
5
        repo_id="org/dataset",
6
        repo_type="dataset",
7
        token_secret="hf-token",
8
    ),
9
)