nemo_gym.comparison.schema

Config and result schema for gym eval compare.

Every candidate-varying field is a list, positionally parallel to ComparisonResult.candidates. v0 only ever compares one candidate, but keeping the shape list-valued means lifting that restriction later is a behavior change rather than a schema change.

Module Contents

Classes

Name	Description
`AgentComparison`	-
`CandidateMetricValue`	A candidate’s reading, plus its movement from the baseline.
`ComparisonConfig`	Compare a baseline eval run against a candidate run.
`ComparisonResult`	The machine-readable artifact written as `compare_report.json`.
`FlipSummary`	-
`MetricRow`	-
`MetricValue`	One side’s reading of a metric, plus whatever uncertainty the run recorded for it.
`RunFile`	One run’s parsed `*_aggregate_metrics.json`, indexed by agent name.
`TaskFlip`	-

Data

MAX_CANDIDATES

MAX_FLIPS_SHOWN

ReportFormat

API

class nemo_gym.comparison.schema.AgentComparison()

Bases: BaseModel

baseline_agent

str

baseline_repeat_count

Optional[int] = None

baseline_task_count

int

candidate_agents

List[str]

candidate_repeat_counts

List[Optional[int]] = Field(default_factory=list)

candidate_task_counts

List[int]

flips

List[FlipSummary] = Field(default_factory=list)

metrics

List[MetricRow] = Field(default_factory=list)

notes

List[str] = Field(default_factory=list)

class nemo_gym.comparison.schema.CandidateMetricValue()

Bases: MetricValue

A candidate’s reading, plus its movement from the baseline.

delta

Optional[float] = None

delta_pct

Optional[float] = None

class nemo_gym.comparison.schema.ComparisonConfig()

Bases: BaseNeMoGymCLIConfig

Compare a baseline eval run against a candidate run.

Reads only each run’s <stem>_aggregate_metrics.json, derived from the rollouts JSONL path you pass, and writes compare_report.md and/or compare_report.json. The rollouts JSONL itself is never opened — it is the run’s identity and the handle the sibling path is derived from.

Examples:

gym eval compare         --baseline outputs/run_a/rollouts.jsonl         --candidates outputs/run_b/rollouts.jsonl

To point at metrics files that do not follow the <stem>_aggregate_metrics.json convention, set baseline_aggregate_metrics_fpath and candidate_aggregate_metrics_fpaths.

agent_name

Optional[str]

baseline_agent_name

Optional[str]

baseline_aggregate_metrics_fpath

Optional[str]

baseline_rollouts_jsonl_fpath

str

candidate_agent_names

Optional[List[str]]

candidate_aggregate_metrics_fpaths

Optional[List[str]]

candidate_rollouts_jsonl_fpaths

List[str]

output_dirpath

Optional[str]

report_format

ReportFormat

nemo_gym.comparison.schema.ComparisonConfig._check_candidate_parallel_lists() -> nemo_gym.comparison.schema.ComparisonConfig

class nemo_gym.comparison.schema.ComparisonResult()

Bases: BaseModel

The machine-readable artifact written as compare_report.json.

baseline

RunFile

candidates

List[RunFile]

command

str

comparisons

List[AgentComparison] = Field(default_factory=list)

generated_at

str

nemo_gym_version

str

schema_version

Literal['1'] = '1'

skipped_agents

Dict[str, List[str]] = Field(default_factory=dict)

warnings

List[str] = Field(default_factory=list)

class nemo_gym.comparison.schema.FlipSummary()

Bases: BaseModel

baseline_only_task_count

int = 0

candidate_index

int

candidate_only_task_count

int = 0

common_task_count

int = 0

fail_to_pass_count

Optional[int] = None

field

str = 'reward'

flips

List[TaskFlip] = Field(default_factory=list)

mode

Literal['binary', 'continuous', 'unavailable']

net

Optional[int] = None

pass_to_fail_count

Optional[int] = None

reason

Optional[str] = None

tied_count

Optional[int] = None

unchanged_count

Optional[int] = None

class nemo_gym.comparison.schema.MetricRow()

Bases: BaseModel

baseline

Optional[MetricValue] = None

candidates

List[Optional[CandidateMetricValue]] = Field(default_factory=list)

is_key_metric

bool

metric

str

present_in

List[str]

class nemo_gym.comparison.schema.MetricValue()

Bases: BaseModel

One side’s reading of a metric, plus whatever uncertainty the run recorded for it.

ci_high

Optional[float] = None

ci_low

Optional[float] = None

mean_across_repeats

Optional[float] = None

se_across_repeats

Optional[float] = None

std_err_across_runs

Optional[float] = None

value

float

class nemo_gym.comparison.schema.RunFile()

Bases: BaseModel

One run’s parsed *_aggregate_metrics.json, indexed by agent name.

agent_names

List[str]

aggregate_metrics_fpath

Path

entries_by_agent

Dict[str, Any]

index

int = 0

label

str

Short display name for the run: the directory its rollouts live in.

model_config

= ConfigDict(frozen=True)

role

Literal['baseline', 'candidate']

rollouts_jsonl_fpath

Path

class nemo_gym.comparison.schema.TaskFlip()

Bases: BaseModel

baseline_rewards

Optional[List[float]] = None

baseline_score

float

candidate_rewards

Optional[List[float]] = None

candidate_score

float

delta

float

direction

Literal['pass_to_fail', 'fail_to_pass', 'changed']

task_index

int

nemo_gym.comparison.schema.MAX_CANDIDATES = 1

nemo_gym.comparison.schema.MAX_FLIPS_SHOWN = 10

nemo_gym.comparison.schema.ReportFormat = Literal['md', 'json', 'both']