nemo_gym.rollout_compare.schema

The vocabulary of the comparison.

The task key candidates, the embedded verifier fields, the sidecar flag, the file name constants, the Category and Side literals, the Roots alias, the input error and the three result dataclasses: one side of a task, one compared task and the whole result. Every other module of the package imports from here and this module imports from none of them.

Module Contents

Classes

Name	Description
`CompareInputError`	An input file could not be read or does not look like rollouts.
`CompareResult`	-
`TaskComparison`	-
`TaskSide`	One side’s rows for a task, reduced to what the comparison needs.

Data

Category

EMBEDDED_VERIFIER_FIELDS

TASK_KEY_CANDIDATES

VERIFIER_STDOUT_FILENAME

_T

API

class nemo_gym.rollout_compare.CompareInputError()

Exception

Bases: Exception

An input file could not be read or does not look like rollouts.

class nemo_gym.rollout_compare.CompareResult(
    tasks: list[nemo_gym.rollout_compare.schema.TaskComparison],
    old_key: str,
    new_key: str,
    old_rows: int,
    new_rows: int,
    old_masked_rows: int = 0,
    new_masked_rows: int = 0,
    old_prefix_stripped: typing.Optional[str] = None,
    new_prefix_stripped: typing.Optional[str] = None,
    keyed_by_taskset: bool = False,
    notes: list[str] = list()
)

Dataclass

keyed_by_taskset

bool = False

new_key

str

new_masked_rows

int = 0

new_prefix_stripped

Optional[str] = None

new_rows

int

notes

list[str] = field(default_factory=list)

old_key

str

old_masked_rows

int = 0

old_prefix_stripped

Optional[str] = None

old_rows

int

tasks

list[TaskComparison]

nemo_gym.rollout_compare.CompareResult.by_category(
    category: nemo_gym.rollout_compare.schema.Category
) -> list[nemo_gym.rollout_compare.schema.TaskComparison]

nemo_gym.rollout_compare.CompareResult.counts() -> dict[str, int]

class nemo_gym.rollout_compare.TaskComparison(
    task: str,
    category: nemo_gym.rollout_compare.schema.Category,
    old: typing.Optional[nemo_gym.rollout_compare.schema.TaskSide] = None,
    new: typing.Optional[nemo_gym.rollout_compare.schema.TaskSide] = None,
    winner: typing.Optional[nemo_gym.rollout_compare.schema.Side] = None
)

Dataclass

Category

loser

Optional[Side]

new

Optional[TaskSide] = None

old

Optional[TaskSide] = None

task

str

winner

Optional[Side] = None

class nemo_gym.rollout_compare.TaskSide(
    rows: list[dict[str, typing.Any]],
    reward: typing.Optional[float],
    masked: typing.Optional[str],
    agent_error: typing.Optional[str],
    reward_rows: int = 0,
    failure_kind: typing.Optional[str] = None
)

Dataclass

One side’s rows for a task, reduced to what the comparison needs.

agent_error

Optional[str]

failure_kind

Optional[str] = None

masked

Optional[str]

repeated

bool

reward

Optional[float]

reward_rows

int = 0

rows

list[dict[str, Any]]

nemo_gym.rollout_compare.TaskSide.reward_text() -> str

nemo_gym.rollout_compare.schema.Category = Literal['identical', 'flipped', 'flipped_agent_error', 'masked', 'missing']

nemo_gym.rollout_compare.schema.EMBEDDED_VERIFIER_FIELDS: tuple[str, ...] = ('test_output', 'verifier_stdout', 'verifier_output')

nemo_gym.rollout_compare.schema.FAILURES_SUFFIX = '_failures.jsonl'

nemo_gym.rollout_compare.schema.Roots = list[Path] | dict[str, list[Path]]

nemo_gym.rollout_compare.schema.SIDECAR_FLAG = '_ng_compare_sidecar'

nemo_gym.rollout_compare.schema.Side = Literal['old', 'new']

nemo_gym.rollout_compare.schema.TASK_KEY_CANDIDATES: tuple[str, ...] = ('_ng_task_id', 'task_name', 'task_id', 'verifier_metadata.task_id', 'verifier_m...

nemo_gym.rollout_compare.schema.VERIFIER_STDOUT_FILENAME = 'test-stdout.txt'

nemo_gym.rollout_compare.schema._T = TypeVar('_T')