nemo_gym.verification.verifier_fixture

In-process contract tests for resources-server verifiers.

Module Contents

Classes

Name	Description
`VerifierFixture`	-
`VerifierFixtureCase`	-
`VerifierFixtureError`	A fixture is invalid or does not match verifier behavior.
`VerifierFixtureResult`	-

Functions

Name	Description
`_assert_equal`	-
`_observe`	-
`_reward_from`	-
`_write_fixture`	-
`exercise_verifier_fixture`	Exercise required scorer cases without starting a resources server.
`load_verifier_fixture`	Load and validate resources-server-owned JSONL cases.

Data

_CaseKind

_FiniteReward

_FixtureInvocation

API

class nemo_gym.verification.verifier_fixture.VerifierFixture(
    server_factory: collections.abc.Callable[[], object],
    request_model: type[pydantic.BaseModel],
    cases_path: str | pathlib.Path,
    invoke: nemo_gym.verification.verifier_fixture._FixtureInvocation | None = None,
    reseed: nemo_gym.verification.verifier_fixture._FixtureInvocation | None = None
)

Dataclass

cases_path

str | Path

invoke

_FixtureInvocation | None = None

request_model

type[BaseModel]

reseed

_FixtureInvocation | None = None

server_factory

Callable[[], object]

class nemo_gym.verification.verifier_fixture.VerifierFixtureCase()

Bases: BaseModel

expected_error

str | None = None

expected_reward

_FiniteReward | None = None

kind

_CaseKind

model_config

name

str = Field(min_length=1)

request

dict[str, Any]

nemo_gym.verification.verifier_fixture.VerifierFixtureCase.validate_expectation() -> 'VerifierFixtureCase'

class nemo_gym.verification.verifier_fixture.VerifierFixtureError()

Bases: ValueError

A fixture is invalid or does not match verifier behavior.

class nemo_gym.verification.verifier_fixture.VerifierFixtureResult(
    name: str,
    kind: nemo_gym.verification.verifier_fixture._CaseKind,
    observed_rewards: tuple[float, ...] = ()
)

Dataclass

kind

_CaseKind

name

str

observed_rewards

tuple[float, ...] = ()

nemo_gym.verification.verifier_fixture._assert_equal(
    actual: float,
    expected: float,
    message: str
) -> None

nemo_gym.verification.verifier_fixture._observe(
    fixture: nemo_gym.verification.verifier_fixture.VerifierFixture,
    case: nemo_gym.verification.verifier_fixture.VerifierFixtureCase,
    expect_error: bool = False,
    reseed: bool = False
) -> float

async

nemo_gym.verification.verifier_fixture._reward_from(
    result: object,
    case: nemo_gym.verification.verifier_fixture.VerifierFixtureCase
) -> float

nemo_gym.verification.verifier_fixture._write_fixture(
    path: pathlib.Path,
    cases: list[nemo_gym.verification.verifier_fixture.VerifierFixtureCase]
) -> None

nemo_gym.verification.verifier_fixture.exercise_verifier_fixture(
    fixture: nemo_gym.verification.verifier_fixture.VerifierFixture,
    reward_range: collections.abc.Sequence[float],
    higher_is_better: bool = True,
    determinism: str | enum.Enum,
    update_expected: bool = False
) -> tuple[nemo_gym.verification.verifier_fixture.VerifierFixtureResult, ...]

async

Exercise required scorer cases without starting a resources server.

nemo_gym.verification.verifier_fixture.load_verifier_fixture(
    path: str | pathlib.Path
) -> list[nemo_gym.verification.verifier_fixture.VerifierFixtureCase]

Load and validate resources-server-owned JSONL cases.

nemo_gym.verification.verifier_fixture._CaseKind = Literal['full_reward', 'zero_reward', 'malformed', 'determinism']

nemo_gym.verification.verifier_fixture._FiniteReward = Annotated[float, Field(strict=True, allow_inf_nan=False)]

nemo_gym.verification.verifier_fixture._FixtureInvocation = Callable[[object, BaseModel], object | Awaitable[object]]