nemo_gym.metrics_config

Metric name components and primary-metric filtering.

Module Contents

Classes

Name	Description
`PassMajorityStat`	-
`Stat`	-

Functions

Name	Description
`is_primary_metric`	Whether a metric name is a point estimate suitable for repeat aggregation and comparison.

Data

ACROSS_REPEATS_MARKER

METRIC_EXCLUDED_NAMES

METRIC_EXCLUDED_PREFIXES

METRIC_EXCLUDED_SUFFIXES

PASS_MAJORITY_STAT_SUFFIXES

STAT_SEPARATOR

API

class nemo_gym.metrics_config.PassMajorityStat

Bases: enum.Enum

AVG_SAMPLE_STD_DEV

= 'avg_sample_std_dev'

STD_DEV_ACROSS_RUNS

= 'std_dev_across_runs'

STD_ERR_ACROSS_RUNS

= 'std_err_across_runs'

suffix

class nemo_gym.metrics_config.Stat

Bases: enum.Enum

CI_HIGH_95

= 'ci_high_95'

CI_LOW_95

= 'ci_low_95'

HISTOGRAM

= 'histogram'

MAX

= 'max'

MEAN

= 'mean'

MEDIAN

= 'median'

MIN

= 'min'

P25

= 'p25'

P75

= 'p75'

= 'se'

SEM

= 'sem'

STD

= 'std'

across_repeats_prefix

prefix

nemo_gym.metrics_config.is_primary_metric(
    name: object
) -> bool

Whether a metric name is a point estimate suitable for repeat aggregation and comparison.

nemo_gym.metrics_config.ACROSS_REPEATS_MARKER = f'_across_repeats{STAT_SEPARATOR}'

nemo_gym.metrics_config.METRIC_EXCLUDED_NAMES = (TASK_INDEX_KEY_NAME, ROLLOUT_INDEX_KEY_NAME, ATTEMPT_INDEX_KEY_NAME, 'sample_co...

nemo_gym.metrics_config.METRIC_EXCLUDED_PREFIXES = tuple((stat.prefix) for stat in Stat if stat != Stat.MEAN)

nemo_gym.metrics_config.METRIC_EXCLUDED_SUFFIXES = (f'/{Stat.MAX}', f'/{Stat.MIN}', f'/{Stat.MEDIAN}', '/p5', f'/{Stat.P25}', f'/{S...

nemo_gym.metrics_config.PASS_MAJORITY_STAT_SUFFIXES: Tuple[str, ...] = tuple((stat.suffix) for stat in PassMajorityStat)

nemo_gym.metrics_config.STAT_SEPARATOR = '/'