nemo_gym.metrics_config

View as Markdown

Metric name components and primary-metric filtering.

Module Contents

Classes

NameDescription
PassMajorityStat-
Stat-

Functions

NameDescription
is_primary_metricWhether a metric name is a point estimate suitable for repeat aggregation and comparison.

Data

ACROSS_REPEATS_MARKER

METRIC_EXCLUDED_NAMES

METRIC_EXCLUDED_PREFIXES

METRIC_EXCLUDED_SUFFIXES

PASS_MAJORITY_STAT_SUFFIXES

STAT_SEPARATOR

API

class nemo_gym.metrics_config.PassMajorityStat

Bases: enum.Enum

AVG_SAMPLE_STD_DEV
= 'avg_sample_std_dev'
STD_DEV_ACROSS_RUNS
= 'std_dev_across_runs'
STD_ERR_ACROSS_RUNS
= 'std_err_across_runs'
suffix
=
class nemo_gym.metrics_config.Stat

Bases: enum.Enum

CI_HIGH_95
= 'ci_high_95'
CI_LOW_95
= 'ci_low_95'
HISTOGRAM
= 'histogram'
MAX
= 'max'
MEAN
= 'mean'
MEDIAN
= 'median'
MIN
= 'min'
P25
= 'p25'
P75
= 'p75'
SE
= 'se'
SEM
= 'sem'
STD
= 'std'
across_repeats_prefix
=
prefix
=
nemo_gym.metrics_config.is_primary_metric(
name: object
) -> bool

Whether a metric name is a point estimate suitable for repeat aggregation and comparison.

nemo_gym.metrics_config.ACROSS_REPEATS_MARKER = f'_across_repeats{STAT_SEPARATOR}'
nemo_gym.metrics_config.METRIC_EXCLUDED_NAMES = (TASK_INDEX_KEY_NAME, ROLLOUT_INDEX_KEY_NAME, ATTEMPT_INDEX_KEY_NAME, 'sample_co...
nemo_gym.metrics_config.METRIC_EXCLUDED_PREFIXES = tuple((stat.prefix) for stat in Stat if stat != Stat.MEAN)
nemo_gym.metrics_config.METRIC_EXCLUDED_SUFFIXES = (f'/{Stat.MAX}', f'/{Stat.MIN}', f'/{Stat.MEDIAN}', '/p5', f'/{Stat.P25}', f'/{S...
nemo_gym.metrics_config.PASS_MAJORITY_STAT_SUFFIXES: Tuple[str, ...] = tuple((stat.suffix) for stat in PassMajorityStat)
nemo_gym.metrics_config.STAT_SEPARATOR = '/'