Index

A

adapter_config (nemo_evaluator.api.api_dataclasses.ApiEndpoint attribute) AdapterConfig (class in nemo_evaluator.adapters.adapter_config) AdapterGlobalContext (class in nemo_evaluator.adapters.types) AdapterRequest (class in nemo_evaluator.adapters.types) AdapterRequestContext (class in nemo_evaluator.adapters.types) AdapterResponse (class in nemo_evaluator.adapters.types) AdapterServer (class in nemo_evaluator.adapters.server) AdapterServerProcess (class in nemo_evaluator.adapters.server)

add() (nemo_evaluator.adapters.caching.diskcaching.Cache method) add_example_files() (in module nemo_evaluator.cli.example) add_reasoning (nemo_evaluator.adapters.interceptors.reasoning_interceptor.ResponseReasoningInterceptor attribute) (nemo_evaluator.adapters.interceptors.reasoning_interceptor.ResponseReasoningInterceptor.Params attribute) aggregate_runtime_metrics() (in module nemo_evaluator.core.resources) api_endpoint (nemo_evaluator.api.api_dataclasses.EvaluationTarget attribute) api_key (nemo_evaluator.api.api_dataclasses.ApiEndpoint attribute) ApiEndpoint (class in nemo_evaluator.api.api_dataclasses) args_to_key() (in module nemo_evaluator.adapters.caching.diskcaching)

B

BaseLoggingParams (class in nemo_evaluator.logging.config)

bind_request_id() (in module nemo_evaluator.logging.context)

C

Cache (class in nemo_evaluator.adapters.caching.diskcaching) cache_dir (nemo_evaluator.adapters.interceptors.caching_interceptor.CachingInterceptor.Params attribute) (nemo_evaluator.adapters.interceptors.reasoning_interceptor.ResponseReasoningInterceptor attribute) (nemo_evaluator.adapters.interceptors.reasoning_interceptor.ResponseReasoningInterceptor.Params attribute) (nemo_evaluator.adapters.interceptors.response_stats_interceptor.ResponseStatsInterceptor.Params attribute) cache_hit (nemo_evaluator.adapters.types.AdapterRequestContext attribute) cache_key (nemo_evaluator.adapters.types.AdapterRequestContext attribute) caching_dir (nemo_evaluator.adapters.adapter_config.AdapterConfig attribute) CachingInterceptor (class in nemo_evaluator.adapters.interceptors.caching_interceptor) CachingInterceptor.Params (class in nemo_evaluator.adapters.interceptors.caching_interceptor) CHAT (nemo_evaluator.api.api_dataclasses.EndpointType attribute) check_adapter_config() (in module nemo_evaluator.core.input) check_endpoint() (in module nemo_evaluator.core.utils) check_health() (in module nemo_evaluator.core.utils) check_required_default_missing() (in module nemo_evaluator.core.input) check_task_invocation() (in module nemo_evaluator.core.input) check_type_compatibility() (in module nemo_evaluator.core.input)

clear() (nemo_evaluator.adapters.caching.diskcaching.Cache method) clear_cache() (nemo_evaluator.adapters.registry.InterceptorRegistry method) close() (nemo_evaluator.adapters.caching.diskcaching.Cache method) collect_finish_reasons (nemo_evaluator.adapters.interceptors.response_stats_interceptor.ResponseStatsInterceptor.Params attribute) collect_token_stats (nemo_evaluator.adapters.interceptors.response_stats_interceptor.ResponseStatsInterceptor.Params attribute) collect_tool_calls (nemo_evaluator.adapters.interceptors.response_stats_interceptor.ResponseStatsInterceptor.Params attribute) command (nemo_evaluator.api.api_dataclasses.Evaluation attribute) COMPLETIONS (nemo_evaluator.api.api_dataclasses.EndpointType attribute) config (nemo_evaluator.adapters.adapter_config.InterceptorConfig attribute) (nemo_evaluator.adapters.adapter_config.PostEvalHookConfig attribute) (nemo_evaluator.api.api_dataclasses.Evaluation attribute) ConfigParams (class in nemo_evaluator.api.api_dataclasses) configure_logging() (in module nemo_evaluator.logging.utils) Constant (class in nemo_evaluator.adapters.caching.diskcaching) count (nemo_evaluator.api.api_dataclasses.ScoreStats attribute) create_tag_index() (nemo_evaluator.adapters.caching.diskcaching.Cache method) cull() (nemo_evaluator.adapters.caching.diskcaching.Cache method) custom_timestamper() (in module nemo_evaluator.logging.utils)

D

DBNAME (in module nemo_evaluator.adapters.caching.diskcaching) decr() (nemo_evaluator.adapters.caching.diskcaching.Cache method) deep_update() (in module nemo_evaluator.core.utils) DEFAULT_ADAPTER_HOST (nemo_evaluator.adapters.server.AdapterServer attribute) DEFAULT_ADAPTER_PORT (nemo_evaluator.adapters.server.AdapterServer attribute) DEFAULT_SETTINGS (in module nemo_evaluator.adapters.caching.diskcaching) description (nemo_evaluator.adapters.registry.InterceptorMetadata attribute) directory (nemo_evaluator.adapters.caching.diskcaching.Cache property)

dirs (nemo_evaluator.adapters.adapter_config.DiscoveryConfig attribute) discover_components() (nemo_evaluator.adapters.registry.InterceptorRegistry method) discovery (nemo_evaluator.adapters.adapter_config.AdapterConfig attribute) DiscoveryConfig (class in nemo_evaluator.adapters.adapter_config) Disk (class in nemo_evaluator.adapters.caching.diskcaching) disk (nemo_evaluator.adapters.caching.diskcaching.Cache property) dotlist_to_dict() (in module nemo_evaluator.core.utils) drop_tag_index() (nemo_evaluator.adapters.caching.diskcaching.Cache method)

E

EMBEDDING (nemo_evaluator.api.api_dataclasses.EndpointType attribute) EmptyDirWarning enable_caching (nemo_evaluator.adapters.interceptors.reasoning_interceptor.ResponseReasoningInterceptor attribute) (nemo_evaluator.adapters.interceptors.reasoning_interceptor.ResponseReasoningInterceptor.Params attribute) enable_reasoning_tracking (nemo_evaluator.adapters.interceptors.reasoning_interceptor.ResponseReasoningInterceptor attribute) (nemo_evaluator.adapters.interceptors.reasoning_interceptor.ResponseReasoningInterceptor.Params attribute) enabled (nemo_evaluator.adapters.adapter_config.InterceptorConfig attribute) (nemo_evaluator.adapters.adapter_config.PostEvalHookConfig attribute) end_reasoning_token (nemo_evaluator.adapters.interceptors.reasoning_interceptor.ResponseReasoningInterceptor attribute) (nemo_evaluator.adapters.interceptors.reasoning_interceptor.ResponseReasoningInterceptor.Params attribute) endpoint_type (nemo_evaluator.adapters.adapter_config.AdapterConfig attribute) EndpointInterceptor (class in nemo_evaluator.adapters.interceptors.endpoint_interceptor) EndpointInterceptor.Params (class in nemo_evaluator.adapters.interceptors.endpoint_interceptor) EndpointType (class in nemo_evaluator.api.api_dataclasses)

ENOVAL (in module nemo_evaluator.adapters.caching.diskcaching) EvalFrameworkBase (class in nemo_evaluator.api.base) EvalFramworkRegistry (class in nemo_evaluator.core.registry) evaluate() (in module nemo_evaluator.core.evaluate) Evaluation (class in nemo_evaluator.api.api_dataclasses) EvaluationConfig (class in nemo_evaluator.api.api_dataclasses) EvaluationResult (class in nemo_evaluator.api.api_dataclasses) EvaluationTarget (class in nemo_evaluator.api.api_dataclasses) evict() (nemo_evaluator.adapters.caching.diskcaching.Cache method) EVICTION_POLICY (in module nemo_evaluator.adapters.caching.diskcaching) exclude_status_codes (nemo_evaluator.adapters.interceptors.raise_client_error_interceptor.RaiseClientErrorInterceptor attribute) (nemo_evaluator.adapters.interceptors.raise_client_error_interceptor.RaiseClientErrorInterceptor.Params attribute) expire() (nemo_evaluator.adapters.caching.diskcaching.Cache method) extra (nemo_evaluator.api.api_dataclasses.ConfigParams attribute)

F

FatalErrorException fetch() (nemo_evaluator.adapters.caching.diskcaching.Disk method) (nemo_evaluator.adapters.caching.diskcaching.JSONDisk method) filename() (nemo_evaluator.adapters.caching.diskcaching.Disk method)

framework_def() (nemo_evaluator.api.base.EvalFrameworkBase static method) framework_name (nemo_evaluator.api.api_dataclasses.Evaluation attribute) from_legacy_config() (nemo_evaluator.adapters.adapter_config.AdapterConfig class method) full_name() (in module nemo_evaluator.adapters.caching.diskcaching)

G

generate_html_report (nemo_evaluator.adapters.adapter_config.AdapterConfig attribute) generate_report() (nemo_evaluator.adapters.server.AdapterServer method) get() (nemo_evaluator.adapters.caching.diskcaching.Cache method) (nemo_evaluator.adapters.caching.diskcaching.Disk method) (nemo_evaluator.adapters.caching.diskcaching.JSONDisk method) get_all_components() (nemo_evaluator.adapters.registry.InterceptorRegistry method) get_args() (in module nemo_evaluator.core.entrypoint) get_available_evaluations() (in module nemo_evaluator.core.input) get_bound_logger() (in module nemo_evaluator.logging.context) get_current_request_id() (in module nemo_evaluator.logging.context) get_entries() (nemo_evaluator.core.registry.EvalFramworkRegistry method) get_evaluation() (in module nemo_evaluator.core.input) get_framework_evaluations() (in module nemo_evaluator.core.input) get_global_registry() (in module nemo_evaluator.core.registry)

get_instance() (nemo_evaluator.adapters.registry.InterceptorRegistry class method) get_interceptor_configs() (nemo_evaluator.adapters.adapter_config.AdapterConfig method) get_interceptors() (nemo_evaluator.adapters.registry.InterceptorRegistry method) get_legacy_defaults() (nemo_evaluator.adapters.adapter_config.AdapterConfig class method) get_logger() (in module nemo_evaluator.logging.utils) get_metadata() (nemo_evaluator.adapters.registry.InterceptorRegistry method) get_post_eval_hook_configs() (nemo_evaluator.adapters.adapter_config.AdapterConfig method) get_post_eval_hooks() (nemo_evaluator.adapters.registry.InterceptorRegistry method) get_token_usage_from_cache() (in module nemo_evaluator.core.resources) get_token_usage_from_cache_db() (in module nemo_evaluator.core.resources) get_validated_config() (nemo_evaluator.adapters.adapter_config.AdapterConfig class method) GroupResult (class in nemo_evaluator.api.api_dataclasses) groups (nemo_evaluator.api.api_dataclasses.EvaluationResult attribute) (nemo_evaluator.api.api_dataclasses.GroupResult attribute)

H

hash() (nemo_evaluator.adapters.caching.diskcaching.Disk method) headers_cache (nemo_evaluator.adapters.interceptors.caching_interceptor.CachingInterceptor attribute)

html_report_size (nemo_evaluator.adapters.adapter_config.AdapterConfig attribute) (nemo_evaluator.adapters.reports.post_eval_report_hook.PostEvalReportHook.Params attribute)

I

include_if_not_finished (nemo_evaluator.adapters.interceptors.reasoning_interceptor.ResponseReasoningInterceptor attribute) (nemo_evaluator.adapters.interceptors.reasoning_interceptor.ResponseReasoningInterceptor.Params attribute) incr() (nemo_evaluator.adapters.caching.diskcaching.Cache method) init_schema (nemo_evaluator.adapters.registry.InterceptorMetadata attribute) intercept_request() (nemo_evaluator.adapters.interceptors.caching_interceptor.CachingInterceptor method) (nemo_evaluator.adapters.interceptors.endpoint_interceptor.EndpointInterceptor method) (nemo_evaluator.adapters.interceptors.logging_interceptor.RequestLoggingInterceptor method) (nemo_evaluator.adapters.interceptors.payload_modifier_interceptor.PayloadParamsModifierInterceptor method) (nemo_evaluator.adapters.interceptors.system_message_interceptor.SystemMessageInterceptor method) (nemo_evaluator.adapters.types.RequestInterceptor method) (nemo_evaluator.adapters.types.RequestToResponseInterceptor method) intercept_response() (nemo_evaluator.adapters.interceptors.caching_interceptor.CachingInterceptor method) (nemo_evaluator.adapters.interceptors.logging_interceptor.ResponseLoggingInterceptor method) (nemo_evaluator.adapters.interceptors.progress_tracking_interceptor.ProgressTrackingInterceptor method) (nemo_evaluator.adapters.interceptors.raise_client_error_interceptor.RaiseClientErrorInterceptor method) (nemo_evaluator.adapters.interceptors.reasoning_interceptor.ResponseReasoningInterceptor method) (nemo_evaluator.adapters.interceptors.response_stats_interceptor.ResponseStatsInterceptor method) (nemo_evaluator.adapters.types.ResponseInterceptor method)

interceptor_class (nemo_evaluator.adapters.registry.InterceptorMetadata attribute) InterceptorConfig (class in nemo_evaluator.adapters.adapter_config) InterceptorMetadata (class in nemo_evaluator.adapters.registry) InterceptorRegistry (class in nemo_evaluator.adapters.registry) interceptors (nemo_evaluator.adapters.adapter_config.AdapterConfig attribute) is_port_open() (in module nemo_evaluator.adapters.server) is_post_eval_hook() (nemo_evaluator.adapters.registry.InterceptorRegistry method) is_request_interceptor() (nemo_evaluator.adapters.registry.InterceptorRegistry method) is_request_to_response_interceptor() (nemo_evaluator.adapters.registry.InterceptorRegistry method) is_response_interceptor() (nemo_evaluator.adapters.registry.InterceptorRegistry method) iterkeys() (nemo_evaluator.adapters.caching.diskcaching.Cache method)

J

JSONDisk (class in nemo_evaluator.adapters.caching.diskcaching)

K

KeyType (in module nemo_evaluator.core.utils)

L

latency_ms (nemo_evaluator.adapters.types.AdapterResponse attribute) LEVEL_MAP (nemo_evaluator.logging.utils.MainConsoleRenderer attribute) limit_samples (nemo_evaluator.api.api_dataclasses.ConfigParams attribute) load_run_config() (in module nemo_evaluator.core.input) log_failed_requests (nemo_evaluator.adapters.adapter_config.AdapterConfig attribute) log_level (nemo_evaluator.logging.config.BaseLoggingParams attribute) log_request_body (nemo_evaluator.adapters.interceptors.logging_interceptor.RequestLoggingInterceptor attribute) (nemo_evaluator.adapters.interceptors.logging_interceptor.RequestLoggingInterceptor.Params attribute) log_request_headers (nemo_evaluator.adapters.interceptors.logging_interceptor.RequestLoggingInterceptor attribute) (nemo_evaluator.adapters.interceptors.logging_interceptor.RequestLoggingInterceptor.Params attribute) log_response_body (nemo_evaluator.adapters.interceptors.logging_interceptor.ResponseLoggingInterceptor attribute) (nemo_evaluator.adapters.interceptors.logging_interceptor.ResponseLoggingInterceptor.Params attribute)

log_response_headers (nemo_evaluator.adapters.interceptors.logging_interceptor.ResponseLoggingInterceptor attribute) (nemo_evaluator.adapters.interceptors.logging_interceptor.ResponseLoggingInterceptor.Params attribute) logger (in module nemo_evaluator.adapters.registry) (in module nemo_evaluator.adapters.server) (in module nemo_evaluator.core.evaluate) (in module nemo_evaluator.core.input) (in module nemo_evaluator.core.utils) (in module nemo_evaluator.logging.utils) logging_aggregated_stats_interval (nemo_evaluator.adapters.interceptors.reasoning_interceptor.ResponseReasoningInterceptor attribute) (nemo_evaluator.adapters.interceptors.reasoning_interceptor.ResponseReasoningInterceptor.Params attribute) (nemo_evaluator.adapters.interceptors.response_stats_interceptor.ResponseStatsInterceptor.Params attribute)

M

main() (in module nemo_evaluator.cli.example) MainConsoleRenderer (class in nemo_evaluator.logging.utils) MAJOR (in module nemo_evaluator.package_info) max (nemo_evaluator.api.api_dataclasses.ScoreStats attribute) max_new_tokens (nemo_evaluator.api.api_dataclasses.ConfigParams attribute) max_requests (nemo_evaluator.adapters.interceptors.logging_interceptor.RequestLoggingInterceptor attribute) (nemo_evaluator.adapters.interceptors.logging_interceptor.RequestLoggingInterceptor.Params attribute) max_responses (nemo_evaluator.adapters.interceptors.logging_interceptor.ResponseLoggingInterceptor attribute) (nemo_evaluator.adapters.interceptors.logging_interceptor.ResponseLoggingInterceptor.Params attribute) max_retries (nemo_evaluator.api.api_dataclasses.ConfigParams attribute) max_saved_requests (nemo_evaluator.adapters.interceptors.caching_interceptor.CachingInterceptor.Params attribute) max_saved_responses (nemo_evaluator.adapters.interceptors.caching_interceptor.CachingInterceptor.Params attribute) mean (nemo_evaluator.api.api_dataclasses.ScoreStats attribute) merge_dicts() (in module nemo_evaluator.core.input) METADATA (in module nemo_evaluator.adapters.caching.diskcaching) MetricResult (class in nemo_evaluator.api.api_dataclasses) metrics (nemo_evaluator.api.api_dataclasses.GroupResult attribute) (nemo_evaluator.api.api_dataclasses.TaskResult attribute) metrics_path (nemo_evaluator.adapters.types.AdapterGlobalContext property) migrate_reasoning_content (nemo_evaluator.adapters.interceptors.reasoning_interceptor.ResponseReasoningInterceptor attribute) (nemo_evaluator.adapters.interceptors.reasoning_interceptor.ResponseReasoningInterceptor.Params attribute) min (nemo_evaluator.api.api_dataclasses.ScoreStats attribute) MINOR (in module nemo_evaluator.package_info) MisconfigurationError MODE_JSON (in module nemo_evaluator.adapters.caching.diskcaching) MODE_NONE (in module nemo_evaluator.adapters.caching.diskcaching) MODE_RAW (in module nemo_evaluator.adapters.caching.diskcaching) model_config (nemo_evaluator.api.api_dataclasses.ApiEndpoint attribute) model_id (nemo_evaluator.api.api_dataclasses.ApiEndpoint attribute) module nemo_evaluator nemo_evaluator.adapters nemo_evaluator.adapters.adapter_config nemo_evaluator.adapters.caching nemo_evaluator.adapters.caching.diskcaching nemo_evaluator.adapters.decorators nemo_evaluator.adapters.interceptors nemo_evaluator.adapters.interceptors.caching_interceptor nemo_evaluator.adapters.interceptors.endpoint_interceptor nemo_evaluator.adapters.interceptors.logging_interceptor nemo_evaluator.adapters.interceptors.payload_modifier_interceptor nemo_evaluator.adapters.interceptors.progress_tracking_interceptor nemo_evaluator.adapters.interceptors.raise_client_error_interceptor nemo_evaluator.adapters.interceptors.reasoning_interceptor nemo_evaluator.adapters.interceptors.response_stats_interceptor nemo_evaluator.adapters.interceptors.system_message_interceptor nemo_evaluator.adapters.registry nemo_evaluator.adapters.reports nemo_evaluator.adapters.reports.post_eval_report_hook nemo_evaluator.adapters.reports.templates nemo_evaluator.adapters.reports.templates.simple_template nemo_evaluator.adapters.server nemo_evaluator.adapters.types nemo_evaluator.api nemo_evaluator.api.api_dataclasses nemo_evaluator.api.base nemo_evaluator.api.run nemo_evaluator.cli nemo_evaluator.cli.example nemo_evaluator.core nemo_evaluator.core.entrypoint nemo_evaluator.core.evaluate nemo_evaluator.core.input nemo_evaluator.core.registry nemo_evaluator.core.resources nemo_evaluator.core.utils nemo_evaluator.logging nemo_evaluator.logging.config nemo_evaluator.logging.context nemo_evaluator.logging.utils nemo_evaluator.package_info

modules (nemo_evaluator.adapters.adapter_config.DiscoveryConfig attribute) monitor_memory_usage() (in module nemo_evaluator.core.resources)

N

name (nemo_evaluator.adapters.adapter_config.InterceptorConfig attribute) (nemo_evaluator.adapters.adapter_config.PostEvalHookConfig attribute) (nemo_evaluator.adapters.registry.InterceptorMetadata attribute) nemo_evaluator module nemo_evaluator.adapters module nemo_evaluator.adapters.adapter_config module nemo_evaluator.adapters.caching module nemo_evaluator.adapters.caching.diskcaching module nemo_evaluator.adapters.decorators module nemo_evaluator.adapters.interceptors module nemo_evaluator.adapters.interceptors.caching_interceptor module nemo_evaluator.adapters.interceptors.endpoint_interceptor module nemo_evaluator.adapters.interceptors.logging_interceptor module nemo_evaluator.adapters.interceptors.payload_modifier_interceptor module nemo_evaluator.adapters.interceptors.progress_tracking_interceptor module nemo_evaluator.adapters.interceptors.raise_client_error_interceptor module nemo_evaluator.adapters.interceptors.reasoning_interceptor module nemo_evaluator.adapters.interceptors.response_stats_interceptor module nemo_evaluator.adapters.interceptors.system_message_interceptor module nemo_evaluator.adapters.registry module nemo_evaluator.adapters.reports module nemo_evaluator.adapters.reports.post_eval_report_hook module nemo_evaluator.adapters.reports.templates module

nemo_evaluator.adapters.reports.templates.simple_template module nemo_evaluator.adapters.server module nemo_evaluator.adapters.types module nemo_evaluator.api module nemo_evaluator.api.api_dataclasses module nemo_evaluator.api.base module nemo_evaluator.api.run module nemo_evaluator.cli module nemo_evaluator.cli.example module nemo_evaluator.core module nemo_evaluator.core.entrypoint module nemo_evaluator.core.evaluate module nemo_evaluator.core.input module nemo_evaluator.core.registry module nemo_evaluator.core.resources module nemo_evaluator.core.utils module nemo_evaluator.logging module nemo_evaluator.logging.config module nemo_evaluator.logging.context module nemo_evaluator.logging.utils module nemo_evaluator.package_info module

O

output_dir (nemo_evaluator.adapters.interceptors.progress_tracking_interceptor.ProgressTrackingInterceptor.Params attribute) (nemo_evaluator.adapters.types.AdapterGlobalContext attribute) (nemo_evaluator.api.api_dataclasses.EvaluationConfig attribute)

P

parallelism (nemo_evaluator.api.api_dataclasses.ConfigParams attribute) params (nemo_evaluator.api.api_dataclasses.EvaluationConfig attribute) params_to_add (nemo_evaluator.adapters.interceptors.payload_modifier_interceptor.PayloadParamsModifierInterceptor.Params attribute) params_to_remove (nemo_evaluator.adapters.interceptors.payload_modifier_interceptor.PayloadParamsModifierInterceptor.Params attribute) params_to_rename (nemo_evaluator.adapters.interceptors.payload_modifier_interceptor.PayloadParamsModifierInterceptor.Params attribute) parse_cli_args() (in module nemo_evaluator.core.input) parse_output() (in module nemo_evaluator.core.evaluate) (nemo_evaluator.api.base.EvalFrameworkBase static method) parse_override_params() (in module nemo_evaluator.core.input) PATCH (in module nemo_evaluator.package_info) PayloadParamsModifierInterceptor (class in nemo_evaluator.adapters.interceptors.payload_modifier_interceptor) PayloadParamsModifierInterceptor.Params (class in nemo_evaluator.adapters.interceptors.payload_modifier_interceptor) pkg_name (nemo_evaluator.api.api_dataclasses.Evaluation attribute) post_eval_hook() (nemo_evaluator.adapters.interceptors.progress_tracking_interceptor.ProgressTrackingInterceptor method) (nemo_evaluator.adapters.interceptors.reasoning_interceptor.ResponseReasoningInterceptor method) (nemo_evaluator.adapters.interceptors.response_stats_interceptor.ResponseStatsInterceptor method) (nemo_evaluator.adapters.reports.post_eval_report_hook.PostEvalReportHook method) (nemo_evaluator.adapters.types.PostEvalHook method)

post_eval_hooks (nemo_evaluator.adapters.adapter_config.AdapterConfig attribute) PostEvalHook (class in nemo_evaluator.adapters.types) PostEvalHookConfig (class in nemo_evaluator.adapters.adapter_config) PostEvalHookConfig.Config (class in nemo_evaluator.adapters.adapter_config) PostEvalReportHook (class in nemo_evaluator.adapters.reports.post_eval_report_hook) PostEvalReportHook.Params (class in nemo_evaluator.adapters.reports.post_eval_report_hook) PostEvalReportHook.Params.Config (class in nemo_evaluator.adapters.reports.post_eval_report_hook) PRE_RELEASE (in module nemo_evaluator.package_info) prepare_output_directory() (in module nemo_evaluator.core.input) progress_filepath (nemo_evaluator.adapters.interceptors.progress_tracking_interceptor.ProgressTrackingInterceptor attribute) progress_tracking_interval (nemo_evaluator.adapters.interceptors.progress_tracking_interceptor.ProgressTrackingInterceptor attribute) (nemo_evaluator.adapters.interceptors.progress_tracking_interceptor.ProgressTrackingInterceptor.Params attribute) progress_tracking_url (nemo_evaluator.adapters.interceptors.progress_tracking_interceptor.ProgressTrackingInterceptor attribute) (nemo_evaluator.adapters.interceptors.progress_tracking_interceptor.ProgressTrackingInterceptor.Params attribute) ProgressTrackingInterceptor (class in nemo_evaluator.adapters.interceptors.progress_tracking_interceptor) ProgressTrackingInterceptor.Params (class in nemo_evaluator.adapters.interceptors.progress_tracking_interceptor) put() (nemo_evaluator.adapters.caching.diskcaching.Disk method) (nemo_evaluator.adapters.caching.diskcaching.JSONDisk method)

R

r (nemo_evaluator.adapters.types.AdapterRequest attribute) (nemo_evaluator.adapters.types.AdapterResponse attribute) RaiseClientErrorInterceptor (class in nemo_evaluator.adapters.interceptors.raise_client_error_interceptor) RaiseClientErrorInterceptor.Params (class in nemo_evaluator.adapters.interceptors.raise_client_error_interceptor) rctx (nemo_evaluator.adapters.types.AdapterRequest attribute) (nemo_evaluator.adapters.types.AdapterResponse attribute) register() (nemo_evaluator.adapters.registry.InterceptorRegistry method) (nemo_evaluator.core.registry.EvalFramworkRegistry method) register_for_adapter() (in module nemo_evaluator.adapters.decorators) register_framework() (in module nemo_evaluator.core.registry) remove() (nemo_evaluator.adapters.caching.diskcaching.Disk method) render_command() (nemo_evaluator.api.api_dataclasses.Evaluation method) report_types (nemo_evaluator.adapters.reports.post_eval_report_hook.PostEvalReportHook.Params attribute) request_context() (in module nemo_evaluator.logging.context) request_id (nemo_evaluator.adapters.types.AdapterRequestContext attribute) request_method (nemo_evaluator.adapters.interceptors.progress_tracking_interceptor.ProgressTrackingInterceptor attribute) (nemo_evaluator.adapters.interceptors.progress_tracking_interceptor.ProgressTrackingInterceptor.Params attribute) request_timeout (nemo_evaluator.api.api_dataclasses.ConfigParams attribute) RequestInterceptor (class in nemo_evaluator.adapters.types) RequestLoggingInterceptor (class in nemo_evaluator.adapters.interceptors.logging_interceptor)

RequestLoggingInterceptor.Params (class in nemo_evaluator.adapters.interceptors.logging_interceptor) requests_cache (nemo_evaluator.adapters.interceptors.caching_interceptor.CachingInterceptor attribute) RequestToResponseInterceptor (class in nemo_evaluator.adapters.types) RESET (nemo_evaluator.logging.utils.MainConsoleRenderer attribute) reset() (nemo_evaluator.adapters.caching.diskcaching.Cache method) (nemo_evaluator.adapters.registry.InterceptorRegistry method) ResponseInterceptor (class in nemo_evaluator.adapters.types) ResponseLoggingInterceptor (class in nemo_evaluator.adapters.interceptors.logging_interceptor) ResponseLoggingInterceptor.Params (class in nemo_evaluator.adapters.interceptors.logging_interceptor) ResponseReasoningInterceptor (class in nemo_evaluator.adapters.interceptors.reasoning_interceptor) ResponseReasoningInterceptor.Params (class in nemo_evaluator.adapters.interceptors.reasoning_interceptor) responses_cache (nemo_evaluator.adapters.interceptors.caching_interceptor.CachingInterceptor attribute) ResponseStatsInterceptor (class in nemo_evaluator.adapters.interceptors.response_stats_interceptor) ResponseStatsInterceptor.Params (class in nemo_evaluator.adapters.interceptors.response_stats_interceptor) reuse_cached_responses (nemo_evaluator.adapters.interceptors.caching_interceptor.CachingInterceptor.Params attribute) run() (in module nemo_evaluator.core.entrypoint) (nemo_evaluator.adapters.server.AdapterServer method) run_command() (in module nemo_evaluator.core.utils) run_eval() (in module nemo_evaluator.core.entrypoint) run_post_eval_hooks() (nemo_evaluator.adapters.server.AdapterServer method)

S

save_individuals (nemo_evaluator.adapters.interceptors.response_stats_interceptor.ResponseStatsInterceptor.Params attribute) save_requests (nemo_evaluator.adapters.interceptors.caching_interceptor.CachingInterceptor.Params attribute) save_responses (nemo_evaluator.adapters.interceptors.caching_interceptor.CachingInterceptor.Params attribute) Score (class in nemo_evaluator.api.api_dataclasses) scores (nemo_evaluator.api.api_dataclasses.MetricResult attribute) ScoreStats (class in nemo_evaluator.api.api_dataclasses) set() (nemo_evaluator.adapters.caching.diskcaching.Cache method) show_available_tasks() (in module nemo_evaluator.core.entrypoint) SIMPLE_TEMPLATE (in module nemo_evaluator.adapters.reports.templates.simple_template) start_reasoning_token (nemo_evaluator.adapters.interceptors.reasoning_interceptor.ResponseReasoningInterceptor attribute) (nemo_evaluator.adapters.interceptors.reasoning_interceptor.ResponseReasoningInterceptor.Params attribute) stats (nemo_evaluator.api.api_dataclasses.Score attribute) stats() (nemo_evaluator.adapters.caching.diskcaching.Cache method) stats_file_saving_interval (nemo_evaluator.adapters.interceptors.reasoning_interceptor.ResponseReasoningInterceptor attribute) (nemo_evaluator.adapters.interceptors.reasoning_interceptor.ResponseReasoningInterceptor.Params attribute) (nemo_evaluator.adapters.interceptors.response_stats_interceptor.ResponseStatsInterceptor.Params attribute) status_code_range_end (nemo_evaluator.adapters.interceptors.raise_client_error_interceptor.RaiseClientErrorInterceptor attribute) (nemo_evaluator.adapters.interceptors.raise_client_error_interceptor.RaiseClientErrorInterceptor.Params attribute) status_code_range_start (nemo_evaluator.adapters.interceptors.raise_client_error_interceptor.RaiseClientErrorInterceptor attribute) (nemo_evaluator.adapters.interceptors.raise_client_error_interceptor.RaiseClientErrorInterceptor.Params attribute)

status_codes (nemo_evaluator.adapters.interceptors.raise_client_error_interceptor.RaiseClientErrorInterceptor attribute) (nemo_evaluator.adapters.interceptors.raise_client_error_interceptor.RaiseClientErrorInterceptor.Params attribute) stddev (nemo_evaluator.api.api_dataclasses.ScoreStats attribute) stderr (nemo_evaluator.api.api_dataclasses.ScoreStats attribute) store() (nemo_evaluator.adapters.caching.diskcaching.Disk method) (nemo_evaluator.adapters.caching.diskcaching.JSONDisk method) stream (nemo_evaluator.api.api_dataclasses.ApiEndpoint attribute) sum (nemo_evaluator.api.api_dataclasses.ScoreStats attribute) sum_squared (nemo_evaluator.api.api_dataclasses.ScoreStats attribute) supported_endpoint_types (nemo_evaluator.api.api_dataclasses.EvaluationConfig attribute) supports_post_eval_hook() (nemo_evaluator.adapters.registry.InterceptorMetadata method) supports_request_interception() (nemo_evaluator.adapters.registry.InterceptorMetadata method) supports_request_to_response_interception() (nemo_evaluator.adapters.registry.InterceptorMetadata method) supports_response_interception() (nemo_evaluator.adapters.registry.InterceptorMetadata method) system_message (nemo_evaluator.adapters.interceptors.system_message_interceptor.SystemMessageInterceptor attribute) (nemo_evaluator.adapters.interceptors.system_message_interceptor.SystemMessageInterceptor.Params attribute) SystemMessageInterceptor (class in nemo_evaluator.adapters.interceptors.system_message_interceptor) SystemMessageInterceptor.Params (class in nemo_evaluator.adapters.interceptors.system_message_interceptor)

T

target (nemo_evaluator.api.api_dataclasses.Evaluation attribute) task (nemo_evaluator.api.api_dataclasses.ConfigParams attribute) TaskResult (class in nemo_evaluator.api.api_dataclasses) tasks (nemo_evaluator.api.api_dataclasses.EvaluationResult attribute) temperature (nemo_evaluator.api.api_dataclasses.ConfigParams attribute) Timeout

timeout (nemo_evaluator.adapters.caching.diskcaching.Cache property) top_p (nemo_evaluator.api.api_dataclasses.ConfigParams attribute) touch() (nemo_evaluator.adapters.caching.diskcaching.Cache method) tracking_requests_stats (nemo_evaluator.adapters.adapter_config.AdapterConfig attribute) transact() (nemo_evaluator.adapters.caching.diskcaching.Cache method) type (nemo_evaluator.api.api_dataclasses.ApiEndpoint attribute) (nemo_evaluator.api.api_dataclasses.EvaluationConfig attribute)

U

UNDEFINED (nemo_evaluator.api.api_dataclasses.EndpointType attribute) UNKNOWN (in module nemo_evaluator.adapters.caching.diskcaching) UnknownFileWarning

url (nemo_evaluator.adapters.types.AdapterGlobalContext attribute) (nemo_evaluator.api.api_dataclasses.ApiEndpoint attribute) use_enum_values (nemo_evaluator.adapters.adapter_config.PostEvalHookConfig.Config attribute) (nemo_evaluator.adapters.reports.post_eval_report_hook.PostEvalReportHook.Params.Config attribute)

V

validate_configuration() (in module nemo_evaluator.core.input) value (nemo_evaluator.api.api_dataclasses.Score attribute) variance (nemo_evaluator.api.api_dataclasses.ScoreStats attribute)

VERSION (in module nemo_evaluator.package_info) VLM (nemo_evaluator.api.api_dataclasses.EndpointType attribute) volume() (nemo_evaluator.adapters.caching.diskcaching.Cache method)

W

wait_for_server() (in module nemo_evaluator.adapters.server)