> For clean Markdown of any page, append .md to the page URL. > For a complete documentation index, see https://docs.nvidia.com/nemo-platform/llms.txt. > For AI client integration (Claude Code, Cursor, etc.), connect to the MCP server at https://docs.nvidia.com/nemo-platform/_mcp/server. # NeMo Platform configuration reference This document describes the structure and defaults for the global config file for the NeMo Platform. All sections are shown in YAML format with inline comments for description, default, and possible values. ## Configuration sections ### `platform` Platform-wide configuration settings. ```yaml wordWrap platform: # Runtime used by the platform. Used to auto-detect default backends to use. | default: 'docker' | values: 'kubernetes' | 'docker' | 'none' runtime: docker # Base URL for the NeMo Platform api. Used as the default URL for all services. | default: 'http://localhost:8080' base_url: http://localhost:8080 # Optional loopback address override for job containers to reach platform services. If not specified, automatically determined based on platform: macOS uses 'host.docker.internal', Docker containers use container hostname, Linux host network uses no override. Can be set via config file or NMP_LOOPBACK_ADDRESS env var. loopback_address: # Global image pull secrets for the platform image_pull_secrets: [] # Docker registry for NeMo Platform images (e.g., 'nvcr.io/nvidia/nemo-platform'). | default: 'my-registry' image_registry: my-registry # Default tag for NeMo Platform images. | default: 'local' image_tag: local # Shared Docker configuration for services using Docker backends. docker: # GPU device IDs to reserve for the Docker GPU pool. Use 'all' to auto-detect and use all available GPUs, 'none' or empty string to disable GPU support, or a comma-separated list of device IDs (e.g., '0,1,2,3'). | default: 'all' reserved_gpu_device_ids: all # Name of the secret containing the default NGC API key. Defaults to 'system/ngc-api-key'. | default: 'system/ngc-api-key' ngc_api_key_secret: system/ngc-api-key # Environment variable name to source the NGC API key from. | default: 'NGC_API_KEY' ngc_api_key_env_var: NGC_API_KEY # When true, GET / returns a 301 redirect to /studio. When false, GET / returns 404. This is used to redirect the root URL to the Studio UI. | default: True redirect_root_to_studio: true ``` ### `service` Common configuration shared by all services. ```yaml wordWrap service: # Format for logs generated by the service | default: 'plain' | values: 'json' | 'plain' log_format: plain # Logging level for the NeMo Platform. | default: 'INFO' | values: 'DEBUG' | 'INFO' | 'WARN' | 'ERROR' log_level: INFO # Scheme for the NeMo Platform service. | default: 'http' scheme: http # Host for the NeMo Platform service. | default: '127.0.0.1' host: 127.0.0.1 # Port for the NeMo Platform service. | default: 8080 port: 8080 ``` ### `auth` Configuration for the Auth Service. ```yaml wordWrap auth: # Master switch for authorization. If False, all requests are allowed. | default: False enabled: false # Base URL for the Policy Decision Point (auth service or external OPA). | default: 'http://localhost:8080' policy_decision_point_base_url: http://localhost:8080 # Policy Decision Point provider: 'embedded' for auth service's built-in WASM engine, 'opa' for external OPA sidecar. | default: 'embedded' | values: 'embedded' | 'opa' policy_decision_point_provider: embedded # HTTP timeout in seconds for outbound Policy Decision Point (PDP) requests | default: 5 policy_decision_point_request_timeout_seconds: 5 # Default polling interval (seconds) used by AuthClient.wait_role and wait_permissions. Lower values reduce role-propagation wait time at the cost of more PDP requests; tests typically override this to a small value. | default: 1.0 propagation_poll_interval_seconds: 1.0 # Allow unsigned JWTs (`alg=none`) for local development/testing. Disabled by default and should not be enabled in production. | default: False allow_unsigned_jwt: false # When auth is enabled with the embedded PDP and policy.wasm is missing, build it automatically from a local NeMo Platform source checkout. Packaged deployments should include policy.wasm at build time and can disable this for fail-fast startup. | default: True embedded_pdp_auto_build_wasm: true # OIDC configuration for native token validation. oidc: # Enable native OIDC token validation. | default: False enabled: false # OIDC issuer URL (e.g., https://sso.nvidia.com). Used for token validation and .well-known discovery. | default: '' issuer: '' # Additional valid issuers for token validation. Useful for Azure AD where access tokens use v1.0 issuer format (https://sts.windows.net/{tenant}/) while endpoints use v2.0. additional_issuers: [] # OAuth client ID for this NeMo Platform deployment. Used for device flow and token audience validation. | default: '' client_id: '' # Override authorization endpoint (defaults to discovery). authorization_endpoint: # Override token endpoint (defaults to discovery). token_endpoint: # Override device authorization endpoint (defaults to discovery). device_authorization_endpoint: # Override JWKS URI for token validation (defaults to discovery). jwks_uri: # Expected token audience. When set, tokens must include this value in their 'aud' claim. When not set, audience validation is skipped entirely. audience: # JWT claim containing the principal email (maps to X-NMP-Principal-Email). Set explicitly for your IdP. | default: 'email' email_claim: email # JWT claim containing user groups. Supports 'groups' (standard) and 'cognito:groups' (AWS Cognito). | default: 'groups' groups_claim: groups # JWT claim to use as principal ID (maps to X-NMP-Principal-Id). Set explicitly for your IdP. | default: 'sub' subject_claim: sub # Space-separated OAuth scopes to request during authentication. For Azure AD with custom API, use: 'api://{app-id}/.default openid profile email' | default: 'openid profile email offline_access' default_scopes: openid profile email offline_access # Enable SDK workload identity token exchange using NMP_WORKLOAD_IDENTITY_TOKEN_FILE subject tokens. | default: False workload_token_exchange_enabled: false # OAuth client ID to use for workload identity token exchange. Defaults to client_id when unset. workload_client_id: # OAuth token endpoint to use for workload identity token exchange. Defaults to token_endpoint. workload_token_endpoint: # RFC 8693 audience requested for workload identity token exchange. workload_audience: # Space-separated OAuth scopes requested for workload identity token exchange. workload_scope: # Issuer to stamp on workload identity access tokens minted by the NeMo auth service. Defaults to the platform auth endpoint origin serving the token exchange request. workload_token_issuer: # Lifetime in seconds for workload identity access tokens minted by the NeMo auth service. | default: 300 workload_token_ttl_seconds: 300 # JWT key id advertised by the NeMo auth service workload identity JWKS endpoint. | default: 'nemo-workload-exchange' workload_token_key_id: nemo-workload-exchange # Path to a PEM-encoded RSA private key used by the NeMo auth service to sign workload identity access tokens. Intended for mounted shared secrets. workload_token_private_key_file: # Additional RFC 8693 audience values accepted by the NeMo auth service workload token exchange endpoint. The configured workload_audience is always accepted. workload_allowed_audiences: [] # JWKS URI used by the NeMo auth service to validate JWT subject tokens for workload token exchange. Leave unset when only Kubernetes TokenReview subject validation is enabled. workload_subject_jwks_uri: # Allowed JWT subject token issuers for workload token exchange. Required when workload_subject_jwks_uri is set. workload_subject_issuers: [] # TTL in seconds for caching workload subject JWKS responses. Set to 0 to disable caching. | default: 3600 workload_subject_jwks_cache_ttl_seconds: 3600 # Allow the NeMo auth service workload token exchange endpoint to validate Kubernetes projected service account subject tokens using the TokenReview API. | default: False workload_kubernetes_token_review_enabled: false # Prefix to strip from token scopes before authorization. For example, if IdP returns 'api://my-app/models:read', set prefix to 'api://my-app/' to normalize to 'models:read'. If not set, scopes are used as-is. scope_prefix: # TTL in seconds for caching IdP discovery document responses. Used by the discovery endpoint to avoid per-request IdP calls. Set to 0 to disable caching. | default: 300 discovery_cache_ttl: 300 # Port to run the service on | default: 8000 port: 8000 # Refresh interval for policy data in seconds | default: 30 policy_data_refresh_interval: 30.0 # Seconds to cache the OPA bundle | default: 5 bundle_cache_seconds: 5.0 # Bootstrap admin email for platform setup admin_email: # Name of the default workspace where all authenticated users get Editor role | default: 'default' default_workspace: default # CPU budget for embedded PDP policy evaluation, in millions of WASM fuel units. Default of 200 provides headroom for full plugin-merged authorization data. | default: 200 embedded_pdp_cpu_limit: 200 # Maximum linear memory (MB) the embedded PDP WASM runtime can consume. | default: 32 embedded_pdp_memory_limit_mb: 32 # Fail-mode for a plugin that contributes invalid HTTP authz. | default: 'hard_fail' | values: 'deny_route' | 'quarantine' | 'hard_fail' on_invalid_plugin: hard_fail ``` ### `entities` Configuration for the Entities Service. ```yaml wordWrap entities: # Run Alembic migrations (upgrade head) at startup. | default: True run_migrations: true database_config: # Full database URL (overrides other settings) url: # Database dialect - either sqlite or postgresql | default: 'postgresql' dialect: postgresql # Database hostname | default: '' host: '' # Database path | default: '' path: '' # Database name | default: '' name: '' # Optional database port port: # Optional database username user: # Optional database password password: # Maximum number of connections in the database connection pool | default: 10 connections_limit: 10 # Connection timeout in seconds. For PostgreSQL (asyncpg) and SQLite, how long to wait when connecting or acquiring a lock. | default: 30 connect_timeout_seconds: 30 # Enable SQLAlchemy echo for the database connection | default: False echo: false # Interval in seconds to run the workspace cleanup routine. | default: 10 workspace_cleanup_interval: 10 # Interval in seconds for the background database health ping (used for readiness). | default: 10.0 db_health_check_interval_seconds: 10.0 # Enable in-process caching of per-principal role-binding list queries. Set via NMP_ENTITIES_PRINCIPAL_BINDINGS_CACHE_ENABLED. | default: True principal_bindings_cache_enabled: true # TTL in seconds for principal role-binding cache entries (NMP_ENTITIES_PRINCIPAL_BINDINGS_CACHE_TTL_SEC). | default: 60.0 principal_bindings_cache_ttl_sec: 60.0 # Max distinct principal keys in the role-binding cache before LRU eviction (NMP_ENTITIES_PRINCIPAL_BINDINGS_CACHE_MAX_SIZE). | default: 256 principal_bindings_cache_max_size: 256 ``` ### `files` Configuration for the Files Service. ```yaml wordWrap files: default_storage_config: # Chunk size in bytes for reading/streaming files. Larger chunks reduce async overhead but increase memory per concurrent download. Default: 1MB. | default: 1048576 read_chunk_size: 1048576 # default: 'local' | values: type: local path: /data/files_storage # How many bytes to buffer before flushing to disk | default: 16777216 write_buffer_size: 16777216 # Comma-separated list of external hosts the Files service is allowed to access. | default: 'https://api.ngc.nvidia.com,https://huggingface.co' allowed_external_hosts: https://api.ngc.nvidia.com,https://huggingface.co # Allow users to explicitly create filesets with local storage config. Security-sensitive: enable only in trusted deployments. | default: False allow_user_local_storage: false # TTL for file locks in seconds (default 5 minutes) | default: 300 file_lock_ttl_seconds: 300 # Maximum concurrent downloads during cache warming | default: 3 cache_warming_max_concurrent: 3 # Maximum Hugging Face request attempts for transient failures. | default: 4 hf_retry_attempts: 4 # Initial Hugging Face retry delay in seconds before exponential backoff. | default: 0.5 hf_retry_initial_delay_seconds: 0.5 # Maximum Hugging Face retry delay in seconds. | default: 5.0 hf_retry_max_delay_seconds: 5.0 ``` ### `inference_gateway` Configuration for the Inference Gateway Service. ```yaml wordWrap inference_gateway: # How frequently (in seconds) to refresh the internal model cache from the Models service. If set to 0, disable automatic refreshing. | default: 3 refresh_model_cache_interval_sec: 3 # Time-to-live (in seconds) for cached secrets before they expire and need refreshing. If set to 0, we will refresh this each time the model providers are refreshed. | default: 0 secrets_ttl_sec: 0 ``` ### `jobs` Configuration for the Jobs Service. ```yaml wordWrap jobs: # List of executor profiles for the Jobs service executors: [] # Default executor profile configurations executor_defaults: # Default Docker execution profile configuration docker: # default: 1800 ttl_seconds_before_active: 1800 # default: 86400 ttl_seconds_active: 86400 # default: 3600 ttl_seconds_after_finished: 3600 # default: True cleanup_completed_jobs_immediately: true # Path to the jobs launcher tool | default: '/tools/jobs-launcher' launcher_tool_path: /tools/jobs-launcher # Default container image for job task pods. Used when a job step omits container.image. When unset, falls back to the platform CPU tasks image (platform.image_registry/nmp-cpu-tasks:platform.image_tag). default_task_image: # Optional env vars applied to all jobs (e.g. HOME=/tmp). Keys must not conflict with platform-reserved names. Job steps may override these variables. env: {} # Docker storage configuration storage: # Name of the Docker volume for persistent storage | default: 'nemo-jobs-storage' volume_name: nemo-jobs-storage # Docker image used to set permissions on the volume | default: 'busybox' volume_permissions_image: busybox # List of additional Docker volume mounts for the job additional_volume_mounts: [] # Docker networking configuration networking: # Docker network for the job container | default: 'host' job_container_network: host # Docker workload identity subject-token issuer configuration. workload_identity: # Enable Docker workload identity token-file injection. Defaults to auth.oidc.workload_token_exchange_enabled. enabled: # OAuth token endpoint used by the Docker demo issuer. Defaults to auth.oidc.token_endpoint. token_endpoint: # OAuth client ID used by the Docker demo issuer. Defaults to auth.oidc.workload_client_id or auth.oidc.client_id. client_id: # OAuth client secret for the Docker demo issuer. client_secret: # Username for the Docker demo issuer password grant. username: # Controller environment variable that contains the Docker demo issuer password grant shared secret. | default: 'AUTHENTIK_WORKLOAD_IDENTITY_PASSWORD' password_env_var: AUTHENTIK_WORKLOAD_IDENTITY_PASSWORD # OAuth scope for the Docker demo issuer. scope: # Fallback subject-token lifetime when the Docker demo issuer response omits expires_in. subject_token_ttl_seconds: 600 # Seconds before subject-token expiry when the Docker refresher issues a replacement token. | default: 60 refresh_margin_seconds: 60 # Default Kubernetes execution profile configuration kubernetes_job: # default: 1800 ttl_seconds_before_active: 1800 # default: 86400 ttl_seconds_active: 86400 # default: 3600 ttl_seconds_after_finished: 3600 # default: True cleanup_completed_jobs_immediately: true # Path to the jobs launcher tool | default: '/tools/jobs-launcher' launcher_tool_path: /tools/jobs-launcher # Default container image for job task pods. Used when a job step omits container.image. When unset, falls back to the platform CPU tasks image (platform.image_registry/nmp-cpu-tasks:platform.image_tag). default_task_image: # Optional env vars applied to all jobs (e.g. HOME=/tmp). Keys must not conflict with platform-reserved names. Job steps may override these variables. env: {} # Kubernetes namespace to submit the job to. If not set, it will be determined from the environment. namespace: # Kubernetes service account name for job pods. Uses the Kubernetes default service account when set to 'default'. | default: 'default' service_account_name: default # Tolerations for the Kubernetes job pods. tolerations: [] # Node selector for the Kubernetes job pods. node_selector: {} # Affinity for the Kubernetes job pods. affinity: {} # Resource requests and limits for the Kubernetes job pods. resources: # Minimum resources requested for the container. requests: # CPU specification (e.g., '250m', '1', '2.5'). cpu: # Memory specification (e.g., '128Mi', '1Gi', '512M'). memory: # Maximum resources the container can use. limits: # CPU specification (e.g., '250m', '1', '2.5'). cpu: # Memory specification (e.g., '128Mi', '1Gi', '512M'). memory: # Number of nodes to use. | default: 1 num_nodes: 1 # Step requesting number of GPUs. num_gpus: # Shared memory (/dev/shm) size as a Kubernetes quantity (e.g. '1Gi', '4Gi'). Used for GPU and distributed-GPU job executors. When unset, defaults to 1Gi per allocated GPU. shm_size: # Pod security context for the Kubernetes job pods. pod_security_context: {} # Image pull secrets for the Kubernetes job pods. image_pull_secrets: [] # Metadata to add to each job object in the Kubernetes job. job_metadata: labels: {} annotations: {} # Metadata to add to each pod in the Kubernetes job. pod_metadata: labels: {} annotations: {} # Storage configuration for the Kubernetes job pods. storage: # Persistent Volume Claim Name to use for job storage. | default: '' pvc_name: '' # Image used to set volume permissions | default: 'busybox' volume_permissions_image: busybox # Additional volumes to mount additional_volumes: [] # Additional volume mounts additional_volume_mounts: [] # Number of GPUs to request for the job | default: 1 num_gpus: 1 # The scheduler name to use for the pod spec. When non-empty, this value is applied to the pod's schedulerName field, enabling custom schedulers such as KAI Scheduler. Empty string omits schedulerName so the cluster default scheduler is used. | default: '' scheduler_name: '' # Container image that contains the jobs-launcher binary. | default: 'nvcr.io/nvidia/nemo-microservices/jobs-launcher:latest' launcher_image: nvcr.io/nvidia/nemo-microservices/jobs-launcher:latest # Requested expirationSeconds for the projected service account token used as the workload identity subject token. | default: 600 workload_identity_token_expiration_seconds: 600 # Audience for the projected service account token. Defaults to auth.oidc.workload_client_id, auth.oidc.client_id, then 'nemo-platform'. workload_identity_token_audience: # Default Volcano execution profile configuration volcano_job: # default: 1800 ttl_seconds_before_active: 1800 # default: 86400 ttl_seconds_active: 86400 # default: 3600 ttl_seconds_after_finished: 3600 # default: True cleanup_completed_jobs_immediately: true # Path to the jobs launcher tool | default: '/tools/jobs-launcher' launcher_tool_path: /tools/jobs-launcher # Default container image for job task pods. Used when a job step omits container.image. When unset, falls back to the platform CPU tasks image (platform.image_registry/nmp-cpu-tasks:platform.image_tag). default_task_image: # Optional env vars applied to all jobs (e.g. HOME=/tmp). Keys must not conflict with platform-reserved names. Job steps may override these variables. env: {} # Kubernetes namespace to submit the job to. If not set, it will be determined from the environment. namespace: # Kubernetes service account name for job pods. Uses the Kubernetes default service account when set to 'default'. | default: 'default' service_account_name: default # Tolerations for the Kubernetes job pods. tolerations: [] # Node selector for the Kubernetes job pods. node_selector: {} # Affinity for the Kubernetes job pods. affinity: {} # Resource requests and limits for the Kubernetes job pods. resources: # Minimum resources requested for the container. requests: # CPU specification (e.g., '250m', '1', '2.5'). cpu: # Memory specification (e.g., '128Mi', '1Gi', '512M'). memory: # Maximum resources the container can use. limits: # CPU specification (e.g., '250m', '1', '2.5'). cpu: # Memory specification (e.g., '128Mi', '1Gi', '512M'). memory: # Number of nodes to use. | default: 1 num_nodes: 1 # Step requesting number of GPUs. num_gpus: # Shared memory (/dev/shm) size as a Kubernetes quantity (e.g. '1Gi', '4Gi'). Used for GPU and distributed-GPU job executors. When unset, defaults to 1Gi per allocated GPU. shm_size: # Pod security context for the Kubernetes job pods. pod_security_context: {} # Image pull secrets for the Kubernetes job pods. image_pull_secrets: [] # Metadata to add to each job object in the Kubernetes job. job_metadata: labels: {} annotations: {} # Metadata to add to each pod in the Kubernetes job. pod_metadata: labels: {} annotations: {} # Storage configuration for the Kubernetes job pods. storage: # Persistent Volume Claim Name to use for job storage. | default: '' pvc_name: '' # Image used to set volume permissions | default: 'busybox' volume_permissions_image: busybox # Additional volumes to mount additional_volumes: [] # Additional volume mounts additional_volume_mounts: [] # Number of GPUs to request for the job | default: 1 num_gpus: 1 # The scheduler name to use for the Volcano job. | default: 'volcano' scheduler_name: volcano # Container image that contains the jobs-launcher binary. | default: 'nvcr.io/nvidia/nemo-microservices/jobs-launcher:latest' launcher_image: nvcr.io/nvidia/nemo-microservices/jobs-launcher:latest # Requested expirationSeconds for the projected service account token used as the workload identity subject token. | default: 600 workload_identity_token_expiration_seconds: 600 # Audience for the projected service account token. Defaults to auth.oidc.workload_client_id, auth.oidc.client_id, then 'nemo-platform'. workload_identity_token_audience: # The Volcano queue to submit the job to. | default: 'default' queue: default # maxRetry indicates the maximum number of retries allowed by the job | default: 0 max_retry: 0 # plugins indicates the plugins used by Volcano when the job is scheduled. We always add the pytorch plugin if more than one node. plugins: {} # Enable multi-node networking injection. Sets annotations to trigger Kyverno policy mutations. | default: True enable_multi_node_networking: true # Default subprocess execution profile configuration subprocess: # default: 1800 ttl_seconds_before_active: 1800 # default: 86400 ttl_seconds_active: 86400 # default: 3600 ttl_seconds_after_finished: 3600 # Keep subprocess working directories by default so runs remain inspectable. | default: False cleanup_completed_jobs_immediately: false # Path to the jobs launcher tool | default: '/tools/jobs-launcher' launcher_tool_path: /tools/jobs-launcher # Default container image for job task pods. Used when a job step omits container.image. When unset, falls back to the platform CPU tasks image (platform.image_registry/nmp-cpu-tasks:platform.image_tag). default_task_image: # Optional env vars applied to all jobs (e.g. HOME=/tmp). Keys must not conflict with platform-reserved names. Job steps may override these variables. env: {} # Root directory for subprocess job state, config, storage, and logs. | default: '/tmp/nmp-subprocess-jobs' working_directory: /tmp/nmp-subprocess-jobs # How long to wait after SIGTERM before force killing the process group. | default: 10 graceful_shutdown_timeout_seconds: 10 # Interval in seconds for the job reconciler to run | default: 2 reconcile_interval_seconds: 2 # Interval in seconds for the job scheduler to run | default: 5 schedule_interval_seconds: 5 # Register the subprocess/default execution profile. When unset, defaults to true for docker/none runtimes and false for kubernetes. enable_subprocess_executor: # Include raw job log lines in controller diagnostics snapshots. Disabled by default because job logs may contain secrets or PII. Enable only for local debugging or test environments. | default: False include_job_logs_in_diagnostics: false ``` ### `models` Consolidated configuration for the Models service. ```yaml wordWrap models: # Image used to pull model weights. Its entrypoint is overridden to the Hugging Face CLI ('hf download ...'), so any image with the 'hf' CLI on PATH works. Defaults to the platform's nmp-api image (registry/tag from platform config); override to use a different puller image. huggingface_model_puller: my-registry/nmp-api:local # Controller service configuration controller: # Interval in seconds for the Models Controller to run its control loop | default: 5 interval_seconds: 5 # Dict of custom backend configurations for the Models Controller backends: deployments_plugin: default_executor: docker_executor: k8s_executor: # Maximum seconds a deployment may stay PENDING before ERROR. | default: 7200 pending_timeout_seconds: 7200 # default: 5 max_restart_count: 5 default_storage_class: # default: '200Gi' default_pvc_size: 200Gi # default: 'nvcr.io/nim/meta/llama-3.1-8b-instruct' default_nimservice_image: nvcr.io/nim/meta/llama-3.1-8b-instruct # default: '1.8.5' default_nimservice_image_tag: 1.8.5 # Default vLLM image repository. Fully qualified so it resolves on runtimes that block docker.io short names. | default: 'docker.io/vllm/vllm-openai' default_vllm_image: docker.io/vllm/vllm-openai # default: 'v0.22.1' default_vllm_image_tag: v0.22.1 # default: 1000 default_user_id: 1000 # default: 2000 default_group_id: 2000 # default: 2000 default_vllm_user_id: 2000 # default: 0 default_vllm_group_id: 0 # default: 30 peft_refresh_interval: 30 # default: 'nmp-api' lora_sidecar_image_name: nmp-api lora_sidecar_command: - nemo - services - run - --sidecars - adapters lora_sidecar_args: [] # BusyBox image repository for LoRA cache init containers. Fully qualified so it resolves on runtimes that block docker.io short names. | default: 'docker.io/library/busybox' busybox_image: docker.io/library/busybox # BusyBox image tag for LoRA cache init containers. | default: 'latest' busybox_image_tag: latest # Maximum seconds a deployment may stay DELETING before ERROR. 0 disables timeout escalation. | default: 60 deleting_timeout_seconds: 60 # default: False enabled: false # Time-to-live in seconds for DELETED deployments before they are permanently removed from the database | default: 30 model_deployment_garbage_collection_ttl_seconds: 30 # Time-to-live in seconds for ERROR deployments before backend resources are garbage collected | default: 10800 error_deployment_ttl_seconds: 10800 # Maximum number of attempts to recover a deployment when backend resources are lost | default: 5 drift_recovery_max_attempts: 5 # Base delay in seconds between drift recovery attempts (used for exponential backoff) | default: 30 drift_recovery_base_delay_seconds: 30 # Maximum delay in seconds between drift recovery attempts (caps exponential backoff) | default: 300 drift_recovery_max_delay_seconds: 300 # Per-request timeout in seconds for GET /v1/models provider autodiscovery via the inference gateway. External providers (e.g. NVIDIA Build) can return large model lists and need longer than the SDK default. | default: 180 provider_discovery_timeout_seconds: 180 # SDK retry count for provider autodiscovery requests. The controller loop already retries on each cycle; disabling SDK retries avoids multi-minute retry storms on slow upstreams. | default: 0 provider_discovery_max_retries: 0 # Parallelism estimation heuristics (model size thresholds, TP/PP/DP/CP/EP costs, balance bonuses) parallelism: # Standard node configuration (e.g., DGX H100) | default: 8 gpus_per_node_default: 8 # Standard GPU memory in GB (e.g., H100 80GB) | default: 80 gpu_memory_gb_default: 80 model_size_thresholds: # >300B: Very large models | default: 300.0 very_large: 300.0 # 100-300B: Large models | default: 100.0 large: 100.0 # 50-100B: Medium models | default: 50.0 medium: 50.0 # <70B: Small models for TP cost | default: 70.0 small_tp: 70.0 # <40B: Small MoE models | default: 40.0 small_moe: 40.0 memory: # Start penalizing above 60% memory usage | default: 0.6 pressure_threshold: 0.6 # Moderate memory pressure | default: 0.5 pressure_moderate: 0.5 # Low memory pressure | default: 0.45 pressure_low: 0.45 # Base penalty for exceeding threshold | default: 1000000000.0 base_penalty: 1000000000.0 # Divisor for quadratic penalty: (excess / divisor) ** 2 | default: 0.1 scale_divisor: 0.1 # Maximum discount factor (70% off) | default: 0.7 pp_discount_max: 0.7 # Scale factor for discount calculation | default: 2.0 pp_discount_scale: 2.0 tensor_parallelism: # For models > 300B | default: 50.0 base_cost_very_large_model: 50.0 # For models <= 300B | default: 100.0 base_cost_standard_model: 100.0 # TP=8 is standard for 340B+ | default: 8 excessive_very_large: 8 # TP=4 is standard for 70B and below | default: 4 excessive_standard: 4 # Penalty for excessive TP on large models (>70B) | default: 100000000.0 penalty_large_model: 100000000.0 # Penalty for excessive TP on small models (<70B) | default: 300000000.0 penalty_small_model: 300000000.0 data_parallelism: # total_parallelism >= 64 | default: -10000.0 bonus_minimal: -10000.0 # total_parallelism >= 32 | default: -50000.0 bonus_small: -50000.0 # total_parallelism <= 2 | default: -50000000.0 bonus_very_strong: -50000000.0 # total_parallelism <= 4 | default: -30000000.0 bonus_strong: -30000000.0 # total_parallelism <= 8 | default: -15000000.0 bonus_moderate: -15000000.0 # total_parallelism > 8 | default: -20000000.0 bonus_medium: -20000000.0 # Smaller MoE models | default: -1000000.0 bonus_small_moe: -1000000.0 # Total parallelism threshold | default: 64 total_parallelism_very_high: 64 # Total parallelism threshold | default: 32 total_parallelism_high: 32 # Total parallelism threshold | default: 2 total_parallelism_very_low: 2 # Total parallelism threshold | default: 4 total_parallelism_low: 4 # Total parallelism threshold | default: 8 total_parallelism_medium: 8 # Double DP bonus for pure DP+CP configurations | default: 2.0 cp_bonus_multiplier: 2.0 pipeline_parallelism: # MoE models (lower cost due to higher compute per stage) | default: 500000.0 cost_moe: 500000.0 # param_count_b > 300 | default: 5000000.0 cost_very_large_model: 5000000.0 # param_count_b > 100 | default: 10000000.0 cost_large_model: 10000000.0 # param_count_b > 50 | default: 30000000.0 cost_medium_model: 30000000.0 # param_count_b <= 50 and tp > 1 | default: 100000000.0 cost_small_with_tp: 100000000.0 # param_count_b <= 50 and tp == 1 | default: 300000000.0 cost_small_without_tp: 300000000.0 context_parallelism: # Strong bonus for optimal CP | default: -300000000.0 bonus_optimal: -300000000.0 # Bonus for CP=2 in medium sequences | default: -200000000.0 bonus_good: -200000000.0 # Penalty: not using enough CP or using CP when not needed | default: 200000000.0 penalty_suboptimal: 200000000.0 # Penalty: too much CP | default: 100000000.0 penalty_too_much: 100000000.0 # Penalty: should use CP but using CP=1 | default: 300000000.0 penalty_should_use: 300000000.0 # Very long sequences | default: 1.0 seq_to_param_ratio_high: 1.0 # Medium sequences | default: 0.3 seq_to_param_ratio_medium: 0.3 # Maximum CP value to consider | default: 8 max_value: 8 # CP=2 is optimal for medium sequences | default: 2 optimal_value: 2 # Multiplier for parameter memory calculation | default: 6.0 param_memory_multiplier: 6.0 # Multiplier for layers in parameter calculation | default: 12.0 param_layers_multiplier: 12.0 # Multiplier for sequence memory calculation | default: 38.0 seq_memory_multiplier: 38.0 # Enable CP for sequences >= 8K | default: 8192 seq_threshold_enable: 8192 # Enable CP=4 for sequences >= 16K | default: 16384 seq_threshold_cp4: 16384 # Enable CP=8 for sequences >= 32K | default: 32768 seq_threshold_cp8: 32768 # Enable CP=16 for sequences >= 128K | default: 131072 seq_threshold_cp16: 131072 expert_parallelism: # Huge penalty for EP > 1 on non-MoE models | default: 1000000000.0 penalty_non_moe: 1000000000.0 # Perfect: 1 routed expert per GPU | default: -500000000.0 bonus_perfect: -500000000.0 # Very efficient: <= 8 experts per GPU | default: -400000000.0 bonus_very_efficient: -400000000.0 # Good: <= 32 experts per GPU | default: -300000000.0 bonus_good: -300000000.0 # Acceptable: <= 64 experts per GPU | default: -200000000.0 bonus_acceptable: -200000000.0 # High expert count per GPU (>64) | default: -100000000.0 bonus_high_count: -100000000.0 # Huge penalty: no EP on MoE | default: 800000000.0 penalty_no_sharding: 800000000.0 # Penalty for non-divisor EP | default: 300000000.0 penalty_non_divisor: 300000000.0 # Experts per GPU threshold | default: 8 experts_per_gpu_very_efficient: 8 # Experts per GPU threshold | default: 32 experts_per_gpu_good: 32 # Experts per GPU threshold | default: 64 experts_per_gpu_acceptable: 64 balance: # TP == PP (perfect balance) | default: 1.0 ratio_perfect: 1.0 # max(TP, PP) / min(TP, PP) <= 2.0 | default: 2.0 ratio_good: 2.0 # Perfect balance for very large dense models (>300B) | default: -500000000.0 bonus_perfect_very_large: -500000000.0 # Good balance for very large dense models | default: -300000000.0 bonus_good_very_large: -300000000.0 # Perfect balance for large models (>100B) | default: -400000000.0 bonus_perfect_large: -400000000.0 # Good balance for large models | default: -200000000.0 bonus_good_large: -200000000.0 # Perfect balance for smaller models | default: -300000000.0 bonus_perfect_small: -300000000.0 # Good balance for smaller models | default: -100000000.0 bonus_good_small: -100000000.0 # Very large MoE with tight memory | default: -500000000.0 bonus_strong_moe: -500000000.0 # Quadratic TP penalty factor | default: 5000000.0 tp_squared_multiplier: 5000000.0 # PP >= 4 is significant | default: 4 pp_significant_threshold: 4 # EP >= 4 is significant | default: 4 ep_significant_threshold: 4 # Configuration for trust_remote_code in the models service trust_remote_code: # List of repo IDs or regex patterns trusted for model loading from HF (direct match or fullmatch). hf_allow_list: - nvidia/.* # List of org/team/name strings or regex patterns for trusted for model loading from NGC (direct match or fullmatch). ngc_allow_list: - nvidia/.* # Whether to allow trust_remote_code anywhere in the platform | default: True enabled: true # Configuration for tool_call_plugin in the models service tool_call_plugin: # Whether to allow custom tool-call parser plugins. When disabled (default), any tool_call_plugin value supplied via API or fileset metadata is rejected or stripped. Enable with caution — plugins execute arbitrary Python code inside the inference container. | default: False enabled: false ``` ### `secrets` Configuration for the Secrets service. ```yaml wordWrap secrets: # Encryption configuration for the Secrets service. encryption: # The type of encryption key used to encrypt/decrypt secrets. | default: '' current_provider: '' # Configuration for all available encryption providers. providers: # Mapping of secret key encryptor names to their configurations. secret_key: default: # Base64-encoded encryption key value: # Indicates if the key should be loaded from an environment variable. If set, 'value' is ignored. from_env: # Mapping of vault encryptor names to their configurations. vault: default: # Name of the key in Vault to use for encryption/decryption. | default: 'nemo-platform-key' key_name: nemo-platform-key # Address of the Vault server. If not specificed, the value of the VAULT_ADDR env variable will be used. address: # Authentication token for Vault. If not specified, the value of the VAULT_TOKEN env variable or the content of the file located at ~/.vault-token will be used. token: ``` ### `automodel` Environment variables use the NMP\_AUTOMODEL\_ prefix. ```yaml wordWrap automodel: # Registry host/path prefix for nmp-customizer-tasks and nmp-automodel-training. Override via NMP_AUTOMODEL_IMAGE_REGISTRY for other environments, defaults to the platform's image registry. image_registry: # Override entire GPU training image (registry/name:tag). training_image: # Override entire CPU tasks image (registry/name:tag). tasks_image: # default: '1' default_job_resource_cpu_request: '1' # default: '8Gi' default_job_resource_memory_request: 8Gi # default: '4' default_job_resource_cpu_limit: '4' # default: '16Gi' default_job_resource_memory_limit: 16Gi # Terminate training if no task progress within this many seconds (0 disables). | default: 3600 training_staleness_timeout_seconds: 3600 # Default GPU execution profile when the job spec omits training.execution_profile. | default: 'gpu' default_training_execution_profile: gpu ``` ### `unsloth` Environment variables use the `NMP_UNSLOTH_` prefix. ```yaml wordWrap unsloth: # Registry host/path prefix for nmp-unsloth-tasks and nmp-unsloth-training. Override via NMP_UNSLOTH_IMAGE_REGISTRY for other environments, defaults to the platform's image registry. image_registry: # Override entire GPU training image (registry/name:tag). training_image: # Override entire CPU tasks image (registry/name:tag). tasks_image: # default: '1' default_job_resource_cpu_request: '1' # default: '8Gi' default_job_resource_memory_request: 8Gi # default: '4' default_job_resource_cpu_limit: '4' # default: '16Gi' default_job_resource_memory_limit: 16Gi # Default GPU execution profile when the job spec omits training.execution_profile. | default: 'gpu' default_training_execution_profile: gpu ``` ### `safe_synthesizer` Configuration for Safe Synthesizer plugin API and task compilation. ```yaml wordWrap safe_synthesizer: # default: '0.0.0.0' host: 0.0.0.0 # default: 8000 port: 8000 entrypoint: - python - -m - nemo_safe_synthesizer_plugin.tasks.safe_synthesizer # default: 'subprocess-local' | values: 'subprocess-local' | 'container' job_mode: subprocess-local # default: 'default' job_executor_profile: default # default: 'safe-synthesizer-tasks' container_image: safe-synthesizer-tasks # Optional fully qualified task image reference. When set, this bypasses platform NMP_IMAGE_REGISTRY / NMP_IMAGE_TAG qualification for Safe Synthesizer jobs. container_image_ref: # default: '.nemo/safe-synthesizer-runtime' runtime_venv: .nemo/safe-synthesizer-runtime # default: 'nemo-safe-synthesizer[engine,cu129]==0.1.7' runtime_package: nemo-safe-synthesizer[engine,cu129]==0.1.7 # default: '3.11' runtime_python_version: '3.11' runtime_python: # default: '16G' default_job_resource_memory_request: 16G # default: '4' default_job_resource_cpu_request: '4' # default: '16G' default_job_resource_memory_limit: 16G # default: '4' default_job_resource_cpu_limit: '4' ``` ### `studio` Configuration for the Studio service. ```yaml wordWrap studio: # Path to the directory containing the built static UI assets. When unset, defaults to the `static/` directory bundled alongside the `nmp.studio` package (populated by the wheel build). static_files_path: # Base URL of the platform. This is used by the Studio UI to make API calls. | default: '' platform_base_url: '' # Enable Studio UI browser telemetry export. | default: False telemetry_enabled: false # Studio UI OpenTelemetry settings. otel: # Internal OTLP/HTTP collector base URL used by the Studio telemetry proxy. | default: '' collector_url: '' # OpenTelemetry service.name resource attribute for Studio UI telemetry. | default: 'nemo-studio-ui' service_name: nemo-studio-ui # Browser origins allowed to post telemetry through the Studio telemetry proxy. Entries use shell-style '*' wildcards. allowed_origins: - http://localhost - http://localhost:* ```