NeMo Platform configuration reference

1
platform:
2
  # Runtime used by the platform. Used to auto-detect default backends to use. | default: 'docker' | values: 'kubernetes' | 'docker' | 'none'
3
  runtime: docker
4
  # Base URL for the NeMo Platform api. Used as the default URL for all services. | default: 'http://localhost:8080'
5
  base_url: http://localhost:8080
6
  # Optional loopback address override for job containers to reach platform services. If not specified, automatically determined based on platform: macOS uses 'host.docker.internal', Docker containers use container hostname, Linux host network uses no override. Can be set via config file or NMP_LOOPBACK_ADDRESS env var.
7
  loopback_address:
8
  # Global image pull secrets for the platform
9
  image_pull_secrets: []
10
  # Docker registry for NeMo Platform images (e.g., 'nvcr.io/nvidia/nemo-platform'). | default: 'my-registry'
11
  image_registry: my-registry
12
  # Default tag for NeMo Platform images. | default: 'local'
13
  image_tag: local
14
  # Shared Docker configuration for services using Docker backends.
15
  docker:
16
    # GPU device IDs to reserve for the Docker GPU pool. Use 'all' to auto-detect and use all available GPUs, 'none' or empty string to disable GPU support, or a comma-separated list of device IDs (e.g., '0,1,2,3'). | default: 'all'
17
    reserved_gpu_device_ids: all
18
  # Name of the secret containing the default NGC API key. Defaults to 'system/ngc-api-key'. | default: 'system/ngc-api-key'
19
  ngc_api_key_secret: system/ngc-api-key
20
  # Environment variable name to source the NGC API key from. | default: 'NGC_API_KEY'
21
  ngc_api_key_env_var: NGC_API_KEY
22
  # When true, GET / returns a 301 redirect to /studio. When false, GET / returns 404. This is used to redirect the root URL to the Studio UI. | default: True
23
  redirect_root_to_studio: true

1
service:
2
  # Format for logs generated by the service | default: 'plain' | values: 'json' | 'plain'
3
  log_format: plain
4
  # Logging level for the NeMo Platform. | default: 'INFO' | values: 'DEBUG' | 'INFO' | 'WARN' | 'ERROR'
5
  log_level: INFO
6
  # Scheme for the NeMo Platform service. | default: 'http'
7
  scheme: http
8
  # Host for the NeMo Platform service. | default: '127.0.0.1'
9
  host: 127.0.0.1
10
  # Port for the NeMo Platform service. | default: 8080
11
  port: 8080

1
auth:
2
  # Master switch for authorization. If False, all requests are allowed. | default: False
3
  enabled: false
4
  # Base URL for the Policy Decision Point (auth service or external OPA). | default: 'http://localhost:8080'
5
  policy_decision_point_base_url: http://localhost:8080
6
  # Policy Decision Point provider: 'embedded' for auth service's built-in WASM engine, 'opa' for external OPA sidecar. | default: 'embedded' | values: 'embedded' | 'opa'
7
  policy_decision_point_provider: embedded
8
  # HTTP timeout in seconds for outbound Policy Decision Point (PDP) requests | default: 5
9
  policy_decision_point_request_timeout_seconds: 5
10
  # Default polling interval (seconds) used by AuthClient.wait_role and wait_permissions. Lower values reduce role-propagation wait time at the cost of more PDP requests; tests typically override this to a small value. | default: 1.0
11
  propagation_poll_interval_seconds: 1.0
12
  # Allow unsigned JWTs (`alg=none`) for local development/testing. Disabled by default and should not be enabled in production. | default: False
13
  allow_unsigned_jwt: false
14
  # When auth is enabled with the embedded PDP and policy.wasm is missing, build it automatically from a local NeMo Platform source checkout. Packaged deployments should include policy.wasm at build time and can disable this for fail-fast startup. | default: True
15
  embedded_pdp_auto_build_wasm: true
16
  # OIDC configuration for native token validation.
17
  oidc:
18
    # Enable native OIDC token validation. | default: False
19
    enabled: false
20
    # OIDC issuer URL (e.g., https://sso.nvidia.com). Used for token validation and .well-known discovery. | default: ''
21
    issuer: ''
22
    # Additional valid issuers for token validation. Useful for Azure AD where access tokens use v1.0 issuer format (https://sts.windows.net/{tenant}/) while endpoints use v2.0.
23
    additional_issuers: []
24
    # OAuth client ID for this NeMo Platform deployment. Used for device flow and token audience validation. | default: ''
25
    client_id: ''
26
    # Override authorization endpoint (defaults to discovery).
27
    authorization_endpoint:
28
    # Override token endpoint (defaults to discovery).
29
    token_endpoint:
30
    # Override device authorization endpoint (defaults to discovery).
31
    device_authorization_endpoint:
32
    # Override JWKS URI for token validation (defaults to discovery).
33
    jwks_uri:
34
    # Expected token audience. When set, tokens must include this value in their 'aud' claim. When not set, audience validation is skipped entirely.
35
    audience:
36
    # JWT claim containing the principal email (maps to X-NMP-Principal-Email). Set explicitly for your IdP. | default: 'email'
37
    email_claim: email
38
    # JWT claim containing user groups. Supports 'groups' (standard) and 'cognito:groups' (AWS Cognito). | default: 'groups'
39
    groups_claim: groups
40
    # JWT claim to use as principal ID (maps to X-NMP-Principal-Id). Set explicitly for your IdP. | default: 'sub'
41
    subject_claim: sub
42
    # Space-separated OAuth scopes to request during authentication. For Azure AD with custom API, use: 'api://{app-id}/.default openid profile email' | default: 'openid profile email offline_access'
43
    default_scopes: openid profile email offline_access
44
    # Enable SDK workload identity token exchange using NMP_WORKLOAD_IDENTITY_TOKEN_FILE subject tokens. | default: False
45
    workload_token_exchange_enabled: false
46
    # OAuth client ID to use for workload identity token exchange. Defaults to client_id when unset.
47
    workload_client_id:
48
    # OAuth token endpoint to use for workload identity token exchange. Defaults to token_endpoint.
49
    workload_token_endpoint:
50
    # RFC 8693 audience requested for workload identity token exchange.
51
    workload_audience:
52
    # Space-separated OAuth scopes requested for workload identity token exchange.
53
    workload_scope:
54
    # Issuer to stamp on workload identity access tokens minted by the NeMo auth service. Defaults to the platform auth endpoint origin serving the token exchange request.
55
    workload_token_issuer:
56
    # Lifetime in seconds for workload identity access tokens minted by the NeMo auth service. | default: 300
57
    workload_token_ttl_seconds: 300
58
    # Workload-specific JWT key id advertised by the NeMo auth service workload identity JWKS endpoint. When unset, workload token exchange uses auth.token_signing.key_id.
59
    workload_token_key_id:
60
    # Path to a PEM-encoded RSA private key used by the NeMo auth service to sign workload identity access tokens. Intended for mounted shared secrets.
61
    workload_token_private_key_file:
62
    # Additional RFC 8693 audience values accepted by the NeMo auth service workload token exchange endpoint. The configured workload_audience is always accepted.
63
    workload_allowed_audiences: []
64
    # JWKS URI used by the NeMo auth service to validate JWT subject tokens for workload token exchange. Leave unset when only Kubernetes TokenReview subject validation is enabled.
65
    workload_subject_jwks_uri:
66
    # Allowed JWT subject token issuers for workload token exchange. Required when workload_subject_jwks_uri is set.
67
    workload_subject_issuers: []
68
    # TTL in seconds for caching workload subject JWKS responses. Set to 0 to disable caching. | default: 3600
69
    workload_subject_jwks_cache_ttl_seconds: 3600
70
    # Allow the NeMo auth service workload token exchange endpoint to validate Kubernetes projected service account subject tokens using the TokenReview API. | default: False
71
    workload_kubernetes_token_review_enabled: false
72
    # Prefix to strip from token scopes before authorization. For example, if IdP returns 'api://my-app/models:read', set prefix to 'api://my-app/' to normalize to 'models:read'. If not set, scopes are used as-is.
73
    scope_prefix:
74
    # TTL in seconds for caching IdP discovery document responses. Used by the discovery endpoint to avoid per-request IdP calls. Set to 0 to disable caching. | default: 300
75
    discovery_cache_ttl: 300
76
  # Shared token signing configuration for NeMo Platform-minted JWTs.
77
  token_signing:
78
    # Shared issuer for NeMo Platform-minted JWTs. Defaults to <platform.base_url>/apis/auth.
79
    issuer:
80
    # Shared JWT key id advertised by NeMo Platform JWKS endpoints. | default: 'nemo-platform-signing'
81
    key_id: nemo-platform-signing
82
    # Path to the PEM-encoded RSA private key used by the auth service to sign NeMo Platform-minted JWTs. Intended for mounted shared secrets.
83
    private_key_file:
84
  # Scoped Access Key configuration.
85
  access_keys:
86
    # Enable NeMo Platform Scoped Access Key creation and validation. | default: False
87
    enabled: false
88
    # Token format to issue for newly created Scoped Access Keys. | default: 'jwt' | values: 'jwt'
89
    issue_format: jwt
90
    # Scoped Access Key token formats accepted by validators.
91
    accepted_formats:
92
    - jwt
93
    # Expected audience for NeMo Platform Scoped Access Key JWTs. | default: 'nemo-platform-access-key'
94
    audience: nemo-platform-access-key
95
    # Default finite lifetime in seconds for newly created Scoped Access Keys when expires_in_seconds is omitted. Set to null to require callers to provide an expiry when max_expires_in_seconds is finite, or to make omitted expiry non-time-delimited when max_expires_in_seconds is also null. | default: 2592000
96
    default_expires_in_seconds: 2592000
97
    # Maximum finite lifetime accepted when creating Scoped Access Keys. Set to null to allow explicit no-expiration requests. | default: 2592000
98
    max_expires_in_seconds: 2592000
99
  # Port to run the service on | default: 8000
100
  port: 8000
101
  # Refresh interval for policy data in seconds | default: 30
102
  policy_data_refresh_interval: 30.0
103
  # Seconds to cache the OPA bundle | default: 5
104
  bundle_cache_seconds: 5.0
105
  # Bootstrap admin email for platform setup
106
  admin_email:
107
  # Name of the default workspace where all authenticated users get Editor role | default: 'default'
108
  default_workspace: default
109
  # CPU budget for embedded PDP policy evaluation, in millions of WASM fuel units. Default of 200 provides headroom for full plugin-merged authorization data. | default: 200
110
  embedded_pdp_cpu_limit: 200
111
  # Maximum linear memory (MB) the embedded PDP WASM runtime can consume. | default: 32
112
  embedded_pdp_memory_limit_mb: 32
113
  # Fail-mode for a plugin that contributes invalid HTTP authz. | default: 'hard_fail' | values: 'deny_route' | 'quarantine' | 'hard_fail'
114
  on_invalid_plugin: hard_fail

1
entities:
2
  # Run Alembic migrations (upgrade head) at startup. | default: True
3
  run_migrations: true
4
  database_config:
5
    # Full database URL (overrides other settings)
6
    url:
7
    # Database dialect - either sqlite or postgresql | default: 'postgresql'
8
    dialect: postgresql
9
    # Database hostname | default: ''
10
    host: ''
11
    # Database path | default: ''
12
    path: ''
13
    # Database name | default: ''
14
    name: ''
15
    # Optional database port
16
    port:
17
    # Optional database username
18
    user:
19
    # Optional database password
20
    password:
21
    # Maximum number of connections in the database connection pool | default: 10
22
    connections_limit: 10
23
    # Connection timeout in seconds. For PostgreSQL (asyncpg) and SQLite, how long to wait when connecting or acquiring a lock. | default: 30
24
    connect_timeout_seconds: 30
25
    # Enable SQLAlchemy echo for the database connection | default: False
26
    echo: false
27
  # Interval in seconds to run the workspace cleanup routine. | default: 10
28
  workspace_cleanup_interval: 10
29
  # Interval in seconds for the background database health ping (used for readiness). | default: 10.0
30
  db_health_check_interval_seconds: 10.0
31
  # Enable in-process caching of per-principal role-binding list queries. Set via NMP_ENTITIES_PRINCIPAL_BINDINGS_CACHE_ENABLED. | default: True
32
  principal_bindings_cache_enabled: true
33
  # TTL in seconds for principal role-binding cache entries (NMP_ENTITIES_PRINCIPAL_BINDINGS_CACHE_TTL_SEC). | default: 60.0
34
  principal_bindings_cache_ttl_sec: 60.0
35
  # Max distinct principal keys in the role-binding cache before LRU eviction (NMP_ENTITIES_PRINCIPAL_BINDINGS_CACHE_MAX_SIZE). | default: 256
36
  principal_bindings_cache_max_size: 256

1
files:
2
  default_storage_config:
3
    # Chunk size in bytes for reading/streaming files. Larger chunks reduce async overhead but increase memory per concurrent download. Default: 1MB. | default: 1048576
4
    read_chunk_size: 1048576
5
    # default: 'local' | values: <StorageConfigType.LOCAL: 'local'>
6
    type: local
7
    path: /data/files_storage
8
    # How many bytes to buffer before flushing to disk | default: 16777216
9
    write_buffer_size: 16777216
10
  # Comma-separated list of external hosts the Files service is allowed to access. | default: 'https://api.ngc.nvidia.com,https://huggingface.co'
11
  allowed_external_hosts: https://api.ngc.nvidia.com,https://huggingface.co
12
  # Allow users to explicitly create filesets with local storage config. Security-sensitive: enable only in trusted deployments. | default: False
13
  allow_user_local_storage: false
14
  # TTL for file locks in seconds (default 5 minutes) | default: 300
15
  file_lock_ttl_seconds: 300
16
  # Maximum concurrent downloads during cache warming | default: 3
17
  cache_warming_max_concurrent: 3
18
  # Maximum Hugging Face request attempts for transient failures. | default: 4
19
  hf_retry_attempts: 4
20
  # Initial Hugging Face retry delay in seconds before exponential backoff. | default: 0.5
21
  hf_retry_initial_delay_seconds: 0.5
22
  # Maximum Hugging Face retry delay in seconds. | default: 5.0
23
  hf_retry_max_delay_seconds: 5.0

1
inference_gateway:
2
  # How frequently (in seconds) to refresh the internal model cache from the Models service. If set to 0, disable automatic refreshing. | default: 3
3
  refresh_model_cache_interval_sec: 3
4
  # Time-to-live (in seconds) for cached secrets before they expire and need refreshing. If set to 0, we will refresh this each time the model providers are refreshed. | default: 0
5
  secrets_ttl_sec: 0

1
jobs:
2
  # List of executor profiles for the Jobs service
3
  executors: []
4
  # Default executor profile configurations
5
  executor_defaults:
6
    # Default Docker execution profile configuration
7
    docker:
8
      # default: 1800
9
      ttl_seconds_before_active: 1800
10
      # default: 86400
11
      ttl_seconds_active: 86400
12
      # default: 3600
13
      ttl_seconds_after_finished: 3600
14
      # default: True
15
      cleanup_completed_jobs_immediately: true
16
      # Path to the jobs launcher tool | default: '/tools/jobs-launcher'
17
      launcher_tool_path: /tools/jobs-launcher
18
      # Default container image for job task pods. Used when a job step omits container.image. When unset, falls back to the platform CPU tasks image (platform.image_registry/nmp-cpu-tasks:platform.image_tag).
19
      default_task_image:
20
      # Optional env vars applied to all jobs (e.g. HOME=/tmp). Keys must not conflict with platform-reserved names. Job steps may override these variables.
21
      env: {}
22
      # Docker storage configuration
23
      storage:
24
        # Name of the Docker volume for persistent storage | default: 'nemo-jobs-storage'
25
        volume_name: nemo-jobs-storage
26
        # Docker image used to set permissions on the volume | default: 'docker.io/library/busybox:stable'
27
        volume_permissions_image: docker.io/library/busybox:stable
28
        # List of additional Docker volume mounts for the job
29
        additional_volume_mounts: []
30
      # Docker networking configuration
31
      networking:
32
        # Docker network for the job container | default: 'host'
33
        job_container_network: host
34
      # Docker workload identity subject-token issuer configuration.
35
      workload_identity:
36
        # Enable Docker workload identity token-file injection. Defaults to auth.oidc.workload_token_exchange_enabled.
37
        enabled:
38
        # OAuth token endpoint used by the Docker demo issuer. Defaults to auth.oidc.token_endpoint.
39
        token_endpoint:
40
        # OAuth client ID used by the Docker demo issuer. Defaults to auth.oidc.workload_client_id or auth.oidc.client_id.
41
        client_id:
42
        # OAuth client secret for the Docker demo issuer.
43
        client_secret:
44
        # Username for the Docker demo issuer password grant.
45
        username:
46
        # Controller environment variable that contains the Docker demo issuer password grant shared secret. | default: 'AUTHENTIK_WORKLOAD_IDENTITY_PASSWORD'
47
        password_env_var: AUTHENTIK_WORKLOAD_IDENTITY_PASSWORD
48
        # OAuth scope for the Docker demo issuer.
49
        scope:
50
        # Fallback subject-token lifetime when the Docker demo issuer response omits expires_in.
51
        subject_token_ttl_seconds: 600
52
        # Seconds before subject-token expiry when the Docker refresher issues a replacement token. | default: 60
53
        refresh_margin_seconds: 60
54
    # Default Kubernetes execution profile configuration
55
    kubernetes_job:
56
      # default: 1800
57
      ttl_seconds_before_active: 1800
58
      # default: 86400
59
      ttl_seconds_active: 86400
60
      # default: 3600
61
      ttl_seconds_after_finished: 3600
62
      # default: True
63
      cleanup_completed_jobs_immediately: true
64
      # Path to the jobs launcher tool | default: '/tools/jobs-launcher'
65
      launcher_tool_path: /tools/jobs-launcher
66
      # Default container image for job task pods. Used when a job step omits container.image. When unset, falls back to the platform CPU tasks image (platform.image_registry/nmp-cpu-tasks:platform.image_tag).
67
      default_task_image:
68
      # Optional env vars applied to all jobs (e.g. HOME=/tmp). Keys must not conflict with platform-reserved names. Job steps may override these variables.
69
      env: {}
70
      # Kubernetes namespace to submit the job to. If not set, it will be determined from the environment.
71
      namespace:
72
      # Kubernetes service account name for job pods. Uses the Kubernetes default service account when set to 'default'. | default: 'default'
73
      service_account_name: default
74
      # Tolerations for the Kubernetes job pods.
75
      tolerations: []
76
      # Node selector for the Kubernetes job pods.
77
      node_selector: {}
78
      # Affinity for the Kubernetes job pods.
79
      affinity: {}
80
      # Resource requests and limits for the Kubernetes job pods.
81
      resources:
82
        # Minimum resources requested for the container.
83
        requests:
84
          # CPU specification (e.g., '250m', '1', '2.5').
85
          cpu:
86
          # Memory specification (e.g., '128Mi', '1Gi', '512M').
87
          memory:
88
        # Maximum resources the container can use.
89
        limits:
90
          # CPU specification (e.g., '250m', '1', '2.5').
91
          cpu:
92
          # Memory specification (e.g., '128Mi', '1Gi', '512M').
93
          memory:
94
        # Number of nodes to use. | default: 1
95
        num_nodes: 1
96
        # Step requesting number of GPUs.
97
        num_gpus:
98
        # Shared memory (/dev/shm) size as a Kubernetes quantity (e.g. '1Gi', '4Gi'). Used for GPU and distributed-GPU job executors. When unset, defaults to 1Gi per allocated GPU.
99
        shm_size:
100
      # Pod security context for the Kubernetes job pods.
101
      pod_security_context: {}
102
      # Image pull secrets for the Kubernetes job pods.
103
      image_pull_secrets: []
104
      # Metadata to add to each job object in the Kubernetes job.
105
      job_metadata:
106
        labels: {}
107
        annotations: {}
108
      # Metadata to add to each pod in the Kubernetes job.
109
      pod_metadata:
110
        labels: {}
111
        annotations: {}
112
      # Storage configuration for the Kubernetes job pods.
113
      storage:
114
        # Persistent Volume Claim Name to use for job storage. | default: ''
115
        pvc_name: ''
116
        # Image used to set volume permissions | default: 'docker.io/library/busybox:stable'
117
        volume_permissions_image: docker.io/library/busybox:stable
118
        # Additional volumes to mount
119
        additional_volumes: []
120
        # Additional volume mounts
121
        additional_volume_mounts: []
122
      # Number of GPUs to request for the job | default: 1
123
      num_gpus: 1
124
      # The scheduler name to use for the pod spec. When non-empty, this value is applied to the pod's schedulerName field, enabling custom schedulers such as KAI Scheduler. Empty string omits schedulerName so the cluster default scheduler is used. | default: ''
125
      scheduler_name: ''
126
      # Container image that contains the jobs-launcher binary. | default: 'nvcr.io/nvidia/nemo-microservices/jobs-launcher:latest'
127
      launcher_image: nvcr.io/nvidia/nemo-microservices/jobs-launcher:latest
128
      # Requested expirationSeconds for the projected service account token used as the workload identity subject token. | default: 600
129
      workload_identity_token_expiration_seconds: 600
130
      # Audience for the projected service account token. Defaults to auth.oidc.workload_client_id, auth.oidc.client_id, then 'nemo-platform'.
131
      workload_identity_token_audience:
132
    # Default Volcano execution profile configuration
133
    volcano_job:
134
      # default: 1800
135
      ttl_seconds_before_active: 1800
136
      # default: 86400
137
      ttl_seconds_active: 86400
138
      # default: 3600
139
      ttl_seconds_after_finished: 3600
140
      # default: True
141
      cleanup_completed_jobs_immediately: true
142
      # Path to the jobs launcher tool | default: '/tools/jobs-launcher'
143
      launcher_tool_path: /tools/jobs-launcher
144
      # Default container image for job task pods. Used when a job step omits container.image. When unset, falls back to the platform CPU tasks image (platform.image_registry/nmp-cpu-tasks:platform.image_tag).
145
      default_task_image:
146
      # Optional env vars applied to all jobs (e.g. HOME=/tmp). Keys must not conflict with platform-reserved names. Job steps may override these variables.
147
      env: {}
148
      # Kubernetes namespace to submit the job to. If not set, it will be determined from the environment.
149
      namespace:
150
      # Kubernetes service account name for job pods. Uses the Kubernetes default service account when set to 'default'. | default: 'default'
151
      service_account_name: default
152
      # Tolerations for the Kubernetes job pods.
153
      tolerations: []
154
      # Node selector for the Kubernetes job pods.
155
      node_selector: {}
156
      # Affinity for the Kubernetes job pods.
157
      affinity: {}
158
      # Resource requests and limits for the Kubernetes job pods.
159
      resources:
160
        # Minimum resources requested for the container.
161
        requests:
162
          # CPU specification (e.g., '250m', '1', '2.5').
163
          cpu:
164
          # Memory specification (e.g., '128Mi', '1Gi', '512M').
165
          memory:
166
        # Maximum resources the container can use.
167
        limits:
168
          # CPU specification (e.g., '250m', '1', '2.5').
169
          cpu:
170
          # Memory specification (e.g., '128Mi', '1Gi', '512M').
171
          memory:
172
        # Number of nodes to use. | default: 1
173
        num_nodes: 1
174
        # Step requesting number of GPUs.
175
        num_gpus:
176
        # Shared memory (/dev/shm) size as a Kubernetes quantity (e.g. '1Gi', '4Gi'). Used for GPU and distributed-GPU job executors. When unset, defaults to 1Gi per allocated GPU.
177
        shm_size:
178
      # Pod security context for the Kubernetes job pods.
179
      pod_security_context: {}
180
      # Image pull secrets for the Kubernetes job pods.
181
      image_pull_secrets: []
182
      # Metadata to add to each job object in the Kubernetes job.
183
      job_metadata:
184
        labels: {}
185
        annotations: {}
186
      # Metadata to add to each pod in the Kubernetes job.
187
      pod_metadata:
188
        labels: {}
189
        annotations: {}
190
      # Storage configuration for the Kubernetes job pods.
191
      storage:
192
        # Persistent Volume Claim Name to use for job storage. | default: ''
193
        pvc_name: ''
194
        # Image used to set volume permissions | default: 'docker.io/library/busybox:stable'
195
        volume_permissions_image: docker.io/library/busybox:stable
196
        # Additional volumes to mount
197
        additional_volumes: []
198
        # Additional volume mounts
199
        additional_volume_mounts: []
200
      # Number of GPUs to request for the job | default: 1
201
      num_gpus: 1
202
      # The scheduler name to use for the Volcano job. | default: 'volcano'
203
      scheduler_name: volcano
204
      # Container image that contains the jobs-launcher binary. | default: 'nvcr.io/nvidia/nemo-microservices/jobs-launcher:latest'
205
      launcher_image: nvcr.io/nvidia/nemo-microservices/jobs-launcher:latest
206
      # Requested expirationSeconds for the projected service account token used as the workload identity subject token. | default: 600
207
      workload_identity_token_expiration_seconds: 600
208
      # Audience for the projected service account token. Defaults to auth.oidc.workload_client_id, auth.oidc.client_id, then 'nemo-platform'.
209
      workload_identity_token_audience:
210
      # The Volcano queue to submit the job to. | default: 'default'
211
      queue: default
212
      # maxRetry indicates the maximum number of retries allowed by the job | default: 0
213
      max_retry: 0
214
      # plugins indicates the plugins used by Volcano when the job is scheduled. We always add the pytorch plugin if more than one node.
215
      plugins: {}
216
      # Enable multi-node networking injection. Sets annotations to trigger Kyverno policy mutations. | default: True
217
      enable_multi_node_networking: true
218
    # Default subprocess execution profile configuration
219
    subprocess:
220
      # default: 1800
221
      ttl_seconds_before_active: 1800
222
      # default: 86400
223
      ttl_seconds_active: 86400
224
      # default: 3600
225
      ttl_seconds_after_finished: 3600
226
      # Keep subprocess working directories by default so runs remain inspectable. | default: False
227
      cleanup_completed_jobs_immediately: false
228
      # Path to the jobs launcher tool | default: '/tools/jobs-launcher'
229
      launcher_tool_path: /tools/jobs-launcher
230
      # Default container image for job task pods. Used when a job step omits container.image. When unset, falls back to the platform CPU tasks image (platform.image_registry/nmp-cpu-tasks:platform.image_tag).
231
      default_task_image:
232
      # Optional env vars applied to all jobs (e.g. HOME=/tmp). Keys must not conflict with platform-reserved names. Job steps may override these variables.
233
      env: {}
234
      # Root directory for subprocess job state, config, storage, and logs. | default: '/tmp/nmp-subprocess-jobs'
235
      working_directory: /tmp/nmp-subprocess-jobs
236
      # How long to wait after SIGTERM before force killing the process group. | default: 10
237
      graceful_shutdown_timeout_seconds: 10
238
  # Interval in seconds for the job reconciler to run | default: 2
239
  reconcile_interval_seconds: 2
240
  # Interval in seconds for the job scheduler to run | default: 5
241
  schedule_interval_seconds: 5
242
  # Register the subprocess/default execution profile. When unset, defaults to true for docker/none runtimes and false for kubernetes.
243
  enable_subprocess_executor:
244
  # Include raw job log lines in controller diagnostics snapshots. Disabled by default because job logs may contain secrets or PII. Enable only for local debugging or test environments. | default: False
245
  include_job_logs_in_diagnostics: false

1
models:
2
  # Image used to pull model weights. Its entrypoint is overridden to the Hugging Face CLI ('hf download ...'), so any image with the 'hf' CLI on PATH works. Defaults to the platform's nmp-api image (registry/tag from platform config); override to use a different puller image.
3
  huggingface_model_puller: my-registry/nmp-api:local
4
  # Controller service configuration
5
  controller:
6
    # Interval in seconds for the Models Controller to run its control loop | default: 5
7
    interval_seconds: 5
8
    # Dict of custom backend configurations for the Models Controller
9
    backends:
10
      deployments_plugin:
11
        default_executor:
12
        docker_executor:
13
        k8s_executor:
14
        # Maximum seconds a deployment may stay PENDING before ERROR. | default: 7200
15
        pending_timeout_seconds: 7200
16
        # default: 5
17
        max_restart_count: 5
18
        default_storage_class:
19
        # default: '200Gi'
20
        default_pvc_size: 200Gi
21
        # default: 'nvcr.io/nim/meta/llama-3.1-8b-instruct'
22
        default_nimservice_image: nvcr.io/nim/meta/llama-3.1-8b-instruct
23
        # default: '1.8.5'
24
        default_nimservice_image_tag: 1.8.5
25
        # Default vLLM image repository. Fully qualified so it resolves on runtimes that block docker.io short names. | default: 'docker.io/vllm/vllm-openai'
26
        default_vllm_image: docker.io/vllm/vllm-openai
27
        # default: 'v0.22.1'
28
        default_vllm_image_tag: v0.22.1
29
        # default: 1000
30
        default_user_id: 1000
31
        # default: 2000
32
        default_group_id: 2000
33
        # default: 2000
34
        default_vllm_user_id: 2000
35
        # default: 0
36
        default_vllm_group_id: 0
37
        # default: 30
38
        peft_refresh_interval: 30
39
        # default: 'nmp-api'
40
        lora_sidecar_image_name: nmp-api
41
        # Container command for the LoRA adapters sidecar. Replaces the image ENTRYPOINT (e.g. `nemo`), so the adapters module is invoked directly and does not write platform instance state under the image $HOME (which is unwritable under the pod's runtime uid).
42
        lora_sidecar_command:
43
        - python
44
        - -m
45
        - nmp.core.models.sidecars.adapters.main
46
        lora_sidecar_args: []
47
        # BusyBox image repository for LoRA cache init containers. Fully qualified so it resolves on runtimes that block docker.io short names. | default: 'docker.io/library/busybox'
48
        busybox_image: docker.io/library/busybox
49
        # BusyBox image tag for LoRA cache init containers. | default: 'latest'
50
        busybox_image_tag: latest
51
        # Maximum seconds a deployment may stay DELETING before ERROR. 0 disables timeout escalation. | default: 60
52
        deleting_timeout_seconds: 60
53
        # default: False
54
        enabled: false
55
    # Time-to-live in seconds for DELETED deployments before they are permanently removed from the database | default: 30
56
    model_deployment_garbage_collection_ttl_seconds: 30
57
    # Time-to-live in seconds for ERROR deployments before backend resources are garbage collected | default: 10800
58
    error_deployment_ttl_seconds: 10800
59
    # Maximum number of attempts to recover a deployment when backend resources are lost | default: 5
60
    drift_recovery_max_attempts: 5
61
    # Base delay in seconds between drift recovery attempts (used for exponential backoff) | default: 30
62
    drift_recovery_base_delay_seconds: 30
63
    # Maximum delay in seconds between drift recovery attempts (caps exponential backoff) | default: 300
64
    drift_recovery_max_delay_seconds: 300
65
    # Per-request timeout in seconds for GET /v1/models provider autodiscovery via the inference gateway. External providers (e.g. NVIDIA Build) can return large model lists and need longer than the SDK default. | default: 180
66
    provider_discovery_timeout_seconds: 180
67
    # SDK retry count for provider autodiscovery requests. The controller loop already retries on each cycle; disabling SDK retries avoids multi-minute retry storms on slow upstreams. | default: 0
68
    provider_discovery_max_retries: 0
69
  # Parallelism estimation heuristics (model size thresholds, TP/PP/DP/CP/EP costs, balance bonuses)
70
  parallelism:
71
    # Standard node configuration (e.g., DGX H100) | default: 8
72
    gpus_per_node_default: 8
73
    # Standard GPU memory in GB (e.g., H100 80GB) | default: 80
74
    gpu_memory_gb_default: 80
75
    model_size_thresholds:
76
      # >300B: Very large models | default: 300.0
77
      very_large: 300.0
78
      # 100-300B: Large models | default: 100.0
79
      large: 100.0
80
      # 50-100B: Medium models | default: 50.0
81
      medium: 50.0
82
      # <70B: Small models for TP cost | default: 70.0
83
      small_tp: 70.0
84
      # <40B: Small MoE models | default: 40.0
85
      small_moe: 40.0
86
    memory:
87
      # Start penalizing above 60% memory usage | default: 0.6
88
      pressure_threshold: 0.6
89
      # Moderate memory pressure | default: 0.5
90
      pressure_moderate: 0.5
91
      # Low memory pressure | default: 0.45
92
      pressure_low: 0.45
93
      # Base penalty for exceeding threshold | default: 1000000000.0
94
      base_penalty: 1000000000.0
95
      # Divisor for quadratic penalty: (excess / divisor) ** 2 | default: 0.1
96
      scale_divisor: 0.1
97
      # Maximum discount factor (70% off) | default: 0.7
98
      pp_discount_max: 0.7
99
      # Scale factor for discount calculation | default: 2.0
100
      pp_discount_scale: 2.0
101
    tensor_parallelism:
102
      # For models > 300B | default: 50.0
103
      base_cost_very_large_model: 50.0
104
      # For models <= 300B | default: 100.0
105
      base_cost_standard_model: 100.0
106
      # TP=8 is standard for 340B+ | default: 8
107
      excessive_very_large: 8
108
      # TP=4 is standard for 70B and below | default: 4
109
      excessive_standard: 4
110
      # Penalty for excessive TP on large models (>70B) | default: 100000000.0
111
      penalty_large_model: 100000000.0
112
      # Penalty for excessive TP on small models (<70B) | default: 300000000.0
113
      penalty_small_model: 300000000.0
114
    data_parallelism:
115
      # total_parallelism >= 64 | default: -10000.0
116
      bonus_minimal: -10000.0
117
      # total_parallelism >= 32 | default: -50000.0
118
      bonus_small: -50000.0
119
      # total_parallelism <= 2 | default: -50000000.0
120
      bonus_very_strong: -50000000.0
121
      # total_parallelism <= 4 | default: -30000000.0
122
      bonus_strong: -30000000.0
123
      # total_parallelism <= 8 | default: -15000000.0
124
      bonus_moderate: -15000000.0
125
      # total_parallelism > 8 | default: -20000000.0
126
      bonus_medium: -20000000.0
127
      # Smaller MoE models | default: -1000000.0
128
      bonus_small_moe: -1000000.0
129
      # Total parallelism threshold | default: 64
130
      total_parallelism_very_high: 64
131
      # Total parallelism threshold | default: 32
132
      total_parallelism_high: 32
133
      # Total parallelism threshold | default: 2
134
      total_parallelism_very_low: 2
135
      # Total parallelism threshold | default: 4
136
      total_parallelism_low: 4
137
      # Total parallelism threshold | default: 8
138
      total_parallelism_medium: 8
139
      # Double DP bonus for pure DP+CP configurations | default: 2.0
140
      cp_bonus_multiplier: 2.0
141
    pipeline_parallelism:
142
      # MoE models (lower cost due to higher compute per stage) | default: 500000.0
143
      cost_moe: 500000.0
144
      # param_count_b > 300 | default: 5000000.0
145
      cost_very_large_model: 5000000.0
146
      # param_count_b > 100 | default: 10000000.0
147
      cost_large_model: 10000000.0
148
      # param_count_b > 50 | default: 30000000.0
149
      cost_medium_model: 30000000.0
150
      # param_count_b <= 50 and tp > 1 | default: 100000000.0
151
      cost_small_with_tp: 100000000.0
152
      # param_count_b <= 50 and tp == 1 | default: 300000000.0
153
      cost_small_without_tp: 300000000.0
154
    context_parallelism:
155
      # Strong bonus for optimal CP | default: -300000000.0
156
      bonus_optimal: -300000000.0
157
      # Bonus for CP=2 in medium sequences | default: -200000000.0
158
      bonus_good: -200000000.0
159
      # Penalty: not using enough CP or using CP when not needed | default: 200000000.0
160
      penalty_suboptimal: 200000000.0
161
      # Penalty: too much CP | default: 100000000.0
162
      penalty_too_much: 100000000.0
163
      # Penalty: should use CP but using CP=1 | default: 300000000.0
164
      penalty_should_use: 300000000.0
165
      # Very long sequences | default: 1.0
166
      seq_to_param_ratio_high: 1.0
167
      # Medium sequences | default: 0.3
168
      seq_to_param_ratio_medium: 0.3
169
      # Maximum CP value to consider | default: 8
170
      max_value: 8
171
      # CP=2 is optimal for medium sequences | default: 2
172
      optimal_value: 2
173
      # Multiplier for parameter memory calculation | default: 6.0
174
      param_memory_multiplier: 6.0
175
      # Multiplier for layers in parameter calculation | default: 12.0
176
      param_layers_multiplier: 12.0
177
      # Multiplier for sequence memory calculation | default: 38.0
178
      seq_memory_multiplier: 38.0
179
      # Enable CP for sequences >= 8K | default: 8192
180
      seq_threshold_enable: 8192
181
      # Enable CP=4 for sequences >= 16K | default: 16384
182
      seq_threshold_cp4: 16384
183
      # Enable CP=8 for sequences >= 32K | default: 32768
184
      seq_threshold_cp8: 32768
185
      # Enable CP=16 for sequences >= 128K | default: 131072
186
      seq_threshold_cp16: 131072
187
    expert_parallelism:
188
      # Huge penalty for EP > 1 on non-MoE models | default: 1000000000.0
189
      penalty_non_moe: 1000000000.0
190
      # Perfect: 1 routed expert per GPU | default: -500000000.0
191
      bonus_perfect: -500000000.0
192
      # Very efficient: <= 8 experts per GPU | default: -400000000.0
193
      bonus_very_efficient: -400000000.0
194
      # Good: <= 32 experts per GPU | default: -300000000.0
195
      bonus_good: -300000000.0
196
      # Acceptable: <= 64 experts per GPU | default: -200000000.0
197
      bonus_acceptable: -200000000.0
198
      # High expert count per GPU (>64) | default: -100000000.0
199
      bonus_high_count: -100000000.0
200
      # Huge penalty: no EP on MoE | default: 800000000.0
201
      penalty_no_sharding: 800000000.0
202
      # Penalty for non-divisor EP | default: 300000000.0
203
      penalty_non_divisor: 300000000.0
204
      # Experts per GPU threshold | default: 8
205
      experts_per_gpu_very_efficient: 8
206
      # Experts per GPU threshold | default: 32
207
      experts_per_gpu_good: 32
208
      # Experts per GPU threshold | default: 64
209
      experts_per_gpu_acceptable: 64
210
    balance:
211
      # TP == PP (perfect balance) | default: 1.0
212
      ratio_perfect: 1.0
213
      # max(TP, PP) / min(TP, PP) <= 2.0 | default: 2.0
214
      ratio_good: 2.0
215
      # Perfect balance for very large dense models (>300B) | default: -500000000.0
216
      bonus_perfect_very_large: -500000000.0
217
      # Good balance for very large dense models | default: -300000000.0
218
      bonus_good_very_large: -300000000.0
219
      # Perfect balance for large models (>100B) | default: -400000000.0
220
      bonus_perfect_large: -400000000.0
221
      # Good balance for large models | default: -200000000.0
222
      bonus_good_large: -200000000.0
223
      # Perfect balance for smaller models | default: -300000000.0
224
      bonus_perfect_small: -300000000.0
225
      # Good balance for smaller models | default: -100000000.0
226
      bonus_good_small: -100000000.0
227
      # Very large MoE with tight memory | default: -500000000.0
228
      bonus_strong_moe: -500000000.0
229
      # Quadratic TP penalty factor | default: 5000000.0
230
      tp_squared_multiplier: 5000000.0
231
      # PP >= 4 is significant | default: 4
232
      pp_significant_threshold: 4
233
      # EP >= 4 is significant | default: 4
234
      ep_significant_threshold: 4
235
  # Configuration for trust_remote_code in the models service
236
  trust_remote_code:
237
    # List of repo IDs or regex patterns trusted for model loading from HF (direct match or fullmatch).
238
    hf_allow_list:
239
    - nvidia/.*
240
    # List of org/team/name strings or regex patterns for trusted for model loading from NGC (direct match or fullmatch).
241
    ngc_allow_list:
242
    - nvidia/.*
243
    # Whether to allow trust_remote_code anywhere in the platform | default: True
244
    enabled: true
245
  # Configuration for tool_call_plugin in the models service
246
  tool_call_plugin:
247
    # Whether to allow custom tool-call parser plugins. When disabled (default), any tool_call_plugin value supplied via API or fileset metadata is rejected or stripped. Enable with caution — plugins execute arbitrary Python code inside the inference container. | default: False
248
    enabled: false

1
secrets:
2
  # Encryption configuration for the Secrets service.
3
  encryption:
4
    # The type of encryption key used to encrypt/decrypt secrets. | default: ''
5
    current_provider: ''
6
    # Configuration for all available encryption providers.
7
    providers:
8
      # Mapping of secret key encryptor names to their configurations.
9
      secret_key:
10
        default:
11
          # Base64-encoded encryption key
12
          value:
13
          # Indicates if the key should be loaded from an environment variable. If set, 'value' is ignored.
14
          from_env:
15
      # Mapping of vault encryptor names to their configurations.
16
      vault:
17
        default:
18
          # Name of the key in Vault to use for encryption/decryption. | default: 'nemo-platform-key'
19
          key_name: nemo-platform-key
20
          # Address of the Vault server. If not specificed, the value of the VAULT_ADDR env variable will be used.
21
          address:
22
          # Authentication token for Vault. If not specified, the value of the VAULT_TOKEN env variable or the content of the file located at ~/.vault-token will be used.
23
          token:

1
automodel:
2
  # Registry host/path prefix for nmp-customizer-tasks and nmp-automodel-training. Override via NMP_AUTOMODEL_IMAGE_REGISTRY for other environments, defaults to the platform's image registry.
3
  image_registry:
4
  # Override entire GPU training image (registry/name:tag).
5
  training_image:
6
  # Override entire CPU tasks image (registry/name:tag).
7
  tasks_image:
8
  # default: '1'
9
  default_job_resource_cpu_request: '1'
10
  # default: '8Gi'
11
  default_job_resource_memory_request: 8Gi
12
  # default: '4'
13
  default_job_resource_cpu_limit: '4'
14
  # default: '16Gi'
15
  default_job_resource_memory_limit: 16Gi
16
  # Terminate training if no task progress within this many seconds (0 disables). | default: 3600
17
  training_staleness_timeout_seconds: 3600
18
  # Default GPU execution profile when the job spec omits training.execution_profile. | default: 'gpu'
19
  default_training_execution_profile: gpu

1
unsloth:
2
  # Registry host/path prefix for nmp-unsloth-tasks and nmp-unsloth-training. Override via NMP_UNSLOTH_IMAGE_REGISTRY for other environments, defaults to the platform's image registry.
3
  image_registry:
4
  # Override entire GPU training image (registry/name:tag).
5
  training_image:
6
  # Override entire CPU tasks image (registry/name:tag).
7
  tasks_image:
8
  # default: '1'
9
  default_job_resource_cpu_request: '1'
10
  # default: '8Gi'
11
  default_job_resource_memory_request: 8Gi
12
  # default: '4'
13
  default_job_resource_cpu_limit: '4'
14
  # default: '16Gi'
15
  default_job_resource_memory_limit: 16Gi
16
  # Default GPU execution profile when the job spec omits training.execution_profile. | default: 'gpu'
17
  default_training_execution_profile: gpu

1
safe_synthesizer:
2
  # default: '0.0.0.0'
3
  host: 0.0.0.0
4
  # default: 8000
5
  port: 8000
6
  entrypoint:
7
  - python
8
  - -m
9
  - nemo_safe_synthesizer_plugin.tasks.safe_synthesizer
10
  # default: 'subprocess-local' | values: 'subprocess-local' | 'container'
11
  job_mode: subprocess-local
12
  # default: 'default'
13
  job_executor_profile: default
14
  # default: 'safe-synthesizer-tasks'
15
  container_image: safe-synthesizer-tasks
16
  # Optional fully qualified task image reference. When set, this bypasses platform NMP_IMAGE_REGISTRY / NMP_IMAGE_TAG qualification for Safe Synthesizer jobs.
17
  container_image_ref:
18
  # default: '.nemo/safe-synthesizer-runtime'
19
  runtime_venv: .nemo/safe-synthesizer-runtime
20
  # default: 'nemo-safe-synthesizer[engine,cu129]==0.1.7'
21
  runtime_package: nemo-safe-synthesizer[engine,cu129]==0.1.7
22
  # default: '3.11'
23
  runtime_python_version: '3.11'
24
  runtime_python:
25
  # default: '16G'
26
  default_job_resource_memory_request: 16G
27
  # default: '4'
28
  default_job_resource_cpu_request: '4'
29
  # default: '16G'
30
  default_job_resource_memory_limit: 16G
31
  # default: '4'
32
  default_job_resource_cpu_limit: '4'

1
studio:
2
  # Path to the directory containing the built static UI assets. When unset, Studio looks for the `static/` directory bundled alongside the `nmp.studio` package (populated by the wheel build), then `/static/studio` (where NeMo Platform container images place the bundle), then `web/packages/studio/dist` in a source checkout.
3
  static_files_path:
4
  # Base URL of the platform. This is used by the Studio UI to make API calls. | default: ''
5
  platform_base_url: ''
6
  # Enable Studio UI browser telemetry export. | default: False
7
  telemetry_enabled: false
8
  # Studio UI OpenTelemetry settings.
9
  otel:
10
    # Internal OTLP/HTTP collector base URL used by the Studio telemetry proxy. | default: ''
11
    collector_url: ''
12
    # OpenTelemetry service.name resource attribute for Studio UI telemetry. | default: 'nemo-studio-ui'
13
    service_name: nemo-studio-ui
14
    # Browser origins allowed to post telemetry through the Studio telemetry proxy. Entries use shell-style '*' wildcards.
15
    allowed_origins:
16
    - http://localhost
17
    - http://localhost:*