vLLM Deployment Templates

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
4
apiVersion: nvidia.com/v1alpha1
5
kind: DynamoGraphDeployment
6
metadata:
7
  name: vllm-agg
8
spec:
9
  services:
10
    Frontend:
11
      envFromSecret: hf-token-secret
12
      componentType: frontend
13
      replicas: 1
14
      extraPodSpec:
15
        mainContainer:
16
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
17
    VllmDecodeWorker:
18
      envFromSecret: hf-token-secret
19
      componentType: worker
20
      replicas: 1
21
      resources:
22
        limits:
23
          gpu: "1"
24
        requests:
25
          custom:
26
            # Increase this value for larger models
27
            ephemeral-storage: "2Gi"
28
      extraPodSpec:
29
        mainContainer:
30
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
31
          workingDir: /workspace/examples/backends/vllm
32
          command:
33
            - python3
34
            - -m
35
            - dynamo.vllm
36
          args:
37
            - --model
38
            - Qwen/Qwen3-0.6B

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
4
apiVersion: nvidia.com/v1alpha1
5
kind: DynamoGraphDeployment
6
metadata:
7
  name: vllm-agg-router
8
spec:
9
  services:
10
    Frontend:
11
      componentType: frontend
12
      replicas: 1
13
      extraPodSpec:
14
        mainContainer:
15
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
16
      envs:
17
        - name: DYN_ROUTER_MODE
18
          value: kv
19
    VllmDecodeWorker:
20
      envFromSecret: hf-token-secret
21
      componentType: worker
22
      replicas: 2
23
      resources:
24
        limits:
25
          gpu: "1"
26
        requests:
27
          custom:
28
            # Increase this value for larger models
29
            ephemeral-storage: "2Gi"
30
      extraPodSpec:
31
        mainContainer:
32
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
33
          workingDir: /workspace/examples/backends/vllm
34
          command:
35
          - python3
36
          - -m
37
          - dynamo.vllm
38
          args:
39
            - --model
40
            - Qwen/Qwen3-0.6B
41
            - --kv-events-config
42
            - '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}'

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
4
## This example demonstrates KV-aware routing with the --no-kv-events flag.
5
## Instead of receiving KV events from workers, the router predicts cache state
6
## locally based on routing decisions with TTL-based expiration and pruning.
7
## Note: This mode does not require NATS during Dynamo platform deployment.
8
apiVersion: nvidia.com/v1alpha1
9
kind: DynamoGraphDeployment
10
metadata:
11
  name: vllm-agg-router-kv-approx
12
spec:
13
  services:
14
    Frontend:
15
      componentType: frontend
16
      replicas: 1
17
      extraPodSpec:
18
        mainContainer:
19
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
20
          command:
21
            - python3
22
          args:
23
            - -m
24
            - dynamo.frontend
25
            - --router-mode
26
            - kv
27
            - --no-kv-events
28
      envs:
29
        - name: DYN_ROUTER_MODE
30
          value: kv
31
    VllmDecodeWorker:
32
      envFromSecret: hf-token-secret
33
      componentType: worker
34
      replicas: 2
35
      resources:
36
        limits:
37
          gpu: "1"
38
      extraPodSpec:
39
        mainContainer:
40
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
41
          workingDir: /workspace/examples/backends/vllm
42
          command:
43
          - python3
44
          - -m
45
          - dynamo.vllm
46
          args:
47
            - --model
48
            - Qwen/Qwen3-0.6B
49
            - --kv-events-config
50
            - '{"enable_kv_cache_events": false}'

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
4
apiVersion: nvidia.com/v1alpha1
5
kind: DynamoGraphDeployment
6
metadata:
7
  name: vllm-agg-kvbm
8
spec:
9
  services:
10
    Frontend:
11
      componentType: frontend
12
      replicas: 1
13
      extraPodSpec:
14
        mainContainer:
15
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
16
    VllmDecodeWorker:
17
      envFromSecret: hf-token-secret
18
      componentType: worker
19
      replicas: 1
20
      resources:
21
        requests:
22
          gpu: "1"
23
          memory: "200Gi"
24
        limits:
25
          gpu: "1"
26
          memory: "250Gi"
27
      envs:
28
        - name: DYN_KVBM_CPU_CACHE_GB
29
          value: "100"
30
      extraPodSpec:
31
        mainContainer:
32
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
33
          workingDir: /workspace/examples/backends/vllm
34
          command:
35
          - python3
36
          - -m
37
          - dynamo.vllm
38
          args:
39
            - --model
40
            - Qwen/Qwen3-8B
41
            - --max-model-len
42
            - "32000"
43
            - --enforce-eager
44
            - --kv-transfer-config
45
            - '{"kv_connector":"DynamoConnector","kv_connector_module_path":"kvbm.vllm_integration.connector","kv_role":"kv_both"}'

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
#
4
# GPU Memory Service (GMS) sidecar example.
5
#
6
# The operator injects a GMS sidecar init container that provides shared GPU
7
# memory access via DRA (Dynamic Resource Allocation). The sidecar runs two GMS
8
# processes per GPU (weights + kv_cache) and communicates with the main container
9
# over UDS sockets on a shared emptyDir volume.
10
#
11
# Requires Kubernetes 1.34+ with DRA v1 enabled and the NVIDIA GPU DRA driver installed.
12
13
apiVersion: nvidia.com/v1alpha1
14
kind: DynamoGraphDeployment
15
metadata:
16
  name: vllm-agg-gms
17
spec:
18
  services:
19
    Frontend:
20
      envFromSecret: hf-token-secret
21
      componentType: frontend
22
      replicas: 1
23
      extraPodSpec:
24
        mainContainer:
25
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
26
    VllmWorker:
27
      envFromSecret: hf-token-secret
28
      componentType: worker
29
      replicas: 1
30
      resources:
31
        limits:
32
          gpu: "1"
33
        requests:
34
          custom:
35
            ephemeral-storage: "2Gi"
36
      gpuMemoryService:
37
        enabled: true
38
      extraPodSpec:
39
        mainContainer:
40
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
41
          workingDir: /workspace/examples/backends/vllm
42
          command:
43
            - python3
44
            - -m
45
            - dynamo.vllm
46
          args:
47
            - --model
48
            - Qwen/Qwen3-0.6B
49
            - --load-format
50
            - gms

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
#
4
# Active-passive GPU failover example.
5
#
6
# The operator clones the worker's main container into two engine containers
7
# (engine-0 active, engine-1 standby) sharing GPUs via DRA and the GMS sidecar.
8
# When the active engine fails, the standby acquires the flock and takes over.
9
#
10
# Requires:
11
#   - gpuMemoryService.enabled: true (GMS sidecar + DRA)
12
#   - nvidia.com/dynamo-kube-discovery-mode: container (per-container K8s discovery)
13
#   - Kubernetes 1.34+ with DRA v1 enabled and the NVIDIA GPU DRA driver installed
14
15
apiVersion: nvidia.com/v1alpha1
16
kind: DynamoGraphDeployment
17
metadata:
18
  name: vllm-agg-failover
19
  annotations:
20
    nvidia.com/dynamo-kube-discovery-mode: container
21
spec:
22
  services:
23
    Frontend:
24
      envFromSecret: hf-token-secret
25
      componentType: frontend
26
      replicas: 1
27
      extraPodSpec:
28
        mainContainer:
29
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
30
    VllmWorker:
31
      envFromSecret: hf-token-secret
32
      componentType: worker
33
      replicas: 1
34
      resources:
35
        limits:
36
          gpu: "2"
37
        requests:
38
          custom:
39
            ephemeral-storage: "2Gi"
40
      gpuMemoryService:
41
        enabled: true
42
      failover:
43
        enabled: true
44
      extraPodSpec:
45
        mainContainer:
46
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
47
          workingDir: /workspace/examples/backends/vllm
48
          command:
49
            - python3
50
            - -m
51
            - dynamo.vllm
52
          args:
53
            - --model
54
            - Qwen/Qwen3-0.6B
55
            - --tensor-parallel-size
56
            - "2"
57
            - --load-format
58
            - gms

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
4
# Aggregated vLLM deployment with OpenTelemetry tracing enabled.
5
# Base deployment: agg.yaml
6
# See docs/observability/tracing.md for setup instructions.
7
8
apiVersion: nvidia.com/v1alpha1
9
kind: DynamoGraphDeployment
10
metadata:
11
  name: vllm-agg-tracing
12
spec:
13
  envs:
14
    - name: DYN_LOGGING_JSONL
15
      value: "true"
16
    - name: OTEL_EXPORT_ENABLED
17
      value: "true"
18
    - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
19
      value: "http://tempo.observability.svc.cluster.local:4317"
20
  services:
21
    Frontend:
22
      envFromSecret: hf-token-secret
23
      componentType: frontend
24
      replicas: 1
25
      extraPodSpec:
26
        mainContainer:
27
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
28
          env:
29
            - name: OTEL_SERVICE_NAME
30
              value: "dynamo-frontend"
31
    VllmDecodeWorker:
32
      envFromSecret: hf-token-secret
33
      componentType: worker
34
      replicas: 1
35
      resources:
36
        limits:
37
          gpu: "1"
38
        requests:
39
          custom:
40
            # Increase this value for larger models
41
            ephemeral-storage: "2Gi"
42
      extraPodSpec:
43
        mainContainer:
44
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
45
          workingDir: /workspace/examples/backends/vllm
46
          command:
47
            - python3
48
            - -m
49
            - dynamo.vllm
50
          args:
51
            - --model
52
            - Qwen/Qwen3-0.6B
53
          env:
54
            - name: OTEL_SERVICE_NAME
55
              value: "dynamo-worker-vllm"

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
4
apiVersion: resource.k8s.io/v1
5
kind: ResourceClaimTemplate
6
metadata:
7
  name: gpu-template
8
spec:
9
  spec:
10
    devices:
11
      requests:
12
        - name: gpu
13
          exactly:
14
            deviceClassName: gpu.intel.com
15
            count: 1
16
---
17
apiVersion: nvidia.com/v1alpha1
18
kind: DynamoGraphDeployment
19
metadata:
20
  name: vllm-agg-xpu-dra
21
spec:
22
  services:
23
    Frontend:
24
      envFromSecret: hf-token-secret
25
      componentType: frontend
26
      replicas: 1
27
      extraPodSpec:
28
        mainContainer:
29
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
30
    VllmDecodeWorker:
31
      envFromSecret: hf-token-secret
32
      componentType: worker
33
      replicas: 1
34
      resources:
35
        requests:
36
          custom:
37
            # Increase this value for larger models
38
            ephemeral-storage: "2Gi"
39
      extraPodSpec:
40
        resourceClaims:
41
          - name: gpu
42
            resourceClaimTemplateName: gpu-template
43
        # NOTE: Uncomment if your environment requires specific group access
44
        # securityContext:
45
        #   runAsUser: 1000
46
        #   runAsGroup: 1000
47
        #   supplementalGroups:
48
        #     - 44   # render group
49
        #     - 991  # video group
50
        mainContainer:
51
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime-xpu:my-tag
52
          resources:
53
            claims:
54
              - name: gpu
55
          env:
56
            - name: VLLM_TARGET_DEVICE
57
              value: xpu
58
          workingDir: /workspace/examples/backends/vllm
59
          command:
60
            - python3
61
            - -m
62
            - dynamo.vllm
63
          args:
64
            - --model
65
            - Qwen/Qwen3-0.6B
66
            - --block-size
67
            - "64"

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
4
apiVersion: nvidia.com/v1alpha1
5
kind: DynamoGraphDeployment
6
metadata:
7
  name: vllm-disagg
8
spec:
9
  services:
10
    Frontend:
11
      componentType: frontend
12
      replicas: 1
13
      extraPodSpec:
14
        mainContainer:
15
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
16
    VllmDecodeWorker:
17
      envFromSecret: hf-token-secret
18
      componentType: worker
19
      subComponentType: decode
20
      replicas: 1
21
      resources:
22
        limits:
23
          gpu: "1"
24
        requests:
25
          custom:
26
            # Increase this value for larger models
27
            ephemeral-storage: "2Gi"
28
      extraPodSpec:
29
        mainContainer:
30
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
31
          workingDir: /workspace/examples/backends/vllm
32
          command:
33
          - python3
34
          - -m
35
          - dynamo.vllm
36
          args:
37
            - --model
38
            - Qwen/Qwen3-0.6B
39
            - --disaggregation-mode
40
            - decode
41
    VllmPrefillWorker:
42
      envFromSecret: hf-token-secret
43
      componentType: worker
44
      subComponentType: prefill
45
      replicas: 1
46
      resources:
47
        limits:
48
          gpu: "1"
49
        requests:
50
          custom:
51
            # Increase this value for larger models
52
            ephemeral-storage: "2Gi"
53
      extraPodSpec:
54
        mainContainer:
55
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
56
          workingDir: /workspace/examples/backends/vllm
57
          command:
58
          - python3
59
          - -m
60
          - dynamo.vllm
61
          args:
62
            - --model
63
            - Qwen/Qwen3-0.6B
64
            - --disaggregation-mode
65
            - prefill
66
            - --kv-transfer-config
67
            - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
4
apiVersion: nvidia.com/v1alpha1
5
kind: DynamoGraphDeployment
6
metadata:
7
  name: vllm-v1-disagg-router
8
spec:
9
  services:
10
    Frontend:
11
      componentType: frontend
12
      replicas: 1
13
      extraPodSpec:
14
        mainContainer:
15
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
16
      envs:
17
        - name: DYN_ROUTER_MODE
18
          value: kv
19
    VllmDecodeWorker:
20
      envFromSecret: hf-token-secret
21
      componentType: worker
22
      replicas: 2
23
      resources:
24
        limits:
25
          gpu: "1"
26
        requests:
27
          custom:
28
            # Increase this value for larger models
29
            ephemeral-storage: "2Gi"
30
      extraPodSpec:
31
        mainContainer:
32
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
33
          workingDir: /workspace/examples/backends/vllm
34
          command:
35
          - python3
36
          - -m
37
          - dynamo.vllm
38
          args:
39
            - --model
40
            - Qwen/Qwen3-0.6B
41
            - --disaggregation-mode
42
            - decode
43
    VllmPrefillWorker:
44
      envFromSecret: hf-token-secret
45
      componentType: worker
46
      replicas: 2
47
      resources:
48
        limits:
49
          gpu: "1"
50
        requests:
51
          custom:
52
            # Increase this value for larger models
53
            ephemeral-storage: "2Gi"
54
      extraPodSpec:
55
        mainContainer:
56
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
57
          workingDir: /workspace/examples/backends/vllm
58
          command:
59
          - python3
60
          - -m
61
          - dynamo.vllm
62
          args:
63
            - --model
64
            - Qwen/Qwen3-0.6B
65
            - --disaggregation-mode
66
            - prefill
67
            - --kv-transfer-config
68
            - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
69
            - --kv-events-config
70
            - '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}'

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
4
apiVersion: nvidia.com/v1alpha1
5
kind: DynamoGraphDeployment
6
metadata:
7
  name: vllm-disagg-planner
8
spec:
9
  services:
10
    Frontend:
11
      componentType: frontend
12
      replicas: 1
13
      extraPodSpec:
14
        mainContainer:
15
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
16
    Planner:
17
      componentType: planner
18
      replicas: 1
19
      extraPodSpec:
20
        mainContainer:
21
          # Planner image selection:
22
          #   Dynamo >= 1.1.0: use the dedicated planner image
23
          #     nvcr.io/nvidia/ai-dynamo/dynamo-planner:<version>
24
          #     (backend runtime images no longer ship planner runtime deps
25
          #     such as kubernetes_asyncio, pmdarima, prophet, aiconfigurator).
26
          #   Dynamo <  1.1.0: use the backend runtime image
27
          #     nvcr.io/nvidia/ai-dynamo/vllm-runtime:<version>.
28
          image: nvcr.io/nvidia/ai-dynamo/dynamo-planner:my-tag
29
          command:
30
          - python3
31
          - -m
32
          - dynamo.planner
33
          args:
34
            - --config
35
            - '{"environment": "kubernetes", "backend": "vllm", "optimization_target": "sla", "enable_throughput_scaling": true, "enable_load_scaling": true, "pre_deployment_sweeping_mode": "none", "throughput_adjustment_interval_seconds": 60, "load_adjustment_interval_seconds": 5}'
36
    VllmDecodeWorker:
37
      envFromSecret: hf-token-secret
38
      componentType: worker
39
      subComponentType: decode
40
      replicas: 1
41
      resources:
42
        limits:
43
          gpu: "1"
44
      extraPodSpec:
45
        mainContainer:
46
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
47
          workingDir: /workspace/examples/backends/vllm
48
          command:
49
            - python3
50
          args:
51
            - -m
52
            - dynamo.vllm
53
            - --model
54
            - Qwen/Qwen3-0.6B
55
    VllmPrefillWorker:
56
      envFromSecret: hf-token-secret
57
      componentType: worker
58
      subComponentType: prefill
59
      replicas: 1
60
      resources:
61
        limits:
62
          gpu: "1"
63
      extraPodSpec:
64
        mainContainer:
65
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
66
          workingDir: /workspace/examples/backends/vllm
67
          command:
68
            - python3
69
          args:
70
            - -m
71
            - dynamo.vllm
72
            - --model
73
            - Qwen/Qwen3-0.6B
74
            - --disaggregation-mode
75
            - prefill
76
            - --kv-transfer-config
77
            - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
4
apiVersion: nvidia.com/v1alpha1
5
kind: DynamoGraphDeployment
6
metadata:
7
  name: vllm-disagg-kvbm
8
spec:
9
  services:
10
    Frontend:
11
      componentType: frontend
12
      replicas: 1
13
      extraPodSpec:
14
        mainContainer:
15
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
16
    VllmDecodeWorker:
17
      envFromSecret: hf-token-secret
18
      componentType: worker
19
      replicas: 1
20
      resources:
21
        limits:
22
          gpu: "1"
23
      extraPodSpec:
24
        mainContainer:
25
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
26
          workingDir: /workspace/examples/backends/vllm
27
          command:
28
          - python3
29
          - -m
30
          - dynamo.vllm
31
          args:
32
            - --model
33
            - Qwen/Qwen3-8B
34
            - --disaggregation-mode
35
            - decode
36
            - --kv-transfer-config
37
            - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
38
            - --max-model-len
39
            - "32000"
40
            - --enforce-eager
41
    VllmPrefillWorker:
42
      envFromSecret: hf-token-secret
43
      componentType: worker
44
      replicas: 1
45
      resources:
46
        requests:
47
          gpu: "1"
48
          memory: "200Gi"
49
        limits:
50
          gpu: "1"
51
          memory: "250Gi"
52
      envs:
53
        - name: DYN_KVBM_CPU_CACHE_GB
54
          value: "100"
55
      extraPodSpec:
56
        mainContainer:
57
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
58
          workingDir: /workspace/examples/backends/vllm
59
          command:
60
          - python3
61
          - -m
62
          - dynamo.vllm
63
          args:
64
            - --model
65
            - Qwen/Qwen3-8B
66
            - --disaggregation-mode
67
            - prefill
68
            - --max-model-len
69
            - "32000"
70
            - --enforce-eager
71
            - --kv-transfer-config
72
            - '{"kv_connector":"PdConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"DynamoConnector","kv_connector_module_path":"kvbm.vllm_integration.connector","kv_role":"kv_both"},{"kv_connector":"NixlConnector","kv_role":"kv_both"}]},"kv_connector_module_path":"kvbm.vllm_integration.connector"}'

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
4
apiVersion: nvidia.com/v1alpha1
5
kind: DynamoGraphDeployment
6
metadata:
7
  name: vllm-disagg-kvbm-tp2
8
spec:
9
  services:
10
    Frontend:
11
      componentType: frontend
12
      replicas: 1
13
      extraPodSpec:
14
        mainContainer:
15
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
16
    VllmDecodeWorker:
17
      envFromSecret: hf-token-secret
18
      componentType: worker
19
      replicas: 1
20
      resources:
21
        requests:
22
          gpu: "2"
23
        limits:
24
          gpu: "2"
25
      extraPodSpec:
26
        mainContainer:
27
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
28
          workingDir: /workspace/examples/backends/vllm
29
          command:
30
          - python3
31
          - -m
32
          - dynamo.vllm
33
          args:
34
            - --model
35
            - Qwen/Qwen3-8B
36
            - --disaggregation-mode
37
            - decode
38
            - --kv-transfer-config
39
            - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
40
            - --gpu-memory-utilization
41
            - "0.23"
42
            - --max-model-len
43
            - "32000"
44
            - --enforce-eager
45
            - --tensor-parallel-size
46
            - "2"
47
    VllmPrefillWorker:
48
      envFromSecret: hf-token-secret
49
      componentType: worker
50
      replicas: 1
51
      resources:
52
        requests:
53
          gpu: "2"
54
          memory: "200Gi"
55
        limits:
56
          gpu: "2"
57
          memory: "250Gi"
58
      envs:
59
        - name: DYN_KVBM_CPU_CACHE_GB
60
          value: "100"
61
      extraPodSpec:
62
        mainContainer:
63
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
64
          workingDir: /workspace/examples/backends/vllm
65
          command:
66
          - python3
67
          - -m
68
          - dynamo.vllm
69
          args:
70
            - --model
71
            - Qwen/Qwen3-8B
72
            - --disaggregation-mode
73
            - prefill
74
            - --gpu-memory-utilization
75
            - "0.23"
76
            - --max-model-len
77
            - "32000"
78
            - --enforce-eager
79
            - --kv-transfer-config
80
            - '{"kv_connector":"PdConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"DynamoConnector","kv_connector_module_path":"kvbm.vllm_integration.connector","kv_role":"kv_both"},{"kv_connector":"NixlConnector","kv_role":"kv_both"}]},"kv_connector_module_path":"kvbm.vllm_integration.connector"}'
81
            - --tensor-parallel-size
82
            - "2"

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
4
apiVersion: nvidia.com/v1alpha1
5
kind: DynamoGraphDeployment
6
metadata:
7
  name: vllm-disagg-kvbm-2p2d
8
spec:
9
  services:
10
    Frontend:
11
      componentType: frontend
12
      replicas: 1
13
      extraPodSpec:
14
        mainContainer:
15
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
16
    VllmDecodeWorker:
17
      envFromSecret: hf-token-secret
18
      componentType: worker
19
      replicas: 2
20
      resources:
21
        limits:
22
          gpu: "1"
23
      extraPodSpec:
24
        mainContainer:
25
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
26
          workingDir: /workspace/examples/backends/vllm
27
          command:
28
          - python3
29
          - -m
30
          - dynamo.vllm
31
          args:
32
            - --model
33
            - Qwen/Qwen3-8B
34
            - --disaggregation-mode
35
            - decode
36
            - --kv-transfer-config
37
            - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
38
            - --max-model-len
39
            - "32000"
40
            - --enforce-eager
41
    VllmPrefillWorker:
42
      envFromSecret: hf-token-secret
43
      componentType: worker
44
      replicas: 2
45
      resources:
46
        requests:
47
          gpu: "1"
48
          memory: "200Gi"
49
        limits:
50
          gpu: "1"
51
          memory: "250Gi"
52
      envs:
53
        - name: DYN_KVBM_CPU_CACHE_GB
54
          value: "100"
55
      extraPodSpec:
56
        mainContainer:
57
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
58
          workingDir: /workspace/examples/backends/vllm
59
          command:
60
          - python3
61
          - -m
62
          - dynamo.vllm
63
          args:
64
            - --model
65
            - Qwen/Qwen3-8B
66
            - --disaggregation-mode
67
            - prefill
68
            - --max-model-len
69
            - "32000"
70
            - --enforce-eager
71
            - --kv-transfer-config
72
            - '{"kv_connector":"PdConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"DynamoConnector","kv_connector_module_path":"kvbm.vllm_integration.connector","kv_role":"kv_both"},{"kv_connector":"NixlConnector","kv_role":"kv_both"}]},"kv_connector_module_path":"kvbm.vllm_integration.connector"}'

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
4
# Disaggregated vLLM deployment with OpenTelemetry tracing enabled.
5
# Base deployment: disagg.yaml
6
# See docs/observability/tracing.md for setup instructions.
7
8
apiVersion: nvidia.com/v1alpha1
9
kind: DynamoGraphDeployment
10
metadata:
11
  name: vllm-disagg-tracing
12
spec:
13
  envs:
14
    - name: DYN_LOGGING_JSONL
15
      value: "true"
16
    - name: OTEL_EXPORT_ENABLED
17
      value: "true"
18
    - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
19
      value: "http://tempo.observability.svc.cluster.local:4317"
20
  services:
21
    Frontend:
22
      componentType: frontend
23
      replicas: 1
24
      extraPodSpec:
25
        mainContainer:
26
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
27
          env:
28
            - name: OTEL_SERVICE_NAME
29
              value: "dynamo-frontend"
30
    VllmDecodeWorker:
31
      envFromSecret: hf-token-secret
32
      componentType: worker
33
      subComponentType: decode
34
      replicas: 1
35
      resources:
36
        limits:
37
          gpu: "1"
38
        requests:
39
          custom:
40
            # Increase this value for larger models
41
            ephemeral-storage: "2Gi"
42
      extraPodSpec:
43
        mainContainer:
44
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
45
          workingDir: /workspace/examples/backends/vllm
46
          command:
47
          - python3
48
          - -m
49
          - dynamo.vllm
50
          args:
51
            - --model
52
            - Qwen/Qwen3-0.6B
53
            - --disaggregation-mode
54
            - decode
55
          env:
56
            - name: OTEL_SERVICE_NAME
57
              value: "dynamo-worker-decode"
58
    VllmPrefillWorker:
59
      envFromSecret: hf-token-secret
60
      componentType: worker
61
      subComponentType: prefill
62
      replicas: 1
63
      resources:
64
        limits:
65
          gpu: "1"
66
        requests:
67
          custom:
68
            # Increase this value for larger models
69
            ephemeral-storage: "2Gi"
70
      extraPodSpec:
71
        mainContainer:
72
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
73
          workingDir: /workspace/examples/backends/vllm
74
          command:
75
          - python3
76
          - -m
77
          - dynamo.vllm
78
          args:
79
            - --model
80
            - Qwen/Qwen3-0.6B
81
            - --disaggregation-mode
82
            - prefill
83
            - --kv-transfer-config
84
            - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
85
          env:
86
            - name: OTEL_SERVICE_NAME
87
              value: "dynamo-worker-prefill"

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
4
apiVersion: nvidia.com/v1alpha1
5
kind: DynamoGraphDeployment
6
metadata:
7
  name: vllm-disagg
8
spec:
9
  services:
10
    Frontend:
11
      componentType: frontend
12
      replicas: 1
13
      extraPodSpec:
14
        mainContainer:
15
          image: my-registry/vllm-runtime:my-tag
16
          workingDir: /workspace/examples/backends/vllm
17
          command:
18
          - python3
19
          - -m
20
          - dynamo.frontend
21
          args:
22
            - --http-port
23
            - "8000"
24
    decode:
25
      envFromSecret: hf-token-secret
26
      componentType: worker
27
      replicas: 1
28
      multinode:
29
        nodeCount: 2
30
      resources:
31
        limits:
32
          gpu: "1"
33
      extraPodSpec:
34
        mainContainer:
35
          image: my-registry/vllm-runtime:my-tag
36
          workingDir: /workspace/examples/backends/vllm
37
          command:
38
          - python3
39
          - -m
40
          - dynamo.vllm
41
          args:
42
          - --model
43
          - Qwen/Qwen3-0.6B
44
          - --tensor-parallel-size
45
          - "2"
46
          - --disaggregation-mode
47
          - decode
48
          - --kv-transfer-config
49
          - '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id":
50
            "vllm-disagg-decode-engine-0abc123"}'
51
    prefill:
52
      envFromSecret: hf-token-secret
53
      componentType: worker
54
      replicas: 1
55
      multinode:
56
        nodeCount: 2
57
      resources:
58
        limits:
59
          gpu: "1"
60
      extraPodSpec:
61
        mainContainer:
62
          image: my-registry/vllm-runtime:my-tag
63
          workingDir: /workspace/examples/backends/vllm
64
          command:
65
          - python3
66
          - -m
67
          - dynamo.vllm
68
          args:
69
          - --model
70
          - Qwen/Qwen3-0.6B
71
          - --tensor-parallel-size
72
          - "2"
73
          - --disaggregation-mode
74
          - prefill
75
          - --kv-transfer-config
76
          - '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id":
77
            "vllm-disagg-prefill-engine-0abc123"}'