vLLM Deployment Templates

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
4
apiVersion: nvidia.com/v1beta1
5
kind: DynamoGraphDeployment
6
metadata:
7
  name: vllm-agg
8
spec:
9
  components:
10
  - name: Frontend
11
    podTemplate:
12
      spec:
13
        containers:
14
        - envFrom:
15
          - secretRef:
16
              name: hf-token-secret
17
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
18
          name: main
19
    replicas: 1
20
    type: frontend
21
  - name: VllmDecodeWorker
22
    podTemplate:
23
      spec:
24
        containers:
25
        - args:
26
          - --model
27
          - Qwen/Qwen3-0.6B
28
          command:
29
          - python3
30
          - -m
31
          - dynamo.vllm
32
          envFrom:
33
          - secretRef:
34
              name: hf-token-secret
35
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
36
          name: main
37
          resources:
38
            limits:
39
              nvidia.com/gpu: "1"
40
            requests:
41
              # Increase this value for larger models.
42
              ephemeral-storage: 2Gi
43
          workingDir: /workspace/examples/backends/vllm
44
    replicas: 1
45
    type: worker

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
4
apiVersion: nvidia.com/v1beta1
5
kind: DynamoGraphDeployment
6
metadata:
7
  name: vllm-agg-router
8
spec:
9
  components:
10
  - name: Frontend
11
    podTemplate:
12
      spec:
13
        containers:
14
        - env:
15
          - name: DYN_ROUTER_MODE
16
            value: kv
17
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
18
          name: main
19
    replicas: 1
20
    type: frontend
21
  - name: VllmDecodeWorker
22
    podTemplate:
23
      spec:
24
        containers:
25
        - args:
26
          - --model
27
          - Qwen/Qwen3-0.6B
28
          - --kv-events-config
29
          - '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}'
30
          command:
31
          - python3
32
          - -m
33
          - dynamo.vllm
34
          envFrom:
35
          - secretRef:
36
              name: hf-token-secret
37
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
38
          name: main
39
          resources:
40
            limits:
41
              nvidia.com/gpu: "1"
42
            requests:
43
              # Increase this value for larger models.
44
              ephemeral-storage: 2Gi
45
          workingDir: /workspace/examples/backends/vllm
46
    replicas: 2
47
    type: worker

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
4
## This example demonstrates KV-aware routing with the --no-kv-events flag.
5
## Instead of receiving KV events from workers, the router predicts cache state
6
## locally based on routing decisions with TTL-based expiration and pruning.
7
## Note: This mode does not require NATS during Dynamo platform deployment.
8
apiVersion: nvidia.com/v1beta1
9
kind: DynamoGraphDeployment
10
metadata:
11
  name: vllm-agg-router-kv-approx
12
spec:
13
  components:
14
  - name: Frontend
15
    podTemplate:
16
      spec:
17
        containers:
18
        - args:
19
          - -m
20
          - dynamo.frontend
21
          - --router-mode
22
          - kv
23
          - --no-kv-events
24
          command:
25
          - python3
26
          env:
27
          - name: DYN_ROUTER_MODE
28
            value: kv
29
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
30
          name: main
31
    replicas: 1
32
    type: frontend
33
  - name: VllmDecodeWorker
34
    podTemplate:
35
      spec:
36
        containers:
37
        - args:
38
          - --model
39
          - Qwen/Qwen3-0.6B
40
          - --kv-events-config
41
          - '{"enable_kv_cache_events": false}'
42
          command:
43
          - python3
44
          - -m
45
          - dynamo.vllm
46
          envFrom:
47
          - secretRef:
48
              name: hf-token-secret
49
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
50
          name: main
51
          resources:
52
            limits:
53
              nvidia.com/gpu: "1"
54
          workingDir: /workspace/examples/backends/vllm
55
    replicas: 2
56
    type: worker

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
4
apiVersion: nvidia.com/v1beta1
5
kind: DynamoGraphDeployment
6
metadata:
7
  name: vllm-agg-kvbm
8
spec:
9
  components:
10
  - name: Frontend
11
    podTemplate:
12
      spec:
13
        containers:
14
        - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
15
          name: main
16
    replicas: 1
17
    type: frontend
18
  - name: VllmDecodeWorker
19
    podTemplate:
20
      spec:
21
        containers:
22
        - args:
23
          - --model
24
          - Qwen/Qwen3-8B
25
          - --max-model-len
26
          - "32000"
27
          - --enforce-eager
28
          - --kv-transfer-config
29
          - '{"kv_connector":"DynamoConnector","kv_connector_module_path":"kvbm.vllm_integration.connector","kv_role":"kv_both"}'
30
          command:
31
          - python3
32
          - -m
33
          - dynamo.vllm
34
          env:
35
          - name: DYN_KVBM_CPU_CACHE_GB
36
            value: "100"
37
          envFrom:
38
          - secretRef:
39
              name: hf-token-secret
40
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
41
          name: main
42
          resources:
43
            limits:
44
              memory: 250Gi
45
              nvidia.com/gpu: "1"
46
            requests:
47
              memory: 200Gi
48
              nvidia.com/gpu: "1"
49
          workingDir: /workspace/examples/backends/vllm
50
    replicas: 1
51
    type: worker

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
#
4
# GPU Memory Service (GMS) sidecar example.
5
#
6
# The operator injects a GMS sidecar init container that provides shared GPU
7
# memory access via DRA (Dynamic Resource Allocation). The sidecar runs two GMS
8
# processes per GPU (weights + kv_cache) and communicates with the main container
9
# over UDS sockets on a shared emptyDir volume.
10
#
11
# Requires Kubernetes 1.34+ with DRA v1 enabled and the NVIDIA GPU DRA driver installed.
12
13
apiVersion: nvidia.com/v1beta1
14
kind: DynamoGraphDeployment
15
metadata:
16
  name: vllm-agg-gms
17
spec:
18
  components:
19
  - name: Frontend
20
    podTemplate:
21
      spec:
22
        containers:
23
        - envFrom:
24
          - secretRef:
25
              name: hf-token-secret
26
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
27
          name: main
28
    replicas: 1
29
    type: frontend
30
  - experimental:
31
      gpuMemoryService: {}
32
    name: VllmWorker
33
    podTemplate:
34
      spec:
35
        containers:
36
        - args:
37
          - --model
38
          - Qwen/Qwen3-0.6B
39
          - --load-format
40
          - gms
41
          command:
42
          - python3
43
          - -m
44
          - dynamo.vllm
45
          envFrom:
46
          - secretRef:
47
              name: hf-token-secret
48
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
49
          name: main
50
          resources:
51
            limits:
52
              nvidia.com/gpu: "1"
53
            requests:
54
              ephemeral-storage: 2Gi
55
          workingDir: /workspace/examples/backends/vllm
56
    replicas: 1
57
    type: worker

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
#
4
# Active-passive GPU failover example.
5
#
6
# The operator clones the worker's main container into two engine containers
7
# (engine-0 active, engine-1 standby) sharing GPUs via DRA and the GMS sidecar.
8
# When the active engine fails, the standby acquires the flock and takes over.
9
#
10
# Requires:
11
#   - experimental.gpuMemoryService: {} (GMS sidecar + DRA)
12
#   - nvidia.com/dynamo-kube-discovery-mode: container (per-container K8s discovery)
13
#   - Kubernetes 1.34+ with DRA v1 enabled and the NVIDIA GPU DRA driver installed
14
15
apiVersion: nvidia.com/v1beta1
16
kind: DynamoGraphDeployment
17
metadata:
18
  annotations:
19
    nvidia.com/dynamo-kube-discovery-mode: container
20
  name: vllm-agg-failover
21
spec:
22
  components:
23
  - name: Frontend
24
    podTemplate:
25
      spec:
26
        containers:
27
        - envFrom:
28
          - secretRef:
29
              name: hf-token-secret
30
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
31
          name: main
32
    replicas: 1
33
    type: frontend
34
  - experimental:
35
      failover: {}
36
      gpuMemoryService: {}
37
    name: VllmWorker
38
    podTemplate:
39
      spec:
40
        containers:
41
        - args:
42
          - --model
43
          - Qwen/Qwen3-0.6B
44
          - --tensor-parallel-size
45
          - "2"
46
          - --load-format
47
          - gms
48
          command:
49
          - python3
50
          - -m
51
          - dynamo.vllm
52
          envFrom:
53
          - secretRef:
54
              name: hf-token-secret
55
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
56
          name: main
57
          resources:
58
            limits:
59
              nvidia.com/gpu: "2"
60
            requests:
61
              ephemeral-storage: 2Gi
62
          workingDir: /workspace/examples/backends/vllm
63
    replicas: 1
64
    type: worker

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
4
# Aggregated vLLM deployment with OpenTelemetry tracing enabled.
5
# Base deployment: agg.yaml
6
# See docs/observability/tracing.md for setup instructions.
7
8
apiVersion: nvidia.com/v1beta1
9
kind: DynamoGraphDeployment
10
metadata:
11
  name: vllm-agg-tracing
12
spec:
13
  components:
14
  - name: Frontend
15
    podTemplate:
16
      spec:
17
        containers:
18
        - env:
19
          - name: OTEL_SERVICE_NAME
20
            value: dynamo-frontend
21
          envFrom:
22
          - secretRef:
23
              name: hf-token-secret
24
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
25
          name: main
26
    replicas: 1
27
    type: frontend
28
  - name: VllmDecodeWorker
29
    podTemplate:
30
      spec:
31
        containers:
32
        - args:
33
          - --model
34
          - Qwen/Qwen3-0.6B
35
          command:
36
          - python3
37
          - -m
38
          - dynamo.vllm
39
          env:
40
          - name: OTEL_SERVICE_NAME
41
            value: dynamo-worker-vllm
42
          envFrom:
43
          - secretRef:
44
              name: hf-token-secret
45
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
46
          name: main
47
          resources:
48
            limits:
49
              nvidia.com/gpu: "1"
50
            requests:
51
              # Increase this value for larger models.
52
              ephemeral-storage: 2Gi
53
          workingDir: /workspace/examples/backends/vllm
54
    replicas: 1
55
    type: worker
56
  env:
57
  - name: DYN_LOGGING_JSONL
58
    value: "true"
59
  - name: OTEL_EXPORT_ENABLED
60
    value: "true"
61
  - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
62
    value: http://tempo.observability.svc.cluster.local:4317

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
4
apiVersion: resource.k8s.io/v1
5
kind: ResourceClaimTemplate
6
metadata:
7
  name: gpu-template
8
spec:
9
  spec:
10
    devices:
11
      requests:
12
        - name: gpu
13
          exactly:
14
            deviceClassName: gpu.intel.com
15
            count: 1
16
---
17
apiVersion: nvidia.com/v1beta1
18
kind: DynamoGraphDeployment
19
metadata:
20
  name: vllm-agg-xpu-dra
21
spec:
22
  components:
23
  - name: Frontend
24
    podTemplate:
25
      spec:
26
        containers:
27
        - envFrom:
28
          - secretRef:
29
              name: hf-token-secret
30
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
31
          name: main
32
    replicas: 1
33
    type: frontend
34
  - name: VllmDecodeWorker
35
    podTemplate:
36
      spec:
37
        containers:
38
        - args:
39
          - --model
40
          - Qwen/Qwen3-0.6B
41
          - --block-size
42
          - "64"
43
          command:
44
          - python3
45
          - -m
46
          - dynamo.vllm
47
          env:
48
          - name: VLLM_TARGET_DEVICE
49
            value: xpu
50
          envFrom:
51
          - secretRef:
52
              name: hf-token-secret
53
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime-xpu:my-tag
54
          name: main
55
          resources:
56
            claims:
57
            - name: gpu
58
            requests:
59
              # Increase this value for larger models.
60
              ephemeral-storage: 2Gi
61
          workingDir: /workspace/examples/backends/vllm
62
        resourceClaims:
63
        - name: gpu
64
          resourceClaimTemplateName: gpu-template
65
        # NOTE: Uncomment if your environment requires specific group access
66
        # securityContext:
67
        #   runAsUser: 1000
68
        #   runAsGroup: 1000
69
        #   supplementalGroups:
70
        #     - 44   # render group
71
        #     - 991  # video group
72
    replicas: 1
73
    type: worker

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
4
apiVersion: nvidia.com/v1beta1
5
kind: DynamoGraphDeployment
6
metadata:
7
  name: vllm-disagg
8
spec:
9
  components:
10
  - name: Frontend
11
    podTemplate:
12
      spec:
13
        containers:
14
        - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
15
          name: main
16
    replicas: 1
17
    type: frontend
18
  - name: VllmDecodeWorker
19
    podTemplate:
20
      spec:
21
        containers:
22
        - args:
23
          - --model
24
          - Qwen/Qwen3-0.6B
25
          - --disaggregation-mode
26
          - decode
27
          - --kv-transfer-config
28
          - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
29
          command:
30
          - python3
31
          - -m
32
          - dynamo.vllm
33
          envFrom:
34
          - secretRef:
35
              name: hf-token-secret
36
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
37
          name: main
38
          resources:
39
            limits:
40
              nvidia.com/gpu: "1"
41
            requests:
42
              # Increase this value for larger models.
43
              ephemeral-storage: 2Gi
44
          workingDir: /workspace/examples/backends/vllm
45
    replicas: 1
46
    type: decode
47
  - name: VllmPrefillWorker
48
    podTemplate:
49
      spec:
50
        containers:
51
        - args:
52
          - --model
53
          - Qwen/Qwen3-0.6B
54
          - --disaggregation-mode
55
          - prefill
56
          - --kv-transfer-config
57
          - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
58
          command:
59
          - python3
60
          - -m
61
          - dynamo.vllm
62
          envFrom:
63
          - secretRef:
64
              name: hf-token-secret
65
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
66
          name: main
67
          resources:
68
            limits:
69
              nvidia.com/gpu: "1"
70
            requests:
71
              # Increase this value for larger models.
72
              ephemeral-storage: 2Gi
73
          workingDir: /workspace/examples/backends/vllm
74
    replicas: 1
75
    type: prefill

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
4
apiVersion: nvidia.com/v1beta1
5
kind: DynamoGraphDeployment
6
metadata:
7
  name: vllm-v1-disagg-router
8
spec:
9
  components:
10
  - name: Frontend
11
    podTemplate:
12
      spec:
13
        containers:
14
        - env:
15
          - name: DYN_ROUTER_MODE
16
            value: kv
17
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
18
          name: main
19
    replicas: 1
20
    type: frontend
21
  - name: VllmDecodeWorker
22
    podTemplate:
23
      spec:
24
        containers:
25
        - args:
26
          - --model
27
          - Qwen/Qwen3-0.6B
28
          - --disaggregation-mode
29
          - decode
30
          - --kv-transfer-config
31
          - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
32
          command:
33
          - python3
34
          - -m
35
          - dynamo.vllm
36
          envFrom:
37
          - secretRef:
38
              name: hf-token-secret
39
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
40
          name: main
41
          resources:
42
            limits:
43
              nvidia.com/gpu: "1"
44
            requests:
45
              # Increase this value for larger models.
46
              ephemeral-storage: 2Gi
47
          workingDir: /workspace/examples/backends/vllm
48
    replicas: 2
49
    type: decode
50
  - name: VllmPrefillWorker
51
    podTemplate:
52
      spec:
53
        containers:
54
        - args:
55
          - --model
56
          - Qwen/Qwen3-0.6B
57
          - --disaggregation-mode
58
          - prefill
59
          - --kv-transfer-config
60
          - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
61
          - --kv-events-config
62
          - '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}'
63
          command:
64
          - python3
65
          - -m
66
          - dynamo.vllm
67
          envFrom:
68
          - secretRef:
69
              name: hf-token-secret
70
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
71
          name: main
72
          resources:
73
            limits:
74
              nvidia.com/gpu: "1"
75
            requests:
76
              # Increase this value for larger models.
77
              ephemeral-storage: 2Gi
78
          workingDir: /workspace/examples/backends/vllm
79
    replicas: 2
80
    type: prefill

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
4
apiVersion: nvidia.com/v1beta1
5
kind: DynamoGraphDeployment
6
metadata:
7
  name: vllm-disagg-planner
8
spec:
9
  components:
10
  - name: Frontend
11
    podTemplate:
12
      spec:
13
        containers:
14
        - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
15
          name: main
16
    replicas: 1
17
    type: frontend
18
  - name: Planner
19
    podTemplate:
20
      spec:
21
        containers:
22
        - args:
23
          - --config
24
          - '{"environment": "kubernetes", "backend": "vllm", "optimization_target": "sla",
25
            "enable_throughput_scaling": true, "enable_load_scaling": true, "pre_deployment_sweeping_mode":
26
            "none", "throughput_adjustment_interval_seconds": 60, "load_adjustment_interval_seconds": 5}'
27
          command:
28
          - python3
29
          - -m
30
          - dynamo.planner
31
          # Planner image selection:
32
          #   Dynamo >= 1.1.0: use the dedicated planner image
33
          #     nvcr.io/nvidia/ai-dynamo/dynamo-planner:<version>
34
          #     (backend runtime images no longer ship planner runtime deps
35
          #     such as kubernetes_asyncio, pmdarima, prophet, aiconfigurator).
36
          #   Dynamo <  1.1.0: use the backend runtime image
37
          #     nvcr.io/nvidia/ai-dynamo/vllm-runtime:<version>.
38
          image: nvcr.io/nvidia/ai-dynamo/dynamo-planner:my-tag
39
          name: main
40
    replicas: 1
41
    type: planner
42
  - name: VllmDecodeWorker
43
    podTemplate:
44
      spec:
45
        containers:
46
        - args:
47
          - -m
48
          - dynamo.vllm
49
          - --model
50
          - Qwen/Qwen3-0.6B
51
          - --disaggregation-mode
52
          - decode
53
          - --kv-transfer-config
54
          - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
55
          command:
56
          - python3
57
          envFrom:
58
          - secretRef:
59
              name: hf-token-secret
60
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
61
          name: main
62
          resources:
63
            limits:
64
              nvidia.com/gpu: "1"
65
          workingDir: /workspace/examples/backends/vllm
66
    replicas: 1
67
    type: decode
68
  - name: VllmPrefillWorker
69
    podTemplate:
70
      spec:
71
        containers:
72
        - args:
73
          - -m
74
          - dynamo.vllm
75
          - --model
76
          - Qwen/Qwen3-0.6B
77
          - --disaggregation-mode
78
          - prefill
79
          - --kv-transfer-config
80
          - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
81
          command:
82
          - python3
83
          envFrom:
84
          - secretRef:
85
              name: hf-token-secret
86
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
87
          name: main
88
          resources:
89
            limits:
90
              nvidia.com/gpu: "1"
91
          workingDir: /workspace/examples/backends/vllm
92
    replicas: 1
93
    type: prefill

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
4
apiVersion: nvidia.com/v1beta1
5
kind: DynamoGraphDeployment
6
metadata:
7
  name: vllm-disagg-kvbm
8
spec:
9
  components:
10
  - name: Frontend
11
    podTemplate:
12
      spec:
13
        containers:
14
        - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
15
          name: main
16
    replicas: 1
17
    type: frontend
18
  - name: VllmDecodeWorker
19
    podTemplate:
20
      spec:
21
        containers:
22
        - args:
23
          - --model
24
          - Qwen/Qwen3-8B
25
          - --disaggregation-mode
26
          - decode
27
          - --disable-hybrid-kv-cache-manager
28
          - --kv-transfer-config
29
          - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
30
          - --max-model-len
31
          - "32000"
32
          - --enforce-eager
33
          command:
34
          - python3
35
          - -m
36
          - dynamo.vllm
37
          envFrom:
38
          - secretRef:
39
              name: hf-token-secret
40
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
41
          name: main
42
          resources:
43
            limits:
44
              nvidia.com/gpu: "1"
45
          workingDir: /workspace/examples/backends/vllm
46
    replicas: 1
47
    type: decode
48
  - name: VllmPrefillWorker
49
    podTemplate:
50
      spec:
51
        containers:
52
        - args:
53
          - --model
54
          - Qwen/Qwen3-8B
55
          - --disaggregation-mode
56
          - prefill
57
          - --disable-hybrid-kv-cache-manager
58
          - --max-model-len
59
          - "32000"
60
          - --enforce-eager
61
          - --kv-transfer-config
62
          - '{"kv_connector":"PdConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"DynamoConnector","kv_connector_module_path":"kvbm.vllm_integration.connector","kv_role":"kv_both"},{"kv_connector":"NixlConnector","kv_role":"kv_both"}]},"kv_connector_module_path":"kvbm.vllm_integration.connector"}'
63
          command:
64
          - python3
65
          - -m
66
          - dynamo.vllm
67
          env:
68
          - name: DYN_KVBM_CPU_CACHE_GB
69
            value: "100"
70
          envFrom:
71
          - secretRef:
72
              name: hf-token-secret
73
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
74
          name: main
75
          resources:
76
            limits:
77
              memory: 250Gi
78
              nvidia.com/gpu: "1"
79
            requests:
80
              memory: 200Gi
81
              nvidia.com/gpu: "1"
82
          workingDir: /workspace/examples/backends/vllm
83
    replicas: 1
84
    type: prefill

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
4
apiVersion: nvidia.com/v1beta1
5
kind: DynamoGraphDeployment
6
metadata:
7
  name: vllm-disagg-kvbm-tp2
8
spec:
9
  components:
10
  - name: Frontend
11
    podTemplate:
12
      spec:
13
        containers:
14
        - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
15
          name: main
16
    replicas: 1
17
    type: frontend
18
  - name: VllmDecodeWorker
19
    podTemplate:
20
      spec:
21
        containers:
22
        - args:
23
          - --model
24
          - Qwen/Qwen3-8B
25
          - --disaggregation-mode
26
          - decode
27
          - --disable-hybrid-kv-cache-manager
28
          - --kv-transfer-config
29
          - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
30
          - --gpu-memory-utilization
31
          - "0.23"
32
          - --max-model-len
33
          - "32000"
34
          - --enforce-eager
35
          - --tensor-parallel-size
36
          - "2"
37
          command:
38
          - python3
39
          - -m
40
          - dynamo.vllm
41
          envFrom:
42
          - secretRef:
43
              name: hf-token-secret
44
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
45
          name: main
46
          resources:
47
            limits:
48
              nvidia.com/gpu: "2"
49
            requests:
50
              nvidia.com/gpu: "2"
51
          workingDir: /workspace/examples/backends/vllm
52
    replicas: 1
53
    type: decode
54
  - name: VllmPrefillWorker
55
    podTemplate:
56
      spec:
57
        containers:
58
        - args:
59
          - --model
60
          - Qwen/Qwen3-8B
61
          - --disaggregation-mode
62
          - prefill
63
          - --disable-hybrid-kv-cache-manager
64
          - --gpu-memory-utilization
65
          - "0.23"
66
          - --max-model-len
67
          - "32000"
68
          - --enforce-eager
69
          - --kv-transfer-config
70
          - '{"kv_connector":"PdConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"DynamoConnector","kv_connector_module_path":"kvbm.vllm_integration.connector","kv_role":"kv_both"},{"kv_connector":"NixlConnector","kv_role":"kv_both"}]},"kv_connector_module_path":"kvbm.vllm_integration.connector"}'
71
          - --tensor-parallel-size
72
          - "2"
73
          command:
74
          - python3
75
          - -m
76
          - dynamo.vllm
77
          env:
78
          - name: DYN_KVBM_CPU_CACHE_GB
79
            value: "100"
80
          envFrom:
81
          - secretRef:
82
              name: hf-token-secret
83
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
84
          name: main
85
          resources:
86
            limits:
87
              memory: 250Gi
88
              nvidia.com/gpu: "2"
89
            requests:
90
              memory: 200Gi
91
              nvidia.com/gpu: "2"
92
          workingDir: /workspace/examples/backends/vllm
93
    replicas: 1
94
    type: prefill

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
4
apiVersion: nvidia.com/v1beta1
5
kind: DynamoGraphDeployment
6
metadata:
7
  name: vllm-disagg-kvbm-2p2d
8
spec:
9
  components:
10
  - name: Frontend
11
    podTemplate:
12
      spec:
13
        containers:
14
        - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
15
          name: main
16
    replicas: 1
17
    type: frontend
18
  - name: VllmDecodeWorker
19
    podTemplate:
20
      spec:
21
        containers:
22
        - args:
23
          - --model
24
          - Qwen/Qwen3-8B
25
          - --disaggregation-mode
26
          - decode
27
          - --disable-hybrid-kv-cache-manager
28
          - --kv-transfer-config
29
          - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
30
          - --max-model-len
31
          - "32000"
32
          - --enforce-eager
33
          command:
34
          - python3
35
          - -m
36
          - dynamo.vllm
37
          envFrom:
38
          - secretRef:
39
              name: hf-token-secret
40
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
41
          name: main
42
          resources:
43
            limits:
44
              nvidia.com/gpu: "1"
45
          workingDir: /workspace/examples/backends/vllm
46
    replicas: 2
47
    type: decode
48
  - name: VllmPrefillWorker
49
    podTemplate:
50
      spec:
51
        containers:
52
        - args:
53
          - --model
54
          - Qwen/Qwen3-8B
55
          - --disaggregation-mode
56
          - prefill
57
          - --disable-hybrid-kv-cache-manager
58
          - --max-model-len
59
          - "32000"
60
          - --enforce-eager
61
          - --kv-transfer-config
62
          - '{"kv_connector":"PdConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"DynamoConnector","kv_connector_module_path":"kvbm.vllm_integration.connector","kv_role":"kv_both"},{"kv_connector":"NixlConnector","kv_role":"kv_both"}]},"kv_connector_module_path":"kvbm.vllm_integration.connector"}'
63
          command:
64
          - python3
65
          - -m
66
          - dynamo.vllm
67
          env:
68
          - name: DYN_KVBM_CPU_CACHE_GB
69
            value: "100"
70
          envFrom:
71
          - secretRef:
72
              name: hf-token-secret
73
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
74
          name: main
75
          resources:
76
            limits:
77
              memory: 250Gi
78
              nvidia.com/gpu: "1"
79
            requests:
80
              memory: 200Gi
81
              nvidia.com/gpu: "1"
82
          workingDir: /workspace/examples/backends/vllm
83
    replicas: 2
84
    type: prefill

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
4
# Disaggregated vLLM deployment with OpenTelemetry tracing enabled.
5
# Base deployment: disagg.yaml
6
# See docs/observability/tracing.md for setup instructions.
7
8
apiVersion: nvidia.com/v1beta1
9
kind: DynamoGraphDeployment
10
metadata:
11
  name: vllm-disagg-tracing
12
spec:
13
  components:
14
  - name: Frontend
15
    podTemplate:
16
      spec:
17
        containers:
18
        - env:
19
          - name: OTEL_SERVICE_NAME
20
            value: dynamo-frontend
21
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
22
          name: main
23
    replicas: 1
24
    type: frontend
25
  - name: VllmDecodeWorker
26
    podTemplate:
27
      spec:
28
        containers:
29
        - args:
30
          - --model
31
          - Qwen/Qwen3-0.6B
32
          - --disaggregation-mode
33
          - decode
34
          - --kv-transfer-config
35
          - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
36
          command:
37
          - python3
38
          - -m
39
          - dynamo.vllm
40
          env:
41
          - name: OTEL_SERVICE_NAME
42
            value: dynamo-worker-decode
43
          envFrom:
44
          - secretRef:
45
              name: hf-token-secret
46
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
47
          name: main
48
          resources:
49
            limits:
50
              nvidia.com/gpu: "1"
51
            requests:
52
              # Increase this value for larger models.
53
              ephemeral-storage: 2Gi
54
          workingDir: /workspace/examples/backends/vllm
55
    replicas: 1
56
    type: decode
57
  - name: VllmPrefillWorker
58
    podTemplate:
59
      spec:
60
        containers:
61
        - args:
62
          - --model
63
          - Qwen/Qwen3-0.6B
64
          - --disaggregation-mode
65
          - prefill
66
          - --kv-transfer-config
67
          - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
68
          command:
69
          - python3
70
          - -m
71
          - dynamo.vllm
72
          env:
73
          - name: OTEL_SERVICE_NAME
74
            value: dynamo-worker-prefill
75
          envFrom:
76
          - secretRef:
77
              name: hf-token-secret
78
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
79
          name: main
80
          resources:
81
            limits:
82
              nvidia.com/gpu: "1"
83
            requests:
84
              # Increase this value for larger models.
85
              ephemeral-storage: 2Gi
86
          workingDir: /workspace/examples/backends/vllm
87
    replicas: 1
88
    type: prefill
89
  env:
90
  - name: DYN_LOGGING_JSONL
91
    value: "true"
92
  - name: OTEL_EXPORT_ENABLED
93
    value: "true"
94
  - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
95
    value: http://tempo.observability.svc.cluster.local:4317