vLLM Deployment Templates

# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
apiVersion: nvidia.com/v1beta1
kind: DynamoGraphDeployment
metadata:
  name: vllm-agg
spec:
  components:
  - name: Frontend
    podTemplate:
      spec:
        containers:
        - envFrom:
          - secretRef:
              name: hf-token-secret
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
    replicas: 1
    type: frontend
  - name: worker
    podTemplate:
      spec:
        containers:
        - args:
          - --model
          - Qwen/Qwen3-0.6B
          command:
          - python3
          - -m
          - dynamo.vllm
          envFrom:
          - secretRef:
              name: hf-token-secret
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
          resources:
            limits:
              nvidia.com/gpu: "1"
            requests:
              # Increase this value for larger models.
              ephemeral-storage: 2Gi
          workingDir: /workspace/examples/backends/vllm
    replicas: 1
    type: worker

# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
apiVersion: nvidia.com/v1beta1
kind: DynamoGraphDeployment
metadata:
  name: vllm-agg-router
spec:
  components:
  - name: Frontend
    podTemplate:
      spec:
        containers:
        - env:
          - name: DYN_ROUTER_MODE
            value: kv
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
    replicas: 1
    type: frontend
  - name: worker
    podTemplate:
      spec:
        containers:
        - args:
          - --model
          - Qwen/Qwen3-0.6B
          - --kv-events-config
          - '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}'
          command:
          - python3
          - -m
          - dynamo.vllm
          envFrom:
          - secretRef:
              name: hf-token-secret
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
          resources:
            limits:
              nvidia.com/gpu: "1"
            requests:
              # Increase this value for larger models.
              ephemeral-storage: 2Gi
          workingDir: /workspace/examples/backends/vllm
    replicas: 2
    type: worker

# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
## This example demonstrates KV-aware routing with the --no-kv-events flag.
## Instead of receiving KV events from workers, the router predicts cache state
## locally based on routing decisions with TTL-based expiration and pruning.
## Note: This mode does not require NATS during Dynamo platform deployment.
apiVersion: nvidia.com/v1beta1
kind: DynamoGraphDeployment
metadata:
  name: vllm-agg-router-kv-approx
spec:
  components:
  - name: Frontend
    podTemplate:
      spec:
        containers:
        - args:
          - -m
          - dynamo.frontend
          - --router-mode
          - kv
          - --no-kv-events
          command:
          - python3
          env:
          - name: DYN_ROUTER_MODE
            value: kv
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
    replicas: 1
    type: frontend
  - name: worker
    podTemplate:
      spec:
        containers:
        - args:
          - --model
          - Qwen/Qwen3-0.6B
          - --kv-events-config
          - '{"enable_kv_cache_events": false}'
          command:
          - python3
          - -m
          - dynamo.vllm
          envFrom:
          - secretRef:
              name: hf-token-secret
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
          resources:
            limits:
              nvidia.com/gpu: "1"
          workingDir: /workspace/examples/backends/vllm
    replicas: 2
    type: worker

# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
apiVersion: nvidia.com/v1beta1
kind: DynamoGraphDeployment
metadata:
  name: vllm-agg-kvbm
spec:
  components:
  - name: Frontend
    podTemplate:
      spec:
        containers:
        - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
    replicas: 1
    type: frontend
  - name: worker
    podTemplate:
      spec:
        containers:
        - args:
          - --model
          - Qwen/Qwen3-8B
          - --max-model-len
          - "32000"
          - --enforce-eager
          - --kv-transfer-config
          - '{"kv_connector":"DynamoConnector","kv_connector_module_path":"kvbm.vllm_integration.connector","kv_role":"kv_both"}'
          command:
          - python3
          - -m
          - dynamo.vllm
          env:
          - name: DYN_KVBM_CPU_CACHE_GB
            value: "100"
          envFrom:
          - secretRef:
              name: hf-token-secret
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
          resources:
            limits:
              memory: 250Gi
              nvidia.com/gpu: "1"
            requests:
              memory: 200Gi
              nvidia.com/gpu: "1"
          workingDir: /workspace/examples/backends/vllm
    replicas: 1
    type: worker

# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# GPU Memory Service (GMS) sidecar example.
#
# The operator injects a GMS sidecar init container that provides shared GPU
# memory access via DRA (Dynamic Resource Allocation). The sidecar runs two GMS
# processes per GPU (weights + kv_cache) and communicates with the main container
# over UDS sockets on a shared emptyDir volume.
#
# Requires Kubernetes 1.34+ with DRA v1 enabled and the NVIDIA GPU DRA driver installed.
apiVersion: nvidia.com/v1beta1
kind: DynamoGraphDeployment
metadata:
  name: vllm-agg-gms
spec:
  components:
  - name: Frontend
    podTemplate:
      spec:
        containers:
        - envFrom:
          - secretRef:
              name: hf-token-secret
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
    replicas: 1
    type: frontend
  - experimental:
      gpuMemoryService: {}
    name: worker
    podTemplate:
      spec:
        containers:
        - args:
          - --model
          - Qwen/Qwen3-0.6B
          - --load-format
          - gms
          command:
          - python3
          - -m
          - dynamo.vllm
          envFrom:
          - secretRef:
              name: hf-token-secret
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
          resources:
            limits:
              nvidia.com/gpu: "1"
            requests:
              ephemeral-storage: 2Gi
          workingDir: /workspace/examples/backends/vllm
    replicas: 1
    type: worker

# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Active-passive GPU failover example.
#
# The operator clones the worker's main container into two engine containers
# (engine-0 active, engine-1 standby) sharing GPUs via DRA and the GMS sidecar.
# When the active engine fails, the standby acquires the flock and takes over.
#
# Requires:
#   - experimental.gpuMemoryService: {} (GMS sidecar + DRA)
#   - nvidia.com/dynamo-kube-discovery-mode: container (per-container K8s discovery)
#   - Kubernetes 1.34+ with DRA v1 enabled and the NVIDIA GPU DRA driver installed
apiVersion: nvidia.com/v1beta1
kind: DynamoGraphDeployment
metadata:
  annotations:
    nvidia.com/dynamo-kube-discovery-mode: container
  name: vllm-agg-failover
spec:
  components:
  - name: Frontend
    podTemplate:
      spec:
        containers:
        - envFrom:
          - secretRef:
              name: hf-token-secret
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
    replicas: 1
    type: frontend
  - experimental:
      failover: {}
      gpuMemoryService: {}
    name: worker
    podTemplate:
      spec:
        containers:
        - args:
          - --model
          - Qwen/Qwen3-0.6B
          - --tensor-parallel-size
          - "2"
          - --load-format
          - gms
          command:
          - python3
          - -m
          - dynamo.vllm
          envFrom:
          - secretRef:
              name: hf-token-secret
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
          resources:
            limits:
              nvidia.com/gpu: "2"
            requests:
              ephemeral-storage: 2Gi
          workingDir: /workspace/examples/backends/vllm
    replicas: 1
    type: worker

# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Aggregated vLLM deployment with OpenTelemetry tracing enabled.
# Base deployment: agg.yaml
# See docs/observability/tracing.md for setup instructions.
apiVersion: nvidia.com/v1beta1
kind: DynamoGraphDeployment
metadata:
  name: vllm-agg-tracing
spec:
  components:
  - name: Frontend
    podTemplate:
      spec:
        containers:
        - env:
          - name: OTEL_SERVICE_NAME
            value: dynamo-frontend
          envFrom:
          - secretRef:
              name: hf-token-secret
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
    replicas: 1
    type: frontend
  - name: worker
    podTemplate:
      spec:
        containers:
        - args:
          - --model
          - Qwen/Qwen3-0.6B
          command:
          - python3
          - -m
          - dynamo.vllm
          env:
          - name: OTEL_SERVICE_NAME
            value: dynamo-worker-vllm
          envFrom:
          - secretRef:
              name: hf-token-secret
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
          resources:
            limits:
              nvidia.com/gpu: "1"
            requests:
              # Increase this value for larger models.
              ephemeral-storage: 2Gi
          workingDir: /workspace/examples/backends/vllm
    replicas: 1
    type: worker
  env:
  - name: DYN_LOGGING_JSONL
    value: "true"
  - name: OTEL_EXPORT_ENABLED
    value: "true"
  - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
    value: http://tempo.observability.svc.cluster.local:4317

# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
apiVersion: resource.k8s.io/v1
kind: ResourceClaimTemplate
metadata:
  name: gpu-template
spec:
  spec:
    devices:
      requests:
        - name: gpu
          exactly:
            deviceClassName: gpu.intel.com
            count: 1
---
apiVersion: nvidia.com/v1beta1
kind: DynamoGraphDeployment
metadata:
  name: vllm-agg-xpu-dra
spec:
  components:
  - name: Frontend
    podTemplate:
      spec:
        containers:
        - envFrom:
          - secretRef:
              name: hf-token-secret
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
    replicas: 1
    type: frontend
  - name: worker
    podTemplate:
      spec:
        containers:
        - args:
          - --model
          - Qwen/Qwen3-0.6B
          - --block-size
          - "64"
          command:
          - python3
          - -m
          - dynamo.vllm
          env:
          - name: VLLM_TARGET_DEVICE
            value: xpu
          envFrom:
          - secretRef:
              name: hf-token-secret
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime-xpu:my-tag
          name: main
          resources:
            claims:
            - name: gpu
            requests:
              # Increase this value for larger models.
              ephemeral-storage: 2Gi
          workingDir: /workspace/examples/backends/vllm
        resourceClaims:
        - name: gpu
          resourceClaimTemplateName: gpu-template
        # NOTE: Uncomment if your environment requires specific group access
        # securityContext:
        #   runAsUser: 1000
        #   runAsGroup: 1000
        #   supplementalGroups:
        #     - 44   # render group
        #     - 991  # video group
    replicas: 1
    type: worker

# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
apiVersion: nvidia.com/v1beta1
kind: DynamoGraphDeployment
metadata:
  name: vllm-disagg
spec:
  components:
  - name: Frontend
    podTemplate:
      spec:
        containers:
        - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
    replicas: 1
    type: frontend
  - name: decode
    podTemplate:
      spec:
        containers:
        - args:
          - --model
          - Qwen/Qwen3-0.6B
          - --disaggregation-mode
          - decode
          - --kv-transfer-config
          - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
          command:
          - python3
          - -m
          - dynamo.vllm
          envFrom:
          - secretRef:
              name: hf-token-secret
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
          resources:
            limits:
              nvidia.com/gpu: "1"
            requests:
              # Increase this value for larger models.
              ephemeral-storage: 2Gi
          workingDir: /workspace/examples/backends/vllm
    replicas: 1
    type: decode
  - name: prefill
    podTemplate:
      spec:
        containers:
        - args:
          - --model
          - Qwen/Qwen3-0.6B
          - --disaggregation-mode
          - prefill
          - --kv-transfer-config
          - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
          command:
          - python3
          - -m
          - dynamo.vllm
          envFrom:
          - secretRef:
              name: hf-token-secret
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
          resources:
            limits:
              nvidia.com/gpu: "1"
            requests:
              # Increase this value for larger models.
              ephemeral-storage: 2Gi
          workingDir: /workspace/examples/backends/vllm
    replicas: 1
    type: prefill

# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
apiVersion: nvidia.com/v1beta1
kind: DynamoGraphDeployment
metadata:
  name: vllm-v1-disagg-router
spec:
  components:
  - name: Frontend
    podTemplate:
      spec:
        containers:
        - env:
          - name: DYN_ROUTER_MODE
            value: kv
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
    replicas: 1
    type: frontend
  - name: decode
    podTemplate:
      spec:
        containers:
        - args:
          - --model
          - Qwen/Qwen3-0.6B
          - --disaggregation-mode
          - decode
          - --kv-transfer-config
          - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
          command:
          - python3
          - -m
          - dynamo.vllm
          envFrom:
          - secretRef:
              name: hf-token-secret
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
          resources:
            limits:
              nvidia.com/gpu: "1"
            requests:
              # Increase this value for larger models.
              ephemeral-storage: 2Gi
          workingDir: /workspace/examples/backends/vllm
    replicas: 2
    type: decode
  - name: prefill
    podTemplate:
      spec:
        containers:
        - args:
          - --model
          - Qwen/Qwen3-0.6B
          - --disaggregation-mode
          - prefill
          - --kv-transfer-config
          - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
          - --kv-events-config
          - '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}'
          command:
          - python3
          - -m
          - dynamo.vllm
          envFrom:
          - secretRef:
              name: hf-token-secret
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
          resources:
            limits:
              nvidia.com/gpu: "1"
            requests:
              # Increase this value for larger models.
              ephemeral-storage: 2Gi
          workingDir: /workspace/examples/backends/vllm
    replicas: 2
    type: prefill

# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
apiVersion: nvidia.com/v1beta1
kind: DynamoGraphDeployment
metadata:
  name: vllm-disagg-planner
spec:
  components:
  - name: Frontend
    podTemplate:
      spec:
        containers:
        - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
    replicas: 1
    type: frontend
  - name: Planner
    podTemplate:
      spec:
        containers:
        - args:
          - --config
          - '{"environment": "kubernetes", "backend": "vllm", "optimization_target": "sla",
            "enable_throughput_scaling": true, "enable_load_scaling": true, "pre_deployment_sweeping_mode":
            "none", "throughput_adjustment_interval_seconds": 60, "load_adjustment_interval_seconds": 5}'
          command:
          - python3
          - -m
          - dynamo.planner
          # Planner image selection:
          #   Dynamo >= 1.1.0: use the dedicated planner image
          #     nvcr.io/nvidia/ai-dynamo/dynamo-planner:<version>
          #     (backend runtime images no longer ship planner runtime deps
          #     such as kubernetes_asyncio, pmdarima, prophet, aiconfigurator).
          #   Dynamo <  1.1.0: use the backend runtime image
          #     nvcr.io/nvidia/ai-dynamo/vllm-runtime:<version>.
          image: nvcr.io/nvidia/ai-dynamo/dynamo-planner:my-tag
          name: main
    replicas: 1
    type: planner
  - name: decode
    podTemplate:
      spec:
        containers:
        - args:
          - -m
          - dynamo.vllm
          - --model
          - Qwen/Qwen3-0.6B
          - --disaggregation-mode
          - decode
          - --kv-transfer-config
          - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
          command:
          - python3
          envFrom:
          - secretRef:
              name: hf-token-secret
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
          resources:
            limits:
              nvidia.com/gpu: "1"
          workingDir: /workspace/examples/backends/vllm
    replicas: 1
    type: decode
  - name: prefill
    podTemplate:
      spec:
        containers:
        - args:
          - -m
          - dynamo.vllm
          - --model
          - Qwen/Qwen3-0.6B
          - --disaggregation-mode
          - prefill
          - --kv-transfer-config
          - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
          command:
          - python3
          envFrom:
          - secretRef:
              name: hf-token-secret
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
          resources:
            limits:
              nvidia.com/gpu: "1"
          workingDir: /workspace/examples/backends/vllm
    replicas: 1
    type: prefill

# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
apiVersion: nvidia.com/v1beta1
kind: DynamoGraphDeployment
metadata:
  name: vllm-disagg-kvbm
spec:
  components:
  - name: Frontend
    podTemplate:
      spec:
        containers:
        - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
    replicas: 1
    type: frontend
  - name: decode
    podTemplate:
      spec:
        containers:
        - args:
          - --model
          - Qwen/Qwen3-8B
          - --disaggregation-mode
          - decode
          - --disable-hybrid-kv-cache-manager
          - --kv-transfer-config
          - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
          - --max-model-len
          - "32000"
          - --enforce-eager
          command:
          - python3
          - -m
          - dynamo.vllm
          envFrom:
          - secretRef:
              name: hf-token-secret
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
          resources:
            limits:
              nvidia.com/gpu: "1"
          workingDir: /workspace/examples/backends/vllm
    replicas: 1
    type: decode
  - name: prefill
    podTemplate:
      spec:
        containers:
        - args:
          - --model
          - Qwen/Qwen3-8B
          - --disaggregation-mode
          - prefill
          - --disable-hybrid-kv-cache-manager
          - --max-model-len
          - "32000"
          - --enforce-eager
          - --kv-transfer-config
          - '{"kv_connector":"PdConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"DynamoConnector","kv_connector_module_path":"kvbm.vllm_integration.connector","kv_role":"kv_both"},{"kv_connector":"NixlConnector","kv_role":"kv_both"}]},"kv_connector_module_path":"kvbm.vllm_integration.connector"}'
          command:
          - python3
          - -m
          - dynamo.vllm
          env:
          - name: DYN_KVBM_CPU_CACHE_GB
            value: "100"
          envFrom:
          - secretRef:
              name: hf-token-secret
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
          resources:
            limits:
              memory: 250Gi
              nvidia.com/gpu: "1"
            requests:
              memory: 200Gi
              nvidia.com/gpu: "1"
          workingDir: /workspace/examples/backends/vllm
    replicas: 1
    type: prefill

# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
apiVersion: nvidia.com/v1beta1
kind: DynamoGraphDeployment
metadata:
  name: vllm-disagg-kvbm-tp2
spec:
  components:
  - name: Frontend
    podTemplate:
      spec:
        containers:
        - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
    replicas: 1
    type: frontend
  - name: decode
    podTemplate:
      spec:
        containers:
        - args:
          - --model
          - Qwen/Qwen3-8B
          - --disaggregation-mode
          - decode
          - --disable-hybrid-kv-cache-manager
          - --kv-transfer-config
          - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
          - --gpu-memory-utilization
          - "0.23"
          - --max-model-len
          - "32000"
          - --enforce-eager
          - --tensor-parallel-size
          - "2"
          command:
          - python3
          - -m
          - dynamo.vllm
          envFrom:
          - secretRef:
              name: hf-token-secret
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
          resources:
            limits:
              nvidia.com/gpu: "2"
            requests:
              nvidia.com/gpu: "2"
          workingDir: /workspace/examples/backends/vllm
    replicas: 1
    type: decode
  - name: prefill
    podTemplate:
      spec:
        containers:
        - args:
          - --model
          - Qwen/Qwen3-8B
          - --disaggregation-mode
          - prefill
          - --disable-hybrid-kv-cache-manager
          - --gpu-memory-utilization
          - "0.23"
          - --max-model-len
          - "32000"
          - --enforce-eager
          - --kv-transfer-config
          - '{"kv_connector":"PdConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"DynamoConnector","kv_connector_module_path":"kvbm.vllm_integration.connector","kv_role":"kv_both"},{"kv_connector":"NixlConnector","kv_role":"kv_both"}]},"kv_connector_module_path":"kvbm.vllm_integration.connector"}'
          - --tensor-parallel-size
          - "2"
          command:
          - python3
          - -m
          - dynamo.vllm
          env:
          - name: DYN_KVBM_CPU_CACHE_GB
            value: "100"
          envFrom:
          - secretRef:
              name: hf-token-secret
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
          resources:
            limits:
              memory: 250Gi
              nvidia.com/gpu: "2"
            requests:
              memory: 200Gi
              nvidia.com/gpu: "2"
          workingDir: /workspace/examples/backends/vllm
    replicas: 1
    type: prefill

# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
apiVersion: nvidia.com/v1beta1
kind: DynamoGraphDeployment
metadata:
  name: vllm-disagg-kvbm-2p2d
spec:
  components:
  - name: Frontend
    podTemplate:
      spec:
        containers:
        - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
    replicas: 1
    type: frontend
  - name: decode
    podTemplate:
      spec:
        containers:
        - args:
          - --model
          - Qwen/Qwen3-8B
          - --disaggregation-mode
          - decode
          - --disable-hybrid-kv-cache-manager
          - --kv-transfer-config
          - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
          - --max-model-len
          - "32000"
          - --enforce-eager
          command:
          - python3
          - -m
          - dynamo.vllm
          envFrom:
          - secretRef:
              name: hf-token-secret
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
          resources:
            limits:
              nvidia.com/gpu: "1"
          workingDir: /workspace/examples/backends/vllm
    replicas: 2
    type: decode
  - name: prefill
    podTemplate:
      spec:
        containers:
        - args:
          - --model
          - Qwen/Qwen3-8B
          - --disaggregation-mode
          - prefill
          - --disable-hybrid-kv-cache-manager
          - --max-model-len
          - "32000"
          - --enforce-eager
          - --kv-transfer-config
          - '{"kv_connector":"PdConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"DynamoConnector","kv_connector_module_path":"kvbm.vllm_integration.connector","kv_role":"kv_both"},{"kv_connector":"NixlConnector","kv_role":"kv_both"}]},"kv_connector_module_path":"kvbm.vllm_integration.connector"}'
          command:
          - python3
          - -m
          - dynamo.vllm
          env:
          - name: DYN_KVBM_CPU_CACHE_GB
            value: "100"
          envFrom:
          - secretRef:
              name: hf-token-secret
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
          resources:
            limits:
              memory: 250Gi
              nvidia.com/gpu: "1"
            requests:
              memory: 200Gi
              nvidia.com/gpu: "1"
          workingDir: /workspace/examples/backends/vllm
    replicas: 2
    type: prefill

# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Disaggregated vLLM deployment with OpenTelemetry tracing enabled.
# Base deployment: disagg.yaml
# See docs/observability/tracing.md for setup instructions.
apiVersion: nvidia.com/v1beta1
kind: DynamoGraphDeployment
metadata:
  name: vllm-disagg-tracing
spec:
  components:
  - name: Frontend
    podTemplate:
      spec:
        containers:
        - env:
          - name: OTEL_SERVICE_NAME
            value: dynamo-frontend
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
    replicas: 1
    type: frontend
  - name: decode
    podTemplate:
      spec:
        containers:
        - args:
          - --model
          - Qwen/Qwen3-0.6B
          - --disaggregation-mode
          - decode
          - --kv-transfer-config
          - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
          command:
          - python3
          - -m
          - dynamo.vllm
          env:
          - name: OTEL_SERVICE_NAME
            value: dynamo-worker-decode
          envFrom:
          - secretRef:
              name: hf-token-secret
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
          resources:
            limits:
              nvidia.com/gpu: "1"
            requests:
              # Increase this value for larger models.
              ephemeral-storage: 2Gi
          workingDir: /workspace/examples/backends/vllm
    replicas: 1
    type: decode
  - name: prefill
    podTemplate:
      spec:
        containers:
        - args:
          - --model
          - Qwen/Qwen3-0.6B
          - --disaggregation-mode
          - prefill
          - --kv-transfer-config
          - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
          command:
          - python3
          - -m
          - dynamo.vllm
          env:
          - name: OTEL_SERVICE_NAME
            value: dynamo-worker-prefill
          envFrom:
          - secretRef:
              name: hf-token-secret
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
          resources:
            limits:
              nvidia.com/gpu: "1"
            requests:
              # Increase this value for larger models.
              ephemeral-storage: 2Gi
          workingDir: /workspace/examples/backends/vllm
    replicas: 1
    type: prefill
  env:
  - name: DYN_LOGGING_JSONL
    value: "true"
  - name: OTEL_EXPORT_ENABLED
    value: "true"
  - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
    value: http://tempo.observability.svc.cluster.local:4317

# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
apiVersion: nvidia.com/v1beta1
kind: DynamoGraphDeployment
metadata:
  name: vllm-disagg
spec:
  components:
  - name: Frontend
    podTemplate:
      spec:
        containers:
        - args:
          - --http-port
          - "8000"
          command:
          - python3
          - -m
          - dynamo.frontend
          image: my-registry/vllm-runtime:my-tag
          name: main
          workingDir: /workspace/examples/backends/vllm
    replicas: 1
    type: frontend
  - multinode:
      nodeCount: 2
    name: decode
    podTemplate:
      spec:
        containers:
        - args:
          - --model
          - Qwen/Qwen3-0.6B
          - --tensor-parallel-size
          - "2"
          - --disaggregation-mode
          - decode
          - --kv-transfer-config
          - '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id":
            "vllm-disagg-decode-engine-0abc123"}'
          command:
          - python3
          - -m
          - dynamo.vllm
          envFrom:
          - secretRef:
              name: hf-token-secret
          image: my-registry/vllm-runtime:my-tag
          name: main
          resources:
            limits:
              nvidia.com/gpu: "1"
          workingDir: /workspace/examples/backends/vllm
    replicas: 1
    type: decode
  - multinode:
      nodeCount: 2
    name: prefill
    podTemplate:
      spec:
        containers:
        - args:
          - --model
          - Qwen/Qwen3-0.6B
          - --tensor-parallel-size
          - "2"
          - --disaggregation-mode
          - prefill
          - --kv-transfer-config
          - '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id":
            "vllm-disagg-prefill-engine-0abc123"}'
          command:
          - python3
          - -m
          - dynamo.vllm
          envFrom:
          - secretRef:
              name: hf-token-secret
          image: my-registry/vllm-runtime:my-tag
          name: main
          resources:
            limits:
              nvidia.com/gpu: "1"
          workingDir: /workspace/examples/backends/vllm
    replicas: 1
    type: prefill

# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
apiVersion: resource.k8s.io/v1
kind: ResourceClaimTemplate
metadata:
  name: gpu-template
spec:
  spec:
    devices:
      requests:
        - name: gpu
          exactly:
            deviceClassName: gpu.intel.com
            count: 1
---
apiVersion: nvidia.com/v1beta1
kind: DynamoGraphDeployment
metadata:
  name: vllm-disagg-xpu-dra
spec:
  components:
  - name: Frontend
    podTemplate:
      spec:
        containers:
        - envFrom:
          - secretRef:
              name: hf-token-secret
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag
          name: main
    replicas: 1
    type: frontend
  - name: decode
    podTemplate:
      spec:
        containers:
        - args:
          - --model
          - Qwen/Qwen3-0.6B
          - --disaggregation-mode
          - decode
          - --kv-transfer-config
          - '{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_buffer_device":"xpu"}'
          - --block-size
          - "64"
          command:
          - python3
          - -m
          - dynamo.vllm
          env:
          - name: VLLM_TARGET_DEVICE
            value: xpu
          envFrom:
          - secretRef:
              name: hf-token-secret
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime-xpu:my-tag
          name: main
          resources:
            claims:
            - name: gpu
            requests:
              # Increase this value for larger models.
              ephemeral-storage: 2Gi
          workingDir: /workspace/examples/backends/vllm
        resourceClaims:
        - name: gpu
          resourceClaimTemplateName: gpu-template
        # NOTE: Uncomment if your environment requires specific group access
        # securityContext:
        #   runAsUser: 1000
        #   runAsGroup: 1000
        #   supplementalGroups:
        #     - 44   # render group
        #     - 991  # video group
    replicas: 1
    type: decode
  - name: prefill
    podTemplate:
      spec:
        containers:
        - args:
          - --model
          - Qwen/Qwen3-0.6B
          - --disaggregation-mode
          - prefill
          - --kv-transfer-config
          - '{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_buffer_device":"xpu"}'
          - --block-size
          - "64"
          command:
          - python3
          - -m
          - dynamo.vllm
          env:
          - name: VLLM_TARGET_DEVICE
            value: xpu
          envFrom:
          - secretRef:
              name: hf-token-secret
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime-xpu:my-tag
          name: main
          resources:
            claims:
            - name: gpu
            requests:
              # Increase this value for larger models.
              ephemeral-storage: 2Gi
          workingDir: /workspace/examples/backends/vllm
        resourceClaims:
        - name: gpu
          resourceClaimTemplateName: gpu-template
        # NOTE: Uncomment if your environment requires specific group access
        # securityContext:
        #   runAsUser: 1000
        #   runAsGroup: 1000
        #   supplementalGroups:
        #     - 44   # render group
        #     - 991  # video group
    replicas: 1
    type: prefill

# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Example: DynamoGraphDeployment with inter-pod GMS (GPU Memory Service)
# failover on vLLM.
#
# Inter-pod GMS failover splits the traditional single-engine pod into:
#   * a dedicated GMS weight-server pod (per rank) that owns the model weights
#     and exposes them over a shared-GPU UDS, and
#   * N engine pods (per rank) that attach to the same GPUs via DRA and race
#     for a flock; the winner becomes primary, the others are hot shadows.
#
# This file contains variants you can use under .spec.components:
#
# Single-node GMS:
#   Creates per PCSG replica:
#     - 1 GMS weight-server pod     (<component>-gms-0)
#     - numShadows + 1 engine pods (<component>, replicas = numShadows + 1)
#   All engine pods + the GMS pod share the same GPUs via DRA ResourceClaims.
#   component.replicas controls how many PCSG replicas are created
#   (horizontal scale).
#
# Multinode GMS (N nodes):
#   Creates per PCSG replica:
#     - 1 GMS weight-server pod per rank   (<component>-gms-<rank>)
#     - numShadows + 1 engine pods per rank
#         rank 0: <component>-ldr   (leader,   replicas = numShadows + 1)
#         rank R: <component>-wkr-R (worker R, replicas = numShadows + 1)
#   Each rank's GMS + engine pods share GPUs via DRA within that node.
#   component.replicas controls horizontal PCSG replicas.
apiVersion: nvidia.com/v1beta1
kind: DynamoGraphDeployment
metadata:
  annotations:
    nvidia.com/dynamo-kube-discovery-mode: container
  name: llm-serving-mn
spec:
  backendFramework: vllm
  components:
  # --- Single-node GMS failover ---
  - experimental:
      failover:
        mode: InterPod
        numShadows: 1 # 1 primary + 1 shadow = 2 engine pods per PCSG replica
      gpuMemoryService:
        mode: InterPod
    name: agg
    podTemplate:
      spec:
        containers:
        - args:
          - --model
          - Qwen/Qwen3-0.6B
          - --tensor-parallel-size
          - "1"
          - --enforce-eager
          - --gpu-memory-utilization
          - "0.85"
          command:
          - python3
          - -m
          - dynamo.vllm
          envFrom:
          - secretRef:
              name: hf-token-secret
          image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:latest
          name: main
          resources:
            limits:
              nvidia.com/gpu: "1"
    replicas: 1
    # sharedMemorySize: 16Gi
    type: worker
  # --- Multinode GMS failover (2 nodes) ---
  # - experimental:
  #     failover:
  #       mode: InterPod
  #       numShadows: 1 # 1 primary + 1 shadow = 2 engine pods per rank
  #     gpuMemoryService:
  #       mode: InterPod
  #   multinode:
  #     nodeCount: 2
  #   name: agg
  #   podTemplate:
  #     spec:
  #       containers:
  #       - args:
  #         # args: ["--model", "Qwen/Qwen3-235B-A22B", "--tensor-parallel-size", "8", "--enforce-eager", "--gpu-memory-utilization", "0.85"]
  #         - --model
  #         - Qwen/Qwen3-0.6B
  #         - --tensor-parallel-size
  #         - "2"
  #         - --enforce-eager
  #         - --gpu-memory-utilization
  #         - "0.85"
  #         command:
  #         - python3
  #         - -m
  #         - dynamo.vllm
  #         envFrom:
  #         - secretRef:
  #             name: hf-token-secret
  #         image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:latest
  #         name: main
  #         resources:
  #           limits:
  #             nvidia.com/gpu: "1"
  #   replicas: 1
  #   # sharedMemorySize: 16Gi
  #   type: worker
  # --- Regular frontend (no failover) ---
  - name: frontend
    podTemplate:
      spec:
        containers:
        - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:latest
          # command: ["python3", "-m", "dynamo.frontend"]
          name: main
    replicas: 1
    type: frontend