> For clean Markdown content of this page, append .md to this URL. For the complete documentation index, see https://docs.nvidia.com/dynamo/llms.txt. For full content including API reference and SDK examples, see https://docs.nvidia.com/dynamo/llms-full.txt. # TensorRT-LLM Deployment Templates Copy-paste `DynamoGraphDeployment` (`nvidia.com/v1beta1`) manifests for the TensorRT-LLM backend, grouped by topology. Each manifest is embedded from [`examples/backends/trtllm/deploy/`](https://github.com/ai-dynamo/dynamo/tree/v1.4.0/examples/backends/trtllm/deploy) — open an entry, use the copy button, then set your image tag and `hf-token-secret` before applying. Apply any template with: ```bash kubectl apply -f agg.yaml ``` Some templates bundle a `ConfigMap` (engine configuration) or a `PersistentVolumeClaim` alongside the deployment. Apply the whole file — every document in it is part of the example. ## Aggregated #### agg.yaml · Baseline aggregated serving ```yaml title={"agg.yaml"} maxLines={0} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 apiVersion: nvidia.com/v1beta1 kind: DynamoGraphDeployment metadata: name: trtllm-agg spec: components: - name: Frontend podTemplate: spec: containers: - image: my-registry/tensorrtllm-runtime:my-tag name: main replicas: 1 type: frontend - name: TRTLLMWorker podTemplate: spec: containers: - args: - --model-path - Qwen/Qwen3-0.6B - --served-model-name - Qwen/Qwen3-0.6B - --extra-engine-args - ./examples/backends/trtllm/engine_configs/qwen3/agg.yaml command: - python3 - -m - dynamo.trtllm envFrom: - secretRef: name: hf-token-secret image: my-registry/tensorrtllm-runtime:my-tag name: main resources: limits: nvidia.com/gpu: "1" requests: # Increase this value for larger models. ephemeral-storage: 2Gi workingDir: /workspace/ replicas: 1 type: worker ``` #### agg\_router.yaml · Aggregated with KV-aware routing ```yaml title={"agg_router.yaml"} maxLines={0} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 apiVersion: nvidia.com/v1beta1 kind: DynamoGraphDeployment metadata: name: trtllm-agg-router spec: components: - name: Frontend podTemplate: spec: containers: - env: - name: DYN_ROUTER_MODE value: kv image: my-registry/tensorrtllm-runtime:my-tag name: main replicas: 1 type: frontend - name: TRTLLMWorker podTemplate: spec: containers: - args: - --model-path - Qwen/Qwen3-0.6B - --served-model-name - Qwen/Qwen3-0.6B - --extra-engine-args - ./examples/backends/trtllm/engine_configs/qwen3/agg.yaml - --publish-events-and-metrics command: - python3 - -m - dynamo.trtllm envFrom: - secretRef: name: hf-token-secret image: my-registry/tensorrtllm-runtime:my-tag name: main resources: limits: nvidia.com/gpu: "1" requests: # Increase this value for larger models. ephemeral-storage: 2Gi workingDir: /workspace/ replicas: 2 type: worker ``` #### agg-with-config.yaml · Aggregated with an external engine ConfigMap (bundles a ConfigMap) ```yaml title={"agg-with-config.yaml"} maxLines={0} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # configmap that contains the custom trtllm configuration apiVersion: v1 kind: ConfigMap metadata: name: nvidia-config data: agg.yaml: | tensor_parallel_size: 1 moe_expert_parallel_size: 1 enable_attention_dp: false max_num_tokens: 8192 max_batch_size: 16 trust_remote_code: true backend: pytorch enable_chunked_prefill: true disable_overlap_scheduler: true kv_cache_config: free_gpu_memory_fraction: 0.95 cuda_graph_config: max_batch_size: 16 --- apiVersion: nvidia.com/v1beta1 kind: DynamoGraphDeployment metadata: name: trtllm-agg spec: components: - name: Frontend podTemplate: spec: containers: - image: my-registry/tensorrtllm-runtime:my-tag name: main replicas: 1 type: frontend - name: TRTLLMWorker podTemplate: spec: containers: - args: - --model-path - Qwen/Qwen3-0.6B - --served-model-name - Qwen/Qwen3-0.6B - --extra-engine-args - /workspace/examples/backends/trtllm/engine_configs/qwen3/agg.yaml command: - python3 - -m - dynamo.trtllm envFrom: - secretRef: name: hf-token-secret image: my-registry/tensorrtllm-runtime:my-tag name: main resources: limits: nvidia.com/gpu: "1" # Mount the ConfigMap key as the engine args file without masking /workspace. volumeMounts: - mountPath: /workspace/examples/backends/trtllm/engine_configs/qwen3/agg.yaml name: nvidia-config readOnly: true subPath: agg.yaml workingDir: /workspace/examples/backends/trtllm # Declare the ConfigMap as a pod volume. volumes: - configMap: name: nvidia-config name: nvidia-config replicas: 1 type: worker ``` ## Disaggregated #### disagg.yaml · Baseline disaggregated prefill/decode ```yaml title={"disagg.yaml"} maxLines={0} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 apiVersion: nvidia.com/v1beta1 kind: DynamoGraphDeployment metadata: name: trtllm-disagg spec: components: - name: Frontend podTemplate: spec: containers: - image: my-registry/tensorrtllm-runtime:my-tag name: main replicas: 1 type: frontend - name: decode podTemplate: spec: containers: - args: - --model-path - Qwen/Qwen3-0.6B - --served-model-name - Qwen/Qwen3-0.6B - --extra-engine-args - ./examples/backends/trtllm/engine_configs/qwen3/decode.yaml - --disaggregation-mode - decode command: - python3 - -m - dynamo.trtllm envFrom: - secretRef: name: hf-token-secret image: my-registry/tensorrtllm-runtime:my-tag name: main resources: limits: nvidia.com/gpu: "1" workingDir: /workspace/ replicas: 1 type: decode - name: prefill podTemplate: spec: containers: - args: - --model-path - Qwen/Qwen3-0.6B - --served-model-name - Qwen/Qwen3-0.6B - --extra-engine-args - ./examples/backends/trtllm/engine_configs/qwen3/prefill.yaml - --disaggregation-mode - prefill command: - python3 - -m - dynamo.trtllm envFrom: - secretRef: name: hf-token-secret image: my-registry/tensorrtllm-runtime:my-tag name: main resources: limits: nvidia.com/gpu: "1" workingDir: /workspace/ replicas: 1 type: prefill ``` #### disagg\_router.yaml · Disaggregated with KV-aware routing ```yaml title={"disagg_router.yaml"} maxLines={0} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 apiVersion: nvidia.com/v1beta1 kind: DynamoGraphDeployment metadata: name: trtllm-v1-disagg-router spec: components: - name: Frontend podTemplate: spec: containers: - env: - name: DYN_ROUTER_MODE value: kv image: my-registry/tensorrtllm-runtime:my-tag name: main replicas: 1 type: frontend - name: decode podTemplate: spec: containers: - args: - --model-path - Qwen/Qwen3-0.6B - --served-model-name - Qwen/Qwen3-0.6B - --extra-engine-args - ./examples/backends/trtllm/engine_configs/qwen3/decode.yaml - --disaggregation-mode - decode command: - python3 - -m - dynamo.trtllm envFrom: - secretRef: name: hf-token-secret image: my-registry/tensorrtllm-runtime:my-tag name: main resources: limits: nvidia.com/gpu: "1" workingDir: /workspace/ replicas: 2 type: decode - name: prefill podTemplate: spec: containers: - args: - --model-path - Qwen/Qwen3-0.6B - --served-model-name - Qwen/Qwen3-0.6B - --extra-engine-args - ./examples/backends/trtllm/engine_configs/qwen3/prefill.yaml - --disaggregation-mode - prefill - --publish-events-and-metrics command: - python3 - -m - dynamo.trtllm envFrom: - secretRef: name: hf-token-secret image: my-registry/tensorrtllm-runtime:my-tag name: main resources: limits: nvidia.com/gpu: "1" workingDir: /workspace/ replicas: 2 type: prefill ``` #### disagg\_planner.yaml · Disaggregated with Dynamo Planner autoscaling ```yaml title={"disagg_planner.yaml"} maxLines={0} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 apiVersion: nvidia.com/v1beta1 kind: DynamoGraphDeployment metadata: name: trtllm-disagg-planner spec: components: - name: Frontend podTemplate: spec: containers: - args: - -m - dynamo.frontend - --http-port - "8000" - --kv-cache-block-size - "128" - --router-mode - kv - --router-kv-overlap-score-credit - "0.0" - --router-temperature - "0.0" - --no-kv-events command: - python3 image: my-registry/tensorrtllm-runtime:my-tag name: main workingDir: /workspace/examples/backends/trtllm replicas: 1 type: frontend - name: Planner podTemplate: spec: containers: - args: - -m - dynamo.planner - --config - '{"environment": "kubernetes", "backend": "trtllm", "optimization_target": "sla", "enable_throughput_scaling": true, "enable_load_scaling": true, "pre_deployment_sweeping_mode": "none", "throughput_adjustment_interval_seconds": 60, "load_adjustment_interval_seconds": 5}' command: - python3 envFrom: - secretRef: name: hf-token-secret # Planner image selection: # Dynamo >= 1.1.0: use the dedicated planner image # /dynamo-planner: # (backend runtime images no longer ship planner runtime deps # such as kubernetes_asyncio, pmdarima, prophet, aiconfigurator). # Dynamo < 1.1.0: use the backend runtime image # /tensorrtllm-runtime:. image: my-registry/dynamo-planner:my-tag name: main ports: - containerPort: 9085 name: metrics replicas: 1 type: planner - name: decode podTemplate: spec: containers: - args: - -m - dynamo.trtllm - --model-path - Qwen/Qwen3-0.6B - --served-model-name - Qwen/Qwen3-0.6B - --extra-engine-args - ./examples/backends/trtllm/engine_configs/qwen3/decode.yaml - --disaggregation-mode - decode command: - python3 envFrom: - secretRef: name: hf-token-secret image: my-registry/tensorrtllm-runtime:my-tag livenessProbe: failureThreshold: 1 httpGet: path: /live port: 9090 periodSeconds: 5 timeoutSeconds: 30 name: main readinessProbe: failureThreshold: 60 httpGet: path: /health port: 9090 periodSeconds: 10 timeoutSeconds: 30 resources: limits: nvidia.com/gpu: "1" workingDir: /workspace/ terminationGracePeriodSeconds: 600 replicas: 1 type: decode - name: prefill podTemplate: spec: containers: - args: - -m - dynamo.trtllm - --model-path - Qwen/Qwen3-0.6B - --served-model-name - Qwen/Qwen3-0.6B - --extra-engine-args - ./examples/backends/trtllm/engine_configs/qwen3/prefill.yaml - --disaggregation-mode - prefill command: - python3 envFrom: - secretRef: name: hf-token-secret image: my-registry/tensorrtllm-runtime:my-tag name: main resources: limits: nvidia.com/gpu: "1" workingDir: /workspace/ terminationGracePeriodSeconds: 600 replicas: 1 type: prefill ``` #### disagg-multinode.yaml · Disaggregated across multiple nodes (bundles a ConfigMap and PersistentVolumeClaim) ```yaml title={"disagg-multinode.yaml"} maxLines={0} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 apiVersion: v1 kind: ConfigMap metadata: name: nvidia-config data: prefill.yaml: | # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. tensor_parallel_size: 8 moe_expert_parallel_size: 1 enable_attention_dp: false max_num_tokens: 8192 trust_remote_code: true backend: pytorch enable_chunked_prefill: true # Overlap scheduler not currently supported in prefill only workers. disable_overlap_scheduler: true kv_cache_config: free_gpu_memory_fraction: 0.80 cache_transceiver_config: backend: DEFAULT decode.yaml: | # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. tensor_parallel_size: 8 moe_expert_parallel_size: 1 enable_attention_dp: false max_num_tokens: 8192 trust_remote_code: true backend: pytorch enable_chunked_prefill: true disable_overlap_scheduler: false kv_cache_config: free_gpu_memory_fraction: 0.80 cache_transceiver_config: backend: DEFAULT --- apiVersion: v1 kind: PersistentVolumeClaim metadata: name: models spec: accessModes: - ReadWriteMany resources: requests: storage: 100Gi --- apiVersion: nvidia.com/v1beta1 kind: DynamoGraphDeployment metadata: name: trtllm-disagg-tp8 spec: backendFramework: trtllm components: - name: Frontend podTemplate: spec: containers: - args: - --http-port - "8000" command: - python3 - -m - dynamo.frontend image: my-registry/tensorrtllm-runtime:my-tag name: main volumeMounts: - mountPath: /models name: models workingDir: /workspace/examples/backends/trtllm volumes: - name: models persistentVolumeClaim: claimName: models replicas: 1 type: frontend - multinode: nodeCount: 2 name: decode podTemplate: spec: containers: - args: - --model-path - Qwen/Qwen3-0.6B - --served-model-name - Qwen/Qwen3-0.6B - --extra-engine-args - /workspace/engine_configs/decode.yaml - --disaggregation-mode - decode command: - python3 - -m - dynamo.trtllm envFrom: - secretRef: name: hf-token-secret image: my-registry/tensorrtllm-runtime:my-tag name: main resources: limits: nvidia.com/gpu: "4" volumeMounts: - mountPath: /models name: models - mountPath: /workspace/engine_configs name: nvidia-config readOnly: true workingDir: /workspace/ volumes: - name: models persistentVolumeClaim: claimName: models - configMap: name: nvidia-config name: nvidia-config replicas: 1 type: decode - multinode: nodeCount: 2 name: prefill podTemplate: spec: containers: - args: - --model-path - Qwen/Qwen3-0.6B - --served-model-name - Qwen/Qwen3-0.6B - --extra-engine-args - /workspace/engine_configs/prefill.yaml - --disaggregation-mode - prefill command: - python3 - -m - dynamo.trtllm envFrom: - secretRef: name: hf-token-secret image: my-registry/tensorrtllm-runtime:my-tag name: main resources: limits: nvidia.com/gpu: "4" volumeMounts: - mountPath: /models name: models - mountPath: /workspace/engine_configs name: nvidia-config readOnly: true workingDir: /workspace/ volumes: - name: models persistentVolumeClaim: claimName: models - configMap: name: nvidia-config name: nvidia-config replicas: 1 type: prefill env: - name: OMPI_ALLOW_RUN_AS_ROOT value: "1" - name: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM value: "1" - name: HF_HOME value: /models ``` ## Source All templates live in [`examples/backends/trtllm/deploy/`](https://github.com/ai-dynamo/dynamo/tree/v1.4.0/examples/backends/trtllm/deploy). For local launch commands, see [TensorRT-LLM Local Deployment Examples](/dynamo/recipes/cli-templates/tensor-rt-llm).