> For clean Markdown content of this page, append .md to this URL. For the complete documentation index, see https://docs.nvidia.com/dynamo/llms.txt. For full content including API reference and SDK examples, see https://docs.nvidia.com/dynamo/llms-full.txt. # TensorRT-LLM Deployment Templates Copy-paste `DynamoGraphDeployment` (`nvidia.com/v1alpha1`) manifests for the TensorRT-LLM backend, grouped by topology. Each manifest is embedded from [`examples/backends/trtllm/deploy/`](https://github.com/ai-dynamo/dynamo/tree/main/examples/backends/trtllm/deploy) — open an entry, use the copy button, then set your image tag and `hf-token-secret` before applying. Apply any template with: ```bash kubectl apply -f agg.yaml ``` Some templates bundle a `ConfigMap` (engine configuration) or a `PersistentVolumeClaim` alongside the deployment. Apply the whole file — every document in it is part of the example. ## Aggregated #### agg.yaml · Baseline aggregated serving ```yaml title={"agg.yaml"} maxLines={0} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 apiVersion: nvidia.com/v1alpha1 kind: DynamoGraphDeployment metadata: name: trtllm-agg spec: services: Frontend: componentType: frontend replicas: 1 extraPodSpec: mainContainer: image: my-registry/tensorrtllm-runtime:my-tag TRTLLMWorker: envFromSecret: hf-token-secret componentType: worker replicas: 1 resources: limits: gpu: "1" requests: custom: # Increase this value for larger models ephemeral-storage: "2Gi" extraPodSpec: mainContainer: image: my-registry/tensorrtllm-runtime:my-tag workingDir: /workspace/ command: - python3 - -m - dynamo.trtllm args: - --model-path - Qwen/Qwen3-0.6B - --served-model-name - Qwen/Qwen3-0.6B - --extra-engine-args - ./examples/backends/trtllm/engine_configs/qwen3/agg.yaml ``` #### agg\_router.yaml · Aggregated with KV-aware routing ```yaml title={"agg_router.yaml"} maxLines={0} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 apiVersion: nvidia.com/v1alpha1 kind: DynamoGraphDeployment metadata: name: trtllm-agg-router spec: services: Frontend: componentType: frontend replicas: 1 extraPodSpec: mainContainer: image: my-registry/tensorrtllm-runtime:my-tag envs: - name: DYN_ROUTER_MODE value: kv TRTLLMWorker: envFromSecret: hf-token-secret componentType: worker replicas: 2 resources: limits: gpu: "1" requests: custom: # Increase this value for larger models ephemeral-storage: "2Gi" extraPodSpec: mainContainer: image: my-registry/tensorrtllm-runtime:my-tag workingDir: /workspace/ command: - python3 - -m - dynamo.trtllm args: - --model-path - Qwen/Qwen3-0.6B - --served-model-name - Qwen/Qwen3-0.6B - --extra-engine-args - ./examples/backends/trtllm/engine_configs/qwen3/agg.yaml - --publish-events-and-metrics ``` #### agg-with-config.yaml · Aggregated with an external engine ConfigMap (bundles a ConfigMap) ```yaml title={"agg-with-config.yaml"} maxLines={0} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # configmap that contains the custom trtllm configuration apiVersion: v1 kind: ConfigMap metadata: name: nvidia-config data: agg.yaml: | tensor_parallel_size: 1 moe_expert_parallel_size: 1 enable_attention_dp: false max_num_tokens: 8192 max_batch_size: 16 trust_remote_code: true backend: pytorch enable_chunked_prefill: true disable_overlap_scheduler: true kv_cache_config: free_gpu_memory_fraction: 0.95 cuda_graph_config: max_batch_size: 16 --- # dynamo graph deployment which uses the custom configuration contained in the configmap apiVersion: nvidia.com/v1alpha1 kind: DynamoGraphDeployment metadata: name: trtllm-agg spec: services: Frontend: componentType: frontend replicas: 1 extraPodSpec: mainContainer: image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:my-tag TRTLLMWorker: envFromSecret: hf-token-secret componentType: worker replicas: 1 resources: limits: gpu: "1" extraPodSpec: # declare the configmap as a volume volumes: - name: nvidia-config configMap: name: nvidia-config mainContainer: image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:my-tag workingDir: /workspace/examples/backends/trtllm # mount the configmap as a volume volumeMounts: - name: nvidia-config mountPath: /workspace/ readOnly: true command: - python3 - -m - dynamo.trtllm args: - --model-path - Qwen/Qwen3-0.6B - --served-model-name - Qwen/Qwen3-0.6B - --extra-engine-args - ./examples/backends/trtllm/engine_configs/qwen3/agg.yaml ``` ## Disaggregated #### disagg.yaml · Baseline disaggregated prefill/decode ```yaml title={"disagg.yaml"} maxLines={0} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 apiVersion: nvidia.com/v1alpha1 kind: DynamoGraphDeployment metadata: name: trtllm-disagg spec: services: Frontend: componentType: frontend replicas: 1 extraPodSpec: mainContainer: image: my-registry/tensorrtllm-runtime:my-tag prefill: envFromSecret: hf-token-secret componentType: worker subComponentType: prefill replicas: 1 resources: limits: gpu: "1" extraPodSpec: mainContainer: image: my-registry/tensorrtllm-runtime:my-tag workingDir: /workspace/ command: - python3 - -m - dynamo.trtllm args: - --model-path - Qwen/Qwen3-0.6B - --served-model-name - Qwen/Qwen3-0.6B - --extra-engine-args - ./examples/backends/trtllm/engine_configs/qwen3/prefill.yaml - --disaggregation-mode - prefill decode: envFromSecret: hf-token-secret componentType: worker subComponentType: decode replicas: 1 resources: limits: gpu: "1" extraPodSpec: mainContainer: image: my-registry/tensorrtllm-runtime:my-tag workingDir: /workspace/ command: - python3 - -m - dynamo.trtllm args: - --model-path - Qwen/Qwen3-0.6B - --served-model-name - Qwen/Qwen3-0.6B - --extra-engine-args - ./examples/backends/trtllm/engine_configs/qwen3/decode.yaml - --disaggregation-mode - decode ``` #### disagg\_router.yaml · Disaggregated with KV-aware routing ```yaml title={"disagg_router.yaml"} maxLines={0} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 apiVersion: nvidia.com/v1alpha1 kind: DynamoGraphDeployment metadata: name: trtllm-v1-disagg-router spec: services: Frontend: componentType: frontend replicas: 1 extraPodSpec: mainContainer: image: my-registry/tensorrtllm-runtime:my-tag envs: - name: DYN_ROUTER_MODE value: kv prefill: envFromSecret: hf-token-secret componentType: worker replicas: 2 resources: limits: gpu: "1" extraPodSpec: mainContainer: image: my-registry/tensorrtllm-runtime:my-tag workingDir: /workspace/ command: - python3 - -m - dynamo.trtllm args: - --model-path - Qwen/Qwen3-0.6B - --served-model-name - Qwen/Qwen3-0.6B - --extra-engine-args - ./examples/backends/trtllm/engine_configs/qwen3/prefill.yaml - --disaggregation-mode - prefill - --publish-events-and-metrics decode: envFromSecret: hf-token-secret componentType: worker replicas: 2 resources: limits: gpu: "1" extraPodSpec: mainContainer: image: my-registry/tensorrtllm-runtime:my-tag workingDir: /workspace/ command: - python3 - -m - dynamo.trtllm args: - --model-path - Qwen/Qwen3-0.6B - --served-model-name - Qwen/Qwen3-0.6B - --extra-engine-args - ./examples/backends/trtllm/engine_configs/qwen3/decode.yaml - --disaggregation-mode - decode ``` #### disagg\_planner.yaml · Disaggregated with Dynamo Planner autoscaling ```yaml title={"disagg_planner.yaml"} maxLines={0} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 apiVersion: nvidia.com/v1alpha1 kind: DynamoGraphDeployment metadata: name: trtllm-disagg-planner spec: services: Frontend: componentType: frontend replicas: 1 extraPodSpec: mainContainer: image: my-registry/tensorrtllm-runtime:my-tag workingDir: /workspace/examples/backends/trtllm command: - python3 args: - -m - dynamo.frontend - --http-port - "8000" - --kv-cache-block-size - "128" - --router-mode - kv - --router-kv-overlap-score-credit - "0.0" - --router-temperature - "0.0" - --no-kv-events Planner: envFromSecret: hf-token-secret componentType: planner replicas: 1 extraPodSpec: mainContainer: # Planner image selection: # Dynamo >= 1.1.0: use the dedicated planner image # /dynamo-planner: # (backend runtime images no longer ship planner runtime deps # such as kubernetes_asyncio, pmdarima, prophet, aiconfigurator). # Dynamo < 1.1.0: use the backend runtime image # /tensorrtllm-runtime:. image: my-registry/dynamo-planner:my-tag ports: - name: metrics containerPort: 9085 command: - python3 args: - -m - dynamo.planner - --config - '{"environment": "kubernetes", "backend": "trtllm", "optimization_target": "sla", "enable_throughput_scaling": true, "enable_load_scaling": true, "pre_deployment_sweeping_mode": "none", "throughput_adjustment_interval_seconds": 60, "load_adjustment_interval_seconds": 5}' decode: envFromSecret: hf-token-secret componentType: worker subComponentType: decode replicas: 1 livenessProbe: httpGet: path: /live port: 9090 periodSeconds: 5 timeoutSeconds: 30 failureThreshold: 1 readinessProbe: httpGet: path: /health port: 9090 periodSeconds: 10 timeoutSeconds: 30 failureThreshold: 60 resources: limits: gpu: "1" extraPodSpec: terminationGracePeriodSeconds: 600 mainContainer: image: my-registry/tensorrtllm-runtime:my-tag workingDir: /workspace/ command: - python3 args: - -m - dynamo.trtllm - --model-path - Qwen/Qwen3-0.6B - --served-model-name - Qwen/Qwen3-0.6B - --extra-engine-args - ./examples/backends/trtllm/engine_configs/qwen3/decode.yaml - --disaggregation-mode - decode prefill: envFromSecret: hf-token-secret componentType: worker subComponentType: prefill replicas: 1 resources: limits: gpu: "1" extraPodSpec: terminationGracePeriodSeconds: 600 mainContainer: image: my-registry/tensorrtllm-runtime:my-tag workingDir: /workspace/ command: - python3 args: - -m - dynamo.trtllm - --model-path - Qwen/Qwen3-0.6B - --served-model-name - Qwen/Qwen3-0.6B - --extra-engine-args - ./examples/backends/trtllm/engine_configs/qwen3/prefill.yaml - --disaggregation-mode - prefill ``` #### disagg-multinode.yaml · Disaggregated across multiple nodes (bundles a ConfigMap and PersistentVolumeClaim) ```yaml title={"disagg-multinode.yaml"} maxLines={0} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 apiVersion: v1 kind: ConfigMap metadata: name: nvidia-config data: prefill.yaml: | # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. tensor_parallel_size: 8 moe_expert_parallel_size: 1 enable_attention_dp: false max_num_tokens: 8192 trust_remote_code: true backend: pytorch enable_chunked_prefill: true # Overlap scheduler not currently supported in prefill only workers. disable_overlap_scheduler: true kv_cache_config: free_gpu_memory_fraction: 0.80 cache_transceiver_config: backend: DEFAULT decode.yaml: | # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. tensor_parallel_size: 8 moe_expert_parallel_size: 1 enable_attention_dp: false max_num_tokens: 8192 trust_remote_code: true backend: pytorch enable_chunked_prefill: true disable_overlap_scheduler: false kv_cache_config: free_gpu_memory_fraction: 0.80 cache_transceiver_config: backend: DEFAULT --- apiVersion: v1 kind: PersistentVolumeClaim metadata: name: models spec: accessModes: - ReadWriteMany resources: requests: storage: 100Gi --- apiVersion: nvidia.com/v1alpha1 kind: DynamoGraphDeployment metadata: name: trtllm-disagg-tp8 spec: backendFramework: trtllm pvcs: - name: models envs: - name: OMPI_ALLOW_RUN_AS_ROOT value: "1" - name: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM value: "1" - name: HF_HOME value: "/models" services: Frontend: volumeMounts: - name: models mountPoint: /models componentType: frontend replicas: 1 extraPodSpec: mainContainer: image: my-registry/tensorrtllm-runtime:my-tag workingDir: /workspace/examples/backends/trtllm command: - python3 - -m - dynamo.frontend args: - --http-port - "8000" prefill: volumeMounts: - name: models mountPoint: /models envFromSecret: hf-token-secret componentType: worker replicas: 1 multinode: nodeCount: 2 resources: limits: gpu: "4" extraPodSpec: volumes: - name: nvidia-config configMap: name: nvidia-config mainContainer: volumeMounts: - name: nvidia-config mountPath: /workspace/ readOnly: true image: my-registry/tensorrtllm-runtime:my-tag workingDir: /workspace/ command: - python3 - -m - dynamo.trtllm args: - --model-path - Qwen/Qwen3-0.6B - --served-model-name - Qwen/Qwen3-0.6B - --extra-engine-args - /workspace/prefill.yaml - --disaggregation-mode - prefill decode: volumeMounts: - name: models mountPoint: /models envFromSecret: hf-token-secret componentType: worker replicas: 1 multinode: nodeCount: 2 resources: limits: gpu: "4" extraPodSpec: volumes: - name: nvidia-config configMap: name: nvidia-config mainContainer: volumeMounts: - name: nvidia-config mountPath: /workspace/ readOnly: true image: my-registry/tensorrtllm-runtime:my-tag workingDir: /workspace/ command: - python3 - -m - dynamo.trtllm args: - --model-path - Qwen/Qwen3-0.6B - --served-model-name - Qwen/Qwen3-0.6B - --extra-engine-args - /workspace/decode.yaml - --disaggregation-mode - decode ``` ## Source All templates live in [`examples/backends/trtllm/deploy/`](https://github.com/ai-dynamo/dynamo/tree/main/examples/backends/trtllm/deploy). For local launch commands, see [TensorRT-LLM Local Deployment Examples](/dynamo/dev/recipes/cli-templates/tensor-rt-llm).