TensorRT-LLM Deployment Templates

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
4
apiVersion: nvidia.com/v1alpha1
5
kind: DynamoGraphDeployment
6
metadata:
7
  name: trtllm-agg
8
spec:
9
  services:
10
    Frontend:
11
      componentType: frontend
12
      replicas: 1
13
      extraPodSpec:
14
        mainContainer:
15
          image: my-registry/tensorrtllm-runtime:my-tag
16
    TRTLLMWorker:
17
      envFromSecret: hf-token-secret
18
      componentType: worker
19
      replicas: 1
20
      resources:
21
        limits:
22
          gpu: "1"
23
        requests:
24
          custom:
25
            # Increase this value for larger models
26
            ephemeral-storage: "2Gi"
27
      extraPodSpec:
28
        mainContainer:
29
          image: my-registry/tensorrtllm-runtime:my-tag
30
          workingDir: /workspace/
31
          command:
32
          - python3
33
          - -m
34
          - dynamo.trtllm
35
          args:
36
            - --model-path
37
            - Qwen/Qwen3-0.6B
38
            - --served-model-name
39
            - Qwen/Qwen3-0.6B
40
            - --extra-engine-args
41
            - ./examples/backends/trtllm/engine_configs/qwen3/agg.yaml

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
4
apiVersion: nvidia.com/v1alpha1
5
kind: DynamoGraphDeployment
6
metadata:
7
  name: trtllm-agg-router
8
spec:
9
  services:
10
    Frontend:
11
      componentType: frontend
12
      replicas: 1
13
      extraPodSpec:
14
        mainContainer:
15
          image: my-registry/tensorrtllm-runtime:my-tag
16
      envs:
17
        - name: DYN_ROUTER_MODE
18
          value: kv
19
    TRTLLMWorker:
20
      envFromSecret: hf-token-secret
21
      componentType: worker
22
      replicas: 2
23
      resources:
24
        limits:
25
          gpu: "1"
26
        requests:
27
          custom:
28
            # Increase this value for larger models
29
            ephemeral-storage: "2Gi"
30
      extraPodSpec:
31
        mainContainer:
32
          image: my-registry/tensorrtllm-runtime:my-tag
33
          workingDir: /workspace/
34
          command:
35
          - python3
36
          - -m
37
          - dynamo.trtllm
38
          args:
39
            - --model-path
40
            - Qwen/Qwen3-0.6B
41
            - --served-model-name
42
            - Qwen/Qwen3-0.6B
43
            - --extra-engine-args
44
            - ./examples/backends/trtllm/engine_configs/qwen3/agg.yaml
45
            - --publish-events-and-metrics

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
# configmap that contains the custom trtllm configuration
4
apiVersion: v1
5
kind: ConfigMap
6
metadata:
7
  name: nvidia-config
8
data:
9
  agg.yaml: |
10
    tensor_parallel_size: 1
11
    moe_expert_parallel_size: 1
12
    enable_attention_dp: false
13
    max_num_tokens: 8192
14
    max_batch_size: 16
15
    trust_remote_code: true
16
    backend: pytorch
17
    enable_chunked_prefill: true
18
    disable_overlap_scheduler: true
19
    kv_cache_config:
20
      free_gpu_memory_fraction: 0.95
21
    cuda_graph_config:
22
      max_batch_size: 16
23
---
24
# dynamo graph deployment which uses the custom configuration contained in the configmap
25
apiVersion: nvidia.com/v1alpha1
26
kind: DynamoGraphDeployment
27
metadata:
28
  name: trtllm-agg
29
spec:
30
  services:
31
    Frontend:
32
      componentType: frontend
33
      replicas: 1
34
      extraPodSpec:
35
        mainContainer:
36
          image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:my-tag
37
    TRTLLMWorker:
38
      envFromSecret: hf-token-secret
39
      componentType: worker
40
      replicas: 1
41
      resources:
42
        limits:
43
          gpu: "1"
44
      extraPodSpec:
45
        # declare the configmap as a volume
46
        volumes:
47
        - name: nvidia-config
48
          configMap:
49
            name: nvidia-config
50
        mainContainer:
51
          image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:my-tag
52
          workingDir: /workspace/examples/backends/trtllm
53
          # mount the configmap as a volume
54
          volumeMounts:
55
          - name: nvidia-config
56
            mountPath: /workspace/
57
            readOnly: true
58
          command:
59
          - python3
60
          - -m
61
          - dynamo.trtllm
62
          args:
63
            - --model-path
64
            - Qwen/Qwen3-0.6B
65
            - --served-model-name
66
            - Qwen/Qwen3-0.6B
67
            - --extra-engine-args
68
            - ./examples/backends/trtllm/engine_configs/qwen3/agg.yaml

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
4
apiVersion: nvidia.com/v1alpha1
5
kind: DynamoGraphDeployment
6
metadata:
7
  name: trtllm-disagg
8
spec:
9
  services:
10
    Frontend:
11
      componentType: frontend
12
      replicas: 1
13
      extraPodSpec:
14
        mainContainer:
15
          image: my-registry/tensorrtllm-runtime:my-tag
16
    prefill:
17
      envFromSecret: hf-token-secret
18
      componentType: worker
19
      subComponentType: prefill
20
      replicas: 1
21
      resources:
22
        limits:
23
          gpu: "1"
24
      extraPodSpec:
25
        mainContainer:
26
          image: my-registry/tensorrtllm-runtime:my-tag
27
          workingDir: /workspace/
28
          command:
29
          - python3
30
          - -m
31
          - dynamo.trtllm
32
          args:
33
            - --model-path
34
            - Qwen/Qwen3-0.6B
35
            - --served-model-name
36
            - Qwen/Qwen3-0.6B
37
            - --extra-engine-args
38
            - ./examples/backends/trtllm/engine_configs/qwen3/prefill.yaml
39
            - --disaggregation-mode
40
            - prefill
41
    decode:
42
      envFromSecret: hf-token-secret
43
      componentType: worker
44
      subComponentType: decode
45
      replicas: 1
46
      resources:
47
        limits:
48
          gpu: "1"
49
      extraPodSpec:
50
        mainContainer:
51
          image: my-registry/tensorrtllm-runtime:my-tag
52
          workingDir: /workspace/
53
          command:
54
          - python3
55
          - -m
56
          - dynamo.trtllm
57
          args:
58
            - --model-path
59
            - Qwen/Qwen3-0.6B
60
            - --served-model-name
61
            - Qwen/Qwen3-0.6B
62
            - --extra-engine-args
63
            - ./examples/backends/trtllm/engine_configs/qwen3/decode.yaml
64
            - --disaggregation-mode
65
            - decode

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
4
apiVersion: nvidia.com/v1alpha1
5
kind: DynamoGraphDeployment
6
metadata:
7
  name: trtllm-v1-disagg-router
8
spec:
9
  services:
10
    Frontend:
11
      componentType: frontend
12
      replicas: 1
13
      extraPodSpec:
14
        mainContainer:
15
          image: my-registry/tensorrtllm-runtime:my-tag
16
      envs:
17
        - name: DYN_ROUTER_MODE
18
          value: kv
19
    prefill:
20
      envFromSecret: hf-token-secret
21
      componentType: worker
22
      replicas: 2
23
      resources:
24
        limits:
25
          gpu: "1"
26
      extraPodSpec:
27
        mainContainer:
28
          image: my-registry/tensorrtllm-runtime:my-tag
29
          workingDir: /workspace/
30
          command:
31
          - python3
32
          - -m
33
          - dynamo.trtllm
34
          args:
35
            - --model-path
36
            - Qwen/Qwen3-0.6B
37
            - --served-model-name
38
            - Qwen/Qwen3-0.6B
39
            - --extra-engine-args
40
            - ./examples/backends/trtllm/engine_configs/qwen3/prefill.yaml
41
            - --disaggregation-mode
42
            - prefill
43
            - --publish-events-and-metrics
44
    decode:
45
      envFromSecret: hf-token-secret
46
      componentType: worker
47
      replicas: 2
48
      resources:
49
        limits:
50
          gpu: "1"
51
      extraPodSpec:
52
        mainContainer:
53
          image: my-registry/tensorrtllm-runtime:my-tag
54
          workingDir: /workspace/
55
          command:
56
          - python3
57
          - -m
58
          - dynamo.trtllm
59
          args:
60
            - --model-path
61
            - Qwen/Qwen3-0.6B
62
            - --served-model-name
63
            - Qwen/Qwen3-0.6B
64
            - --extra-engine-args
65
            - ./examples/backends/trtllm/engine_configs/qwen3/decode.yaml
66
            - --disaggregation-mode
67
            - decode

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
4
apiVersion: nvidia.com/v1alpha1
5
kind: DynamoGraphDeployment
6
metadata:
7
  name: trtllm-disagg-planner
8
spec:
9
  services:
10
    Frontend:
11
      componentType: frontend
12
      replicas: 1
13
      extraPodSpec:
14
        mainContainer:
15
          image: my-registry/tensorrtllm-runtime:my-tag
16
          workingDir: /workspace/examples/backends/trtllm
17
          command:
18
            - python3
19
          args:
20
            - -m
21
            - dynamo.frontend
22
            - --http-port
23
            - "8000"
24
            - --kv-cache-block-size
25
            - "128"
26
            - --router-mode
27
            - kv
28
            - --router-kv-overlap-score-credit
29
            - "0.0"
30
            - --router-temperature
31
            - "0.0"
32
            - --no-kv-events
33
    Planner:
34
      envFromSecret: hf-token-secret
35
      componentType: planner
36
      replicas: 1
37
      extraPodSpec:
38
        mainContainer:
39
          # Planner image selection:
40
          #   Dynamo >= 1.1.0: use the dedicated planner image
41
          #     <registry>/dynamo-planner:<version>
42
          #     (backend runtime images no longer ship planner runtime deps
43
          #     such as kubernetes_asyncio, pmdarima, prophet, aiconfigurator).
44
          #   Dynamo <  1.1.0: use the backend runtime image
45
          #     <registry>/tensorrtllm-runtime:<version>.
46
          image: my-registry/dynamo-planner:my-tag
47
          ports:
48
            - name: metrics
49
              containerPort: 9085
50
          command:
51
            - python3
52
          args:
53
            - -m
54
            - dynamo.planner
55
            - --config
56
            - '{"environment": "kubernetes", "backend": "trtllm", "optimization_target": "sla", "enable_throughput_scaling": true, "enable_load_scaling": true, "pre_deployment_sweeping_mode": "none", "throughput_adjustment_interval_seconds": 60, "load_adjustment_interval_seconds": 5}'
57
    decode:
58
      envFromSecret: hf-token-secret
59
      componentType: worker
60
      subComponentType: decode
61
      replicas: 1
62
      livenessProbe:
63
        httpGet:
64
          path: /live
65
          port: 9090
66
        periodSeconds: 5
67
        timeoutSeconds: 30
68
        failureThreshold: 1
69
      readinessProbe:
70
        httpGet:
71
          path: /health
72
          port: 9090
73
        periodSeconds: 10
74
        timeoutSeconds: 30
75
        failureThreshold: 60
76
      resources:
77
        limits:
78
          gpu: "1"
79
      extraPodSpec:
80
        terminationGracePeriodSeconds: 600
81
        mainContainer:
82
          image: my-registry/tensorrtllm-runtime:my-tag
83
          workingDir: /workspace/
84
          command:
85
            - python3
86
          args:
87
            - -m
88
            - dynamo.trtllm
89
            - --model-path
90
            - Qwen/Qwen3-0.6B
91
            - --served-model-name
92
            - Qwen/Qwen3-0.6B
93
            - --extra-engine-args
94
            - ./examples/backends/trtllm/engine_configs/qwen3/decode.yaml
95
            - --disaggregation-mode
96
            - decode
97
    prefill:
98
      envFromSecret: hf-token-secret
99
      componentType: worker
100
      subComponentType: prefill
101
      replicas: 1
102
      resources:
103
        limits:
104
          gpu: "1"
105
      extraPodSpec:
106
        terminationGracePeriodSeconds: 600
107
        mainContainer:
108
          image: my-registry/tensorrtllm-runtime:my-tag
109
          workingDir: /workspace/
110
          command:
111
            - python3
112
          args:
113
            - -m
114
            - dynamo.trtllm
115
            - --model-path
116
            - Qwen/Qwen3-0.6B
117
            - --served-model-name
118
            - Qwen/Qwen3-0.6B
119
            - --extra-engine-args
120
            - ./examples/backends/trtllm/engine_configs/qwen3/prefill.yaml
121
            - --disaggregation-mode
122
            - prefill

1
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
apiVersion: v1
4
kind: ConfigMap
5
metadata:
6
  name: nvidia-config
7
data:
8
  prefill.yaml: |
9
    # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
10
    # SPDX-License-Identifier: Apache-2.0
11
    #
12
    # Licensed under the Apache License, Version 2.0 (the "License");
13
    # you may not use this file except in compliance with the License.
14
    # You may obtain a copy of the License at
15
    #
16
    # http://www.apache.org/licenses/LICENSE-2.0
17
    #
18
    # Unless required by applicable law or agreed to in writing, software
19
    # distributed under the License is distributed on an "AS IS" BASIS,
20
    # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21
    # See the License for the specific language governing permissions and
22
    # limitations under the License.
23
    tensor_parallel_size: 8
24
    moe_expert_parallel_size: 1
25
    enable_attention_dp: false
26
    max_num_tokens: 8192
27
    trust_remote_code: true
28
    backend: pytorch
29
    enable_chunked_prefill: true
30
    # Overlap scheduler not currently supported in prefill only workers.
31
    disable_overlap_scheduler: true
32
    kv_cache_config:
33
      free_gpu_memory_fraction: 0.80
34
    cache_transceiver_config:
35
      backend: DEFAULT
36
37
  decode.yaml: |
38
    # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
39
    # SPDX-License-Identifier: Apache-2.0
40
    #
41
    # Licensed under the Apache License, Version 2.0 (the "License");
42
    # you may not use this file except in compliance with the License.
43
    # You may obtain a copy of the License at
44
    #
45
    # http://www.apache.org/licenses/LICENSE-2.0
46
    #
47
    # Unless required by applicable law or agreed to in writing, software
48
    # distributed under the License is distributed on an "AS IS" BASIS,
49
    # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
50
    # See the License for the specific language governing permissions and
51
    # limitations under the License.
52
    tensor_parallel_size: 8
53
    moe_expert_parallel_size: 1
54
    enable_attention_dp: false
55
    max_num_tokens: 8192
56
    trust_remote_code: true
57
    backend: pytorch
58
    enable_chunked_prefill: true
59
    disable_overlap_scheduler: false
60
    kv_cache_config:
61
      free_gpu_memory_fraction: 0.80
62
    cache_transceiver_config:
63
      backend: DEFAULT
64
---
65
apiVersion: v1
66
kind: PersistentVolumeClaim
67
metadata:
68
  name: models
69
spec:
70
  accessModes:
71
    - ReadWriteMany
72
  resources:
73
    requests:
74
      storage: 100Gi
75
---
76
apiVersion: nvidia.com/v1alpha1
77
kind: DynamoGraphDeployment
78
metadata:
79
  name: trtllm-disagg-tp8
80
spec:
81
  backendFramework: trtllm
82
  pvcs:
83
    - name: models
84
  envs:
85
    - name: OMPI_ALLOW_RUN_AS_ROOT
86
      value: "1"
87
    - name: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM
88
      value: "1"
89
    - name: HF_HOME
90
      value: "/models"
91
  services:
92
    Frontend:
93
      volumeMounts:
94
        - name: models
95
          mountPoint: /models
96
      componentType: frontend
97
      replicas: 1
98
      extraPodSpec:
99
        mainContainer:
100
          image: my-registry/tensorrtllm-runtime:my-tag
101
          workingDir: /workspace/examples/backends/trtllm
102
          command:
103
          - python3
104
          - -m
105
          - dynamo.frontend
106
          args:
107
            - --http-port
108
            - "8000"
109
    prefill:
110
      volumeMounts:
111
        - name: models
112
          mountPoint: /models
113
      envFromSecret: hf-token-secret
114
      componentType: worker
115
      replicas: 1
116
      multinode:
117
        nodeCount: 2
118
      resources:
119
        limits:
120
          gpu: "4"
121
      extraPodSpec:
122
        volumes:
123
          - name: nvidia-config
124
            configMap:
125
              name: nvidia-config
126
        mainContainer:
127
          volumeMounts:
128
            - name: nvidia-config
129
              mountPath: /workspace/
130
              readOnly: true
131
          image: my-registry/tensorrtllm-runtime:my-tag
132
          workingDir: /workspace/
133
          command:
134
          - python3
135
          - -m
136
          - dynamo.trtllm
137
          args:
138
            - --model-path
139
            - Qwen/Qwen3-0.6B
140
            - --served-model-name
141
            - Qwen/Qwen3-0.6B
142
            - --extra-engine-args
143
            - /workspace/prefill.yaml
144
            - --disaggregation-mode
145
            - prefill
146
    decode:
147
      volumeMounts:
148
        - name: models
149
          mountPoint: /models
150
      envFromSecret: hf-token-secret
151
      componentType: worker
152
      replicas: 1
153
      multinode:
154
        nodeCount: 2
155
      resources:
156
        limits:
157
          gpu: "4"
158
      extraPodSpec:
159
        volumes:
160
          - name: nvidia-config
161
            configMap:
162
              name: nvidia-config
163
        mainContainer:
164
          volumeMounts:
165
            - name: nvidia-config
166
              mountPath: /workspace/
167
              readOnly: true
168
          image: my-registry/tensorrtllm-runtime:my-tag
169
          workingDir: /workspace/
170
          command:
171
          - python3
172
          - -m
173
          - dynamo.trtllm
174
          args:
175
            - --model-path
176
            - Qwen/Qwen3-0.6B
177
            - --served-model-name
178
            - Qwen/Qwen3-0.6B
179
            - --extra-engine-args
180
            - /workspace/decode.yaml
181
            - --disaggregation-mode
182
            - decode