NVIDIA DeepStream SDK API Reference
9.2 Release
sources/libs/nvmultiobjecttracker/include/modules/Segmenter/SegmenterProfile.hpp
Go to the documentation of this file.
1 /*
2  * SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3  * SPDX-License-Identifier: Apache-2.0
4  *
5  * Licensed under the Apache License, Version 2.0 (the "License");
6  * you may not use this file except in compliance with the License.
7  * You may obtain a copy of the License at
8  *
9  * http://www.apache.org/licenses/LICENSE-2.0
10  *
11  * Unless required by applicable law or agreed to in writing, software
12  * distributed under the License is distributed on an "AS IS" BASIS,
13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  * See the License for the specific language governing permissions and
15  * limitations under the License.
16  */
17 
18 #include <NvInfer.h>
19 
27 static void createOnnxProfile4NanoSamDecoder(
28  nvinfer1::INetworkDefinition *network, nvinfer1::IBuilder *builder, nvinfer1::IBuilderConfig *config, int batchSize
29 )
30 {
31  nvinfer1::IOptimizationProfile* profile = builder->createOptimizationProfile();
32 
33  nvinfer1::ITensor* input = network->getInput(1);
34  const char* inputNamePts = input->getName();
35  profile->setDimensions(inputNamePts, nvinfer1::OptProfileSelector::kMIN, nvinfer1::Dims3(1, 1, 2));
36  profile->setDimensions(inputNamePts, nvinfer1::OptProfileSelector::kOPT, nvinfer1::Dims3(1, 1, 2));
37  profile->setDimensions(inputNamePts, nvinfer1::OptProfileSelector::kMAX, nvinfer1::Dims3(1, 10, 2));
38 
39  input = network->getInput(2);
40  const char* inputNamePtLabels = input->getName();
41  profile->setDimensions(inputNamePtLabels, nvinfer1::OptProfileSelector::kMIN, nvinfer1::Dims2(1, 1));
42  profile->setDimensions(inputNamePtLabels, nvinfer1::OptProfileSelector::kOPT, nvinfer1::Dims2(1, 1));
43  profile->setDimensions(inputNamePtLabels, nvinfer1::OptProfileSelector::kMAX, nvinfer1::Dims2(1, 10));
44 
45  config->addOptimizationProfile(profile);
46 }
47 
53 static void setRuntimeDim4NanoSamDecoder(TrtModule *trtModule, const std::vector<int>& sizes)
54 {
55  trtModule->m_context->setOptimizationProfileAsync(0, trtModule->m_cuStream);
56  trtModule->m_context->setInputShape(trtModule->m_engine->getIOTensorName(1), nvinfer1::Dims3(1, 2, 2));
57  trtModule->m_context->setInputShape(trtModule->m_engine->getIOTensorName(2), nvinfer1::Dims2(1, 2));
58 }
59 
67 static void createOnnxProfile4EVSamEncoder(
68  nvinfer1::INetworkDefinition *network, nvinfer1::IBuilder *builder, nvinfer1::IBuilderConfig *config, int batchSize
69 )
70 {
71  nvinfer1::IOptimizationProfile* profile = builder->createOptimizationProfile();
72 
73  nvinfer1::ITensor* input = network->getInput(0);
74 
75  const char* inputName = input->getName();
76  nvinfer1::Dims inputDim = input->getDimensions();
77  inputDim.d[0] = 1;
78  profile->setDimensions(inputName, nvinfer1::OptProfileSelector::kMIN, inputDim);
79  inputDim.d[0] = batchSize;
80  profile->setDimensions(inputName, nvinfer1::OptProfileSelector::kOPT, inputDim);
81  profile->setDimensions(inputName, nvinfer1::OptProfileSelector::kMAX, inputDim);
82 
83  config->addOptimizationProfile(profile);
84 }
85 
91 static void setRuntimeDim4EVSamEncoder(TrtModule *trtModule, const std::vector<int>& sizes)
92 {
93  for (int ind = 0; ind < trtModule->m_engine->getNbIOTensors(); ++ind)
94  {
95  if (trtModule->m_engine->getTensorIOMode(trtModule->m_engine->getIOTensorName(ind)) == nvinfer1::TensorIOMode::kINPUT)
96  {
97  nvinfer1::Dims lastInputDims = trtModule->m_context->getTensorShape(
98  trtModule->m_engine->getIOTensorName(ind)
99  );
100  lastInputDims.d[0] = 1;
101  trtModule->m_context->setInputShape(trtModule->m_engine->getIOTensorName(ind), lastInputDims);
102  }
103  }
104 }
105 
113 static void createOnnxProfile4EVSamDecoder(
114  nvinfer1::INetworkDefinition *network, nvinfer1::IBuilder *builder, nvinfer1::IBuilderConfig *config, int batchSize
115 )
116 {
117  nvinfer1::IOptimizationProfile* profile = builder->createOptimizationProfile();
118  int maxPointNum = batchSize * 2;
119  nvinfer1::ITensor* input = network->getInput(1);
120  const char* inputNamePts = input->getName();
121  profile->setDimensions(inputNamePts, nvinfer1::OptProfileSelector::kMIN, nvinfer1::Dims2(1, 2 * 2));
122  profile->setDimensions(inputNamePts, nvinfer1::OptProfileSelector::kOPT, nvinfer1::Dims2(1, maxPointNum * 2));
123  profile->setDimensions(inputNamePts, nvinfer1::OptProfileSelector::kMAX, nvinfer1::Dims2(1, maxPointNum * 2));
124 
125  input = network->getInput(2);
126  const char* inputNamePtLabels = input->getName();
127  profile->setDimensions(inputNamePtLabels, nvinfer1::OptProfileSelector::kMIN, nvinfer1::Dims2(1, 2));
128  profile->setDimensions(inputNamePtLabels, nvinfer1::OptProfileSelector::kOPT, nvinfer1::Dims2(1, maxPointNum));
129  profile->setDimensions(inputNamePtLabels, nvinfer1::OptProfileSelector::kMAX, nvinfer1::Dims2(1, maxPointNum));
130 
131  config->addOptimizationProfile(profile);
132 }
133 
139 static void setRuntimeDim4EVSamDecoder(TrtModule *trtModule, const std::vector<int>& sizes)
140 {
141  trtModule->m_context->setOptimizationProfileAsync(0, trtModule->m_cuStream);
142  trtModule->m_context->setInputShape(trtModule->m_engine->getIOTensorName(1), nvinfer1::Dims2(1, 2 * 2));
143  trtModule->m_context->setInputShape(trtModule->m_engine->getIOTensorName(2), nvinfer1::Dims2(1, 2));
144 }
145 
153 static void createOnnxProfile4Sam2MaskDecoder(
154  nvinfer1::INetworkDefinition *network, nvinfer1::IBuilder *builder, nvinfer1::IBuilderConfig *config, int batchSize
155 )
156 {
157  nvinfer1::IOptimizationProfile* profile = builder->createOptimizationProfile();
158 
159  nvinfer1::ITensor* input = network->getInput(0);
160  const char* inputNamePts = input->getName();
161  profile->setDimensions(inputNamePts, nvinfer1::OptProfileSelector::kMIN, nvinfer1::Dims3(1, 1, 2));
162  profile->setDimensions(inputNamePts, nvinfer1::OptProfileSelector::kOPT, nvinfer1::Dims3(batchSize, 2, 2));
163  profile->setDimensions(inputNamePts, nvinfer1::OptProfileSelector::kMAX, nvinfer1::Dims3(batchSize, 2, 2));
164 
165  input = network->getInput(1);
166  const char* inputNamePtLabels = input->getName();
167  profile->setDimensions(inputNamePtLabels, nvinfer1::OptProfileSelector::kMIN, nvinfer1::Dims2(1, 1));
168  profile->setDimensions(inputNamePtLabels, nvinfer1::OptProfileSelector::kOPT, nvinfer1::Dims2(batchSize, 2));
169  profile->setDimensions(inputNamePtLabels, nvinfer1::OptProfileSelector::kMAX, nvinfer1::Dims2(batchSize, 2));
170 
171  input = network->getInput(2);
172  const char* inputNamePixFeatWithMem = input->getName();
173  profile->setDimensions(
174  inputNamePixFeatWithMem, nvinfer1::OptProfileSelector::kMIN, nvinfer1::Dims4(1, 256, 64, 64)
175  );
176  profile->setDimensions(
177  inputNamePixFeatWithMem, nvinfer1::OptProfileSelector::kOPT, nvinfer1::Dims4(batchSize, 256, 64, 64)
178  );
179  profile->setDimensions(
180  inputNamePixFeatWithMem, nvinfer1::OptProfileSelector::kMAX, nvinfer1::Dims4(batchSize, 256, 64, 64)
181  );
182 
183  config->addOptimizationProfile(profile);
184 }
185 
191 static void setRuntimeDim4Sam2MaskDecoder(TrtModule *trtModule, const std::vector<int>& sizes)
192 {
193  int pointNum = sizes.at(0);
194  int batchSize = (sizes.size() >= 2) ? sizes.at(1) : 1;
195  trtModule->m_context->setOptimizationProfileAsync(0, trtModule->m_cuStream);
196  trtModule->m_context->setInputShape(
197  trtModule->m_engine->getIOTensorName(0), nvinfer1::Dims3(batchSize, pointNum, 2)
198  );
199  trtModule->m_context->setInputShape(trtModule->m_engine->getIOTensorName(1), nvinfer1::Dims2(batchSize, pointNum));
200  trtModule->m_context->setInputShape(
201  trtModule->m_engine->getIOTensorName(2), nvinfer1::Dims4(batchSize, 256, 64, 64)
202  );
203 }
204 
212 static void createOnnxProfile4Sam2MemAttn(
213  nvinfer1::INetworkDefinition *network, nvinfer1::IBuilder *builder, nvinfer1::IBuilderConfig *config, int batchSize
214 )
215 {
216  nvinfer1::IOptimizationProfile* profile = builder->createOptimizationProfile();
217  int numObjPtr = 16, numMaskMem = 7;
219  nvinfer1::ITensor* input = network->getInput(2);
220  const char* inputName2 = input->getName();
221  profile->setDimensions(inputName2, nvinfer1::OptProfileSelector::kMIN, nvinfer1::Dims3(1, 1, 256));
222  profile->setDimensions(inputName2, nvinfer1::OptProfileSelector::kOPT, nvinfer1::Dims3(batchSize, numObjPtr, 256));
223  profile->setDimensions(inputName2, nvinfer1::OptProfileSelector::kMAX, nvinfer1::Dims3(batchSize, numObjPtr, 256));
224 
226  input = network->getInput(3);
227  const char* inputName3 = input->getName();
228  nvinfer1::Dims inputDim = {5, {1,1,64,64,64}};
229  profile->setDimensions(inputName3, nvinfer1::OptProfileSelector::kMIN, inputDim);
230  inputDim.d[0] = batchSize, inputDim.d[1] = numMaskMem;
231  profile->setDimensions(inputName3, nvinfer1::OptProfileSelector::kOPT, inputDim);
232  inputDim.d[0] = batchSize, inputDim.d[1] = numMaskMem;
233  profile->setDimensions(inputName3, nvinfer1::OptProfileSelector::kMAX, inputDim);
234 
237  input = network->getInput(4);
238  const char* inputName4 = input->getName();
239  profile->setDimensions(inputName4, nvinfer1::OptProfileSelector::kMIN, nvinfer1::Dims3(1, 4100, 64));
240  profile->setDimensions(
241  inputName4, nvinfer1::OptProfileSelector::kOPT, nvinfer1::Dims3(batchSize, numMaskMem*4096 + 4*numObjPtr, 64)
242  );
243  profile->setDimensions(
244  inputName4, nvinfer1::OptProfileSelector::kMAX, nvinfer1::Dims3(batchSize, numMaskMem*4096 + 4*numObjPtr, 64)
245  );
246 
247  config->addOptimizationProfile(profile);
248 }
249 
255 static void setRuntimeDim4Sam2MemAttn(TrtModule *trtModule, const std::vector<int>& sizes)
256 {
257  int batchSize = sizes.at(0), numObjPtrs = sizes.at(1), numMaskMem = sizes.at(2), numFrames = sizes.at(3);
258  trtModule->m_context->setOptimizationProfileAsync(0, trtModule->m_cuStream);
259  trtModule->m_context->setInputShape(
260  trtModule->m_engine->getIOTensorName(2), nvinfer1::Dims3(batchSize, numObjPtrs, 256)
261  );
262  nvinfer1::Dims inputDim = {5, {batchSize,numMaskMem,64,64,64}};
263  trtModule->m_context->setInputShape(trtModule->m_engine->getIOTensorName(3), inputDim);
264  trtModule->m_context->setInputShape(
265  trtModule->m_engine->getIOTensorName(4), nvinfer1::Dims3(batchSize, numMaskMem*4096 + 4*numFrames, 64)
266  );
267 }
268 
276 static void createOnnxProfile4Sam2MemEncoder(
277  nvinfer1::INetworkDefinition *network, nvinfer1::IBuilder *builder, nvinfer1::IBuilderConfig *config, int batchSize
278 )
279 {
280  nvinfer1::IOptimizationProfile* profile = builder->createOptimizationProfile();
281 
282  nvinfer1::ITensor* input = network->getInput(0);
283  const char* inputNameMaskForMem = input->getName();
284  profile->setDimensions(inputNameMaskForMem, nvinfer1::OptProfileSelector::kMIN, nvinfer1::Dims4(1, 1, 1024, 1024));
285  profile->setDimensions(
286  inputNameMaskForMem, nvinfer1::OptProfileSelector::kOPT, nvinfer1::Dims4(batchSize, 1, 1024, 1024)
287  );
288  profile->setDimensions(
289  inputNameMaskForMem, nvinfer1::OptProfileSelector::kMAX, nvinfer1::Dims4(batchSize, 1, 1024, 1024)
290  );
291 
292  input = network->getInput(2);
293  const char* inputNameOccLogit = input->getName();
294  profile->setDimensions(inputNameOccLogit, nvinfer1::OptProfileSelector::kMIN, nvinfer1::Dims2(1, 1));
295  profile->setDimensions(inputNameOccLogit, nvinfer1::OptProfileSelector::kOPT, nvinfer1::Dims2(batchSize, 1));
296  profile->setDimensions(inputNameOccLogit, nvinfer1::OptProfileSelector::kMAX, nvinfer1::Dims2(batchSize, 1));
297  config->addOptimizationProfile(profile);
298 }
299 
305 static void setRuntimeDim4Sam2MemEncoder(TrtModule *trtModule, const std::vector<int>& sizes)
306 {
307  int batchSize = sizes.at(0);
308  trtModule->m_context->setOptimizationProfileAsync(0, trtModule->m_cuStream);
309  trtModule->m_context->setInputShape(
310  trtModule->m_engine->getIOTensorName(0), nvinfer1::Dims4(batchSize, 1, 1024, 1024)
311  );
312  trtModule->m_context->setInputShape(trtModule->m_engine->getIOTensorName(2), nvinfer1::Dims2(batchSize, 1));
313 }
TrtModule::m_context
std::shared_ptr< nvinfer1::IExecutionContext > m_context
TensorRT execution context.
Definition: sources/libs/nvmultiobjecttracker/include/modules/FrameTransform/trtModule.hpp:202
setRuntimeDim4Sam2MemAttn
static void setRuntimeDim4Sam2MemAttn(TrtModule *trtModule, const std::vector< int > &sizes)
Set runtime input shapes for SAM2 memory attention.
Definition: sources/libs/nvmultiobjecttracker/include/modules/Segmenter/SegmenterProfile.hpp:255
createOnnxProfile4NanoSamDecoder
static void createOnnxProfile4NanoSamDecoder(nvinfer1::INetworkDefinition *network, nvinfer1::IBuilder *builder, nvinfer1::IBuilderConfig *config, int batchSize)
Configure dynamic input shapes for the NanoSAM decoder.
Definition: sources/libs/nvmultiobjecttracker/include/modules/Segmenter/SegmenterProfile.hpp:27
createOnnxProfile4Sam2MaskDecoder
static void createOnnxProfile4Sam2MaskDecoder(nvinfer1::INetworkDefinition *network, nvinfer1::IBuilder *builder, nvinfer1::IBuilderConfig *config, int batchSize)
Configure dynamic input shapes for the SAM2 mask decoder.
Definition: sources/libs/nvmultiobjecttracker/include/modules/Segmenter/SegmenterProfile.hpp:153
setRuntimeDim4NanoSamDecoder
static void setRuntimeDim4NanoSamDecoder(TrtModule *trtModule, const std::vector< int > &sizes)
Set runtime input shapes for the NanoSAM decoder.
Definition: sources/libs/nvmultiobjecttracker/include/modules/Segmenter/SegmenterProfile.hpp:53
createOnnxProfile4EVSamEncoder
static void createOnnxProfile4EVSamEncoder(nvinfer1::INetworkDefinition *network, nvinfer1::IBuilder *builder, nvinfer1::IBuilderConfig *config, int batchSize)
Configure dynamic input shapes for the EfficientViT-SAM encoder.
Definition: sources/libs/nvmultiobjecttracker/include/modules/Segmenter/SegmenterProfile.hpp:67
createOnnxProfile4Sam2MemEncoder
static void createOnnxProfile4Sam2MemEncoder(nvinfer1::INetworkDefinition *network, nvinfer1::IBuilder *builder, nvinfer1::IBuilderConfig *config, int batchSize)
Configure dynamic input shapes for the SAM2 memory encoder.
Definition: sources/libs/nvmultiobjecttracker/include/modules/Segmenter/SegmenterProfile.hpp:276
TrtModule
TRT module to perform various deep learning inference.
Definition: sources/libs/nvmultiobjecttracker/include/modules/FrameTransform/trtModule.hpp:137
setRuntimeDim4Sam2MemEncoder
static void setRuntimeDim4Sam2MemEncoder(TrtModule *trtModule, const std::vector< int > &sizes)
Set runtime input shapes for the SAM2 memory encoder.
Definition: sources/libs/nvmultiobjecttracker/include/modules/Segmenter/SegmenterProfile.hpp:305
setRuntimeDim4EVSamEncoder
static void setRuntimeDim4EVSamEncoder(TrtModule *trtModule, const std::vector< int > &sizes)
Set runtime input shapes for the EfficientViT-SAM encoder.
Definition: sources/libs/nvmultiobjecttracker/include/modules/Segmenter/SegmenterProfile.hpp:91
TrtModule::m_cuStream
cudaStream_t m_cuStream
Cuda stream to run all GPU tasks.
Definition: sources/libs/nvmultiobjecttracker/include/modules/FrameTransform/trtModule.hpp:203
setRuntimeDim4Sam2MaskDecoder
static void setRuntimeDim4Sam2MaskDecoder(TrtModule *trtModule, const std::vector< int > &sizes)
Set runtime input shapes for the SAM2 mask decoder.
Definition: sources/libs/nvmultiobjecttracker/include/modules/Segmenter/SegmenterProfile.hpp:191
createOnnxProfile4EVSamDecoder
static void createOnnxProfile4EVSamDecoder(nvinfer1::INetworkDefinition *network, nvinfer1::IBuilder *builder, nvinfer1::IBuilderConfig *config, int batchSize)
Configure dynamic input shapes for the EfficientViT-SAM decoder.
Definition: sources/libs/nvmultiobjecttracker/include/modules/Segmenter/SegmenterProfile.hpp:113
createOnnxProfile4Sam2MemAttn
static void createOnnxProfile4Sam2MemAttn(nvinfer1::INetworkDefinition *network, nvinfer1::IBuilder *builder, nvinfer1::IBuilderConfig *config, int batchSize)
Configure dynamic input shapes for SAM2 memory attention.
Definition: sources/libs/nvmultiobjecttracker/include/modules/Segmenter/SegmenterProfile.hpp:212
setRuntimeDim4EVSamDecoder
static void setRuntimeDim4EVSamDecoder(TrtModule *trtModule, const std::vector< int > &sizes)
Set runtime input shapes for the EfficientViT-SAM decoder.
Definition: sources/libs/nvmultiobjecttracker/include/modules/Segmenter/SegmenterProfile.hpp:139
TrtModule::m_engine
std::shared_ptr< nvinfer1::ICudaEngine > m_engine
TensorRT engine for reid.
Definition: sources/libs/nvmultiobjecttracker/include/modules/FrameTransform/trtModule.hpp:201