NVIDIA DeepStream SDK API Reference

9.2 Release
sources/gst-plugins/gst-nvdsvideotemplate/ds-sparse4d/infer/tensor_rt_engine.h
Go to the documentation of this file.
1 /*
2  * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3  * SPDX-License-Identifier: Apache-2.0
4  *
5  * Licensed under the Apache License, Version 2.0 (the "License");
6  * you may not use this file except in compliance with the License.
7  * You may obtain a copy of the License at
8  *
9  * http://www.apache.org/licenses/LICENSE-2.0
10  *
11  * Unless required by applicable law or agreed to in writing, software
12  * distributed under the License is distributed on an "AS IS" BASIS,
13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  * See the License for the specific language governing permissions and
15  * limitations under the License.
16  */
17 
18 #ifndef TENSOR_RT_ENGINE_H
19 #define TENSOR_RT_ENGINE_H
20 
21 #include <string>
22 #include <vector>
23 #include <memory>
24 #include <fstream>
25 #include <iostream>
26 #include <iomanip>
27 #include <sstream>
28 #include <NvInfer.h>
29 #include <cuda_runtime_api.h>
30 #include "../utils/log_utils.h"
31 #include "../utils/tensor.h"
32 #include "../utils/config_properties.h"
33 
40 public:
45 
50 
56  bool isInitialized() const { return m_Initialized; }
57 
63  void configure(const ConfigProperties& config) {
64  m_GpuId = config.gpu_id;
65  m_LoggerSeverity = config.trt_logger_severity;
66  m_EnableProfiler = config.enable_profiler;
67  }
68 
74  int getGpuId() const { return m_GpuId; }
75 
81  nvinfer1::ICudaEngine* getEngine() const { return m_Engine; }
82 
88  cudaStream_t getCudaStream() const { return m_CudaStream; }
89 
96  int getTensorBindingIndex(const std::string& name) const {
97  if (!m_Initialized || !m_Engine) {
98  return -1;
99  }
100 
101  for (int i = 0; i < static_cast<int>(m_Engine->getNbIOTensors()); i++) {
102  if (name == m_Engine->getIOTensorName(i)) {
103  return i;
104  }
105  }
106  return -1;
107  }
108 
115  nvinfer1::Dims getBindingDimensions(int index) const {
116  if (m_Engine && index >= 0 && index < static_cast<int>(m_Engine->getNbIOTensors())) {
117  return m_Engine->getTensorShape(m_Engine->getIOTensorName(index));
118  }
119  nvinfer1::Dims emptyDims;
120  emptyDims.nbDims = 0;
121  return emptyDims;
122  }
123 
130  void* getDeviceBuffer(const std::string& name) const {
131  int idx = getTensorBindingIndex(name);
132  if (idx >= 0 && static_cast<size_t>(idx) < m_DeviceBuffers.size()) {
133  return m_DeviceBuffers[idx];
134  }
135  return nullptr;
136  }
137 
143  std::vector<std::string> getInputTensorNames() const {
144  if (!m_Initialized || !m_Engine) {
145  return std::vector<std::string>();
146  }
147 
148  // Return cached tensor names if available
149  if (!m_InputTensorNames.empty()) {
150  return m_InputTensorNames;
151  }
152 
153  // Otherwise, build the list
154  std::vector<std::string> inputTensorNames;
155  int nbIOTensors = m_Engine->getNbIOTensors();
156 
157  for (int i = 0; i < nbIOTensors; i++) {
158  const char* tensorName = m_Engine->getIOTensorName(i);
159  nvinfer1::TensorIOMode tensorMode = m_Engine->getTensorIOMode(tensorName);
160 
161  if (tensorMode == nvinfer1::TensorIOMode::kINPUT) {
162  inputTensorNames.push_back(tensorName);
163  }
164  }
165 
166  // Cache the results
167  const_cast<TensorRTEngine*>(this)->m_InputTensorNames = inputTensorNames;
168  return inputTensorNames;
169  }
170 
176  std::vector<std::string> getOutputTensorNames() const {
177  if (!m_Initialized || !m_Engine) {
178  return std::vector<std::string>();
179  }
180 
181  // Return cached tensor names if available
182  if (!m_OutputTensorNames.empty()) {
183  return m_OutputTensorNames;
184  }
185 
186  // Otherwise, build the list
187  std::vector<std::string> outputTensorNames;
188  int nbIOTensors = m_Engine->getNbIOTensors();
189 
190  for (int i = 0; i < nbIOTensors; i++) {
191  const char* tensorName = m_Engine->getIOTensorName(i);
192  nvinfer1::TensorIOMode tensorMode = m_Engine->getTensorIOMode(tensorName);
193 
194  if (tensorMode == nvinfer1::TensorIOMode::kOUTPUT) {
195  outputTensorNames.push_back(tensorName);
196  }
197  }
198 
199  // Cache the results
200  const_cast<TensorRTEngine*>(this)->m_OutputTensorNames = outputTensorNames;
201  return outputTensorNames;
202  }
203 
215  bool updateBindingPointer(const std::string& tensorName, void* newBuffer) {
216  if (!m_Initialized || !m_Engine || !m_Context) {
217  return false;
218  }
219 
220  int bindingIdx = getTensorBindingIndex(tensorName);
221  if (bindingIdx < 0) {
222  return false;
223  }
224 
225  // // Free old buffer memory to prevent memory leaks
226  // void* oldBuffer = m_DeviceBuffers[bindingIdx];
227  // if (oldBuffer != nullptr && oldBuffer != newBuffer) {
228  // cudaError_t err = cudaFree(oldBuffer);
229  // if (err != cudaSuccess) {
230  // // Log error but continue with update
231  // std::cerr << "Warning: Failed to free old buffer for tensor '"
232  // << tensorName << "': " << cudaGetErrorString(err) << std::endl;
233  // }
234  // }
235 
236  m_DeviceBuffers[bindingIdx] = newBuffer;
237  m_Context->setTensorAddress(tensorName.c_str(), newBuffer);
238  return true;
239  }
240 
251  bool deserializeEngine(const std::string& engineFilePath, TensorCache* tensorCache, int batchSize, int numSensors, PrettyLogger* logger = nullptr);
252 
261  bool runInference(int batch_size, int num_sensors, PrettyLogger* logger = nullptr);
262 
263 private:
264  // TensorRT logger
265  class Logger : public nvinfer1::ILogger {
266  public:
267  Logger(TensorRTEngine* parent) : m_Parent(parent) {}
268  void log(Severity severity, const char* msg) noexcept override;
269  private:
270  TensorRTEngine* m_Parent;
271  };
272 
273  // Simple profiler for runtime information
274  class SimpleProfiler : public nvinfer1::IProfiler {
275  public:
276  void reportLayerTime(const char* layerName, float timeMs) noexcept override {
277  std::cout << "TensorRT [PROFILE] Layer: " << layerName
278  << " - Time: " << std::fixed << std::setprecision(3)
279  << timeMs << " ms" << std::endl;
280  }
281  };
282 
283  // TensorRT engine and execution context
284  nvinfer1::ICudaEngine* m_Engine;
285  nvinfer1::IExecutionContext* m_Context;
286 
287  // TensorRT runtime objects
288  nvinfer1::IRuntime* m_Runtime;
289  Logger m_Logger{this}; // Initialize with parent pointer
290  SimpleProfiler m_Profiler;
291 
292  // Device buffers for input/output tensors
293  std::vector<void*> m_DeviceBuffers;
294 
295  // CUDA stream for asynchronous operations
296  cudaStream_t m_CudaStream;
297 
298  // GPU ID to use for inference
299  int m_GpuId;
300 
301  // Flag to track initialization status
302  bool m_Initialized;
303 
304  // Cached input and output tensor names
305  std::vector<std::string> m_InputTensorNames;
306  std::vector<std::string> m_OutputTensorNames;
307 
308  // Configuration properties for logger and profiler
309  int m_LoggerSeverity; // Logger severity level
310  bool m_EnableProfiler; // Whether to enable profiler
311 
318  size_t getElementSize(nvinfer1::DataType dataType) const {
319  switch (dataType) {
320  case nvinfer1::DataType::kFLOAT:
321  return 4; // 4 bytes for float
322  case nvinfer1::DataType::kHALF:
323  return 2; // 2 bytes for half
324  case nvinfer1::DataType::kINT8:
325  return 1; // 1 byte for int8
326  case nvinfer1::DataType::kINT32:
327  return 4; // 4 bytes for int32
328  case nvinfer1::DataType::kBOOL:
329  return 1; // 1 byte for bool
330  default:
331  return 4; // Default to 4 bytes
332  }
333  }
334 
341  std::string getDataTypeString(nvinfer1::DataType dataType) const {
342  switch (dataType) {
343  case nvinfer1::DataType::kFLOAT: return "FP32";
344  case nvinfer1::DataType::kHALF: return "FP16";
345  case nvinfer1::DataType::kINT8: return "INT8";
346  case nvinfer1::DataType::kINT32: return "INT32";
347  case nvinfer1::DataType::kBOOL: return "BOOL";
348  default: return "UNKNOWN";
349  }
350  }
351 
362  std::vector<size_t> printTensorInfo(int nbIOTensors, TensorCache* tensorCache, int batchSize, int numSensors, PrettyLogger* logger);
363 
364 };
365 
366 #endif // TENSOR_RT_ENGINE_H
TensorRTEngine::runInference
bool runInference(int batch_size, int num_sensors, PrettyLogger *logger=nullptr)
Run inference using pre-allocated output tensors.
TensorRTEngine::getEngine
nvinfer1::ICudaEngine * getEngine() const
Get the TensorRT engine object.
Definition: sources/gst-plugins/gst-nvdsvideotemplate/ds-sparse4d/infer/tensor_rt_engine.h:81
PrettyLogger
Definition: sources/gst-plugins/gst-nvdsvideotemplate/ds-sparse4d/utils/log_utils.h:57
TensorRTEngine::getGpuId
int getGpuId() const
Get the GPU ID used for inference.
Definition: sources/gst-plugins/gst-nvdsvideotemplate/ds-sparse4d/infer/tensor_rt_engine.h:74
TensorCache
Class to manage multiple tensors with string-based lookup.
Definition: sources/gst-plugins/gst-nvdsvideotemplate/ds-sparse4d/utils/tensor.h:511
ds3d::DataType
DataType
Definition: includes/ds3d/common/idatatype.h:82
TensorRTEngine::deserializeEngine
bool deserializeEngine(const std::string &engineFilePath, TensorCache *tensorCache, int batchSize, int numSensors, PrettyLogger *logger=nullptr)
Deserialize a TensorRT engine from a file.
cudaStream_t
struct CUstream_st * cudaStream_t
Forward declaration of cudaStream_t.
Definition: includes/nvbufsurftransform.h:40
TensorRTEngine::getDeviceBuffer
void * getDeviceBuffer(const std::string &name) const
Get device buffer for a tensor by name.
Definition: sources/gst-plugins/gst-nvdsvideotemplate/ds-sparse4d/infer/tensor_rt_engine.h:130
ConfigProperties::enable_profiler
bool enable_profiler
Definition: sources/gst-plugins/gst-nvdsvideotemplate/ds-sparse4d/utils/config_properties.h:113
half_float::log
half log(half arg)
Natural logarithm.
Definition: includes/half.hpp:3067
TensorRTEngine::getInputTensorNames
std::vector< std::string > getInputTensorNames() const
Get all input tensor names.
Definition: sources/gst-plugins/gst-nvdsvideotemplate/ds-sparse4d/infer/tensor_rt_engine.h:143
ConfigProperties
Structure containing all configurable properties from the YAML config file.
Definition: sources/gst-plugins/gst-nvdsvideotemplate/ds-sparse4d/utils/config_properties.h:32
TensorRTEngine::getTensorBindingIndex
int getTensorBindingIndex(const std::string &name) const
Get binding index for a tensor by name.
Definition: sources/gst-plugins/gst-nvdsvideotemplate/ds-sparse4d/infer/tensor_rt_engine.h:96
TensorRTEngine::configure
void configure(const ConfigProperties &config)
Configure the TensorRT engine with properties from config.
Definition: sources/gst-plugins/gst-nvdsvideotemplate/ds-sparse4d/infer/tensor_rt_engine.h:63
TensorRTEngine::TensorRTEngine
TensorRTEngine()
Constructor.
TensorRTEngine::getCudaStream
cudaStream_t getCudaStream() const
Get CUDA stream.
Definition: sources/gst-plugins/gst-nvdsvideotemplate/ds-sparse4d/infer/tensor_rt_engine.h:88
logger
Logger logger
TensorRTEngine::updateBindingPointer
bool updateBindingPointer(const std::string &tensorName, void *newBuffer)
Updates the binding pointer for a specific tensor.
Definition: sources/gst-plugins/gst-nvdsvideotemplate/ds-sparse4d/infer/tensor_rt_engine.h:215
logs::Severity
Severity
Severity Levels assigned to log events, based on Log Data Model, with the addition of kInvalid (mappe...
Definition: includes/opentelemetry/logs/severity.h:22
ConfigProperties::gpu_id
int gpu_id
Definition: sources/gst-plugins/gst-nvdsvideotemplate/ds-sparse4d/utils/config_properties.h:45
Logger
Definition: sources/gst-plugins/gst-nvdsvideotemplate/ds-sparse4d/infer/engine_builder.h:50
TensorRTEngine
TensorRT inference engine class.
Definition: sources/gst-plugins/gst-nvdsvideotemplate/ds-sparse4d/infer/tensor_rt_engine.h:39
TensorRTEngine::isInitialized
bool isInitialized() const
Check if the engine is initialized.
Definition: sources/gst-plugins/gst-nvdsvideotemplate/ds-sparse4d/infer/tensor_rt_engine.h:56
TensorRTEngine::getOutputTensorNames
std::vector< std::string > getOutputTensorNames() const
Get all output tensor names.
Definition: sources/gst-plugins/gst-nvdsvideotemplate/ds-sparse4d/infer/tensor_rt_engine.h:176
TensorRTEngine::~TensorRTEngine
~TensorRTEngine()
Destructor - frees allocated resources.
TensorRTEngine::getBindingDimensions
nvinfer1::Dims getBindingDimensions(int index) const
Get binding dimensions for a tensor by index.
Definition: sources/gst-plugins/gst-nvdsvideotemplate/ds-sparse4d/infer/tensor_rt_engine.h:115
ConfigProperties::trt_logger_severity
int trt_logger_severity
Definition: sources/gst-plugins/gst-nvdsvideotemplate/ds-sparse4d/utils/config_properties.h:112