18 #ifndef TENSOR_RT_ENGINE_H
19 #define TENSOR_RT_ENGINE_H
29 #include <cuda_runtime_api.h>
30 #include "../utils/log_utils.h"
31 #include "../utils/tensor.h"
32 #include "../utils/config_properties.h"
81 nvinfer1::ICudaEngine*
getEngine()
const {
return m_Engine; }
97 if (!m_Initialized || !m_Engine) {
101 for (
int i = 0; i < static_cast<int>(m_Engine->getNbIOTensors()); i++) {
102 if (name == m_Engine->getIOTensorName(i)) {
116 if (m_Engine && index >= 0 && index <
static_cast<int>(m_Engine->getNbIOTensors())) {
117 return m_Engine->getTensorShape(m_Engine->getIOTensorName(index));
119 nvinfer1::Dims emptyDims;
120 emptyDims.nbDims = 0;
132 if (idx >= 0 &&
static_cast<size_t>(idx) < m_DeviceBuffers.size()) {
133 return m_DeviceBuffers[idx];
144 if (!m_Initialized || !m_Engine) {
145 return std::vector<std::string>();
149 if (!m_InputTensorNames.empty()) {
150 return m_InputTensorNames;
154 std::vector<std::string> inputTensorNames;
155 int nbIOTensors = m_Engine->getNbIOTensors();
157 for (
int i = 0; i < nbIOTensors; i++) {
158 const char* tensorName = m_Engine->getIOTensorName(i);
159 nvinfer1::TensorIOMode tensorMode = m_Engine->getTensorIOMode(tensorName);
161 if (tensorMode == nvinfer1::TensorIOMode::kINPUT) {
162 inputTensorNames.push_back(tensorName);
167 const_cast<TensorRTEngine*
>(
this)->m_InputTensorNames = inputTensorNames;
168 return inputTensorNames;
177 if (!m_Initialized || !m_Engine) {
178 return std::vector<std::string>();
182 if (!m_OutputTensorNames.empty()) {
183 return m_OutputTensorNames;
187 std::vector<std::string> outputTensorNames;
188 int nbIOTensors = m_Engine->getNbIOTensors();
190 for (
int i = 0; i < nbIOTensors; i++) {
191 const char* tensorName = m_Engine->getIOTensorName(i);
192 nvinfer1::TensorIOMode tensorMode = m_Engine->getTensorIOMode(tensorName);
194 if (tensorMode == nvinfer1::TensorIOMode::kOUTPUT) {
195 outputTensorNames.push_back(tensorName);
200 const_cast<TensorRTEngine*
>(
this)->m_OutputTensorNames = outputTensorNames;
201 return outputTensorNames;
216 if (!m_Initialized || !m_Engine || !m_Context) {
221 if (bindingIdx < 0) {
236 m_DeviceBuffers[bindingIdx] = newBuffer;
237 m_Context->setTensorAddress(tensorName.c_str(), newBuffer);
265 class Logger :
public nvinfer1::ILogger {
268 void log(
Severity severity,
const char* msg) noexcept
override;
274 class SimpleProfiler :
public nvinfer1::IProfiler {
276 void reportLayerTime(
const char* layerName,
float timeMs) noexcept
override {
277 std::cout <<
"TensorRT [PROFILE] Layer: " << layerName
278 <<
" - Time: " << std::fixed << std::setprecision(3)
279 << timeMs <<
" ms" << std::endl;
284 nvinfer1::ICudaEngine* m_Engine;
285 nvinfer1::IExecutionContext* m_Context;
288 nvinfer1::IRuntime* m_Runtime;
290 SimpleProfiler m_Profiler;
293 std::vector<void*> m_DeviceBuffers;
305 std::vector<std::string> m_InputTensorNames;
306 std::vector<std::string> m_OutputTensorNames;
309 int m_LoggerSeverity;
310 bool m_EnableProfiler;
320 case nvinfer1::DataType::kFLOAT:
322 case nvinfer1::DataType::kHALF:
324 case nvinfer1::DataType::kINT8:
326 case nvinfer1::DataType::kINT32:
328 case nvinfer1::DataType::kBOOL:
343 case nvinfer1::DataType::kFLOAT:
return "FP32";
344 case nvinfer1::DataType::kHALF:
return "FP16";
345 case nvinfer1::DataType::kINT8:
return "INT8";
346 case nvinfer1::DataType::kINT32:
return "INT32";
347 case nvinfer1::DataType::kBOOL:
return "BOOL";
348 default:
return "UNKNOWN";
366 #endif // TENSOR_RT_ENGINE_H