|
NVIDIA DeepStream SDK API Reference
|
7.0 Release
|
Go to the documentation of this file.
13 #ifndef __NVDSINFER_MODEL_BUILDER_H__
14 #define __NVDSINFER_MODEL_BUILDER_H__
18 #include <condition_variable>
24 #include <unordered_map>
26 #include <NvCaffeParser.h>
28 #include <NvInferRuntime.h>
29 #include <NvOnnxParser.h>
56 const std::shared_ptr<DlLibHandle>& dllib)
59 virtual bool isValid()
const = 0;
77 const std::shared_ptr<DlLibHandle>& handle =
nullptr);
79 bool isValid()
const override {
return m_CaffeParser.get(); }
80 const char*
getModelName()
const override {
return m_ModelPath.c_str(); }
89 std::string m_ProtoPath;
90 std::string m_ModelPath;
91 std::vector<std::string> m_OutputLayers;
93 UniquePtrWDestroy<nvcaffeparser1::ICaffeParser> m_CaffeParser;
114 const std::shared_ptr<DlLibHandle>& handle =
nullptr);
138 const std::shared_ptr<DlLibHandle>& handle =
nullptr)
140 m_ModelName(initParams.onnxFilePath) {}
142 bool isValid()
const override {
return !m_ModelName.empty(); }
143 const char*
getModelName()
const override {
return m_ModelName.c_str(); }
148 std::string m_ModelName;
165 const std::shared_ptr<DlLibHandle>& handle);
171 return (
bool)m_CustomParser;
182 return m_CustomParser->hasFullDimsSupported();
186 std::unique_ptr<IModelParser> m_CustomParser;
190 class TrtModelBuilder;
199 std::tuple<nvinfer1::DataType, nvinfer1::TensorFormats>;
201 std::tuple<nvinfer1::DataType, nvinfer1::DeviceType>;
228 bool sanityCheck()
const override;
232 nvinfer1::EnumMax<nvinfer1::OptProfileSelector>()>;
250 bool sanityCheck()
const override;
260 : m_Engine(std::move(
engine)), m_DlaCore(dlaCore) {}
264 const std::shared_ptr<DlLibHandle>& dlHandle =
nullptr,
265 nvinfer1::IPluginFactory* pluginFactory =
nullptr);
269 bool hasDla()
const {
return m_DlaCore >= 0; }
273 std::vector<NvDsInferBatchDimsLayerInfo>& layersInfo);
275 int profileIdx, std::vector<NvDsInferBatchDimsLayerInfo>& layersInfo);
289 return m_Engine.get();
297 std::shared_ptr<DlLibHandle> m_DlHandle;
298 nvinfer1::IPluginFactory* m_RuntimePluginFactory =
nullptr;
302 nvinfer1::IBuilderConfig *
const builderConfig,
305 nvinfer1::ICudaEngine *& cudaEngine);
320 const std::shared_ptr<DlLibHandle>& dlHandle =
nullptr);
328 m_Int8Calibrator = std::move(calibrator);
340 std::string& suggestedPathName);
346 std::unique_ptr<TrtEngine> buildEngine(
347 nvinfer1::INetworkDefinition& network,
BuildParams& options);
352 const std::string& path, nvinfer1::ICudaEngine& engine);
357 const std::string& path,
int dla = -1);
368 std::unique_ptr<TrtEngine> buildEngine();
372 std::unique_ptr<TrtEngine> getCudaEngineFromCustomLib(
384 std::unique_ptr<BuildParams> createImplicitParams(
386 std::unique_ptr<BuildParams> createDynamicParams(
388 void initCommonParams(
394 nvinfer1::ILogger& m_Logger;
395 std::shared_ptr<DlLibHandle> m_DlLib;
396 std::shared_ptr<BaseModelParser> m_Parser;
397 std::unique_ptr<BuildParams> m_Options;
401 std::shared_ptr<nvinfer1::IInt8Calibrator> m_Int8Calibrator;
408 nvinfer1::IBuilderConfig *
const builderConfig,
411 nvinfer1::ICudaEngine *& cudaEngine);
std::vector< std::string > inputNames
~CaffeModelParser() override
UffModelParser(const NvDsInferContextInitParams &initParams, const std::shared_ptr< DlLibHandle > &handle=nullptr)
~OnnxModelParser() override=default
bool hasFullDimsSupported() const override
std::unordered_map< std::string, TensorIOFormat > inputFormats
bool isValid() const override
Implementation of ModelParser for caffemodels derived from BaseModelParser.
NvDsInferStatus getLayerInfo(int idx, NvDsInferLayerInfo &layer)
NvDsInferStatus getImplicitLayersInfo(std::vector< NvDsInferBatchDimsLayerInfo > &layersInfo)
virtual bool sanityCheck() const
NvDsInferTensorOrder
Defines UFF input layer orders.
Holds build parameters required for implicit batch dimension network.
Implementation of ModelParser for custom models.
Helper class written on top of nvinfer1::ICudaEngine.
bool hasFullDimsSupported() const override
NvDsInferStatus parseModel(nvinfer1::INetworkDefinition &network) override
std::shared_ptr< DlLibHandle > m_LibHandle
ModelParams m_ModelParams
NvDsInferStatus parseModel(nvinfer1::INetworkDefinition &network) override
std::vector< ProfileDims > inputProfileDims
std::vector< nvinfer1::Dims > inputDims
Implementation of ModelParser for ONNX models derived from BaseModelParser.
bool NvDsInferCudaEngineGetFromTltModel(nvinfer1::IBuilder *const builder, nvinfer1::IBuilderConfig *const builderConfig, const NvDsInferContextInitParams *const initParams, nvinfer1::DataType dataType, nvinfer1::ICudaEngine *&cudaEngine)
Decodes and creates a CUDA engine file from a TLT encoded model.
Helper class to build models and generate the TensorRT ICudaEngine required for inference.
std::unordered_map< std::string, TensorIOFormat > outputFormats
BaseModelParser(const NvDsInferContextInitParams ¶ms, const std::shared_ptr< DlLibHandle > &dllib)
std::string int8CalibrationFilePath
const char * getModelName() const override
nvuffparser::UffInputOrder inputOrder
NvDsInferStatus serializeEngine(const std::string &path, nvinfer1::ICudaEngine &engine)
NvDsInferNetworkMode networkMode
TrtEngine(UniquePtrWDestroy< nvinfer1::ICudaEngine > &&engine, int dlaCore=-1)
bool NvDsInferCudaEngineGet(nvinfer1::IBuilder *builder, NvDsInferContextInitParams *initParams, nvinfer1::DataType dataType, nvinfer1::ICudaEngine *&cudaEngine) __attribute__((deprecated("Use 'engine-create-func-name' config parameter instead")))
The NvDsInferCudaEngineGet interface has been deprecated and has been replaced by NvDsInferEngineCrea...
NvDsInferNetworkMode
Defines internal data formats used by the inference engine.
std::tuple< nvinfer1::DataType, nvinfer1::TensorFormats > TensorIOFormat
UniquePtrWDestroy< nvonnxparser::IParser > m_OnnxParser
@ NvDsInferNetworkMode_FP32
UniquePtrWDestroy< nvuffparser::IUffParser > m_UffParser
NvDsInferStatus parseModel(nvinfer1::INetworkDefinition &network) override
nvinfer1::ICudaEngine & engine()
bool(* NvDsInferEngineCreateCustomFunc)(nvinfer1::IBuilder *const builder, nvinfer1::IBuilderConfig *const builderConfig, const NvDsInferContextInitParams *const initParams, nvinfer1::DataType dataType, nvinfer1::ICudaEngine *&cudaEngine)
Type definition for functions that build and return a CudaEngine for custom models.
CaffeModelParser(const NvDsInferContextInitParams &initParams, const std::shared_ptr< DlLibHandle > &handle=nullptr)
virtual bool isValid() const =0
Holds information about one layer in the model.
bool isValid() const override
std::unique_ptr< TrtEngine > deserializeEngine(const std::string &path, int dla=-1)
std::array< nvinfer1::Dims, nvinfer1::EnumMax< nvinfer1::OptProfileSelector >()> ProfileDims
Holds a pointer to a heap-allocated Plugin Factory object required during Caffe model parsing.
virtual ~BaseModelParser()
NvDsInferContextInitParams m_ModelParams
std::tuple< nvinfer1::DataType, nvinfer1::DeviceType > LayerDevicePrecision
bool isValid() const override
const char * getModelName() const override
~UffModelParser() override
@ NvDsInferTensorOrder_kNCHW
OnnxModelParser(const NvDsInferContextInitParams &initParams, const std::shared_ptr< DlLibHandle > &handle=nullptr)
std::unique_ptr< TrtEngine > buildModel(const NvDsInferContextInitParams &initParams, std::string &suggestedPathName)
const char * getModelName() const override
Implementation of ModelParser for UFF models derived from BaseModelParser.
Holds the initialization parameters required for the NvDsInferContext interface.
virtual NvDsInferStatus configBuilder(TrtModelBuilder &builder)=0
decltype(&NvDsInferCudaEngineGet) NvDsInferCudaEngineGetFcnDeprecated
NvDsInferStatus getFullDimsLayersInfo(int profileIdx, std::vector< NvDsInferBatchDimsLayerInfo > &layersInfo)
const char * getModelName() const override
std::vector< std::string > outputNames
nvinfer1::ICudaEngine * operator->()
const char * safeStr(const char *str)
bool isValid() const override
std::vector< nvinfer1::Dims > inputDims
std::unordered_map< std::string, LayerDevicePrecision > layerDevicePrecisions
Holds build parameters required for full dimensions network.
Holds build parameters common to implicit batch dimension/full dimension networks.
NvDsInferTensorOrder inputOrder
static const size_t kWorkSpaceSize
CustomModelParser(const NvDsInferContextInitParams &initParams, const std::shared_ptr< DlLibHandle > &handle)
NvDsInferStatus parseModel(nvinfer1::INetworkDefinition &network) override
bool hasFullDimsSupported() const override
TrtModelBuilder(int gpuId, nvinfer1::ILogger &logger, const std::shared_ptr< DlLibHandle > &dlHandle=nullptr)
bool hasFullDimsSupported() const override
void setInt8Calibrator(std::unique_ptr< nvinfer1::IInt8Calibrator > &&calibrator)
NvDsInferStatus
Enum for the status codes returned by NvDsInferContext.
NvDsInferStatus initParser()