|
NVIDIA DeepStream SDK API Reference
|
6.4 Release
|
Go to the documentation of this file.
12 #ifndef __NVDSINFER_MODEL_BUILDER_H__
13 #define __NVDSINFER_MODEL_BUILDER_H__
17 #include <condition_variable>
23 #include <unordered_map>
25 #include <NvCaffeParser.h>
27 #include <NvInferRuntime.h>
28 #include <NvOnnxParser.h>
55 const std::shared_ptr<DlLibHandle>& dllib)
58 virtual bool isValid()
const = 0;
76 const std::shared_ptr<DlLibHandle>& handle =
nullptr);
78 bool isValid()
const override {
return m_CaffeParser.get(); }
79 const char*
getModelName()
const override {
return m_ModelPath.c_str(); }
88 std::string m_ProtoPath;
89 std::string m_ModelPath;
90 std::vector<std::string> m_OutputLayers;
92 UniquePtrWDestroy<nvcaffeparser1::ICaffeParser> m_CaffeParser;
113 const std::shared_ptr<DlLibHandle>& handle =
nullptr);
137 const std::shared_ptr<DlLibHandle>& handle =
nullptr)
139 m_ModelName(initParams.onnxFilePath) {}
141 bool isValid()
const override {
return !m_ModelName.empty(); }
142 const char*
getModelName()
const override {
return m_ModelName.c_str(); }
147 std::string m_ModelName;
164 const std::shared_ptr<DlLibHandle>& handle);
170 return (
bool)m_CustomParser;
181 return m_CustomParser->hasFullDimsSupported();
185 std::unique_ptr<IModelParser> m_CustomParser;
189 class TrtModelBuilder;
198 std::tuple<nvinfer1::DataType, nvinfer1::TensorFormats>;
200 std::tuple<nvinfer1::DataType, nvinfer1::DeviceType>;
227 bool sanityCheck()
const override;
231 nvinfer1::EnumMax<nvinfer1::OptProfileSelector>()>;
249 bool sanityCheck()
const override;
259 : m_Engine(std::move(
engine)), m_DlaCore(dlaCore) {}
263 const std::shared_ptr<DlLibHandle>& dlHandle =
nullptr,
264 nvinfer1::IPluginFactory* pluginFactory =
nullptr);
268 bool hasDla()
const {
return m_DlaCore >= 0; }
272 std::vector<NvDsInferBatchDimsLayerInfo>& layersInfo);
274 int profileIdx, std::vector<NvDsInferBatchDimsLayerInfo>& layersInfo);
288 return m_Engine.get();
296 std::shared_ptr<DlLibHandle> m_DlHandle;
297 nvinfer1::IPluginFactory* m_RuntimePluginFactory =
nullptr;
301 nvinfer1::IBuilderConfig *
const builderConfig,
304 nvinfer1::ICudaEngine *& cudaEngine);
319 const std::shared_ptr<DlLibHandle>& dlHandle =
nullptr);
327 m_Int8Calibrator = std::move(calibrator);
339 std::string& suggestedPathName);
345 std::unique_ptr<TrtEngine> buildEngine(
346 nvinfer1::INetworkDefinition& network,
BuildParams& options);
351 const std::string& path, nvinfer1::ICudaEngine& engine);
356 const std::string& path,
int dla = -1);
367 std::unique_ptr<TrtEngine> buildEngine();
371 std::unique_ptr<TrtEngine> getCudaEngineFromCustomLib(
383 std::unique_ptr<BuildParams> createImplicitParams(
385 std::unique_ptr<BuildParams> createDynamicParams(
387 void initCommonParams(
393 nvinfer1::ILogger& m_Logger;
394 std::shared_ptr<DlLibHandle> m_DlLib;
395 std::shared_ptr<BaseModelParser> m_Parser;
396 std::unique_ptr<BuildParams> m_Options;
400 std::shared_ptr<nvinfer1::IInt8Calibrator> m_Int8Calibrator;
407 nvinfer1::IBuilderConfig *
const builderConfig,
410 nvinfer1::ICudaEngine *& cudaEngine);
std::vector< std::string > inputNames
~CaffeModelParser() override
UffModelParser(const NvDsInferContextInitParams &initParams, const std::shared_ptr< DlLibHandle > &handle=nullptr)
~OnnxModelParser() override=default
bool hasFullDimsSupported() const override
std::unordered_map< std::string, TensorIOFormat > inputFormats
bool isValid() const override
Implementation of ModelParser for caffemodels derived from BaseModelParser.
NvDsInferStatus getLayerInfo(int idx, NvDsInferLayerInfo &layer)
NvDsInferStatus getImplicitLayersInfo(std::vector< NvDsInferBatchDimsLayerInfo > &layersInfo)
virtual bool sanityCheck() const
NvDsInferTensorOrder
Defines UFF input layer orders.
Holds build parameters required for implicit batch dimension network.
Implementation of ModelParser for custom models.
Helper class written on top of nvinfer1::ICudaEngine.
bool hasFullDimsSupported() const override
NvDsInferStatus parseModel(nvinfer1::INetworkDefinition &network) override
std::shared_ptr< DlLibHandle > m_LibHandle
ModelParams m_ModelParams
NvDsInferStatus parseModel(nvinfer1::INetworkDefinition &network) override
std::vector< ProfileDims > inputProfileDims
std::vector< nvinfer1::Dims > inputDims
Implementation of ModelParser for ONNX models derived from BaseModelParser.
bool NvDsInferCudaEngineGetFromTltModel(nvinfer1::IBuilder *const builder, nvinfer1::IBuilderConfig *const builderConfig, const NvDsInferContextInitParams *const initParams, nvinfer1::DataType dataType, nvinfer1::ICudaEngine *&cudaEngine)
Decodes and creates a CUDA engine file from a TLT encoded model.
Helper class to build models and generate the TensorRT ICudaEngine required for inference.
std::unordered_map< std::string, TensorIOFormat > outputFormats
BaseModelParser(const NvDsInferContextInitParams ¶ms, const std::shared_ptr< DlLibHandle > &dllib)
std::string int8CalibrationFilePath
const char * getModelName() const override
nvuffparser::UffInputOrder inputOrder
NvDsInferStatus serializeEngine(const std::string &path, nvinfer1::ICudaEngine &engine)
NvDsInferNetworkMode networkMode
TrtEngine(UniquePtrWDestroy< nvinfer1::ICudaEngine > &&engine, int dlaCore=-1)
bool NvDsInferCudaEngineGet(nvinfer1::IBuilder *builder, NvDsInferContextInitParams *initParams, nvinfer1::DataType dataType, nvinfer1::ICudaEngine *&cudaEngine) __attribute__((deprecated("Use 'engine-create-func-name' config parameter instead")))
The NvDsInferCudaEngineGet interface has been deprecated and has been replaced by NvDsInferEngineCrea...
NvDsInferNetworkMode
Defines internal data formats used by the inference engine.
Copyright (c) 2019-2021, NVIDIA CORPORATION.
std::tuple< nvinfer1::DataType, nvinfer1::TensorFormats > TensorIOFormat
UniquePtrWDestroy< nvonnxparser::IParser > m_OnnxParser
@ NvDsInferNetworkMode_FP32
UniquePtrWDestroy< nvuffparser::IUffParser > m_UffParser
NvDsInferStatus parseModel(nvinfer1::INetworkDefinition &network) override
nvinfer1::ICudaEngine & engine()
bool(* NvDsInferEngineCreateCustomFunc)(nvinfer1::IBuilder *const builder, nvinfer1::IBuilderConfig *const builderConfig, const NvDsInferContextInitParams *const initParams, nvinfer1::DataType dataType, nvinfer1::ICudaEngine *&cudaEngine)
Type definition for functions that build and return a CudaEngine for custom models.
CaffeModelParser(const NvDsInferContextInitParams &initParams, const std::shared_ptr< DlLibHandle > &handle=nullptr)
virtual bool isValid() const =0
Holds information about one layer in the model.
bool isValid() const override
std::unique_ptr< TrtEngine > deserializeEngine(const std::string &path, int dla=-1)
std::array< nvinfer1::Dims, nvinfer1::EnumMax< nvinfer1::OptProfileSelector >()> ProfileDims
Holds a pointer to a heap-allocated Plugin Factory object required during Caffe model parsing.
virtual ~BaseModelParser()
NvDsInferContextInitParams m_ModelParams
std::tuple< nvinfer1::DataType, nvinfer1::DeviceType > LayerDevicePrecision
bool isValid() const override
const char * getModelName() const override
~UffModelParser() override
@ NvDsInferTensorOrder_kNCHW
OnnxModelParser(const NvDsInferContextInitParams &initParams, const std::shared_ptr< DlLibHandle > &handle=nullptr)
std::unique_ptr< TrtEngine > buildModel(const NvDsInferContextInitParams &initParams, std::string &suggestedPathName)
const char * getModelName() const override
Implementation of ModelParser for UFF models derived from BaseModelParser.
Holds the initialization parameters required for the NvDsInferContext interface.
virtual NvDsInferStatus configBuilder(TrtModelBuilder &builder)=0
decltype(&NvDsInferCudaEngineGet) NvDsInferCudaEngineGetFcnDeprecated
NvDsInferStatus getFullDimsLayersInfo(int profileIdx, std::vector< NvDsInferBatchDimsLayerInfo > &layersInfo)
const char * getModelName() const override
std::vector< std::string > outputNames
nvinfer1::ICudaEngine * operator->()
const char * safeStr(const char *str)
bool isValid() const override
std::vector< nvinfer1::Dims > inputDims
std::unordered_map< std::string, LayerDevicePrecision > layerDevicePrecisions
Holds build parameters required for full dimensions network.
Holds build parameters common to implicit batch dimension/full dimension networks.
NvDsInferTensorOrder inputOrder
static const size_t kWorkSpaceSize
CustomModelParser(const NvDsInferContextInitParams &initParams, const std::shared_ptr< DlLibHandle > &handle)
NvDsInferStatus parseModel(nvinfer1::INetworkDefinition &network) override
bool hasFullDimsSupported() const override
TrtModelBuilder(int gpuId, nvinfer1::ILogger &logger, const std::shared_ptr< DlLibHandle > &dlHandle=nullptr)
bool hasFullDimsSupported() const override
void setInt8Calibrator(std::unique_ptr< nvinfer1::IInt8Calibrator > &&calibrator)
NvDsInferStatus
Enum for the status codes returned by NvDsInferContext.
NvDsInferStatus initParser()