|
NVIDIA DeepStream SDK API Reference
|
6.4 Release
|
Go to the documentation of this file.
12 #ifndef __NVDSINFER_BACKEND_H__
13 #define __NVDSINFER_BACKEND_H__
16 #include <condition_variable>
21 #include <cuda_runtime_api.h>
23 #include <NvCaffeParser.h>
25 #include <NvInferRuntime.h>
42 explicit CudaStream(uint flag = cudaStreamDefault,
int priority = 0);
51 m_Stream = o.m_Stream;
65 explicit CudaEvent(uint flag = cudaEventDefault);
67 operator cudaEvent_t() {
return m_Event; }
68 cudaEvent_t&
ptr() {
return m_Event; }
79 cudaEvent_t m_Event =
nullptr;
150 virtual NvDsInferBatchDims
getBatchDims(
int bindingIndex = 0)
const = 0;
179 virtual const NvDsInferBatchDimsLayerInfo&
getLayerInfo(
int bindingIdx) = 0;
181 virtual int getLayerIdx(
const std::string& bindingName) = 0;
186 int bindingIdx,
const NvDsInferBatchDims& batchDims) = 0;
195 const std::shared_ptr<InferBatchBuffer>& buffer,
CudaStream& stream,
215 std::shared_ptr<TrtEngine> engine);
217 int getLayerIdx(
const std::string& bindingName)
override;
220 const NvDsInferBatchDimsLayerInfo&
getLayerInfo(
int bindingIdx)
override
227 int bindingIdx,
const NvDsInferBatchDims& batchDims)
override;
232 return m_AllLayers[bindingIdx].profileDims[kSELECTOR_MAX];
237 return m_AllLayers[bindingIdx].profileDims[kSELECTOR_MIN];
242 return m_AllLayers[bindingIdx].profileDims[kSELECTOR_OPT];
263 std::shared_ptr<TrtEngine> engine);
269 const std::shared_ptr<InferBatchBuffer>& buffer,
CudaStream& stream,
274 int bindingIdx,
const NvDsInferBatchDims& batchDims)
override;
287 std::shared_ptr<TrtEngine> engine,
int profile = 0);
293 const std::shared_ptr<InferBatchBuffer>& buffer,
CudaStream& stream,
308 std::shared_ptr<TrtEngine> engine);
311 const std::shared_ptr<InferBatchBuffer>& buffer,
CudaStream& stream,
322 std::shared_ptr<TrtEngine> engine,
int profile = 0);
325 const std::shared_ptr<InferBatchBuffer>& buffer,
CudaStream& stream,
329 static std::mutex sExecutionMutex;
342 const std::shared_ptr<TrtEngine>& engine);
Backend context for full dimensions network.
virtual NvDsInferStatus enqueueBuffer(const std::shared_ptr< InferBatchBuffer > &buffer, CudaStream &stream, CudaEvent *consumeEvent)=0
Helper base class for managing Cuda allocated buffers.
CudaDeviceBuffer(size_t size)
virtual NvDsInferBatchDims getBatchDims(int bindingIndex=0) const =0
Helper class written on top of nvinfer1::ICudaEngine.
virtual NvDsInferBatchDims getMaxBatchDims(int bindingIdx)=0
InferBatchBuffer()=default
const NvDsInferBatchDimsLayerInfo & getLayerInfo(int bindingIdx) override
Abstract interface for managing the actual inferencing implementation.
virtual NvDsInferBatchDims getOptBatchDims(int bindingIdx)=0
Helper class for managing Cuda Streams.
virtual ~BackendContext()=default
DlaFullDimTrtBackendContext(UniquePtrWDestroy< nvinfer1::IExecutionContext > &&ctx, std::shared_ptr< TrtEngine > engine, int profile=0)
bool canSupportBatchDims(int bindingIdx, const NvDsInferBatchDims &batchDims) override
Backend context for implicit batch dimension network.
Abstract interface to manage a batched buffer for inference.
virtual NvDsInferBatchDims getMinBatchDims(int bindingIdx)=0
NvDsInferDataType
Specifies the data type of a layer.
virtual std::vector< void * > & getDeviceBuffers()=0
Copyright (c) 2019-2021, NVIDIA CORPORATION.
virtual NvDsInferStatus initialize()=0
Helper class for managing Cuda Streams.
virtual const NvDsInferBatchDimsLayerInfo & getLayerInfo(int bindingIdx)=0
int getNumBoundLayers() override
Base class for implementations of the BackendContext interface.
FullDimTrtBackendContext(UniquePtrWDestroy< nvinfer1::IExecutionContext > &&ctx, std::shared_ptr< TrtEngine > engine, int profile=0)
virtual NvDsInferDataType getDataType(int bindingIndex=0) const =0
void move_copy(CudaBuffer &&o)
std::vector< NvDsInferBatchDimsLayerInfo > m_AllLayers
int getLayerIdx(const std::string &bindingName) override
CudaEvent(uint flag=cudaEventDefault)
DISABLE_CLASS_COPY(CudaBuffer)
virtual int getNumBoundLayers()=0
TrtBackendContext(UniquePtrWDestroy< nvinfer1::IExecutionContext > &&ctx, std::shared_ptr< TrtEngine > engine)
virtual NvDsInferBatchDims getOptBatchDims(int bindingIdx) override
NvDsInferStatus enqueueBuffer(const std::shared_ptr< InferBatchBuffer > &buffer, CudaStream &stream, CudaEvent *consumeEvent) override
Backend context for implicit batch dimension network inferencing on DLA.
CudaHostBuffer(size_t size)
virtual bool canSupportBatchDims(int bindingIdx, const NvDsInferBatchDims &batchDims)=0
ImplicitTrtBackendContext(UniquePtrWDestroy< nvinfer1::IExecutionContext > &&ctx, std::shared_ptr< TrtEngine > engine)
CudaStream(uint flag=cudaStreamDefault, int priority=0)
Backend context for implicit batch dimension network inferencing on DLA.
#define SIMPLE_MOVE_COPY(Cls)
static std::mutex sDLAExecutionMutex
std::shared_ptr< TrtEngine > m_CudaEngine
virtual NvDsInferBatchDims getMaxBatchDims(int bindingIdx) override
virtual ~CudaBuffer()=default
DlaImplicitTrtBackendContext(UniquePtrWDestroy< nvinfer1::IExecutionContext > &&ctx, std::shared_ptr< TrtEngine > engine)
virtual NvDsInferBatchDims getMinBatchDims(int bindingIdx) override
NvDsInferStatus enqueueBuffer(const std::shared_ptr< InferBatchBuffer > &buffer, CudaStream &stream, CudaEvent *consumeEvent) override
virtual ~InferBatchBuffer()=default
virtual int getLayerIdx(const std::string &bindingName)=0
UniquePtrWDestroy< nvinfer1::IExecutionContext > m_Context
Helper class for managing Cuda events.
bool canSupportBatchDims(int bindingIdx, const NvDsInferBatchDims &batchDims) override
std::unique_ptr< TrtBackendContext > createBackendContext(const std::shared_ptr< TrtEngine > &engine)
Create an instance of a BackendContext.
NvDsInferStatus
Enum for the status codes returned by NvDsInferContext.