50 #ifndef NV_INFER_RUNTIME_COMMON_H 51 #define NV_INFER_RUNTIME_COMMON_H 55 #include "NvInferVersion.h" 57 #if __cplusplus >= 201103L 58 #define _TENSORRT_FINAL final 59 #define _TENSORRT_OVERRIDE override 61 #define _TENSORRT_FINAL 62 #define _TENSORRT_OVERRIDE 66 #if __cplusplus >= 201402L 67 #define TRT_DEPRECATED [[deprecated]] 69 #define TRT_DEPRECATED_ENUM 71 #define TRT_DEPRECATED_ENUM TRT_DEPRECATED 74 #define TRT_DEPRECATED_API __declspec(dllexport) 76 #define TRT_DEPRECATED_API [[deprecated]] __attribute__((visibility("default"))) 80 #define TRT_DEPRECATED 81 #define TRT_DEPRECATED_ENUM 82 #define TRT_DEPRECATED_API __declspec(dllexport) 84 #define TRT_DEPRECATED __attribute__((deprecated)) 85 #define TRT_DEPRECATED_ENUM 86 #define TRT_DEPRECATED_API __attribute__((deprecated, visibility("default"))) 91 #ifdef TENSORRT_BUILD_LIB 93 #define TENSORRTAPI __declspec(dllexport) 95 #define TENSORRTAPI __attribute__((visibility("default"))) 109 struct cublasContext;
115 static const int NV_TENSORRT_VERSION = (NV_TENSORRT_MAJOR * 1000) + (NV_TENSORRT_MINOR * 100) + NV_TENSORRT_PATCH;
125 class IErrorRecorder;
149 template <
typename T>
150 constexpr
inline int EnumMax();
209 static const int MAX_DIMS = 8;
243 kNCHW TRT_DEPRECATED_ENUM =
kLINEAR,
252 kNC2HW2 TRT_DEPRECATED_ENUM =
kCHW2,
261 kNHWC8 TRT_DEPRECATED_ENUM =
kHWC8,
354 return NV_TENSORRT_VERSION;
361 virtual const char* getPluginType()
const TRTNOEXCEPT = 0;
367 virtual const char* getPluginVersion()
const TRTNOEXCEPT = 0;
376 virtual int getNbOutputs()
const TRTNOEXCEPT = 0;
387 virtual Dims getOutputDimensions(
int index,
const Dims* inputs,
int nbInputDims) TRTNOEXCEPT = 0;
425 virtual void configureWithFormat(
const Dims* inputDims,
int nbInputs,
const Dims* outputDims,
int nbOutputs,
DataType type,
PluginFormat format,
int maxBatchSize) TRTNOEXCEPT = 0;
432 virtual int initialize() TRTNOEXCEPT = 0;
438 virtual void terminate() TRTNOEXCEPT = 0;
448 virtual size_t getWorkspaceSize(
int maxBatchSize)
const TRTNOEXCEPT = 0;
461 virtual int enqueue(
int batchSize,
const void*
const* inputs,
void** outputs,
void* workspace,
cudaStream_t stream) TRTNOEXCEPT = 0;
468 virtual size_t getSerializationSize()
const TRTNOEXCEPT = 0;
477 virtual void serialize(
void* buffer)
const TRTNOEXCEPT = 0;
482 virtual void destroy() TRTNOEXCEPT = 0;
487 virtual IPluginV2* clone()
const TRTNOEXCEPT = 0;
493 virtual void setPluginNamespace(
const char* pluginNamespace) TRTNOEXCEPT = 0;
498 virtual const char* getPluginNamespace()
const TRTNOEXCEPT = 0;
535 virtual bool isOutputBroadcastAcrossBatch(
int outputIndex,
const bool* inputIsBroadcasted,
int nbInputs)
const TRTNOEXCEPT = 0;
550 virtual bool canBroadcastInputAcrossBatch(
int inputIndex)
const TRTNOEXCEPT = 0;
579 virtual void configurePlugin(
const Dims* inputDims,
int nbInputs,
const Dims* outputDims,
581 const bool* inputIsBroadcast,
const bool* outputIsBroadcast,
PluginFormat floatFormat,
int maxBatchSize) TRTNOEXCEPT = 0;
688 virtual bool supportsFormatCombination(
int pos,
const PluginTensorDesc* inOut,
int nbInputs,
int nbOutputs)
const TRTNOEXCEPT = 0;
765 const char* name{
nullptr};
769 const void* data{
nullptr};
780 PluginField(
const char* name_ =
nullptr,
const void* data_ =
nullptr,
const PluginFieldType type_ = PluginFieldType::kUNKNOWN, int32_t length_ = 0)
814 virtual const char* getPluginName()
const TRTNOEXCEPT = 0;
819 virtual const char* getPluginVersion()
const TRTNOEXCEPT = 0;
835 virtual IPluginV2* deserializePlugin(
const char* name,
const void* serialData,
size_t serialLength) TRTNOEXCEPT = 0;
843 virtual void setPluginNamespace(
const char* pluginNamespace) TRTNOEXCEPT = 0;
848 virtual const char* getPluginNamespace()
const TRTNOEXCEPT = 0;
875 virtual bool registerCreator(
IPluginCreator& creator,
const char* pluginNamespace) noexcept = 0;
881 virtual IPluginCreator*
const* getPluginCreatorList(
int* numCreators)
const noexcept = 0;
887 virtual IPluginCreator* getPluginCreator(
const char* pluginType,
const char* pluginVersion,
const char* pluginNamespace =
"") noexcept = 0;
905 virtual void setErrorRecorder(
IErrorRecorder* recorder) noexcept = 0;
960 virtual void* allocate(uint64_t size, uint64_t alignment, uint32_t flags) TRTNOEXCEPT = 0;
969 virtual void free(
void* memory) TRTNOEXCEPT = 0;
1009 virtual void log(
Severity severity,
const char* msg) TRTNOEXCEPT = 0;
1015 constexpr
inline int EnumMax<ILogger::Severity>()
1172 virtual int32_t getNbErrors()
const noexcept = 0;
1186 virtual ErrorCode getErrorCode(int32_t errorIdx)
const noexcept = 0;
1202 virtual ErrorDesc getErrorDesc(int32_t errorIdx)
const noexcept = 0;
1213 virtual bool hasOverflowed()
const noexcept = 0;
1224 virtual void clear() noexcept = 0;
1254 virtual RefCount incRefCount() noexcept = 0;
1267 virtual RefCount decRefCount() noexcept = 0;
1277 extern "C" TENSORRTAPI
void* createSafeInferRuntime_INTERNAL(
void* logger,
int version);
1305 template <
typename T>
1314 #define REGISTER_TENSORRT_PLUGIN(name) \ 1315 static nvinfer1::PluginRegistrar<name> pluginRegistrar##name {} 1319 #endif // NV_INFER_RUNTIME_COMMON_H Perform the normal matrix multiplication in the first recurrent layer.
DataType
The type of weights and tensors.
Definition: NvInferRuntimeCommon.h:162
constexpr int EnumMax< ErrorCode >()
Maximum number of elements in ErrorCode enum.
Definition: NvInferRuntimeCommon.h:1115
int32_t RefCount
Definition: NvInferRuntimeCommon.h:1153
virtual int getTensorRTVersion() const
Return the API version with which this plugin was built.
Definition: NvInferRuntimeCommon.h:352
Plugin class for user-implemented layers.
Definition: NvInferRuntimeCommon.h:344
TRT_DEPRECATED void configurePlugin(const Dims *, int, const Dims *, int, const DataType *, const DataType *, const bool *, const bool *, PluginFormat, int) _TENSORRT_OVERRIDE _TENSORRT_FINAL
Deprecated interface inheriting from base class. Derived classes should not implement this...
Definition: NvInferRuntimeCommon.h:719
Single gate RNN w/ TANH activation function.
struct CUstream_st * cudaStream_t
Forward declaration of cudaStream_t.
Definition: NvInferRuntimeCommon.h:112
PluginFieldType
Definition: NvInferRuntimeCommon.h:740
TRT_DEPRECATED void configureWithFormat(const Dims *, int, const Dims *, int, DataType, PluginFormat, int) _TENSORRT_OVERRIDE _TENSORRT_FINAL
Deprecated interface inheriting from base class. Derived classes should not implement this...
Definition: NvInferRuntimeCommon.h:709
Parametric softplus activation: alpha*log(exp(beta*x)+1)
const PluginField * fields
Pointer to PluginField entries.
Definition: NvInferRuntimeCommon.h:792
nvinfer1::IPluginRegistry * getPluginRegistry()
Return the plugin registry.
Severity
Definition: NvInferRuntimeCommon.h:994
Definition: NvInferRuntimeCommon.h:789
TensorLocation
The location for tensor data storage, device or host.
Definition: NvInferRuntimeCommon.h:926
constexpr int EnumMax< TensorFormat >()
Maximum number of elements in TensorFormat enum.
Definition: NvInferRuntimeCommon.h:296
virtual void attachToContext(cudnnContext *, cublasContext *, IGpuAllocator *)
Attach the plugin object to an execution context and grant the plugin the access to some context reso...
Definition: NvInferRuntimeCommon.h:596
Register the plugin creator to the registry The static registry object will be instantiated when the ...
Definition: NvInferRuntimeCommon.h:1306
Scaled tanh activation: alpha*tanh(beta*x)
Per-channel coefficients.
Elements correspond to different spatial data.
Clip activation: max(alpha, min(beta, x))
TRT_DEPRECATED int getTensorRTVersion() const _TENSORRT_OVERRIDE
Return the API version with which this plugin was built. The upper byte is reserved by TensorRT and i...
Definition: NvInferRuntimeCommon.h:699
The TensorRT API version 1 namespace.
Elements correspond to different sequence values.
virtual ~IGpuAllocator()
Definition: NvInferRuntimeCommon.h:975
virtual void detachFromContext()
Detach the plugin object from its execution context.
Definition: NvInferRuntimeCommon.h:604
Single registration point for all plugins in an application. It is used to find plugin implementation...
Definition: NvInferRuntimeCommon.h:868
Plugin creator class for user implemented layers.
Definition: NvInferRuntimeCommon.h:803
Elu activation: x>=0 ? x : alpha * (exp(x) - 1).
Structure containing plugin attribute field names and associated data This information can be parsed ...
Definition: NvInferRuntimeCommon.h:760
virtual int getTensorRTVersion() const
Return the version of the API the plugin creator was compiled with.
Definition: NvInferRuntimeCommon.h:809
nvinfer1::Dims field type.
ActivationType
Forward declare IGpuAllocator for use in other interfaces.
Definition: NvInferRuntimeCommon.h:133
int getInferLibVersion()
Return the library version number.
constexpr int EnumMax< TensorLocation >()
Maximum number of elements in TensorLocation enum.
Definition: NvInferRuntimeCommon.h:933
Reference counted application-implemented error reporting interface for TensorRT objects.
Definition: NvInferRuntimeCommon.h:1142
Structure to define the dimensions of a tensor.
Definition: NvInferRuntimeCommon.h:206
const char * ErrorDesc
Definition: NvInferRuntimeCommon.h:1148
constexpr int EnumMax< DimensionType >()
Maximum number of elements in DimensionType enum.
Definition: NvInferRuntimeCommon.h:190
int getTensorRTVersion() const _TENSORRT_OVERRIDE
Return the API version with which this plugin was built. The upper byte reserved by TensorRT and is u...
Definition: NvInferRuntimeCommon.h:620
void configureWithFormat(const Dims *, int, const Dims *, int, DataType, PluginFormat, int) _TENSORRT_OVERRIDE
Derived classes should not implement this. In a C++11 API it would be override final.
Definition: NvInferRuntimeCommon.h:628
Single gate RNN w/ ReLU activation function.
uint32_t TensorFormats
It is capable of representing one or more TensorFormat by binary OR operations, e.g., 1U << TensorFormats::kCHW4 | 1U << TensorFormats::kCHW32.
Definition: NvInferRuntimeCommon.h:221
ErrorCode
Error codes that can be returned by TensorRT during execution.
Definition: NvInferRuntimeCommon.h:1025
<– Deprecated, used for backward compatibility
constexpr int EnumMax< DataType >()
Maximum number of elements in DataType enum.
Definition: NvInferRuntimeCommon.h:172
Plugin class for user-implemented layers.
Definition: NvInferRuntimeCommon.h:641
Enable Int8 layer selection, with FP32 fallback with FP16 fallback if kFP16 also specified.
int nbDims
The number of dimensions.
Definition: NvInferRuntimeCommon.h:210
TensorFormat
Format of the input/output tensors.
Definition: NvInferRuntimeCommon.h:236
Application-implemented logging interface for the builder, engine and runtime.
Definition: NvInferRuntimeCommon.h:986
TRT_DEPRECATED bool supportsFormat(DataType, PluginFormat) const _TENSORRT_OVERRIDE _TENSORRT_FINAL
Deprecated interface inheriting from base class. Derived classes should not implement this...
Definition: NvInferRuntimeCommon.h:729
Fields that a plugin might see for an input or output.
Definition: NvInferRuntimeCommon.h:311
virtual bool registerCreator(IPluginCreator &creator, const char *pluginNamespace) noexcept=0
Register a plugin creator. Returns false if one with same type is already registered.
Selu activation: x>0 ? beta * x : beta * (alpha*exp(x) - alpha)
constexpr int EnumMax()
Maximum number of elements in an enumeration type.
DimensionType
The type of data encoded across this dimension.
Definition: NvInferRuntimeCommon.h:181
Thresholded ReLU activation: x>alpha ? x : 0.
#define _TENSORRT_OVERRIDE
Items that are marked as deprecated will be removed in a future release.
Definition: NvInferRuntimeCommon.h:62
Application-implemented class for controlling allocation on the GPU.
Definition: NvInferRuntimeCommon.h:943
<– Deprecated, used for backward compatibility
nvinfer1::ILogger * getLogger()
Return the logger object.
Definition of plugin versions.
Elements correspond to different batch index.
LeakyRelu activation: x>=0 ? x : alpha * x.
constexpr int EnumMax< ActivationType >()
Maximum number of elements in ActivationType enum.
Definition: NvInferRuntimeCommon.h:153
Plugin class for user-implemented layers.
Definition: NvInferRuntimeCommon.h:514
Hard sigmoid activation: max(0, min(1, alpha*x+beta))
<– Deprecated, used for backward compatibility
int nbFields
Number of PluginField entries.
Definition: NvInferRuntimeCommon.h:791
struct CUevent_st * cudaEvent_t
Forward declaration of cudaEvent_t.
Definition: NvInferRuntimeCommon.h:113
Softsign activation: x / (1+|x|)