Go to the documentation of this file.
50 #ifndef NV_INFER_RUNTIME_COMMON_H
51 #define NV_INFER_RUNTIME_COMMON_H
57 #if __cplusplus >= 201103L
58 #define _TENSORRT_FINAL final
59 #define _TENSORRT_OVERRIDE override
61 #define _TENSORRT_FINAL
62 #define _TENSORRT_OVERRIDE
66 #if __cplusplus >= 201402L
67 #define TRT_DEPRECATED [[deprecated]]
69 #define TRT_DEPRECATED_ENUM
71 #define TRT_DEPRECATED_ENUM TRT_DEPRECATED
74 #define TRT_DEPRECATED_API __declspec(dllexport)
76 #define TRT_DEPRECATED_API [[deprecated]] __attribute__((visibility("default")))
80 #define TRT_DEPRECATED
81 #define TRT_DEPRECATED_ENUM
82 #define TRT_DEPRECATED_API __declspec(dllexport)
84 #define TRT_DEPRECATED __attribute__((deprecated))
85 #define TRT_DEPRECATED_ENUM
86 #define TRT_DEPRECATED_API __attribute__((deprecated, visibility("default")))
91 #ifdef TENSORRT_BUILD_LIB
93 #define TENSORRTAPI __declspec(dllexport)
95 #define TENSORRTAPI __attribute__((visibility("default")))
109 struct cublasContext;
125 class IErrorRecorder;
149 template <
typename T>
150 constexpr
inline int EnumMax();
381 return NV_TENSORRT_VERSION;
469 virtual
void terminate() TRTNOEXCEPT = 0;
492 virtual
int enqueue(
int batchSize, const
void* const* inputs,
void** outputs,
void* workspace,
cudaStream_t stream) TRTNOEXCEPT = 0;
508 virtual
void serialize(
void* buffer) const TRTNOEXCEPT = 0;
513 virtual
void destroy() TRTNOEXCEPT = 0;
614 const bool* inputIsBroadcast,
const bool* outputIsBroadcast,
PluginFormat floatFormat,
int maxBatchSize) TRTNOEXCEPT = 0;
814 PluginField(
const char* name_ =
nullptr,
const void* data_ =
nullptr,
const PluginFieldType type_ = PluginFieldType::kUNKNOWN, int32_t length_ = 0)
994 virtual void*
allocate(uint64_t size, uint64_t alignment, uint32_t flags) TRTNOEXCEPT = 0;
1003 virtual void free(
void* memory) TRTNOEXCEPT = 0;
1043 virtual void log(
Severity severity,
const char* msg) TRTNOEXCEPT = 0;
1049 constexpr
inline int EnumMax<ILogger::Severity>()
1257 virtual
void clear() noexcept = 0;
1310 extern "C" TENSORRTAPI
void* createSafeInferRuntime_INTERNAL(
void* logger,
int version);
1338 template <
typename T>
1347 #define REGISTER_TENSORRT_PLUGIN(name) \
1348 static nvinfer1::PluginRegistrar<name> pluginRegistrar##name {}
1352 #endif // NV_INFER_RUNTIME_COMMON_H
virtual void free(void *memory)=0
Scaled tanh activation: alpha*tanh(beta*x)
ActivationType
Forward declare IGpuAllocator for use in other interfaces.
Definition: NvInferRuntimeCommon.h:133
An application error has occurred.
virtual bool supportsFormat(DataType type, PluginFormat format) const =0
Check format support.
virtual const char * getPluginType() const =0
Return the plugin type. Should match the plugin name returned by the corresponding plugin creator.
virtual IErrorRecorder * getErrorRecorder() const noexcept=0
set the ErrorRecorder assigned to this interface.
virtual int getNbOutputs() const =0
Get the number of outputs from the layer.
Severity
Definition: NvInferRuntimeCommon.h:1028
virtual const char * getPluginNamespace() const =0
Return the namespace of the plugin creator object.
virtual bool isOutputBroadcastAcrossBatch(int outputIndex, const bool *inputIsBroadcasted, int nbInputs) const =0
Return true if output tensor is broadcast across a batch.
virtual const char * getPluginName() const =0
Return the plugin name.
virtual IPluginV2Ext * clone() const _TENSORRT_OVERRIDE=0
Clone the plugin object. This copies over internal plugin parameters as well and returns a new plugin...
Register the plugin creator to the registry The static registry object will be instantiated when the ...
Definition: NvInferRuntimeCommon.h:1339
virtual void destroy()=0
Destroy the plugin object. This will be called when the network, builder or engine is destroyed.
const PluginField * fields
Pointer to PluginField entries.
Definition: NvInferRuntimeCommon.h:826
32-bit floating point format.
Parametric softplus activation: alpha*log(exp(beta*x)+1)
constexpr int EnumMax()
Maximum number of elements in an enumeration type.
Thresholded ReLU activation: x>alpha ? x : 0.
virtual ErrorCode getErrorCode(int32_t errorIdx) const noexcept=0
Returns the ErrorCode enumeration.
virtual void setPluginNamespace(const char *pluginNamespace)=0
Set the namespace that this plugin object belongs to. Ideally, all plugin objects from the same plugi...
Plugin class for user-implemented layers.
Definition: NvInferRuntimeCommon.h:371
TensorLocation
The location for tensor data storage, device or host.
Definition: NvInferRuntimeCommon.h:960
virtual const char * getPluginVersion() const =0
Return the plugin version. Should match the plugin version returned by the corresponding plugin creat...
uint32_t TensorFormats
It is capable of representing one or more TensorFormat by binary OR operations, e....
Definition: NvInferRuntimeCommon.h:234
int32_t RefCount
Definition: NvInferRuntimeCommon.h:1186
constexpr int EnumMax< DataType >()
Maximum number of elements in DataType enum.
Definition: NvInferRuntimeCommon.h:182
int nbDims
The number of dimensions.
Definition: NvInferRuntimeCommon.h:223
virtual RefCount decRefCount() noexcept=0
Decrements the refcount for the current ErrorRecorder.
virtual ErrorDesc getErrorDesc(int32_t errorIdx) const noexcept=0
Returns the c-style string description of the error.
Enable Int8 layer selection, with FP32 fallback with FP16 fallback if kFP16 also specified.
virtual void setErrorRecorder(IErrorRecorder *recorder) noexcept=0
Set the ErrorRecorder for this interface.
virtual nvinfer1::DataType getOutputDataType(int index, const nvinfer1::DataType *inputTypes, int nbInputs) const =0
Return the DataType of the plugin output at the requested index. The default behavior should be to re...
8-bit boolean. 0 = false, 1 = true, other values undefined.
Clip activation: max(alpha, min(beta, x))
constexpr int EnumMax< DimensionType >()
Maximum number of elements in DimensionType enum.
Definition: NvInferRuntimeCommon.h:200
virtual int enqueue(int batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream)=0
Execute the layer.
PluginFieldType
Definition: NvInferRuntimeCommon.h:773
virtual const char * getPluginVersion() const =0
Return the plugin version.
Plugin creator class for user implemented layers.
Definition: NvInferRuntimeCommon.h:837
Single registration point for all plugins in an application. It is used to find plugin implementation...
Definition: NvInferRuntimeCommon.h:902
Structure to define the dimensions of a tensor.
Definition: NvInferRuntimeCommon.h:219
Application-implemented logging interface for the builder, engine and runtime.
Definition: NvInferRuntimeCommon.h:1020
virtual void serialize(void *buffer) const =0
Serialize the layer.
Hard sigmoid activation: max(0, min(1, alpha*x+beta))
virtual bool canBroadcastInputAcrossBatch(int inputIndex) const =0
Return true if plugin can use input that is broadcast across batch without replication.
TRT_DEPRECATED int getTensorRTVersion() const _TENSORRT_OVERRIDE
Return the API version with which this plugin was built. The upper byte is reserved by TensorRT and i...
Definition: NvInferRuntimeCommon.h:732
TRT_DEPRECATED void configurePlugin(const Dims *, int, const Dims *, int, const DataType *, const DataType *, const bool *, const bool *, PluginFormat, int) _TENSORRT_OVERRIDE _TENSORRT_FINAL
Deprecated interface inheriting from base class. Derived classes should not implement this....
Definition: NvInferRuntimeCommon.h:752
Elu activation: x>=0 ? x : alpha * (exp(x) - 1).
int32_t length
Number of data entries in the Plugin attribute.
Definition: NvInferRuntimeCommon.h:812
DimensionType
The type of data encoded across this dimension.
Definition: NvInferRuntimeCommon.h:191
nvinfer1::ILogger * getLogger()
Return the logger object.
PluginFieldType type
Plugin field attribute type.
Definition: NvInferRuntimeCommon.h:808
DataType type
Definition: NvInferRuntimeCommon.h:341
TRT_DEPRECATED DimensionType type[MAX_DIMS]
The type of each dimension.
Definition: NvInferRuntimeCommon.h:225
virtual bool reportError(ErrorCode val, ErrorDesc desc) noexcept=0
report an error to the error recorder with the corresponding enum and description.
Perform the normal matrix multiplication in the first recurrent layer.
virtual void log(Severity severity, const char *msg)=0
constexpr int EnumMax< ActivationType >()
Maximum number of elements in ActivationType enum.
Definition: NvInferRuntimeCommon.h:153
Per-channel coefficients.
The TensorRT API version 1 namespace.
TensorFormat
Format of the input/output tensors.
Definition: NvInferRuntimeCommon.h:249
int getTensorRTVersion() const _TENSORRT_OVERRIDE
Return the API version with which this plugin was built. The upper byte reserved by TensorRT and is u...
Definition: NvInferRuntimeCommon.h:653
virtual void * allocate(uint64_t size, uint64_t alignment, uint32_t flags)=0
Deprecated name of kCHW2, provided for backwards compatibility.
LeakyRelu activation: x>=0 ? x : alpha * x.
const char * ErrorDesc
Definition: NvInferRuntimeCommon.h:1181
virtual int initialize()=0
Initialize the layer for execution. This is called when the engine is created.
#define NV_TENSORRT_MINOR
TensorRT minor version.
Definition: NvInferVersion.h:60
Softsign activation: x / (1+|x|)
struct CUstream_st * cudaStream_t
Forward declaration of cudaStream_t.
Definition: NvInferRuntimeCommon.h:112
virtual void terminate()=0
Release resources acquired during plugin layer initialization. This is called when the engine is dest...
Deprecated name of kHWC8, provided for backwards compatibility.
An internal error has occurred. Execution is unrecoverable.
virtual bool registerCreator(IPluginCreator &creator, const char *pluginNamespace) noexcept=0
Register a plugin creator. Returns false if one with same type is already registered.
#define _TENSORRT_OVERRIDE
Items that are marked as deprecated will be removed in a future release.
Definition: NvInferRuntimeCommon.h:62
struct CUevent_st * cudaEvent_t
Forward declaration of cudaEvent_t.
Definition: NvInferRuntimeCommon.h:113
virtual void configureWithFormat(const Dims *inputDims, int nbInputs, const Dims *outputDims, int nbOutputs, DataType type, PluginFormat format, int maxBatchSize)=0
Configure the layer.
DataType
The type of weights and tensors.
Definition: NvInferRuntimeCommon.h:163
Definition of plugin versions.
Single gate RNN w/ ReLU activation function.
constexpr int EnumMax< ErrorCode >()
Maximum number of elements in ErrorCode enum.
Definition: NvInferRuntimeCommon.h:1149
virtual void setPluginNamespace(const char *pluginNamespace)=0
Set the namespace of the plugin creator based on the plugin library it belongs to....
Deprecated name of kLINEAR, provided for backwards compatibility.
virtual RefCount incRefCount() noexcept=0
Increments the refcount for the current ErrorRecorder.
constexpr int EnumMax< TensorLocation >()
Maximum number of elements in TensorLocation enum.
Definition: NvInferRuntimeCommon.h:967
virtual size_t getSerializationSize() const =0
Find the size of the serialization buffer required.
virtual size_t getWorkspaceSize(int maxBatchSize) const =0
Find the workspace size required by the layer.
virtual IPluginV2 * createPlugin(const char *name, const PluginFieldCollection *fc)=0
Return a plugin object. Return nullptr in case of error.
TRT_DEPRECATED void configureWithFormat(const Dims *, int, const Dims *, int, DataType, PluginFormat, int) _TENSORRT_OVERRIDE _TENSORRT_FINAL
Deprecated interface inheriting from base class. Derived classes should not implement this....
Definition: NvInferRuntimeCommon.h:742
Elements correspond to different sequence values.
virtual bool supportsFormatCombination(int pos, const PluginTensorDesc *inOut, int nbInputs, int nbOutputs) const =0
Return true if plugin supports the format and datatype for the input/output indexed by pos.
Plugin class for user-implemented layers.
Definition: NvInferRuntimeCommon.h:674
Single gate RNN w/ TANH activation function.
int d[MAX_DIMS]
The extent of each dimension.
Definition: NvInferRuntimeCommon.h:224
virtual bool hasOverflowed() const noexcept=0
Determine if the error stack has overflowed.
Reference counted application-implemented error reporting interface for TensorRT objects.
Definition: NvInferRuntimeCommon.h:1175
nvinfer1::IPluginRegistry * getPluginRegistry()
Return the plugin registry.
virtual void configurePlugin(const Dims *inputDims, int nbInputs, const Dims *outputDims, int nbOutputs, const DataType *inputTypes, const DataType *outputTypes, const bool *inputIsBroadcast, const bool *outputIsBroadcast, PluginFormat floatFormat, int maxBatchSize)=0
Configure the layer with input and output data types.
virtual int getTensorRTVersion() const
Return the API version with which this plugin was built.
Definition: NvInferRuntimeCommon.h:379
Verbose messages with debugging information.
Plugin class for user-implemented layers.
Definition: NvInferRuntimeCommon.h:545
constexpr int EnumMax< TensorFormat >()
Maximum number of elements in TensorFormat enum.
Definition: NvInferRuntimeCommon.h:323
virtual int32_t getNbErrors() const noexcept=0
Return the number of errors.
Informational messages with instructional information.
An application error has been discovered, but TensorRT has recovered or fallen back to a default.
static const int MAX_DIMS
The maximum number of dimensions supported for a tensor.
Definition: NvInferRuntimeCommon.h:222
virtual int getTensorRTVersion() const
Return the version of the API the plugin creator was compiled with.
Definition: NvInferRuntimeCommon.h:843
int getInferLibVersion()
Return the library version number.
virtual void detachFromContext()
Detach the plugin object from its execution context.
Definition: NvInferRuntimeCommon.h:637
virtual IPluginV2 * clone() const =0
Clone the plugin object. This copies over internal plugin parameters and returns a new plugin object ...
virtual IPluginCreator *const * getPluginCreatorList(int *numCreators) const noexcept=0
Return all the registered plugin creators and the number of registered plugin creators....
int nbFields
Number of PluginField entries.
Definition: NvInferRuntimeCommon.h:825
virtual IPluginCreator * getPluginCreator(const char *pluginType, const char *pluginVersion, const char *pluginNamespace="") noexcept=0
Return plugin creator based on plugin type, version and namespace associated with plugin during netwo...
virtual const char * getPluginNamespace() const =0
Return the namespace of the plugin object.
Elements correspond to different batch index.
Fields that a plugin might see for an input or output.
Definition: NvInferRuntimeCommon.h:338
#define NV_TENSORRT_MAJOR
TensorRT major version.
Definition: NvInferVersion.h:59
virtual ~IGpuAllocator()
Definition: NvInferRuntimeCommon.h:1009
virtual void configurePlugin(const PluginTensorDesc *in, int nbInput, const PluginTensorDesc *out, int nbOutput)=0
Configure the layer.
virtual IPluginV2 * deserializePlugin(const char *name, const void *serialData, size_t serialLength)=0
Called during deserialization of plugin layer. Return a plugin object.
TRT_DEPRECATED bool supportsFormat(DataType, PluginFormat) const _TENSORRT_OVERRIDE _TENSORRT_FINAL
Deprecated interface inheriting from base class. Derived classes should not implement this....
Definition: NvInferRuntimeCommon.h:762
IEEE 16-bit floating-point format.
virtual void clear() noexcept=0
Clear the error stack on the error recorder.
nvinfer1::Dims field type.
virtual const PluginFieldCollection * getFieldNames()=0
Return a list of fields that needs to be passed to createPlugin.
Selu activation: x>0 ? beta * x : beta * (alpha*exp(x) - alpha)
Application-implemented class for controlling allocation on the GPU.
Definition: NvInferRuntimeCommon.h:977
virtual Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims)=0
Get the dimension of an output tensor.
const void * data
Plugin field attribute data.
Definition: NvInferRuntimeCommon.h:803
#define NV_TENSORRT_PATCH
TensorRT patch version.
Definition: NvInferVersion.h:61
ErrorCode
Error codes that can be returned by TensorRT during execution.
Definition: NvInferRuntimeCommon.h:1059
Definition: NvInferRuntimeCommon.h:823
Structure containing plugin attribute field names and associated data This information can be parsed ...
Definition: NvInferRuntimeCommon.h:793
Signed 32-bit integer format.
void configureWithFormat(const Dims *, int, const Dims *, int, DataType, PluginFormat, int) _TENSORRT_OVERRIDE
Derived classes should not implement this. In a C++11 API it would be override final.
Definition: NvInferRuntimeCommon.h:661
Elements correspond to different spatial data.
virtual void attachToContext(cudnnContext *, cublasContext *, IGpuAllocator *)
Attach the plugin object to an execution context and grant the plugin the access to some context reso...
Definition: NvInferRuntimeCommon.h:629
const char * name
Plugin field attribute name.
Definition: NvInferRuntimeCommon.h:799