Go to the documentation of this file.
50 #ifndef NV_INFER_RUNTIME_COMMON_H
51 #define NV_INFER_RUNTIME_COMMON_H
57 #if __cplusplus >= 201103L
58 #define _TENSORRT_FINAL final
59 #define _TENSORRT_OVERRIDE override
61 #define _TENSORRT_FINAL
62 #define _TENSORRT_OVERRIDE
66 #if __cplusplus >= 201402L
67 #define TRT_DEPRECATED [[deprecated]]
69 #define TRT_DEPRECATED_ENUM
71 #define TRT_DEPRECATED_ENUM TRT_DEPRECATED
74 #define TRT_DEPRECATED_API __declspec(dllexport)
76 #define TRT_DEPRECATED_API [[deprecated]] __attribute__((visibility("default")))
80 #define TRT_DEPRECATED
81 #define TRT_DEPRECATED_ENUM
82 #define TRT_DEPRECATED_API __declspec(dllexport)
84 #define TRT_DEPRECATED __attribute__((deprecated))
85 #define TRT_DEPRECATED_ENUM
86 #define TRT_DEPRECATED_API __attribute__((deprecated, visibility("default")))
91 #ifdef TENSORRT_BUILD_LIB
93 #define TENSORRTAPI __declspec(dllexport)
95 #define TENSORRTAPI __attribute__((visibility("default")))
112 struct cublasContext;
118 static const int32_t NV_TENSORRT_VERSION
129 class IErrorRecorder;
133 template <
typename T>
134 constexpr
inline int32_t
EnumMax();
444 return NV_TENSORRT_VERSION;
560 virtual int32_t
enqueue(int32_t batchSize, const
void* const* inputs,
void** outputs,
void* workspace,
638 int32_t outputIndex,
const bool* inputIsBroadcasted, int32_t nbInputs)
const TRTNOEXCEPT = 0;
683 virtual void configurePlugin(
const Dims* inputDims, int32_t nbInputs,
const Dims* outputDims, int32_t nbOutputs,
684 const DataType* inputTypes,
const DataType* outputTypes,
const bool* inputIsBroadcast,
900 PluginField(
const char* name_ =
nullptr,
const void* data_ =
nullptr,
const PluginFieldType type_ = PluginFieldType::kUNKNOWN, int32_t length_ = 0)
931 return NV_TENSORRT_VERSION;
1083 virtual void*
allocate(uint64_t size, uint64_t alignment, uint32_t flags)
TRTNOEXCEPT = 0;
1139 constexpr
inline int32_t EnumMax<ILogger::Severity>()
1348 virtual
void clear() noexcept = 0;
1401 extern "C" TENSORRTAPI
void* createSafeInferRuntime_INTERNAL(
void* logger, int32_t version);
1429 template <
typename T>
1438 #define REGISTER_TENSORRT_PLUGIN(name) \
1439 static nvinfer1::PluginRegistrar<name> pluginRegistrar##name {}
1443 #endif // NV_INFER_RUNTIME_COMMON_H
Hard sigmoid activation: max(0, min(1, alpha*x+beta))
Application error has been discovered. TensorRT has recovered or fallen back to a default.
virtual void free(void *memory)=0
virtual int32_t getTensorRTVersion() const
Return the version of the API the plugin creator was compiled with.
Definition: NvInferRuntimeCommon.h:929
Internal error has occurred. Execution is unrecoverable.
Parametric softplus activation: alpha*log(exp(beta*x)+1)
virtual bool supportsFormat(DataType type, PluginFormat format) const =0
Check format support.
virtual const char * getPluginType() const =0
Return the plugin type. Should match the plugin name returned by the corresponding plugin creator.
virtual IErrorRecorder * getErrorRecorder() const noexcept=0
set the ErrorRecorder assigned to this interface.
int32_t nbDims
The number of dimensions.
Definition: NvInferRuntimeCommon.h:231
virtual IPluginCreator *const * getPluginCreatorList(int32_t *numCreators) const noexcept=0
Return all the registered plugin creators and the number of registered plugin creators....
virtual const char * getPluginNamespace() const =0
Return the namespace of the plugin creator object.
virtual bool isOutputBroadcastAcrossBatch(int32_t outputIndex, const bool *inputIsBroadcasted, int32_t nbInputs) const =0
Return true if output tensor is broadcast across a batch.
virtual const char * getPluginName() const =0
Return the plugin name.
virtual IPluginV2Ext * clone() const _TENSORRT_OVERRIDE=0
Clone the plugin object. This copies over internal plugin parameters as well and returns a new plugin...
Signed 32-bit integer format.
Per-channel coefficients.
int32_t getTensorRTVersion() const _TENSORRT_OVERRIDE
Return the API version with which this plugin was built. The upper byte reserved by TensorRT and is u...
Definition: NvInferRuntimeCommon.h:725
void configureWithFormat(const Dims *, int32_t, const Dims *, int32_t, DataType, PluginFormat, int32_t) _TENSORRT_OVERRIDE
Derived classes should not implement this. In a C++11 API it would be override final.
Definition: NvInferRuntimeCommon.h:733
Register the plugin creator to the registry The static registry object will be instantiated when the ...
Definition: NvInferRuntimeCommon.h:1430
virtual void destroy()=0
Destroy the plugin object. This will be called when the network, builder or engine is destroyed.
const PluginField * fields
Pointer to PluginField entries.
Definition: NvInferRuntimeCommon.h:912
ErrorCode
Error codes that can be returned by TensorRT during execution.
Definition: NvInferRuntimeCommon.h:1149
nvinfer1::Dims field type.
TRT_DEPRECATED int32_t getTensorRTVersion() const _TENSORRT_OVERRIDE
Return the API version with which this plugin was built. The upper byte is reserved by TensorRT and i...
Definition: NvInferRuntimeCommon.h:812
Selu activation: x>0 ? beta * x : beta * (alpha*exp(x) - alpha)
8-bit boolean. 0 = false, 1 = true, other values undefined.
virtual ErrorCode getErrorCode(int32_t errorIdx) const noexcept=0
Returns the ErrorCode enumeration.
virtual void setPluginNamespace(const char *pluginNamespace)=0
Set the namespace that this plugin object belongs to. Ideally, all plugin objects from the same plugi...
Clip activation: max(alpha, min(beta, x))
Plugin class for user-implemented layers.
Definition: NvInferRuntimeCommon.h:433
Scaled tanh activation: alpha*tanh(beta*x)
virtual const char * getPluginVersion() const =0
Return the plugin version. Should match the plugin version returned by the corresponding plugin creat...
uint32_t TensorFormats
It is capable of representing one or more TensorFormat by binary OR operations, e....
Definition: NvInferRuntimeCommon.h:243
Softsign activation: x / (1+|x|)
int32_t RefCount
Definition: NvInferRuntimeCommon.h:1277
virtual bool supportsFormatCombination(int32_t pos, const PluginTensorDesc *inOut, int32_t nbInputs, int32_t nbOutputs) const =0
Return true if plugin supports the format and datatype for the input/output indexed by pos.
IEEE 16-bit floating-point format.
virtual RefCount decRefCount() noexcept=0
Decrements the refcount for the current ErrorRecorder.
virtual ErrorDesc getErrorDesc(int32_t errorIdx) const noexcept=0
Returns the c-style string description of the error.
Enable Int8 layer selection, with FP32 fallback with FP16 fallback if kFP16 also specified.
virtual void setErrorRecorder(IErrorRecorder *recorder) noexcept=0
Set the ErrorRecorder for this interface.
virtual const char * getPluginVersion() const =0
Return the plugin version.
TensorFormat
Format of the input/output tensors.
Definition: NvInferRuntimeCommon.h:258
Plugin creator class for user implemented layers.
Definition: NvInferRuntimeCommon.h:923
Single registration point for all plugins in an application. It is used to find plugin implementation...
Definition: NvInferRuntimeCommon.h:991
Structure to define the dimensions of a tensor.
Definition: NvInferRuntimeCommon.h:227
virtual bool canBroadcastInputAcrossBatch(int32_t inputIndex) const =0
Return true if plugin can use input that is broadcast across batch without replication.
PluginFieldType
Definition: NvInferRuntimeCommon.h:859
Application-implemented logging interface for the builder, engine and runtime.
Definition: NvInferRuntimeCommon.h:1109
virtual void serialize(void *buffer) const =0
Serialize the layer.
constexpr int32_t EnumMax< ErrorCode >()
Maximum number of elements in ErrorCode enum.
Definition: NvInferRuntimeCommon.h:1240
ActivationType
Enumerates the types of activation to perform in an activation layer.
Definition: NvInferRuntimeCommon.h:141
constexpr int32_t EnumMax< TensorLocation >()
Maximum number of elements in TensorLocation enum.
Definition: NvInferRuntimeCommon.h:1056
Application error has occurred.
#define TRTNOEXCEPT
Definition: NvInferRuntimeCommon.h:103
int32_t d[MAX_DIMS]
The extent of each dimension.
Definition: NvInferRuntimeCommon.h:232
TRT_DEPRECATED void configurePlugin(const Dims *, int32_t, const Dims *, int32_t, const DataType *, const DataType *, const bool *, const bool *, PluginFormat, int32_t) _TENSORRT_OVERRIDE _TENSORRT_FINAL
Deprecated interface inheriting from base class. Derived classes should not implement this....
Definition: NvInferRuntimeCommon.h:836
Single gate RNN w/ ReLU activation function.
virtual nvinfer1::DataType getOutputDataType(int32_t index, const nvinfer1::DataType *inputTypes, int32_t nbInputs) const =0
Return the DataType of the plugin output at the requested index. The default behavior should be to re...
int32_t length
Number of data entries in the Plugin attribute.
Definition: NvInferRuntimeCommon.h:898
nvinfer1::ILogger * getLogger()
Return the logger object.
constexpr int32_t EnumMax< DimensionType >()
Maximum number of elements in DimensionType enum.
Definition: NvInferRuntimeCommon.h:208
PluginFieldType type
Plugin field attribute type.
Definition: NvInferRuntimeCommon.h:894
DataType type
Definition: NvInferRuntimeCommon.h:403
TRT_DEPRECATED DimensionType type[MAX_DIMS]
Definition: NvInferRuntimeCommon.h:233
virtual bool reportError(ErrorCode val, ErrorDesc desc) noexcept=0
report an error to the error recorder with the corresponding enum and description.
constexpr int32_t EnumMax< TensorFormat >()
Maximum number of elements in TensorFormat enum.
Definition: NvInferRuntimeCommon.h:385
virtual void log(Severity severity, const char *msg)=0
constexpr int32_t EnumMax< DataType >()
Maximum number of elements in DataType enum.
Definition: NvInferRuntimeCommon.h:189
The TensorRT API version 1 namespace.
virtual void * allocate(uint64_t size, uint64_t alignment, uint32_t flags)=0
Thresholded ReLU activation: x>alpha ? x : 0.
const char * ErrorDesc
Definition: NvInferRuntimeCommon.h:1272
#define NV_TENSORRT_MINOR
TensorRT minor version.
Definition: NvInferVersion.h:60
virtual int32_t getTensorRTVersion() const
Return the API version with which this plugin was built.
Definition: NvInferRuntimeCommon.h:442
struct CUstream_st * cudaStream_t
Forward declaration of cudaStream_t.
Definition: NvInferRuntimeCommon.h:115
virtual void terminate()=0
Release resources acquired during plugin layer initialization. This is called when the engine is dest...
virtual bool registerCreator(IPluginCreator &creator, const char *pluginNamespace) noexcept=0
Register a plugin creator. Returns false if one with same type is already registered.
#define _TENSORRT_OVERRIDE
Items that are marked as deprecated will be removed in a future release.
Definition: NvInferRuntimeCommon.h:62
virtual void configurePlugin(const PluginTensorDesc *in, int32_t nbInput, const PluginTensorDesc *out, int32_t nbOutput)=0
Configure the layer.
struct CUevent_st * cudaEvent_t
Forward declaration of cudaEvent_t.
Definition: NvInferRuntimeCommon.h:116
Definition of plugin versions.
Perform the normal matrix multiplication in the first recurrent layer.
virtual void setPluginNamespace(const char *pluginNamespace)=0
Set the namespace of the plugin creator based on the plugin library it belongs to....
int32_t nbFields
Number of PluginField entries.
Definition: NvInferRuntimeCommon.h:911
virtual RefCount incRefCount() noexcept=0
Increments the refcount for the current ErrorRecorder.
virtual size_t getSerializationSize() const =0
Find the size of the serialization buffer required.
DataType
The type of weights and tensors.
Definition: NvInferRuntimeCommon.h:169
Elu activation: x>=0 ? x : alpha * (exp(x) - 1).
virtual size_t getWorkspaceSize(int32_t maxBatchSize) const =0
Find the workspace size required by the layer.
virtual IPluginV2 * createPlugin(const char *name, const PluginFieldCollection *fc)=0
Return a plugin object. Return nullptr in case of error.
Severity
Definition: NvInferRuntimeCommon.h:1117
static const int32_t MAX_DIMS
The maximum number of dimensions supported for a tensor.
Definition: NvInferRuntimeCommon.h:230
Plugin class for user-implemented layers.
Definition: NvInferRuntimeCommon.h:749
LeakyRelu activation: x>=0 ? x : alpha * x.
int32_t getInferLibVersion()
Return the library version number.
virtual bool hasOverflowed() const noexcept=0
Determine if the error stack has overflowed.
constexpr int32_t EnumMax< ActivationType >()
Maximum number of elements in ActivationType enum.
Definition: NvInferRuntimeCommon.h:159
Reference counted application-implemented error reporting interface for TensorRT objects.
Definition: NvInferRuntimeCommon.h:1266
nvinfer1::IPluginRegistry * getPluginRegistry()
Return the plugin registry.
Verbose messages with debugging information.
virtual Dims getOutputDimensions(int32_t index, const Dims *inputs, int32_t nbInputDims)=0
Get the dimension of an output tensor.
Plugin class for user-implemented layers.
Definition: NvInferRuntimeCommon.h:614
virtual int32_t getNbErrors() const noexcept=0
Return the number of errors.
32-bit floating point format.
virtual void detachFromContext()
Detach the plugin object from its execution context.
Definition: NvInferRuntimeCommon.h:708
virtual IPluginV2 * clone() const =0
Clone the plugin object. This copies over internal plugin parameters and returns a new plugin object ...
DimensionType
The type of data encoded across this dimension.
Definition: NvInferRuntimeCommon.h:198
virtual int32_t enqueue(int32_t batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream)=0
Execute the layer.
TRT_DEPRECATED void configureWithFormat(const Dims *, int32_t, const Dims *, int32_t, DataType, PluginFormat, int32_t) _TENSORRT_OVERRIDE _TENSORRT_FINAL
Deprecated interface inheriting from base class. Derived classes should not implement this....
Definition: NvInferRuntimeCommon.h:824
virtual IPluginCreator * getPluginCreator(const char *pluginType, const char *pluginVersion, const char *pluginNamespace="") noexcept=0
Return plugin creator based on plugin type, version and namespace associated with plugin during netwo...
Informational messages with instructional information.
virtual const char * getPluginNamespace() const =0
Return the namespace of the plugin object.
Fields that a plugin might see for an input or output.
Definition: NvInferRuntimeCommon.h:400
#define NV_TENSORRT_MAJOR
TensorRT major version.
Definition: NvInferVersion.h:59
virtual ~IGpuAllocator()
Definition: NvInferRuntimeCommon.h:1098
virtual IPluginV2 * deserializePlugin(const char *name, const void *serialData, size_t serialLength)=0
Called during deserialization of plugin layer. Return a plugin object.
TRT_DEPRECATED bool supportsFormat(DataType, PluginFormat) const _TENSORRT_OVERRIDE _TENSORRT_FINAL
Deprecated interface inheriting from base class. Derived classes should not implement this....
Definition: NvInferRuntimeCommon.h:848
virtual void clear() noexcept=0
Clear the error stack on the error recorder.
constexpr int32_t EnumMax()
Forward declare IGpuAllocator for use in other interfaces.
Single gate RNN w/ TANH activation function.
virtual void configureWithFormat(const Dims *inputDims, int32_t nbInputs, const Dims *outputDims, int32_t nbOutputs, DataType type, PluginFormat format, int32_t maxBatchSize)=0
Configure the layer.
virtual const PluginFieldCollection * getFieldNames()=0
Return a list of fields that needs to be passed to createPlugin.
Elements correspond to different sequence values.
Application-implemented class for controlling allocation on the GPU.
Definition: NvInferRuntimeCommon.h:1066
virtual int32_t initialize()=0
Initialize the layer for execution. This is called when the engine is created.
Elements correspond to different spatial data.
const void * data
Plugin field attribute data.
Definition: NvInferRuntimeCommon.h:889
#define NV_TENSORRT_PATCH
TensorRT patch version.
Definition: NvInferVersion.h:61
Definition: NvInferRuntimeCommon.h:909
Structure containing plugin attribute field names and associated data This information can be parsed ...
Definition: NvInferRuntimeCommon.h:879
Elements correspond to different batch index.
TensorLocation
The location for tensor data storage, device or host.
Definition: NvInferRuntimeCommon.h:1048
virtual void configurePlugin(const Dims *inputDims, int32_t nbInputs, const Dims *outputDims, int32_t nbOutputs, const DataType *inputTypes, const DataType *outputTypes, const bool *inputIsBroadcast, const bool *outputIsBroadcast, PluginFormat floatFormat, int32_t maxBatchSize)=0
Configure the layer with input and output data types.
virtual void attachToContext(cudnnContext *, cublasContext *, IGpuAllocator *)
Attach the plugin object to an execution context and grant the plugin the access to some context reso...
Definition: NvInferRuntimeCommon.h:700
const char * name
Plugin field attribute name.
Definition: NvInferRuntimeCommon.h:885
virtual int32_t getNbOutputs() const =0
Get the number of outputs from the layer.