13#ifndef NV_INFER_RUNTIME_COMMON_H
14#define NV_INFER_RUNTIME_COMMON_H
19#include <cuda_runtime_api.h>
22#if __cplusplus >= 201402L
23#define TRT_DEPRECATED [[deprecated]]
25#define TRT_DEPRECATED_ENUM
27#define TRT_DEPRECATED_ENUM TRT_DEPRECATED
30#define TRT_DEPRECATED_API __declspec(dllexport)
32#define TRT_DEPRECATED_API [[deprecated]] __attribute__((visibility("default")))
37#define TRT_DEPRECATED_ENUM
38#define TRT_DEPRECATED_API __declspec(dllexport)
40#define TRT_DEPRECATED __attribute__((deprecated))
41#define TRT_DEPRECATED_ENUM
42#define TRT_DEPRECATED_API __attribute__((deprecated, visibility("default")))
47#ifdef TENSORRT_BUILD_LIB
49#define TENSORRTAPI __declspec(dllexport)
51#define TENSORRTAPI __attribute__((visibility("default")))
73#define NV_TENSORRT_VERSION nvinfer1::kNV_TENSORRT_VERSION_IMPL
82static constexpr int32_t kNV_TENSORRT_VERSION_IMPL
138 static constexpr int32_t kVALUE = 5;
317 static constexpr int32_t kVALUE = 12;
572 virtual int32_t
enqueue(int32_t batchSize,
void const* const* inputs,
void* const* outputs,
void* workspace,
573 cudaStream_t stream) noexcept
601 virtual
void serialize(
void* buffer) const noexcept = 0;
717 int32_t outputIndex,
bool const* inputIsBroadcasted, int32_t nbInputs)
const noexcept
775 DataType const* inputTypes,
DataType const* outputTypes,
bool const* inputIsBroadcast,
776 bool const* outputIsBroadcast,
PluginFormat floatFormat, int32_t maxBatchSize)
noexcept
944 int32_t pos,
PluginTensorDesc const* inOut, int32_t nbInputs, int32_t nbOutputs)
const noexcept
1148 virtual
IPluginV2* deserializePlugin(
AsciiChar const* name,
void const* serialData,
size_t serialLength) noexcept
1247 AsciiChar const*
const pluginNamespace =
"") noexcept
1326 static constexpr int32_t kVALUE = 1;
1530 static constexpr int32_t kVALUE = 5;
1637 static constexpr int32_t kVALUE = 11;
1675 static constexpr size_t kMAX_DESC_LENGTH{127U};
1705 virtual int32_t getNbErrors() const noexcept = 0;
1724 virtual
ErrorCode getErrorCode(int32_t errorIdx) const noexcept = 0;
1745 virtual
ErrorDesc getErrorDesc(int32_t errorIdx) const noexcept = 0;
1761 virtual
bool hasOverflowed() const noexcept = 0;
1777 virtual
void clear() noexcept = 0;
#define TENSORRTAPI
Definition: NvInferRuntimeCommon.h:54
int32_t getInferLibVersion() noexcept
Return the library version number.
#define NV_TENSORRT_VERSION
Definition: NvInferRuntimeCommon.h:73
#define TRT_DEPRECATED
Definition: NvInferRuntimeCommon.h:40
#define NV_TENSORRT_MINOR
TensorRT minor version.
Definition: NvInferVersion.h:23
#define NV_TENSORRT_MAJOR
TensorRT major version.
Definition: NvInferVersion.h:22
#define NV_TENSORRT_PATCH
TensorRT patch version.
Definition: NvInferVersion.h:24
Definition: NvInferRuntimeCommon.h:153
int32_t nbDims
The rank (number of dimensions).
Definition: NvInferRuntimeCommon.h:158
static constexpr int32_t MAX_DIMS
The maximum rank (number of dimensions) supported for a tensor.
Definition: NvInferRuntimeCommon.h:156
int32_t d[MAX_DIMS]
The extent of each dimension.
Definition: NvInferRuntimeCommon.h:160
Reference counted application-implemented error reporting interface for TensorRT objects.
Definition: NvInferRuntimeCommon.h:1665
virtual ~IErrorRecorder() noexcept=default
char const * ErrorDesc
Definition: NvInferRuntimeCommon.h:1670
int32_t RefCount
Definition: NvInferRuntimeCommon.h:1680
Application-implemented class for controlling allocation on the GPU.
Definition: NvInferRuntimeCommon.h:1338
virtual bool deallocate(void *const memory) noexcept
Definition: NvInferRuntimeCommon.h:1448
virtual void * reallocate(void *, uint64_t, uint64_t) noexcept
Definition: NvInferRuntimeCommon.h:1423
virtual ~IGpuAllocator()=default
virtual void * allocate(uint64_t const size, uint64_t const alignment, AllocatorFlags const flags) noexcept=0
virtual TRT_DEPRECATED void free(void *const memory) noexcept=0
Application-implemented logging interface for the builder, refitter and runtime.
Definition: NvInferRuntimeCommon.h:1476
virtual ~ILogger()=default
Severity
Definition: NvInferRuntimeCommon.h:1484
virtual void log(Severity severity, AsciiChar const *msg) noexcept=0
Plugin creator class for user implemented layers.
Definition: NvInferRuntimeCommon.h:1074
virtual int32_t getTensorRTVersion() const noexcept
Return the version of the API the plugin creator was compiled with.
Definition: NvInferRuntimeCommon.h:1083
virtual AsciiChar const * getPluginName() const noexcept=0
Return the plugin name.
Single registration point for all plugins in an application. It is used to find plugin implementation...
Definition: NvInferRuntimeCommon.h:1210
virtual bool registerCreator(IPluginCreator &creator, AsciiChar const *const pluginNamespace) noexcept=0
Register a plugin creator. Returns false if one with same type is already registered.
virtual IPluginCreator * getPluginCreator(AsciiChar const *const pluginName, AsciiChar const *const pluginVersion, AsciiChar const *const pluginNamespace="") noexcept=0
Return plugin creator based on plugin name, version, and namespace associated with plugin during netw...
virtual IPluginCreator *const * getPluginCreatorList(int32_t *const numCreators) const noexcept=0
Return all the registered plugin creators and the number of registered plugin creators....
Plugin class for user-implemented layers.
Definition: NvInferRuntimeCommon.h:680
~IPluginV2Ext() override=default
virtual bool canBroadcastInputAcrossBatch(int32_t inputIndex) const noexcept=0
Return true if plugin can use input that is broadcast across batch without replication.
void configureWithFormat(Dims const *, int32_t, Dims const *, int32_t, DataType, PluginFormat, int32_t) noexcept override
Derived classes should not implement this. In a C++11 API it would be override final.
Definition: NvInferRuntimeCommon.h:867
virtual bool isOutputBroadcastAcrossBatch(int32_t outputIndex, bool const *inputIsBroadcasted, int32_t nbInputs) const noexcept=0
Return true if output tensor is broadcast across a batch.
IPluginV2Ext * clone() const noexcept override=0
Clone the plugin object. This copies over internal plugin parameters as well and returns a new plugin...
virtual void configurePlugin(Dims const *inputDims, int32_t nbInputs, Dims const *outputDims, int32_t nbOutputs, DataType const *inputTypes, DataType const *outputTypes, bool const *inputIsBroadcast, bool const *outputIsBroadcast, PluginFormat floatFormat, int32_t maxBatchSize) noexcept=0
Configure the layer with input and output data types.
virtual void detachFromContext() noexcept
Detach the plugin object from its execution context.
Definition: NvInferRuntimeCommon.h:823
virtual void attachToContext(cudnnContext *, cublasContext *, IGpuAllocator *) noexcept
Attach the plugin object to an execution context and grant the plugin the access to some context reso...
Definition: NvInferRuntimeCommon.h:805
virtual nvinfer1::DataType getOutputDataType(int32_t index, nvinfer1::DataType const *inputTypes, int32_t nbInputs) const noexcept=0
Return the DataType of the plugin output at the requested index.
Plugin class for user-implemented layers.
Definition: NvInferRuntimeCommon.h:373
virtual AsciiChar const * getPluginType() const noexcept=0
Return the plugin type. Should match the plugin name returned by the corresponding plugin creator.
virtual void terminate() noexcept=0
Release resources acquired during plugin layer initialization. This is called when the engine is dest...
virtual int32_t getTensorRTVersion() const noexcept
Return the API version with which this plugin was built.
Definition: NvInferRuntimeCommon.h:385
virtual void setPluginNamespace(AsciiChar const *pluginNamespace) noexcept=0
Set the namespace that this plugin object belongs to. Ideally, all plugin objects from the same plugi...
virtual void configureWithFormat(Dims const *inputDims, int32_t nbInputs, Dims const *outputDims, int32_t nbOutputs, DataType type, PluginFormat format, int32_t maxBatchSize) noexcept=0
Configure the layer.
virtual void serialize(void *buffer) const noexcept=0
Serialize the layer.
virtual void destroy() noexcept=0
Destroy the plugin object. This will be called when the network, builder or engine is destroyed.
virtual AsciiChar const * getPluginVersion() const noexcept=0
Return the plugin version. Should match the plugin version returned by the corresponding plugin creat...
virtual Dims getOutputDimensions(int32_t index, Dims const *inputs, int32_t nbInputDims) noexcept=0
Get the dimension of an output tensor.
virtual IPluginV2 * clone() const noexcept=0
Clone the plugin object. This copies over internal plugin parameters and returns a new plugin object ...
virtual int32_t enqueue(int32_t batchSize, void const *const *inputs, void *const *outputs, void *workspace, cudaStream_t stream) noexcept=0
Execute the layer.
virtual size_t getSerializationSize() const noexcept=0
Find the size of the serialization buffer required.
virtual size_t getWorkspaceSize(int32_t maxBatchSize) const noexcept=0
Find the workspace size required by the layer.
virtual int32_t getNbOutputs() const noexcept=0
Get the number of outputs from the layer.
virtual AsciiChar const * getPluginNamespace() const noexcept=0
Return the namespace of the plugin object.
virtual int32_t initialize() noexcept=0
Initialize the layer for execution. This is called when the engine is created.
virtual bool supportsFormat(DataType type, PluginFormat format) const noexcept=0
Check format support.
Plugin class for user-implemented layers.
Definition: NvInferRuntimeCommon.h:883
int32_t getTensorRTVersion() const noexcept override
Return the API version with which this plugin was built. The upper byte is reserved by TensorRT and i...
Definition: NvInferRuntimeCommon.h:971
virtual void configurePlugin(PluginTensorDesc const *in, int32_t nbInput, PluginTensorDesc const *out, int32_t nbOutput) noexcept=0
Configure the layer.
virtual bool supportsFormatCombination(int32_t pos, PluginTensorDesc const *inOut, int32_t nbInputs, int32_t nbOutputs) const noexcept=0
Return true if plugin supports the format and datatype for the input/output indexed by pos.
Structure containing plugin attribute field names and associated data This information can be parsed ...
Definition: NvInferRuntimeCommon.h:1026
AsciiChar const * name
Plugin field attribute name.
Definition: NvInferRuntimeCommon.h:1031
PluginField(AsciiChar const *const name_=nullptr, void const *const data_=nullptr, PluginFieldType const type_=PluginFieldType::kUNKNOWN, int32_t const length_=0) noexcept
Definition: NvInferRuntimeCommon.h:1046
void const * data
Plugin field attribute data.
Definition: NvInferRuntimeCommon.h:1035
int32_t length
Number of data entries in the Plugin attribute.
Definition: NvInferRuntimeCommon.h:1044
PluginFieldType type
Plugin field attribute type.
Definition: NvInferRuntimeCommon.h:1040
The TensorRT API version 1 namespace.
ErrorCode
Error codes that can be returned by TensorRT during execution.
Definition: NvInferRuntimeCommon.h:1540
PluginFieldType
Definition: NvInferRuntimeCommon.h:997
@ kUNKNOWN
Unknown field type.
@ kFLOAT32
FP32 field type.
@ kINT16
INT16 field type.
@ kDIMS
nvinfer1::Dims field type.
@ kFLOAT64
FP64 field type.
@ kFLOAT16
FP16 field type.
char_t AsciiChar
AsciiChar is the type used by TensorRT to represent valid ASCII characters.
Definition: NvInferRuntimeCommon.h:88
char char_t
char_t is the type used by TensorRT to represent all valid characters.
Definition: NvInferRuntimeCommon.h:86
@ kV2_DYNAMICEXT
IPluginV2DynamicExt.
@ kV2_IOEXT
IPluginV2IOExt.
DataType
The type of weights and tensors.
Definition: NvInferRuntimeCommon.h:114
@ kFLOAT
32-bit floating point format.
@ kBOOL
8-bit boolean. 0 = false, 1 = true, other values undefined.
@ kHALF
IEEE 16-bit floating-point format.
@ kINT8
8-bit integer representing a quantized floating-point value.
@ kINT32
Signed 32-bit integer format.
TensorFormat PluginFormat
PluginFormat is reserved for backward compatibility.
Definition: NvInferRuntimeCommon.h:308
@ kINT8
Enable Int8 layer selection, with FP32 fallback with FP16 fallback if kFP16 also specified.
TensorFormat
Format of the input/output tensors.
Definition: NvInferRuntimeCommon.h:183
constexpr int32_t EnumMax() noexcept
Maximum number of elements in an enumeration type.
Definition: NvInferRuntimeCommon.h:104
AllocatorFlag
Definition: NvInferRuntimeCommon.h:1316
@ kRESIZABLE
TensorRT may call realloc() on this allocation.
uint32_t AllocatorFlags
Definition: NvInferRuntimeCommon.h:1330
Definition of plugin versions.
Plugin field collection struct.
Definition: NvInferRuntimeCommon.h:1058
PluginField const * fields
Pointer to PluginField entries.
Definition: NvInferRuntimeCommon.h:1062
int32_t nbFields
Number of PluginField entries.
Definition: NvInferRuntimeCommon.h:1060
Fields that a plugin might see for an input or output.
Definition: NvInferRuntimeCommon.h:332
DataType type
Definition: NvInferRuntimeCommon.h:336
Dims dims
Dimensions.
Definition: NvInferRuntimeCommon.h:334
TensorFormat format
Tensor format.
Definition: NvInferRuntimeCommon.h:338
float scale
Scale for INT8 data type.
Definition: NvInferRuntimeCommon.h:340
Declaration of EnumMaxImpl struct to store maximum number of elements in an enumeration type.
Definition: NvInferRuntimeCommon.h:99