56 #define NV_TENSORRT_MAJOR 5
57 #define NV_TENSORRT_MINOR 1
58 #define NV_TENSORRT_PATCH 5
59 #define NV_TENSORRT_BUILD 0
61 #define NV_TENSORRT_SONAME_MAJOR 5
62 #define NV_TENSORRT_SONAME_MINOR 1
63 #define NV_TENSORRT_SONAME_PATCH 5
65 #if __cplusplus > 201103L
66 #define _TENSORRT_FINAL final
67 #define _TENSORRT_OVERRIDE override
69 #define _TENSORRT_FINAL
70 #define _TENSORRT_OVERRIDE
74 #ifdef TENSORRT_BUILD_LIB
76 #define TENSORRTAPI __declspec(dllexport)
78 #define TENSORRTAPI __attribute__((visibility("default")))
102 struct cublasContext;
118 template <
typename T>
243 :
Dims2(height, width)
253 int&
h() {
return d[0]; }
260 int h()
const {
return d[0]; }
267 int&
w() {
return d[1]; }
274 int w()
const {
return d[1]; }
290 d[0] =
d[1] =
d[2] = 0;
334 :
Dims3(channels, height, width)
345 int&
c() {
return d[0]; }
352 int c()
const {
return d[0]; }
359 int&
h() {
return d[1]; }
366 int h()
const {
return d[1]; }
373 int&
w() {
return d[2]; }
380 int w()
const {
return d[2]; }
396 d[0] =
d[1] =
d[2] =
d[3] = 0;
407 Dims4(
int d0,
int d1,
int d2,
int d3)
443 DimsNCHW(
int batchSize,
int channels,
int height,
int width)
444 :
Dims4(batchSize, channels, height, width)
456 int&
n() {
return d[0]; }
463 int n()
const {
return d[0]; }
470 int&
c() {
return d[1]; }
477 int c()
const {
return d[1]; }
484 int&
h() {
return d[2]; }
491 int h()
const {
return d[2]; }
498 int&
w() {
return d[3]; }
505 int w()
const {
return d[3]; }
537 virtual void*
data()
const = 0;
538 virtual std::size_t
size()
const = 0;
627 virtual void setName(
const char* name) = 0;
636 virtual const char*
getName()
const = 0;
820 virtual void setName(
const char* name) = 0;
828 virtual const char*
getName()
const = 0;
1347 virtual void setAlpha(
float alpha) = 0;
1358 virtual void setBeta(
float beta) = 0;
1364 virtual float getAlpha()
const = 0;
1370 virtual float getBeta()
const = 0;
1382 kMAX_AVERAGE_BLEND = 2
1605 virtual void setAlpha(
float alpha) = 0;
1612 virtual float getAlpha()
const = 0;
1620 virtual void setBeta(
float beta) = 0;
1627 virtual float getBeta()
const = 0;
1635 virtual void setK(
float k) = 0;
1642 virtual float getK()
const = 0;
1782 virtual void setAxes(uint32_t axes) = 0;
1789 virtual uint32_t
getAxes()
const = 0;
1817 virtual void setAxis(
int axis) = 0;
1824 virtual int getAxis()
const = 0;
2625 virtual int32_t getLayerCount()
const = 0;
2626 virtual int32_t getHiddenSize()
const = 0;
2627 virtual int32_t getMaxSeqLength()
const = 0;
2628 virtual int32_t getDataLength()
const = 0;
2888 virtual void configure(
const Dims* inputDims,
int nbInputs,
const Dims* outputDims,
int nbOutputs,
int maxBatchSize) = 0;
2924 virtual int enqueue(
int batchSize,
const void*
const* inputs,
void** outputs,
void* workspace,
cudaStream_t stream) = 0;
2940 virtual void serialize(
void* buffer) = 0;
2963 return NV_TENSORRT_VERSION;
3026 return NV_TENSORRT_VERSION;
3125 virtual int enqueue(
int batchSize,
const void*
const* inputs,
void** outputs,
void* workspace,
cudaStream_t stream) = 0;
3141 virtual void serialize(
void* buffer)
const = 0;
3168 class IGpuAllocator;
3243 const bool* inputIsBroadcast,
const bool* outputIsBroadcast,
PluginFormat floatFormat,
int maxBatchSize)
3285 return (0x01000000 | (NV_TENSORRT_VERSION & 0xFFFFFF));
3386 PluginField(
const char* name_ =
nullptr,
const void* data_ =
nullptr,
const PluginFieldType type_ = PluginFieldType::kUNKNOWN,
int length_ = 0)
3892 virtual void setK(
int k) = 0;
3899 virtual int getK()
const = 0;
4830 virtual bool execute(
int batchSize,
void** bindings) = 0;
4900 virtual void setName(
const char* name) = 0;
4907 virtual const char*
getName()
const = 0;
5092 kLEGACY_CALIBRATION = 0,
5093 kENTROPY_CALIBRATION = 1,
5094 kENTROPY_CALIBRATION_2 = 2
5137 virtual bool getBatch(
void* bindings[],
const char* names[],
int nbBindings) = 0;
5296 virtual void*
allocate(uint64_t size, uint64_t alignment, uint32_t flags) = 0;
5305 virtual void free(
void* memory) = 0;
5689 virtual bool setWeights(
const char* layerName,
5732 virtual void destroy() = 0;
5761 virtual IPlugin*
createPlugin(
const char* layerName,
const void* serialData,
size_t serialLength) = 0;
5854 virtual void log(
Severity severity,
const char* msg) = 0;
5860 inline int EnumMax<ILogger::Severity>()
5897 inline IBuilder* createInferBuilder(ILogger& logger)
5907 inline IRefitter* createInferRefitter(ICudaEngine& engine, ILogger& logger)
5917 inline IRuntime* createInferRuntime(ILogger& logger)
5929 template <
typename T>
5938 #define REGISTER_TENSORRT_PLUGIN(name) \
5939 static nvinfer1::PluginRegistrar<name> pluginRegistrar##name {}
DimensionType type[MAX_DIMS]
The type of each dimension.
Definition: NvInfer.h:187
int EnumMax< PluginFormat >()
Maximum number of elements in PluginFormat enum.
Definition: NvInfer.h:2837
virtual bool enqueue(int batchSize, void **bindings, cudaStream_t stream, cudaEvent_t *inputConsumed)=0
Asynchronously execute inference on a batch.
int w() const
Get the width.
Definition: NvInfer.h:505
int n() const
Get the index count.
Definition: NvInfer.h:463
Use SAME padding with prePadding <= postPadding.
An engine for executing inference on a built network.
Definition: NvInfer.h:4930
Substract the second element from the first.
Perform the normal matrix multiplication in the first recurrent layer.
virtual void configureWithFormat(const Dims *inputDims, int nbInputs, const Dims *outputDims, int nbOutputs, DataType type, PluginFormat format, int maxBatchSize)=0
Configure the layer.
DataType
The type of weights and tensors.
Definition: NvInfer.h:125
virtual void configurePlugin(const Dims *inputDims, int nbInputs, const Dims *outputDims, int nbOutputs, const DataType *inputTypes, const DataType *outputTypes, const bool *inputIsBroadcast, const bool *outputIsBroadcast, PluginFormat floatFormat, int maxBatchSize)=0
Configure the layer with input and output data types.
PaddingMode
Enumerates the modes of padding to perform in convolution, deconvolution and pooling layer...
Definition: NvInfer.h:962
virtual void setAverageCountExcludesPadding(bool exclusive)=0
Set whether average pooling uses as a denominator the overlap area between the window and the unpadde...
virtual bool isNetworkInput() const =0
Whether the tensor is a network input.
virtual DimsHW getDilation() const =0
Get the dilation for a convolution.
Severity
Definition: NvInfer.h:5839
virtual void setMinFindIterations(int minFind)=0
Set the number of minimization iterations used when timing layers.
virtual int getMaxBatchSize() const =0
Get the maximum batch size.
int EnumMax< TensorLocation >()
Maximum number of elements in TensorLocation enum.
Definition: NvInfer.h:598
virtual ITensor * getCellState() const =0
Get the initial cell state of the RNN.
virtual size_t getSerializationSize() const =0
Find the size of the serialization buffer required.
virtual IPlugin * createPlugin(const char *layerName, const void *serialData, size_t serialLength)=0
Create a plugin from serialized data.
virtual ITensor * addInput(const char *name, DataType type, Dims dimensions)=0
Add an input tensor to the network.
virtual uint32_t getAxes() const =0
Get the axis along which softmax occurs.
virtual void resetPrecision()=0
reset the computational precision for this layer
virtual void setBias(Weights bias)=0
Set the bias parameters for the RNN.
DimsNCHW(int batchSize, int channels, int height, int width)
Construct a DimsNCHW given batch size, channel count, height and width.
Definition: NvInfer.h:443
virtual void setKernelWeights(Weights weights)=0
Set the kernel weights for the deconvolution.
virtual ReduceOperation getOperation() const =0
Get the reduce operation for the layer.
#define NV_TENSORRT_MAJOR
TensorRT major version.
Definition: NvInfer.h:56
virtual IExecutionContext * createExecutionContext()=0
Create an execution context.
virtual IPluginV2 * createPlugin(const char *name, const PluginFieldCollection *fc)=0
Return a plugin object. Return nullptr in case of error.
virtual int getBatchSize() const =0
Get the batch size used for calibration batches.
RNNOperation
Enumerates the RNN operations that may be performed by an RNN layer.
Definition: NvInfer.h:2193
virtual int getTensorRTVersion() const
Return the API version with which this plugin was built.
Definition: NvInfer.h:3024
virtual void setOperation(ElementWiseOperation type)=0
Set the binary operation for the layer.
virtual void setBroadcastAcrossBatch(bool broadcastAcrossBatch)=0
Set whether to enable broadcast of tensor across the batch.
A Softmax layer in a network definition.
Definition: NvInfer.h:1762
virtual Dims getPrePadding() const =0
Get the pre-padding.
virtual int getNbGroups() const =0
Set the number of groups for a convolution.
virtual Weights getPower() const =0
Get the power value.
virtual void setWeightsForGate(int layerIndex, RNNGateType gate, bool isW, Weights weights)=0
Set the weight parameters for an individual gate in the RNN.
MatrixOperation
Enumerates the operations that may be performed on a tensor by IMatrixMultiplyLayer before multiplica...
Definition: NvInfer.h:3925
Definition: NvInfer.h:3688
Plugin class for user-implemented layers.
Definition: NvInfer.h:3016
virtual double getQuantile() const =0
The quantile (between 0 and 1) that will be used to select the region maximum when the quantile metho...
virtual std::size_t getWorkspaceSize() const =0
Get the amount of workspace the engine uses.
An application error has occurred.
An application error has been discovered, but TensorRT has recovered or fallen back to a default...
virtual Weights getBias() const =0
Get the bias parameter vector for the RNN.
virtual void destroy()=0
Destroy the allocated memory.
virtual bool bindingIsInput(int bindingIndex) const =0
Determine whether a binding is an input binding.
virtual int getGatherAxis() const =0
Get the non-batch dimension axis to gather on.
virtual void setInputMode(RNNInputMode op)=0
Set the input mode of the RNN layer.
virtual const char * getName() const =0
Return the name of a layer.
virtual Weights getScale() const =0
Get the scale value.
virtual DimsHW getKernelSize() const =0
Get the HW kernel size of the deconvolution.
Layer that represents an unary operation.
Definition: NvInfer.h:3542
virtual void destroy()=0
Destroy this object.
virtual DimsHW getStride() const =0
Get the stride of the deconvolution.
PluginFieldType
Definition: NvInfer.h:3346
virtual void setDLACore(int dlaCore)=0
Set the DLA core that the deserialized engine must execute on.
Rectified linear activation.
virtual bool getDebugSync() const =0
Get the debug sync flag.
virtual bool execute(int batchSize, void **bindings)=0
Synchronously execute inference on a batch.
virtual void destroy()=0
Destroy this INetworkDefinition object.
virtual const PluginFieldCollection * getFieldNames()=0
Return a list of fields that needs to be passed to createPlugin.
virtual void setPrePadding(Dims padding)=0
Set the pre-padding.
An Activation layer in a network definition.
Definition: NvInfer.h:1317
TENSORRTAPI void * createInferRuntime_INTERNAL(void *logger, int version)
Internal C entry point for creating IRuntime.
int w() const
Get the width.
Definition: NvInfer.h:380
virtual Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims)=0
Get the dimension of an output tensor.
RNNDirection
Enumerates the RNN direction that may be performed by an RNN layer.
Definition: NvInfer.h:2214
int EnumMax< DataType >()
Maximum number of elements in DataType enum.
Definition: NvInfer.h:134
virtual PaddingMode getPaddingMode() const =0
Get the padding mode.
Layer that represents a Matrix Multiplication.
Definition: NvInfer.h:3971
virtual DimsHW getKernelSize() const =0
Get the HW kernel size of the convolution.
virtual DataType type() const =0
The type of the memory that was allocated.
virtual int getAverageFindIterations() const =0
Query the number of averaging iterations.
Safety restricted capability, TensorRT flow that can only run on DLA/PVA devices. ...
DimsCHW()
Construct an empty DimsCHW object.
Definition: NvInfer.h:319
No operation is performed on the first recurrent layer.
const void * values
The weight values, in a contiguous array.
Definition: NvInfer.h:520
int c() const
Get the channel count.
Definition: NvInfer.h:477
void configure(const Dims *, int, const Dims *, int, int) _TENSORRT_FINAL
Derived classes should not implement this. In a C++11 API it would be override final.
Definition: NvInfer.h:3002
virtual bool getDebugSync() const =0
Query whether the builder will use debug synchronization.
virtual void setAlpha(float alpha)=0
Set the LRN alpha value.
virtual int initialize()=0
Initialize the layer for execution. This is called when the engine is created.
struct CUevent_st * cudaEvent_t
Forward declaration of cudaEvent_t.
Definition: NvInfer.h:106
CalibrationAlgoType getAlgorithm() override
Definition: NvInfer.h:5196
virtual float getDynamicRangeMin() const =0
Get minimum of dynamic range.
virtual Dims getBindingDimensions(int bindingIndex) const =0
Get the dimensions of a binding.
int h() const
Get the height.
Definition: NvInfer.h:491
virtual void setPrecision(DataType dataType)=0
Set the computational precision of this layer.
A convolution layer in a network definition.
Definition: NvInfer.h:991
virtual ITopKLayer * addTopK(ITensor &input, TopKOperation op, int k, uint32_t reduceAxes)=0
Add a TopK layer to the network.
Allows a serialized engine to be deserialized.
Definition: NvInfer.h:5771
virtual void setStart(Dims start)=0
Set the start offset.
virtual Dims getReshapeDimensions() const =0
Get the reshaped dimensions.
virtual void writeHistogramCache(const void *ptr, std::size_t length)=0
Save a histogram cache.
virtual void setPoolingOutputDimensionsFormula(IOutputDimensionsFormula *formula)=0
Set the pooling output dimensions formula.
virtual int getDLACore() const =0
Get the DLA core that the engine executes on.
A RaggedSoftmax layer in a network definition.
Definition: NvInfer.h:4026
virtual unsigned getLayerCount() const =0
Get the number of layers in the RNN.
virtual int getNbOutputs() const =0
Get the number of outputs from the layer.
virtual Dims getStart() const =0
Get the start offset.
virtual void configure(const Dims *inputDims, int nbInputs, const Dims *outputDims, int nbOutputs, int maxBatchSize)=0
Configure the layer.
ScaleMode
Controls how shift, scale and power are applied in a Scale layer.
Definition: NvInfer.h:1653
Plugin class for user-implemented layers.
Definition: NvInfer.h:2849
Layer that represents a constant value.
Definition: NvInfer.h:4052
virtual Dims getPrePadding() const =0
Get the pre-padding.
virtual void setBeta(float beta)=0
Set the beta parameter (must be finite).
virtual DataType getPrecision() const =0
get the computational precision of this layer
virtual ILayer * getLayer(int index) const =0
Get the layer specified by the given index.
A Scale layer in a network definition.
Definition: NvInfer.h:1688
virtual float getBeta() const =0
Get the beta parameter.
virtual bool getBatch(void *bindings[], const char *names[], int nbBindings)=0
Get a batch of input for calibration.
Parametric softplus activation: alpha*log(exp(beta*x)+1)
virtual IRaggedSoftMaxLayer * addRaggedSoftMax(ITensor &input, ITensor &bounds)=0
Add a RaggedSoftMax layer to the network.
virtual int getNbGroups() const =0
Get the number of groups for a deconvolution.
Use SAME padding, with prePadding >= postPadding.
NHWC where C must be a multiple of 8.
virtual bool refitCudaEngine()=0
Updates associated engine. Return true if successful.
int EnumMax< EngineCapability >()
Maximum number of elements in EngineCapability enum.
Definition: NvInfer.h:5269
virtual void setNbOutputMaps(int nbOutputMaps)=0
Set the number of output maps for the convolution.
virtual void log(Severity severity, const char *msg)=0
virtual const char * getBindingName(int bindingIndex) const =0
Retrieve the name corresponding to a binding index.
virtual IRNNv2Layer * addRNNv2(ITensor &input, int32_t layerCount, int32_t hiddenSize, int32_t maxSeqLen, RNNOperation op)=0
Add an layerCount deep RNN layer to the network with hiddenSize internal states that can take a batch...
const PluginField * fields
Pointer to PluginField entries.
Definition: NvInfer.h:3398
virtual bool getFp16Mode() const =0
Query whether 16-bit kernels are permitted.
virtual IScaleLayer * addScale(ITensor &input, ScaleMode mode, Weights shift, Weights scale, Weights power)=0
Add a Scale layer to the network.
virtual void setHiddenState(ITensor &hidden)=0
Set the initial hidden state of the RNN with the provided hidden ITensor.
virtual void setPluginNamespace(const char *pluginNamespace)=0
Set the namespace that this plugin object belongs to. Ideally, all plugin objects from the same plugi...
int EnumMax< RNNOperation >()
Maximum number of elements in RNNOperation enum.
Definition: NvInfer.h:2202
int EnumMax< LayerType >()
Maximum number of elements in LayerType enum.
Definition: NvInfer.h:582
virtual void setAxes(uint32_t axes)=0
Set the axis along which softmax is computed. Currently, only one axis can be set.
static const int MAX_DIMS
The maximum number of dimensions supported for a tensor.
Definition: NvInfer.h:184
virtual void setPrePadding(DimsHW padding)=0
Set the padding that is applied at the start of the tensor.
virtual float getDynamicRange() const =0
Get dynamic range for the tensor.
virtual DataType getBindingDataType(int bindingIndex) const =0
Determine the required data type for a buffer from its binding index.
A fully connected layer in a network definition. This layer expects an input tensor of three or more ...
Definition: NvInfer.h:1226
virtual ITensor * getSequenceLengths() const =0
Get the sequence lengths specified for the RNN.
virtual Dims getPrePadding() const =0
Get the pre-padding.
Use CAFFE padding, rounding output size down.
virtual void resetDeviceType(ILayer *layer)=0
reset the DeviceType for this layer
virtual DataType getType() const =0
Get the data type of a tensor.
virtual bool supportsFormat(DataType type, PluginFormat format) const =0
Check format support.
PluginFormat
Definition: NvInfer.h:2822
TENSORRTAPI void * createInferBuilder_INTERNAL(void *logger, int version)
Internal C entry point for creating IBuilder.
virtual void setStride(Dims stride)=0
Set the slicing stride.
virtual void setKeepDimensions(bool keepDimensions)=0
Set the boolean that specifies whether or not to keep the reduced dimensions for the layer...
virtual void setBeta(float beta)=0
Set the LRN beta value.
int w() const
Get the width.
Definition: NvInfer.h:274
ReduceOperation
Enumerates the reduce operations that may be performed by a Reduce layer.
Definition: NvInfer.h:3568
virtual const char * getPluginNamespace() const =0
Return the namespace of the plugin creator object.
Definition: NvInfer.h:3395
An internal error has occurred. Execution is unrecoverable.
virtual const char * getName() const =0
Return the name of the execution context.
A LRN layer in a network definition.
Definition: NvInfer.h:1581
bias for IConvolutionLayer, IDeconvolutionLayer, or IFullyConnectedLayer
int EnumMax< DeviceType >()
Maximum number of elements in DeviceType enum.
Definition: NvInfer.h:150
Descriptor for three-dimensional data.
Definition: NvInfer.h:281
virtual bool getInt8Mode() const =0
Query whether Int8 mode is used.
virtual int getNbLayers() const =0
Get the number of layers in the network.
virtual ElementWiseOperation getOperation() const =0
Get the binary operation for the layer.
virtual void setOperation(UnaryOperation op)=0
Set the unary operation for the layer.
virtual IPaddingLayer * addPadding(ITensor &input, DimsHW prePadding, DimsHW postPadding)=0
Add a padding layer to the network.
TensorLocation
The location for tensor data storage, device or host.
Definition: NvInfer.h:591
shift part of IScaleLayer
Definition: NvInfer.h:2092
Builds an engine from a network definition.
Definition: NvInfer.h:5321
Verbose messages with debugging information.
virtual void markOutput(ITensor &tensor)=0
Mark a tensor as a network output.
virtual DimsHW getPadding() const =0
Get the padding of the deconvolution.
virtual void setDLACore(int dlaCore)=0
Set the DLA core that the engine must execute on.
virtual void setOperation(RNNOperation op)=0
Set the operation of the RNN layer.
virtual void writeCalibrationCache(const void *ptr, std::size_t length)=0
Save a calibration cache.
virtual void setBiasWeights(Weights weights)=0
Set the bias weights for the convolution.
TENSORRTAPI nvinfer1::ILogger * getLogger()
Return the logger object.
virtual TopKOperation getOperation() const =0
Get the operation for the layer.
virtual void setRefittable(bool canRefit)=0
virtual void attachToContext(cudnnContext *, cublasContext *, IGpuAllocator *)
Attach the plugin object to an execution context and grant the plugin the access to some context reso...
Definition: NvInfer.h:3259
Layer that represents a TopK reduction.
Definition: NvInfer.h:3868
virtual PaddingMode getPaddingMode() const =0
Get the padding mode.
virtual RNNInputMode getInputMode() const =0
Get the input mode of the RNN layer.
int EnumMax< ActivationType >()
Maximum number of elements in ActivationType enum.
Definition: NvInfer.h:1301
virtual ISliceLayer * addSlice(ITensor &input, Dims start, Dims size, Dims stride)=0
Add a slice layer to the network.
virtual void setInt8Mode(bool mode)=0
Set the maximum value for a region.
virtual void setReshapeDimensions(Dims dimensions)=0
Set the reshaped dimensions.
DataType type
The type of the weights.
Definition: NvInfer.h:519
virtual float getBeta() const =0
Get the LRN beta value.
virtual void setPadding(DimsHW padding)=0
Set the padding of the deconvolution.
virtual int enqueue(int batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream)=0
Execute the layer.
virtual TensorLocation getLocation() const =0
Get the storage location of a tensor.
An RNN layer in a network definition, version 2.
Definition: NvInfer.h:2622
Register the plugin creator to the registry The static registry object will be instantiated when the ...
Definition: NvInfer.h:5930
Scaled tanh activation: alpha*tanh(beta*x)
virtual void setType(DataType type)=0
Set the data type of a tensor.
virtual ITensor * getOutput(int index) const =0
Get the output tensor specified by the given index.
virtual Permutation getSecondTranspose() const =0
Get the permutation applied by the second transpose operation.
int & h()
Get the height.
Definition: NvInfer.h:359
Elements correspond to different spatial data.
virtual DimsHW getPadding() const =0
Get the padding of the convolution. If the padding is asymmetric, the pre-padding is returned...
virtual nvinfer1::INetworkDefinition * createNetwork()=0
Create a network definition object.
Clip activation: max(alpha, min(beta, x))
virtual int getNbOutputChannels() const =0
Get the number of output channels K from the fully connected layer.
int EnumMax< WeightsRole >()
Maximum number of elements in WeightsRole enum.
Definition: NvInfer.h:5664
virtual bool getTranspose(int index) const =0
Get the transpose flag for an input tensor.
virtual Weights getWeights() const =0
Get the weights for the layer.
virtual bool getRefittable() const =0
Query whether or not engines will be refittable.
virtual void setFp16Mode(bool mode)=0
Set whether or not 16-bit kernels are permitted.
virtual void reportLayerTime(const char *layerName, float ms)=0
Layer time reporting callback.
virtual void setK(int k)=0
Set the k value for the layer.
virtual IDeconvolutionLayer * addDeconvolution(ITensor &input, int nbOutputMaps, DimsHW kernelSize, Weights kernelWeights, Weights biasWeights)=0
Add a deconvolution layer to the network.
virtual IActivationLayer * addActivation(ITensor &input, ActivationType type)=0
Add an activation layer to the network.
virtual Weights getKernelWeights() const =0
Get the kernel weights.
virtual void setName(const char *name)=0
Set the name of the execution context.
virtual int getNbOutputs() const =0
Get the number of outputs in the network.
virtual void setPluginNamespace(const char *pluginNamespace)=0
Set the namespace of the plugin creator based on the plugin library it belongs to. This can be set while registering the plugin creator.
int & n()
Get the index count.
Definition: NvInfer.h:456
int EnumMax< TopKOperation >()
Maximum number of elements in TopKOperation enum.
Definition: NvInfer.h:3856
Elements correspond to different sequence values.
virtual void serialize(void *buffer) const =0
Serialize the layer.
virtual std::size_t size() const =0
The size in bytes of the data that was allocated.
int EnumMax()
Maximum number of elements in an enumeration type.
virtual bool isDeviceTypeSet(const ILayer *layer) const =0
whether the DeviceType has been explicitly set for this layer
int c() const
Get the channel count.
Definition: NvInfer.h:352
int & w()
Get the width.
Definition: NvInfer.h:373
virtual IElementWiseLayer * addElementWise(ITensor &input1, ITensor &input2, ElementWiseOperation op)=0
Add an elementwise layer to the network.
virtual ~IGpuAllocator()
Definition: NvInfer.h:5311
virtual bool getAverageCountExcludesPadding() const =0
Get whether exclusive pooling uses as a denominator the overlap area betwen the window and the unpadd...
Layer that represents a reduction operator.
Definition: NvInfer.h:3590
virtual PaddingMode getPaddingMode() const =0
Set the padding mode.
virtual void setPrePadding(Dims padding)=0
Set the pre-padding.
Definition: NvInfer.h:5205
PoolingType
The type of pooling to perform in a pooling layer.
Definition: NvInfer.h:1378
int h() const
Get the height.
Definition: NvInfer.h:366
virtual const char * getPluginVersion() const =0
Return the plugin version. Should match the plugin version returned by the corresponding plugin creat...
virtual void detachFromContext()
Detach the plugin object from its execution context.
Definition: NvInfer.h:3267
Single registration point for all plugins in an application. It is used to find plugin implementation...
Definition: NvInfer.h:3474
virtual float getBlendFactor() const =0
Get the blending factor for the max_average_blend mode: max_average_blendPool = (1-blendFactor)*maxPo...
int order[Dims::MAX_DIMS]
Definition: NvInfer.h:3696
int EnumMax< RNNGateType >()
Maximum number of elements in RNNGateType enum.
Definition: NvInfer.h:2608
virtual void setPostPadding(Dims padding)=0
Set the post-padding.
virtual void setMode(ScaleMode mode)=0
Set the scale mode.
virtual ITensor * getInput(int index) const =0
Get the layer input corresponding to the given index.
virtual bool supportsFormat(DataType type, PluginFormat format) const =0
Check format support.
Inverse hyperbolic tangent.
virtual void setConvolutionOutputDimensionsFormula(IOutputDimensionsFormula *formula)=0
Set the convolution output dimensions formula.
Plugin creator class for user implemented layers.
Definition: NvInfer.h:3409
Elu activation: x>=0 ? x : alpha * (exp(x) - 1).
int & w()
Get the width.
Definition: NvInfer.h:267
virtual void setSequenceLengths(ITensor &seqLengths)=0
Specify individual sequence lengths in the batch with the ITensor pointed to by seqLengths.
Plugin factory for deserialization.
Definition: NvInfer.h:5744
virtual int getNbInputs() const =0
Get the number of inputs in the network.
virtual IPluginV2Ext * clone() const _TENSORRT_OVERRIDE=0
Clone the plugin object. This copies over internal plugin parameters as well and returns a new plugin...
virtual ITensor * getHiddenState() const =0
Get the initial hidden state of the RNN.
int length
Number of data entries in the Plugin attribute.
Definition: NvInfer.h:3384
virtual void setOperation(TopKOperation op)=0
Set the operation for the layer.
Descriptor for data with one channel dimension and two spatial dimensions.
Definition: NvInfer.h:313
virtual DimsHW getWindowSize() const =0
Get the window size for pooling.
The first element to the power of the second element.
A network definition for input to the builder.
Definition: NvInfer.h:4102
virtual int getNbLayers() const =0
Get the number of layers in the network.
virtual IHostMemory * serialize() const =0
Serialize the network to a stream.
virtual CalibrationAlgoType getAlgorithm()
Definition: NvInfer.h:5211
virtual IUnaryLayer * addUnary(ITensor &input, UnaryOperation operation)=0
Add a unary layer to the network.
Like kNONE, but transpose the matrix dimensions.
Structure containing plugin attribute field names and associated data This information can be parsed ...
Definition: NvInfer.h:3366
virtual void setOperation(ReduceOperation op)=0
Set the reduce operation for the layer.
virtual void resetDynamicRange()=0
Undo effect of setDynamicRange.
virtual int getTensorRTVersion() const
Return the version of the API the plugin creator was compiled with.
Definition: NvInfer.h:3415
virtual void destroy()=0
Destroy this object.
int EnumMax< CalibrationAlgoType >()
Maximum number of elements in CalibrationAlgoType enum.
Definition: NvInfer.h:5098
virtual RNNDirection getDirection() const =0
Get the direction of the RNN layer.
virtual DimsHW getPadding() const =0
Get the padding for pooling.
virtual const char * getPluginType() const =0
Return the plugin type. Should match the plugin name returned by the corresponding plugin creator...
virtual size_t getSerializationSize()=0
Find the size of the serialization buffer required.
virtual ITensor * getHiddenState() const =0
Get the initial hidden state of the RNN.
virtual IConstantLayer * addConstant(Dims dimensions, Weights weights)=0
Add a constant layer to the network.
virtual void setPaddingMode(PaddingMode paddingMode)=0
Set the padding mode.
virtual void setGpuAllocator(IGpuAllocator *allocator)=0
Set the GPU allocator.
virtual bool getStrictTypeConstraints() const =0
Query whether or not type constraints are strict.
Divide the first element by the second.
Layer type for pluginV2.
Definition: NvInfer.h:3327
virtual int getMinFindIterations() const =0
Query the number of minimization iterations.
Full capability, TensorRT mode without any restrictions.
virtual LayerType getType() const =0
Return the type of a layer.
virtual void setDeviceMemory(void *memory)=0
set the device memory for use by this execution context.
EngineCapability
List of supported engine capability flows.
Definition: NvInfer.h:5261
virtual float getK() const =0
Get the LRN K value.
virtual void resetOutputType(int index)=0
reset the output type for this layer
virtual RNNDirection getDirection() const =0
Get the direction of the RNN layer.
Product of the two elements.
Dims2()
Construct an empty Dims2 object.
Definition: NvInfer.h:200
virtual IProfiler * getProfiler() const =0
Get the profiler.
virtual void setPrePadding(Dims padding)=0
Set the pre-padding.
virtual IPluginV2 & getPlugin()=0
Get the plugin for the layer.
TopKOperation
Enumerates the operations that may be performed by a TopK layer.
Definition: NvInfer.h:3849
virtual Weights getWeightsForGate(int layerIndex, RNNGateType gate, bool isW) const =0
Get the weight parameters for an individual gate in the RNN.
struct CUstream_st * cudaStream_t
Forward declaration of cudaStream_t.
Definition: NvInfer.h:105
Dims3(int d0, int d1, int d2)
Construct a Dims3 from 3 elements.
Definition: NvInfer.h:300
virtual void setStride(DimsHW stride)=0
Set the stride for pooling.
virtual int getSeqLength() const =0
Get the sequence length.
Descriptor for four-dimensional data.
Definition: NvInfer.h:387
virtual DimsHW getStride() const =0
Get the stride for pooling.
Elements correspond to different channels.
virtual void terminate()=0
Release resources acquired during plugin layer initialization. This is called when the engine is dest...
virtual void setStrictTypeConstraints(bool mode)=0
Set whether or not type constraints are strict.
int EnumMax< ReduceOperation >()
Maximum number of elements in ReduceOperation enum.
Definition: NvInfer.h:3578
int EnumMax< DimensionType >()
Maximum number of elements in DimensionType enum.
Definition: NvInfer.h:168
nvinfer1::Dims field type.
ActivationType
Enumerates the types of activation to perform in an activation layer.
Definition: NvInfer.h:1284
virtual void setDeconvolutionOutputDimensionsFormula(IOutputDimensionsFormula *formula)=0
Set the deconvolution output dimensions formula.
virtual const char * getPluginNamespace() const =0
Return the namespace of the plugin object.
Descriptor for data with one index dimension, one channel dimension and two spatial dimensions...
Definition: NvInfer.h:421
virtual void setKernelWeights(Weights weights)=0
Set the kernel weights, given as a KxC matrix in row-major order.
virtual bool setDynamicRange(float min, float max)=0
Set dynamic range for the tensor.
virtual void setMaxBatchSize(int batchSize)=0
Set the maximum batch size.
virtual Dims getPostPadding() const =0
Get the padding.
virtual int getBindingIndex(const char *name) const =0
Retrieve the binding index for a named tensor.
Inverse hyperbolic cosine.
virtual IFullyConnectedLayer * addFullyConnected(ITensor &input, int nbOutputs, Weights kernelWeights, Weights biasWeights)=0
Add a fully connected layer to the network.
TENSORRTAPI nvinfer1::IPluginRegistry * getPluginRegistry()
Return the plugin registry.
virtual MatrixOperation getOperation(int index) const =0
Get the operation for an input tensor.
virtual void setGatherAxis(int axis)=0
Set the non-batch dimension axis to gather on. The axis must be less than the number of non-batch dim...
virtual int getK() const =0
Get the k value for the layer.
virtual const void * readCalibrationCache(std::size_t &length)=0
Load a calibration cache.
virtual void setBiasForGate(int layerIndex, RNNGateType gate, bool isW, Weights bias)=0
Set the bias parameters for an individual gate in the RNN.
virtual void setDeviceType(ILayer *layer, DeviceType deviceType)=0
Set the device that this layer must execute on.
virtual IReduceLayer * addReduce(ITensor &input, ReduceOperation operation, uint32_t reduceAxes, bool keepDimensions)=0
Add a reduce layer to the network.
virtual void setPadding(DimsHW padding)=0
Set the padding for pooling.
DimsHW()
Construct an empty DimsHW object.
Definition: NvInfer.h:230
virtual void setWindowSize(DimsHW windowSize)=0
Set the window size for pooling.
virtual bool registerCreator(IPluginCreator &creator, const char *pluginNamespace)=0
Register a plugin creator. Returns false if one with same type is already registered.
Use explicit padding, rounding output size up.
virtual int getDLACore() const =0
Get the DLA core that the engine executes on.
virtual void setPaddingMode(PaddingMode paddingMode)=0
Set the padding mode.
Base class for all layer classes in a network definition.
Definition: NvInfer.h:803
virtual void setTranspose(int index, bool val)=0
Set the transpose flag for an input tensor.
Dims4(int d0, int d1, int d2, int d3)
Construct a Dims4 from 4 elements.
Definition: NvInfer.h:407
int EnumMax< PoolingType >()
Maximum number of elements in PoolingType enum.
Definition: NvInfer.h:1386
Definition: NvInfer.h:3788
int & h()
Get the height.
Definition: NvInfer.h:484
virtual int getWindowSize() const =0
Get the LRN window size.
int & h()
Get the height.
Definition: NvInfer.h:253
virtual void setAlpha(float alpha)=0
Set the alpha parameter (must be finite).
virtual void free(void *memory)=0
virtual void reset(nvinfer1::INetworkDefinition &network)=0
Resets the builder state.
virtual void setActivationType(ActivationType type)=0
Set the type of activation to be performed.
TENSORRTAPI int getInferLibVersion()
Return the library version number.
virtual std::size_t getHiddenSize() const =0
Get the size of the hidden layers.
int & c()
Get the channel count.
Definition: NvInfer.h:470
virtual void setStride(DimsHW stride)=0
Get the stride of the deconvolution.
Structure to define the dimensions of a tensor.
Definition: NvInfer.h:181
Network iterates from first to last and vice versa and outputs concatenated.
#define NV_TENSORRT_PATCH
TensorRT patch version.
Definition: NvInfer.h:58
virtual IRNNLayer * addRNN(ITensor &inputs, int layerCount, std::size_t hiddenSize, int maxSeqLen, RNNOperation op, RNNInputMode mode, RNNDirection dir, Weights weights, Weights bias)=0
Add an layerCount deep RNN layer to the network with a sequence length of maxSeqLen and hiddenSize in...
virtual bool precisionIsSet() const =0
whether the computational precision has been set for this layer
virtual float getDynamicRangeMax() const =0
Get maximum of dynamic range.
virtual void setReduceAxes(uint32_t reduceAxes)=0
Set which axes to reduce for the layer.
virtual void setGpuAllocator(IGpuAllocator *allocator)=0
Set the GPU allocator.
virtual bool canBroadcastInputAcrossBatch(int inputIndex) const =0
Return true if plugin can use input that is broadcast across batch without replication.
Layer type for shuffling data.
Definition: NvInfer.h:3708
virtual bool platformHasFastFp16() const =0
Determine whether the platform has fast native fp16.
DimsCHW(int channels, int height, int width)
Construct a DimsCHW given channel count, height and width.
Definition: NvInfer.h:333
Definition: NvInfer.h:5190
A elementwise layer in a network definition.
Definition: NvInfer.h:2062
A Pooling layer in a network definition.
Definition: NvInfer.h:1401
virtual void setDefaultDeviceType(DeviceType deviceType)=0
Sets the default DeviceType to be used by the builder. It ensures that all the layers that can run on...
virtual RNNOperation getOperation() const =0
Get the operation of the RNN layer.
virtual void serialize(void *buffer)=0
Serialize the layer.
virtual void setDilation(DimsHW dims)=0
Set the dilation for a convolution.
virtual void setOperation(RNNOperation op)=0
Set the operation of the RNN layer.
virtual void configureWithFormat(const Dims *inputDims, int nbInputs, const Dims *outputDims, int nbOutputs, DataType type, PluginFormat format, int maxBatchSize)=0
Configure the layer.
virtual DimsHW getPrePadding() const =0
Set the padding that is applied at the start of the tensor.
virtual void setDirection(RNNDirection op)=0
Set the direction of the RNN layer.
int d[MAX_DIMS]
The extent of each dimension.
Definition: NvInfer.h:186
Minimum of the two elements.
virtual Weights getBiasWeights() const =0
Get the bias weights.
#define NV_TENSORRT_MINOR
TensorRT minor version.
Definition: NvInfer.h:57
virtual Dims getPostPadding() const =0
Get the post-padding.
virtual int getNbBindings() const =0
Get the number of binding indices.
virtual void setPadding(DimsHW padding)=0
Set the padding of the convolution.
virtual Dims getDimensions() const =0
Get the dimensions of a tensor.
virtual void removeTensor(ITensor &tensor)=0
remove a tensor from the network definition.
int getTensorRTVersion() const _TENSORRT_OVERRIDE
Return the API version with which this plugin was built. The upper byte reserved by TensorRT and is u...
Definition: NvInfer.h:3283
ElementWiseOperation
Enumerates the binary operations that may be performed by an ElementWise layer.
Definition: NvInfer.h:2034
int64_t count
The number of weights in the array.
Definition: NvInfer.h:521
virtual int getMaxDLABatchSize() const =0
Get the maximum batch size DLA can support. For any tensor the total volume of index dimensions combi...
Three-gate network consisting of Gated Recurrent Units.
virtual IPluginV2 * deserializePlugin(const char *name, const void *serialData, size_t serialLength)=0
Called during deserialization of plugin layer. Return a plugin object.
virtual const char * getName() const =0
Get the tensor name.
RNNGateType
Identifies an individual gate within an RNN cell.
Definition: NvInfer.h:2596
Network iterations from first input to last input.
virtual void setBiasWeights(Weights weights)=0
Set the bias weights for the deconvolution.
int & c()
Get the channel count.
Definition: NvInfer.h:345
virtual void setSecondTranspose(Permutation permutation)=0
Set the permutation applied by the second transpose operation.
virtual void setEngineCapability(EngineCapability capability)=0
Configure the builder to target specified EngineCapability flow.
virtual bool getKeepDimensions() const =0
Get the boolean that specifies whether or not to keep the reduced dimensions for the layer...
virtual bool isNetworkOutput() const =0
Whether the tensor is a network output.
Dims3()
Construct an empty Dims3 object.
Definition: NvInfer.h:287
virtual bool getBroadcastAcrossBatch() const =0
Check if tensor is broadcast across the batch.
virtual IPluginV2 * clone() const =0
Clone the plugin object. This copies over internal plugin parameters and returns a new plugin object ...
virtual ITensor * getCellState() const =0
Get the initial cell state of the RNN.
virtual void setCellState(ITensor &cell)=0
Set the initial cell state of the LSTM with the provided cell ITensor.
A tensor in a network definition.
Definition: NvInfer.h:612
virtual void destroy()=0
Destroy this object;.
virtual void setLocation(TensorLocation location)=0
Set the storage location of a tensor.
virtual IPluginCreator *const * getPluginCreatorList(int *numCreators) const =0
Return all the registered plugin creators and the number of registered plugin creators. Returns nullptr if none found.
kernel for IConvolutionLayer, IDeconvolutionLayer, or IFullyConnectedLayer
int EnumMax< UnaryOperation >()
Maximum number of elements in UnaryOperation enum.
Definition: NvInfer.h:3530
virtual void setBlendFactor(float blendFactor)=0
Set the blending factor for the max_average_blend mode: max_average_blendPool = (1-blendFactor)*maxPo...
An array of weights used as a layer parameter.
Definition: NvInfer.h:516
int & w()
Get the width.
Definition: NvInfer.h:498
const char * name
Plugin field attribute name.
Definition: NvInfer.h:3371
virtual int getNbOutputs() const =0
Get the number of outputs from the layer.
int h() const
Get the height.
Definition: NvInfer.h:260
virtual int getDataLength() const =0
Get the length of the data being processed by the RNN for use in computing other values.
virtual DeviceType getDeviceType(const ILayer *layer) const =0
Get the device that this layer executes on.
virtual IMatrixMultiplyLayer * addMatrixMultiply(ITensor &input0, MatrixOperation op0, ITensor &input1, MatrixOperation op1)=0
Add a MatrixMultiply layer to the network.
virtual nvinfer1::ICudaEngine * deserializeCudaEngine(const void *blob, std::size_t size, IPluginFactory *pluginFactory)=0
Deserialize an engine from a stream.
virtual EngineCapability getEngineCapability() const =0
Query EngineCapability flow configured for the builder.
virtual nvinfer1::ICudaEngine * buildCudaEngine(nvinfer1::INetworkDefinition &network)=0
Build a CUDA engine from a network definition.
Dims2(int d0, int d1)
Construct a Dims2 from 2 elements.
Definition: NvInfer.h:212
virtual Dims getSize() const =0
Get the output dimension.
virtual Weights getShift() const =0
Get the shift value.
virtual void setAxis(int axis)=0
Set the axis along which concatenation occurs.
virtual int getNbOutputMaps() const =0
Get the number of output feature maps for the deconvolution.
Dims4()
Construct an empty Dims2 object.
Definition: NvInfer.h:393
virtual void setScale(Weights scale)=0
Set the scale value.
virtual void setInput(int index, ITensor &tensor)=0
replace an input of this layer with a specific tensor
virtual IPoolingLayer * addPooling(ITensor &input, PoolingType type, DimsHW windowSize)=0
Add a pooling layer to the network.
int nbDims
The number of dimensions.
Definition: NvInfer.h:185
virtual bool platformHasFastInt8() const =0
Determine whether the platform has fast native int8.
virtual int enqueue(int batchSize, const void *const *inputs, void **outputs, void *workspace, cudaStream_t stream)=0
Execute the layer.
virtual DimsHW getStride() const =0
Get the stride of the convolution.
Application-implemented interface for calibration.
Definition: NvInfer.h:5114
virtual void destroy()=0
Destroy the plugin object. This will be called when the network, builder or engine is destroyed...
#define _TENSORRT_OVERRIDE
Defines which symbols are exported.
Definition: NvInfer.h:70
Application-implemented logging interface for the builder, engine and runtime.
Definition: NvInfer.h:5831
virtual DataType getOutputType(int index) const =0
get the output type of this layer
virtual IPluginLayer * addPluginExt(ITensor *const *inputs, int nbInputs, IPluginExt &plugin)=0
Add a plugin layer to the network using an IPluginExt interface.
virtual void setDirection(RNNDirection op)=0
Set the direction of the RNN layer.
virtual bool setWeights(const char *layerName, WeightsRole role, Weights weights)=0
Specify new weights for a layer of given name. Returns true on success, or false if new weights are r...
virtual void setInt8Calibrator(IInt8Calibrator *calibrator)=0
Set Int8 Calibration interface.
virtual bool outputTypeIsSet(int index) const =0
whether the output type has been set for this layer
DimsHW(int height, int width)
Construct a DimsHW given height and width.
Definition: NvInfer.h:242
virtual void * data() const =0
A pointer to the raw data that is owned by the library.
Identical coefficients across all elements of the tensor.
int EnumMax< ElementWiseOperation >()
Maximum number of elements in ElementWiseOperation enum.
Definition: NvInfer.h:2046
Plugin class for user-implemented layers.
Definition: NvInfer.h:2953
Layer that represents a padding operation.
Definition: NvInfer.h:3649
virtual void setBiasWeights(Weights weights)=0
Set the bias weights.
virtual IIdentityLayer * addIdentity(ITensor &input)=0
Add an identity layer.
virtual void setNbOutputMaps(int nbOutputMaps)=0
Set the number of output feature maps for the deconvolution.
virtual size_t getWorkspaceSize(int maxBatchSize) const =0
Find the workspace size required by the layer.
virtual void setNbOutputChannels(int nbOutputs)=0
Set the number of output channels K from the fully connected layer.
virtual void setKernelWeights(Weights weights)=0
Set the kernel weights for the convolution.
virtual Weights getBiasWeights() const =0
Get the bias weights for the convolution.
virtual size_t getWorkspaceSize(int maxBatchSize) const =0
Find the workspace size required by the layer.
Safety restricted capability, TensorRT flow that can only run on GPU devices.
int EnumMax< MatrixOperation >()
Maximum number of elements in MatrixOperation enum.
Definition: NvInfer.h:3941
virtual uint32_t getReduceAxes() const =0
Get the axes over which to reduce for the layer.
virtual ITensor * getOutput(int index) const =0
Get the layer output corresponding to the given index.
virtual IPlugin & getPlugin()=0
Get the plugin for the layer.
virtual void setInputMode(RNNInputMode op)=0
Set the operation of the RNN layer.
virtual void setNbGroups(int nbGroups)=0
Set the number of groups for a convolution.
virtual Dims getStride() const =0
Get the slicing stride.
virtual void setCellState(ITensor &cell)=0
Set the initial cell state of the RNN with the provided cell ITensor.
virtual bool isRefittable() const =0
Return true if engine can be refit.
virtual void setKernelSize(DimsHW kernelSize)=0
Set the HW kernel size of the convolution.
CalibrationAlgoType
Version of calibration algorithm to use.
Definition: NvInfer.h:5090
virtual void setStride(DimsHW stride)=0
Get the stride of the convolution.
virtual float getAlpha() const =0
Get the LRN alpha value.
virtual void setReduceAxes(uint32_t reduceAxes)=0
Set the axes over which to reduce.
Informational messages with instructional information.
virtual CalibrationAlgoType getAlgorithm()
Definition: NvInfer.h:5182
Use explicit padding, rounding output size down.
virtual IConvolutionLayer * addConvolution(ITensor &input, int nbOutputMaps, DimsHW kernelSize, Weights kernelWeights, Weights biasWeights)=0
Add a convolution layer to the network.
virtual void setFirstTranspose(Permutation permutation)=0
Set the permutation applied by the first transpose operation.
DimsNCHW()
Construct an empty DimsNCHW object.
Definition: NvInfer.h:427
Class to handle library allocated memory that is accessible to the user.
Definition: NvInfer.h:534
LayerType
The type values of layer classes.
Definition: NvInfer.h:552
Selu activation: x>0 ? beta * x : beta * (alpha*exp(x) - alpha)
int EnumMax< PaddingMode >()
Maximum number of elements in PaddingMode enum.
Definition: NvInfer.h:973
virtual void setPostPadding(Dims padding)=0
Set the post-padding.
int EnumMax< ScaleMode >()
Maximum number of elements in ScaleMode enum.
Definition: NvInfer.h:1661
A layer that represents the identity function.
Definition: NvInfer.h:4040
virtual bool canRunOnDLA(const ILayer *layer) const =0
Checks if a layer can run on DLA.
virtual void setDimensions(Dims dimensions)=0
Set the dimensions for the layer.
virtual Weights getBiasWeights() const =0
Get the bias weights for the deconvolution.
virtual void setProfiler(IProfiler *)=0
Set the profiler.
virtual void setWeights(Weights weights)=0
Set the weights for the layer.
virtual void setSize(Dims size)=0
Set the output dimension.
RNNInputMode
Enumerates the RNN input modes that may occur with an RNN layer.
Definition: NvInfer.h:2241
A RNN layer in a network definition.
Definition: NvInfer.h:2264
virtual void setName(const char *name)=0
Set the name of a layer.
virtual ActivationType getActivationType() const =0
Get the type of activation to be performed.
virtual Dims getDimensions() const =0
Get the dimensions for the layer.
virtual void setDebugSync(bool sync)=0
Set the debug sync flag.
virtual void destroy()=0
Destroy this object.
virtual void unmarkOutput(ITensor &tensor)=0
unmark a tensor as a network output.
virtual void setPostPadding(DimsHW padding)=0
Set the padding that is applied at the end of the tensor.
void configureWithFormat(const Dims *, int, const Dims *, int, DataType, PluginFormat, int) _TENSORRT_OVERRIDE _TENSORRT_FINAL
Derived classes should not implement this. In a C++11 API it would be override final.
Definition: NvInfer.h:3291
Layer type for plugins.
Definition: NvInfer.h:3304
virtual UnaryOperation getOperation() const =0
Get the unary operation for the layer.
virtual bool isOutputBroadcastAcrossBatch(int outputIndex, const bool *inputIsBroadcasted, int nbInputs) const =0
Return true if output tensor is broadcast across a batch.
virtual void setOperation(int index, MatrixOperation op)=0
Set the operation for an input tensor.
virtual void setPaddingMode(PaddingMode paddingMode)=0
Set the padding mode.
virtual void setPostPadding(Dims padding)=0
Set the post-padding.
DimensionType
The type of data encoded across this dimension.
Definition: NvInfer.h:159
virtual IConcatenationLayer * addConcatenation(ITensor *const *inputs, int nbInputs)=0
Add a concatenation layer to the network.
Four-gate LSTM network w/o peephole connections.
virtual DimsHW getPostPadding() const =0
Set the padding that is applied at the end of the tensor.
Thresholded ReLU activation: x>alpha : x : 0.
virtual const ICudaEngine & getEngine() const =0
Get the associated engine.
virtual void * allocate(uint64_t size, uint64_t alignment, uint32_t flags)=0
virtual void setHiddenState(ITensor &hidden)=0
Set the initial hidden state of the RNN with the provided hidden ITensor.
virtual TensorLocation getLocation(int bindingIndex) const =0
Get location of binding.
virtual ScaleMode getMode() const =0
Set the scale mode.
virtual ILRNLayer * addLRN(ITensor &input, int window, float alpha, float beta, float k)=0
Add a LRN layer to the network.
virtual IPluginCreator * getPluginCreator(const char *pluginType, const char *pluginVersion, const char *pluginNamespace="")=0
Return plugin creator based on plugin type, version and namespace associated with plugin during netwo...
virtual int getTensorRTVersion() const
Return the API version with which this plugin was built.
Definition: NvInfer.h:2961
virtual int getAll(int size, const char **layerNames, WeightsRole *roles)=0
Get description of all weights that could be refit.
virtual void setMaxWorkspaceSize(std::size_t workspaceSize)=0
Set the maximum workspace size.
virtual void setShift(Weights shift)=0
Set the shift value.
virtual IPluginV2Layer * addPluginV2(ITensor *const *inputs, int nbInputs, IPluginV2 &plugin)=0
Add a plugin layer to the network using the IPluginV2 interface.
virtual float getAlpha() const =0
Get the alpha parameter.
virtual size_t getDeviceMemorySize() const =0
Return the amount of device memory required by an execution context.
virtual Weights getKernelWeights() const =0
Get the kernel weights for the deconvolution.
virtual int getNbDLACores() const =0
Returns number of DLA hardware cores accessible.
virtual Dims getOutputDimensions(int index, const Dims *inputs, int nbInputDims)=0
Get the dimension of an output tensor.
Application-implemented class for controlling allocation on the GPU.
Definition: NvInfer.h:5279
Context for executing inference using an engine.
Definition: NvInfer.h:4816
virtual ITensor * getInput(int index) const =0
Get the input tensor specified by the given index.
Updates weights in an engine.
Definition: NvInfer.h:5676
virtual void setPower(Weights power)=0
Set the power value.
virtual double getRegressionCutoff() const =0
The fraction (between 0 and 1) of the maximum used to define the regression cutoff when using regress...
virtual void setHalf2Mode(bool mode)=0
Set whether half2 mode is used.
virtual void setOutputType(int index, DataType dataType)=0
Set the output type of this layer.
virtual std::size_t getMaxWorkspaceSize() const =0
Get the maximum workspace size.
virtual int getAxis() const =0
Get the axis along which concatenation occurs.
Descriptor for two-dimensional spatial data.
Definition: NvInfer.h:224
UnaryOperation
Enumerates the unary operations that may be performed by a Unary layer.
Definition: NvInfer.h:3506
virtual void setAverageFindIterations(int avgFind)=0
Set the number of averaging iterations used when timing layers.
virtual int getNbOutputMaps() const =0
Get the number of output maps for the convolution.
virtual int getNbDLACores() const =0
Returns number of DLA hardware cores accessible.
virtual DeviceType getDefaultDeviceType() const =0
Get the default DeviceType which was set by setDefaultDeviceType.
virtual RNNOperation getOperation() const =0
Get the operation of the RNN layer.
virtual nvinfer1::DataType getOutputDataType(int index, const nvinfer1::DataType *inputTypes, int nbInputs) const =0
Return the DataType of the plugin output at the requested index. The default behavior should be to re...
DeviceType
The device that this layer/network will execute on.
Definition: NvInfer.h:144
virtual Dims getPostPadding() const =0
Get the padding.
virtual bool getHalf2Mode() const =0
Query whether half2 mode is used.
virtual int getNbInputs() const =0
Get the number of inputs of a layer.
virtual ISoftMaxLayer * addSoftMax(ITensor &input)=0
Add a SoftMax layer to the network.
Elements correspond to different batch index.
virtual Weights getBiasForGate(int layerIndex, RNNGateType gate, bool isW) const =0
Get the bias parameters for an individual gate in the RNN.
Descriptor for two-dimensional data.
Definition: NvInfer.h:194
virtual int getMaxBatchSize() const =0
Get the maximum batch size which can be used for inference.
virtual void setName(const char *name)=0
Set the tensor name.
virtual IOutputDimensionsFormula & getPoolingOutputDimensionsFormula() const =0
Get the pooling output dimensions formula.
LeakyRelu activation: x>=0 ? x : alpha * x.
virtual int getNbOutputs() const =0
Get the number of outputs of a layer.
virtual IGatherLayer * addGather(ITensor &data, ITensor &indices, int axis)=0
Add a gather layer to the network.
virtual IOutputDimensionsFormula & getDeconvolutionOutputDimensionsFormula() const =0
Get the deconvolution output dimensions formula.
virtual void setDimensions(Dims dimensions)=0
Set the dimensions of a tensor.
virtual void setWindowSize(int windowSize)=0
Set the LRN window size.
virtual Weights getKernelWeights() const =0
Get the kernel weights for the convolution.
virtual CalibrationAlgoType getAlgorithm()=0
Get the algorithm used by this calibrator.
Application-implemented interface for profiling.
Definition: NvInfer.h:4790
virtual RNNInputMode getInputMode() const =0
Get the operation of the RNN layer.
A concatenation layer in a network definition.
Definition: NvInfer.h:1803
virtual void setK(float k)=0
Set the LRN K value.
Plugin class for user-implemented layers.
Definition: NvInfer.h:3180
virtual void setNbGroups(int nbGroups)=0
Set the number of groups for a deconvolution.
const void * data
Plugin field attribute data.
Definition: NvInfer.h:3375
Hard sigmoid activation: max(0, min(1, alpha*x+beta))
virtual IPluginLayer * addPlugin(ITensor *const *inputs, int nbInputs, IPlugin &plugin)=0
Add a plugin layer to the network.
int EnumMax< RNNDirection >()
Maximum number of elements in RNNDirection enum.
Definition: NvInfer.h:2221
virtual const void * readHistogramCache(std::size_t &length)=0
Load a histogram.
virtual void allowGPUFallback(bool setFallBackMode)=0
Sets the builder to use GPU if a layer that was supposed to run on DLA can not run on DLA...
TENSORRTAPI void * createInferRefitter_INTERNAL(void *engine, void *logger, int version)
Internal C entry point for creating IRefitter.
int nbFields
Number of PluginField entries.
Definition: NvInfer.h:3397
virtual Permutation getFirstTranspose() const =0
Get the permutation applied by the first transpose operation.
virtual const char * getPluginVersion() const =0
Return the plugin version.
WeightsRole
How a layer uses particular Weights.
Definition: NvInfer.h:5654
virtual IExecutionContext * createExecutionContextWithoutDeviceMemory()=0
create an execution context without any device memory allocated
virtual void setKernelSize(DimsHW kernelSize)=0
Set the HW kernel size of the convolution.
virtual void setWeights(Weights weights)=0
Set the weight parameters for the RNN.
virtual Weights getWeights() const =0
Get the W weights for the RNN.
virtual const char * getPluginName() const =0
Return the plugin name.
virtual void terminate()=0
Release resources acquired during plugin layer initialization. This is called when the engine is dest...
virtual bool dynamicRangeIsSet() const =0
Query whether dynamic range is set.
virtual PoolingType getPoolingType() const =0
Get the type of activation to be performed.
virtual int getMissing(int size, const char **layerNames, WeightsRole *roles)=0
Get description of missing weights.
PluginFieldType type
Plugin field attribute type.
Definition: NvInfer.h:3380
A deconvolution layer in a network definition.
Definition: NvInfer.h:1836
virtual IShuffleLayer * addShuffle(ITensor &input)=0
Add a shuffle layer to the network.
virtual void setPoolingType(PoolingType type)=0
Set the type of activation to be performed.
virtual IOutputDimensionsFormula & getConvolutionOutputDimensionsFormula() const =0
Get the convolution output dimensions formula.
virtual void setDebugSync(bool sync)=0
Set whether the builder should use debug synchronization.
Softsign activation: x / (1+|x|)
virtual uint32_t getReduceAxes() const =0
Get the axes to reduce for the layer.
virtual int initialize()=0
Initialize the layer for execution. This is called when the engine is created.
Use CAFFE padding, rounding output size up.
Definition: NvInfer.h:5176
int EnumMax< RNNInputMode >()
Maximum number of elements in RNNInputMode enum.
Definition: NvInfer.h:2248