TensorRT  8.0.0
NvInferImpl.h
Go to the documentation of this file.
1 /*
2  * Copyright 1993-2021 NVIDIA Corporation. All rights reserved.
3  *
4  * NOTICE TO LICENSEE:
5  *
6  * This source code and/or documentation ("Licensed Deliverables") are
7  * subject to NVIDIA intellectual property rights under U.S. and
8  * international Copyright laws.
9  *
10  * These Licensed Deliverables contained herein is PROPRIETARY and
11  * CONFIDENTIAL to NVIDIA and is being provided under the terms and
12  * conditions of a form of NVIDIA software license agreement by and
13  * between NVIDIA and Licensee ("License Agreement") or electronically
14  * accepted by Licensee. Notwithstanding any terms or conditions to
15  * the contrary in the License Agreement, reproduction or disclosure
16  * of the Licensed Deliverables to any third party without the express
17  * written consent of NVIDIA is prohibited.
18  *
19  * NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
20  * LICENSE AGREEMENT, NVIDIA MAKES NO REPRESENTATION ABOUT THE
21  * SUITABILITY OF THESE LICENSED DELIVERABLES FOR ANY PURPOSE. IT IS
22  * PROVIDED "AS IS" WITHOUT EXPRESS OR IMPLIED WARRANTY OF ANY KIND.
23  * NVIDIA DISCLAIMS ALL WARRANTIES WITH REGARD TO THESE LICENSED
24  * DELIVERABLES, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY,
25  * NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE.
26  * NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
27  * LICENSE AGREEMENT, IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY
28  * SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, OR ANY
29  * DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
30  * WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
31  * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
32  * OF THESE LICENSED DELIVERABLES.
33  *
34  * U.S. Government End Users. These Licensed Deliverables are a
35  * "commercial item" as that term is defined at 48 C.F.R. 2.101 (OCT
36  * 1995), consisting of "commercial computer software" and "commercial
37  * computer software documentation" as such terms are used in 48
38  * C.F.R. 12.212 (SEPT 1995) and is provided to the U.S. Government
39  * only as a commercial end item. Consistent with 48 C.F.R.12.212 and
40  * 48 C.F.R. 227.7202-1 through 227.7202-4 (JUNE 1995), all
41  * U.S. Government End Users acquire the Licensed Deliverables with
42  * only those rights set forth herein.
43  *
44  * Any use of the Licensed Deliverables in individual and commercial
45  * software must include, in the user documentation and internal
46  * comments to the code, the above Disclaimer and U.S. Government End
47  * Users Notice.
48  */
49 
50 #ifndef NV_INFER_IMPL_H
51 #define NV_INFER_IMPL_H
52 
53 #include "NvInferLegacyDims.h"
54 #include "NvInferRuntimeCommon.h"
55 
56 namespace nvinfer1
57 {
58 
59 class IActivationLayer;
60 class IAlgorithm;
61 class IAlgorithmContext;
62 class IAlgorithmIOInfo;
63 class IAlgorithmVariant;
64 class IAlgorithmSelector;
65 class IBuilderConfig;
66 class IConcatenationLayer;
67 class IConstantLayer;
68 class IConstantLayer;
69 class IConvolutionLayer;
70 class IConvolutionLayer;
71 class ICudaEngine;
72 class IDeconvolutionLayer;
73 class IDeconvolutionLayer;
74 class IDequantizeLayer;
75 class IDimensionExpr;
76 class IElementWiseLayer;
77 class IExecutionContext;
78 class IFillLayer;
79 class IFullyConnectedLayer;
80 class IGatherLayer;
81 class IHostMemory;
82 class IIdentityLayer;
83 class IInt8Calibrator;
84 class IIteratorLayer;
85 class ILayer;
86 class ILoop;
87 class ILoopOutputLayer;
88 class ILRNLayer;
89 class IMatrixMultiplyLayer;
90 class INetworkDefinition;
91 class IOptimizationProfile;
92 class IPaddingLayer;
93 class IPaddingLayer;
94 class IParametricReLULayer;
95 class IPlugin;
96 class IPluginExt;
97 class IPluginFactory;
98 class IPluginLayer;
99 class IPluginLayer;
100 class IPluginV2Layer;
101 class IPoolingLayer;
102 class IPoolingLayer;
103 class IProfiler;
104 class IQuantizeLayer;
105 class IRaggedSoftMaxLayer;
106 class IRecurrenceLayer;
107 class IReduceLayer;
108 class IResizeLayer;
109 class IRNNv2Layer;
110 class IScaleLayer;
111 class ISelectLayer;
112 class IShapeLayer;
113 class IShuffleLayer;
114 class ISliceLayer;
115 class ISoftMaxLayer;
116 class ITensor;
117 class ITimingCache;
118 class ITopKLayer;
119 class ITripLimitLayer;
120 class IUnaryLayer;
121 struct Permutation;
122 class Weights;
123 
124 enum class ActivationType : int32_t;
125 enum class BuilderFlag : int32_t;
126 enum class CalibrationAlgoType : int32_t;
127 enum class DeviceType : int32_t;
128 enum class DimensionOperation : int32_t;
129 enum class ElementWiseOperation : int32_t;
130 enum class EngineCapability : int32_t;
131 enum class FillOperation : int32_t;
132 enum class LayerType : int32_t;
133 enum class LoopOutput : int32_t;
134 enum class MatrixOperation : int32_t;
135 enum class NetworkDefinitionCreationFlag : int32_t;
136 enum class OptProfileSelector : int32_t;
137 enum class PaddingMode : int32_t;
138 enum class PoolingType : int32_t;
139 enum class ProfilingVerbosity : int32_t;
140 enum class QuantizationFlag : int32_t;
141 enum class ReduceOperation : int32_t;
142 enum class ResizeCoordinateTransformation : int32_t;
143 enum class ResizeMode : int32_t;
144 enum class ResizeRoundMode : int32_t;
145 enum class ResizeSelector : int32_t;
146 enum class RNNDirection : int32_t;
147 enum class RNNGateType : int32_t;
148 enum class RNNInputMode : int32_t;
149 enum class RNNOperation : int32_t;
150 enum class ScaleMode : int32_t;
151 enum class SliceMode : int32_t;
152 enum class TensorLocation : int32_t;
153 enum class TopKOperation : int32_t;
154 enum class TripLimit : int32_t;
155 enum class UnaryOperation : int32_t;
156 enum class WeightsRole : int32_t;
157 
158 using TacticSources = uint32_t;
159 using TensorFormats = uint32_t;
160 using BuilderFlags = uint32_t;
161 using NetworkDefinitionCreationFlags = uint32_t;
162 using QuantizationFlags = uint32_t;
163 
171 
172 namespace apiv
173 {
174 
175 class VRoot
176 {
177 public:
178  virtual ~VRoot() noexcept = default;
179 };
180 
181 class VHostMemory : public VRoot
182 {
183 public:
184  virtual void* data() const noexcept = 0;
185  virtual std::size_t size() const noexcept = 0;
186  virtual DataType type() const noexcept = 0;
187 };
188 
189 class VDimensionExpr : public VRoot
190 {
191 public:
192  virtual bool isConstant() const = 0;
193  virtual int32_t getConstantValue() const = 0;
194 };
195 
196 class VExprBuilder : public VRoot
197 {
198 public:
199  virtual const IDimensionExpr* constant(int32_t value) = 0;
200  virtual const IDimensionExpr* operation(
201  DimensionOperation op, const IDimensionExpr& first, const IDimensionExpr& second)
202  = 0;
203 };
204 
205 class VRuntime : public VRoot
206 {
207 public:
208  virtual nvinfer1::ICudaEngine* deserializeCudaEngine(
209  const void* blob, std::size_t size, IPluginFactory* pluginFactory) noexcept
210  = 0;
211  virtual void setDLACore(int32_t dlaCore) noexcept = 0;
212  virtual int32_t getDLACore() const noexcept = 0;
213  virtual int32_t getNbDLACores() const noexcept = 0;
214  virtual void setGpuAllocator(IGpuAllocator* allocator) noexcept = 0;
215  virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
216  virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
217 };
218 
219 class VRefitter : public VRoot
220 {
221 public:
222  virtual bool setWeights(const char* layerName, WeightsRole role, const Weights weights) noexcept = 0;
223  virtual bool refitCudaEngine() noexcept = 0;
224  virtual int32_t getMissing(int32_t size, const char** layerNames, WeightsRole* roles) noexcept = 0;
225  virtual int32_t getAll(int32_t size, const char** layerNames, WeightsRole* roles) noexcept = 0;
226  virtual bool setDynamicRange(const char* tensorName, float min, float max) noexcept = 0;
227  virtual float getDynamicRangeMin(const char* tensorName) const noexcept = 0;
228  virtual float getDynamicRangeMax(const char* tensorName) const noexcept = 0;
229  virtual int32_t getTensorsWithDynamicRange(int32_t size, const char** tensorNames) const noexcept = 0;
230  virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
231  virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
232  virtual bool setNamedWeights(const char* name, Weights weights) noexcept = 0;
233  virtual int32_t getMissingWeights(int32_t size, const char** weightsNames) noexcept = 0;
234  virtual int32_t getAllWeights(int32_t size, const char** weightsNames) noexcept = 0;
235 };
236 
237 class VOptimizationProfile : public VRoot
238 {
239 public:
240  virtual bool setDimensions(const char* inputName, OptProfileSelector select, Dims dims) noexcept = 0;
241  virtual Dims getDimensions(const char* inputName, OptProfileSelector select) const noexcept = 0;
242  virtual bool setShapeValues(
243  const char* inputName, OptProfileSelector select, const int32_t* values, int32_t nbValues) noexcept
244  = 0;
245  virtual int32_t getNbShapeValues(const char* inputName) const noexcept = 0;
246  virtual int32_t const* getShapeValues(const char* inputName, OptProfileSelector select) const noexcept = 0;
247  virtual bool setExtraMemoryTarget(float target) noexcept = 0;
248  virtual float getExtraMemoryTarget() const noexcept = 0;
249  virtual bool isValid() const noexcept = 0;
250 };
251 
252 class VCudaEngine : public VRoot
253 {
254 public:
255  virtual int32_t getNbBindings() const noexcept = 0;
256  virtual int32_t getBindingIndex(const char* name) const noexcept = 0;
257  virtual const char* getBindingName(int32_t bindingIndex) const noexcept = 0;
258  virtual bool bindingIsInput(int32_t bindingIndex) const noexcept = 0;
259  virtual Dims getBindingDimensions(int32_t bindingIndex) const noexcept = 0;
260  virtual DataType getBindingDataType(int32_t bindingIndex) const noexcept = 0;
261  virtual int32_t getMaxBatchSize() const noexcept = 0;
262  virtual int32_t getNbLayers() const noexcept = 0;
263  virtual IHostMemory* serialize() const noexcept = 0;
264  virtual IExecutionContext* createExecutionContext() noexcept = 0;
265  virtual TensorLocation getLocation(int32_t bindingIndex) const noexcept = 0;
266  virtual IExecutionContext* createExecutionContextWithoutDeviceMemory() noexcept = 0;
267  virtual size_t getDeviceMemorySize() const noexcept = 0;
268  virtual bool isRefittable() const noexcept = 0;
269  virtual int32_t getBindingBytesPerComponent(int32_t bindingIndex) const noexcept = 0;
270  virtual int32_t getBindingComponentsPerElement(int32_t bindingIndex) const noexcept = 0;
271  virtual TensorFormat getBindingFormat(int32_t bindingIndex) const noexcept = 0;
272  virtual const char* getBindingFormatDesc(int32_t bindingIndex) const noexcept = 0;
273  virtual int32_t getBindingVectorizedDim(int32_t bindingIndex) const noexcept = 0;
274  virtual const char* getName() const noexcept = 0;
275  virtual int32_t getNbOptimizationProfiles() const noexcept = 0;
276  virtual Dims getProfileDimensions(int32_t bindingIndex, int32_t profileIndex, OptProfileSelector select) const
277  noexcept
278  = 0;
279  virtual const int32_t* getProfileShapeValues(
280  int32_t profileIndex, int32_t inputIndex, OptProfileSelector select) const noexcept
281  = 0;
282  virtual bool isShapeBinding(int32_t bindingIndex) const noexcept = 0;
283  virtual bool isExecutionBinding(int32_t bindingIndex) const noexcept = 0;
284  virtual EngineCapability getEngineCapability() const noexcept = 0;
285  virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
286  virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
287  virtual bool hasImplicitBatchDimension() const noexcept = 0;
288  virtual TacticSources getTacticSources() const noexcept = 0;
289 };
290 
291 class VExecutionContext : public VRoot
292 {
293 public:
294  virtual bool execute(int32_t batchSize, void* const* bindings) noexcept = 0;
295  virtual bool enqueue(
296  int32_t batchSize, void* const* bindings, cudaStream_t stream, cudaEvent_t* inputConsumed) noexcept
297  = 0;
298  virtual void setDebugSync(bool sync) noexcept = 0;
299  virtual bool getDebugSync() const noexcept = 0;
300  virtual void setProfiler(IProfiler*) noexcept = 0;
301  virtual IProfiler* getProfiler() const noexcept = 0;
302  virtual const ICudaEngine& getEngine() const noexcept = 0;
303  virtual void setName(const char* name) noexcept = 0;
304  virtual const char* getName() const noexcept = 0;
305  virtual void setDeviceMemory(void* memory) noexcept = 0;
306  virtual Dims getStrides(int32_t bindingIndex) const noexcept = 0;
307  virtual bool setOptimizationProfile(int32_t profileIndex) noexcept = 0;
308  virtual int32_t getOptimizationProfile() const noexcept = 0;
309  virtual bool setBindingDimensions(int32_t bindingIndex, Dims dimensions) noexcept = 0;
310  virtual Dims getBindingDimensions(int32_t bindingIndex) const noexcept = 0;
311  virtual bool setInputShapeBinding(int32_t bindingIndex, int32_t const* data) noexcept = 0;
312  virtual bool getShapeBinding(int32_t bindingIndex, int32_t* data) const noexcept = 0;
313  virtual bool allInputDimensionsSpecified() const noexcept = 0;
314  virtual bool allInputShapesSpecified() const noexcept = 0;
315  virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
316  virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
317  virtual bool executeV2(void* const* bindings) noexcept = 0;
318  virtual bool enqueueV2(void* const* bindings, cudaStream_t stream, cudaEvent_t* inputConsumed) noexcept = 0;
319  virtual bool setOptimizationProfileAsync(int32_t profileIndex, cudaStream_t stream) noexcept = 0;
320 };
321 
322 class VTensor : public VRoot
323 {
324 public:
325  virtual void setName(const char* name) noexcept = 0;
326  virtual const char* getName() const noexcept = 0;
327  virtual void setDimensions(Dims dimensions) noexcept = 0;
328  virtual Dims getDimensions() const noexcept = 0;
329  virtual void setType(DataType type) noexcept = 0;
330  virtual DataType getType() const noexcept = 0;
331  virtual bool setDynamicRange(float min, float max) noexcept = 0;
332  virtual bool isNetworkInput() const noexcept = 0;
333  virtual bool isNetworkOutput() const noexcept = 0;
334  virtual void setBroadcastAcrossBatch(bool broadcastAcrossBatch) noexcept = 0;
335  virtual bool getBroadcastAcrossBatch() const noexcept = 0;
336  virtual TensorLocation getLocation() const noexcept = 0;
337  virtual void setLocation(TensorLocation location) noexcept = 0;
338  virtual bool dynamicRangeIsSet() const noexcept = 0;
339  virtual void resetDynamicRange() noexcept = 0;
340  virtual float getDynamicRangeMin() const noexcept = 0;
341  virtual float getDynamicRangeMax() const noexcept = 0;
342  virtual void setAllowedFormats(TensorFormats formats) noexcept = 0;
343  virtual TensorFormats getAllowedFormats() const noexcept = 0;
344  virtual bool isShapeTensor() const noexcept = 0;
345  virtual bool isExecutionTensor() const noexcept = 0;
346 };
347 class VLayer : public VRoot
348 {
349 public:
350  virtual LayerType getType() const noexcept = 0;
351  virtual void setName(const char* name) noexcept = 0;
352  virtual const char* getName() const noexcept = 0;
353  virtual int32_t getNbInputs() const noexcept = 0;
354  virtual ITensor* getInput(int32_t index) const noexcept = 0;
355  virtual int32_t getNbOutputs() const noexcept = 0;
356  virtual ITensor* getOutput(int32_t index) const noexcept = 0;
357  virtual void setInput(int32_t index, ITensor& tensor) noexcept = 0;
358  virtual void setPrecision(DataType dataType) noexcept = 0;
359  virtual DataType getPrecision() const noexcept = 0;
360  virtual bool precisionIsSet() const noexcept = 0;
361  virtual void resetPrecision() noexcept = 0;
362  virtual void setOutputType(int32_t index, DataType dataType) noexcept = 0;
363  virtual DataType getOutputType(int32_t index) const noexcept = 0;
364  virtual bool outputTypeIsSet(int32_t index) const noexcept = 0;
365  virtual void resetOutputType(int32_t index) noexcept = 0;
366 };
367 
368 class VConvolutionLayer : public VRoot
369 {
370 public:
371  virtual void setKernelSize(DimsHW kernelSize) noexcept = 0;
372  virtual DimsHW getKernelSize() const noexcept = 0;
373  virtual void setNbOutputMaps(int32_t nbOutputMaps) noexcept = 0;
374  virtual int32_t getNbOutputMaps() const noexcept = 0;
375  virtual void setStride(DimsHW stride) noexcept = 0;
376  virtual DimsHW getStride() const noexcept = 0;
377  virtual void setPadding(DimsHW padding) noexcept = 0;
378  virtual DimsHW getPadding() const noexcept = 0;
379  virtual void setNbGroups(int32_t nbGroups) noexcept = 0;
380  virtual int32_t getNbGroups() const noexcept = 0;
381  virtual void setKernelWeights(Weights weights) noexcept = 0;
382  virtual Weights getKernelWeights() const noexcept = 0;
383  virtual void setBiasWeights(Weights weights) noexcept = 0;
384  virtual Weights getBiasWeights() const noexcept = 0;
385  virtual void setDilation(DimsHW dilation) noexcept = 0;
386  virtual DimsHW getDilation() const noexcept = 0;
387  virtual void setPrePadding(Dims padding) noexcept = 0;
388  virtual Dims getPrePadding() const noexcept = 0;
389  virtual void setPostPadding(Dims padding) noexcept = 0;
390  virtual Dims getPostPadding() const noexcept = 0;
391  virtual void setPaddingMode(PaddingMode paddingMode) noexcept = 0;
392  virtual PaddingMode getPaddingMode() const noexcept = 0;
393  virtual void setKernelSizeNd(Dims kernelSize) noexcept = 0;
394  virtual Dims getKernelSizeNd() const noexcept = 0;
395  virtual void setStrideNd(Dims stride) noexcept = 0;
396  virtual Dims getStrideNd() const noexcept = 0;
397  virtual void setPaddingNd(Dims padding) noexcept = 0;
398  virtual Dims getPaddingNd() const noexcept = 0;
399  virtual void setDilationNd(Dims dilation) noexcept = 0;
400  virtual Dims getDilationNd() const noexcept = 0;
401 };
402 
403 class VFullyConnectedLayer : public VRoot
404 {
405 public:
406  virtual void setNbOutputChannels(int32_t nbOutputs) noexcept = 0;
407  virtual int32_t getNbOutputChannels() const noexcept = 0;
408  virtual void setKernelWeights(Weights weights) noexcept = 0;
409  virtual Weights getKernelWeights() const noexcept = 0;
410  virtual void setBiasWeights(Weights weights) noexcept = 0;
411  virtual Weights getBiasWeights() const noexcept = 0;
412 };
413 
414 class VActivationLayer : public VRoot
415 {
416 public:
417  virtual void setActivationType(ActivationType type) noexcept = 0;
418  virtual ActivationType getActivationType() const noexcept = 0;
419  virtual void setAlpha(float alpha) noexcept = 0;
420  virtual void setBeta(float beta) noexcept = 0;
421  virtual float getAlpha() const noexcept = 0;
422  virtual float getBeta() const noexcept = 0;
423 };
424 
425 class VPoolingLayer : public VRoot
426 {
427 public:
428  virtual void setPoolingType(PoolingType type) noexcept = 0;
429  virtual PoolingType getPoolingType() const noexcept = 0;
430  virtual void setWindowSize(DimsHW windowSize) noexcept = 0;
431  virtual DimsHW getWindowSize() const noexcept = 0;
432  virtual void setStride(DimsHW stride) noexcept = 0;
433  virtual DimsHW getStride() const noexcept = 0;
434  virtual void setPadding(DimsHW padding) noexcept = 0;
435  virtual DimsHW getPadding() const noexcept = 0;
436  virtual void setBlendFactor(float blendFactor) noexcept = 0;
437  virtual float getBlendFactor() const noexcept = 0;
438  virtual void setAverageCountExcludesPadding(bool exclusive) noexcept = 0;
439  virtual bool getAverageCountExcludesPadding() const noexcept = 0;
440  virtual void setPrePadding(Dims padding) noexcept = 0;
441  virtual Dims getPrePadding() const noexcept = 0;
442  virtual void setPostPadding(Dims padding) noexcept = 0;
443  virtual Dims getPostPadding() const noexcept = 0;
444  virtual void setPaddingMode(PaddingMode paddingMode) noexcept = 0;
445  virtual PaddingMode getPaddingMode() const noexcept = 0;
446  virtual void setWindowSizeNd(Dims windowSize) noexcept = 0;
447  virtual Dims getWindowSizeNd() const noexcept = 0;
448  virtual void setStrideNd(Dims stride) noexcept = 0;
449  virtual Dims getStrideNd() const noexcept = 0;
450  virtual void setPaddingNd(Dims padding) noexcept = 0;
451  virtual Dims getPaddingNd() const noexcept = 0;
452 };
453 
454 class VLRNLayer : public VRoot
455 {
456 public:
457  virtual void setWindowSize(int32_t windowSize) noexcept = 0;
458  virtual int32_t getWindowSize() const noexcept = 0;
459  virtual void setAlpha(float alpha) noexcept = 0;
460  virtual float getAlpha() const noexcept = 0;
461  virtual void setBeta(float beta) noexcept = 0;
462  virtual float getBeta() const noexcept = 0;
463  virtual void setK(float k) noexcept = 0;
464  virtual float getK() const noexcept = 0;
465 };
466 
467 class VScaleLayer : public VRoot
468 {
469 public:
470  virtual void setMode(ScaleMode mode) noexcept = 0;
471  virtual ScaleMode getMode() const noexcept = 0;
472  virtual void setShift(Weights shift) noexcept = 0;
473  virtual Weights getShift() const noexcept = 0;
474  virtual void setScale(Weights scale) noexcept = 0;
475  virtual Weights getScale() const noexcept = 0;
476  virtual void setPower(Weights power) noexcept = 0;
477  virtual Weights getPower() const noexcept = 0;
478  virtual int32_t getChannelAxis() const noexcept = 0;
479  virtual void setChannelAxis(int32_t channelAxis) noexcept = 0;
480 };
481 
482 class VSoftMaxLayer : public VRoot
483 {
484 public:
485  virtual void setAxes(uint32_t axes) noexcept = 0;
486  virtual uint32_t getAxes() const noexcept = 0;
487 };
488 
489 class VConcatenationLayer : public VRoot
490 {
491 public:
492  virtual void setAxis(int32_t axis) noexcept = 0;
493  virtual int32_t getAxis() const noexcept = 0;
494 };
495 
496 class VDeconvolutionLayer : public VRoot
497 {
498 public:
499  virtual void setKernelSize(DimsHW kernelSize) noexcept = 0;
500  virtual DimsHW getKernelSize() const noexcept = 0;
501  virtual void setNbOutputMaps(int32_t nbOutputMaps) noexcept = 0;
502  virtual int32_t getNbOutputMaps() const noexcept = 0;
503  virtual void setStride(DimsHW stride) noexcept = 0;
504  virtual DimsHW getStride() const noexcept = 0;
505  virtual void setPadding(DimsHW padding) noexcept = 0;
506  virtual DimsHW getPadding() const noexcept = 0;
507  virtual void setNbGroups(int32_t nbGroups) noexcept = 0;
508  virtual int32_t getNbGroups() const noexcept = 0;
509  virtual void setKernelWeights(Weights weights) noexcept = 0;
510  virtual Weights getKernelWeights() const noexcept = 0;
511  virtual void setBiasWeights(Weights weights) noexcept = 0;
512  virtual Weights getBiasWeights() const noexcept = 0;
513  virtual void setPrePadding(Dims padding) noexcept = 0;
514  virtual Dims getPrePadding() const noexcept = 0;
515  virtual void setPostPadding(Dims padding) noexcept = 0;
516  virtual Dims getPostPadding() const noexcept = 0;
517  virtual void setPaddingMode(PaddingMode paddingMode) noexcept = 0;
518  virtual PaddingMode getPaddingMode() const noexcept = 0;
519  virtual void setKernelSizeNd(Dims kernelSize) noexcept = 0;
520  virtual Dims getKernelSizeNd() const noexcept = 0;
521  virtual void setStrideNd(Dims stride) noexcept = 0;
522  virtual Dims getStrideNd() const noexcept = 0;
523  virtual void setPaddingNd(Dims padding) noexcept = 0;
524  virtual Dims getPaddingNd() const noexcept = 0;
525  virtual void setDilationNd(Dims dilation) noexcept = 0;
526  virtual Dims getDilationNd() const noexcept = 0;
527 };
528 
529 class VElementWiseLayer : public VRoot
530 {
531 public:
532  virtual void setOperation(ElementWiseOperation op) noexcept = 0;
533  virtual ElementWiseOperation getOperation() const noexcept = 0;
534 };
535 
536 class VGatherLayer : public VRoot
537 {
538 public:
539  virtual void setGatherAxis(int32_t axis) noexcept = 0;
540  virtual int32_t getGatherAxis() const noexcept = 0;
541  virtual void setNbElementWiseDims(int32_t k) noexcept = 0;
542  virtual int32_t getNbElementWiseDims() const noexcept = 0;
543 };
544 
545 class VRNNv2Layer : public VRoot
546 {
547 public:
548  virtual int32_t getLayerCount() const noexcept = 0;
549  virtual int32_t getHiddenSize() const noexcept = 0;
550  virtual int32_t getMaxSeqLength() const noexcept = 0;
551  virtual int32_t getDataLength() const noexcept = 0;
552  virtual void setSequenceLengths(ITensor& seqLengths) noexcept = 0;
553  virtual ITensor* getSequenceLengths() const noexcept = 0;
554  virtual void setOperation(RNNOperation op) noexcept = 0;
555  virtual RNNOperation getOperation() const noexcept = 0;
556  virtual void setInputMode(RNNInputMode op) noexcept = 0;
557  virtual RNNInputMode getInputMode() const noexcept = 0;
558  virtual void setDirection(RNNDirection op) noexcept = 0;
559  virtual RNNDirection getDirection() const noexcept = 0;
560  virtual void setWeightsForGate(int32_t layerIndex, RNNGateType gate, bool isW, Weights weights) noexcept = 0;
561  virtual Weights getWeightsForGate(int32_t layerIndex, RNNGateType gate, bool isW) const noexcept = 0;
562  virtual void setBiasForGate(int32_t layerIndex, RNNGateType gate, bool isW, Weights bias) noexcept = 0;
563  virtual Weights getBiasForGate(int32_t layerIndex, RNNGateType gate, bool isW) const noexcept = 0;
564  virtual void setHiddenState(ITensor& hidden) noexcept = 0;
565  virtual ITensor* getHiddenState() const noexcept = 0;
566  virtual void setCellState(ITensor& cell) noexcept = 0;
567  virtual ITensor* getCellState() const noexcept = 0;
568 };
569 
570 class VPluginLayer : public VRoot
571 {
572 public:
573  virtual IPlugin& getPlugin() noexcept = 0;
574 };
575 
576 class VPluginV2Layer : public VRoot
577 {
578 public:
579  virtual IPluginV2& getPlugin() noexcept = 0;
580 };
581 
582 class VUnaryLayer : public VRoot
583 {
584 public:
585  virtual void setOperation(UnaryOperation op) noexcept = 0;
586  virtual UnaryOperation getOperation() const noexcept = 0;
587 };
588 
589 class VReduceLayer : public VRoot
590 {
591 public:
592  virtual void setOperation(ReduceOperation op) noexcept = 0;
593  virtual ReduceOperation getOperation() const noexcept = 0;
594  virtual void setReduceAxes(uint32_t reduceAxes) noexcept = 0;
595  virtual uint32_t getReduceAxes() const noexcept = 0;
596  virtual void setKeepDimensions(bool keepDimensions) noexcept = 0;
597  virtual bool getKeepDimensions() const noexcept = 0;
598 };
599 
600 class VPaddingLayer : public VRoot
601 {
602 public:
603  virtual void setPrePadding(DimsHW padding) noexcept = 0;
604  virtual DimsHW getPrePadding() const noexcept = 0;
605  virtual void setPostPadding(DimsHW padding) noexcept = 0;
606  virtual DimsHW getPostPadding() const noexcept = 0;
607  virtual void setPrePaddingNd(Dims padding) noexcept = 0;
608  virtual Dims getPrePaddingNd() const noexcept = 0;
609  virtual void setPostPaddingNd(Dims padding) noexcept = 0;
610  virtual Dims getPostPaddingNd() const noexcept = 0;
611 };
612 
613 class VShuffleLayer : public VRoot
614 {
615 public:
616  virtual void setFirstTranspose(const Permutation& permutation) noexcept = 0;
617  virtual const Permutation& getFirstTranspose() const noexcept = 0;
618  virtual void setReshapeDimensions(Dims dimensions) noexcept = 0;
619  virtual Dims getReshapeDimensions() const noexcept = 0;
620  virtual void setSecondTranspose(const Permutation& permutation) noexcept = 0;
621  virtual const Permutation& getSecondTranspose() const noexcept = 0;
622  virtual void setZeroIsPlaceholder(bool zeroIsPlaceholder) = 0;
623  virtual bool getZeroIsPlaceholder() const = 0;
624 };
625 
626 class VSliceLayer : public VRoot
627 {
628 public:
629  virtual void setStart(Dims start) noexcept = 0;
630  virtual Dims getStart() const noexcept = 0;
631  virtual void setSize(Dims size) noexcept = 0;
632  virtual Dims getSize() const noexcept = 0;
633  virtual void setStride(Dims stride) noexcept = 0;
634  virtual Dims getStride() const noexcept = 0;
635  virtual void setMode(SliceMode mode) noexcept = 0;
636  virtual SliceMode getMode() const noexcept = 0;
637 };
638 
639 class VShapeLayer : public VRoot
640 {
641 public:
642 };
643 
644 class VTopKLayer : public VRoot
645 {
646 public:
647  virtual void setOperation(TopKOperation op) noexcept = 0;
648  virtual TopKOperation getOperation() const noexcept = 0;
649  virtual void setK(int32_t k) noexcept = 0;
650  virtual int32_t getK() const noexcept = 0;
651  virtual void setReduceAxes(uint32_t reduceAxes) noexcept = 0;
652  virtual uint32_t getReduceAxes() const noexcept = 0;
653 };
654 
655 class VMatrixMultiplyLayer : public VRoot
656 {
657 public:
658  virtual void setOperation(int32_t index, MatrixOperation op) noexcept = 0;
659  virtual MatrixOperation getOperation(int32_t index) const noexcept = 0;
660 };
661 
662 class VRaggedSoftMaxLayer : public VRoot
663 {
664 public:
665 };
666 
667 class VIdentityLayer : public VRoot
668 {
669 public:
670 };
671 
672 class VConstantLayer : public VRoot
673 {
674 public:
675  virtual void setWeights(Weights weights) noexcept = 0;
676  virtual Weights getWeights() const noexcept = 0;
677  virtual void setDimensions(Dims dimensions) noexcept = 0;
678  virtual Dims getDimensions() const noexcept = 0;
679 };
680 
681 class VParametricReLULayer : public VRoot
682 {
683 public:
684 };
685 
686 class VResizeLayer : public VRoot
687 {
688 public:
689  virtual void setOutputDimensions(Dims dimensions) noexcept = 0;
690  virtual Dims getOutputDimensions() const noexcept = 0;
691  virtual void setScales(const float* scales, int32_t nbScales) noexcept = 0;
692  virtual int32_t getScales(int32_t size, float* scales) const noexcept = 0;
693  virtual void setResizeMode(ResizeMode resizeMode) noexcept = 0;
694  virtual ResizeMode getResizeMode() const noexcept = 0;
695  virtual void setAlignCorners(bool alignCorners) noexcept = 0;
696  virtual bool getAlignCorners() const noexcept = 0;
697  virtual void setCoordinateTransformation(ResizeCoordinateTransformation coordTransform) noexcept = 0;
698  virtual ResizeCoordinateTransformation getCoordinateTransformation() const noexcept = 0;
699  virtual void setSelectorForSinglePixel(ResizeSelector selector) noexcept = 0;
700  virtual ResizeSelector getSelectorForSinglePixel() const noexcept = 0;
701  virtual void setNearestRounding(ResizeRoundMode value) noexcept = 0;
702  virtual ResizeRoundMode getNearestRounding() const noexcept = 0;
703 };
704 
705 class VLoopBoundaryLayer : public VRoot
706 {
707 public:
708  virtual ILoop* getLoop() const noexcept = 0;
709 };
710 
711 class VRecurrenceLayer : public VRoot
712 {
713 public:
714 };
715 
716 class VLoopOutputLayer : public VRoot
717 {
718 public:
719  virtual LoopOutput getLoopOutput() const noexcept = 0;
720  virtual void setAxis(int32_t axis) noexcept = 0;
721  virtual int32_t getAxis() const noexcept = 0;
722 };
723 
724 class VTripLimitLayer : public VRoot
725 {
726 public:
727  virtual TripLimit getTripLimit() const noexcept = 0;
728 };
729 
730 class VIteratorLayer : public VRoot
731 {
732 public:
733  virtual void setAxis(int32_t axis) noexcept = 0;
734  virtual int32_t getAxis() const noexcept = 0;
735  virtual void setReverse(bool reverse) noexcept = 0;
736  virtual bool getReverse() const noexcept = 0;
737 };
738 class VLoop : public VRoot
739 {
740 public:
741  virtual IRecurrenceLayer* addRecurrence(ITensor& initialValue) noexcept = 0;
742  virtual ITripLimitLayer* addTripLimit(ITensor& tensor, TripLimit limit) noexcept = 0;
743  virtual IIteratorLayer* addIterator(ITensor& tensor, int32_t axis = 0, bool reverse = false) noexcept = 0;
744  virtual ILoopOutputLayer* addLoopOutput(ITensor& tensor, LoopOutput outputKind, int32_t axis = 0) noexcept = 0;
745  virtual void setName(const char* name) noexcept = 0;
746  virtual const char* getName() const noexcept = 0;
747 };
748 class VSelectLayer : public VRoot
749 {
750 };
751 
752 class VFillLayer : public VRoot
753 {
754 public:
755  virtual void setDimensions(Dims dimensions) noexcept = 0;
756  virtual Dims getDimensions() const noexcept = 0;
757  virtual void setOperation(FillOperation op) noexcept = 0;
758  virtual FillOperation getOperation() const noexcept = 0;
759  virtual void setAlpha(double alpha) noexcept = 0;
760  virtual double getAlpha() const noexcept = 0;
761  virtual void setBeta(double beta) noexcept = 0;
762  virtual double getBeta() const noexcept = 0;
763 };
764 
765 class VQuantizeLayer : public VRoot
766 {
767 public:
768  virtual int32_t getAxis() const noexcept = 0;
769  virtual void setAxis(int32_t axis) noexcept = 0;
770 };
771 
772 class VDequantizeLayer : public VRoot
773 {
774 public:
775  virtual int32_t getAxis() const noexcept = 0;
776  virtual void setAxis(int32_t axis) noexcept = 0;
777 };
778 
779 class VNetworkDefinition : public VRoot
780 {
781 public:
782  virtual ITensor* addInput(const char* name, DataType type, Dims dimensions) noexcept = 0;
783  virtual void markOutput(ITensor& tensor) noexcept = 0;
784  virtual IConvolutionLayer* addConvolution(
785  ITensor& input, int32_t nbOutputMaps, DimsHW kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
786  = 0;
787  virtual IFullyConnectedLayer* addFullyConnected(
788  ITensor& input, int32_t nbOutputs, Weights kernelWeights, Weights biasWeights) noexcept
789  = 0;
790  virtual IActivationLayer* addActivation(ITensor& input, ActivationType type) noexcept = 0;
791  virtual IPoolingLayer* addPooling(ITensor& input, PoolingType type, DimsHW windowSize) noexcept = 0;
792  virtual ILRNLayer* addLRN(ITensor& input, int32_t window, float alpha, float beta, float k) noexcept = 0;
793  virtual IScaleLayer* addScale(ITensor& input, ScaleMode mode, Weights shift, Weights scale, Weights power) noexcept
794  = 0;
795  virtual ISoftMaxLayer* addSoftMax(ITensor& input) noexcept = 0;
796  virtual IConcatenationLayer* addConcatenation(ITensor* const* inputs, int32_t nbInputs) noexcept = 0;
797  virtual IDeconvolutionLayer* addDeconvolution(
798  ITensor& input, int32_t nbOutputMaps, DimsHW kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
799  = 0;
800  virtual IElementWiseLayer* addElementWise(ITensor& input1, ITensor& input2, ElementWiseOperation op) noexcept = 0;
801  virtual IUnaryLayer* addUnary(ITensor& input, UnaryOperation operation) noexcept = 0;
802  virtual IPaddingLayer* addPadding(ITensor& input, DimsHW prePadding, DimsHW postPadding) noexcept = 0;
803  virtual IShuffleLayer* addShuffle(ITensor& input) noexcept = 0;
804  virtual int32_t getNbLayers() const noexcept = 0;
805  virtual ILayer* getLayer(int32_t index) const noexcept = 0;
806  virtual int32_t getNbInputs() const noexcept = 0;
807  virtual ITensor* getInput(int32_t index) const noexcept = 0;
808  virtual int32_t getNbOutputs() const noexcept = 0;
809  virtual ITensor* getOutput(int32_t index) const noexcept = 0;
810  virtual IReduceLayer* addReduce(
811  ITensor& input, ReduceOperation operation, uint32_t reduceAxes, bool keepDimensions) noexcept
812  = 0;
813  virtual ITopKLayer* addTopK(ITensor& input, TopKOperation op, int32_t k, uint32_t reduceAxes) noexcept = 0;
814  virtual IGatherLayer* addGather(ITensor& data, ITensor& indices, int32_t axis) noexcept = 0;
815  virtual IRaggedSoftMaxLayer* addRaggedSoftMax(ITensor& input, ITensor& bounds) noexcept = 0;
816  virtual IMatrixMultiplyLayer* addMatrixMultiply(
817  ITensor& input0, MatrixOperation op0, ITensor& input1, MatrixOperation op1) noexcept
818  = 0;
819  virtual IConstantLayer* addConstant(Dims dimensions, Weights weights) noexcept = 0;
820  virtual IRNNv2Layer* addRNNv2(
821  ITensor& input, int32_t layerCount, int32_t hiddenSize, int32_t maxSeqLen, RNNOperation op) noexcept
822  = 0;
823  virtual IIdentityLayer* addIdentity(ITensor& input) noexcept = 0;
824  virtual void removeTensor(ITensor& tensor) noexcept = 0;
825  virtual void unmarkOutput(ITensor& tensor) noexcept = 0;
826  virtual IPluginV2Layer* addPluginV2(ITensor* const* inputs, int32_t nbInputs, IPluginV2& plugin) noexcept = 0;
827  virtual ISliceLayer* addSlice(ITensor& input, Dims start, Dims size, Dims stride) noexcept = 0;
828  virtual void setName(const char* name) noexcept = 0;
829  virtual const char* getName() const noexcept = 0;
830  virtual IShapeLayer* addShape(ITensor& input) noexcept = 0;
831  virtual bool hasImplicitBatchDimension() const noexcept = 0;
832  virtual bool markOutputForShapes(ITensor& tensor) noexcept = 0;
833  virtual bool unmarkOutputForShapes(ITensor& tensor) noexcept = 0;
834  virtual IParametricReLULayer* addParametricReLU(ITensor& input, ITensor& slope) noexcept = 0;
835  virtual IConvolutionLayer* addConvolutionNd(
836  ITensor& input, int32_t nbOutputMaps, Dims kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
837  = 0;
838  virtual IPoolingLayer* addPoolingNd(ITensor& input, PoolingType type, Dims windowSize) noexcept = 0;
839  virtual IDeconvolutionLayer* addDeconvolutionNd(
840  ITensor& input, int32_t nbOutputMaps, Dims kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
841  = 0;
842  virtual IScaleLayer* addScaleNd(
843  ITensor& input, ScaleMode mode, Weights shift, Weights scale, Weights power, int32_t channelAxis) noexcept
844  = 0;
845  virtual IResizeLayer* addResize(ITensor& input) noexcept = 0;
846  virtual bool hasExplicitPrecision() const noexcept = 0;
847  virtual ILoop* addLoop() noexcept = 0;
848  virtual ISelectLayer* addSelect(ITensor& condition, ITensor& thenInput, ITensor& elseInput) noexcept = 0;
849  virtual IFillLayer* addFill(Dims dimensions, FillOperation op) noexcept = 0;
850  virtual IPaddingLayer* addPaddingNd(ITensor& input, Dims prePadding, Dims postPadding) noexcept = 0;
851  virtual bool setWeightsName(Weights weights, const char* name) noexcept = 0;
852  virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
853  virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
854  virtual IDequantizeLayer* addDequantize(ITensor& input, ITensor& scale) noexcept = 0;
855  virtual IQuantizeLayer* addQuantize(ITensor& input, ITensor& scale) noexcept = 0;
856 };
857 
858 class VAlgorithmIOInfo : public VRoot
859 {
860 public:
861  virtual TensorFormat getTensorFormat() const noexcept = 0;
862  virtual DataType getDataType() const noexcept = 0;
863  virtual Dims getStrides() const noexcept = 0;
864 };
865 
866 class VAlgorithmVariant : public VRoot
867 {
868 public:
869  virtual int64_t getImplementation() const noexcept = 0;
870  virtual int64_t getTactic() const noexcept = 0;
871 };
872 
873 class VAlgorithmContext : public VRoot
874 {
875 public:
876  virtual const char* getName() const noexcept = 0;
877  virtual Dims getDimensions(int32_t index, OptProfileSelector select) const noexcept = 0;
878  virtual int32_t getNbInputs() const noexcept = 0;
879  virtual int32_t getNbOutputs() const noexcept = 0;
880 };
881 
882 class VAlgorithm : public VRoot
883 {
884 public:
885  virtual const IAlgorithmIOInfo& getAlgorithmIOInfo(int32_t index) const noexcept = 0;
886  virtual const IAlgorithmVariant& getAlgorithmVariant() const noexcept = 0;
887  virtual float getTimingMSec() const noexcept = 0;
888  virtual std::size_t getWorkspaceSize() const noexcept = 0;
889  virtual const IAlgorithmIOInfo* getAlgorithmIOInfoByIndex(int32_t index) const noexcept = 0;
890 };
891 
892 class VTimingCache : public VRoot
893 {
894 public:
895  virtual nvinfer1::IHostMemory* serialize() const noexcept = 0;
896  virtual bool combine(const ITimingCache& inputCache, bool ignoreMismatch) noexcept = 0;
897  virtual bool reset() noexcept = 0;
898 };
899 
900 class VBuilderConfig : public VRoot
901 {
902 public:
903  virtual void setMinTimingIterations(int32_t minTiming) noexcept = 0;
904  virtual int32_t getMinTimingIterations() const noexcept = 0;
905  virtual void setAvgTimingIterations(int32_t avgTiming) noexcept = 0;
906  virtual int32_t getAvgTimingIterations() const noexcept = 0;
907  virtual void setEngineCapability(EngineCapability capability) noexcept = 0;
908  virtual EngineCapability getEngineCapability() const noexcept = 0;
909  virtual void setInt8Calibrator(IInt8Calibrator* calibrator) noexcept = 0;
910  virtual IInt8Calibrator* getInt8Calibrator() const noexcept = 0;
911  virtual void setMaxWorkspaceSize(std::size_t workspaceSize) noexcept = 0;
912  virtual std::size_t getMaxWorkspaceSize() const noexcept = 0;
913  virtual void setFlags(BuilderFlags builderFlags) noexcept = 0;
914  virtual BuilderFlags getFlags() const noexcept = 0;
915  virtual void clearFlag(BuilderFlag builderFlag) noexcept = 0;
916  virtual void setFlag(BuilderFlag builderFlag) noexcept = 0;
917  virtual bool getFlag(BuilderFlag builderFlag) const noexcept = 0;
918  virtual void setDeviceType(const ILayer* layer, DeviceType deviceType) noexcept = 0;
919  virtual DeviceType getDeviceType(const ILayer* layer) const noexcept = 0;
920  virtual bool isDeviceTypeSet(const ILayer* layer) const noexcept = 0;
921  virtual void resetDeviceType(const ILayer* layer) noexcept = 0;
922  virtual bool canRunOnDLA(const ILayer* layer) const noexcept = 0;
923  virtual void setDLACore(int32_t dlaCore) noexcept = 0;
924  virtual int32_t getDLACore() const noexcept = 0;
925  virtual void setDefaultDeviceType(DeviceType deviceType) noexcept = 0;
926  virtual DeviceType getDefaultDeviceType() const noexcept = 0;
927  virtual void reset() noexcept = 0;
928  virtual void setProfileStream(const cudaStream_t stream) noexcept = 0;
929  virtual cudaStream_t getProfileStream() const noexcept = 0;
930  virtual int32_t addOptimizationProfile(const IOptimizationProfile* profile) noexcept = 0;
931  virtual int32_t getNbOptimizationProfiles() const noexcept = 0;
932  virtual void setProfilingVerbosity(ProfilingVerbosity verbosity) noexcept = 0;
933  virtual ProfilingVerbosity getProfilingVerbosity() const noexcept = 0;
934  virtual void setAlgorithmSelector(IAlgorithmSelector* selector) noexcept = 0;
935  virtual IAlgorithmSelector* getAlgorithmSelector() const noexcept = 0;
936  virtual bool setCalibrationProfile(const IOptimizationProfile* profile) noexcept = 0;
937  virtual const IOptimizationProfile* getCalibrationProfile() noexcept = 0;
938  virtual void setQuantizationFlags(QuantizationFlags flags) noexcept = 0;
939  virtual QuantizationFlags getQuantizationFlags() const noexcept = 0;
940  virtual void clearQuantizationFlag(QuantizationFlag flag) noexcept = 0;
941  virtual void setQuantizationFlag(QuantizationFlag flag) noexcept = 0;
942  virtual bool getQuantizationFlag(QuantizationFlag flag) const noexcept = 0;
943  virtual bool setTacticSources(TacticSources tacticSources) noexcept = 0;
944  virtual TacticSources getTacticSources() const noexcept = 0;
945  virtual nvinfer1::ITimingCache* createTimingCache(const void* blob, std::size_t size) const noexcept = 0;
946  virtual bool setTimingCache(const ITimingCache& cache, bool ignoreMismatch) noexcept = 0;
947  virtual const nvinfer1::ITimingCache* getTimingCache() const noexcept = 0;
948 };
949 
950 class VBuilder : public VRoot
951 {
952 public:
953  virtual void setMaxBatchSize(int32_t batchSize) noexcept = 0;
954  virtual int32_t getMaxBatchSize() const noexcept = 0;
955  virtual bool platformHasFastFp16() const noexcept = 0;
956  virtual bool platformHasFastInt8() const noexcept = 0;
957  virtual int32_t getMaxDLABatchSize() const noexcept = 0;
958  virtual int32_t getNbDLACores() const noexcept = 0;
959  virtual void setGpuAllocator(IGpuAllocator* allocator) noexcept = 0;
960  virtual nvinfer1::IBuilderConfig* createBuilderConfig() noexcept = 0;
961  virtual nvinfer1::ICudaEngine* buildEngineWithConfig(INetworkDefinition& network, IBuilderConfig& config) noexcept
962  = 0;
963  virtual nvinfer1::INetworkDefinition* createNetworkV2(NetworkDefinitionCreationFlags flags) noexcept = 0;
964  virtual nvinfer1::IOptimizationProfile* createOptimizationProfile() noexcept = 0;
965  virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
966  virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
967  virtual void reset() noexcept = 0;
968  virtual bool platformHasTf32() const noexcept = 0;
969  virtual nvinfer1::IHostMemory* buildSerializedNetwork(INetworkDefinition& network, IBuilderConfig& config) noexcept
970  = 0;
971 };
972 
973 } // namespace apiv
974 } // namespace nvinfer1
975 
976 #endif // NV_INFER_RUNTIME_IMPL_H
nvinfer1::LoopOutput
LoopOutput
Enum that describes kinds of loop outputs.
Definition: NvInfer.h:4739
nvinfer1::apiv::VAlgorithmIOInfo
Definition: NvInferImpl.h:858
nvinfer1::apiv::VShuffleLayer
Definition: NvInferImpl.h:613
nvinfer1::SliceMode
SliceMode
Controls how ISliceLayer handles out of bounds coordinates.
Definition: NvInfer.h:3863
nvinfer1::IFullyConnectedLayer
A fully connected layer in a network definition. This layer expects an input tensor of three or more ...
Definition: NvInfer.h:1439
nvinfer1::Permutation
Definition: NvInfer.h:3684
nvinfer1::IConstantLayer
Layer that represents a constant value.
Definition: NvInfer.h:4290
nvinfer1::IParametricReLULayer
Layer that represents a parametric ReLU operation.
Definition: NvInfer.h:4353
nvinfer1::DimensionOperation
DimensionOperation
An operation on two IDimensionExpr, which represent integer expressions used in dimension computation...
Definition: NvInferRuntime.h:211
nvinfer1::apiv::VLoopBoundaryLayer
Definition: NvInferImpl.h:705
nvinfer1::INetworkDefinition
A network definition for input to the builder.
Definition: NvInfer.h:5435
nvinfer1::Weights
An array of weights used as a layer parameter.
Definition: NvInferRuntime.h:145
nvinfer1::IAlgorithmVariant
provides a unique 128-bit identifier, which along with the input and output information denotes the v...
Definition: NvInfer.h:6830
nvinfer1::IPaddingLayer
Layer that represents a padding operation.
Definition: NvInfer.h:3572
nvinfer1::apiv::VAlgorithmVariant
Definition: NvInferImpl.h:866
nvinfer1::IConvolutionLayer
A convolution layer in a network definition.
Definition: NvInfer.h:987
nvinfer1::IHostMemory
Class to handle library allocated memory that is accessible to the user.
Definition: NvInferRuntime.h:163
nvinfer1::IIdentityLayer
A layer that represents the identity function.
Definition: NvInfer.h:4276
nvinfer1::DeviceType
DeviceType
The device that this layer/network will execute on.
Definition: NvInferRuntime.h:621
nvinfer1::ProfilingVerbosity
ProfilingVerbosity
List of verbosity levels of layer information exposed in NVTX annotations.
Definition: NvInfer.h:7098
nvinfer1::apiv::VFullyConnectedLayer
Definition: NvInferImpl.h:403
nvinfer1::IAlgorithmSelector
Interface implemented by application for selecting and reporting algorithms of a layer provided by th...
Definition: NvInfer.h:6983
nvinfer1::RNNInputMode
RNNInputMode
Enumerates the RNN input modes that may occur with an RNN layer.
Definition: NvInfer.h:3032
nvinfer1::ITripLimitLayer
Definition: NvInfer.h:4892
nvinfer1::IPluginV2
Plugin class for user-implemented layers.
Definition: NvInferRuntimeCommon.h:409
nvinfer1::IAlgorithmIOInfo
Carries information about input or output of the algorithm. IAlgorithmIOInfo for all the input and ou...
Definition: NvInfer.h:6787
nvinfer1::Dims32
Definition: NvInferRuntimeCommon.h:189
nvinfer1::apiv::VScaleLayer
Definition: NvInferImpl.h:467
nvinfer1::TensorFormats
uint32_t TensorFormats
It is capable of representing one or more TensorFormat by binary OR operations, e....
Definition: NvInfer.h:141
nvinfer1::apiv::VFillLayer
Definition: NvInferImpl.h:752
nvinfer1::apiv::VGatherLayer
Definition: NvInferImpl.h:536
nvinfer1::IRNNv2Layer
An RNN layer in a network definition, version 2.
Definition: NvInfer.h:3080
nvinfer1::DimsHW
Descriptor for two-dimensional spatial data.
Definition: NvInferLegacyDims.h:98
nvinfer1::apiv::VTripLimitLayer
Definition: NvInferImpl.h:724
nvinfer1::ResizeMode
ResizeMode
Enumerates various modes of resize in the resize layer. Resize mode set using setResizeMode().
Definition: NvInfer.h:4365
nvinfer1::TensorFormat
TensorFormat
Format of the input/output tensors.
Definition: NvInferRuntimeCommon.h:220
nvinfer1::ILoopOutputLayer
Definition: NvInfer.h:4834
nvinfer1::apiv::VConcatenationLayer
Definition: NvInferImpl.h:489
nvinfer1::apiv::VCudaEngine
Definition: NvInferImpl.h:252
nvinfer1::apiv::VLoop
Definition: NvInferImpl.h:738
nvinfer1::ICudaEngine
An engine for executing inference on a built network, with functionally unsafe features.
Definition: NvInferRuntime.h:1231
nvinfer1::QuantizationFlags
uint32_t QuantizationFlags
Represents one or more QuantizationFlag values using binary OR operations.
Definition: NvInfer.h:7025
nvinfer1::ActivationType
ActivationType
Enumerates the types of activation to perform in an activation layer.
Definition: NvInfer.h:148
nvinfer1::IRaggedSoftMaxLayer
A RaggedSoftmax layer in a network definition.
Definition: NvInfer.h:4258
nvinfer1::apiv::VExprBuilder
Definition: NvInferImpl.h:196
nvinfer1::apiv::VIdentityLayer
Definition: NvInferImpl.h:667
nvinfer1::ElementWiseOperation
ElementWiseOperation
Enumerates the binary operations that may be performed by an ElementWise layer.
Definition: NvInfer.h:2773
nvinfer1::apiv::VReduceLayer
Definition: NvInferImpl.h:589
nvinfer1::ISliceLayer
Slices an input tensor into an output tensor based on the offset and strides.
Definition: NvInfer.h:3906
nvinfer1::TacticSources
int32_t uint32_t TacticSources
Represents a collection of one or more TacticSource values combine using bitwise-OR operations.
Definition: NvInferImpl.h:158
nvinfer1::apiv::VAlgorithm
Definition: NvInferImpl.h:882
nvinfer1::apiv::VSoftMaxLayer
Definition: NvInferImpl.h:482
nvinfer1::ScaleMode
ScaleMode
Controls how shift, scale and power are applied in a Scale layer.
Definition: NvInfer.h:2088
nvinfer1::apiv::VSliceLayer
Definition: NvInferImpl.h:626
nvinfer1::ReduceOperation
ReduceOperation
Enumerates the reduce operations that may be performed by a Reduce layer.
Definition: NvInfer.h:3471
nvinfer1::IIteratorLayer
Definition: NvInfer.h:4905
nvinfer1::apiv::VQuantizeLayer
Definition: NvInferImpl.h:765
nvinfer1::BuilderFlag
BuilderFlag
List of valid modes that the builder can enable when creating an engine from a network definition.
Definition: NvInfer.h:7064
nvinfer1
The TensorRT API version 1 namespace.
nvinfer1::ILayer
Base class for all layer classes in a network definition.
Definition: NvInfer.h:511
nvinfer1::apiv::VLayer
Definition: NvInferImpl.h:347
nvinfer1::ILoop
Definition: NvInfer.h:4946
nvinfer1::IPluginV2Layer
Layer type for pluginV2.
Definition: NvInfer.h:3361
nvinfer1::apiv::VConstantLayer
Definition: NvInferImpl.h:672
nvinfer1::IElementWiseLayer
A elementwise layer in a network definition.
Definition: NvInfer.h:2815
nvinfer1::apiv::VRaggedSoftMaxLayer
Definition: NvInferImpl.h:662
nvinfer1::RNNGateType
RNNGateType
Identifies an individual gate within an RNN cell.
Definition: NvInfer.h:3052
cudaStream_t
struct CUstream_st * cudaStream_t
Forward declaration of cudaStream_t.
Definition: NvInferRuntimeCommon.h:107
nvinfer1::apiv::VTensor
Definition: NvInferImpl.h:322
nvinfer1::UnaryOperation
UnaryOperation
Enumerates the unary operations that may be performed by a Unary layer.
Definition: NvInfer.h:3386
nvinfer1::ITimingCache
Class to handle tactic timing info collected from builder.
Definition: NvInfer.h:7142
nvinfer1::apiv::VDequantizeLayer
Definition: NvInferImpl.h:772
nvinfer1::IShapeLayer
Layer type for getting shape of a tensor.
Definition: NvInfer.h:4053
nvinfer1::apiv::VRuntime
Definition: NvInferImpl.h:205
cudaEvent_t
struct CUevent_st * cudaEvent_t
Forward declaration of cudaEvent_t.
Definition: NvInferRuntimeCommon.h:110
nvinfer1::apiv::VTopKLayer
Definition: NvInferImpl.h:644
nvinfer1::NetworkDefinitionCreationFlag
NetworkDefinitionCreationFlag
List of immutable network properties expressed at network creation time. NetworkDefinitionCreationFla...
Definition: NvInfer.h:7816
nvinfer1::PaddingMode
PaddingMode
Enumerates the modes of padding to perform in convolution, deconvolution and pooling layer,...
Definition: NvInfer.h:955
nvinfer1::MatrixOperation
MatrixOperation
Enumerates the operations that may be performed on a tensor by IMatrixMultiplyLayer before multiplica...
Definition: NvInfer.h:4161
nvinfer1::IOptimizationProfile
Optimization profile for dynamic input dimensions and shape tensors.
Definition: NvInferRuntime.h:1055
nvinfer1::PoolingType
PoolingType
The type of pooling to perform in a pooling layer.
Definition: NvInfer.h:1622
nvinfer1::ResizeSelector
ResizeSelector
The coordinate selector when resize to single pixel output.
Definition: NvInfer.h:4438
nvinfer1::apiv::VActivationLayer
Definition: NvInferImpl.h:414
nvinfer1::apiv::VOptimizationProfile
Definition: NvInferImpl.h:237
nvinfer1::DataType
DataType
The type of weights and tensors.
Definition: NvInferRuntimeCommon.h:150
nvinfer1::IConcatenationLayer
A concatenation layer in a network definition.
Definition: NvInfer.h:2327
nvinfer1::IShuffleLayer
Layer type for shuffling data.
Definition: NvInfer.h:3707
nvinfer1::apiv::VConvolutionLayer
Definition: NvInferImpl.h:368
nvinfer1::RNNOperation
RNNOperation
Enumerates the RNN operations that may be performed by an RNN layer.
Definition: NvInfer.h:2982
nvinfer1::ITensor
A tensor in a network definition.
Definition: NvInfer.h:187
nvinfer1::IRecurrenceLayer
Definition: NvInfer.h:4789
nvinfer1::apiv::VPaddingLayer
Definition: NvInferImpl.h:600
nvinfer1::apiv::VUnaryLayer
Definition: NvInferImpl.h:582
nvinfer1::LayerType
LayerType
The type values of layer classes.
Definition: NvInfer.h:89
nvinfer1::apiv::VHostMemory
Definition: NvInferImpl.h:181
nvinfer1::apiv::VBuilderConfig
Definition: NvInferImpl.h:900
nvinfer1::ILRNLayer
A LRN layer in a network definition.
Definition: NvInfer.h:1988
nvinfer1::RNNDirection
RNNDirection
Enumerates the RNN direction that may be performed by an RNN layer.
Definition: NvInfer.h:3004
nvinfer1::TripLimit
TripLimit
Enum that describes kinds of trip limits.
Definition: NvInfer.h:4759
nvinfer1::IReduceLayer
Layer that represents a reduction operator across Shape, Int32, Float, and Half tensors.
Definition: NvInfer.h:3494
nvinfer1::IMatrixMultiplyLayer
Layer that represents a Matrix Multiplication.
Definition: NvInfer.h:4215
nvinfer1::apiv::VPoolingLayer
Definition: NvInferImpl.h:425
nvinfer1::FillOperation
FillOperation
Enumerates the tensor fill operations that may performed by a fill layer.
Definition: NvInfer.h:5051
nvinfer1::apiv::VMatrixMultiplyLayer
Definition: NvInferImpl.h:655
nvinfer1::IFillLayer
Generate an output tensor with specified mode.
Definition: NvInfer.h:5089
NvInferRuntimeCommon.h
nvinfer1::QuantizationFlag
QuantizationFlag
List of valid flags for quantizing the network to int8.
Definition: NvInfer.h:7034
nvinfer1::IInt8Calibrator
Application-implemented interface for calibration.
Definition: NvInfer.h:6605
nvinfer1::apiv::VPluginLayer
Definition: NvInferImpl.h:570
nvinfer1::IErrorRecorder
Reference counted application-implemented error reporting interface for TensorRT objects.
Definition: NvInferRuntimeCommon.h:1353
nvinfer1::apiv::VDeconvolutionLayer
Definition: NvInferImpl.h:496
nvinfer1::ITopKLayer
Layer that represents a TopK reduction.
Definition: NvInfer.h:4085
nvinfer1::ResizeRoundMode
ResizeRoundMode
The rounding mode for nearest neighbor resize.
Definition: NvInfer.h:4465
nvinfer1::OptProfileSelector
OptProfileSelector
When setting or querying optimization profile parameters (such as shape tensor inputs or dynamic dime...
Definition: NvInferRuntime.h:1019
nvinfer1::WeightsRole
WeightsRole
How a layer uses particular Weights.
Definition: NvInferRuntime.h:599
nvinfer1::ISelectLayer
Definition: NvInfer.h:5037
nvinfer1::apiv::VSelectLayer
Definition: NvInferImpl.h:748
nvinfer1::NetworkDefinitionCreationFlags
uint32_t NetworkDefinitionCreationFlags
Represents one or more NetworkDefinitionCreationFlag flags using binary OR operations....
Definition: NvInfer.h:7805
nvinfer1::IGatherLayer
Definition: NvInfer.h:2852
nvinfer1::apiv::VElementWiseLayer
Definition: NvInferImpl.h:529
nvinfer1::IDimensionExpr
Definition: NvInferRuntime.h:263
nvinfer1::apiv::VRNNv2Layer
Definition: NvInferImpl.h:545
nvinfer1::BuilderFlags
uint32_t BuilderFlags
Represents one or more QuantizationFlag values using binary OR operations, e.g., 1U << BuilderFlag::k...
Definition: NvInfer.h:7055
nvinfer1::EngineCapability
EngineCapability
List of supported engine capability flows.
Definition: NvInferRuntime.h:103
nvinfer1::apiv::VRecurrenceLayer
Definition: NvInferImpl.h:711
nvinfer1::ResizeCoordinateTransformation
ResizeCoordinateTransformation
The resize coordinate transformation function.
Definition: NvInfer.h:4388
nvinfer1::apiv::VLRNLayer
Definition: NvInferImpl.h:454
nvinfer1::IProfiler
Application-implemented interface for profiling.
Definition: NvInferRuntime.h:579
nvinfer1::apiv::VTimingCache
Definition: NvInferImpl.h:892
nvinfer1::apiv::VPluginV2Layer
Definition: NvInferImpl.h:576
nvinfer1::IScaleLayer
A Scale layer in a network definition.
Definition: NvInfer.h:2128
nvinfer1::ISoftMaxLayer
A Softmax layer in a network definition.
Definition: NvInfer.h:2263
nvinfer1::IBuilderConfig
Holds properties for configuring a builder to produce an engine.
Definition: NvInfer.h:7204
nvinfer1::apiv::VDimensionExpr
Definition: NvInferImpl.h:189
nvinfer1::apiv::VParametricReLULayer
Definition: NvInferImpl.h:681
nvinfer1::apiv::VRoot
Definition: NvInferImpl.h:175
nvinfer1::apiv::VExecutionContext
Definition: NvInferImpl.h:291
nvinfer1::IResizeLayer
A resize layer in a network definition.
Definition: NvInfer.h:4511
nvinfer1::apiv::VShapeLayer
Definition: NvInferImpl.h:639
nvinfer1::apiv::VResizeLayer
Definition: NvInferImpl.h:686
nvinfer1::IPoolingLayer
A Pooling layer in a network definition.
Definition: NvInfer.h:1650
nvinfer1::apiv::VNetworkDefinition
Definition: NvInferImpl.h:779
nvinfer1::apiv::VAlgorithmContext
Definition: NvInferImpl.h:873
nvinfer1::IGpuAllocator
Application-implemented class for controlling allocation on the GPU.
Definition: NvInferRuntimeCommon.h:1093
nvinfer1::IUnaryLayer
Layer that represents an unary operation.
Definition: NvInfer.h:3425
nvinfer1::apiv::VBuilder
Definition: NvInferImpl.h:950
nvinfer1::apiv::VLoopOutputLayer
Definition: NvInferImpl.h:716
nvinfer1::IQuantizeLayer
A Quantize layer in a network definition.
Definition: NvInfer.h:5297
nvinfer1::apiv::VRefitter
Definition: NvInferImpl.h:219
nvinfer1::TopKOperation
TopKOperation
Enumerates the operations that may be performed by a TopK layer.
Definition: NvInfer.h:4065
nvinfer1::TensorLocation
TensorLocation
The location for tensor data storage, device or host.
Definition: NvInferRuntime.h:235
nvinfer1::IExecutionContext
Context for executing inference using an engine, with functionally unsafe features.
Definition: NvInferRuntime.h:1745
nvinfer1::IActivationLayer
An Activation layer in a network definition.
Definition: NvInfer.h:1540
nvinfer1::CalibrationAlgoType
CalibrationAlgoType
Version of calibration algorithm to use.
Definition: NvInfer.h:6579
nvinfer1::apiv::VIteratorLayer
Definition: NvInferImpl.h:730
nvinfer1::IDequantizeLayer
A Dequantize layer in a network definition.
Definition: NvInfer.h:5384
nvinfer1::IDeconvolutionLayer
A deconvolution layer in a network definition.
Definition: NvInfer.h:2367