TensorRT  8.0.3
NvInferImpl.h
Go to the documentation of this file.
1 /*
2  * Copyright 1993-2021 NVIDIA Corporation. All rights reserved.
3  *
4  * NOTICE TO LICENSEE:
5  *
6  * This source code and/or documentation ("Licensed Deliverables") are
7  * subject to NVIDIA intellectual property rights under U.S. and
8  * international Copyright laws.
9  *
10  * These Licensed Deliverables contained herein is PROPRIETARY and
11  * CONFIDENTIAL to NVIDIA and is being provided under the terms and
12  * conditions of a form of NVIDIA software license agreement by and
13  * between NVIDIA and Licensee ("License Agreement") or electronically
14  * accepted by Licensee. Notwithstanding any terms or conditions to
15  * the contrary in the License Agreement, reproduction or disclosure
16  * of the Licensed Deliverables to any third party without the express
17  * written consent of NVIDIA is prohibited.
18  *
19  * NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
20  * LICENSE AGREEMENT, NVIDIA MAKES NO REPRESENTATION ABOUT THE
21  * SUITABILITY OF THESE LICENSED DELIVERABLES FOR ANY PURPOSE. IT IS
22  * PROVIDED "AS IS" WITHOUT EXPRESS OR IMPLIED WARRANTY OF ANY KIND.
23  * NVIDIA DISCLAIMS ALL WARRANTIES WITH REGARD TO THESE LICENSED
24  * DELIVERABLES, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY,
25  * NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE.
26  * NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
27  * LICENSE AGREEMENT, IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY
28  * SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, OR ANY
29  * DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
30  * WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
31  * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
32  * OF THESE LICENSED DELIVERABLES.
33  *
34  * U.S. Government End Users. These Licensed Deliverables are a
35  * "commercial item" as that term is defined at 48 C.F.R. 2.101 (OCT
36  * 1995), consisting of "commercial computer software" and "commercial
37  * computer software documentation" as such terms are used in 48
38  * C.F.R. 12.212 (SEPT 1995) and is provided to the U.S. Government
39  * only as a commercial end item. Consistent with 48 C.F.R.12.212 and
40  * 48 C.F.R. 227.7202-1 through 227.7202-4 (JUNE 1995), all
41  * U.S. Government End Users acquire the Licensed Deliverables with
42  * only those rights set forth herein.
43  *
44  * Any use of the Licensed Deliverables in individual and commercial
45  * software must include, in the user documentation and internal
46  * comments to the code, the above Disclaimer and U.S. Government End
47  * Users Notice.
48  */
49 
50 #ifndef NV_INFER_IMPL_H
51 #define NV_INFER_IMPL_H
52 
53 #include "NvInferLegacyDims.h"
54 #include "NvInferRuntimeCommon.h"
55 
56 namespace nvinfer1
57 {
58 
59 class IActivationLayer;
60 class IAlgorithm;
61 class IAlgorithmContext;
62 class IAlgorithmIOInfo;
63 class IAlgorithmSelector;
64 class IAlgorithmVariant;
65 class IBuilderConfig;
66 class IConcatenationLayer;
67 class IConstantLayer;
68 class IConvolutionLayer;
69 class ICudaEngine;
70 class IDeconvolutionLayer;
71 class IDequantizeLayer;
72 class IDimensionExpr;
73 class IElementWiseLayer;
74 class IExecutionContext;
75 class IFillLayer;
76 class IFullyConnectedLayer;
77 class IGatherLayer;
78 class IHostMemory;
79 class IIdentityLayer;
80 class IInt8Calibrator;
81 class IIteratorLayer;
82 class ILayer;
83 class ILoop;
84 class ILoopOutputLayer;
85 class ILRNLayer;
86 class IMatrixMultiplyLayer;
87 class INetworkDefinition;
88 class IOptimizationProfile;
89 class IPaddingLayer;
90 class IParametricReLULayer;
91 class IPlugin;
92 class IPluginExt;
93 class IPluginFactory;
94 class IPluginLayer;
95 class IPluginV2Layer;
96 class IPoolingLayer;
97 class IProfiler;
98 class IQuantizeLayer;
99 class IRaggedSoftMaxLayer;
100 class IRecurrenceLayer;
101 class IReduceLayer;
102 class IResizeLayer;
103 class IRNNv2Layer;
104 class IScaleLayer;
105 class ISelectLayer;
106 class IShapeLayer;
107 class IShuffleLayer;
108 class ISliceLayer;
109 class ISoftMaxLayer;
110 class ITensor;
111 class ITimingCache;
112 class ITopKLayer;
113 class ITripLimitLayer;
114 class IUnaryLayer;
115 struct Permutation;
116 class Weights;
117 
118 enum class ActivationType : int32_t;
119 enum class BuilderFlag : int32_t;
120 enum class CalibrationAlgoType : int32_t;
121 enum class DeviceType : int32_t;
122 enum class DimensionOperation : int32_t;
123 enum class ElementWiseOperation : int32_t;
124 enum class EngineCapability : int32_t;
125 enum class FillOperation : int32_t;
126 enum class LayerType : int32_t;
127 enum class LoopOutput : int32_t;
128 enum class MatrixOperation : int32_t;
129 enum class NetworkDefinitionCreationFlag : int32_t;
130 enum class OptProfileSelector : int32_t;
131 enum class PaddingMode : int32_t;
132 enum class PoolingType : int32_t;
133 enum class ProfilingVerbosity : int32_t;
134 enum class QuantizationFlag : int32_t;
135 enum class ReduceOperation : int32_t;
136 enum class ResizeCoordinateTransformation : int32_t;
137 enum class ResizeMode : int32_t;
138 enum class ResizeRoundMode : int32_t;
139 enum class ResizeSelector : int32_t;
140 enum class RNNDirection : int32_t;
141 enum class RNNGateType : int32_t;
142 enum class RNNInputMode : int32_t;
143 enum class RNNOperation : int32_t;
144 enum class ScaleMode : int32_t;
145 enum class SliceMode : int32_t;
146 enum class TensorLocation : int32_t;
147 enum class TopKOperation : int32_t;
148 enum class TripLimit : int32_t;
149 enum class UnaryOperation : int32_t;
150 enum class WeightsRole : int32_t;
151 
152 using TacticSources = uint32_t;
153 using TensorFormats = uint32_t;
154 using BuilderFlags = uint32_t;
155 using NetworkDefinitionCreationFlags = uint32_t;
156 using QuantizationFlags = uint32_t;
157 
165 
166 namespace apiv
167 {
168 
169 class VRoot
170 {
171 public:
172  virtual ~VRoot() noexcept = default;
173 };
174 
175 class VHostMemory : public VRoot
176 {
177 public:
178  virtual void* data() const noexcept = 0;
179  virtual std::size_t size() const noexcept = 0;
180  virtual DataType type() const noexcept = 0;
181 };
182 
183 class VDimensionExpr : public VRoot
184 {
185 public:
186  virtual bool isConstant() const = 0;
187  virtual int32_t getConstantValue() const = 0;
188 };
189 
190 class VExprBuilder : public VRoot
191 {
192 public:
193  virtual const IDimensionExpr* constant(int32_t value) = 0;
194  virtual const IDimensionExpr* operation(
195  DimensionOperation op, const IDimensionExpr& first, const IDimensionExpr& second)
196  = 0;
197 };
198 
199 class VRuntime : public VRoot
200 {
201 public:
202  virtual nvinfer1::ICudaEngine* deserializeCudaEngine(
203  const void* blob, std::size_t size, IPluginFactory* pluginFactory) noexcept
204  = 0;
205  virtual void setDLACore(int32_t dlaCore) noexcept = 0;
206  virtual int32_t getDLACore() const noexcept = 0;
207  virtual int32_t getNbDLACores() const noexcept = 0;
208  virtual void setGpuAllocator(IGpuAllocator* allocator) noexcept = 0;
209  virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
210  virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
211 };
212 
213 class VRefitter : public VRoot
214 {
215 public:
216  virtual bool setWeights(const char* layerName, WeightsRole role, const Weights weights) noexcept = 0;
217  virtual bool refitCudaEngine() noexcept = 0;
218  virtual int32_t getMissing(int32_t size, const char** layerNames, WeightsRole* roles) noexcept = 0;
219  virtual int32_t getAll(int32_t size, const char** layerNames, WeightsRole* roles) noexcept = 0;
220  virtual bool setDynamicRange(const char* tensorName, float min, float max) noexcept = 0;
221  virtual float getDynamicRangeMin(const char* tensorName) const noexcept = 0;
222  virtual float getDynamicRangeMax(const char* tensorName) const noexcept = 0;
223  virtual int32_t getTensorsWithDynamicRange(int32_t size, const char** tensorNames) const noexcept = 0;
224  virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
225  virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
226  virtual bool setNamedWeights(const char* name, Weights weights) noexcept = 0;
227  virtual int32_t getMissingWeights(int32_t size, const char** weightsNames) noexcept = 0;
228  virtual int32_t getAllWeights(int32_t size, const char** weightsNames) noexcept = 0;
229 };
230 
231 class VOptimizationProfile : public VRoot
232 {
233 public:
234  virtual bool setDimensions(const char* inputName, OptProfileSelector select, Dims dims) noexcept = 0;
235  virtual Dims getDimensions(const char* inputName, OptProfileSelector select) const noexcept = 0;
236  virtual bool setShapeValues(
237  const char* inputName, OptProfileSelector select, const int32_t* values, int32_t nbValues) noexcept
238  = 0;
239  virtual int32_t getNbShapeValues(const char* inputName) const noexcept = 0;
240  virtual int32_t const* getShapeValues(const char* inputName, OptProfileSelector select) const noexcept = 0;
241  virtual bool setExtraMemoryTarget(float target) noexcept = 0;
242  virtual float getExtraMemoryTarget() const noexcept = 0;
243  virtual bool isValid() const noexcept = 0;
244 };
245 
246 class VCudaEngine : public VRoot
247 {
248 public:
249  virtual int32_t getNbBindings() const noexcept = 0;
250  virtual int32_t getBindingIndex(const char* name) const noexcept = 0;
251  virtual const char* getBindingName(int32_t bindingIndex) const noexcept = 0;
252  virtual bool bindingIsInput(int32_t bindingIndex) const noexcept = 0;
253  virtual Dims getBindingDimensions(int32_t bindingIndex) const noexcept = 0;
254  virtual DataType getBindingDataType(int32_t bindingIndex) const noexcept = 0;
255  virtual int32_t getMaxBatchSize() const noexcept = 0;
256  virtual int32_t getNbLayers() const noexcept = 0;
257  virtual IHostMemory* serialize() const noexcept = 0;
258  virtual IExecutionContext* createExecutionContext() noexcept = 0;
259  virtual TensorLocation getLocation(int32_t bindingIndex) const noexcept = 0;
260  virtual IExecutionContext* createExecutionContextWithoutDeviceMemory() noexcept = 0;
261  virtual size_t getDeviceMemorySize() const noexcept = 0;
262  virtual bool isRefittable() const noexcept = 0;
263  virtual int32_t getBindingBytesPerComponent(int32_t bindingIndex) const noexcept = 0;
264  virtual int32_t getBindingComponentsPerElement(int32_t bindingIndex) const noexcept = 0;
265  virtual TensorFormat getBindingFormat(int32_t bindingIndex) const noexcept = 0;
266  virtual const char* getBindingFormatDesc(int32_t bindingIndex) const noexcept = 0;
267  virtual int32_t getBindingVectorizedDim(int32_t bindingIndex) const noexcept = 0;
268  virtual const char* getName() const noexcept = 0;
269  virtual int32_t getNbOptimizationProfiles() const noexcept = 0;
270  virtual Dims getProfileDimensions(int32_t bindingIndex, int32_t profileIndex, OptProfileSelector select) const
271  noexcept
272  = 0;
273  virtual const int32_t* getProfileShapeValues(
274  int32_t profileIndex, int32_t inputIndex, OptProfileSelector select) const noexcept
275  = 0;
276  virtual bool isShapeBinding(int32_t bindingIndex) const noexcept = 0;
277  virtual bool isExecutionBinding(int32_t bindingIndex) const noexcept = 0;
278  virtual EngineCapability getEngineCapability() const noexcept = 0;
279  virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
280  virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
281  virtual bool hasImplicitBatchDimension() const noexcept = 0;
282  virtual TacticSources getTacticSources() const noexcept = 0;
283 };
284 
285 class VExecutionContext : public VRoot
286 {
287 public:
288  virtual bool execute(int32_t batchSize, void* const* bindings) noexcept = 0;
289  virtual bool enqueue(
290  int32_t batchSize, void* const* bindings, cudaStream_t stream, cudaEvent_t* inputConsumed) noexcept
291  = 0;
292  virtual void setDebugSync(bool sync) noexcept = 0;
293  virtual bool getDebugSync() const noexcept = 0;
294  virtual void setProfiler(IProfiler*) noexcept = 0;
295  virtual IProfiler* getProfiler() const noexcept = 0;
296  virtual const ICudaEngine& getEngine() const noexcept = 0;
297  virtual void setName(const char* name) noexcept = 0;
298  virtual const char* getName() const noexcept = 0;
299  virtual void setDeviceMemory(void* memory) noexcept = 0;
300  virtual Dims getStrides(int32_t bindingIndex) const noexcept = 0;
301  virtual bool setOptimizationProfile(int32_t profileIndex) noexcept = 0;
302  virtual int32_t getOptimizationProfile() const noexcept = 0;
303  virtual bool setBindingDimensions(int32_t bindingIndex, Dims dimensions) noexcept = 0;
304  virtual Dims getBindingDimensions(int32_t bindingIndex) const noexcept = 0;
305  virtual bool setInputShapeBinding(int32_t bindingIndex, int32_t const* data) noexcept = 0;
306  virtual bool getShapeBinding(int32_t bindingIndex, int32_t* data) const noexcept = 0;
307  virtual bool allInputDimensionsSpecified() const noexcept = 0;
308  virtual bool allInputShapesSpecified() const noexcept = 0;
309  virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
310  virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
311  virtual bool executeV2(void* const* bindings) noexcept = 0;
312  virtual bool enqueueV2(void* const* bindings, cudaStream_t stream, cudaEvent_t* inputConsumed) noexcept = 0;
313  virtual bool setOptimizationProfileAsync(int32_t profileIndex, cudaStream_t stream) noexcept = 0;
314 };
315 
316 class VTensor : public VRoot
317 {
318 public:
319  virtual void setName(const char* name) noexcept = 0;
320  virtual const char* getName() const noexcept = 0;
321  virtual void setDimensions(Dims dimensions) noexcept = 0;
322  virtual Dims getDimensions() const noexcept = 0;
323  virtual void setType(DataType type) noexcept = 0;
324  virtual DataType getType() const noexcept = 0;
325  virtual bool setDynamicRange(float min, float max) noexcept = 0;
326  virtual bool isNetworkInput() const noexcept = 0;
327  virtual bool isNetworkOutput() const noexcept = 0;
328  virtual void setBroadcastAcrossBatch(bool broadcastAcrossBatch) noexcept = 0;
329  virtual bool getBroadcastAcrossBatch() const noexcept = 0;
330  virtual TensorLocation getLocation() const noexcept = 0;
331  virtual void setLocation(TensorLocation location) noexcept = 0;
332  virtual bool dynamicRangeIsSet() const noexcept = 0;
333  virtual void resetDynamicRange() noexcept = 0;
334  virtual float getDynamicRangeMin() const noexcept = 0;
335  virtual float getDynamicRangeMax() const noexcept = 0;
336  virtual void setAllowedFormats(TensorFormats formats) noexcept = 0;
337  virtual TensorFormats getAllowedFormats() const noexcept = 0;
338  virtual bool isShapeTensor() const noexcept = 0;
339  virtual bool isExecutionTensor() const noexcept = 0;
340 };
341 class VLayer : public VRoot
342 {
343 public:
344  virtual LayerType getType() const noexcept = 0;
345  virtual void setName(const char* name) noexcept = 0;
346  virtual const char* getName() const noexcept = 0;
347  virtual int32_t getNbInputs() const noexcept = 0;
348  virtual ITensor* getInput(int32_t index) const noexcept = 0;
349  virtual int32_t getNbOutputs() const noexcept = 0;
350  virtual ITensor* getOutput(int32_t index) const noexcept = 0;
351  virtual void setInput(int32_t index, ITensor& tensor) noexcept = 0;
352  virtual void setPrecision(DataType dataType) noexcept = 0;
353  virtual DataType getPrecision() const noexcept = 0;
354  virtual bool precisionIsSet() const noexcept = 0;
355  virtual void resetPrecision() noexcept = 0;
356  virtual void setOutputType(int32_t index, DataType dataType) noexcept = 0;
357  virtual DataType getOutputType(int32_t index) const noexcept = 0;
358  virtual bool outputTypeIsSet(int32_t index) const noexcept = 0;
359  virtual void resetOutputType(int32_t index) noexcept = 0;
360 };
361 
362 class VConvolutionLayer : public VRoot
363 {
364 public:
365  virtual void setKernelSize(DimsHW kernelSize) noexcept = 0;
366  virtual DimsHW getKernelSize() const noexcept = 0;
367  virtual void setNbOutputMaps(int32_t nbOutputMaps) noexcept = 0;
368  virtual int32_t getNbOutputMaps() const noexcept = 0;
369  virtual void setStride(DimsHW stride) noexcept = 0;
370  virtual DimsHW getStride() const noexcept = 0;
371  virtual void setPadding(DimsHW padding) noexcept = 0;
372  virtual DimsHW getPadding() const noexcept = 0;
373  virtual void setNbGroups(int32_t nbGroups) noexcept = 0;
374  virtual int32_t getNbGroups() const noexcept = 0;
375  virtual void setKernelWeights(Weights weights) noexcept = 0;
376  virtual Weights getKernelWeights() const noexcept = 0;
377  virtual void setBiasWeights(Weights weights) noexcept = 0;
378  virtual Weights getBiasWeights() const noexcept = 0;
379  virtual void setDilation(DimsHW dilation) noexcept = 0;
380  virtual DimsHW getDilation() const noexcept = 0;
381  virtual void setPrePadding(Dims padding) noexcept = 0;
382  virtual Dims getPrePadding() const noexcept = 0;
383  virtual void setPostPadding(Dims padding) noexcept = 0;
384  virtual Dims getPostPadding() const noexcept = 0;
385  virtual void setPaddingMode(PaddingMode paddingMode) noexcept = 0;
386  virtual PaddingMode getPaddingMode() const noexcept = 0;
387  virtual void setKernelSizeNd(Dims kernelSize) noexcept = 0;
388  virtual Dims getKernelSizeNd() const noexcept = 0;
389  virtual void setStrideNd(Dims stride) noexcept = 0;
390  virtual Dims getStrideNd() const noexcept = 0;
391  virtual void setPaddingNd(Dims padding) noexcept = 0;
392  virtual Dims getPaddingNd() const noexcept = 0;
393  virtual void setDilationNd(Dims dilation) noexcept = 0;
394  virtual Dims getDilationNd() const noexcept = 0;
395 };
396 
397 class VFullyConnectedLayer : public VRoot
398 {
399 public:
400  virtual void setNbOutputChannels(int32_t nbOutputs) noexcept = 0;
401  virtual int32_t getNbOutputChannels() const noexcept = 0;
402  virtual void setKernelWeights(Weights weights) noexcept = 0;
403  virtual Weights getKernelWeights() const noexcept = 0;
404  virtual void setBiasWeights(Weights weights) noexcept = 0;
405  virtual Weights getBiasWeights() const noexcept = 0;
406 };
407 
408 class VActivationLayer : public VRoot
409 {
410 public:
411  virtual void setActivationType(ActivationType type) noexcept = 0;
412  virtual ActivationType getActivationType() const noexcept = 0;
413  virtual void setAlpha(float alpha) noexcept = 0;
414  virtual void setBeta(float beta) noexcept = 0;
415  virtual float getAlpha() const noexcept = 0;
416  virtual float getBeta() const noexcept = 0;
417 };
418 
419 class VPoolingLayer : public VRoot
420 {
421 public:
422  virtual void setPoolingType(PoolingType type) noexcept = 0;
423  virtual PoolingType getPoolingType() const noexcept = 0;
424  virtual void setWindowSize(DimsHW windowSize) noexcept = 0;
425  virtual DimsHW getWindowSize() const noexcept = 0;
426  virtual void setStride(DimsHW stride) noexcept = 0;
427  virtual DimsHW getStride() const noexcept = 0;
428  virtual void setPadding(DimsHW padding) noexcept = 0;
429  virtual DimsHW getPadding() const noexcept = 0;
430  virtual void setBlendFactor(float blendFactor) noexcept = 0;
431  virtual float getBlendFactor() const noexcept = 0;
432  virtual void setAverageCountExcludesPadding(bool exclusive) noexcept = 0;
433  virtual bool getAverageCountExcludesPadding() const noexcept = 0;
434  virtual void setPrePadding(Dims padding) noexcept = 0;
435  virtual Dims getPrePadding() const noexcept = 0;
436  virtual void setPostPadding(Dims padding) noexcept = 0;
437  virtual Dims getPostPadding() const noexcept = 0;
438  virtual void setPaddingMode(PaddingMode paddingMode) noexcept = 0;
439  virtual PaddingMode getPaddingMode() const noexcept = 0;
440  virtual void setWindowSizeNd(Dims windowSize) noexcept = 0;
441  virtual Dims getWindowSizeNd() const noexcept = 0;
442  virtual void setStrideNd(Dims stride) noexcept = 0;
443  virtual Dims getStrideNd() const noexcept = 0;
444  virtual void setPaddingNd(Dims padding) noexcept = 0;
445  virtual Dims getPaddingNd() const noexcept = 0;
446 };
447 
448 class VLRNLayer : public VRoot
449 {
450 public:
451  virtual void setWindowSize(int32_t windowSize) noexcept = 0;
452  virtual int32_t getWindowSize() const noexcept = 0;
453  virtual void setAlpha(float alpha) noexcept = 0;
454  virtual float getAlpha() const noexcept = 0;
455  virtual void setBeta(float beta) noexcept = 0;
456  virtual float getBeta() const noexcept = 0;
457  virtual void setK(float k) noexcept = 0;
458  virtual float getK() const noexcept = 0;
459 };
460 
461 class VScaleLayer : public VRoot
462 {
463 public:
464  virtual void setMode(ScaleMode mode) noexcept = 0;
465  virtual ScaleMode getMode() const noexcept = 0;
466  virtual void setShift(Weights shift) noexcept = 0;
467  virtual Weights getShift() const noexcept = 0;
468  virtual void setScale(Weights scale) noexcept = 0;
469  virtual Weights getScale() const noexcept = 0;
470  virtual void setPower(Weights power) noexcept = 0;
471  virtual Weights getPower() const noexcept = 0;
472  virtual int32_t getChannelAxis() const noexcept = 0;
473  virtual void setChannelAxis(int32_t channelAxis) noexcept = 0;
474 };
475 
476 class VSoftMaxLayer : public VRoot
477 {
478 public:
479  virtual void setAxes(uint32_t axes) noexcept = 0;
480  virtual uint32_t getAxes() const noexcept = 0;
481 };
482 
483 class VConcatenationLayer : public VRoot
484 {
485 public:
486  virtual void setAxis(int32_t axis) noexcept = 0;
487  virtual int32_t getAxis() const noexcept = 0;
488 };
489 
490 class VDeconvolutionLayer : public VRoot
491 {
492 public:
493  virtual void setKernelSize(DimsHW kernelSize) noexcept = 0;
494  virtual DimsHW getKernelSize() const noexcept = 0;
495  virtual void setNbOutputMaps(int32_t nbOutputMaps) noexcept = 0;
496  virtual int32_t getNbOutputMaps() const noexcept = 0;
497  virtual void setStride(DimsHW stride) noexcept = 0;
498  virtual DimsHW getStride() const noexcept = 0;
499  virtual void setPadding(DimsHW padding) noexcept = 0;
500  virtual DimsHW getPadding() const noexcept = 0;
501  virtual void setNbGroups(int32_t nbGroups) noexcept = 0;
502  virtual int32_t getNbGroups() const noexcept = 0;
503  virtual void setKernelWeights(Weights weights) noexcept = 0;
504  virtual Weights getKernelWeights() const noexcept = 0;
505  virtual void setBiasWeights(Weights weights) noexcept = 0;
506  virtual Weights getBiasWeights() const noexcept = 0;
507  virtual void setPrePadding(Dims padding) noexcept = 0;
508  virtual Dims getPrePadding() const noexcept = 0;
509  virtual void setPostPadding(Dims padding) noexcept = 0;
510  virtual Dims getPostPadding() const noexcept = 0;
511  virtual void setPaddingMode(PaddingMode paddingMode) noexcept = 0;
512  virtual PaddingMode getPaddingMode() const noexcept = 0;
513  virtual void setKernelSizeNd(Dims kernelSize) noexcept = 0;
514  virtual Dims getKernelSizeNd() const noexcept = 0;
515  virtual void setStrideNd(Dims stride) noexcept = 0;
516  virtual Dims getStrideNd() const noexcept = 0;
517  virtual void setPaddingNd(Dims padding) noexcept = 0;
518  virtual Dims getPaddingNd() const noexcept = 0;
519  virtual void setDilationNd(Dims dilation) noexcept = 0;
520  virtual Dims getDilationNd() const noexcept = 0;
521 };
522 
523 class VElementWiseLayer : public VRoot
524 {
525 public:
526  virtual void setOperation(ElementWiseOperation op) noexcept = 0;
527  virtual ElementWiseOperation getOperation() const noexcept = 0;
528 };
529 
530 class VGatherLayer : public VRoot
531 {
532 public:
533  virtual void setGatherAxis(int32_t axis) noexcept = 0;
534  virtual int32_t getGatherAxis() const noexcept = 0;
535  virtual void setNbElementWiseDims(int32_t k) noexcept = 0;
536  virtual int32_t getNbElementWiseDims() const noexcept = 0;
537 };
538 
539 class VRNNv2Layer : public VRoot
540 {
541 public:
542  virtual int32_t getLayerCount() const noexcept = 0;
543  virtual int32_t getHiddenSize() const noexcept = 0;
544  virtual int32_t getMaxSeqLength() const noexcept = 0;
545  virtual int32_t getDataLength() const noexcept = 0;
546  virtual void setSequenceLengths(ITensor& seqLengths) noexcept = 0;
547  virtual ITensor* getSequenceLengths() const noexcept = 0;
548  virtual void setOperation(RNNOperation op) noexcept = 0;
549  virtual RNNOperation getOperation() const noexcept = 0;
550  virtual void setInputMode(RNNInputMode op) noexcept = 0;
551  virtual RNNInputMode getInputMode() const noexcept = 0;
552  virtual void setDirection(RNNDirection op) noexcept = 0;
553  virtual RNNDirection getDirection() const noexcept = 0;
554  virtual void setWeightsForGate(int32_t layerIndex, RNNGateType gate, bool isW, Weights weights) noexcept = 0;
555  virtual Weights getWeightsForGate(int32_t layerIndex, RNNGateType gate, bool isW) const noexcept = 0;
556  virtual void setBiasForGate(int32_t layerIndex, RNNGateType gate, bool isW, Weights bias) noexcept = 0;
557  virtual Weights getBiasForGate(int32_t layerIndex, RNNGateType gate, bool isW) const noexcept = 0;
558  virtual void setHiddenState(ITensor& hidden) noexcept = 0;
559  virtual ITensor* getHiddenState() const noexcept = 0;
560  virtual void setCellState(ITensor& cell) noexcept = 0;
561  virtual ITensor* getCellState() const noexcept = 0;
562 };
563 
564 class VPluginLayer : public VRoot
565 {
566 public:
567  virtual IPlugin& getPlugin() noexcept = 0;
568 };
569 
570 class VPluginV2Layer : public VRoot
571 {
572 public:
573  virtual IPluginV2& getPlugin() noexcept = 0;
574 };
575 
576 class VUnaryLayer : public VRoot
577 {
578 public:
579  virtual void setOperation(UnaryOperation op) noexcept = 0;
580  virtual UnaryOperation getOperation() const noexcept = 0;
581 };
582 
583 class VReduceLayer : public VRoot
584 {
585 public:
586  virtual void setOperation(ReduceOperation op) noexcept = 0;
587  virtual ReduceOperation getOperation() const noexcept = 0;
588  virtual void setReduceAxes(uint32_t reduceAxes) noexcept = 0;
589  virtual uint32_t getReduceAxes() const noexcept = 0;
590  virtual void setKeepDimensions(bool keepDimensions) noexcept = 0;
591  virtual bool getKeepDimensions() const noexcept = 0;
592 };
593 
594 class VPaddingLayer : public VRoot
595 {
596 public:
597  virtual void setPrePadding(DimsHW padding) noexcept = 0;
598  virtual DimsHW getPrePadding() const noexcept = 0;
599  virtual void setPostPadding(DimsHW padding) noexcept = 0;
600  virtual DimsHW getPostPadding() const noexcept = 0;
601  virtual void setPrePaddingNd(Dims padding) noexcept = 0;
602  virtual Dims getPrePaddingNd() const noexcept = 0;
603  virtual void setPostPaddingNd(Dims padding) noexcept = 0;
604  virtual Dims getPostPaddingNd() const noexcept = 0;
605 };
606 
607 class VShuffleLayer : public VRoot
608 {
609 public:
610  virtual void setFirstTranspose(const Permutation& permutation) noexcept = 0;
611  virtual const Permutation& getFirstTranspose() const noexcept = 0;
612  virtual void setReshapeDimensions(Dims dimensions) noexcept = 0;
613  virtual Dims getReshapeDimensions() const noexcept = 0;
614  virtual void setSecondTranspose(const Permutation& permutation) noexcept = 0;
615  virtual const Permutation& getSecondTranspose() const noexcept = 0;
616  virtual void setZeroIsPlaceholder(bool zeroIsPlaceholder) = 0;
617  virtual bool getZeroIsPlaceholder() const = 0;
618 };
619 
620 class VSliceLayer : public VRoot
621 {
622 public:
623  virtual void setStart(Dims start) noexcept = 0;
624  virtual Dims getStart() const noexcept = 0;
625  virtual void setSize(Dims size) noexcept = 0;
626  virtual Dims getSize() const noexcept = 0;
627  virtual void setStride(Dims stride) noexcept = 0;
628  virtual Dims getStride() const noexcept = 0;
629  virtual void setMode(SliceMode mode) noexcept = 0;
630  virtual SliceMode getMode() const noexcept = 0;
631 };
632 
633 class VShapeLayer : public VRoot
634 {
635 public:
636 };
637 
638 class VTopKLayer : public VRoot
639 {
640 public:
641  virtual void setOperation(TopKOperation op) noexcept = 0;
642  virtual TopKOperation getOperation() const noexcept = 0;
643  virtual void setK(int32_t k) noexcept = 0;
644  virtual int32_t getK() const noexcept = 0;
645  virtual void setReduceAxes(uint32_t reduceAxes) noexcept = 0;
646  virtual uint32_t getReduceAxes() const noexcept = 0;
647 };
648 
649 class VMatrixMultiplyLayer : public VRoot
650 {
651 public:
652  virtual void setOperation(int32_t index, MatrixOperation op) noexcept = 0;
653  virtual MatrixOperation getOperation(int32_t index) const noexcept = 0;
654 };
655 
656 class VRaggedSoftMaxLayer : public VRoot
657 {
658 public:
659 };
660 
661 class VIdentityLayer : public VRoot
662 {
663 public:
664 };
665 
666 class VConstantLayer : public VRoot
667 {
668 public:
669  virtual void setWeights(Weights weights) noexcept = 0;
670  virtual Weights getWeights() const noexcept = 0;
671  virtual void setDimensions(Dims dimensions) noexcept = 0;
672  virtual Dims getDimensions() const noexcept = 0;
673 };
674 
675 class VParametricReLULayer : public VRoot
676 {
677 public:
678 };
679 
680 class VResizeLayer : public VRoot
681 {
682 public:
683  virtual void setOutputDimensions(Dims dimensions) noexcept = 0;
684  virtual Dims getOutputDimensions() const noexcept = 0;
685  virtual void setScales(const float* scales, int32_t nbScales) noexcept = 0;
686  virtual int32_t getScales(int32_t size, float* scales) const noexcept = 0;
687  virtual void setResizeMode(ResizeMode resizeMode) noexcept = 0;
688  virtual ResizeMode getResizeMode() const noexcept = 0;
689  virtual void setAlignCorners(bool alignCorners) noexcept = 0;
690  virtual bool getAlignCorners() const noexcept = 0;
691  virtual void setCoordinateTransformation(ResizeCoordinateTransformation coordTransform) noexcept = 0;
692  virtual ResizeCoordinateTransformation getCoordinateTransformation() const noexcept = 0;
693  virtual void setSelectorForSinglePixel(ResizeSelector selector) noexcept = 0;
694  virtual ResizeSelector getSelectorForSinglePixel() const noexcept = 0;
695  virtual void setNearestRounding(ResizeRoundMode value) noexcept = 0;
696  virtual ResizeRoundMode getNearestRounding() const noexcept = 0;
697 };
698 
699 class VLoopBoundaryLayer : public VRoot
700 {
701 public:
702  virtual ILoop* getLoop() const noexcept = 0;
703 };
704 
705 class VRecurrenceLayer : public VRoot
706 {
707 public:
708 };
709 
710 class VLoopOutputLayer : public VRoot
711 {
712 public:
713  virtual LoopOutput getLoopOutput() const noexcept = 0;
714  virtual void setAxis(int32_t axis) noexcept = 0;
715  virtual int32_t getAxis() const noexcept = 0;
716 };
717 
718 class VTripLimitLayer : public VRoot
719 {
720 public:
721  virtual TripLimit getTripLimit() const noexcept = 0;
722 };
723 
724 class VIteratorLayer : public VRoot
725 {
726 public:
727  virtual void setAxis(int32_t axis) noexcept = 0;
728  virtual int32_t getAxis() const noexcept = 0;
729  virtual void setReverse(bool reverse) noexcept = 0;
730  virtual bool getReverse() const noexcept = 0;
731 };
732 class VLoop : public VRoot
733 {
734 public:
735  virtual IRecurrenceLayer* addRecurrence(ITensor& initialValue) noexcept = 0;
736  virtual ITripLimitLayer* addTripLimit(ITensor& tensor, TripLimit limit) noexcept = 0;
737  virtual IIteratorLayer* addIterator(ITensor& tensor, int32_t axis = 0, bool reverse = false) noexcept = 0;
738  virtual ILoopOutputLayer* addLoopOutput(ITensor& tensor, LoopOutput outputKind, int32_t axis = 0) noexcept = 0;
739  virtual void setName(const char* name) noexcept = 0;
740  virtual const char* getName() const noexcept = 0;
741 };
742 class VSelectLayer : public VRoot
743 {
744 };
745 
746 class VFillLayer : public VRoot
747 {
748 public:
749  virtual void setDimensions(Dims dimensions) noexcept = 0;
750  virtual Dims getDimensions() const noexcept = 0;
751  virtual void setOperation(FillOperation op) noexcept = 0;
752  virtual FillOperation getOperation() const noexcept = 0;
753  virtual void setAlpha(double alpha) noexcept = 0;
754  virtual double getAlpha() const noexcept = 0;
755  virtual void setBeta(double beta) noexcept = 0;
756  virtual double getBeta() const noexcept = 0;
757 };
758 
759 class VQuantizeLayer : public VRoot
760 {
761 public:
762  virtual int32_t getAxis() const noexcept = 0;
763  virtual void setAxis(int32_t axis) noexcept = 0;
764 };
765 
766 class VDequantizeLayer : public VRoot
767 {
768 public:
769  virtual int32_t getAxis() const noexcept = 0;
770  virtual void setAxis(int32_t axis) noexcept = 0;
771 };
772 
773 class VNetworkDefinition : public VRoot
774 {
775 public:
776  virtual ITensor* addInput(const char* name, DataType type, Dims dimensions) noexcept = 0;
777  virtual void markOutput(ITensor& tensor) noexcept = 0;
778  virtual IConvolutionLayer* addConvolution(
779  ITensor& input, int32_t nbOutputMaps, DimsHW kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
780  = 0;
781  virtual IFullyConnectedLayer* addFullyConnected(
782  ITensor& input, int32_t nbOutputs, Weights kernelWeights, Weights biasWeights) noexcept
783  = 0;
784  virtual IActivationLayer* addActivation(ITensor& input, ActivationType type) noexcept = 0;
785  virtual IPoolingLayer* addPooling(ITensor& input, PoolingType type, DimsHW windowSize) noexcept = 0;
786  virtual ILRNLayer* addLRN(ITensor& input, int32_t window, float alpha, float beta, float k) noexcept = 0;
787  virtual IScaleLayer* addScale(ITensor& input, ScaleMode mode, Weights shift, Weights scale, Weights power) noexcept
788  = 0;
789  virtual ISoftMaxLayer* addSoftMax(ITensor& input) noexcept = 0;
790  virtual IConcatenationLayer* addConcatenation(ITensor* const* inputs, int32_t nbInputs) noexcept = 0;
791  virtual IDeconvolutionLayer* addDeconvolution(
792  ITensor& input, int32_t nbOutputMaps, DimsHW kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
793  = 0;
794  virtual IElementWiseLayer* addElementWise(ITensor& input1, ITensor& input2, ElementWiseOperation op) noexcept = 0;
795  virtual IUnaryLayer* addUnary(ITensor& input, UnaryOperation operation) noexcept = 0;
796  virtual IPaddingLayer* addPadding(ITensor& input, DimsHW prePadding, DimsHW postPadding) noexcept = 0;
797  virtual IShuffleLayer* addShuffle(ITensor& input) noexcept = 0;
798  virtual int32_t getNbLayers() const noexcept = 0;
799  virtual ILayer* getLayer(int32_t index) const noexcept = 0;
800  virtual int32_t getNbInputs() const noexcept = 0;
801  virtual ITensor* getInput(int32_t index) const noexcept = 0;
802  virtual int32_t getNbOutputs() const noexcept = 0;
803  virtual ITensor* getOutput(int32_t index) const noexcept = 0;
804  virtual IReduceLayer* addReduce(
805  ITensor& input, ReduceOperation operation, uint32_t reduceAxes, bool keepDimensions) noexcept
806  = 0;
807  virtual ITopKLayer* addTopK(ITensor& input, TopKOperation op, int32_t k, uint32_t reduceAxes) noexcept = 0;
808  virtual IGatherLayer* addGather(ITensor& data, ITensor& indices, int32_t axis) noexcept = 0;
809  virtual IRaggedSoftMaxLayer* addRaggedSoftMax(ITensor& input, ITensor& bounds) noexcept = 0;
810  virtual IMatrixMultiplyLayer* addMatrixMultiply(
811  ITensor& input0, MatrixOperation op0, ITensor& input1, MatrixOperation op1) noexcept
812  = 0;
813  virtual IConstantLayer* addConstant(Dims dimensions, Weights weights) noexcept = 0;
814  virtual IRNNv2Layer* addRNNv2(
815  ITensor& input, int32_t layerCount, int32_t hiddenSize, int32_t maxSeqLen, RNNOperation op) noexcept
816  = 0;
817  virtual IIdentityLayer* addIdentity(ITensor& input) noexcept = 0;
818  virtual void removeTensor(ITensor& tensor) noexcept = 0;
819  virtual void unmarkOutput(ITensor& tensor) noexcept = 0;
820  virtual IPluginV2Layer* addPluginV2(ITensor* const* inputs, int32_t nbInputs, IPluginV2& plugin) noexcept = 0;
821  virtual ISliceLayer* addSlice(ITensor& input, Dims start, Dims size, Dims stride) noexcept = 0;
822  virtual void setName(const char* name) noexcept = 0;
823  virtual const char* getName() const noexcept = 0;
824  virtual IShapeLayer* addShape(ITensor& input) noexcept = 0;
825  virtual bool hasImplicitBatchDimension() const noexcept = 0;
826  virtual bool markOutputForShapes(ITensor& tensor) noexcept = 0;
827  virtual bool unmarkOutputForShapes(ITensor& tensor) noexcept = 0;
828  virtual IParametricReLULayer* addParametricReLU(ITensor& input, ITensor& slope) noexcept = 0;
829  virtual IConvolutionLayer* addConvolutionNd(
830  ITensor& input, int32_t nbOutputMaps, Dims kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
831  = 0;
832  virtual IPoolingLayer* addPoolingNd(ITensor& input, PoolingType type, Dims windowSize) noexcept = 0;
833  virtual IDeconvolutionLayer* addDeconvolutionNd(
834  ITensor& input, int32_t nbOutputMaps, Dims kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
835  = 0;
836  virtual IScaleLayer* addScaleNd(
837  ITensor& input, ScaleMode mode, Weights shift, Weights scale, Weights power, int32_t channelAxis) noexcept
838  = 0;
839  virtual IResizeLayer* addResize(ITensor& input) noexcept = 0;
840  virtual bool hasExplicitPrecision() const noexcept = 0;
841  virtual ILoop* addLoop() noexcept = 0;
842  virtual ISelectLayer* addSelect(ITensor& condition, ITensor& thenInput, ITensor& elseInput) noexcept = 0;
843  virtual IFillLayer* addFill(Dims dimensions, FillOperation op) noexcept = 0;
844  virtual IPaddingLayer* addPaddingNd(ITensor& input, Dims prePadding, Dims postPadding) noexcept = 0;
845  virtual bool setWeightsName(Weights weights, const char* name) noexcept = 0;
846  virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
847  virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
848  virtual IDequantizeLayer* addDequantize(ITensor& input, ITensor& scale) noexcept = 0;
849  virtual IQuantizeLayer* addQuantize(ITensor& input, ITensor& scale) noexcept = 0;
850 };
851 
852 class VAlgorithmIOInfo : public VRoot
853 {
854 public:
855  virtual TensorFormat getTensorFormat() const noexcept = 0;
856  virtual DataType getDataType() const noexcept = 0;
857  virtual Dims getStrides() const noexcept = 0;
858 };
859 
860 class VAlgorithmVariant : public VRoot
861 {
862 public:
863  virtual int64_t getImplementation() const noexcept = 0;
864  virtual int64_t getTactic() const noexcept = 0;
865 };
866 
867 class VAlgorithmContext : public VRoot
868 {
869 public:
870  virtual const char* getName() const noexcept = 0;
871  virtual Dims getDimensions(int32_t index, OptProfileSelector select) const noexcept = 0;
872  virtual int32_t getNbInputs() const noexcept = 0;
873  virtual int32_t getNbOutputs() const noexcept = 0;
874 };
875 
876 class VAlgorithm : public VRoot
877 {
878 public:
879  virtual const IAlgorithmIOInfo& getAlgorithmIOInfo(int32_t index) const noexcept = 0;
880  virtual const IAlgorithmVariant& getAlgorithmVariant() const noexcept = 0;
881  virtual float getTimingMSec() const noexcept = 0;
882  virtual std::size_t getWorkspaceSize() const noexcept = 0;
883  virtual const IAlgorithmIOInfo* getAlgorithmIOInfoByIndex(int32_t index) const noexcept = 0;
884 };
885 
886 class VTimingCache : public VRoot
887 {
888 public:
889  virtual nvinfer1::IHostMemory* serialize() const noexcept = 0;
890  virtual bool combine(const ITimingCache& inputCache, bool ignoreMismatch) noexcept = 0;
891  virtual bool reset() noexcept = 0;
892 };
893 
894 class VBuilderConfig : public VRoot
895 {
896 public:
897  virtual void setMinTimingIterations(int32_t minTiming) noexcept = 0;
898  virtual int32_t getMinTimingIterations() const noexcept = 0;
899  virtual void setAvgTimingIterations(int32_t avgTiming) noexcept = 0;
900  virtual int32_t getAvgTimingIterations() const noexcept = 0;
901  virtual void setEngineCapability(EngineCapability capability) noexcept = 0;
902  virtual EngineCapability getEngineCapability() const noexcept = 0;
903  virtual void setInt8Calibrator(IInt8Calibrator* calibrator) noexcept = 0;
904  virtual IInt8Calibrator* getInt8Calibrator() const noexcept = 0;
905  virtual void setMaxWorkspaceSize(std::size_t workspaceSize) noexcept = 0;
906  virtual std::size_t getMaxWorkspaceSize() const noexcept = 0;
907  virtual void setFlags(BuilderFlags builderFlags) noexcept = 0;
908  virtual BuilderFlags getFlags() const noexcept = 0;
909  virtual void clearFlag(BuilderFlag builderFlag) noexcept = 0;
910  virtual void setFlag(BuilderFlag builderFlag) noexcept = 0;
911  virtual bool getFlag(BuilderFlag builderFlag) const noexcept = 0;
912  virtual void setDeviceType(const ILayer* layer, DeviceType deviceType) noexcept = 0;
913  virtual DeviceType getDeviceType(const ILayer* layer) const noexcept = 0;
914  virtual bool isDeviceTypeSet(const ILayer* layer) const noexcept = 0;
915  virtual void resetDeviceType(const ILayer* layer) noexcept = 0;
916  virtual bool canRunOnDLA(const ILayer* layer) const noexcept = 0;
917  virtual void setDLACore(int32_t dlaCore) noexcept = 0;
918  virtual int32_t getDLACore() const noexcept = 0;
919  virtual void setDefaultDeviceType(DeviceType deviceType) noexcept = 0;
920  virtual DeviceType getDefaultDeviceType() const noexcept = 0;
921  virtual void reset() noexcept = 0;
922  virtual void setProfileStream(const cudaStream_t stream) noexcept = 0;
923  virtual cudaStream_t getProfileStream() const noexcept = 0;
924  virtual int32_t addOptimizationProfile(const IOptimizationProfile* profile) noexcept = 0;
925  virtual int32_t getNbOptimizationProfiles() const noexcept = 0;
926  virtual void setProfilingVerbosity(ProfilingVerbosity verbosity) noexcept = 0;
927  virtual ProfilingVerbosity getProfilingVerbosity() const noexcept = 0;
928  virtual void setAlgorithmSelector(IAlgorithmSelector* selector) noexcept = 0;
929  virtual IAlgorithmSelector* getAlgorithmSelector() const noexcept = 0;
930  virtual bool setCalibrationProfile(const IOptimizationProfile* profile) noexcept = 0;
931  virtual const IOptimizationProfile* getCalibrationProfile() noexcept = 0;
932  virtual void setQuantizationFlags(QuantizationFlags flags) noexcept = 0;
933  virtual QuantizationFlags getQuantizationFlags() const noexcept = 0;
934  virtual void clearQuantizationFlag(QuantizationFlag flag) noexcept = 0;
935  virtual void setQuantizationFlag(QuantizationFlag flag) noexcept = 0;
936  virtual bool getQuantizationFlag(QuantizationFlag flag) const noexcept = 0;
937  virtual bool setTacticSources(TacticSources tacticSources) noexcept = 0;
938  virtual TacticSources getTacticSources() const noexcept = 0;
939  virtual nvinfer1::ITimingCache* createTimingCache(const void* blob, std::size_t size) const noexcept = 0;
940  virtual bool setTimingCache(const ITimingCache& cache, bool ignoreMismatch) noexcept = 0;
941  virtual const nvinfer1::ITimingCache* getTimingCache() const noexcept = 0;
942 };
943 
944 class VBuilder : public VRoot
945 {
946 public:
947  virtual void setMaxBatchSize(int32_t batchSize) noexcept = 0;
948  virtual int32_t getMaxBatchSize() const noexcept = 0;
949  virtual bool platformHasFastFp16() const noexcept = 0;
950  virtual bool platformHasFastInt8() const noexcept = 0;
951  virtual int32_t getMaxDLABatchSize() const noexcept = 0;
952  virtual int32_t getNbDLACores() const noexcept = 0;
953  virtual void setGpuAllocator(IGpuAllocator* allocator) noexcept = 0;
954  virtual nvinfer1::IBuilderConfig* createBuilderConfig() noexcept = 0;
955  virtual nvinfer1::ICudaEngine* buildEngineWithConfig(INetworkDefinition& network, IBuilderConfig& config) noexcept
956  = 0;
957  virtual nvinfer1::INetworkDefinition* createNetworkV2(NetworkDefinitionCreationFlags flags) noexcept = 0;
958  virtual nvinfer1::IOptimizationProfile* createOptimizationProfile() noexcept = 0;
959  virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
960  virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
961  virtual void reset() noexcept = 0;
962  virtual bool platformHasTf32() const noexcept = 0;
963  virtual nvinfer1::IHostMemory* buildSerializedNetwork(INetworkDefinition& network, IBuilderConfig& config) noexcept
964  = 0;
965  virtual bool isNetworkSupported(INetworkDefinition const& network, IBuilderConfig const& config) const noexcept
966  = 0;
967 };
968 
969 } // namespace apiv
970 } // namespace nvinfer1
971 
972 #endif // NV_INFER_RUNTIME_IMPL_H
nvinfer1::LoopOutput
LoopOutput
Enum that describes kinds of loop outputs.
Definition: NvInfer.h:4754
nvinfer1::apiv::VAlgorithmIOInfo
Definition: NvInferImpl.h:852
nvinfer1::apiv::VShuffleLayer
Definition: NvInferImpl.h:607
nvinfer1::SliceMode
SliceMode
Controls how ISliceLayer handles out of bounds coordinates.
Definition: NvInfer.h:3878
cudaEvent_t
struct CUevent_st * cudaEvent_t
Forward declaration of cudaEvent_t.
Definition: NvInferRuntimeCommon.h:112
nvinfer1::IFullyConnectedLayer
A fully connected layer in a network definition. This layer expects an input tensor of three or more ...
Definition: NvInfer.h:1439
nvinfer1::Permutation
Definition: NvInfer.h:3690
nvinfer1::IConstantLayer
Layer that represents a constant value.
Definition: NvInfer.h:4305
nvinfer1::IParametricReLULayer
Layer that represents a parametric ReLU operation.
Definition: NvInfer.h:4368
nvinfer1::DimensionOperation
DimensionOperation
An operation on two IDimensionExpr, which represent integer expressions used in dimension computation...
Definition: NvInferRuntime.h:218
nvinfer1::apiv::VLoopBoundaryLayer
Definition: NvInferImpl.h:699
nvinfer1::INetworkDefinition
A network definition for input to the builder.
Definition: NvInfer.h:5450
nvinfer1::Weights
An array of weights used as a layer parameter.
Definition: NvInferRuntime.h:152
nvinfer1::IAlgorithmVariant
provides a unique 128-bit identifier, which along with the input and output information denotes the v...
Definition: NvInfer.h:6845
nvinfer1::IPaddingLayer
Layer that represents a padding operation.
Definition: NvInfer.h:3578
nvinfer1::apiv::VAlgorithmVariant
Definition: NvInferImpl.h:860
nvinfer1::IConvolutionLayer
A convolution layer in a network definition.
Definition: NvInfer.h:987
nvinfer1::IHostMemory
Class to handle library allocated memory that is accessible to the user.
Definition: NvInferRuntime.h:170
nvinfer1::IIdentityLayer
A layer that represents the identity function.
Definition: NvInfer.h:4291
nvinfer1::DeviceType
DeviceType
The device that this layer/network will execute on.
Definition: NvInferRuntime.h:628
nvinfer1::ProfilingVerbosity
ProfilingVerbosity
List of verbosity levels of layer information exposed in NVTX annotations.
Definition: NvInfer.h:7120
nvinfer1::apiv::VFullyConnectedLayer
Definition: NvInferImpl.h:397
nvinfer1::IAlgorithmSelector
Interface implemented by application for selecting and reporting algorithms of a layer provided by th...
Definition: NvInfer.h:6998
nvinfer1::RNNInputMode
RNNInputMode
Enumerates the RNN input modes that may occur with an RNN layer.
Definition: NvInfer.h:3038
nvinfer1::ITripLimitLayer
Definition: NvInfer.h:4907
nvinfer1::IPluginV2
Plugin class for user-implemented layers.
Definition: NvInferRuntimeCommon.h:414
nvinfer1::IAlgorithmIOInfo
Carries information about input or output of the algorithm. IAlgorithmIOInfo for all the input and ou...
Definition: NvInfer.h:6802
nvinfer1::Dims32
Definition: NvInferRuntimeCommon.h:194
nvinfer1::apiv::VScaleLayer
Definition: NvInferImpl.h:461
nvinfer1::TensorFormats
uint32_t TensorFormats
It is capable of representing one or more TensorFormat by binary OR operations, e....
Definition: NvInfer.h:141
nvinfer1::apiv::VFillLayer
Definition: NvInferImpl.h:746
nvinfer1::apiv::VGatherLayer
Definition: NvInferImpl.h:530
nvinfer1::IRNNv2Layer
An RNN layer in a network definition, version 2.
Definition: NvInfer.h:3086
nvinfer1::DimsHW
Descriptor for two-dimensional spatial data.
Definition: NvInferLegacyDims.h:110
nvinfer1::apiv::VTripLimitLayer
Definition: NvInferImpl.h:718
nvinfer1::ResizeMode
ResizeMode
Enumerates various modes of resize in the resize layer. Resize mode set using setResizeMode().
Definition: NvInfer.h:4380
nvinfer1::TensorFormat
TensorFormat
Format of the input/output tensors.
Definition: NvInferRuntimeCommon.h:225
nvinfer1::ILoopOutputLayer
Definition: NvInfer.h:4849
nvinfer1::apiv::VConcatenationLayer
Definition: NvInferImpl.h:483
nvinfer1::apiv::VCudaEngine
Definition: NvInferImpl.h:246
nvinfer1::apiv::VLoop
Definition: NvInferImpl.h:732
nvinfer1::ICudaEngine
An engine for executing inference on a built network, with functionally unsafe features.
Definition: NvInferRuntime.h:1258
nvinfer1::QuantizationFlags
uint32_t QuantizationFlags
Represents one or more QuantizationFlag values using binary OR operations.
Definition: NvInfer.h:7040
nvinfer1::ActivationType
ActivationType
Enumerates the types of activation to perform in an activation layer.
Definition: NvInfer.h:148
nvinfer1::IRaggedSoftMaxLayer
A RaggedSoftmax layer in a network definition.
Definition: NvInfer.h:4273
nvinfer1::apiv::VExprBuilder
Definition: NvInferImpl.h:190
nvinfer1::apiv::VIdentityLayer
Definition: NvInferImpl.h:661
nvinfer1::ElementWiseOperation
ElementWiseOperation
Enumerates the binary operations that may be performed by an ElementWise layer.
Definition: NvInfer.h:2774
nvinfer1::apiv::VReduceLayer
Definition: NvInferImpl.h:583
nvinfer1::ISliceLayer
Slices an input tensor into an output tensor based on the offset and strides.
Definition: NvInfer.h:3921
nvinfer1::TacticSources
int32_t uint32_t TacticSources
Represents a collection of one or more TacticSource values combine using bitwise-OR operations.
Definition: NvInferImpl.h:152
nvinfer1::apiv::VAlgorithm
Definition: NvInferImpl.h:876
nvinfer1::apiv::VSoftMaxLayer
Definition: NvInferImpl.h:476
nvinfer1::ScaleMode
ScaleMode
Controls how shift, scale and power are applied in a Scale layer.
Definition: NvInfer.h:2088
nvinfer1::apiv::VSliceLayer
Definition: NvInferImpl.h:620
nvinfer1::ReduceOperation
ReduceOperation
Enumerates the reduce operations that may be performed by a Reduce layer.
Definition: NvInfer.h:3477
nvinfer1::IIteratorLayer
Definition: NvInfer.h:4920
nvinfer1::apiv::VQuantizeLayer
Definition: NvInferImpl.h:759
nvinfer1::BuilderFlag
BuilderFlag
List of valid modes that the builder can enable when creating an engine from a network definition.
Definition: NvInfer.h:7079
nvinfer1
The TensorRT API version 1 namespace.
nvinfer1::ILayer
Base class for all layer classes in a network definition.
Definition: NvInfer.h:511
nvinfer1::apiv::VLayer
Definition: NvInferImpl.h:341
nvinfer1::ILoop
Definition: NvInfer.h:4961
nvinfer1::IPluginV2Layer
Layer type for pluginV2.
Definition: NvInfer.h:3367
nvinfer1::apiv::VConstantLayer
Definition: NvInferImpl.h:666
nvinfer1::IElementWiseLayer
A elementwise layer in a network definition.
Definition: NvInfer.h:2821
nvinfer1::apiv::VRaggedSoftMaxLayer
Definition: NvInferImpl.h:656
nvinfer1::RNNGateType
RNNGateType
Identifies an individual gate within an RNN cell.
Definition: NvInfer.h:3058
nvinfer1::apiv::VTensor
Definition: NvInferImpl.h:316
nvinfer1::UnaryOperation
UnaryOperation
Enumerates the unary operations that may be performed by a Unary layer.
Definition: NvInfer.h:3392
nvinfer1::ITimingCache
Class to handle tactic timing info collected from builder.
Definition: NvInfer.h:7164
nvinfer1::apiv::VDequantizeLayer
Definition: NvInferImpl.h:766
nvinfer1::IShapeLayer
Layer type for getting shape of a tensor.
Definition: NvInfer.h:4068
nvinfer1::apiv::VRuntime
Definition: NvInferImpl.h:199
nvinfer1::apiv::VTopKLayer
Definition: NvInferImpl.h:638
nvinfer1::NetworkDefinitionCreationFlag
NetworkDefinitionCreationFlag
List of immutable network properties expressed at network creation time. NetworkDefinitionCreationFla...
Definition: NvInfer.h:7838
nvinfer1::PaddingMode
PaddingMode
Enumerates the modes of padding to perform in convolution, deconvolution and pooling layer,...
Definition: NvInfer.h:955
nvinfer1::MatrixOperation
MatrixOperation
Enumerates the operations that may be performed on a tensor by IMatrixMultiplyLayer before multiplica...
Definition: NvInfer.h:4176
nvinfer1::IOptimizationProfile
Optimization profile for dynamic input dimensions and shape tensors.
Definition: NvInferRuntime.h:1067
nvinfer1::PoolingType
PoolingType
The type of pooling to perform in a pooling layer.
Definition: NvInfer.h:1622
nvinfer1::ResizeSelector
ResizeSelector
The coordinate selector when resize to single pixel output.
Definition: NvInfer.h:4453
nvinfer1::apiv::VActivationLayer
Definition: NvInferImpl.h:408
nvinfer1::apiv::VOptimizationProfile
Definition: NvInferImpl.h:231
nvinfer1::DataType
DataType
The type of weights and tensors.
Definition: NvInferRuntimeCommon.h:155
nvinfer1::IConcatenationLayer
A concatenation layer in a network definition.
Definition: NvInfer.h:2328
nvinfer1::IShuffleLayer
Layer type for shuffling data.
Definition: NvInfer.h:3713
nvinfer1::apiv::VConvolutionLayer
Definition: NvInferImpl.h:362
nvinfer1::RNNOperation
RNNOperation
Enumerates the RNN operations that may be performed by an RNN layer.
Definition: NvInfer.h:2988
nvinfer1::ITensor
A tensor in a network definition.
Definition: NvInfer.h:187
nvinfer1::IRecurrenceLayer
Definition: NvInfer.h:4804
nvinfer1::apiv::VPaddingLayer
Definition: NvInferImpl.h:594
nvinfer1::apiv::VUnaryLayer
Definition: NvInferImpl.h:576
nvinfer1::LayerType
LayerType
The type values of layer classes.
Definition: NvInfer.h:89
nvinfer1::apiv::VHostMemory
Definition: NvInferImpl.h:175
nvinfer1::apiv::VBuilderConfig
Definition: NvInferImpl.h:894
nvinfer1::ILRNLayer
A LRN layer in a network definition.
Definition: NvInfer.h:1988
nvinfer1::RNNDirection
RNNDirection
Enumerates the RNN direction that may be performed by an RNN layer.
Definition: NvInfer.h:3010
nvinfer1::TripLimit
TripLimit
Enum that describes kinds of trip limits.
Definition: NvInfer.h:4774
nvinfer1::IReduceLayer
Layer that represents a reduction operator across Shape, Int32, Float, and Half tensors.
Definition: NvInfer.h:3500
nvinfer1::IMatrixMultiplyLayer
Layer that represents a Matrix Multiplication.
Definition: NvInfer.h:4230
nvinfer1::apiv::VPoolingLayer
Definition: NvInferImpl.h:419
nvinfer1::FillOperation
FillOperation
Enumerates the tensor fill operations that may performed by a fill layer.
Definition: NvInfer.h:5066
nvinfer1::apiv::VMatrixMultiplyLayer
Definition: NvInferImpl.h:649
nvinfer1::IFillLayer
Generate an output tensor with specified mode.
Definition: NvInfer.h:5104
NvInferRuntimeCommon.h
nvinfer1::QuantizationFlag
QuantizationFlag
List of valid flags for quantizing the network to int8.
Definition: NvInfer.h:7049
nvinfer1::IInt8Calibrator
Application-implemented interface for calibration.
Definition: NvInfer.h:6620
nvinfer1::apiv::VPluginLayer
Definition: NvInferImpl.h:564
nvinfer1::IErrorRecorder
Reference counted application-implemented error reporting interface for TensorRT objects.
Definition: NvInferRuntimeCommon.h:1373
nvinfer1::apiv::VDeconvolutionLayer
Definition: NvInferImpl.h:490
nvinfer1::ITopKLayer
Layer that represents a TopK reduction.
Definition: NvInfer.h:4100
nvinfer1::ResizeRoundMode
ResizeRoundMode
The rounding mode for nearest neighbor resize.
Definition: NvInfer.h:4480
nvinfer1::OptProfileSelector
OptProfileSelector
When setting or querying optimization profile parameters (such as shape tensor inputs or dynamic dime...
Definition: NvInferRuntime.h:1031
nvinfer1::WeightsRole
WeightsRole
How a layer uses particular Weights.
Definition: NvInferRuntime.h:606
nvinfer1::ISelectLayer
Definition: NvInfer.h:5052
NvInferLegacyDims.h
nvinfer1::apiv::VSelectLayer
Definition: NvInferImpl.h:742
nvinfer1::NetworkDefinitionCreationFlags
uint32_t NetworkDefinitionCreationFlags
Represents one or more NetworkDefinitionCreationFlag flags using binary OR operations....
Definition: NvInfer.h:7827
nvinfer1::IGatherLayer
Definition: NvInfer.h:2858
nvinfer1::apiv::VElementWiseLayer
Definition: NvInferImpl.h:523
nvinfer1::IDimensionExpr
Definition: NvInferRuntime.h:270
nvinfer1::apiv::VRNNv2Layer
Definition: NvInferImpl.h:539
nvinfer1::BuilderFlags
uint32_t BuilderFlags
Represents one or more QuantizationFlag values using binary OR operations, e.g., 1U << BuilderFlag::k...
Definition: NvInfer.h:7070
nvinfer1::EngineCapability
EngineCapability
List of supported engine capability flows.
Definition: NvInferRuntime.h:104
nvinfer1::apiv::VRecurrenceLayer
Definition: NvInferImpl.h:705
nvinfer1::ResizeCoordinateTransformation
ResizeCoordinateTransformation
The resize coordinate transformation function.
Definition: NvInfer.h:4403
nvinfer1::apiv::VLRNLayer
Definition: NvInferImpl.h:448
nvinfer1::IProfiler
Application-implemented interface for profiling.
Definition: NvInferRuntime.h:586
nvinfer1::apiv::VTimingCache
Definition: NvInferImpl.h:886
nvinfer1::apiv::VPluginV2Layer
Definition: NvInferImpl.h:570
cudaStream_t
struct CUstream_st * cudaStream_t
Forward declaration of cudaStream_t.
Definition: NvInferRuntimeCommon.h:109
nvinfer1::IScaleLayer
A Scale layer in a network definition.
Definition: NvInfer.h:2128
nvinfer1::ISoftMaxLayer
A Softmax layer in a network definition.
Definition: NvInfer.h:2263
nvinfer1::IBuilderConfig
Holds properties for configuring a builder to produce an engine.
Definition: NvInfer.h:7226
nvinfer1::apiv::VDimensionExpr
Definition: NvInferImpl.h:183
nvinfer1::apiv::VParametricReLULayer
Definition: NvInferImpl.h:675
nvinfer1::apiv::VRoot
Definition: NvInferImpl.h:169
nvinfer1::apiv::VExecutionContext
Definition: NvInferImpl.h:285
nvinfer1::IResizeLayer
A resize layer in a network definition.
Definition: NvInfer.h:4526
nvinfer1::apiv::VShapeLayer
Definition: NvInferImpl.h:633
nvinfer1::apiv::VResizeLayer
Definition: NvInferImpl.h:680
nvinfer1::IPoolingLayer
A Pooling layer in a network definition.
Definition: NvInfer.h:1650
nvinfer1::apiv::VNetworkDefinition
Definition: NvInferImpl.h:773
nvinfer1::apiv::VAlgorithmContext
Definition: NvInferImpl.h:867
nvinfer1::IGpuAllocator
Application-implemented class for controlling allocation on the GPU.
Definition: NvInferRuntimeCommon.h:1103
nvinfer1::IUnaryLayer
Layer that represents an unary operation.
Definition: NvInfer.h:3431
nvinfer1::apiv::VBuilder
Definition: NvInferImpl.h:944
nvinfer1::apiv::VLoopOutputLayer
Definition: NvInferImpl.h:710
nvinfer1::IQuantizeLayer
A Quantize layer in a network definition.
Definition: NvInfer.h:5312
nvinfer1::apiv::VRefitter
Definition: NvInferImpl.h:213
nvinfer1::TopKOperation
TopKOperation
Enumerates the operations that may be performed by a TopK layer.
Definition: NvInfer.h:4080
nvinfer1::TensorLocation
TensorLocation
The location for tensor data storage, device or host.
Definition: NvInferRuntime.h:242
nvinfer1::IExecutionContext
Context for executing inference using an engine, with functionally unsafe features.
Definition: NvInferRuntime.h:1772
nvinfer1::IActivationLayer
An Activation layer in a network definition.
Definition: NvInfer.h:1540
nvinfer1::CalibrationAlgoType
CalibrationAlgoType
Version of calibration algorithm to use.
Definition: NvInfer.h:6594
nvinfer1::apiv::VIteratorLayer
Definition: NvInferImpl.h:724
nvinfer1::IDequantizeLayer
A Dequantize layer in a network definition.
Definition: NvInfer.h:5399
nvinfer1::IDeconvolutionLayer
A deconvolution layer in a network definition.
Definition: NvInfer.h:2368

  Copyright © 2024 NVIDIA Corporation
  Privacy Policy | Manage My Privacy | Do Not Sell or Share My Data | Terms of Service | Accessibility | Corporate Policies | Product Security | Contact