TensorRT  8.2.0
NvInferImpl.h
1 /*
2  * Copyright 1993-2021 NVIDIA Corporation. All rights reserved.
3  *
4  * NOTICE TO LICENSEE:
5  *
6  * This source code and/or documentation ("Licensed Deliverables") are
7  * subject to NVIDIA intellectual property rights under U.S. and
8  * international Copyright laws.
9  *
10  * These Licensed Deliverables contained herein is PROPRIETARY and
11  * CONFIDENTIAL to NVIDIA and is being provided under the terms and
12  * conditions of a form of NVIDIA software license agreement by and
13  * between NVIDIA and Licensee ("License Agreement") or electronically
14  * accepted by Licensee. Notwithstanding any terms or conditions to
15  * the contrary in the License Agreement, reproduction or disclosure
16  * of the Licensed Deliverables to any third party without the express
17  * written consent of NVIDIA is prohibited.
18  *
19  * NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
20  * LICENSE AGREEMENT, NVIDIA MAKES NO REPRESENTATION ABOUT THE
21  * SUITABILITY OF THESE LICENSED DELIVERABLES FOR ANY PURPOSE. IT IS
22  * PROVIDED "AS IS" WITHOUT EXPRESS OR IMPLIED WARRANTY OF ANY KIND.
23  * NVIDIA DISCLAIMS ALL WARRANTIES WITH REGARD TO THESE LICENSED
24  * DELIVERABLES, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY,
25  * NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE.
26  * NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
27  * LICENSE AGREEMENT, IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY
28  * SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, OR ANY
29  * DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
30  * WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
31  * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
32  * OF THESE LICENSED DELIVERABLES.
33  *
34  * U.S. Government End Users. These Licensed Deliverables are a
35  * "commercial item" as that term is defined at 48 C.F.R. 2.101 (OCT
36  * 1995), consisting of "commercial computer software" and "commercial
37  * computer software documentation" as such terms are used in 48
38  * C.F.R. 12.212 (SEPT 1995) and is provided to the U.S. Government
39  * only as a commercial end item. Consistent with 48 C.F.R.12.212 and
40  * 48 C.F.R. 227.7202-1 through 227.7202-4 (JUNE 1995), all
41  * U.S. Government End Users acquire the Licensed Deliverables with
42  * only those rights set forth herein.
43  *
44  * Any use of the Licensed Deliverables in individual and commercial
45  * software must include, in the user documentation and internal
46  * comments to the code, the above Disclaimer and U.S. Government End
47  * Users Notice.
48  */
49 
50 #ifndef NV_INFER_IMPL_H
51 #define NV_INFER_IMPL_H
52 
53 #include "NvInferLegacyDims.h"
54 #include "NvInferRuntimeCommon.h"
55 
56 // @cond SuppressDoxyWarnings
57 
58 namespace nvinfer1
59 {
60 
61 class IActivationLayer;
62 class IAlgorithm;
63 class IAlgorithmContext;
64 class IAlgorithmIOInfo;
65 class IAlgorithmSelector;
66 class IAlgorithmVariant;
67 class IAssertionLayer;
68 class IBuilderConfig;
69 class IConcatenationLayer;
70 class IIfConditional;
71 class IConditionLayer;
72 class IIfConditionalOutputLayer;
73 class IIfConditionalInputLayer;
74 class IConstantLayer;
75 class IConvolutionLayer;
76 class ICudaEngine;
77 class IDeconvolutionLayer;
78 class IDequantizeLayer;
79 class IDimensionExpr;
80 class IEinsumLayer;
81 class IElementWiseLayer;
82 class IExecutionContext;
83 class IFillLayer;
84 class IFullyConnectedLayer;
85 class IGatherLayer;
86 class IHostMemory;
87 class IIdentityLayer;
88 class IIfConditional;
89 class IInt8Calibrator;
90 class IIteratorLayer;
91 class ILayer;
92 class ILoop;
93 class ILoopOutputLayer;
94 class ILRNLayer;
95 class IMatrixMultiplyLayer;
96 class INetworkDefinition;
97 class IOptimizationProfile;
98 class IPaddingLayer;
99 class IParametricReLULayer;
100 class IPlugin;
101 class IPluginExt;
102 class IPluginFactory;
103 class IPluginLayer;
104 class IPluginV2Layer;
105 class IPoolingLayer;
106 class IProfiler;
107 class IQuantizeLayer;
108 class IRaggedSoftMaxLayer;
109 class IRecurrenceLayer;
110 class IReduceLayer;
111 class IResizeLayer;
112 class IRNNv2Layer;
113 class IScaleLayer;
114 class IScatterLayer;
115 class ISelectLayer;
116 class IShapeLayer;
117 class IShuffleLayer;
118 class ISliceLayer;
119 class ISoftMaxLayer;
120 class IEngineInspector;
121 class ITensor;
122 class ITimingCache;
123 class ITopKLayer;
124 class ITripLimitLayer;
125 class IUnaryLayer;
126 struct Permutation;
127 class Weights;
128 
129 enum class ActivationType : int32_t;
130 enum class BuilderFlag : int32_t;
131 enum class CalibrationAlgoType : int32_t;
132 enum class DeviceType : int32_t;
133 enum class DimensionOperation : int32_t;
134 enum class ElementWiseOperation : int32_t;
135 enum class EngineCapability : int32_t;
136 enum class FillOperation : int32_t;
137 enum class GatherMode : int32_t;
138 enum class LayerInformationFormat : int32_t;
139 enum class LayerType : int32_t;
140 enum class LoopOutput : int32_t;
141 enum class MatrixOperation : int32_t;
142 enum class NetworkDefinitionCreationFlag : int32_t;
143 enum class OptProfileSelector : int32_t;
144 enum class PaddingMode : int32_t;
145 enum class PoolingType : int32_t;
146 enum class ProfilingVerbosity : int32_t;
147 enum class QuantizationFlag : int32_t;
148 enum class ReduceOperation : int32_t;
149 enum class ResizeCoordinateTransformation : int32_t;
150 enum class ResizeMode : int32_t;
151 enum class ResizeRoundMode : int32_t;
152 enum class ResizeSelector : int32_t;
153 enum class RNNDirection : int32_t;
154 enum class RNNGateType : int32_t;
155 enum class RNNInputMode : int32_t;
156 enum class RNNOperation : int32_t;
157 enum class ScaleMode : int32_t;
158 enum class ScatterMode : int32_t;
159 enum class SliceMode : int32_t;
160 enum class TensorLocation : int32_t;
161 enum class TopKOperation : int32_t;
162 enum class TripLimit : int32_t;
163 enum class UnaryOperation : int32_t;
164 enum class WeightsRole : int32_t;
165 
166 using TacticSources = uint32_t;
167 using TensorFormats = uint32_t;
168 using BuilderFlags = uint32_t;
169 using NetworkDefinitionCreationFlags = uint32_t;
170 using QuantizationFlags = uint32_t;
171 
179 
180 namespace apiv
181 {
182 
183 class VRoot
184 {
185 public:
186  virtual ~VRoot() noexcept = default;
187 };
188 
189 class VHostMemory : public VRoot
190 {
191 public:
192  virtual void* data() const noexcept = 0;
193  virtual std::size_t size() const noexcept = 0;
194  virtual DataType type() const noexcept = 0;
195 };
196 
197 class VDimensionExpr : public VRoot
198 {
199 public:
200  virtual bool isConstant() const = 0;
201  virtual int32_t getConstantValue() const = 0;
202 };
203 
204 class VExprBuilder : public VRoot
205 {
206 public:
207  virtual const IDimensionExpr* constant(int32_t value) = 0;
208  virtual const IDimensionExpr* operation(
209  DimensionOperation op, const IDimensionExpr& first, const IDimensionExpr& second)
210  = 0;
211 };
212 
213 class VRuntime : public VRoot
214 {
215 public:
216  virtual nvinfer1::ICudaEngine* deserializeCudaEngine(
217  const void* blob, std::size_t size, IPluginFactory* pluginFactory) noexcept
218  = 0;
219  virtual void setDLACore(int32_t dlaCore) noexcept = 0;
220  virtual int32_t getDLACore() const noexcept = 0;
221  virtual int32_t getNbDLACores() const noexcept = 0;
222  virtual void setGpuAllocator(IGpuAllocator* allocator) noexcept = 0;
223  virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
224  virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
225  virtual ILogger* getLogger() const noexcept = 0;
226 };
227 
228 class VRefitter : public VRoot
229 {
230 public:
231  virtual bool setWeights(const char* layerName, WeightsRole role, const Weights weights) noexcept = 0;
232  virtual bool refitCudaEngine() noexcept = 0;
233  virtual int32_t getMissing(int32_t size, const char** layerNames, WeightsRole* roles) noexcept = 0;
234  virtual int32_t getAll(int32_t size, const char** layerNames, WeightsRole* roles) noexcept = 0;
235  virtual bool setDynamicRange(const char* tensorName, float min, float max) noexcept = 0;
236  virtual float getDynamicRangeMin(const char* tensorName) const noexcept = 0;
237  virtual float getDynamicRangeMax(const char* tensorName) const noexcept = 0;
238  virtual int32_t getTensorsWithDynamicRange(int32_t size, const char** tensorNames) const noexcept = 0;
239  virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
240  virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
241  virtual bool setNamedWeights(const char* name, Weights weights) noexcept = 0;
242  virtual int32_t getMissingWeights(int32_t size, const char** weightsNames) noexcept = 0;
243  virtual int32_t getAllWeights(int32_t size, const char** weightsNames) noexcept = 0;
244  virtual ILogger* getLogger() const noexcept = 0;
245 };
246 
247 class VOptimizationProfile : public VRoot
248 {
249 public:
250  virtual bool setDimensions(const char* inputName, OptProfileSelector select, Dims dims) noexcept = 0;
251  virtual Dims getDimensions(const char* inputName, OptProfileSelector select) const noexcept = 0;
252  virtual bool setShapeValues(
253  const char* inputName, OptProfileSelector select, const int32_t* values, int32_t nbValues) noexcept
254  = 0;
255  virtual int32_t getNbShapeValues(const char* inputName) const noexcept = 0;
256  virtual int32_t const* getShapeValues(const char* inputName, OptProfileSelector select) const noexcept = 0;
257  virtual bool setExtraMemoryTarget(float target) noexcept = 0;
258  virtual float getExtraMemoryTarget() const noexcept = 0;
259  virtual bool isValid() const noexcept = 0;
260 };
261 
262 class VCudaEngine : public VRoot
263 {
264 public:
265  virtual int32_t getNbBindings() const noexcept = 0;
266  virtual int32_t getBindingIndex(const char* name) const noexcept = 0;
267  virtual const char* getBindingName(int32_t bindingIndex) const noexcept = 0;
268  virtual bool bindingIsInput(int32_t bindingIndex) const noexcept = 0;
269  virtual Dims getBindingDimensions(int32_t bindingIndex) const noexcept = 0;
270  virtual DataType getBindingDataType(int32_t bindingIndex) const noexcept = 0;
271  virtual int32_t getMaxBatchSize() const noexcept = 0;
272  virtual int32_t getNbLayers() const noexcept = 0;
273  virtual IHostMemory* serialize() const noexcept = 0;
274  virtual IExecutionContext* createExecutionContext() noexcept = 0;
275  virtual TensorLocation getLocation(int32_t bindingIndex) const noexcept = 0;
276  virtual IExecutionContext* createExecutionContextWithoutDeviceMemory() noexcept = 0;
277  virtual size_t getDeviceMemorySize() const noexcept = 0;
278  virtual bool isRefittable() const noexcept = 0;
279  virtual int32_t getBindingBytesPerComponent(int32_t bindingIndex) const noexcept = 0;
280  virtual int32_t getBindingComponentsPerElement(int32_t bindingIndex) const noexcept = 0;
281  virtual TensorFormat getBindingFormat(int32_t bindingIndex) const noexcept = 0;
282  virtual const char* getBindingFormatDesc(int32_t bindingIndex) const noexcept = 0;
283  virtual int32_t getBindingVectorizedDim(int32_t bindingIndex) const noexcept = 0;
284  virtual const char* getName() const noexcept = 0;
285  virtual int32_t getNbOptimizationProfiles() const noexcept = 0;
286  virtual Dims getProfileDimensions(int32_t bindingIndex, int32_t profileIndex, OptProfileSelector select) const
287  noexcept
288  = 0;
289  virtual const int32_t* getProfileShapeValues(
290  int32_t profileIndex, int32_t inputIndex, OptProfileSelector select) const noexcept
291  = 0;
292  virtual bool isShapeBinding(int32_t bindingIndex) const noexcept = 0;
293  virtual bool isExecutionBinding(int32_t bindingIndex) const noexcept = 0;
294  virtual EngineCapability getEngineCapability() const noexcept = 0;
295  virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
296  virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
297  virtual bool hasImplicitBatchDimension() const noexcept = 0;
298  virtual TacticSources getTacticSources() const noexcept = 0;
299  virtual ProfilingVerbosity getProfilingVerbosity() const noexcept = 0;
300  virtual IEngineInspector* createEngineInspector() const noexcept = 0;
301 };
302 
303 class VExecutionContext : public VRoot
304 {
305 public:
306  virtual bool execute(int32_t batchSize, void* const* bindings) noexcept = 0;
307  virtual bool enqueue(
308  int32_t batchSize, void* const* bindings, cudaStream_t stream, cudaEvent_t* inputConsumed) noexcept
309  = 0;
310  virtual void setDebugSync(bool sync) noexcept = 0;
311  virtual bool getDebugSync() const noexcept = 0;
312  virtual void setProfiler(IProfiler*) noexcept = 0;
313  virtual IProfiler* getProfiler() const noexcept = 0;
314  virtual const ICudaEngine& getEngine() const noexcept = 0;
315  virtual void setName(const char* name) noexcept = 0;
316  virtual const char* getName() const noexcept = 0;
317  virtual void setDeviceMemory(void* memory) noexcept = 0;
318  virtual Dims getStrides(int32_t bindingIndex) const noexcept = 0;
319  virtual bool setOptimizationProfile(int32_t profileIndex) noexcept = 0;
320  virtual int32_t getOptimizationProfile() const noexcept = 0;
321  virtual bool setBindingDimensions(int32_t bindingIndex, Dims dimensions) noexcept = 0;
322  virtual Dims getBindingDimensions(int32_t bindingIndex) const noexcept = 0;
323  virtual bool setInputShapeBinding(int32_t bindingIndex, int32_t const* data) noexcept = 0;
324  virtual bool getShapeBinding(int32_t bindingIndex, int32_t* data) const noexcept = 0;
325  virtual bool allInputDimensionsSpecified() const noexcept = 0;
326  virtual bool allInputShapesSpecified() const noexcept = 0;
327  virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
328  virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
329  virtual bool executeV2(void* const* bindings) noexcept = 0;
330  virtual bool enqueueV2(void* const* bindings, cudaStream_t stream, cudaEvent_t* inputConsumed) noexcept = 0;
331  virtual bool setOptimizationProfileAsync(int32_t profileIndex, cudaStream_t stream) noexcept = 0;
332  virtual void setEnqueueEmitsProfile(bool enqueueEmitsProfile) noexcept = 0;
333  virtual bool getEnqueueEmitsProfile() const noexcept = 0;
334  virtual bool reportToProfiler() const noexcept = 0;
335 };
336 
337 class VEngineInspector : public VRoot
338 {
339 public:
340  virtual bool setExecutionContext(IExecutionContext const* context) noexcept = 0;
341  virtual IExecutionContext const* getExecutionContext() const noexcept = 0;
342  virtual AsciiChar const* getLayerInformation(int32_t layerIndex, LayerInformationFormat format) noexcept = 0;
343  virtual AsciiChar const* getEngineInformation(LayerInformationFormat format) noexcept = 0;
344  virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
345  virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
346 };
347 
348 class VTensor : public VRoot
349 {
350 public:
351  virtual void setName(const char* name) noexcept = 0;
352  virtual const char* getName() const noexcept = 0;
353  virtual void setDimensions(Dims dimensions) noexcept = 0;
354  virtual Dims getDimensions() const noexcept = 0;
355  virtual void setType(DataType type) noexcept = 0;
356  virtual DataType getType() const noexcept = 0;
357  virtual bool setDynamicRange(float min, float max) noexcept = 0;
358  virtual bool isNetworkInput() const noexcept = 0;
359  virtual bool isNetworkOutput() const noexcept = 0;
360  virtual void setBroadcastAcrossBatch(bool broadcastAcrossBatch) noexcept = 0;
361  virtual bool getBroadcastAcrossBatch() const noexcept = 0;
362  virtual TensorLocation getLocation() const noexcept = 0;
363  virtual void setLocation(TensorLocation location) noexcept = 0;
364  virtual bool dynamicRangeIsSet() const noexcept = 0;
365  virtual void resetDynamicRange() noexcept = 0;
366  virtual float getDynamicRangeMin() const noexcept = 0;
367  virtual float getDynamicRangeMax() const noexcept = 0;
368  virtual void setAllowedFormats(TensorFormats formats) noexcept = 0;
369  virtual TensorFormats getAllowedFormats() const noexcept = 0;
370  virtual bool isShapeTensor() const noexcept = 0;
371  virtual bool isExecutionTensor() const noexcept = 0;
372 };
373 class VLayer : public VRoot
374 {
375 public:
376  virtual LayerType getType() const noexcept = 0;
377  virtual void setName(const char* name) noexcept = 0;
378  virtual const char* getName() const noexcept = 0;
379  virtual int32_t getNbInputs() const noexcept = 0;
380  virtual ITensor* getInput(int32_t index) const noexcept = 0;
381  virtual int32_t getNbOutputs() const noexcept = 0;
382  virtual ITensor* getOutput(int32_t index) const noexcept = 0;
383  virtual void setInput(int32_t index, ITensor& tensor) noexcept = 0;
384  virtual void setPrecision(DataType dataType) noexcept = 0;
385  virtual DataType getPrecision() const noexcept = 0;
386  virtual bool precisionIsSet() const noexcept = 0;
387  virtual void resetPrecision() noexcept = 0;
388  virtual void setOutputType(int32_t index, DataType dataType) noexcept = 0;
389  virtual DataType getOutputType(int32_t index) const noexcept = 0;
390  virtual bool outputTypeIsSet(int32_t index) const noexcept = 0;
391  virtual void resetOutputType(int32_t index) noexcept = 0;
392 };
393 
394 class VConvolutionLayer : public VRoot
395 {
396 public:
397  virtual void setKernelSize(DimsHW kernelSize) noexcept = 0;
398  virtual DimsHW getKernelSize() const noexcept = 0;
399  virtual void setNbOutputMaps(int32_t nbOutputMaps) noexcept = 0;
400  virtual int32_t getNbOutputMaps() const noexcept = 0;
401  virtual void setStride(DimsHW stride) noexcept = 0;
402  virtual DimsHW getStride() const noexcept = 0;
403  virtual void setPadding(DimsHW padding) noexcept = 0;
404  virtual DimsHW getPadding() const noexcept = 0;
405  virtual void setNbGroups(int32_t nbGroups) noexcept = 0;
406  virtual int32_t getNbGroups() const noexcept = 0;
407  virtual void setKernelWeights(Weights weights) noexcept = 0;
408  virtual Weights getKernelWeights() const noexcept = 0;
409  virtual void setBiasWeights(Weights weights) noexcept = 0;
410  virtual Weights getBiasWeights() const noexcept = 0;
411  virtual void setDilation(DimsHW dilation) noexcept = 0;
412  virtual DimsHW getDilation() const noexcept = 0;
413  virtual void setPrePadding(Dims padding) noexcept = 0;
414  virtual Dims getPrePadding() const noexcept = 0;
415  virtual void setPostPadding(Dims padding) noexcept = 0;
416  virtual Dims getPostPadding() const noexcept = 0;
417  virtual void setPaddingMode(PaddingMode paddingMode) noexcept = 0;
418  virtual PaddingMode getPaddingMode() const noexcept = 0;
419  virtual void setKernelSizeNd(Dims kernelSize) noexcept = 0;
420  virtual Dims getKernelSizeNd() const noexcept = 0;
421  virtual void setStrideNd(Dims stride) noexcept = 0;
422  virtual Dims getStrideNd() const noexcept = 0;
423  virtual void setPaddingNd(Dims padding) noexcept = 0;
424  virtual Dims getPaddingNd() const noexcept = 0;
425  virtual void setDilationNd(Dims dilation) noexcept = 0;
426  virtual Dims getDilationNd() const noexcept = 0;
427 };
428 
429 class VFullyConnectedLayer : public VRoot
430 {
431 public:
432  virtual void setNbOutputChannels(int32_t nbOutputs) noexcept = 0;
433  virtual int32_t getNbOutputChannels() const noexcept = 0;
434  virtual void setKernelWeights(Weights weights) noexcept = 0;
435  virtual Weights getKernelWeights() const noexcept = 0;
436  virtual void setBiasWeights(Weights weights) noexcept = 0;
437  virtual Weights getBiasWeights() const noexcept = 0;
438 };
439 
440 class VActivationLayer : public VRoot
441 {
442 public:
443  virtual void setActivationType(ActivationType type) noexcept = 0;
444  virtual ActivationType getActivationType() const noexcept = 0;
445  virtual void setAlpha(float alpha) noexcept = 0;
446  virtual void setBeta(float beta) noexcept = 0;
447  virtual float getAlpha() const noexcept = 0;
448  virtual float getBeta() const noexcept = 0;
449 };
450 
451 class VPoolingLayer : public VRoot
452 {
453 public:
454  virtual void setPoolingType(PoolingType type) noexcept = 0;
455  virtual PoolingType getPoolingType() const noexcept = 0;
456  virtual void setWindowSize(DimsHW windowSize) noexcept = 0;
457  virtual DimsHW getWindowSize() const noexcept = 0;
458  virtual void setStride(DimsHW stride) noexcept = 0;
459  virtual DimsHW getStride() const noexcept = 0;
460  virtual void setPadding(DimsHW padding) noexcept = 0;
461  virtual DimsHW getPadding() const noexcept = 0;
462  virtual void setBlendFactor(float blendFactor) noexcept = 0;
463  virtual float getBlendFactor() const noexcept = 0;
464  virtual void setAverageCountExcludesPadding(bool exclusive) noexcept = 0;
465  virtual bool getAverageCountExcludesPadding() const noexcept = 0;
466  virtual void setPrePadding(Dims padding) noexcept = 0;
467  virtual Dims getPrePadding() const noexcept = 0;
468  virtual void setPostPadding(Dims padding) noexcept = 0;
469  virtual Dims getPostPadding() const noexcept = 0;
470  virtual void setPaddingMode(PaddingMode paddingMode) noexcept = 0;
471  virtual PaddingMode getPaddingMode() const noexcept = 0;
472  virtual void setWindowSizeNd(Dims windowSize) noexcept = 0;
473  virtual Dims getWindowSizeNd() const noexcept = 0;
474  virtual void setStrideNd(Dims stride) noexcept = 0;
475  virtual Dims getStrideNd() const noexcept = 0;
476  virtual void setPaddingNd(Dims padding) noexcept = 0;
477  virtual Dims getPaddingNd() const noexcept = 0;
478 };
479 
480 class VLRNLayer : public VRoot
481 {
482 public:
483  virtual void setWindowSize(int32_t windowSize) noexcept = 0;
484  virtual int32_t getWindowSize() const noexcept = 0;
485  virtual void setAlpha(float alpha) noexcept = 0;
486  virtual float getAlpha() const noexcept = 0;
487  virtual void setBeta(float beta) noexcept = 0;
488  virtual float getBeta() const noexcept = 0;
489  virtual void setK(float k) noexcept = 0;
490  virtual float getK() const noexcept = 0;
491 };
492 
493 class VScaleLayer : public VRoot
494 {
495 public:
496  virtual void setMode(ScaleMode mode) noexcept = 0;
497  virtual ScaleMode getMode() const noexcept = 0;
498  virtual void setShift(Weights shift) noexcept = 0;
499  virtual Weights getShift() const noexcept = 0;
500  virtual void setScale(Weights scale) noexcept = 0;
501  virtual Weights getScale() const noexcept = 0;
502  virtual void setPower(Weights power) noexcept = 0;
503  virtual Weights getPower() const noexcept = 0;
504  virtual int32_t getChannelAxis() const noexcept = 0;
505  virtual void setChannelAxis(int32_t channelAxis) noexcept = 0;
506 };
507 
508 class VSoftMaxLayer : public VRoot
509 {
510 public:
511  virtual void setAxes(uint32_t axes) noexcept = 0;
512  virtual uint32_t getAxes() const noexcept = 0;
513 };
514 
515 class VConcatenationLayer : public VRoot
516 {
517 public:
518  virtual void setAxis(int32_t axis) noexcept = 0;
519  virtual int32_t getAxis() const noexcept = 0;
520 };
521 
522 class VDeconvolutionLayer : public VRoot
523 {
524 public:
525  virtual void setKernelSize(DimsHW kernelSize) noexcept = 0;
526  virtual DimsHW getKernelSize() const noexcept = 0;
527  virtual void setNbOutputMaps(int32_t nbOutputMaps) noexcept = 0;
528  virtual int32_t getNbOutputMaps() const noexcept = 0;
529  virtual void setStride(DimsHW stride) noexcept = 0;
530  virtual DimsHW getStride() const noexcept = 0;
531  virtual void setPadding(DimsHW padding) noexcept = 0;
532  virtual DimsHW getPadding() const noexcept = 0;
533  virtual void setNbGroups(int32_t nbGroups) noexcept = 0;
534  virtual int32_t getNbGroups() const noexcept = 0;
535  virtual void setKernelWeights(Weights weights) noexcept = 0;
536  virtual Weights getKernelWeights() const noexcept = 0;
537  virtual void setBiasWeights(Weights weights) noexcept = 0;
538  virtual Weights getBiasWeights() const noexcept = 0;
539  virtual void setPrePadding(Dims padding) noexcept = 0;
540  virtual Dims getPrePadding() const noexcept = 0;
541  virtual void setPostPadding(Dims padding) noexcept = 0;
542  virtual Dims getPostPadding() const noexcept = 0;
543  virtual void setPaddingMode(PaddingMode paddingMode) noexcept = 0;
544  virtual PaddingMode getPaddingMode() const noexcept = 0;
545  virtual void setKernelSizeNd(Dims kernelSize) noexcept = 0;
546  virtual Dims getKernelSizeNd() const noexcept = 0;
547  virtual void setStrideNd(Dims stride) noexcept = 0;
548  virtual Dims getStrideNd() const noexcept = 0;
549  virtual void setPaddingNd(Dims padding) noexcept = 0;
550  virtual Dims getPaddingNd() const noexcept = 0;
551  virtual void setDilationNd(Dims dilation) noexcept = 0;
552  virtual Dims getDilationNd() const noexcept = 0;
553 };
554 
555 class VElementWiseLayer : public VRoot
556 {
557 public:
558  virtual void setOperation(ElementWiseOperation op) noexcept = 0;
559  virtual ElementWiseOperation getOperation() const noexcept = 0;
560 };
561 
562 class VGatherLayer : public VRoot
563 {
564 public:
565  virtual void setGatherAxis(int32_t axis) noexcept = 0;
566  virtual int32_t getGatherAxis() const noexcept = 0;
567  virtual void setNbElementWiseDims(int32_t k) noexcept = 0;
568  virtual int32_t getNbElementWiseDims() const noexcept = 0;
569  virtual void setMode(GatherMode mode) noexcept = 0;
570  virtual GatherMode getMode() const noexcept = 0;
571 };
572 
573 class VRNNv2Layer : public VRoot
574 {
575 public:
576  virtual int32_t getLayerCount() const noexcept = 0;
577  virtual int32_t getHiddenSize() const noexcept = 0;
578  virtual int32_t getMaxSeqLength() const noexcept = 0;
579  virtual int32_t getDataLength() const noexcept = 0;
580  virtual void setSequenceLengths(ITensor& seqLengths) noexcept = 0;
581  virtual ITensor* getSequenceLengths() const noexcept = 0;
582  virtual void setOperation(RNNOperation op) noexcept = 0;
583  virtual RNNOperation getOperation() const noexcept = 0;
584  virtual void setInputMode(RNNInputMode op) noexcept = 0;
585  virtual RNNInputMode getInputMode() const noexcept = 0;
586  virtual void setDirection(RNNDirection op) noexcept = 0;
587  virtual RNNDirection getDirection() const noexcept = 0;
588  virtual void setWeightsForGate(int32_t layerIndex, RNNGateType gate, bool isW, Weights weights) noexcept = 0;
589  virtual Weights getWeightsForGate(int32_t layerIndex, RNNGateType gate, bool isW) const noexcept = 0;
590  virtual void setBiasForGate(int32_t layerIndex, RNNGateType gate, bool isW, Weights bias) noexcept = 0;
591  virtual Weights getBiasForGate(int32_t layerIndex, RNNGateType gate, bool isW) const noexcept = 0;
592  virtual void setHiddenState(ITensor& hidden) noexcept = 0;
593  virtual ITensor* getHiddenState() const noexcept = 0;
594  virtual void setCellState(ITensor& cell) noexcept = 0;
595  virtual ITensor* getCellState() const noexcept = 0;
596 };
597 
598 class VPluginLayer : public VRoot
599 {
600 public:
601  virtual IPlugin& getPlugin() noexcept = 0;
602 };
603 
604 class VPluginV2Layer : public VRoot
605 {
606 public:
607  virtual IPluginV2& getPlugin() noexcept = 0;
608 };
609 
610 class VUnaryLayer : public VRoot
611 {
612 public:
613  virtual void setOperation(UnaryOperation op) noexcept = 0;
614  virtual UnaryOperation getOperation() const noexcept = 0;
615 };
616 
617 class VReduceLayer : public VRoot
618 {
619 public:
620  virtual void setOperation(ReduceOperation op) noexcept = 0;
621  virtual ReduceOperation getOperation() const noexcept = 0;
622  virtual void setReduceAxes(uint32_t reduceAxes) noexcept = 0;
623  virtual uint32_t getReduceAxes() const noexcept = 0;
624  virtual void setKeepDimensions(bool keepDimensions) noexcept = 0;
625  virtual bool getKeepDimensions() const noexcept = 0;
626 };
627 
628 class VPaddingLayer : public VRoot
629 {
630 public:
631  virtual void setPrePadding(DimsHW padding) noexcept = 0;
632  virtual DimsHW getPrePadding() const noexcept = 0;
633  virtual void setPostPadding(DimsHW padding) noexcept = 0;
634  virtual DimsHW getPostPadding() const noexcept = 0;
635  virtual void setPrePaddingNd(Dims padding) noexcept = 0;
636  virtual Dims getPrePaddingNd() const noexcept = 0;
637  virtual void setPostPaddingNd(Dims padding) noexcept = 0;
638  virtual Dims getPostPaddingNd() const noexcept = 0;
639 };
640 
641 class VShuffleLayer : public VRoot
642 {
643 public:
644  virtual void setFirstTranspose(const Permutation& permutation) noexcept = 0;
645  virtual const Permutation& getFirstTranspose() const noexcept = 0;
646  virtual void setReshapeDimensions(Dims dimensions) noexcept = 0;
647  virtual Dims getReshapeDimensions() const noexcept = 0;
648  virtual void setSecondTranspose(const Permutation& permutation) noexcept = 0;
649  virtual const Permutation& getSecondTranspose() const noexcept = 0;
650  virtual void setZeroIsPlaceholder(bool zeroIsPlaceholder) = 0;
651  virtual bool getZeroIsPlaceholder() const = 0;
652 };
653 
654 class VSliceLayer : public VRoot
655 {
656 public:
657  virtual void setStart(Dims start) noexcept = 0;
658  virtual Dims getStart() const noexcept = 0;
659  virtual void setSize(Dims size) noexcept = 0;
660  virtual Dims getSize() const noexcept = 0;
661  virtual void setStride(Dims stride) noexcept = 0;
662  virtual Dims getStride() const noexcept = 0;
663  virtual void setMode(SliceMode mode) noexcept = 0;
664  virtual SliceMode getMode() const noexcept = 0;
665 };
666 
667 class VShapeLayer : public VRoot
668 {
669 public:
670 };
671 
672 class VTopKLayer : public VRoot
673 {
674 public:
675  virtual void setOperation(TopKOperation op) noexcept = 0;
676  virtual TopKOperation getOperation() const noexcept = 0;
677  virtual void setK(int32_t k) noexcept = 0;
678  virtual int32_t getK() const noexcept = 0;
679  virtual void setReduceAxes(uint32_t reduceAxes) noexcept = 0;
680  virtual uint32_t getReduceAxes() const noexcept = 0;
681 };
682 
683 class VMatrixMultiplyLayer : public VRoot
684 {
685 public:
686  virtual void setOperation(int32_t index, MatrixOperation op) noexcept = 0;
687  virtual MatrixOperation getOperation(int32_t index) const noexcept = 0;
688 };
689 
690 class VRaggedSoftMaxLayer : public VRoot
691 {
692 public:
693 };
694 
695 class VIdentityLayer : public VRoot
696 {
697 public:
698 };
699 
700 class VConstantLayer : public VRoot
701 {
702 public:
703  virtual void setWeights(Weights weights) noexcept = 0;
704  virtual Weights getWeights() const noexcept = 0;
705  virtual void setDimensions(Dims dimensions) noexcept = 0;
706  virtual Dims getDimensions() const noexcept = 0;
707 };
708 
709 class VParametricReLULayer : public VRoot
710 {
711 public:
712 };
713 
714 class VResizeLayer : public VRoot
715 {
716 public:
717  virtual void setOutputDimensions(Dims dimensions) noexcept = 0;
718  virtual Dims getOutputDimensions() const noexcept = 0;
719  virtual void setScales(const float* scales, int32_t nbScales) noexcept = 0;
720  virtual int32_t getScales(int32_t size, float* scales) const noexcept = 0;
721  virtual void setResizeMode(ResizeMode resizeMode) noexcept = 0;
722  virtual ResizeMode getResizeMode() const noexcept = 0;
723  virtual void setAlignCorners(bool alignCorners) noexcept = 0;
724  virtual bool getAlignCorners() const noexcept = 0;
725  virtual void setCoordinateTransformation(ResizeCoordinateTransformation coordTransform) noexcept = 0;
726  virtual ResizeCoordinateTransformation getCoordinateTransformation() const noexcept = 0;
727  virtual void setSelectorForSinglePixel(ResizeSelector selector) noexcept = 0;
728  virtual ResizeSelector getSelectorForSinglePixel() const noexcept = 0;
729  virtual void setNearestRounding(ResizeRoundMode value) noexcept = 0;
730  virtual ResizeRoundMode getNearestRounding() const noexcept = 0;
731 };
732 
733 class VLoopBoundaryLayer : public VRoot
734 {
735 public:
736  virtual ILoop* getLoop() const noexcept = 0;
737 };
738 
739 class VRecurrenceLayer : public VRoot
740 {
741 public:
742 };
743 
744 class VLoopOutputLayer : public VRoot
745 {
746 public:
747  virtual LoopOutput getLoopOutput() const noexcept = 0;
748  virtual void setAxis(int32_t axis) noexcept = 0;
749  virtual int32_t getAxis() const noexcept = 0;
750 };
751 
752 class VTripLimitLayer : public VRoot
753 {
754 public:
755  virtual TripLimit getTripLimit() const noexcept = 0;
756 };
757 
758 class VIteratorLayer : public VRoot
759 {
760 public:
761  virtual void setAxis(int32_t axis) noexcept = 0;
762  virtual int32_t getAxis() const noexcept = 0;
763  virtual void setReverse(bool reverse) noexcept = 0;
764  virtual bool getReverse() const noexcept = 0;
765 };
766 class VLoop : public VRoot
767 {
768 public:
769  virtual IRecurrenceLayer* addRecurrence(ITensor& initialValue) noexcept = 0;
770  virtual ITripLimitLayer* addTripLimit(ITensor& tensor, TripLimit limit) noexcept = 0;
771  virtual IIteratorLayer* addIterator(ITensor& tensor, int32_t axis = 0, bool reverse = false) noexcept = 0;
772  virtual ILoopOutputLayer* addLoopOutput(ITensor& tensor, LoopOutput outputKind, int32_t axis = 0) noexcept = 0;
773  virtual void setName(const char* name) noexcept = 0;
774  virtual const char* getName() const noexcept = 0;
775 };
776 
777 class VConditionalBoundaryLayer : public VRoot
778 {
779 public:
780  virtual IIfConditional* getConditional() const noexcept = 0;
781 };
782 
783 class VConditionLayer : public VRoot
784 {
785 public:
786 };
787 
788 class VConditionalInputLayer : public VRoot
789 {
790 public:
791 };
792 
793 class VConditionalOutputLayer : public VRoot
794 {
795 public:
796 };
797 
798 class VIfConditional : public VRoot
799 {
800 public:
801  virtual IConditionLayer* setCondition(ITensor& tensor) noexcept = 0;
802  virtual IIfConditionalInputLayer* addInput(ITensor& tensor) noexcept = 0;
803  virtual IIfConditionalOutputLayer* addOutput(ITensor& trueTensor, ITensor& falseTensor) noexcept = 0;
804  virtual void setName(const char* name) noexcept = 0;
805  virtual const char* getName() const noexcept = 0;
806 };
807 
808 class VSelectLayer : public VRoot
809 {
810 };
811 
812 class VAssertionLayer : public VRoot
813 {
814 public:
815  virtual void setMessage(const char* message) noexcept = 0;
816  virtual const char* getMessage() const noexcept = 0;
817 };
818 
819 class VFillLayer : public VRoot
820 {
821 public:
822  virtual void setDimensions(Dims dimensions) noexcept = 0;
823  virtual Dims getDimensions() const noexcept = 0;
824  virtual void setOperation(FillOperation op) noexcept = 0;
825  virtual FillOperation getOperation() const noexcept = 0;
826  virtual void setAlpha(double alpha) noexcept = 0;
827  virtual double getAlpha() const noexcept = 0;
828  virtual void setBeta(double beta) noexcept = 0;
829  virtual double getBeta() const noexcept = 0;
830 };
831 
832 class VQuantizeLayer : public VRoot
833 {
834 public:
835  virtual int32_t getAxis() const noexcept = 0;
836  virtual void setAxis(int32_t axis) noexcept = 0;
837 };
838 
839 class VDequantizeLayer : public VRoot
840 {
841 public:
842  virtual int32_t getAxis() const noexcept = 0;
843  virtual void setAxis(int32_t axis) noexcept = 0;
844 };
845 
846 class VScatterLayer : public VRoot
847 {
848 public:
849  virtual void setMode(ScatterMode mode) noexcept = 0;
850  virtual ScatterMode getMode() const noexcept = 0;
851  virtual void setAxis(int32_t axis) noexcept = 0;
852  virtual int32_t getAxis() const noexcept = 0;
853 }; // class VScatterLayer
854 
855 class VEinsumLayer : public VRoot
856 {
857 public:
858  virtual bool setEquation(const char* equation) noexcept = 0;
859  virtual const char* getEquation() const noexcept = 0;
860 };
861 
862 class VNetworkDefinition : public VRoot
863 {
864 public:
865  virtual ITensor* addInput(const char* name, DataType type, Dims dimensions) noexcept = 0;
866  virtual void markOutput(ITensor& tensor) noexcept = 0;
867  virtual IConvolutionLayer* addConvolution(
868  ITensor& input, int32_t nbOutputMaps, DimsHW kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
869  = 0;
870  virtual IFullyConnectedLayer* addFullyConnected(
871  ITensor& input, int32_t nbOutputs, Weights kernelWeights, Weights biasWeights) noexcept
872  = 0;
873  virtual IActivationLayer* addActivation(ITensor& input, ActivationType type) noexcept = 0;
874  virtual IPoolingLayer* addPooling(ITensor& input, PoolingType type, DimsHW windowSize) noexcept = 0;
875  virtual ILRNLayer* addLRN(ITensor& input, int32_t window, float alpha, float beta, float k) noexcept = 0;
876  virtual IScaleLayer* addScale(ITensor& input, ScaleMode mode, Weights shift, Weights scale, Weights power) noexcept
877  = 0;
878  virtual ISoftMaxLayer* addSoftMax(ITensor& input) noexcept = 0;
879  virtual IConcatenationLayer* addConcatenation(ITensor* const* inputs, int32_t nbInputs) noexcept = 0;
880  virtual IDeconvolutionLayer* addDeconvolution(
881  ITensor& input, int32_t nbOutputMaps, DimsHW kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
882  = 0;
883  virtual IElementWiseLayer* addElementWise(ITensor& input1, ITensor& input2, ElementWiseOperation op) noexcept = 0;
884  virtual IUnaryLayer* addUnary(ITensor& input, UnaryOperation operation) noexcept = 0;
885  virtual IPaddingLayer* addPadding(ITensor& input, DimsHW prePadding, DimsHW postPadding) noexcept = 0;
886  virtual IShuffleLayer* addShuffle(ITensor& input) noexcept = 0;
887  virtual int32_t getNbLayers() const noexcept = 0;
888  virtual ILayer* getLayer(int32_t index) const noexcept = 0;
889  virtual int32_t getNbInputs() const noexcept = 0;
890  virtual ITensor* getInput(int32_t index) const noexcept = 0;
891  virtual int32_t getNbOutputs() const noexcept = 0;
892  virtual ITensor* getOutput(int32_t index) const noexcept = 0;
893  virtual IReduceLayer* addReduce(
894  ITensor& input, ReduceOperation operation, uint32_t reduceAxes, bool keepDimensions) noexcept
895  = 0;
896  virtual ITopKLayer* addTopK(ITensor& input, TopKOperation op, int32_t k, uint32_t reduceAxes) noexcept = 0;
897  virtual IGatherLayer* addGather(ITensor& data, ITensor& indices, int32_t axis) noexcept = 0;
898  virtual IRaggedSoftMaxLayer* addRaggedSoftMax(ITensor& input, ITensor& bounds) noexcept = 0;
899  virtual IMatrixMultiplyLayer* addMatrixMultiply(
900  ITensor& input0, MatrixOperation op0, ITensor& input1, MatrixOperation op1) noexcept
901  = 0;
902  virtual IConstantLayer* addConstant(Dims dimensions, Weights weights) noexcept = 0;
903  virtual IRNNv2Layer* addRNNv2(
904  ITensor& input, int32_t layerCount, int32_t hiddenSize, int32_t maxSeqLen, RNNOperation op) noexcept
905  = 0;
906  virtual IIdentityLayer* addIdentity(ITensor& input) noexcept = 0;
907  virtual void removeTensor(ITensor& tensor) noexcept = 0;
908  virtual void unmarkOutput(ITensor& tensor) noexcept = 0;
909  virtual IPluginV2Layer* addPluginV2(ITensor* const* inputs, int32_t nbInputs, IPluginV2& plugin) noexcept = 0;
910  virtual ISliceLayer* addSlice(ITensor& input, Dims start, Dims size, Dims stride) noexcept = 0;
911  virtual void setName(const char* name) noexcept = 0;
912  virtual const char* getName() const noexcept = 0;
913  virtual IShapeLayer* addShape(ITensor& input) noexcept = 0;
914  virtual bool hasImplicitBatchDimension() const noexcept = 0;
915  virtual bool markOutputForShapes(ITensor& tensor) noexcept = 0;
916  virtual bool unmarkOutputForShapes(ITensor& tensor) noexcept = 0;
917  virtual IParametricReLULayer* addParametricReLU(ITensor& input, ITensor& slope) noexcept = 0;
918  virtual IConvolutionLayer* addConvolutionNd(
919  ITensor& input, int32_t nbOutputMaps, Dims kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
920  = 0;
921  virtual IPoolingLayer* addPoolingNd(ITensor& input, PoolingType type, Dims windowSize) noexcept = 0;
922  virtual IDeconvolutionLayer* addDeconvolutionNd(
923  ITensor& input, int32_t nbOutputMaps, Dims kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
924  = 0;
925  virtual IScaleLayer* addScaleNd(
926  ITensor& input, ScaleMode mode, Weights shift, Weights scale, Weights power, int32_t channelAxis) noexcept
927  = 0;
928  virtual IResizeLayer* addResize(ITensor& input) noexcept = 0;
929  virtual bool hasExplicitPrecision() const noexcept = 0;
930  virtual ILoop* addLoop() noexcept = 0;
931  virtual ISelectLayer* addSelect(ITensor& condition, ITensor& thenInput, ITensor& elseInput) noexcept = 0;
932  virtual IFillLayer* addFill(Dims dimensions, FillOperation op) noexcept = 0;
933  virtual IPaddingLayer* addPaddingNd(ITensor& input, Dims prePadding, Dims postPadding) noexcept = 0;
934  virtual bool setWeightsName(Weights weights, const char* name) noexcept = 0;
935  virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
936  virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
937  virtual IDequantizeLayer* addDequantize(ITensor& input, ITensor& scale) noexcept = 0;
938  virtual IQuantizeLayer* addQuantize(ITensor& input, ITensor& scale) noexcept = 0;
939  virtual IGatherLayer* addGatherV2(ITensor& data, ITensor& indices, GatherMode mode) noexcept = 0;
940  virtual IIfConditional* addIfConditional() noexcept = 0;
941  virtual IScatterLayer* addScatter(ITensor& data, ITensor& indices, ITensor& updates, ScatterMode mode) noexcept = 0;
942  virtual IEinsumLayer* addEinsum(ITensor* const* inputs, int32_t nbInputs, const char* equation) noexcept = 0;
943  virtual IAssertionLayer* addAssertion(ITensor& condition, const char* message) noexcept = 0;
944 };
945 
946 class VAlgorithmIOInfo : public VRoot
947 {
948 public:
949  virtual TensorFormat getTensorFormat() const noexcept = 0;
950  virtual DataType getDataType() const noexcept = 0;
951  virtual Dims getStrides() const noexcept = 0;
952 };
953 
954 class VAlgorithmVariant : public VRoot
955 {
956 public:
957  virtual int64_t getImplementation() const noexcept = 0;
958  virtual int64_t getTactic() const noexcept = 0;
959 };
960 
961 class VAlgorithmContext : public VRoot
962 {
963 public:
964  virtual const char* getName() const noexcept = 0;
965  virtual Dims getDimensions(int32_t index, OptProfileSelector select) const noexcept = 0;
966  virtual int32_t getNbInputs() const noexcept = 0;
967  virtual int32_t getNbOutputs() const noexcept = 0;
968 };
969 
970 class VAlgorithm : public VRoot
971 {
972 public:
973  virtual const IAlgorithmIOInfo& getAlgorithmIOInfo(int32_t index) const noexcept = 0;
974  virtual const IAlgorithmVariant& getAlgorithmVariant() const noexcept = 0;
975  virtual float getTimingMSec() const noexcept = 0;
976  virtual std::size_t getWorkspaceSize() const noexcept = 0;
977  virtual const IAlgorithmIOInfo* getAlgorithmIOInfoByIndex(int32_t index) const noexcept = 0;
978 };
979 
980 class VTimingCache : public VRoot
981 {
982 public:
983  virtual nvinfer1::IHostMemory* serialize() const noexcept = 0;
984  virtual bool combine(const ITimingCache& inputCache, bool ignoreMismatch) noexcept = 0;
985  virtual bool reset() noexcept = 0;
986 };
987 
988 class VBuilderConfig : public VRoot
989 {
990 public:
991  virtual void setMinTimingIterations(int32_t minTiming) noexcept = 0;
992  virtual int32_t getMinTimingIterations() const noexcept = 0;
993  virtual void setAvgTimingIterations(int32_t avgTiming) noexcept = 0;
994  virtual int32_t getAvgTimingIterations() const noexcept = 0;
995  virtual void setEngineCapability(EngineCapability capability) noexcept = 0;
996  virtual EngineCapability getEngineCapability() const noexcept = 0;
997  virtual void setInt8Calibrator(IInt8Calibrator* calibrator) noexcept = 0;
998  virtual IInt8Calibrator* getInt8Calibrator() const noexcept = 0;
999  virtual void setMaxWorkspaceSize(std::size_t workspaceSize) noexcept = 0;
1000  virtual std::size_t getMaxWorkspaceSize() const noexcept = 0;
1001  virtual void setFlags(BuilderFlags builderFlags) noexcept = 0;
1002  virtual BuilderFlags getFlags() const noexcept = 0;
1003  virtual void clearFlag(BuilderFlag builderFlag) noexcept = 0;
1004  virtual void setFlag(BuilderFlag builderFlag) noexcept = 0;
1005  virtual bool getFlag(BuilderFlag builderFlag) const noexcept = 0;
1006  virtual void setDeviceType(const ILayer* layer, DeviceType deviceType) noexcept = 0;
1007  virtual DeviceType getDeviceType(const ILayer* layer) const noexcept = 0;
1008  virtual bool isDeviceTypeSet(const ILayer* layer) const noexcept = 0;
1009  virtual void resetDeviceType(const ILayer* layer) noexcept = 0;
1010  virtual bool canRunOnDLA(const ILayer* layer) const noexcept = 0;
1011  virtual void setDLACore(int32_t dlaCore) noexcept = 0;
1012  virtual int32_t getDLACore() const noexcept = 0;
1013  virtual void setDefaultDeviceType(DeviceType deviceType) noexcept = 0;
1014  virtual DeviceType getDefaultDeviceType() const noexcept = 0;
1015  virtual void reset() noexcept = 0;
1016  virtual void setProfileStream(const cudaStream_t stream) noexcept = 0;
1017  virtual cudaStream_t getProfileStream() const noexcept = 0;
1018  virtual int32_t addOptimizationProfile(const IOptimizationProfile* profile) noexcept = 0;
1019  virtual int32_t getNbOptimizationProfiles() const noexcept = 0;
1020  virtual void setProfilingVerbosity(ProfilingVerbosity verbosity) noexcept = 0;
1021  virtual ProfilingVerbosity getProfilingVerbosity() const noexcept = 0;
1022  virtual void setAlgorithmSelector(IAlgorithmSelector* selector) noexcept = 0;
1023  virtual IAlgorithmSelector* getAlgorithmSelector() const noexcept = 0;
1024  virtual bool setCalibrationProfile(const IOptimizationProfile* profile) noexcept = 0;
1025  virtual const IOptimizationProfile* getCalibrationProfile() noexcept = 0;
1026  virtual void setQuantizationFlags(QuantizationFlags flags) noexcept = 0;
1027  virtual QuantizationFlags getQuantizationFlags() const noexcept = 0;
1028  virtual void clearQuantizationFlag(QuantizationFlag flag) noexcept = 0;
1029  virtual void setQuantizationFlag(QuantizationFlag flag) noexcept = 0;
1030  virtual bool getQuantizationFlag(QuantizationFlag flag) const noexcept = 0;
1031  virtual bool setTacticSources(TacticSources tacticSources) noexcept = 0;
1032  virtual TacticSources getTacticSources() const noexcept = 0;
1033  virtual nvinfer1::ITimingCache* createTimingCache(const void* blob, std::size_t size) const noexcept = 0;
1034  virtual bool setTimingCache(const ITimingCache& cache, bool ignoreMismatch) noexcept = 0;
1035  virtual const nvinfer1::ITimingCache* getTimingCache() const noexcept = 0;
1036 };
1037 
1038 class VBuilder : public VRoot
1039 {
1040 public:
1041  virtual void setMaxBatchSize(int32_t batchSize) noexcept = 0;
1042  virtual int32_t getMaxBatchSize() const noexcept = 0;
1043  virtual bool platformHasFastFp16() const noexcept = 0;
1044  virtual bool platformHasFastInt8() const noexcept = 0;
1045  virtual int32_t getMaxDLABatchSize() const noexcept = 0;
1046  virtual int32_t getNbDLACores() const noexcept = 0;
1047  virtual void setGpuAllocator(IGpuAllocator* allocator) noexcept = 0;
1048  virtual nvinfer1::IBuilderConfig* createBuilderConfig() noexcept = 0;
1049  virtual nvinfer1::ICudaEngine* buildEngineWithConfig(INetworkDefinition& network, IBuilderConfig& config) noexcept
1050  = 0;
1051  virtual nvinfer1::INetworkDefinition* createNetworkV2(NetworkDefinitionCreationFlags flags) noexcept = 0;
1052  virtual nvinfer1::IOptimizationProfile* createOptimizationProfile() noexcept = 0;
1053  virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
1054  virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
1055  virtual void reset() noexcept = 0;
1056  virtual bool platformHasTf32() const noexcept = 0;
1057  virtual nvinfer1::IHostMemory* buildSerializedNetwork(INetworkDefinition& network, IBuilderConfig& config) noexcept
1058  = 0;
1059  virtual bool isNetworkSupported(INetworkDefinition const& network, IBuilderConfig const& config) const noexcept = 0;
1060  virtual ILogger* getLogger() const noexcept = 0;
1061 };
1062 
1063 } // namespace apiv
1064 } // namespace nvinfer1
1065 
1066 // @endcond
1067 
1068 #endif // NV_INFER_RUNTIME_IMPL_H
nvinfer1::LoopOutput
LoopOutput
Enum that describes kinds of loop outputs.
Definition: NvInfer.h:4930
nvinfer1::SliceMode
SliceMode
Controls how ISliceLayer handles out of bounds coordinates.
Definition: NvInfer.h:4029
Dims
Structure to define the dimensions of a tensor.
nvinfer1::AsciiChar
char_t AsciiChar
AsciiChar is the type used by TensorRT to represent valid ASCII characters.
Definition: NvInferRuntimeCommon.h:125
nvinfer1::DimensionOperation
DimensionOperation
An operation on two IDimensionExpr, which represent integer expressions used in dimension computation...
Definition: NvInferRuntime.h:221
getLogger
nvinfer1::ILogger * getLogger() noexcept
Return the logger object.
nvinfer1::LayerInformationFormat
LayerInformationFormat
The format in which the IEngineInspector prints the layer information.
Definition: NvInferRuntime.h:2417
nvinfer1::IHostMemory
Class to handle library allocated memory that is accessible to the user.
Definition: NvInferRuntime.h:173
nvinfer1::DeviceType
DeviceType
The device that this layer/network will execute on.
Definition: NvInferRuntime.h:631
nvinfer1::ProfilingVerbosity
ProfilingVerbosity
List of verbosity levels of layer information exposed in NVTX annotations and in IEngineInspector.
Definition: NvInferRuntime.h:1285
nvinfer1::RNNInputMode
RNNInputMode
Enumerates the RNN input modes that may occur with an RNN layer.
Definition: NvInfer.h:3187
nvinfer1::ResizeMode
ResizeMode
Enumerates various modes of resize in the resize layer. Resize mode set using setResizeMode().
Definition: NvInfer.h:4540
nvinfer1::TensorFormat
TensorFormat
Format of the input/output tensors.
Definition: NvInferRuntimeCommon.h:220
nvinfer1::ICudaEngine
An engine for executing inference on a built network, with functionally unsafe features.
Definition: NvInferRuntime.h:1308
nvinfer1::ActivationType
ActivationType
Enumerates the types of activation to perform in an activation layer.
Definition: NvInfer.h:154
nvinfer1::ElementWiseOperation
ElementWiseOperation
Enumerates the binary operations that may be performed by an ElementWise layer.
Definition: NvInfer.h:2798
nvinfer1::ScaleMode
ScaleMode
Controls how shift, scale and power are applied in a Scale layer.
Definition: NvInfer.h:2102
nvinfer1::ReduceOperation
ReduceOperation
Enumerates the reduce operations that may be performed by a Reduce layer.
Definition: NvInfer.h:3628
nvinfer1::BuilderFlag
BuilderFlag
List of valid modes that the builder can enable when creating an engine from a network definition.
Definition: NvInfer.h:7725
nvinfer1
The TensorRT API version 1 namespace.
nvinfer1::RNNGateType
RNNGateType
Identifies an individual gate within an RNN cell.
Definition: NvInfer.h:3207
nvinfer1::UnaryOperation
UnaryOperation
Enumerates the unary operations that may be performed by a Unary layer.
Definition: NvInfer.h:3541
nvinfer1::NetworkDefinitionCreationFlag
NetworkDefinitionCreationFlag
List of immutable network properties expressed at network creation time. NetworkDefinitionCreationFla...
Definition: NvInfer.h:8442
nvinfer1::PaddingMode
PaddingMode
Enumerates the modes of padding to perform in convolution, deconvolution and pooling layer,...
Definition: NvInfer.h:961
nvinfer1::MatrixOperation
MatrixOperation
Enumerates the operations that may be performed on a tensor by IMatrixMultiplyLayer before multiplica...
Definition: NvInfer.h:4335
nvinfer1::PoolingType
PoolingType
The type of pooling to perform in a pooling layer.
Definition: NvInfer.h:1636
nvinfer1::ResizeSelector
ResizeSelector
The coordinate selector when resize to single pixel output.
Definition: NvInfer.h:4613
nvinfer1::TacticSources
uint32_t TacticSources
Represents a collection of one or more TacticSource values combine using bitwise-OR operations.
Definition: NvInferRuntime.h:1274
nvinfer1::DataType
DataType
The type of weights and tensors.
Definition: NvInferRuntimeCommon.h:150
nvinfer1::RNNOperation
RNNOperation
Enumerates the RNN operations that may be performed by an RNN layer.
Definition: NvInfer.h:3137
nvinfer1::QuantizationFlags
uint32_t QuantizationFlags
Represents one or more QuantizationFlag values using binary OR operations.
Definition: NvInfer.h:7686
nvinfer1::LayerType
LayerType
The type values of layer classes.
Definition: NvInfer.h:89
nvinfer1::RNNDirection
RNNDirection
Enumerates the RNN direction that may be performed by an RNN layer.
Definition: NvInfer.h:3159
nvinfer1::TripLimit
TripLimit
Enum that describes kinds of trip limits.
Definition: NvInfer.h:4950
nvinfer1::FillOperation
FillOperation
Enumerates the tensor fill operations that may performed by a fill layer.
Definition: NvInfer.h:5437
NvInferRuntimeCommon.h
nvinfer1::QuantizationFlag
QuantizationFlag
List of valid flags for quantizing the network to int8.
Definition: NvInfer.h:7695
nvinfer1::ResizeRoundMode
ResizeRoundMode
The rounding mode for nearest neighbor resize.
Definition: NvInfer.h:4639
nvinfer1::OptProfileSelector
OptProfileSelector
When setting or querying optimization profile parameters (such as shape tensor inputs or dynamic dime...
Definition: NvInferRuntime.h:1056
nvinfer1::WeightsRole
WeightsRole
How a layer uses particular Weights.
Definition: NvInferRuntime.h:609
nvinfer1::BuilderFlags
uint32_t BuilderFlags
Represents one or more QuantizationFlag values using binary OR operations, e.g., 1U << BuilderFlag::k...
Definition: NvInfer.h:7716
nvinfer1::NetworkDefinitionCreationFlags
uint32_t NetworkDefinitionCreationFlags
Represents one or more NetworkDefinitionCreationFlag flags using binary OR operations....
Definition: NvInfer.h:8431
NvInferLegacyDims.h
nvinfer1::ScatterMode
ScatterMode
Control form of IScatterLayer.
Definition: NvInfer.h:5875
nvinfer1::TensorFormats
uint32_t TensorFormats
It is capable of representing one or more TensorFormat by binary OR operations, e....
Definition: NvInfer.h:147
nvinfer1::EngineCapability
EngineCapability
List of supported engine capability flows.
Definition: NvInferRuntime.h:105
nvinfer1::ResizeCoordinateTransformation
ResizeCoordinateTransformation
The resize coordinate transformation function.
Definition: NvInfer.h:4563
nvinfer1::GatherMode
GatherMode
Control form of IGatherLayer.
Definition: NvInfer.h:2884
nvinfer1::TopKOperation
TopKOperation
Enumerates the operations that may be performed by a TopK layer.
Definition: NvInfer.h:4239
nvinfer1::TensorLocation
TensorLocation
The location for tensor data storage, device or host.
Definition: NvInferRuntime.h:245
nvinfer1::CalibrationAlgoType
CalibrationAlgoType
Version of calibration algorithm to use.
Definition: NvInfer.h:7240