TensorRT 10.4.0
NvInferImpl.h
Go to the documentation of this file.
1/*
2 * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: Apache-2.0
4 *
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
8 *
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 */
17
18#ifndef NV_INFER_IMPL_H
19#define NV_INFER_IMPL_H
20
21#include "NvInferLegacyDims.h"
22#include "NvInferRuntimeCommon.h"
23
24// @cond SuppressDoxyWarnings
25
26namespace nvinfer1
27{
28
29namespace v_1_0
30{
31class IProgressMonitor;
32}
33using IProgressMonitor = v_1_0::IProgressMonitor;
34
35namespace v_1_0
36{
37class IAlgorithmSelector;
38}
39using IAlgorithmSelector = v_1_0::IAlgorithmSelector;
40
41namespace v_1_0
42{
43class IProfiler;
44}
45using IProfiler = v_1_0::IProfiler;
46
47namespace v_1_0
48{
49class IOutputAllocator;
50}
51using IOutputAllocator = v_1_0::IOutputAllocator;
52
53namespace v_1_0
54{
55class IDebugListener;
56}
57using IDebugListener = v_1_0::IDebugListener;
58
59class IActivationLayer;
60class IAlgorithm;
61class IAlgorithmContext;
62class IAlgorithmIOInfo;
63class IAlgorithmVariant;
64class IAssertionLayer;
65class IBuilder;
66class IBuilderConfig;
67class IConcatenationLayer;
68class IConditionLayer;
69class IConstantLayer;
70class IConvolutionLayer;
71class ICudaEngine;
72class IDeconvolutionLayer;
73class IDequantizeLayer;
74class IDimensionExpr;
75class IEinsumLayer;
76class IElementWiseLayer;
77class IEngineInspector;
78class IExecutionContext;
79class IFillLayer;
80class IGatherLayer;
81class IGridSampleLayer;
82class IHostMemory;
83class IIdentityLayer;
84class ICastLayer;
85class IIfConditional;
86class IIfConditionalInputLayer;
87class IIfConditionalOutputLayer;
88class IInt8Calibrator;
89class IIteratorLayer;
90class ILayer;
91class ILoop;
92class ILoopOutputLayer;
93class ILRNLayer;
94class IMatrixMultiplyLayer;
95class INetworkDefinition;
96class INormalizationLayer;
97class INMSLayer;
98class INonZeroLayer;
99class IOneHotLayer;
100class IOptimizationProfile;
101class IPaddingLayer;
102class IParametricReLULayer;
103class IPlugin;
104class IPluginExt;
105class IPluginFactory;
106class IPluginLayer;
107class IPluginRegistry;
108class IPluginV2Layer;
109
110namespace v_1_0
111{
112class IPluginV3;
113} // namespace v_1_0
114using IPluginV3 = v_1_0::IPluginV3;
115
116class IPluginV3Layer;
117class IPoolingLayer;
118class IQuantizeLayer;
119class IRaggedSoftMaxLayer;
120class IRecurrenceLayer;
121class IReduceLayer;
122class IRefitter;
123class IResizeLayer;
124class IReverseSequenceLayer;
125class IRuntime;
126class IScaleLayer;
127class IScatterLayer;
128class ISelectLayer;
129class ISerializationConfig;
130class IShapeLayer;
131class IShuffleLayer;
132class ISliceLayer;
133class ISoftMaxLayer;
134class ITensor;
135class ITimingCache;
136class ITopKLayer;
137class ITripLimitLayer;
138class IUnaryLayer;
139struct Permutation;
140class Weights;
141
142enum class ActivationType : int32_t;
143enum class BoundingBoxFormat : int32_t;
144enum class BuilderFlag : int32_t;
145enum class CalibrationAlgoType : int32_t;
146enum class DeviceType : int32_t;
147enum class DimensionOperation : int32_t;
148enum class ElementWiseOperation : int32_t;
149enum class EngineCapability : int32_t;
150enum class FillOperation : int32_t;
151enum class GatherMode : int32_t;
152enum class LayerInformationFormat : int32_t;
153enum class LayerType : int32_t;
154enum class LoopOutput : int32_t;
155enum class MatrixOperation : int32_t;
156enum class MemoryPoolType : int32_t;
157enum class NetworkDefinitionCreationFlag : int32_t;
158enum class OptProfileSelector : int32_t;
159enum class PaddingMode : int32_t;
160enum class PoolingType : int32_t;
161enum class ProfilingVerbosity : int32_t;
162enum class QuantizationFlag : int32_t;
163enum class ReduceOperation : int32_t;
164enum class ResizeCoordinateTransformation : int32_t;
165enum class InterpolationMode : int32_t;
166enum class ResizeRoundMode : int32_t;
167enum class ResizeSelector : int32_t;
168enum class ScaleMode : int32_t;
169enum class ScatterMode : int32_t;
170enum class SampleMode : int32_t;
171enum class SerializationFlag : int32_t;
172enum class TensorIOMode : int32_t;
173enum class TensorLocation : int32_t;
174enum class TopKOperation : int32_t;
175enum class TripLimit : int32_t;
176enum class UnaryOperation : int32_t;
177enum class WeightsRole : int32_t;
178enum class PreviewFeature : int32_t;
179enum class HardwareCompatibilityLevel : int32_t;
180enum class ExecutionContextAllocationStrategy : int32_t;
181enum class RuntimePlatform : int32_t;
182
183using TacticSources = uint32_t;
184using TensorFormats = uint32_t;
185using BuilderFlags = uint32_t;
186using NetworkDefinitionCreationFlags = uint32_t;
187using QuantizationFlags = uint32_t;
188using TempfileControlFlags = uint32_t;
189using SerializationFlags = uint32_t;
190
198
199namespace apiv
200{
201
202class VRoot
203{
204public:
205 virtual ~VRoot() noexcept = default;
206};
207
208class VHostMemory : public VRoot
209{
210public:
211 virtual void* data() const noexcept = 0;
212 virtual std::size_t size() const noexcept = 0;
213 virtual DataType type() const noexcept = 0;
214};
215
216class VDimensionExpr : public VRoot
217{
218public:
219 virtual bool isConstant() const = 0;
220 virtual int64_t getConstantValue() const = 0;
221 virtual bool isSizeTensor() const = 0;
222};
223
224class VExprBuilder : public VRoot
225{
226public:
227 virtual IDimensionExpr const* constant(int64_t value) = 0;
228 virtual IDimensionExpr const* operation(
229 DimensionOperation op, IDimensionExpr const& first, IDimensionExpr const& second)
230 = 0;
231 virtual IDimensionExpr const* declareSizeTensor(
232 int32_t outputIndex, IDimensionExpr const& opt, IDimensionExpr const& upper)
233 = 0;
234};
235
236class VRuntime : public VRoot
237{
238public:
239 virtual IRuntime* getPImpl() noexcept = 0;
240 virtual nvinfer1::ICudaEngine* deserializeCudaEngine(void const* blob, std::size_t size) noexcept = 0;
241 virtual nvinfer1::ICudaEngine* deserializeCudaEngine(IStreamReader& streamReader) noexcept = 0;
242 virtual void setDLACore(int32_t dlaCore) noexcept = 0;
243 virtual int32_t getDLACore() const noexcept = 0;
244 virtual int32_t getNbDLACores() const noexcept = 0;
245 virtual void setGpuAllocator(IGpuAllocator* allocator) noexcept = 0;
246 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
247 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
248 virtual ILogger* getLogger() const noexcept = 0;
249 virtual bool setMaxThreads(int32_t maxThreads) noexcept = 0;
250 virtual int32_t getMaxThreads() const noexcept = 0;
251 virtual void setTemporaryDirectory(char const*) noexcept = 0;
252 virtual char const* getTemporaryDirectory() const noexcept = 0;
253 virtual void setTempfileControlFlags(TempfileControlFlags) noexcept = 0;
254 virtual TempfileControlFlags getTempfileControlFlags() const noexcept = 0;
255 virtual IPluginRegistry& getPluginRegistry() noexcept = 0;
256 virtual void setPluginRegistryParent(IPluginRegistry* parent) noexcept = 0;
257 virtual IRuntime* loadRuntime(char const* path) noexcept = 0;
258 virtual void setEngineHostCodeAllowed(bool allowed) noexcept = 0;
259 virtual bool getEngineHostCodeAllowed() const noexcept = 0;
260};
261
262class VRefitter : public VRoot
263{
264public:
265 virtual IRefitter* getPImpl() noexcept = 0;
266 virtual bool setWeights(char const* layerName, WeightsRole role, const Weights weights) noexcept = 0;
267 virtual bool refitCudaEngine() noexcept = 0;
268 virtual int32_t getMissing(int32_t size, char const** layerNames, WeightsRole* roles) noexcept = 0;
269 virtual int32_t getAll(int32_t size, char const** layerNames, WeightsRole* roles) noexcept = 0;
270 virtual bool setDynamicRange(char const* tensorName, float min, float max) noexcept = 0;
271 virtual float getDynamicRangeMin(char const* tensorName) const noexcept = 0;
272 virtual float getDynamicRangeMax(char const* tensorName) const noexcept = 0;
273 virtual int32_t getTensorsWithDynamicRange(int32_t size, char const** tensorNames) const noexcept = 0;
274 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
275 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
276 virtual bool setNamedWeights(char const* name, Weights weights) noexcept = 0;
277 virtual int32_t getMissingWeights(int32_t size, char const** weightsNames) noexcept = 0;
278 virtual int32_t getAllWeights(int32_t size, char const** weightsNames) noexcept = 0;
279 virtual ILogger* getLogger() const noexcept = 0;
280 virtual bool setMaxThreads(int32_t maxThreads) noexcept = 0;
281 virtual int32_t getMaxThreads() const noexcept = 0;
282 virtual bool setNamedWeightsWithLocation(char const* name, Weights weights, TensorLocation location) noexcept = 0;
283 virtual Weights getNamedWeights(char const* weightsName) const noexcept = 0;
284 virtual TensorLocation getWeightsLocation(char const* weightsName) const noexcept = 0;
285 virtual bool unsetNamedWeights(char const* weightsName) noexcept = 0;
286 virtual void setWeightsValidation(bool weightsValidation) noexcept = 0;
287 virtual bool getWeightsValidation() const noexcept = 0;
288 virtual bool refitCudaEngineAsync(cudaStream_t stream) noexcept = 0;
289 virtual Weights getWeightsPrototype(char const* weightsName) const noexcept = 0;
290};
291
292class VOptimizationProfile : public VRoot
293{
294public:
295 virtual bool setDimensions(char const* inputName, OptProfileSelector select, Dims const& dims) noexcept = 0;
296 virtual Dims getDimensions(char const* inputName, OptProfileSelector select) const noexcept = 0;
297 virtual bool setShapeValues(
298 char const* inputName, OptProfileSelector select, int32_t const* values, int32_t nbValues) noexcept = 0;
299 virtual int32_t getNbShapeValues(char const* inputName) const noexcept = 0;
300 virtual int32_t const* getShapeValues(char const* inputName, OptProfileSelector select) const noexcept = 0;
301 virtual bool setExtraMemoryTarget(float target) noexcept = 0;
302 virtual float getExtraMemoryTarget() const noexcept = 0;
303 virtual bool isValid() const noexcept = 0;
304};
305
306class VCudaEngine : public VRoot
307{
308public:
309 virtual ICudaEngine* getPImpl() noexcept = 0;
310 virtual int32_t getNbLayers() const noexcept = 0;
311 virtual IHostMemory* serialize() const noexcept = 0;
312 virtual IExecutionContext* createExecutionContext(ExecutionContextAllocationStrategy strategy) noexcept = 0;
313 virtual IExecutionContext* createExecutionContextWithoutDeviceMemory() noexcept = 0;
314 virtual size_t getDeviceMemorySize() const noexcept = 0;
315 virtual bool isRefittable() const noexcept = 0;
316 virtual char const* getName() const noexcept = 0;
317 virtual int32_t getNbOptimizationProfiles() const noexcept = 0;
318 virtual int32_t const* getProfileTensorValues(
319 char const* tensorName, int32_t profileIndex, OptProfileSelector select) const noexcept = 0;
320 virtual EngineCapability getEngineCapability() const noexcept = 0;
321 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
322 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
323 virtual bool hasImplicitBatchDimension() const noexcept = 0;
324 virtual TacticSources getTacticSources() const noexcept = 0;
325 virtual ProfilingVerbosity getProfilingVerbosity() const noexcept = 0;
326 virtual IEngineInspector* createEngineInspector() const noexcept = 0;
327 virtual Dims getTensorShape(char const* tensorName) const noexcept = 0;
328 virtual DataType getTensorDataType(char const* tensorName) const noexcept = 0;
329 virtual TensorLocation getTensorLocation(char const* tensorName) const noexcept = 0;
330 virtual bool isShapeInferenceIO(char const* tensorName) const noexcept = 0;
331 virtual TensorIOMode getTensorIOMode(char const* tensorName) const noexcept = 0;
332 virtual int32_t getTensorBytesPerComponent(char const* tensorName) const noexcept = 0;
333 virtual int32_t getTensorComponentsPerElement(char const* tensorName) const noexcept = 0;
334 virtual TensorFormat getTensorFormat(char const* tensorName) const noexcept = 0;
335 virtual char const* getTensorFormatDesc(char const* tensorName) const noexcept = 0;
336 virtual int32_t getTensorVectorizedDim(char const* tensorName) const noexcept = 0;
337 virtual Dims getProfileShape(
338 char const* tensorName, int32_t profileIndex, OptProfileSelector select) const noexcept = 0;
339 virtual int32_t getNbIOTensors() const noexcept = 0;
340 virtual char const* getIOTensorName(int32_t index) const noexcept = 0;
341 virtual HardwareCompatibilityLevel getHardwareCompatibilityLevel() const noexcept = 0;
342 virtual int32_t getNbAuxStreams() const noexcept = 0;
343
344 virtual int32_t getTensorBytesPerComponentV2(char const* tensorName, int32_t profileIndex) const noexcept = 0;
345 virtual int32_t getTensorComponentsPerElementV2(char const* tensorName, int32_t profileIndex) const noexcept = 0;
346 virtual TensorFormat getTensorFormatV2(char const* tensorName, int32_t profileIndex) const noexcept = 0;
347 virtual char const* getTensorFormatDescV2(char const* tensorName, int32_t profileIndex) const noexcept = 0;
348 virtual int32_t getTensorVectorizedDimV2(char const* tensorName, int32_t profileIndex) const noexcept = 0;
349
350 virtual ISerializationConfig* createSerializationConfig() noexcept = 0;
351 virtual IHostMemory* serializeWithConfig(ISerializationConfig& config) const noexcept = 0;
352
353 virtual size_t getDeviceMemorySizeForProfile(int32_t profileIndex) const noexcept = 0;
354 virtual IRefitter* createRefitter(ILogger& logger) noexcept = 0;
355
356 virtual bool setWeightStreamingBudget(int64_t gpuMemoryBudget) noexcept = 0;
357 virtual int64_t getWeightStreamingBudget() const noexcept = 0;
358 virtual int64_t getMinimumWeightStreamingBudget() const noexcept = 0;
359 virtual int64_t getStreamableWeightsSize() const noexcept = 0;
360
361 virtual bool isDebugTensor(char const* name) const noexcept = 0;
362
363 // Added in TensorRT 10.1
364 virtual bool setWeightStreamingBudgetV2(int64_t gpuMemoryBudget) noexcept = 0;
365 virtual int64_t getWeightStreamingBudgetV2() const noexcept = 0;
366 virtual int64_t getWeightStreamingAutomaticBudget() const noexcept = 0;
367 virtual int64_t getWeightStreamingScratchMemorySize() const noexcept = 0;
368 virtual int64_t getDeviceMemorySizeV2() const noexcept = 0;
369 virtual int64_t getDeviceMemorySizeForProfileV2(int32_t profileIndex) const noexcept = 0;
370};
371
372class VExecutionContext : public VRoot
373{
374public:
375 virtual IExecutionContext* getPImpl() noexcept = 0;
376 virtual void setDebugSync(bool sync) noexcept = 0;
377 virtual bool getDebugSync() const noexcept = 0;
378 virtual void setProfiler(IProfiler*) noexcept = 0;
379 virtual IProfiler* getProfiler() const noexcept = 0;
380 virtual ICudaEngine const& getEngine() const noexcept = 0;
381 virtual void setName(char const* name) noexcept = 0;
382 virtual char const* getName() const noexcept = 0;
383 virtual void setDeviceMemory(void* memory) noexcept = 0;
384 virtual int32_t getOptimizationProfile() const noexcept = 0;
385 virtual bool allInputDimensionsSpecified() const noexcept = 0;
386 virtual bool allInputShapesSpecified() const noexcept = 0;
387 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
388 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
389 virtual bool executeV2(void* const* bindings) noexcept = 0;
390 virtual bool setOptimizationProfileAsync(int32_t profileIndex, cudaStream_t stream) noexcept = 0;
391 virtual void setEnqueueEmitsProfile(bool enqueueEmitsProfile) noexcept = 0;
392 virtual bool getEnqueueEmitsProfile() const noexcept = 0;
393 virtual bool reportToProfiler() const noexcept = 0;
394 virtual bool setInputShape(char const* tensorName, Dims const& dims) noexcept = 0;
395 virtual Dims getTensorShape(char const* tensorName) const noexcept = 0;
396 virtual Dims getTensorStrides(char const* tensorName) const noexcept = 0;
397 virtual bool setTensorAddress(char const* tensorName, void* data) noexcept = 0;
398 virtual void const* getTensorAddress(char const* tensorName) const noexcept = 0;
399 virtual bool setInputTensorAddress(char const* tensorName, void const* data) noexcept = 0;
400 virtual bool setOutputTensorAddress(char const* tensorName, void* data) noexcept = 0;
401 virtual int32_t inferShapes(int32_t nbMaxNames, char const** tensorNames) noexcept = 0;
402 virtual bool setInputConsumedEvent(cudaEvent_t event) noexcept = 0;
403 virtual cudaEvent_t getInputConsumedEvent() const noexcept = 0;
404 virtual void* getOutputTensorAddress(char const* tensorName) const noexcept = 0;
405 virtual bool setOutputAllocator(char const* tensorName, IOutputAllocator* outputAllocator) noexcept = 0;
406 virtual IOutputAllocator* getOutputAllocator(char const* name) noexcept = 0;
407 virtual int64_t getMaxOutputSize(char const* tensorName) const noexcept = 0;
408 virtual bool setTemporaryStorageAllocator(IGpuAllocator* allocator) noexcept = 0;
409 virtual IGpuAllocator* getTemporaryStorageAllocator() const noexcept = 0;
410 virtual bool enqueueV3(cudaStream_t stream) noexcept = 0;
411 virtual void setPersistentCacheLimit(size_t size) noexcept = 0;
412 virtual size_t getPersistentCacheLimit() const noexcept = 0;
413 virtual bool setNvtxVerbosity(ProfilingVerbosity verbosity) noexcept = 0;
414 virtual ProfilingVerbosity getNvtxVerbosity() const noexcept = 0;
415 virtual void setAuxStreams(cudaStream_t* auxStreams, int32_t nbStreams) noexcept = 0;
416 virtual bool setDebugListener(IDebugListener* listener) noexcept = 0;
417 virtual IDebugListener* getDebugListener() noexcept = 0;
418 virtual bool setTensorDebugState(char const* name, bool flag) noexcept = 0;
419 virtual bool getDebugState(char const* name) const noexcept = 0;
420 virtual bool setAllTensorsDebugState(bool flag) noexcept = 0;
421 virtual size_t updateDeviceMemorySizeForShapes() noexcept = 0;
422
423 // Added in TensorRT 10.1
424 virtual void setDeviceMemoryV2(void* memory, int64_t size) noexcept = 0;
425};
426
427class VEngineInspector : public VRoot
428{
429public:
430 virtual IEngineInspector* getPImpl() noexcept = 0;
431 virtual bool setExecutionContext(IExecutionContext const* context) noexcept = 0;
432 virtual IExecutionContext const* getExecutionContext() const noexcept = 0;
433 virtual char const* getLayerInformation(int32_t layerIndex, LayerInformationFormat format) const noexcept = 0;
434 virtual char const* getEngineInformation(LayerInformationFormat format) const noexcept = 0;
435 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
436 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
437};
438
439class VTensor : public VRoot
440{
441public:
442 virtual void setName(char const* name) noexcept = 0;
443 virtual char const* getName() const noexcept = 0;
444 virtual void setDimensions(Dims const& dimensions) noexcept = 0;
445 virtual Dims getDimensions() const noexcept = 0;
446 virtual void setType(DataType type) noexcept = 0;
447 virtual DataType getType() const noexcept = 0;
448 virtual bool setDynamicRange(float min, float max) noexcept = 0;
449 virtual bool isNetworkInput() const noexcept = 0;
450 virtual bool isNetworkOutput() const noexcept = 0;
451 virtual void setBroadcastAcrossBatch(bool broadcastAcrossBatch) noexcept = 0;
452 virtual bool getBroadcastAcrossBatch() const noexcept = 0;
453 virtual TensorLocation getLocation() const noexcept = 0;
454 virtual void setLocation(TensorLocation location) noexcept = 0;
455 virtual bool dynamicRangeIsSet() const noexcept = 0;
456 virtual void resetDynamicRange() noexcept = 0;
457 virtual float getDynamicRangeMin() const noexcept = 0;
458 virtual float getDynamicRangeMax() const noexcept = 0;
459 virtual void setAllowedFormats(TensorFormats formats) noexcept = 0;
460 virtual TensorFormats getAllowedFormats() const noexcept = 0;
461 virtual bool isShapeTensor() const noexcept = 0;
462 virtual bool isExecutionTensor() const noexcept = 0;
463 virtual void setDimensionName(int32_t index, char const* name) noexcept = 0;
464 virtual char const* getDimensionName(int32_t index) const noexcept = 0;
465};
466class VLayer : public VRoot
467{
468public:
469 virtual LayerType getType() const noexcept = 0;
470 virtual void setName(char const* name) noexcept = 0;
471 virtual char const* getName() const noexcept = 0;
472 virtual int32_t getNbInputs() const noexcept = 0;
473 virtual ITensor* getInput(int32_t index) const noexcept = 0;
474 virtual int32_t getNbOutputs() const noexcept = 0;
475 virtual ITensor* getOutput(int32_t index) const noexcept = 0;
476 virtual void setInput(int32_t index, ITensor& tensor) noexcept = 0;
477 virtual void setPrecision(DataType dataType) noexcept = 0;
478 virtual DataType getPrecision() const noexcept = 0;
479 virtual bool precisionIsSet() const noexcept = 0;
480 virtual void resetPrecision() noexcept = 0;
481 virtual void setOutputType(int32_t index, DataType dataType) noexcept = 0;
482 virtual DataType getOutputType(int32_t index) const noexcept = 0;
483 virtual bool outputTypeIsSet(int32_t index) const noexcept = 0;
484 virtual void resetOutputType(int32_t index) noexcept = 0;
485 virtual void setMetadata(char const* docString) noexcept = 0;
486 virtual char const* getMetadata() const noexcept = 0;
487};
488
489class VConvolutionLayer : public VRoot
490{
491public:
492 virtual void setNbOutputMaps(int64_t nbOutputMaps) noexcept = 0;
493 virtual int64_t getNbOutputMaps() const noexcept = 0;
494 virtual void setNbGroups(int64_t nbGroups) noexcept = 0;
495 virtual int64_t getNbGroups() const noexcept = 0;
496 virtual void setKernelWeights(Weights weights) noexcept = 0;
497 virtual Weights getKernelWeights() const noexcept = 0;
498 virtual void setBiasWeights(Weights weights) noexcept = 0;
499 virtual Weights getBiasWeights() const noexcept = 0;
500 virtual void setPrePadding(Dims const& padding) noexcept = 0;
501 virtual Dims getPrePadding() const noexcept = 0;
502 virtual void setPostPadding(Dims const& padding) noexcept = 0;
503 virtual Dims getPostPadding() const noexcept = 0;
504 virtual void setPaddingMode(PaddingMode paddingMode) noexcept = 0;
505 virtual PaddingMode getPaddingMode() const noexcept = 0;
506 virtual void setKernelSizeNd(Dims const& kernelSize) noexcept = 0;
507 virtual Dims getKernelSizeNd() const noexcept = 0;
508 virtual void setStrideNd(Dims const& stride) noexcept = 0;
509 virtual Dims getStrideNd() const noexcept = 0;
510 virtual void setPaddingNd(Dims const& padding) noexcept = 0;
511 virtual Dims getPaddingNd() const noexcept = 0;
512 virtual void setDilationNd(Dims const& dilation) noexcept = 0;
513 virtual Dims getDilationNd() const noexcept = 0;
514};
515
516class VActivationLayer : public VRoot
517{
518public:
519 virtual void setActivationType(ActivationType type) noexcept = 0;
520 virtual ActivationType getActivationType() const noexcept = 0;
521 virtual void setAlpha(float alpha) noexcept = 0;
522 virtual void setBeta(float beta) noexcept = 0;
523 virtual float getAlpha() const noexcept = 0;
524 virtual float getBeta() const noexcept = 0;
525};
526
527class VPoolingLayer : public VRoot
528{
529public:
530 virtual void setPoolingType(PoolingType type) noexcept = 0;
531 virtual PoolingType getPoolingType() const noexcept = 0;
532 virtual void setBlendFactor(float blendFactor) noexcept = 0;
533 virtual float getBlendFactor() const noexcept = 0;
534 virtual void setAverageCountExcludesPadding(bool exclusive) noexcept = 0;
535 virtual bool getAverageCountExcludesPadding() const noexcept = 0;
536 virtual void setPrePadding(Dims const& padding) noexcept = 0;
537 virtual Dims getPrePadding() const noexcept = 0;
538 virtual void setPostPadding(Dims const& padding) noexcept = 0;
539 virtual Dims getPostPadding() const noexcept = 0;
540 virtual void setPaddingMode(PaddingMode paddingMode) noexcept = 0;
541 virtual PaddingMode getPaddingMode() const noexcept = 0;
542 virtual void setWindowSizeNd(Dims const& windowSize) noexcept = 0;
543 virtual Dims getWindowSizeNd() const noexcept = 0;
544 virtual void setStrideNd(Dims const& stride) noexcept = 0;
545 virtual Dims getStrideNd() const noexcept = 0;
546 virtual void setPaddingNd(Dims const& padding) noexcept = 0;
547 virtual Dims getPaddingNd() const noexcept = 0;
548};
549
550class VLRNLayer : public VRoot
551{
552public:
553 virtual void setWindowSize(int64_t windowSize) noexcept = 0;
554 virtual int64_t getWindowSize() const noexcept = 0;
555 virtual void setAlpha(float alpha) noexcept = 0;
556 virtual float getAlpha() const noexcept = 0;
557 virtual void setBeta(float beta) noexcept = 0;
558 virtual float getBeta() const noexcept = 0;
559 virtual void setK(float k) noexcept = 0;
560 virtual float getK() const noexcept = 0;
561};
562
563class VScaleLayer : public VRoot
564{
565public:
566 virtual void setMode(ScaleMode mode) noexcept = 0;
567 virtual ScaleMode getMode() const noexcept = 0;
568 virtual void setShift(Weights shift) noexcept = 0;
569 virtual Weights getShift() const noexcept = 0;
570 virtual void setScale(Weights scale) noexcept = 0;
571 virtual Weights getScale() const noexcept = 0;
572 virtual void setPower(Weights power) noexcept = 0;
573 virtual Weights getPower() const noexcept = 0;
574 virtual int32_t getChannelAxis() const noexcept = 0;
575 virtual void setChannelAxis(int32_t channelAxis) noexcept = 0;
576};
577
578class VSoftMaxLayer : public VRoot
579{
580public:
581 virtual void setAxes(uint32_t axes) noexcept = 0;
582 virtual uint32_t getAxes() const noexcept = 0;
583};
584
585class VConcatenationLayer : public VRoot
586{
587public:
588 virtual void setAxis(int32_t axis) noexcept = 0;
589 virtual int32_t getAxis() const noexcept = 0;
590};
591
592class VDeconvolutionLayer : public VRoot
593{
594public:
595 virtual void setNbOutputMaps(int64_t nbOutputMaps) noexcept = 0;
596 virtual int64_t getNbOutputMaps() const noexcept = 0;
597 virtual void setNbGroups(int64_t nbGroups) noexcept = 0;
598 virtual int64_t getNbGroups() const noexcept = 0;
599 virtual void setKernelWeights(Weights weights) noexcept = 0;
600 virtual Weights getKernelWeights() const noexcept = 0;
601 virtual void setBiasWeights(Weights weights) noexcept = 0;
602 virtual Weights getBiasWeights() const noexcept = 0;
603 virtual void setPrePadding(Dims const& padding) noexcept = 0;
604 virtual Dims getPrePadding() const noexcept = 0;
605 virtual void setPostPadding(Dims const& padding) noexcept = 0;
606 virtual Dims getPostPadding() const noexcept = 0;
607 virtual void setPaddingMode(PaddingMode paddingMode) noexcept = 0;
608 virtual PaddingMode getPaddingMode() const noexcept = 0;
609 virtual void setKernelSizeNd(Dims const& kernelSize) noexcept = 0;
610 virtual Dims getKernelSizeNd() const noexcept = 0;
611 virtual void setStrideNd(Dims const& stride) noexcept = 0;
612 virtual Dims getStrideNd() const noexcept = 0;
613 virtual void setPaddingNd(Dims const& padding) noexcept = 0;
614 virtual Dims getPaddingNd() const noexcept = 0;
615 virtual void setDilationNd(Dims const& dilation) noexcept = 0;
616 virtual Dims getDilationNd() const noexcept = 0;
617};
618
619class VElementWiseLayer : public VRoot
620{
621public:
622 virtual void setOperation(ElementWiseOperation op) noexcept = 0;
623 virtual ElementWiseOperation getOperation() const noexcept = 0;
624};
625
626class VGatherLayer : public VRoot
627{
628public:
629 virtual void setGatherAxis(int32_t axis) noexcept = 0;
630 virtual int32_t getGatherAxis() const noexcept = 0;
631 virtual void setNbElementWiseDims(int32_t k) noexcept = 0;
632 virtual int32_t getNbElementWiseDims() const noexcept = 0;
633 virtual void setMode(GatherMode mode) noexcept = 0;
634 virtual GatherMode getMode() const noexcept = 0;
635};
636
637class VPluginLayer : public VRoot
638{
639public:
640 virtual IPlugin& getPlugin() noexcept = 0;
641};
642
643class VPluginV2Layer : public VRoot
644{
645public:
646 virtual IPluginV2& getPlugin() noexcept = 0;
647};
648
649class VPluginV3Layer : public VRoot
650{
651public:
652 virtual IPluginV3& getPlugin() noexcept = 0;
653};
654
655class VUnaryLayer : public VRoot
656{
657public:
658 virtual void setOperation(UnaryOperation op) noexcept = 0;
659 virtual UnaryOperation getOperation() const noexcept = 0;
660};
661
662class VReduceLayer : public VRoot
663{
664public:
665 virtual void setOperation(ReduceOperation op) noexcept = 0;
666 virtual ReduceOperation getOperation() const noexcept = 0;
667 virtual void setReduceAxes(uint32_t reduceAxes) noexcept = 0;
668 virtual uint32_t getReduceAxes() const noexcept = 0;
669 virtual void setKeepDimensions(bool keepDimensions) noexcept = 0;
670 virtual bool getKeepDimensions() const noexcept = 0;
671};
672
673class VPaddingLayer : public VRoot
674{
675public:
676 virtual void setPrePaddingNd(Dims const& padding) noexcept = 0;
677 virtual Dims getPrePaddingNd() const noexcept = 0;
678 virtual void setPostPaddingNd(Dims const& padding) noexcept = 0;
679 virtual Dims getPostPaddingNd() const noexcept = 0;
680};
681
682class VShuffleLayer : public VRoot
683{
684public:
685 virtual void setFirstTranspose(Permutation const& permutation) noexcept = 0;
686 virtual Permutation const& getFirstTranspose() const noexcept = 0;
687 virtual void setReshapeDimensions(Dims const& dimensions) noexcept = 0;
688 virtual Dims getReshapeDimensions() const noexcept = 0;
689 virtual void setSecondTranspose(Permutation const& permutation) noexcept = 0;
690 virtual Permutation const& getSecondTranspose() const noexcept = 0;
691 virtual void setZeroIsPlaceholder(bool zeroIsPlaceholder) noexcept = 0;
692 virtual bool getZeroIsPlaceholder() const noexcept = 0;
693};
694
695class VSliceLayer : public VRoot
696{
697public:
698 virtual void setStart(Dims const& start) noexcept = 0;
699 virtual Dims getStart() const noexcept = 0;
700 virtual void setSize(Dims const& size) noexcept = 0;
701 virtual Dims getSize() const noexcept = 0;
702 virtual void setStride(Dims const& stride) noexcept = 0;
703 virtual Dims getStride() const noexcept = 0;
704 virtual void setMode(SampleMode mode) noexcept = 0;
705 virtual SampleMode getMode() const noexcept = 0;
706 virtual void setAxes(Dims const& axes) noexcept = 0;
707 virtual Dims getAxes() const noexcept = 0;
708};
709
710class VShapeLayer : public VRoot
711{
712public:
713};
714
715class VTopKLayer : public VRoot
716{
717public:
718 virtual void setOperation(TopKOperation op) noexcept = 0;
719 virtual TopKOperation getOperation() const noexcept = 0;
720 virtual void setK(int32_t k) noexcept = 0;
721 virtual int32_t getK() const noexcept = 0;
722 virtual void setReduceAxes(uint32_t reduceAxes) noexcept = 0;
723 virtual uint32_t getReduceAxes() const noexcept = 0;
724};
725
726class VMatrixMultiplyLayer : public VRoot
727{
728public:
729 virtual void setOperation(int32_t index, MatrixOperation op) noexcept = 0;
730 virtual MatrixOperation getOperation(int32_t index) const noexcept = 0;
731};
732
733class VNonZeroLayer : public VRoot
734{
735public:
736};
737
738class VRaggedSoftMaxLayer : public VRoot
739{
740public:
741};
742
743class VIdentityLayer : public VRoot
744{
745public:
746};
747
748class VCastLayer : public VRoot
749{
750public:
751 virtual void setToType(DataType toType) noexcept = 0;
752 virtual DataType getToType() const noexcept = 0;
753};
754
755class VConstantLayer : public VRoot
756{
757public:
758 virtual void setWeights(Weights weights) noexcept = 0;
759 virtual Weights getWeights() const noexcept = 0;
760 virtual void setDimensions(Dims const& dimensions) noexcept = 0;
761 virtual Dims getDimensions() const noexcept = 0;
762};
763
764class VParametricReLULayer : public VRoot
765{
766public:
767};
768
769class VResizeLayer : public VRoot
770{
771public:
772 virtual void setOutputDimensions(Dims const& dimensions) noexcept = 0;
773 virtual Dims getOutputDimensions() const noexcept = 0;
774 virtual void setScales(float const* scales, int32_t nbScales) noexcept = 0;
775 virtual int32_t getScales(int32_t size, float* scales) const noexcept = 0;
776 virtual void setResizeMode(InterpolationMode interpolationMode) noexcept = 0;
777 virtual InterpolationMode getResizeMode() const noexcept = 0;
778 virtual void setCoordinateTransformation(ResizeCoordinateTransformation coordTransform) noexcept = 0;
779 virtual ResizeCoordinateTransformation getCoordinateTransformation() const noexcept = 0;
780 virtual void setSelectorForSinglePixel(ResizeSelector selector) noexcept = 0;
781 virtual ResizeSelector getSelectorForSinglePixel() const noexcept = 0;
782 virtual void setNearestRounding(ResizeRoundMode value) noexcept = 0;
783 virtual ResizeRoundMode getNearestRounding() const noexcept = 0;
784 virtual void setCubicCoeff(float value) noexcept = 0;
785 virtual float getCubicCoeff() const noexcept = 0;
786 virtual void setExcludeOutside(bool value) noexcept = 0;
787 virtual bool getExcludeOutside() const noexcept = 0;
788};
789
790class VLoopBoundaryLayer : public VRoot
791{
792public:
793 virtual ILoop* getLoop() const noexcept = 0;
794};
795
796class VRecurrenceLayer : public VRoot
797{
798public:
799};
800
801class VLoopOutputLayer : public VRoot
802{
803public:
804 virtual LoopOutput getLoopOutput() const noexcept = 0;
805 virtual void setAxis(int32_t axis) noexcept = 0;
806 virtual int32_t getAxis() const noexcept = 0;
807};
808
809class VTripLimitLayer : public VRoot
810{
811public:
812 virtual TripLimit getTripLimit() const noexcept = 0;
813};
814
815class VIteratorLayer : public VRoot
816{
817public:
818 virtual void setAxis(int32_t axis) noexcept = 0;
819 virtual int32_t getAxis() const noexcept = 0;
820 virtual void setReverse(bool reverse) noexcept = 0;
821 virtual bool getReverse() const noexcept = 0;
822};
823class VLoop : public VRoot
824{
825public:
826 virtual IRecurrenceLayer* addRecurrence(ITensor& initialValue) noexcept = 0;
827 virtual ITripLimitLayer* addTripLimit(ITensor& tensor, TripLimit limit) noexcept = 0;
828 virtual IIteratorLayer* addIterator(ITensor& tensor, int32_t axis = 0, bool reverse = false) noexcept = 0;
829 virtual ILoopOutputLayer* addLoopOutput(ITensor& tensor, LoopOutput outputKind, int32_t axis = 0) noexcept = 0;
830 virtual void setName(char const* name) noexcept = 0;
831 virtual char const* getName() const noexcept = 0;
832};
833
834class VConditionalBoundaryLayer : public VRoot
835{
836public:
837 virtual IIfConditional* getConditional() const noexcept = 0;
838};
839
840class VConditionLayer : public VRoot
841{
842public:
843};
844
845class VConditionalInputLayer : public VRoot
846{
847public:
848};
849
850class VConditionalOutputLayer : public VRoot
851{
852public:
853};
854
855class VIfConditional : public VRoot
856{
857public:
858 virtual IConditionLayer* setCondition(ITensor& tensor) noexcept = 0;
859 virtual IIfConditionalInputLayer* addInput(ITensor& tensor) noexcept = 0;
860 virtual IIfConditionalOutputLayer* addOutput(ITensor& trueTensor, ITensor& falseTensor) noexcept = 0;
861 virtual void setName(char const* name) noexcept = 0;
862 virtual char const* getName() const noexcept = 0;
863};
864
865class VSelectLayer : public VRoot
866{
867};
868
869class VAssertionLayer : public VRoot
870{
871public:
872 virtual void setMessage(char const* message) noexcept = 0;
873 virtual char const* getMessage() const noexcept = 0;
874};
875
876class VFillLayer : public VRoot
877{
878public:
879 virtual void setDimensions(Dims const& dimensions) noexcept = 0;
880 virtual Dims getDimensions() const noexcept = 0;
881 virtual void setOperation(FillOperation op) noexcept = 0;
882 virtual FillOperation getOperation() const noexcept = 0;
883 virtual void setAlpha(double alpha) noexcept = 0;
884 virtual double getAlpha() const noexcept = 0;
885 virtual void setBeta(double beta) noexcept = 0;
886 virtual double getBeta() const noexcept = 0;
887 virtual void setAlphaInt64(int64_t alpha) noexcept = 0;
888 virtual int64_t getAlphaInt64() const noexcept = 0;
889 virtual void setBetaInt64(int64_t beta) noexcept = 0;
890 virtual int64_t getBetaInt64() const noexcept = 0;
891 virtual bool isAlphaBetaInt64() const noexcept = 0;
892 virtual DataType getToType() const noexcept = 0;
893 virtual void setToType(DataType toType) noexcept = 0;
894};
895
896class VQuantizeLayer : public VRoot
897{
898public:
899 virtual int32_t getAxis() const noexcept = 0;
900 virtual void setAxis(int32_t axis) noexcept = 0;
901 virtual DataType getToType() const noexcept = 0;
902 virtual void setToType(DataType toType) noexcept = 0;
903};
904
905class VDequantizeLayer : public VRoot
906{
907public:
908 virtual int32_t getAxis() const noexcept = 0;
909 virtual void setAxis(int32_t axis) noexcept = 0;
910 virtual DataType getToType() const noexcept = 0;
911 virtual void setToType(DataType toType) noexcept = 0;
912};
913
914
915class VScatterLayer : public VRoot
916{
917public:
918 virtual void setMode(ScatterMode mode) noexcept = 0;
919 virtual ScatterMode getMode() const noexcept = 0;
920 virtual void setAxis(int32_t axis) noexcept = 0;
921 virtual int32_t getAxis() const noexcept = 0;
922}; // class VScatterLayer
923
924class VEinsumLayer : public VRoot
925{
926public:
927 virtual bool setEquation(char const* equation) noexcept = 0;
928 virtual char const* getEquation() const noexcept = 0;
929};
930
931class VOneHotLayer : public VRoot
932{
933public:
934 virtual int32_t getAxis() const noexcept = 0;
935 virtual void setAxis(int32_t axis) noexcept = 0;
936}; // class VOneHotLayer
937
938class VGridSampleLayer : public VRoot
939{
940public:
941 virtual void setInterpolationMode(InterpolationMode mode) noexcept = 0;
942 virtual InterpolationMode getInterpolationMode() const noexcept = 0;
943 virtual void setAlignCorners(bool alignCorners) noexcept = 0;
944 virtual bool getAlignCorners() const noexcept = 0;
945 virtual bool setSampleMode(SampleMode mode) noexcept = 0;
946 virtual SampleMode getSampleMode() const noexcept = 0;
947}; // class VGridSampleLayer
948
949class VNMSLayer : public VRoot
950{
951public:
952 virtual void setBoundingBoxFormat(BoundingBoxFormat fmt) noexcept = 0;
953 virtual BoundingBoxFormat getBoundingBoxFormat() const noexcept = 0;
954 virtual void setTopKBoxLimit(int32_t limit) noexcept = 0;
955 virtual int32_t getTopKBoxLimit() const noexcept = 0;
956}; // class VNMSLayer
957
958class VReverseSequenceLayer : public VRoot
959{
960public:
961 virtual void setBatchAxis(int32_t batchAxis) noexcept = 0;
962 virtual int32_t getBatchAxis() const noexcept = 0;
963
964 virtual void setSequenceAxis(int32_t sequenceAxis) noexcept = 0;
965 virtual int32_t getSequenceAxis() const noexcept = 0;
966}; // class VReverseSequenceLayer
967
968class VNormalizationLayer : public VRoot
969{
970public:
971 virtual void setEpsilon(float eps) noexcept = 0;
972 virtual float getEpsilon() const noexcept = 0;
973 virtual void setAxes(uint32_t axesMask) noexcept = 0;
974 virtual uint32_t getAxes() const noexcept = 0;
975 virtual void setNbGroups(int64_t nbGroups) noexcept = 0;
976 virtual int64_t getNbGroups() const noexcept = 0;
977 virtual void setComputePrecision(DataType type) noexcept = 0;
978 virtual DataType getComputePrecision() const noexcept = 0;
979}; // class VNormalizationLayer
980
981class VNetworkDefinition : public VRoot
982{
983public:
984 virtual ITensor* addInput(char const* name, DataType type, Dims const& dimensions) noexcept = 0;
985 virtual void markOutput(ITensor& tensor) noexcept = 0;
986 virtual IActivationLayer* addActivation(ITensor& input, ActivationType type) noexcept = 0;
987 virtual ILRNLayer* addLRN(ITensor& input, int64_t window, float alpha, float beta, float k) noexcept = 0;
988 virtual IScaleLayer* addScale(
989 ITensor& input, ScaleMode mode, Weights shift, Weights scale, Weights power) noexcept = 0;
990 virtual ISoftMaxLayer* addSoftMax(ITensor& input) noexcept = 0;
991 virtual IConcatenationLayer* addConcatenation(ITensor* const* inputs, int32_t nbInputs) noexcept = 0;
992 virtual IElementWiseLayer* addElementWise(ITensor& input1, ITensor& input2, ElementWiseOperation op) noexcept = 0;
993 virtual IUnaryLayer* addUnary(ITensor& input, UnaryOperation operation) noexcept = 0;
994 virtual IShuffleLayer* addShuffle(ITensor& input) noexcept = 0;
995 virtual int32_t getNbLayers() const noexcept = 0;
996 virtual ILayer* getLayer(int32_t index) const noexcept = 0;
997 virtual int32_t getNbInputs() const noexcept = 0;
998 virtual ITensor* getInput(int32_t index) const noexcept = 0;
999 virtual int32_t getNbOutputs() const noexcept = 0;
1000 virtual ITensor* getOutput(int32_t index) const noexcept = 0;
1001 virtual IReduceLayer* addReduce(
1002 ITensor& input, ReduceOperation operation, uint32_t reduceAxes, bool keepDimensions) noexcept
1003 = 0;
1004 virtual ITopKLayer* addTopK(ITensor& input, TopKOperation op, int32_t k, uint32_t reduceAxes) noexcept = 0;
1005 virtual IGatherLayer* addGather(ITensor& data, ITensor& indices, int32_t axis) noexcept = 0;
1006 virtual IRaggedSoftMaxLayer* addRaggedSoftMax(ITensor& input, ITensor& bounds) noexcept = 0;
1007 virtual IMatrixMultiplyLayer* addMatrixMultiply(
1008 ITensor& input0, MatrixOperation op0, ITensor& input1, MatrixOperation op1) noexcept = 0;
1009 virtual IConstantLayer* addConstant(Dims const& dimensions, Weights weights) noexcept = 0;
1010 virtual IIdentityLayer* addIdentity(ITensor& input) noexcept = 0;
1011 virtual void removeTensor(ITensor& tensor) noexcept = 0;
1012 virtual void unmarkOutput(ITensor& tensor) noexcept = 0;
1013 virtual IPluginV2Layer* addPluginV2(ITensor* const* inputs, int32_t nbInputs, IPluginV2& plugin) noexcept = 0;
1014 virtual IPluginV3Layer* addPluginV3(ITensor* const* inputs, int32_t nbInputs, ITensor* const* shapeInputs,
1015 int32_t nbShapeInputs, IPluginV3& plugin) noexcept = 0;
1016 virtual ISliceLayer* addSlice(ITensor& input, Dims const& start, Dims const& size, Dims const& stride) noexcept = 0;
1017 virtual void setName(char const* name) noexcept = 0;
1018 virtual char const* getName() const noexcept = 0;
1019 virtual IShapeLayer* addShape(ITensor& input) noexcept = 0;
1020 virtual bool hasImplicitBatchDimension() const noexcept = 0;
1021 virtual bool markOutputForShapes(ITensor& tensor) noexcept = 0;
1022 virtual bool unmarkOutputForShapes(ITensor& tensor) noexcept = 0;
1023 virtual IParametricReLULayer* addParametricReLU(ITensor& input, ITensor& slope) noexcept = 0;
1024 virtual IConvolutionLayer* addConvolutionNd(
1025 ITensor& input, int64_t nbOutputMaps, Dims const& kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
1026 = 0;
1027 virtual IPoolingLayer* addPoolingNd(ITensor& input, PoolingType type, Dims const& windowSize) noexcept = 0;
1028 virtual IDeconvolutionLayer* addDeconvolutionNd(
1029 ITensor& input, int64_t nbOutputMaps, Dims const& kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
1030 = 0;
1031 virtual IScaleLayer* addScaleNd(
1032 ITensor& input, ScaleMode mode, Weights shift, Weights scale, Weights power, int32_t channelAxis) noexcept = 0;
1033 virtual IResizeLayer* addResize(ITensor& input) noexcept = 0;
1034 virtual ILoop* addLoop() noexcept = 0;
1035 virtual ISelectLayer* addSelect(ITensor& condition, ITensor& thenInput, ITensor& elseInput) noexcept = 0;
1036 virtual IFillLayer* addFill(Dims const& dimensions, FillOperation op) noexcept = 0;
1037 virtual IPaddingLayer* addPaddingNd(ITensor& input, Dims const& prePadding, Dims const& postPadding) noexcept = 0;
1038 virtual bool setWeightsName(Weights weights, char const* name) noexcept = 0;
1039 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
1040 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
1041 virtual IDequantizeLayer* addDequantize(ITensor& input, ITensor& scale) noexcept = 0;
1042 virtual IQuantizeLayer* addQuantize(ITensor& input, ITensor& scale) noexcept = 0;
1043 virtual IGatherLayer* addGatherV2(ITensor& data, ITensor& indices, GatherMode mode) noexcept = 0;
1044 virtual IIfConditional* addIfConditional() noexcept = 0;
1045 virtual IScatterLayer* addScatter(ITensor& data, ITensor& indices, ITensor& updates, ScatterMode mode) noexcept = 0;
1046 virtual IEinsumLayer* addEinsum(ITensor* const* inputs, int32_t nbInputs, char const* equation) noexcept = 0;
1047 virtual IAssertionLayer* addAssertion(ITensor& condition, char const* message) noexcept = 0;
1048 virtual IOneHotLayer* addOneHot(ITensor& indices, ITensor& values, ITensor& depth, int32_t axis) noexcept = 0;
1049 virtual INonZeroLayer* addNonZero(ITensor& input) noexcept = 0;
1050 virtual IGridSampleLayer* addGridSample(ITensor& input, ITensor& grid) noexcept = 0;
1051 virtual INMSLayer* addNMS(ITensor& boxes, ITensor& scores, ITensor& maxOutputBoxesPerClass) noexcept = 0;
1052 virtual IReverseSequenceLayer* addReverseSequence(ITensor& input, ITensor& sequenceLens) noexcept = 0;
1053 virtual INormalizationLayer* addNormalization(
1054 ITensor& input, ITensor& scale, ITensor& bias, uint32_t axesMask) noexcept = 0;
1055 virtual ICastLayer* addCast(ITensor& input, DataType toType) noexcept = 0;
1056 virtual IBuilder& getBuilder() const noexcept = 0;
1057 virtual NetworkDefinitionCreationFlags getFlags() const noexcept = 0;
1058 virtual bool getFlag(NetworkDefinitionCreationFlag networkDefinitionCreationFlag) const noexcept = 0;
1059 virtual IQuantizeLayer* addQuantizeV2(ITensor& input, ITensor& scale, DataType outputType) noexcept = 0;
1060 virtual IDequantizeLayer* addDequantizeV2(ITensor& input, ITensor& scale, DataType outputType) noexcept = 0;
1061 virtual IFillLayer* addFillV2(Dims const& dimensions, FillOperation op, DataType outputType) noexcept = 0;
1062 virtual bool markDebug(ITensor& tensor) noexcept = 0;
1063 virtual bool unmarkDebug(ITensor& tensor) noexcept = 0;
1064 virtual bool isDebugTensor(nvinfer1::ITensor const& tensor) const noexcept = 0;
1065 virtual bool markWeightsRefittable(char const* name) noexcept = 0;
1066 virtual bool unmarkWeightsRefittable(char const* name) noexcept = 0;
1067 virtual bool areWeightsMarkedRefittable(char const* name) const noexcept = 0;
1068};
1069
1070class VAlgorithmIOInfo : public VRoot
1071{
1072public:
1073 virtual DataType getDataType() const noexcept = 0;
1074 virtual Dims getStrides() const noexcept = 0;
1075 virtual int64_t getVectorizedDim() const noexcept = 0;
1076 virtual int64_t getComponentsPerElement() const noexcept = 0;
1077};
1078
1079class VAlgorithmVariant : public VRoot
1080{
1081public:
1082 virtual int64_t getImplementation() const noexcept = 0;
1083 virtual int64_t getTactic() const noexcept = 0;
1084};
1085
1086class VAlgorithmContext : public VRoot
1087{
1088public:
1089 virtual char const* getName() const noexcept = 0;
1090 virtual Dims getDimensions(int32_t index, OptProfileSelector select) const noexcept = 0;
1091 virtual int32_t getNbInputs() const noexcept = 0;
1092 virtual int32_t getNbOutputs() const noexcept = 0;
1093};
1094
1095class VAlgorithm : public VRoot
1096{
1097public:
1098 virtual IAlgorithmVariant const& getAlgorithmVariant() const noexcept = 0;
1099 virtual float getTimingMSec() const noexcept = 0;
1100 virtual std::size_t getWorkspaceSize() const noexcept = 0;
1101 virtual IAlgorithmIOInfo const* getAlgorithmIOInfoByIndex(int32_t index) const noexcept = 0;
1102};
1103
1104class VTimingCache : public VRoot
1105{
1106public:
1107 virtual nvinfer1::IHostMemory* serialize() const noexcept = 0;
1108 virtual bool combine(ITimingCache const& inputCache, bool ignoreMismatch) noexcept = 0;
1109 virtual bool reset() noexcept = 0;
1110};
1111
1112class VBuilderConfig : public VRoot
1113{
1114public:
1115 virtual void setAvgTimingIterations(int32_t avgTiming) noexcept = 0;
1116 virtual int32_t getAvgTimingIterations() const noexcept = 0;
1117 virtual void setEngineCapability(EngineCapability capability) noexcept = 0;
1118 virtual EngineCapability getEngineCapability() const noexcept = 0;
1119 virtual void setInt8Calibrator(IInt8Calibrator* calibrator) noexcept = 0;
1120 virtual IInt8Calibrator* getInt8Calibrator() const noexcept = 0;
1121 virtual void setFlags(BuilderFlags builderFlags) noexcept = 0;
1122 virtual BuilderFlags getFlags() const noexcept = 0;
1123 virtual void clearFlag(BuilderFlag builderFlag) noexcept = 0;
1124 virtual void setFlag(BuilderFlag builderFlag) noexcept = 0;
1125 virtual bool getFlag(BuilderFlag builderFlag) const noexcept = 0;
1126 virtual void setDeviceType(ILayer const* layer, DeviceType deviceType) noexcept = 0;
1127 virtual DeviceType getDeviceType(ILayer const* layer) const noexcept = 0;
1128 virtual bool isDeviceTypeSet(ILayer const* layer) const noexcept = 0;
1129 virtual void resetDeviceType(ILayer const* layer) noexcept = 0;
1130 virtual bool canRunOnDLA(ILayer const* layer) const noexcept = 0;
1131 virtual void setDLACore(int32_t dlaCore) noexcept = 0;
1132 virtual int32_t getDLACore() const noexcept = 0;
1133 virtual void setDefaultDeviceType(DeviceType deviceType) noexcept = 0;
1134 virtual DeviceType getDefaultDeviceType() const noexcept = 0;
1135 virtual void reset() noexcept = 0;
1136 virtual void setProfileStream(const cudaStream_t stream) noexcept = 0;
1137 virtual cudaStream_t getProfileStream() const noexcept = 0;
1138 virtual int32_t addOptimizationProfile(IOptimizationProfile const* profile) noexcept = 0;
1139 virtual int32_t getNbOptimizationProfiles() const noexcept = 0;
1140 virtual void setProfilingVerbosity(ProfilingVerbosity verbosity) noexcept = 0;
1141 virtual ProfilingVerbosity getProfilingVerbosity() const noexcept = 0;
1142 virtual void setAlgorithmSelector(IAlgorithmSelector* selector) noexcept = 0;
1143 virtual IAlgorithmSelector* getAlgorithmSelector() const noexcept = 0;
1144 virtual bool setCalibrationProfile(IOptimizationProfile const* profile) noexcept = 0;
1145 virtual IOptimizationProfile const* getCalibrationProfile() noexcept = 0;
1146 virtual void setQuantizationFlags(QuantizationFlags flags) noexcept = 0;
1147 virtual QuantizationFlags getQuantizationFlags() const noexcept = 0;
1148 virtual void clearQuantizationFlag(QuantizationFlag flag) noexcept = 0;
1149 virtual void setQuantizationFlag(QuantizationFlag flag) noexcept = 0;
1150 virtual bool getQuantizationFlag(QuantizationFlag flag) const noexcept = 0;
1151 virtual bool setTacticSources(TacticSources tacticSources) noexcept = 0;
1152 virtual TacticSources getTacticSources() const noexcept = 0;
1153 virtual nvinfer1::ITimingCache* createTimingCache(void const* blob, std::size_t size) const noexcept = 0;
1154 virtual bool setTimingCache(ITimingCache const& cache, bool ignoreMismatch) noexcept = 0;
1155 virtual nvinfer1::ITimingCache const* getTimingCache() const noexcept = 0;
1156 virtual void setMemoryPoolLimit(MemoryPoolType pool, std::size_t poolSize) noexcept = 0;
1157 virtual std::size_t getMemoryPoolLimit(MemoryPoolType pool) const noexcept = 0;
1158 virtual void setPreviewFeature(PreviewFeature feature, bool enable) noexcept = 0;
1159 virtual bool getPreviewFeature(PreviewFeature feature) const noexcept = 0;
1160 virtual void setBuilderOptimizationLevel(int32_t level) noexcept = 0;
1161 virtual int32_t getBuilderOptimizationLevel() const noexcept = 0;
1162 virtual void setHardwareCompatibilityLevel(HardwareCompatibilityLevel hardwareCompatibilityLevel) noexcept = 0;
1163 virtual HardwareCompatibilityLevel getHardwareCompatibilityLevel() const noexcept = 0;
1164 virtual void setPluginsToSerialize(char const* const* paths, int32_t nbPaths) noexcept = 0;
1165 virtual char const* getPluginToSerialize(int32_t index) const noexcept = 0;
1166 virtual int32_t getNbPluginsToSerialize() const noexcept = 0;
1167 virtual void setMaxAuxStreams(int32_t nbStreams) noexcept = 0;
1168 virtual int32_t getMaxAuxStreams() const noexcept = 0;
1169 virtual void setProgressMonitor(IProgressMonitor* monitor) noexcept = 0;
1170 virtual IProgressMonitor* getProgressMonitor() const noexcept = 0;
1171 virtual void setRuntimePlatform(RuntimePlatform runtimePlatform) noexcept = 0;
1172 virtual RuntimePlatform getRuntimePlatform() const noexcept = 0;
1173 virtual void setMaxNbTactics(int32_t maxTactics) noexcept = 0;
1174 virtual int32_t getMaxNbTactics() const noexcept = 0;
1175};
1176
1177class VSerializationConfig : public VRoot
1178{
1179public:
1180 virtual bool setFlags(SerializationFlags serializationFlags) noexcept = 0;
1181 virtual SerializationFlags getFlags() const noexcept = 0;
1182 virtual bool clearFlag(SerializationFlag serializationFlag) noexcept = 0;
1183 virtual bool setFlag(SerializationFlag serializationFlag) noexcept = 0;
1184 virtual bool getFlag(SerializationFlag serializationFlag) const noexcept = 0;
1185};
1186
1187class VBuilder : public VRoot
1188{
1189public:
1190 virtual bool platformHasFastFp16() const noexcept = 0;
1191 virtual bool platformHasFastInt8() const noexcept = 0;
1192 virtual int32_t getMaxDLABatchSize() const noexcept = 0;
1193 virtual int32_t getNbDLACores() const noexcept = 0;
1194 virtual void setGpuAllocator(IGpuAllocator* allocator) noexcept = 0;
1195 virtual nvinfer1::IBuilderConfig* createBuilderConfig() noexcept = 0;
1196 virtual nvinfer1::INetworkDefinition* createNetworkV2(NetworkDefinitionCreationFlags flags) noexcept = 0;
1197 virtual nvinfer1::IOptimizationProfile* createOptimizationProfile() noexcept = 0;
1198 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
1199 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
1200 virtual void reset() noexcept = 0;
1201 virtual bool platformHasTf32() const noexcept = 0;
1202 virtual nvinfer1::IHostMemory* buildSerializedNetwork(INetworkDefinition& network, IBuilderConfig& config) noexcept
1203 = 0;
1204 virtual bool isNetworkSupported(INetworkDefinition const& network, IBuilderConfig const& config) const noexcept = 0;
1205 virtual ILogger* getLogger() const noexcept = 0;
1206 virtual bool setMaxThreads(int32_t maxThreads) noexcept = 0;
1207 virtual int32_t getMaxThreads() const noexcept = 0;
1208 virtual IPluginRegistry& getPluginRegistry() noexcept = 0;
1209};
1210
1211} // namespace apiv
1212} // namespace nvinfer1
1213
1214// @endcond
1215
1216#endif // NV_INFER_RUNTIME_IMPL_H
getLogger
nvinfer1::ILogger * getLogger() noexcept
Return the logger object.
getPluginRegistry
nvinfer1::IPluginRegistry * getPluginRegistry() noexcept
Return the plugin registry.
Dims
Structure to define the dimensions of a tensor.
IAlgorithmSelector
Interface implemented by application for selecting and reporting algorithms of a layer provided by th...
IDebugListener
User-implemented callback for notification when value of a debug tensor is updated.
IErrorRecorder
Reference counted application-implemented error reporting interface for TensorRT objects.
IGpuAllocator
Application-implemented class for controlling allocation on the GPU.
IOutputAllocator
Callback from ExecutionContext::enqueueV3()
IPluginV3
Plugin class for the V3 generation of user-implemented layers.
IProfiler
Application-implemented interface for profiling.
IProgressMonitor
Application-implemented progress reporting interface for TensorRT.
IStreamReader
Application-implemented class for reading data in a stream-based manner.
nvinfer1::IHostMemory
Class to handle library allocated memory that is accessible to the user.
Definition: NvInferRuntime.h:139
nvinfer1
The TensorRT API version 1 namespace.
nvinfer1::TacticSources
uint32_t TacticSources
Represents a collection of one or more TacticSource values combine using bitwise-OR operations.
Definition: NvInferRuntime.h:2377
nvinfer1::IOutputAllocator
v_1_0::IOutputAllocator IOutputAllocator
Definition: NvInferRuntime.h:3512
nvinfer1::ResizeSelector
ResizeSelector
The coordinate selector when resize to single pixel output.
Definition: NvInfer.h:3997
nvinfer1::EngineCapability
EngineCapability
List of supported engine capability flows.
Definition: NvInferRuntime.h:73
nvinfer1::DimensionOperation
DimensionOperation
An operation on two IDimensionExpr, which represent integer expressions used in dimension computation...
Definition: NvInferRuntime.h:176
nvinfer1::MemoryPoolType
MemoryPoolType
The type for memory pools used by TensorRT.
Definition: NvInfer.h:8528
nvinfer1::ScaleMode
ScaleMode
Controls how shift, scale and power are applied in a Scale layer.
Definition: NvInfer.h:1810
nvinfer1::RuntimePlatform
RuntimePlatform
Describes the intended runtime platform (operating system and CPU architecture) for the execution of ...
Definition: NvInfer.h:8263
nvinfer1::QuantizationFlags
uint32_t QuantizationFlags
Represents one or more QuantizationFlag values using binary OR operations.
Definition: NvInfer.h:8215
nvinfer1::TensorIOMode
TensorIOMode
Definition of tensor IO Mode.
Definition: NvInferRuntimeBase.h:1100
nvinfer1::HardwareCompatibilityLevel
HardwareCompatibilityLevel
Describes requirements of compatibility with GPU architectures other than that of the GPU on which th...
Definition: NvInfer.h:8642
nvinfer1::SerializationFlag
SerializationFlag
List of valid flags that the engine can enable when serializing the bytes.
Definition: NvInferRuntime.h:2418
nvinfer1::BoundingBoxFormat
BoundingBoxFormat
Representation of bounding box data used for the Boxes input tensor in INMSLayer.
Definition: NvInfer.h:5920
nvinfer1::IPluginV3
v_1_0::IPluginV3 IPluginV3
Definition: NvInferRuntime.h:736
nvinfer1::UnaryOperation
UnaryOperation
Enumerates the unary operations that may be performed by a Unary layer.
Definition: NvInfer.h:2755
nvinfer1::IProfiler
v_1_0::IProfiler IProfiler
Definition: NvInferRuntime.h:1298
nvinfer1::IAlgorithmSelector
v_1_0::IAlgorithmSelector IAlgorithmSelector
Definition: NvInfer.h:8207
nvinfer1::ActivationType
ActivationType
Enumerates the types of activation to perform in an activation layer.
Definition: NvInfer.h:133
nvinfer1::TempfileControlFlags
uint32_t TempfileControlFlags
Represents a collection of one or more TempfileControlFlag values combined using bitwise-OR operation...
Definition: NvInferRuntime.h:1376
nvinfer1::FillOperation
FillOperation
Enumerates the tensor fill operations that may performed by a fill layer.
Definition: NvInfer.h:4972
nvinfer1::ResizeRoundMode
ResizeRoundMode
The rounding mode for nearest neighbor resize.
Definition: NvInfer.h:4027
nvinfer1::PaddingMode
PaddingMode
Enumerates the modes of padding to perform in convolution, deconvolution and pooling layer,...
Definition: NvInfer.h:988
nvinfer1::TripLimit
TripLimit
Definition: NvInfer.h:4382
nvinfer1::NetworkDefinitionCreationFlags
uint32_t NetworkDefinitionCreationFlags
Represents one or more NetworkDefinitionCreationFlag flags using binary OR operations....
Definition: NvInfer.h:9628
nvinfer1::PreviewFeature
PreviewFeature
Define preview features.
Definition: NvInfer.h:8603
nvinfer1::DataType
DataType
The type of weights and tensors.
Definition: NvInferRuntimeBase.h:135
nvinfer1::BuilderFlags
uint32_t BuilderFlags
Represents one or more BuilderFlag values using binary OR operations, e.g., 1U << BuilderFlag::kFP16 ...
Definition: NvInfer.h:8293
nvinfer1::DeviceType
DeviceType
The device that this layer/network will execute on.
Definition: NvInferRuntime.h:1330
nvinfer1::CalibrationAlgoType
CalibrationAlgoType
Version of calibration algorithm to use.
Definition: NvInfer.h:7651
nvinfer1::LayerType
LayerType
The type values of layer classes.
Definition: NvInfer.h:58
nvinfer1::SampleMode
SampleMode
Controls how ISliceLayer and IGridSample handle out-of-bounds coordinates.
Definition: NvInfer.h:3213
nvinfer1::GatherMode
GatherMode
Control form of IGatherLayer.
Definition: NvInfer.h:2504
nvinfer1::IDebugListener
v_1_0::IDebugListener IDebugListener
Definition: NvInferRuntime.h:3553
nvinfer1::WeightsRole
WeightsRole
How a layer uses particular Weights.
Definition: NvInferRuntime.h:1308
nvinfer1::TensorFormats
uint32_t TensorFormats
It is capable of representing one or more TensorFormat by binary OR operations, e....
Definition: NvInfer.h:125
nvinfer1::ProfilingVerbosity
ProfilingVerbosity
List of verbosity levels of layer information exposed in NVTX annotations and in IEngineInspector.
Definition: NvInferRuntime.h:2389
nvinfer1::NetworkDefinitionCreationFlag
NetworkDefinitionCreationFlag
List of immutable network properties expressed at network creation time. NetworkDefinitionCreationFla...
Definition: NvInfer.h:9639
nvinfer1::ElementWiseOperation
ElementWiseOperation
Enumerates the binary operations that may be performed by an ElementWise layer.
Definition: NvInfer.h:2414
nvinfer1::QuantizationFlag
QuantizationFlag
List of valid flags for quantizing the network to int8.
Definition: NvInfer.h:8227
nvinfer1::SerializationFlags
uint32_t SerializationFlags
Represents one or more SerializationFlag values using binary OR operations, e.g., 1U << Serialization...
Definition: NvInferRuntime.h:2408
nvinfer1::InterpolationMode
InterpolationMode
Enumerates various modes of interpolation.
Definition: NvInfer.h:3915
nvinfer1::BuilderFlag
BuilderFlag
List of valid modes that the builder can enable when creating an engine from a network definition.
Definition: NvInfer.h:8303
nvinfer1::TensorFormat
TensorFormat
Format of the input/output tensors.
Definition: NvInferRuntimeBase.h:250
nvinfer1::ExecutionContextAllocationStrategy
ExecutionContextAllocationStrategy
Different memory allocation behaviors for IExecutionContext.
Definition: NvInferRuntime.h:2522
nvinfer1::TopKOperation
TopKOperation
Enumerates the operations that may be performed by a TopK layer.
Definition: NvInfer.h:3494
nvinfer1::ReduceOperation
ReduceOperation
Enumerates the reduce operations that may be performed by a Reduce layer.
Definition: NvInfer.h:2850
nvinfer1::LayerInformationFormat
LayerInformationFormat
The format in which the IEngineInspector prints the layer information.
Definition: NvInferRuntime.h:4445
nvinfer1::ScatterMode
ScatterMode
Control form of IScatterLayer.
Definition: NvInfer.h:5655
nvinfer1::MatrixOperation
MatrixOperation
Enumerates the operations that may be performed on a tensor by IMatrixMultiplyLayer before multiplica...
Definition: NvInfer.h:3620
nvinfer1::ResizeCoordinateTransformation
ResizeCoordinateTransformation
The resize coordinate transformation function.
Definition: NvInfer.h:3943
nvinfer1::LoopOutput
LoopOutput
Definition: NvInfer.h:4356
nvinfer1::PoolingType
PoolingType
The type of pooling to perform in a pooling layer.
Definition: NvInfer.h:1424
nvinfer1::IProgressMonitor
v_1_0::IProgressMonitor IProgressMonitor
Definition: NvInfer.h:8759
nvinfer1::TensorLocation
TensorLocation
The location for tensor data storage, device or host.
Definition: NvInferRuntime.h:201
nvinfer1::OptProfileSelector
OptProfileSelector
When setting or querying optimization profile parameters (such as shape tensor inputs or dynamic dime...
Definition: NvInferRuntime.h:2121

  Copyright © 2024 NVIDIA Corporation
  Privacy Policy | Manage My Privacy | Do Not Sell or Share My Data | Terms of Service | Accessibility | Corporate Policies | Product Security | Contact