TensorRT 10.12.0
NvInferImpl.h
Go to the documentation of this file.
1/*
2 * SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: Apache-2.0
4 *
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
8 *
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 */
17
18#ifndef NV_INFER_IMPL_H
19#define NV_INFER_IMPL_H
20
21#include "NvInferLegacyDims.h"
22#include "NvInferRuntimeCommon.h"
23
24// @cond SuppressDoxyWarnings
25
26namespace nvinfer1
27{
28
29class ILogger;
30
31namespace v_1_0
32{
33class IProgressMonitor;
34}
35using IProgressMonitor = v_1_0::IProgressMonitor;
36
37namespace v_1_0
38{
39class IAlgorithmSelector;
40}
41using IAlgorithmSelector = v_1_0::IAlgorithmSelector;
42
43namespace v_1_0
44{
45class IProfiler;
46}
47using IProfiler = v_1_0::IProfiler;
48
49namespace v_1_0
50{
51class IOutputAllocator;
52}
53using IOutputAllocator = v_1_0::IOutputAllocator;
54
55namespace v_1_0
56{
57class IDebugListener;
58}
59using IDebugListener = v_1_0::IDebugListener;
60
61class IActivationLayer;
62class IAlgorithm;
63class IAlgorithmContext;
64class IAlgorithmIOInfo;
65class IAlgorithmVariant;
66class IAssertionLayer;
67class IBuilder;
68class IBuilderConfig;
69class IConcatenationLayer;
70class IConditionLayer;
71class IConstantLayer;
72class IConvolutionLayer;
73class ICudaEngine;
74class ICumulativeLayer;
75class IDeconvolutionLayer;
76class IDequantizeLayer;
77class IDimensionExpr;
78class IDynamicQuantizeLayer;
79class IEinsumLayer;
80class IElementWiseLayer;
81class IEngineInspector;
82class IExecutionContext;
83class IFillLayer;
84class IGatherLayer;
85class IGridSampleLayer;
86class IHostMemory;
87class IIdentityLayer;
88class ICastLayer;
89class IIfConditional;
90class IIfConditionalInputLayer;
91class IIfConditionalOutputLayer;
92class IInt8Calibrator;
93class IIteratorLayer;
94class ILayer;
95class ILoop;
96class ILoopOutputLayer;
97class ILRNLayer;
98class IMatrixMultiplyLayer;
99class INetworkDefinition;
100class INormalizationLayer;
101class INMSLayer;
102class INonZeroLayer;
103class IOneHotLayer;
104class IOptimizationProfile;
105class IPaddingLayer;
106class IParametricReLULayer;
107class IPlugin;
108class IPluginExt;
109class IPluginFactory;
110class IPluginLayer;
111class IPluginRegistry;
112class IPluginV2Layer;
113class IRuntimeConfig;
114
115namespace v_1_0
116{
117class IPluginV3;
118} // namespace v_1_0
119using IPluginV3 = v_1_0::IPluginV3;
120
121namespace v_1_0
122{
123class IStreamReader;
124} // namespace v_1_0
125using IStreamReader = v_1_0::IStreamReader;
126namespace v_1_0
127{
128class IStreamReaderV2;
129} // namespace v_1_0
130using IStreamReaderV2 = v_1_0::IStreamReaderV2;
131
132class IPluginV3Layer;
133class IPoolingLayer;
134class IQuantizeLayer;
135class IRaggedSoftMaxLayer;
136class IRecurrenceLayer;
137class IReduceLayer;
138class IRefitter;
139class IResizeLayer;
140class IReverseSequenceLayer;
141class IRuntime;
142class IScaleLayer;
143class IScatterLayer;
144class ISelectLayer;
145class ISerializationConfig;
146class IShapeLayer;
147class IShuffleLayer;
148class ISliceLayer;
149class ISoftMaxLayer;
150class ISqueezeLayer;
151class ITensor;
152
153namespace v_1_0
154{
155struct TimingCacheKey;
156struct TimingCacheValue;
157} // namespace v_1_0
158using TimingCacheKey = v_1_0::TimingCacheKey;
159using TimingCacheValue = v_1_0::TimingCacheValue;
160
161class ITimingCache;
162class ITopKLayer;
163class ITripLimitLayer;
164class IUnaryLayer;
165class IUnsqueezeLayer;
166struct Permutation;
167class Weights;
168
169enum class ActivationType : int32_t;
170enum class BoundingBoxFormat : int32_t;
171enum class BuilderFlag : int32_t;
172enum class CalibrationAlgoType : int32_t;
173enum class CumulativeOperation : int32_t;
174enum class DeviceType : int32_t;
175enum class DimensionOperation : int32_t;
176enum class ElementWiseOperation : int32_t;
177enum class EngineCapability : int32_t;
178enum class FillOperation : int32_t;
179enum class GatherMode : int32_t;
180enum class LayerInformationFormat : int32_t;
181enum class LayerType : int32_t;
182enum class LoopOutput : int32_t;
183enum class MatrixOperation : int32_t;
184enum class MemoryPoolType : int32_t;
185enum class NetworkDefinitionCreationFlag : int32_t;
186enum class OptProfileSelector : int32_t;
187enum class PaddingMode : int32_t;
188enum class PoolingType : int32_t;
189enum class ProfilingVerbosity : int32_t;
190enum class QuantizationFlag : int32_t;
191enum class ReduceOperation : int32_t;
192enum class ResizeCoordinateTransformation : int32_t;
193enum class InterpolationMode : int32_t;
194enum class ResizeRoundMode : int32_t;
195enum class ResizeSelector : int32_t;
196enum class ScaleMode : int32_t;
197enum class ScatterMode : int32_t;
198enum class SampleMode : int32_t;
199enum class SerializationFlag : int32_t;
200enum class TensorIOMode : int32_t;
201enum class TensorLocation : int32_t;
202enum class TopKOperation : int32_t;
203enum class TripLimit : int32_t;
204enum class UnaryOperation : int32_t;
205enum class WeightsRole : int32_t;
206enum class PreviewFeature : int32_t;
207enum class HardwareCompatibilityLevel : int32_t;
208enum class ExecutionContextAllocationStrategy : int32_t;
209enum class RuntimePlatform : int32_t;
210enum class TilingOptimizationLevel : int32_t;
211
212
213using TacticSources = uint32_t;
214using TensorFormats = uint32_t;
215using BuilderFlags = uint32_t;
216using NetworkDefinitionCreationFlags = uint32_t;
217using QuantizationFlags = uint32_t;
218using TempfileControlFlags = uint32_t;
219using SerializationFlags = uint32_t;
220
228
229namespace apiv
230{
231
232class VRoot
233{
234public:
235 virtual ~VRoot() noexcept = default;
236};
237
238class VHostMemory : public VRoot
239{
240public:
241 virtual void* data() const noexcept = 0;
242 virtual std::size_t size() const noexcept = 0;
243 virtual DataType type() const noexcept = 0;
244};
245
246class VDimensionExpr : public VRoot
247{
248public:
249 virtual bool isConstant() const = 0;
250 virtual int64_t getConstantValue() const = 0;
251 virtual bool isSizeTensor() const = 0;
252};
253
254class VExprBuilder : public VRoot
255{
256public:
257 virtual IDimensionExpr const* constant(int64_t value) = 0;
258 virtual IDimensionExpr const* operation(
259 DimensionOperation op, IDimensionExpr const& first, IDimensionExpr const& second)
260 = 0;
261 virtual IDimensionExpr const* declareSizeTensor(
262 int32_t outputIndex, IDimensionExpr const& opt, IDimensionExpr const& upper)
263 = 0;
264};
265
266class VRuntime : public VRoot
267{
268public:
269 virtual IRuntime* getPImpl() noexcept = 0;
270 virtual nvinfer1::ICudaEngine* deserializeCudaEngine(void const* blob, std::size_t size) noexcept = 0;
271 virtual nvinfer1::ICudaEngine* deserializeCudaEngine(IStreamReader& streamReader) noexcept = 0;
272 virtual void setDLACore(int32_t dlaCore) noexcept = 0;
273 virtual int32_t getDLACore() const noexcept = 0;
274 virtual int32_t getNbDLACores() const noexcept = 0;
275 virtual void setGpuAllocator(IGpuAllocator* allocator) noexcept = 0;
276 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
277 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
278 virtual ILogger* getLogger() const noexcept = 0;
279 virtual bool setMaxThreads(int32_t maxThreads) noexcept = 0;
280 virtual int32_t getMaxThreads() const noexcept = 0;
281 virtual void setTemporaryDirectory(char const*) noexcept = 0;
282 virtual char const* getTemporaryDirectory() const noexcept = 0;
283 virtual void setTempfileControlFlags(TempfileControlFlags) noexcept = 0;
284 virtual TempfileControlFlags getTempfileControlFlags() const noexcept = 0;
285 virtual IPluginRegistry& getPluginRegistry() noexcept = 0;
286 virtual void setPluginRegistryParent(IPluginRegistry* parent) noexcept = 0;
287 virtual IRuntime* loadRuntime(char const* path) noexcept = 0;
288 virtual void setEngineHostCodeAllowed(bool allowed) noexcept = 0;
289 virtual bool getEngineHostCodeAllowed() const noexcept = 0;
290 // Added in TensorRT version 10.7
291 virtual nvinfer1::ICudaEngine* deserializeCudaEngineV2(IStreamReaderV2& streamReader) noexcept = 0;
292};
293
294class VRefitter : public VRoot
295{
296public:
297 virtual IRefitter* getPImpl() noexcept = 0;
298 virtual bool setWeights(char const* layerName, WeightsRole role, const Weights weights) noexcept = 0;
299 virtual bool refitCudaEngine() noexcept = 0;
300 virtual int32_t getMissing(int32_t size, char const** layerNames, WeightsRole* roles) noexcept = 0;
301 virtual int32_t getAll(int32_t size, char const** layerNames, WeightsRole* roles) noexcept = 0;
302 virtual bool setDynamicRange(char const* tensorName, float min, float max) noexcept = 0;
303 virtual float getDynamicRangeMin(char const* tensorName) const noexcept = 0;
304 virtual float getDynamicRangeMax(char const* tensorName) const noexcept = 0;
305 virtual int32_t getTensorsWithDynamicRange(int32_t size, char const** tensorNames) const noexcept = 0;
306 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
307 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
308 virtual bool setNamedWeights(char const* name, Weights weights) noexcept = 0;
309 virtual int32_t getMissingWeights(int32_t size, char const** weightsNames) noexcept = 0;
310 virtual int32_t getAllWeights(int32_t size, char const** weightsNames) noexcept = 0;
311 virtual ILogger* getLogger() const noexcept = 0;
312 virtual bool setMaxThreads(int32_t maxThreads) noexcept = 0;
313 virtual int32_t getMaxThreads() const noexcept = 0;
314 virtual bool setNamedWeightsWithLocation(char const* name, Weights weights, TensorLocation location) noexcept = 0;
315 virtual Weights getNamedWeights(char const* weightsName) const noexcept = 0;
316 virtual TensorLocation getWeightsLocation(char const* weightsName) const noexcept = 0;
317 virtual bool unsetNamedWeights(char const* weightsName) noexcept = 0;
318 virtual void setWeightsValidation(bool weightsValidation) noexcept = 0;
319 virtual bool getWeightsValidation() const noexcept = 0;
320 virtual bool refitCudaEngineAsync(cudaStream_t stream) noexcept = 0;
321 virtual Weights getWeightsPrototype(char const* weightsName) const noexcept = 0;
322};
323
324class VOptimizationProfile : public VRoot
325{
326public:
327 virtual bool setDimensions(char const* inputName, OptProfileSelector select, Dims const& dims) noexcept = 0;
328 virtual Dims getDimensions(char const* inputName, OptProfileSelector select) const noexcept = 0;
329 virtual bool setShapeValues(
330 char const* inputName, OptProfileSelector select, int32_t const* values, int32_t nbValues) noexcept = 0;
331 virtual int32_t getNbShapeValues(char const* inputName) const noexcept = 0;
332 virtual int32_t const* getShapeValues(char const* inputName, OptProfileSelector select) const noexcept = 0;
333 virtual bool setExtraMemoryTarget(float target) noexcept = 0;
334 virtual float getExtraMemoryTarget() const noexcept = 0;
335 virtual bool isValid() const noexcept = 0;
336 // Added in TensorRT 10.11
337 TRT_NODISCARD virtual bool setShapeValuesV2(
338 char const* inputName, OptProfileSelector select, int64_t const* values, int32_t nbValues) noexcept = 0;
339 TRT_NODISCARD virtual int64_t const* getShapeValuesV2(
340 char const* inputName, OptProfileSelector select) const noexcept = 0;
341};
342
343class VCudaEngine : public VRoot
344{
345public:
346 virtual ICudaEngine* getPImpl() noexcept = 0;
347 virtual int32_t getNbLayers() const noexcept = 0;
348 virtual IHostMemory* serialize() const noexcept = 0;
349 virtual IExecutionContext* createExecutionContext(ExecutionContextAllocationStrategy strategy) noexcept = 0;
350 virtual IExecutionContext* createExecutionContextWithoutDeviceMemory() noexcept = 0;
351 virtual size_t getDeviceMemorySize() const noexcept = 0;
352 virtual bool isRefittable() const noexcept = 0;
353 virtual char const* getName() const noexcept = 0;
354 virtual int32_t getNbOptimizationProfiles() const noexcept = 0;
355 virtual int32_t const* getProfileTensorValues(
356 char const* tensorName, int32_t profileIndex, OptProfileSelector select) const noexcept = 0;
357 virtual EngineCapability getEngineCapability() const noexcept = 0;
358 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
359 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
360 virtual bool hasImplicitBatchDimension() const noexcept = 0;
361 virtual TacticSources getTacticSources() const noexcept = 0;
362 virtual ProfilingVerbosity getProfilingVerbosity() const noexcept = 0;
363 virtual IEngineInspector* createEngineInspector() const noexcept = 0;
364 virtual Dims getTensorShape(char const* tensorName) const noexcept = 0;
365 virtual DataType getTensorDataType(char const* tensorName) const noexcept = 0;
366 virtual TensorLocation getTensorLocation(char const* tensorName) const noexcept = 0;
367 virtual bool isShapeInferenceIO(char const* tensorName) const noexcept = 0;
368 virtual TensorIOMode getTensorIOMode(char const* tensorName) const noexcept = 0;
369 virtual int32_t getTensorBytesPerComponent(char const* tensorName) const noexcept = 0;
370 virtual int32_t getTensorComponentsPerElement(char const* tensorName) const noexcept = 0;
371 virtual TensorFormat getTensorFormat(char const* tensorName) const noexcept = 0;
372 virtual char const* getTensorFormatDesc(char const* tensorName) const noexcept = 0;
373 virtual int32_t getTensorVectorizedDim(char const* tensorName) const noexcept = 0;
374 virtual Dims getProfileShape(
375 char const* tensorName, int32_t profileIndex, OptProfileSelector select) const noexcept = 0;
376 virtual int32_t getNbIOTensors() const noexcept = 0;
377 virtual char const* getIOTensorName(int32_t index) const noexcept = 0;
378 virtual HardwareCompatibilityLevel getHardwareCompatibilityLevel() const noexcept = 0;
379 virtual int32_t getNbAuxStreams() const noexcept = 0;
380
381 virtual int32_t getTensorBytesPerComponentV2(char const* tensorName, int32_t profileIndex) const noexcept = 0;
382 virtual int32_t getTensorComponentsPerElementV2(char const* tensorName, int32_t profileIndex) const noexcept = 0;
383 virtual TensorFormat getTensorFormatV2(char const* tensorName, int32_t profileIndex) const noexcept = 0;
384 virtual char const* getTensorFormatDescV2(char const* tensorName, int32_t profileIndex) const noexcept = 0;
385 virtual int32_t getTensorVectorizedDimV2(char const* tensorName, int32_t profileIndex) const noexcept = 0;
386
387 virtual ISerializationConfig* createSerializationConfig() noexcept = 0;
388 virtual IHostMemory* serializeWithConfig(ISerializationConfig& config) const noexcept = 0;
389
390 virtual size_t getDeviceMemorySizeForProfile(int32_t profileIndex) const noexcept = 0;
391 virtual IRefitter* createRefitter(ILogger& logger) noexcept = 0;
392
393 virtual bool setWeightStreamingBudget(int64_t gpuMemoryBudget) noexcept = 0;
394 virtual int64_t getWeightStreamingBudget() const noexcept = 0;
395 virtual int64_t getMinimumWeightStreamingBudget() const noexcept = 0;
396 virtual int64_t getStreamableWeightsSize() const noexcept = 0;
397
398 virtual bool isDebugTensor(char const* name) const noexcept = 0;
399
400 // Added in TensorRT 10.1
401 virtual bool setWeightStreamingBudgetV2(int64_t gpuMemoryBudget) noexcept = 0;
402 virtual int64_t getWeightStreamingBudgetV2() const noexcept = 0;
403 virtual int64_t getWeightStreamingAutomaticBudget() const noexcept = 0;
404 virtual int64_t getWeightStreamingScratchMemorySize() const noexcept = 0;
405 virtual int64_t getDeviceMemorySizeV2() const noexcept = 0;
406 virtual int64_t getDeviceMemorySizeForProfileV2(int32_t profileIndex) const noexcept = 0;
407 // Added in TensorRT 10.11
408 TRT_NODISCARD virtual int64_t const* getProfileTensorValuesV2(
409 char const* tensorName, int32_t profileIndex, OptProfileSelector select) const noexcept = 0;
410 TRT_NODISCARD virtual IExecutionContext* createExecutionContextWithRuntimeConfig(
411 IRuntimeConfig* runtimeConfig) noexcept = 0;
412 TRT_NODISCARD virtual IRuntimeConfig* createRuntimeConfig() noexcept = 0;
413};
414
415class VExecutionContext : public VRoot
416{
417public:
418 virtual IExecutionContext* getPImpl() noexcept = 0;
419 virtual void setDebugSync(bool sync) noexcept = 0;
420 virtual bool getDebugSync() const noexcept = 0;
421 virtual void setProfiler(IProfiler*) noexcept = 0;
422 virtual IProfiler* getProfiler() const noexcept = 0;
423 virtual ICudaEngine const& getEngine() const noexcept = 0;
424 virtual void setName(char const* name) noexcept = 0;
425 virtual char const* getName() const noexcept = 0;
426 virtual void setDeviceMemory(void* memory) noexcept = 0;
427 virtual int32_t getOptimizationProfile() const noexcept = 0;
428 virtual bool allInputDimensionsSpecified() const noexcept = 0;
429 virtual bool allInputShapesSpecified() const noexcept = 0;
430 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
431 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
432 virtual bool executeV2(void* const* bindings) noexcept = 0;
433 virtual bool setOptimizationProfileAsync(int32_t profileIndex, cudaStream_t stream) noexcept = 0;
434 virtual void setEnqueueEmitsProfile(bool enqueueEmitsProfile) noexcept = 0;
435 virtual bool getEnqueueEmitsProfile() const noexcept = 0;
436 virtual bool reportToProfiler() const noexcept = 0;
437 virtual bool setInputShape(char const* tensorName, Dims const& dims) noexcept = 0;
438 virtual Dims getTensorShape(char const* tensorName) const noexcept = 0;
439 virtual Dims getTensorStrides(char const* tensorName) const noexcept = 0;
440 virtual bool setTensorAddress(char const* tensorName, void* data) noexcept = 0;
441 virtual void const* getTensorAddress(char const* tensorName) const noexcept = 0;
442 virtual bool setInputTensorAddress(char const* tensorName, void const* data) noexcept = 0;
443 virtual bool setOutputTensorAddress(char const* tensorName, void* data) noexcept = 0;
444 virtual int32_t inferShapes(int32_t nbMaxNames, char const** tensorNames) noexcept = 0;
445 virtual bool setInputConsumedEvent(cudaEvent_t event) noexcept = 0;
446 virtual cudaEvent_t getInputConsumedEvent() const noexcept = 0;
447 virtual void* getOutputTensorAddress(char const* tensorName) const noexcept = 0;
448 virtual bool setOutputAllocator(char const* tensorName, IOutputAllocator* outputAllocator) noexcept = 0;
449 virtual IOutputAllocator* getOutputAllocator(char const* name) noexcept = 0;
450 virtual int64_t getMaxOutputSize(char const* tensorName) const noexcept = 0;
451 virtual bool setTemporaryStorageAllocator(IGpuAllocator* allocator) noexcept = 0;
452 virtual IGpuAllocator* getTemporaryStorageAllocator() const noexcept = 0;
453 virtual bool enqueueV3(cudaStream_t stream) noexcept = 0;
454 virtual void setPersistentCacheLimit(size_t size) noexcept = 0;
455 virtual size_t getPersistentCacheLimit() const noexcept = 0;
456 virtual bool setNvtxVerbosity(ProfilingVerbosity verbosity) noexcept = 0;
457 virtual ProfilingVerbosity getNvtxVerbosity() const noexcept = 0;
458 virtual void setAuxStreams(cudaStream_t* auxStreams, int32_t nbStreams) noexcept = 0;
459 virtual bool setDebugListener(IDebugListener* listener) noexcept = 0;
460 virtual IDebugListener* getDebugListener() noexcept = 0;
461 virtual bool setTensorDebugState(char const* name, bool flag) noexcept = 0;
462 virtual bool getDebugState(char const* name) const noexcept = 0;
463 virtual bool setAllTensorsDebugState(bool flag) noexcept = 0;
464 virtual size_t updateDeviceMemorySizeForShapes() noexcept = 0;
465 virtual void setDeviceMemoryV2(void* memory, int64_t size) noexcept = 0;
466 TRT_NODISCARD virtual IRuntimeConfig* getRuntimeConfig() const noexcept = 0;
467 virtual bool setUnfusedTensorsDebugState(bool flag) noexcept = 0;
468 virtual bool getUnfusedTensorsDebugState() const noexcept = 0;
469};
470
471class VEngineInspector : public VRoot
472{
473public:
474 virtual IEngineInspector* getPImpl() noexcept = 0;
475 virtual bool setExecutionContext(IExecutionContext const* context) noexcept = 0;
476 virtual IExecutionContext const* getExecutionContext() const noexcept = 0;
477 virtual char const* getLayerInformation(int32_t layerIndex, LayerInformationFormat format) const noexcept = 0;
478 virtual char const* getEngineInformation(LayerInformationFormat format) const noexcept = 0;
479 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
480 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
481};
482
483class VTensor : public VRoot
484{
485public:
486 virtual void setName(char const* name) noexcept = 0;
487 virtual char const* getName() const noexcept = 0;
488 virtual void setDimensions(Dims const& dimensions) noexcept = 0;
489 virtual Dims getDimensions() const noexcept = 0;
490 virtual void setType(DataType type) noexcept = 0;
491 virtual DataType getType() const noexcept = 0;
492 virtual bool setDynamicRange(float min, float max) noexcept = 0;
493 virtual bool isNetworkInput() const noexcept = 0;
494 virtual bool isNetworkOutput() const noexcept = 0;
495 virtual void setBroadcastAcrossBatch(bool broadcastAcrossBatch) noexcept = 0;
496 virtual bool getBroadcastAcrossBatch() const noexcept = 0;
497 virtual TensorLocation getLocation() const noexcept = 0;
498 virtual void setLocation(TensorLocation location) noexcept = 0;
499 virtual bool dynamicRangeIsSet() const noexcept = 0;
500 virtual void resetDynamicRange() noexcept = 0;
501 virtual float getDynamicRangeMin() const noexcept = 0;
502 virtual float getDynamicRangeMax() const noexcept = 0;
503 virtual void setAllowedFormats(TensorFormats formats) noexcept = 0;
504 virtual TensorFormats getAllowedFormats() const noexcept = 0;
505 virtual bool isShapeTensor() const noexcept = 0;
506 virtual bool isExecutionTensor() const noexcept = 0;
507 virtual void setDimensionName(int32_t index, char const* name) noexcept = 0;
508 virtual char const* getDimensionName(int32_t index) const noexcept = 0;
509};
510
511class VLayer : public VRoot
512{
513public:
514 virtual LayerType getType() const noexcept = 0;
515 virtual void setName(char const* name) noexcept = 0;
516 virtual char const* getName() const noexcept = 0;
517 virtual int32_t getNbInputs() const noexcept = 0;
518 virtual ITensor* getInput(int32_t index) const noexcept = 0;
519 virtual int32_t getNbOutputs() const noexcept = 0;
520 virtual ITensor* getOutput(int32_t index) const noexcept = 0;
521 virtual void setInput(int32_t index, ITensor& tensor) noexcept = 0;
522 virtual void setPrecision(DataType dataType) noexcept = 0;
523 virtual DataType getPrecision() const noexcept = 0;
524 virtual bool precisionIsSet() const noexcept = 0;
525 virtual void resetPrecision() noexcept = 0;
526 virtual void setOutputType(int32_t index, DataType dataType) noexcept = 0;
527 virtual DataType getOutputType(int32_t index) const noexcept = 0;
528 virtual bool outputTypeIsSet(int32_t index) const noexcept = 0;
529 virtual void resetOutputType(int32_t index) noexcept = 0;
530 virtual void setMetadata(char const* docString) noexcept = 0;
531 virtual char const* getMetadata() const noexcept = 0;
532};
533
534class VConvolutionLayer : public VRoot
535{
536public:
537 virtual void setNbOutputMaps(int64_t nbOutputMaps) noexcept = 0;
538 virtual int64_t getNbOutputMaps() const noexcept = 0;
539 virtual void setNbGroups(int64_t nbGroups) noexcept = 0;
540 virtual int64_t getNbGroups() const noexcept = 0;
541 virtual void setKernelWeights(Weights weights) noexcept = 0;
542 virtual Weights getKernelWeights() const noexcept = 0;
543 virtual void setBiasWeights(Weights weights) noexcept = 0;
544 virtual Weights getBiasWeights() const noexcept = 0;
545 virtual void setPrePadding(Dims const& padding) noexcept = 0;
546 virtual Dims getPrePadding() const noexcept = 0;
547 virtual void setPostPadding(Dims const& padding) noexcept = 0;
548 virtual Dims getPostPadding() const noexcept = 0;
549 virtual void setPaddingMode(PaddingMode paddingMode) noexcept = 0;
550 virtual PaddingMode getPaddingMode() const noexcept = 0;
551 virtual void setKernelSizeNd(Dims const& kernelSize) noexcept = 0;
552 virtual Dims getKernelSizeNd() const noexcept = 0;
553 virtual void setStrideNd(Dims const& stride) noexcept = 0;
554 virtual Dims getStrideNd() const noexcept = 0;
555 virtual void setPaddingNd(Dims const& padding) noexcept = 0;
556 virtual Dims getPaddingNd() const noexcept = 0;
557 virtual void setDilationNd(Dims const& dilation) noexcept = 0;
558 virtual Dims getDilationNd() const noexcept = 0;
559};
560
561class VActivationLayer : public VRoot
562{
563public:
564 virtual void setActivationType(ActivationType type) noexcept = 0;
565 virtual ActivationType getActivationType() const noexcept = 0;
566 virtual void setAlpha(float alpha) noexcept = 0;
567 virtual void setBeta(float beta) noexcept = 0;
568 virtual float getAlpha() const noexcept = 0;
569 virtual float getBeta() const noexcept = 0;
570};
571
572class VPoolingLayer : public VRoot
573{
574public:
575 virtual void setPoolingType(PoolingType type) noexcept = 0;
576 virtual PoolingType getPoolingType() const noexcept = 0;
577 virtual void setBlendFactor(float blendFactor) noexcept = 0;
578 virtual float getBlendFactor() const noexcept = 0;
579 virtual void setAverageCountExcludesPadding(bool exclusive) noexcept = 0;
580 virtual bool getAverageCountExcludesPadding() const noexcept = 0;
581 virtual void setPrePadding(Dims const& padding) noexcept = 0;
582 virtual Dims getPrePadding() const noexcept = 0;
583 virtual void setPostPadding(Dims const& padding) noexcept = 0;
584 virtual Dims getPostPadding() const noexcept = 0;
585 virtual void setPaddingMode(PaddingMode paddingMode) noexcept = 0;
586 virtual PaddingMode getPaddingMode() const noexcept = 0;
587 virtual void setWindowSizeNd(Dims const& windowSize) noexcept = 0;
588 virtual Dims getWindowSizeNd() const noexcept = 0;
589 virtual void setStrideNd(Dims const& stride) noexcept = 0;
590 virtual Dims getStrideNd() const noexcept = 0;
591 virtual void setPaddingNd(Dims const& padding) noexcept = 0;
592 virtual Dims getPaddingNd() const noexcept = 0;
593};
594
595class VLRNLayer : public VRoot
596{
597public:
598 virtual void setWindowSize(int64_t windowSize) noexcept = 0;
599 virtual int64_t getWindowSize() const noexcept = 0;
600 virtual void setAlpha(float alpha) noexcept = 0;
601 virtual float getAlpha() const noexcept = 0;
602 virtual void setBeta(float beta) noexcept = 0;
603 virtual float getBeta() const noexcept = 0;
604 virtual void setK(float k) noexcept = 0;
605 virtual float getK() const noexcept = 0;
606};
607
608class VScaleLayer : public VRoot
609{
610public:
611 virtual void setMode(ScaleMode mode) noexcept = 0;
612 virtual ScaleMode getMode() const noexcept = 0;
613 virtual void setShift(Weights shift) noexcept = 0;
614 virtual Weights getShift() const noexcept = 0;
615 virtual void setScale(Weights scale) noexcept = 0;
616 virtual Weights getScale() const noexcept = 0;
617 virtual void setPower(Weights power) noexcept = 0;
618 virtual Weights getPower() const noexcept = 0;
619 virtual int32_t getChannelAxis() const noexcept = 0;
620 virtual void setChannelAxis(int32_t channelAxis) noexcept = 0;
621};
622
623class VSoftMaxLayer : public VRoot
624{
625public:
626 virtual void setAxes(uint32_t axes) noexcept = 0;
627 virtual uint32_t getAxes() const noexcept = 0;
628};
629
630class VConcatenationLayer : public VRoot
631{
632public:
633 virtual void setAxis(int32_t axis) noexcept = 0;
634 virtual int32_t getAxis() const noexcept = 0;
635};
636
637class VDeconvolutionLayer : public VRoot
638{
639public:
640 virtual void setNbOutputMaps(int64_t nbOutputMaps) noexcept = 0;
641 virtual int64_t getNbOutputMaps() const noexcept = 0;
642 virtual void setNbGroups(int64_t nbGroups) noexcept = 0;
643 virtual int64_t getNbGroups() const noexcept = 0;
644 virtual void setKernelWeights(Weights weights) noexcept = 0;
645 virtual Weights getKernelWeights() const noexcept = 0;
646 virtual void setBiasWeights(Weights weights) noexcept = 0;
647 virtual Weights getBiasWeights() const noexcept = 0;
648 virtual void setPrePadding(Dims const& padding) noexcept = 0;
649 virtual Dims getPrePadding() const noexcept = 0;
650 virtual void setPostPadding(Dims const& padding) noexcept = 0;
651 virtual Dims getPostPadding() const noexcept = 0;
652 virtual void setPaddingMode(PaddingMode paddingMode) noexcept = 0;
653 virtual PaddingMode getPaddingMode() const noexcept = 0;
654 virtual void setKernelSizeNd(Dims const& kernelSize) noexcept = 0;
655 virtual Dims getKernelSizeNd() const noexcept = 0;
656 virtual void setStrideNd(Dims const& stride) noexcept = 0;
657 virtual Dims getStrideNd() const noexcept = 0;
658 virtual void setPaddingNd(Dims const& padding) noexcept = 0;
659 virtual Dims getPaddingNd() const noexcept = 0;
660 virtual void setDilationNd(Dims const& dilation) noexcept = 0;
661 virtual Dims getDilationNd() const noexcept = 0;
662};
663
664class VElementWiseLayer : public VRoot
665{
666public:
667 virtual void setOperation(ElementWiseOperation op) noexcept = 0;
668 virtual ElementWiseOperation getOperation() const noexcept = 0;
669};
670
671class VGatherLayer : public VRoot
672{
673public:
674 virtual void setGatherAxis(int32_t axis) noexcept = 0;
675 virtual int32_t getGatherAxis() const noexcept = 0;
676 virtual void setNbElementWiseDims(int32_t k) noexcept = 0;
677 virtual int32_t getNbElementWiseDims() const noexcept = 0;
678 virtual void setMode(GatherMode mode) noexcept = 0;
679 virtual GatherMode getMode() const noexcept = 0;
680};
681
682class VPluginLayer : public VRoot
683{
684public:
685 virtual IPlugin& getPlugin() noexcept = 0;
686};
687
688class VPluginV2Layer : public VRoot
689{
690public:
691 virtual IPluginV2& getPlugin() noexcept = 0;
692};
693
694class VPluginV3Layer : public VRoot
695{
696public:
697 virtual IPluginV3& getPlugin() noexcept = 0;
698};
699
700class VUnaryLayer : public VRoot
701{
702public:
703 virtual void setOperation(UnaryOperation op) noexcept = 0;
704 virtual UnaryOperation getOperation() const noexcept = 0;
705};
706
707class VReduceLayer : public VRoot
708{
709public:
710 virtual void setOperation(ReduceOperation op) noexcept = 0;
711 virtual ReduceOperation getOperation() const noexcept = 0;
712 virtual void setReduceAxes(uint32_t reduceAxes) noexcept = 0;
713 virtual uint32_t getReduceAxes() const noexcept = 0;
714 virtual void setKeepDimensions(bool keepDimensions) noexcept = 0;
715 virtual bool getKeepDimensions() const noexcept = 0;
716};
717
718class VPaddingLayer : public VRoot
719{
720public:
721 virtual void setPrePaddingNd(Dims const& padding) noexcept = 0;
722 virtual Dims getPrePaddingNd() const noexcept = 0;
723 virtual void setPostPaddingNd(Dims const& padding) noexcept = 0;
724 virtual Dims getPostPaddingNd() const noexcept = 0;
725};
726
727class VShuffleLayer : public VRoot
728{
729public:
730 virtual void setFirstTranspose(Permutation const& permutation) noexcept = 0;
731 virtual Permutation const& getFirstTranspose() const noexcept = 0;
732 virtual void setReshapeDimensions(Dims const& dimensions) noexcept = 0;
733 virtual Dims getReshapeDimensions() const noexcept = 0;
734 virtual void setSecondTranspose(Permutation const& permutation) noexcept = 0;
735 virtual Permutation const& getSecondTranspose() const noexcept = 0;
736 virtual void setZeroIsPlaceholder(bool zeroIsPlaceholder) noexcept = 0;
737 virtual bool getZeroIsPlaceholder() const noexcept = 0;
738};
739
740class VSliceLayer : public VRoot
741{
742public:
743 virtual void setStart(Dims const& start) noexcept = 0;
744 virtual Dims getStart() const noexcept = 0;
745 virtual void setSize(Dims const& size) noexcept = 0;
746 virtual Dims getSize() const noexcept = 0;
747 virtual void setStride(Dims const& stride) noexcept = 0;
748 virtual Dims getStride() const noexcept = 0;
749 virtual void setMode(SampleMode mode) noexcept = 0;
750 virtual SampleMode getMode() const noexcept = 0;
751 virtual void setAxes(Dims const& axes) noexcept = 0;
752 virtual Dims getAxes() const noexcept = 0;
753};
754
755class VShapeLayer : public VRoot
756{
757public:
758};
759
760class VTopKLayer : public VRoot
761{
762public:
763 virtual void setOperation(TopKOperation op) noexcept = 0;
764 virtual TopKOperation getOperation() const noexcept = 0;
765 virtual void setK(int32_t k) noexcept = 0;
766 virtual int32_t getK() const noexcept = 0;
767 virtual void setReduceAxes(uint32_t reduceAxes) noexcept = 0;
768 virtual uint32_t getReduceAxes() const noexcept = 0;
769};
770
771class VMatrixMultiplyLayer : public VRoot
772{
773public:
774 virtual void setOperation(int32_t index, MatrixOperation op) noexcept = 0;
775 virtual MatrixOperation getOperation(int32_t index) const noexcept = 0;
776};
777
778class VNonZeroLayer : public VRoot
779{
780public:
781};
782
783class VRaggedSoftMaxLayer : public VRoot
784{
785public:
786};
787
788class VIdentityLayer : public VRoot
789{
790public:
791};
792
793class VCastLayer : public VRoot
794{
795public:
796 virtual void setToType(DataType toType) noexcept = 0;
797 virtual DataType getToType() const noexcept = 0;
798};
799
800class VConstantLayer : public VRoot
801{
802public:
803 virtual void setWeights(Weights weights) noexcept = 0;
804 virtual Weights getWeights() const noexcept = 0;
805 virtual void setDimensions(Dims const& dimensions) noexcept = 0;
806 virtual Dims getDimensions() const noexcept = 0;
807};
808
809class VParametricReLULayer : public VRoot
810{
811public:
812};
813
814class VResizeLayer : public VRoot
815{
816public:
817 virtual void setOutputDimensions(Dims const& dimensions) noexcept = 0;
818 virtual Dims getOutputDimensions() const noexcept = 0;
819 virtual void setScales(float const* scales, int32_t nbScales) noexcept = 0;
820 virtual int32_t getScales(int32_t size, float* scales) const noexcept = 0;
821 virtual void setResizeMode(InterpolationMode interpolationMode) noexcept = 0;
822 virtual InterpolationMode getResizeMode() const noexcept = 0;
823 virtual void setCoordinateTransformation(ResizeCoordinateTransformation coordTransform) noexcept = 0;
824 virtual ResizeCoordinateTransformation getCoordinateTransformation() const noexcept = 0;
825 virtual void setSelectorForSinglePixel(ResizeSelector selector) noexcept = 0;
826 virtual ResizeSelector getSelectorForSinglePixel() const noexcept = 0;
827 virtual void setNearestRounding(ResizeRoundMode value) noexcept = 0;
828 virtual ResizeRoundMode getNearestRounding() const noexcept = 0;
829 virtual void setCubicCoeff(float value) noexcept = 0;
830 virtual float getCubicCoeff() const noexcept = 0;
831 virtual void setExcludeOutside(bool value) noexcept = 0;
832 virtual bool getExcludeOutside() const noexcept = 0;
833};
834
835class VLoopBoundaryLayer : public VRoot
836{
837public:
838 virtual ILoop* getLoop() const noexcept = 0;
839};
840
841class VRecurrenceLayer : public VRoot
842{
843public:
844};
845
846class VLoopOutputLayer : public VRoot
847{
848public:
849 virtual LoopOutput getLoopOutput() const noexcept = 0;
850 virtual void setAxis(int32_t axis) noexcept = 0;
851 virtual int32_t getAxis() const noexcept = 0;
852};
853
854class VTripLimitLayer : public VRoot
855{
856public:
857 virtual TripLimit getTripLimit() const noexcept = 0;
858};
859
860class VIteratorLayer : public VRoot
861{
862public:
863 virtual void setAxis(int32_t axis) noexcept = 0;
864 virtual int32_t getAxis() const noexcept = 0;
865 virtual void setReverse(bool reverse) noexcept = 0;
866 virtual bool getReverse() const noexcept = 0;
867};
868class VLoop : public VRoot
869{
870public:
871 virtual IRecurrenceLayer* addRecurrence(ITensor& initialValue) noexcept = 0;
872 virtual ITripLimitLayer* addTripLimit(ITensor& tensor, TripLimit limit) noexcept = 0;
873 virtual IIteratorLayer* addIterator(ITensor& tensor, int32_t axis = 0, bool reverse = false) noexcept = 0;
874 virtual ILoopOutputLayer* addLoopOutput(ITensor& tensor, LoopOutput outputKind, int32_t axis = 0) noexcept = 0;
875 virtual void setName(char const* name) noexcept = 0;
876 virtual char const* getName() const noexcept = 0;
877};
878
879class VConditionalBoundaryLayer : public VRoot
880{
881public:
882 virtual IIfConditional* getConditional() const noexcept = 0;
883};
884
885class VConditionLayer : public VRoot
886{
887public:
888};
889
890class VConditionalInputLayer : public VRoot
891{
892public:
893};
894
895class VConditionalOutputLayer : public VRoot
896{
897public:
898};
899
900class VIfConditional : public VRoot
901{
902public:
903 virtual IConditionLayer* setCondition(ITensor& tensor) noexcept = 0;
904 virtual IIfConditionalInputLayer* addInput(ITensor& tensor) noexcept = 0;
905 virtual IIfConditionalOutputLayer* addOutput(ITensor& trueTensor, ITensor& falseTensor) noexcept = 0;
906 virtual void setName(char const* name) noexcept = 0;
907 virtual char const* getName() const noexcept = 0;
908};
909
910class VSelectLayer : public VRoot
911{
912};
913
914class VAssertionLayer : public VRoot
915{
916public:
917 virtual void setMessage(char const* message) noexcept = 0;
918 virtual char const* getMessage() const noexcept = 0;
919};
920
921class VFillLayer : public VRoot
922{
923public:
924 virtual void setDimensions(Dims const& dimensions) noexcept = 0;
925 virtual Dims getDimensions() const noexcept = 0;
926 virtual void setOperation(FillOperation op) noexcept = 0;
927 virtual FillOperation getOperation() const noexcept = 0;
928 virtual void setAlpha(double alpha) noexcept = 0;
929 virtual double getAlpha() const noexcept = 0;
930 virtual void setBeta(double beta) noexcept = 0;
931 virtual double getBeta() const noexcept = 0;
932 virtual void setAlphaInt64(int64_t alpha) noexcept = 0;
933 virtual int64_t getAlphaInt64() const noexcept = 0;
934 virtual void setBetaInt64(int64_t beta) noexcept = 0;
935 virtual int64_t getBetaInt64() const noexcept = 0;
936 virtual bool isAlphaBetaInt64() const noexcept = 0;
937 virtual DataType getToType() const noexcept = 0;
938 virtual void setToType(DataType toType) noexcept = 0;
939};
940
941class VQuantizeLayer : public VRoot
942{
943public:
944 virtual int32_t getAxis() const noexcept = 0;
945 virtual void setAxis(int32_t axis) noexcept = 0;
946 virtual DataType getToType() const noexcept = 0;
947 virtual void setToType(DataType toType) noexcept = 0;
948};
949
950class VDequantizeLayer : public VRoot
951{
952public:
953 virtual int32_t getAxis() const noexcept = 0;
954 virtual void setAxis(int32_t axis) noexcept = 0;
955 virtual DataType getToType() const noexcept = 0;
956 virtual void setToType(DataType toType) noexcept = 0;
957};
958
959class VDynamicQuantizeLayer : public VRoot
960{
961public:
962 virtual int32_t getAxis() const noexcept = 0;
963 virtual void setAxis(int32_t axis) noexcept = 0;
964 virtual int32_t getBlockSize() const noexcept = 0;
965 virtual void setBlockSize(int32_t axis) noexcept = 0;
966 virtual DataType getScaleType() const noexcept = 0;
967 virtual void setScaleType(DataType axis) noexcept = 0;
968 virtual DataType getToType() const noexcept = 0;
969 virtual void setToType(DataType toType) noexcept = 0;
970};
971
972class VScatterLayer : public VRoot
973{
974public:
975 virtual void setMode(ScatterMode mode) noexcept = 0;
976 virtual ScatterMode getMode() const noexcept = 0;
977 virtual void setAxis(int32_t axis) noexcept = 0;
978 virtual int32_t getAxis() const noexcept = 0;
979}; // class VScatterLayer
980
981class VEinsumLayer : public VRoot
982{
983public:
984 virtual bool setEquation(char const* equation) noexcept = 0;
985 virtual char const* getEquation() const noexcept = 0;
986};
987
988class VOneHotLayer : public VRoot
989{
990public:
991 virtual int32_t getAxis() const noexcept = 0;
992 virtual void setAxis(int32_t axis) noexcept = 0;
993}; // class VOneHotLayer
994
995class VGridSampleLayer : public VRoot
996{
997public:
998 virtual void setInterpolationMode(InterpolationMode mode) noexcept = 0;
999 virtual InterpolationMode getInterpolationMode() const noexcept = 0;
1000 virtual void setAlignCorners(bool alignCorners) noexcept = 0;
1001 virtual bool getAlignCorners() const noexcept = 0;
1002 virtual bool setSampleMode(SampleMode mode) noexcept = 0;
1003 virtual SampleMode getSampleMode() const noexcept = 0;
1004}; // class VGridSampleLayer
1005
1006class VNMSLayer : public VRoot
1007{
1008public:
1009 virtual void setBoundingBoxFormat(BoundingBoxFormat fmt) noexcept = 0;
1010 virtual BoundingBoxFormat getBoundingBoxFormat() const noexcept = 0;
1011 virtual void setTopKBoxLimit(int32_t limit) noexcept = 0;
1012 virtual int32_t getTopKBoxLimit() const noexcept = 0;
1013}; // class VNMSLayer
1014
1015class VReverseSequenceLayer : public VRoot
1016{
1017public:
1018 virtual void setBatchAxis(int32_t batchAxis) noexcept = 0;
1019 virtual int32_t getBatchAxis() const noexcept = 0;
1020
1021 virtual void setSequenceAxis(int32_t sequenceAxis) noexcept = 0;
1022 virtual int32_t getSequenceAxis() const noexcept = 0;
1023}; // class VReverseSequenceLayer
1024
1025class VNormalizationLayer : public VRoot
1026{
1027public:
1028 virtual void setEpsilon(float eps) noexcept = 0;
1029 virtual float getEpsilon() const noexcept = 0;
1030 virtual void setAxes(uint32_t axesMask) noexcept = 0;
1031 virtual uint32_t getAxes() const noexcept = 0;
1032 virtual void setNbGroups(int64_t nbGroups) noexcept = 0;
1033 virtual int64_t getNbGroups() const noexcept = 0;
1034 virtual void setComputePrecision(DataType type) noexcept = 0;
1035 virtual DataType getComputePrecision() const noexcept = 0;
1036}; // class VNormalizationLayer
1037
1038class VSqueezeLayer : public VRoot
1039{
1040};
1041
1042class VUnsqueezeLayer : public VRoot
1043{
1044};
1045
1046class VCumulativeLayer : public VRoot
1047{
1048public:
1049 virtual bool setOperation(CumulativeOperation op) noexcept = 0;
1050 virtual CumulativeOperation getOperation() const noexcept = 0;
1051 virtual void setExclusive(bool exclusive) noexcept = 0;
1052 virtual bool getExclusive() const noexcept = 0;
1053 virtual void setReverse(bool reverse) noexcept = 0;
1054 virtual bool getReverse() const noexcept = 0;
1055}; // class VCumulativeLayer
1056
1057
1058class VNetworkDefinition : public VRoot
1059{
1060public:
1061 virtual ITensor* addInput(char const* name, DataType type, Dims const& dimensions) noexcept = 0;
1062 virtual void markOutput(ITensor& tensor) noexcept = 0;
1063 virtual IActivationLayer* addActivation(ITensor& input, ActivationType type) noexcept = 0;
1064 virtual ILRNLayer* addLRN(ITensor& input, int64_t window, float alpha, float beta, float k) noexcept = 0;
1065 virtual IScaleLayer* addScale(
1066 ITensor& input, ScaleMode mode, Weights shift, Weights scale, Weights power) noexcept = 0;
1067 virtual ISoftMaxLayer* addSoftMax(ITensor& input) noexcept = 0;
1068 virtual IConcatenationLayer* addConcatenation(ITensor* const* inputs, int32_t nbInputs) noexcept = 0;
1069 virtual IElementWiseLayer* addElementWise(ITensor& input1, ITensor& input2, ElementWiseOperation op) noexcept = 0;
1070 virtual IUnaryLayer* addUnary(ITensor& input, UnaryOperation operation) noexcept = 0;
1071 virtual IShuffleLayer* addShuffle(ITensor& input) noexcept = 0;
1072 virtual int32_t getNbLayers() const noexcept = 0;
1073 virtual ILayer* getLayer(int32_t index) const noexcept = 0;
1074 virtual int32_t getNbInputs() const noexcept = 0;
1075 virtual ITensor* getInput(int32_t index) const noexcept = 0;
1076 virtual int32_t getNbOutputs() const noexcept = 0;
1077 virtual ITensor* getOutput(int32_t index) const noexcept = 0;
1078 virtual IReduceLayer* addReduce(
1079 ITensor& input, ReduceOperation operation, uint32_t reduceAxes, bool keepDimensions) noexcept
1080 = 0;
1081 virtual ITopKLayer* addTopK(ITensor& input, TopKOperation op, int32_t k, uint32_t reduceAxes) noexcept = 0;
1082 virtual IGatherLayer* addGather(ITensor& data, ITensor& indices, int32_t axis) noexcept = 0;
1083 virtual IRaggedSoftMaxLayer* addRaggedSoftMax(ITensor& input, ITensor& bounds) noexcept = 0;
1084 virtual IMatrixMultiplyLayer* addMatrixMultiply(
1085 ITensor& input0, MatrixOperation op0, ITensor& input1, MatrixOperation op1) noexcept = 0;
1086 virtual IConstantLayer* addConstant(Dims const& dimensions, Weights weights) noexcept = 0;
1087 virtual IIdentityLayer* addIdentity(ITensor& input) noexcept = 0;
1088 virtual void removeTensor(ITensor& tensor) noexcept = 0;
1089 virtual void unmarkOutput(ITensor& tensor) noexcept = 0;
1090 virtual IPluginV2Layer* addPluginV2(ITensor* const* inputs, int32_t nbInputs, IPluginV2& plugin) noexcept = 0;
1091 virtual IPluginV3Layer* addPluginV3(ITensor* const* inputs, int32_t nbInputs, ITensor* const* shapeInputs,
1092 int32_t nbShapeInputs, IPluginV3& plugin) noexcept = 0;
1093 virtual ISliceLayer* addSlice(ITensor& input, Dims const& start, Dims const& size, Dims const& stride) noexcept = 0;
1094 virtual void setName(char const* name) noexcept = 0;
1095 virtual char const* getName() const noexcept = 0;
1096 virtual IShapeLayer* addShape(ITensor& input) noexcept = 0;
1097 virtual bool hasImplicitBatchDimension() const noexcept = 0;
1098 virtual bool markOutputForShapes(ITensor& tensor) noexcept = 0;
1099 virtual bool unmarkOutputForShapes(ITensor& tensor) noexcept = 0;
1100 virtual IParametricReLULayer* addParametricReLU(ITensor& input, ITensor& slope) noexcept = 0;
1101 virtual IConvolutionLayer* addConvolutionNd(
1102 ITensor& input, int64_t nbOutputMaps, Dims const& kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
1103 = 0;
1104 virtual IPoolingLayer* addPoolingNd(ITensor& input, PoolingType type, Dims const& windowSize) noexcept = 0;
1105 virtual IDeconvolutionLayer* addDeconvolutionNd(
1106 ITensor& input, int64_t nbOutputMaps, Dims const& kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
1107 = 0;
1108 virtual IScaleLayer* addScaleNd(
1109 ITensor& input, ScaleMode mode, Weights shift, Weights scale, Weights power, int32_t channelAxis) noexcept = 0;
1110 virtual IResizeLayer* addResize(ITensor& input) noexcept = 0;
1111 virtual ILoop* addLoop() noexcept = 0;
1112 virtual ISelectLayer* addSelect(ITensor& condition, ITensor& thenInput, ITensor& elseInput) noexcept = 0;
1113 virtual IFillLayer* addFill(Dims const& dimensions, FillOperation op) noexcept = 0;
1114 virtual IPaddingLayer* addPaddingNd(ITensor& input, Dims const& prePadding, Dims const& postPadding) noexcept = 0;
1115 virtual bool setWeightsName(Weights weights, char const* name) noexcept = 0;
1116 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
1117 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
1118 virtual IDequantizeLayer* addDequantize(ITensor& input, ITensor& scale) noexcept = 0;
1119 virtual IQuantizeLayer* addQuantize(ITensor& input, ITensor& scale) noexcept = 0;
1120 virtual IGatherLayer* addGatherV2(ITensor& data, ITensor& indices, GatherMode mode) noexcept = 0;
1121 virtual IIfConditional* addIfConditional() noexcept = 0;
1122 virtual IScatterLayer* addScatter(ITensor& data, ITensor& indices, ITensor& updates, ScatterMode mode) noexcept = 0;
1123 virtual IEinsumLayer* addEinsum(ITensor* const* inputs, int32_t nbInputs, char const* equation) noexcept = 0;
1124 virtual IAssertionLayer* addAssertion(ITensor& condition, char const* message) noexcept = 0;
1125 virtual IOneHotLayer* addOneHot(ITensor& indices, ITensor& values, ITensor& depth, int32_t axis) noexcept = 0;
1126 virtual INonZeroLayer* addNonZero(ITensor& input) noexcept = 0;
1127 virtual IGridSampleLayer* addGridSample(ITensor& input, ITensor& grid) noexcept = 0;
1128 virtual INMSLayer* addNMS(ITensor& boxes, ITensor& scores, ITensor& maxOutputBoxesPerClass) noexcept = 0;
1129 virtual IReverseSequenceLayer* addReverseSequence(ITensor& input, ITensor& sequenceLens) noexcept = 0;
1130 virtual INormalizationLayer* addNormalization(
1131 ITensor& input, ITensor& scale, ITensor& bias, uint32_t axesMask) noexcept = 0;
1132 virtual ICastLayer* addCast(ITensor& input, DataType toType) noexcept = 0;
1133 virtual IBuilder& getBuilder() const noexcept = 0;
1134 virtual NetworkDefinitionCreationFlags getFlags() const noexcept = 0;
1135 virtual bool getFlag(NetworkDefinitionCreationFlag networkDefinitionCreationFlag) const noexcept = 0;
1136 virtual IQuantizeLayer* addQuantizeV2(ITensor& input, ITensor& scale, DataType outputType) noexcept = 0;
1137 virtual IDequantizeLayer* addDequantizeV2(ITensor& input, ITensor& scale, DataType outputType) noexcept = 0;
1138 virtual IFillLayer* addFillV2(Dims const& dimensions, FillOperation op, DataType outputType) noexcept = 0;
1139 virtual bool markDebug(ITensor& tensor) noexcept = 0;
1140 virtual bool unmarkDebug(ITensor& tensor) noexcept = 0;
1141 virtual bool isDebugTensor(nvinfer1::ITensor const& tensor) const noexcept = 0;
1142 virtual bool markWeightsRefittable(char const* name) noexcept = 0;
1143 virtual bool unmarkWeightsRefittable(char const* name) noexcept = 0;
1144 virtual bool areWeightsMarkedRefittable(char const* name) const noexcept = 0;
1145 virtual ISqueezeLayer* addSqueeze(ITensor& input, ITensor& axes) noexcept = 0;
1146 virtual IUnsqueezeLayer* addUnsqueeze(ITensor& input, ITensor& axes) noexcept = 0;
1147 virtual IDynamicQuantizeLayer* addDynamicQuantize(
1148 ITensor& input, int32_t axis, int32_t blockSize, DataType toType, DataType scaleType) noexcept = 0;
1149 virtual ICumulativeLayer* addCumulative(
1150 ITensor& input, ITensor& axis, CumulativeOperation operation, bool exclusive, bool reverse) noexcept = 0;
1151 virtual bool markUnfusedTensorsAsDebugTensors() noexcept = 0;
1152 virtual bool unmarkUnfusedTensorsAsDebugTensors() noexcept = 0;
1153};
1154
1155class VAlgorithmIOInfo : public VRoot
1156{
1157public:
1158 virtual DataType getDataType() const noexcept = 0;
1159 virtual Dims getStrides() const noexcept = 0;
1160 virtual int64_t getVectorizedDim() const noexcept = 0;
1161 virtual int64_t getComponentsPerElement() const noexcept = 0;
1162};
1163
1164class VAlgorithmVariant : public VRoot
1165{
1166public:
1167 virtual int64_t getImplementation() const noexcept = 0;
1168 virtual int64_t getTactic() const noexcept = 0;
1169};
1170
1171class VAlgorithmContext : public VRoot
1172{
1173public:
1174 virtual char const* getName() const noexcept = 0;
1175 virtual Dims getDimensions(int32_t index, OptProfileSelector select) const noexcept = 0;
1176 virtual int32_t getNbInputs() const noexcept = 0;
1177 virtual int32_t getNbOutputs() const noexcept = 0;
1178};
1179
1180class VAlgorithm : public VRoot
1181{
1182public:
1183 virtual IAlgorithmVariant const& getAlgorithmVariant() const noexcept = 0;
1184 virtual float getTimingMSec() const noexcept = 0;
1185 virtual std::size_t getWorkspaceSize() const noexcept = 0;
1186 virtual IAlgorithmIOInfo const* getAlgorithmIOInfoByIndex(int32_t index) const noexcept = 0;
1187};
1188
1189
1190class VTimingCache : public VRoot
1191{
1192public:
1193 virtual nvinfer1::IHostMemory* serialize() const noexcept = 0;
1194 virtual bool combine(ITimingCache const& inputCache, bool ignoreMismatch) noexcept = 0;
1195 virtual bool reset() noexcept = 0;
1196 virtual int64_t queryKeys(TimingCacheKey* keyBuffer, int64_t capacity) const noexcept = 0;
1197 virtual TimingCacheValue query(TimingCacheKey const& key) const noexcept = 0;
1198 virtual bool update(TimingCacheKey const& key, TimingCacheValue const& value) noexcept = 0;
1199};
1200
1201class VBuilderConfig : public VRoot
1202{
1203public:
1204 virtual void setAvgTimingIterations(int32_t avgTiming) noexcept = 0;
1205 virtual int32_t getAvgTimingIterations() const noexcept = 0;
1206 virtual void setEngineCapability(EngineCapability capability) noexcept = 0;
1207 virtual EngineCapability getEngineCapability() const noexcept = 0;
1208 virtual void setInt8Calibrator(IInt8Calibrator* calibrator) noexcept = 0;
1209 virtual IInt8Calibrator* getInt8Calibrator() const noexcept = 0;
1210 virtual void setFlags(BuilderFlags builderFlags) noexcept = 0;
1211 virtual BuilderFlags getFlags() const noexcept = 0;
1212 virtual void clearFlag(BuilderFlag builderFlag) noexcept = 0;
1213 virtual void setFlag(BuilderFlag builderFlag) noexcept = 0;
1214 virtual bool getFlag(BuilderFlag builderFlag) const noexcept = 0;
1215 virtual void setDeviceType(ILayer const* layer, DeviceType deviceType) noexcept = 0;
1216 virtual DeviceType getDeviceType(ILayer const* layer) const noexcept = 0;
1217 virtual bool isDeviceTypeSet(ILayer const* layer) const noexcept = 0;
1218 virtual void resetDeviceType(ILayer const* layer) noexcept = 0;
1219 virtual bool canRunOnDLA(ILayer const* layer) const noexcept = 0;
1220 virtual void setDLACore(int32_t dlaCore) noexcept = 0;
1221 virtual int32_t getDLACore() const noexcept = 0;
1222 virtual void setDefaultDeviceType(DeviceType deviceType) noexcept = 0;
1223 virtual DeviceType getDefaultDeviceType() const noexcept = 0;
1224 virtual void reset() noexcept = 0;
1225 virtual void setProfileStream(const cudaStream_t stream) noexcept = 0;
1226 virtual cudaStream_t getProfileStream() const noexcept = 0;
1227 virtual int32_t addOptimizationProfile(IOptimizationProfile const* profile) noexcept = 0;
1228 virtual int32_t getNbOptimizationProfiles() const noexcept = 0;
1229 virtual void setProfilingVerbosity(ProfilingVerbosity verbosity) noexcept = 0;
1230 virtual ProfilingVerbosity getProfilingVerbosity() const noexcept = 0;
1231 virtual void setAlgorithmSelector(IAlgorithmSelector* selector) noexcept = 0;
1232 virtual IAlgorithmSelector* getAlgorithmSelector() const noexcept = 0;
1233 virtual bool setCalibrationProfile(IOptimizationProfile const* profile) noexcept = 0;
1234 virtual IOptimizationProfile const* getCalibrationProfile() noexcept = 0;
1235 virtual void setQuantizationFlags(QuantizationFlags flags) noexcept = 0;
1236 virtual QuantizationFlags getQuantizationFlags() const noexcept = 0;
1237 virtual void clearQuantizationFlag(QuantizationFlag flag) noexcept = 0;
1238 virtual void setQuantizationFlag(QuantizationFlag flag) noexcept = 0;
1239 virtual bool getQuantizationFlag(QuantizationFlag flag) const noexcept = 0;
1240 virtual bool setTacticSources(TacticSources tacticSources) noexcept = 0;
1241 virtual TacticSources getTacticSources() const noexcept = 0;
1242 virtual nvinfer1::ITimingCache* createTimingCache(void const* blob, std::size_t size) const noexcept = 0;
1243 virtual bool setTimingCache(ITimingCache const& cache, bool ignoreMismatch) noexcept = 0;
1244 virtual nvinfer1::ITimingCache const* getTimingCache() const noexcept = 0;
1245 virtual void setMemoryPoolLimit(MemoryPoolType pool, std::size_t poolSize) noexcept = 0;
1246 virtual std::size_t getMemoryPoolLimit(MemoryPoolType pool) const noexcept = 0;
1247 virtual void setPreviewFeature(PreviewFeature feature, bool enable) noexcept = 0;
1248 virtual bool getPreviewFeature(PreviewFeature feature) const noexcept = 0;
1249 virtual void setBuilderOptimizationLevel(int32_t level) noexcept = 0;
1250 virtual int32_t getBuilderOptimizationLevel() const noexcept = 0;
1251 virtual void setHardwareCompatibilityLevel(HardwareCompatibilityLevel hardwareCompatibilityLevel) noexcept = 0;
1252 virtual HardwareCompatibilityLevel getHardwareCompatibilityLevel() const noexcept = 0;
1253 virtual void setPluginsToSerialize(char const* const* paths, int32_t nbPaths) noexcept = 0;
1254 virtual char const* getPluginToSerialize(int32_t index) const noexcept = 0;
1255 virtual int32_t getNbPluginsToSerialize() const noexcept = 0;
1256 virtual void setMaxAuxStreams(int32_t nbStreams) noexcept = 0;
1257 virtual int32_t getMaxAuxStreams() const noexcept = 0;
1258 virtual void setProgressMonitor(IProgressMonitor* monitor) noexcept = 0;
1259 virtual IProgressMonitor* getProgressMonitor() const noexcept = 0;
1260 virtual void setRuntimePlatform(RuntimePlatform runtimePlatform) noexcept = 0;
1261 virtual RuntimePlatform getRuntimePlatform() const noexcept = 0;
1262 virtual void setMaxNbTactics(int32_t maxTactics) noexcept = 0;
1263 virtual int32_t getMaxNbTactics() const noexcept = 0;
1264 virtual bool setTilingOptimizationLevel(TilingOptimizationLevel level) noexcept = 0;
1265 virtual TilingOptimizationLevel getTilingOptimizationLevel() const noexcept = 0;
1266 virtual bool setL2LimitForTiling(int64_t size) noexcept = 0;
1267 virtual int64_t getL2LimitForTiling() const noexcept = 0;
1268};
1269
1270class VSerializationConfig : public VRoot
1271{
1272public:
1273 virtual bool setFlags(SerializationFlags serializationFlags) noexcept = 0;
1274 virtual SerializationFlags getFlags() const noexcept = 0;
1275 virtual bool clearFlag(SerializationFlag serializationFlag) noexcept = 0;
1276 virtual bool setFlag(SerializationFlag serializationFlag) noexcept = 0;
1277 virtual bool getFlag(SerializationFlag serializationFlag) const noexcept = 0;
1278};
1279
1280class VBuilder : public VRoot
1281{
1282public:
1283 virtual bool platformHasFastFp16() const noexcept = 0;
1284 virtual bool platformHasFastInt8() const noexcept = 0;
1285 virtual int32_t getMaxDLABatchSize() const noexcept = 0;
1286 virtual int32_t getNbDLACores() const noexcept = 0;
1287 virtual void setGpuAllocator(IGpuAllocator* allocator) noexcept = 0;
1288 virtual nvinfer1::IBuilderConfig* createBuilderConfig() noexcept = 0;
1289 virtual nvinfer1::INetworkDefinition* createNetworkV2(NetworkDefinitionCreationFlags flags) noexcept = 0;
1290 virtual nvinfer1::IOptimizationProfile* createOptimizationProfile() noexcept = 0;
1291 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
1292 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
1293 virtual void reset() noexcept = 0;
1294 virtual bool platformHasTf32() const noexcept = 0;
1295 virtual nvinfer1::IHostMemory* buildSerializedNetwork(
1296 INetworkDefinition& network, IBuilderConfig& config) noexcept = 0;
1297 virtual bool isNetworkSupported(INetworkDefinition const& network, IBuilderConfig const& config) const noexcept = 0;
1298 virtual ILogger* getLogger() const noexcept = 0;
1299 virtual bool setMaxThreads(int32_t maxThreads) noexcept = 0;
1300 virtual int32_t getMaxThreads() const noexcept = 0;
1301 virtual IPluginRegistry& getPluginRegistry() noexcept = 0;
1302 virtual ICudaEngine* buildEngineWithConfig(INetworkDefinition& network, IBuilderConfig& config) noexcept = 0;
1303};
1304
1305class VRuntimeConfig : public VRoot
1306{
1307public:
1308 virtual IRuntimeConfig* getPImpl() noexcept = 0;
1309 virtual void setExecutionContextAllocationStrategy(ExecutionContextAllocationStrategy strategy) noexcept = 0;
1310 virtual ExecutionContextAllocationStrategy getExecutionContextAllocationStrategy() const noexcept = 0;
1311};
1312
1313
1314} // namespace apiv
1315} // namespace nvinfer1
1316
1317// @endcond
1318
1319#endif // NV_INFER_RUNTIME_IMPL_H
getLogger
nvinfer1::ILogger * getLogger() noexcept
Return the logger object.
getPluginRegistry
nvinfer1::IPluginRegistry * getPluginRegistry() noexcept
Return the plugin registry.
TRT_NODISCARD
#define TRT_NODISCARD
A stand-in for [[nodiscard]] and [[nodiscard(REASON)]] that works with older compilers.
Definition: NvInferRuntimeBase.h:57
Dims
Structure to define the dimensions of a tensor.
IAlgorithmSelector
Interface implemented by application for selecting and reporting algorithms of a layer provided by th...
IDebugListener
User-implemented callback for notification when value of a debug tensor is updated.
IErrorRecorder
Reference counted application-implemented error reporting interface for TensorRT objects.
IGpuAllocator
Application-implemented class for controlling allocation on the GPU.
IOutputAllocator
Callback from ExecutionContext::enqueueV3()
IPluginV3
Plugin class for the V3 generation of user-implemented layers.
IProfiler
Application-implemented interface for profiling.
IProgressMonitor
Application-implemented progress reporting interface for TensorRT.
IStreamReader
Application-implemented class for reading data in a stream-based manner.
IStreamReaderV2
Application-implemented class for reading data in a stream-based manner asynchronously....
nvinfer1::IHostMemory
Class to handle library allocated memory that is accessible to the user.
Definition: NvInferRuntime.h:142
nvinfer1
The TensorRT API version 1 namespace.
nvinfer1::TacticSources
uint32_t TacticSources
Represents a collection of one or more TacticSource values combine using bitwise-OR operations.
Definition: NvInferRuntime.h:2913
nvinfer1::IOutputAllocator
v_1_0::IOutputAllocator IOutputAllocator
Definition: NvInferRuntime.h:4136
nvinfer1::ResizeSelector
ResizeSelector
The coordinate selector when resize to single pixel output.
Definition: NvInfer.h:4054
nvinfer1::EngineCapability
EngineCapability
List of supported engine capability flows.
Definition: NvInferRuntime.h:76
nvinfer1::DimensionOperation
DimensionOperation
An operation on two IDimensionExpr, which represent integer expressions used in dimension computation...
Definition: NvInferRuntime.h:179
nvinfer1::MemoryPoolType
MemoryPoolType
The type for memory pools used by TensorRT.
Definition: NvInfer.h:9205
nvinfer1::ScaleMode
ScaleMode
Controls how shift, scale and power are applied in a Scale layer.
Definition: NvInfer.h:1850
nvinfer1::RuntimePlatform
RuntimePlatform
Describes the intended runtime platform (operating system and CPU architecture) for the execution of ...
Definition: NvInfer.h:8805
nvinfer1::QuantizationFlags
uint32_t QuantizationFlags
Represents one or more QuantizationFlag values using binary OR operations.
Definition: NvInfer.h:8757
nvinfer1::TensorIOMode
TensorIOMode
Definition of tensor IO Mode.
Definition: NvInferRuntimeBase.h:656
nvinfer1::HardwareCompatibilityLevel
HardwareCompatibilityLevel
Describes requirements of compatibility with GPU architectures other than that of the GPU on which th...
Definition: NvInfer.h:9324
nvinfer1::CumulativeOperation
CumulativeOperation
Enumerates the cumulative operations that may be performed by a Cumulative layer.
Definition: NvInfer.h:6538
nvinfer1::SerializationFlag
SerializationFlag
List of valid flags that the engine can enable when serializing the bytes.
Definition: NvInferRuntime.h:2954
nvinfer1::BoundingBoxFormat
BoundingBoxFormat
Representation of bounding box data used for the Boxes input tensor in INMSLayer.
Definition: NvInfer.h:6123
nvinfer1::IPluginV3
v_1_0::IPluginV3 IPluginV3
Definition: NvInferPluginBase.h:273
nvinfer1::UnaryOperation
UnaryOperation
Enumerates the unary operations that may be performed by a Unary layer.
Definition: NvInfer.h:2795
nvinfer1::IProfiler
v_1_0::IProfiler IProfiler
Definition: NvInferRuntime.h:1272
nvinfer1::IAlgorithmSelector
v_1_0::IAlgorithmSelector IAlgorithmSelector
Definition: NvInfer.h:8749
nvinfer1::IStreamReaderV2
v_1_0::IStreamReaderV2 IStreamReaderV2
Definition: NvInferRuntime.h:751
nvinfer1::ActivationType
ActivationType
Enumerates the types of activation to perform in an activation layer.
Definition: NvInfer.h:137
nvinfer1::TempfileControlFlags
uint32_t TempfileControlFlags
Represents a collection of one or more TempfileControlFlag values combined using bitwise-OR operation...
Definition: NvInferRuntime.h:1350
nvinfer1::FillOperation
FillOperation
Enumerates the tensor fill operations that may performed by a fill layer.
Definition: NvInfer.h:5035
nvinfer1::ResizeRoundMode
ResizeRoundMode
The rounding mode for nearest neighbor resize.
Definition: NvInfer.h:4084
nvinfer1::PaddingMode
PaddingMode
Enumerates the modes of padding to perform in convolution, deconvolution and pooling layer,...
Definition: NvInfer.h:1028
nvinfer1::TripLimit
TripLimit
Enum that describes kinds of trip limits.
Definition: NvInfer.h:4443
nvinfer1::NetworkDefinitionCreationFlags
uint32_t NetworkDefinitionCreationFlags
Represents one or more NetworkDefinitionCreationFlag flags using binary OR operations....
Definition: NvInfer.h:10435
nvinfer1::PreviewFeature
PreviewFeature
Define preview features.
Definition: NvInfer.h:9280
nvinfer1::TilingOptimizationLevel
TilingOptimizationLevel
Define the optimization levels for Tiling.
Definition: NvInfer.h:9377
nvinfer1::DataType
DataType
The type of weights and tensors.
Definition: NvInferRuntimeBase.h:143
nvinfer1::BuilderFlags
uint32_t BuilderFlags
Represents one or more BuilderFlag values using binary OR operations, e.g., 1U << BuilderFlag::kFP16 ...
Definition: NvInfer.h:8835
nvinfer1::DeviceType
DeviceType
The device that this layer/network will execute on.
Definition: NvInferRuntime.h:1304
nvinfer1::CalibrationAlgoType
CalibrationAlgoType
Version of calibration algorithm to use.
Definition: NvInfer.h:8183
nvinfer1::LayerType
LayerType
The type values of layer classes.
Definition: NvInfer.h:58
nvinfer1::SampleMode
SampleMode
Controls how ISliceLayer and IGridSample handle out-of-bounds coordinates.
Definition: NvInfer.h:3253
nvinfer1::GatherMode
GatherMode
Control form of IGatherLayer.
Definition: NvInfer.h:2542
nvinfer1::IDebugListener
v_1_0::IDebugListener IDebugListener
Definition: NvInferRuntime.h:4177
nvinfer1::WeightsRole
WeightsRole
How a layer uses particular Weights.
Definition: NvInferRuntime.h:1282
nvinfer1::TensorFormats
uint32_t TensorFormats
It is capable of representing one or more TensorFormat by binary OR operations, e....
Definition: NvInfer.h:129
nvinfer1::ProfilingVerbosity
ProfilingVerbosity
List of verbosity levels of layer information exposed in NVTX annotations and in IEngineInspector.
Definition: NvInferRuntime.h:2925
nvinfer1::NetworkDefinitionCreationFlag
NetworkDefinitionCreationFlag
List of immutable network properties expressed at network creation time. NetworkDefinitionCreationFla...
Definition: NvInfer.h:10446
nvinfer1::ElementWiseOperation
ElementWiseOperation
Enumerates the binary operations that may be performed by an ElementWise layer.
Definition: NvInfer.h:2452
nvinfer1::QuantizationFlag
QuantizationFlag
List of valid flags for quantizing the network to int8.
Definition: NvInfer.h:8769
nvinfer1::SerializationFlags
uint32_t SerializationFlags
Represents one or more SerializationFlag values using binary OR operations, e.g., 1U << Serialization...
Definition: NvInferRuntime.h:2944
nvinfer1::InterpolationMode
InterpolationMode
Enumerates various modes of interpolation.
Definition: NvInfer.h:3972
nvinfer1::BuilderFlag
BuilderFlag
List of valid modes that the builder can enable when creating an engine from a network definition.
Definition: NvInfer.h:8845
nvinfer1::TensorFormat
TensorFormat
Format of the input/output tensors.
Definition: NvInferRuntime.h:1384
nvinfer1::ExecutionContextAllocationStrategy
ExecutionContextAllocationStrategy
Different memory allocation behaviors for IExecutionContext.
Definition: NvInferRuntime.h:3058
nvinfer1::TopKOperation
TopKOperation
Enumerates the operations that may be performed by a TopK layer.
Definition: NvInfer.h:3534
nvinfer1::ReduceOperation
ReduceOperation
Enumerates the reduce operations that may be performed by a Reduce layer.
Definition: NvInfer.h:2890
nvinfer1::LayerInformationFormat
LayerInformationFormat
The format in which the IEngineInspector prints the layer information.
Definition: NvInferRuntime.h:5110
nvinfer1::ScatterMode
ScatterMode
Control form of IScatterLayer.
Definition: NvInfer.h:5857
nvinfer1::MatrixOperation
MatrixOperation
Enumerates the operations that may be performed on a tensor by IMatrixMultiplyLayer before multiplica...
Definition: NvInfer.h:3660
nvinfer1::ResizeCoordinateTransformation
ResizeCoordinateTransformation
The resize coordinate transformation function.
Definition: NvInfer.h:4000
nvinfer1::LoopOutput
LoopOutput
Enum that describes kinds of loop outputs.
Definition: NvInfer.h:4415
nvinfer1::IStreamReader
v_1_0::IStreamReader IStreamReader
Definition: NvInferRuntime.h:674
nvinfer1::PoolingType
PoolingType
The type of pooling to perform in a pooling layer.
Definition: NvInfer.h:1464
nvinfer1::IProgressMonitor
v_1_0::IProgressMonitor IProgressMonitor
Definition: NvInfer.h:9493
nvinfer1::TensorLocation
TensorLocation
The location for tensor data storage, device or host.
Definition: NvInferRuntime.h:204
nvinfer1::OptProfileSelector
OptProfileSelector
When setting or querying optimization profile parameters (such as shape tensor inputs or dynamic dime...
Definition: NvInferRuntime.h:2587

  Copyright © 2024 NVIDIA Corporation
  Privacy Policy | Manage My Privacy | Do Not Sell or Share My Data | Terms of Service | Accessibility | Corporate Policies | Product Security | Contact