TensorRT-RTX 1.0.0
NvInferImpl.h
Go to the documentation of this file.
1/*
2 * SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: Apache-2.0
4 *
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
8 *
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 */
17
18#ifndef NV_INFER_IMPL_H
19#define NV_INFER_IMPL_H
20
21#include "NvInferLegacyDims.h"
22#include "NvInferRuntimeCommon.h"
23
24// @cond SuppressDoxyWarnings
25
26namespace nvinfer1
27{
28
29class ILogger;
30
31namespace v_1_0
32{
33class IProgressMonitor;
34}
35using IProgressMonitor = v_1_0::IProgressMonitor;
36
37
38namespace v_1_0
39{
40class IProfiler;
41}
42using IProfiler = v_1_0::IProfiler;
43
44namespace v_1_0
45{
46class IOutputAllocator;
47}
48using IOutputAllocator = v_1_0::IOutputAllocator;
49
50namespace v_1_0
51{
52class IDebugListener;
53}
54using IDebugListener = v_1_0::IDebugListener;
55
56class IActivationLayer;
57class IAssertionLayer;
58class IBuilder;
59class IBuilderConfig;
60class IConcatenationLayer;
61class IConditionLayer;
62class IConstantLayer;
63class IConvolutionLayer;
64class ICudaEngine;
65class ICumulativeLayer;
66class IDeconvolutionLayer;
67class IDequantizeLayer;
68class IDimensionExpr;
69class IDynamicQuantizeLayer;
70class IEinsumLayer;
71class IElementWiseLayer;
72class IEngineInspector;
73class IExecutionContext;
74class IFillLayer;
75class IGatherLayer;
76class IGridSampleLayer;
77class IHostMemory;
78class IIdentityLayer;
79class ICastLayer;
80class IIfConditional;
81class IIfConditionalInputLayer;
82class IIfConditionalOutputLayer;
83class IIteratorLayer;
84class ILayer;
85class ILoop;
86class ILoopOutputLayer;
87class ILRNLayer;
88class IMatrixMultiplyLayer;
89class INetworkDefinition;
90class INormalizationLayer;
91class INMSLayer;
92class INonZeroLayer;
93class IOneHotLayer;
94class IOptimizationProfile;
95class IPaddingLayer;
96class IParametricReLULayer;
97class IRuntimeCache;
98class IRuntimeConfig;
99
100
101namespace v_1_0
102{
103class IStreamReader;
104} // namespace v_1_0
105using IStreamReader = v_1_0::IStreamReader;
106namespace v_1_0
107{
108class IStreamReaderV2;
109} // namespace v_1_0
110using IStreamReaderV2 = v_1_0::IStreamReaderV2;
111
112class IPoolingLayer;
113class IQuantizeLayer;
114class IRaggedSoftMaxLayer;
115class IRecurrenceLayer;
116class IReduceLayer;
117class IRefitter;
118class IResizeLayer;
119class IReverseSequenceLayer;
120class IRuntime;
121class IScaleLayer;
122class IScatterLayer;
123class ISelectLayer;
124class ISerializationConfig;
125class IShapeLayer;
126class IShuffleLayer;
127class ISliceLayer;
128class ISoftMaxLayer;
129class ISqueezeLayer;
130class ITensor;
131
132namespace v_1_0
133{
134struct TimingCacheKey;
135struct TimingCacheValue;
136} // namespace v_1_0
137using TimingCacheKey = v_1_0::TimingCacheKey;
138using TimingCacheValue = v_1_0::TimingCacheValue;
139
140class ITimingCache;
141class ITopKLayer;
142class ITripLimitLayer;
143class IUnaryLayer;
144class IUnsqueezeLayer;
145struct Permutation;
146class Weights;
147
148enum class ActivationType : int32_t;
149enum class BoundingBoxFormat : int32_t;
150enum class BuilderFlag : int32_t;
151enum class CumulativeOperation : int32_t;
152enum class DeviceType : int32_t;
153enum class DimensionOperation : int32_t;
154enum class ElementWiseOperation : int32_t;
155enum class EngineCapability : int32_t;
156enum class FillOperation : int32_t;
157enum class GatherMode : int32_t;
158enum class LayerInformationFormat : int32_t;
159enum class LayerType : int32_t;
160enum class LoopOutput : int32_t;
161enum class MatrixOperation : int32_t;
162enum class MemoryPoolType : int32_t;
163enum class NetworkDefinitionCreationFlag : int32_t;
164enum class OptProfileSelector : int32_t;
165enum class PaddingMode : int32_t;
166enum class PoolingType : int32_t;
167enum class ProfilingVerbosity : int32_t;
168enum class QuantizationFlag : int32_t;
169enum class ReduceOperation : int32_t;
170enum class ResizeCoordinateTransformation : int32_t;
171enum class InterpolationMode : int32_t;
172enum class ResizeRoundMode : int32_t;
173enum class ResizeSelector : int32_t;
174enum class ScaleMode : int32_t;
175enum class ScatterMode : int32_t;
176enum class SampleMode : int32_t;
177enum class SerializationFlag : int32_t;
178enum class TensorIOMode : int32_t;
179enum class TensorLocation : int32_t;
180enum class TopKOperation : int32_t;
181enum class TripLimit : int32_t;
182enum class UnaryOperation : int32_t;
183enum class WeightsRole : int32_t;
184enum class PreviewFeature : int32_t;
185enum class HardwareCompatibilityLevel : int32_t;
186enum class ExecutionContextAllocationStrategy : int32_t;
187enum class RuntimePlatform : int32_t;
188enum class TilingOptimizationLevel : int32_t;
189enum class ComputeCapability : int32_t;
190
191enum class DynamicShapesKernelSpecializationStrategy : int32_t;
192
193using TacticSources = uint32_t;
194using TensorFormats = uint32_t;
195using BuilderFlags = uint32_t;
196using NetworkDefinitionCreationFlags = uint32_t;
197using QuantizationFlags = uint32_t;
198using TempfileControlFlags = uint32_t;
199using SerializationFlags = uint32_t;
200
208
209namespace apiv
210{
211
212class VRoot
213{
214public:
215 virtual ~VRoot() noexcept = default;
216};
217
218class VHostMemory : public VRoot
219{
220public:
221 virtual void* data() const noexcept = 0;
222 virtual std::size_t size() const noexcept = 0;
223 virtual DataType type() const noexcept = 0;
224};
225
226class VDimensionExpr : public VRoot
227{
228public:
229 virtual bool isConstant() const = 0;
230 virtual int64_t getConstantValue() const = 0;
231 virtual bool isSizeTensor() const = 0;
232};
233
234class VExprBuilder : public VRoot
235{
236public:
237 virtual IDimensionExpr const* constant(int64_t value) = 0;
238 virtual IDimensionExpr const* operation(
239 DimensionOperation op, IDimensionExpr const& first, IDimensionExpr const& second)
240 = 0;
241 virtual IDimensionExpr const* declareSizeTensor(
242 int32_t outputIndex, IDimensionExpr const& opt, IDimensionExpr const& upper)
243 = 0;
244};
245
246class VRuntime : public VRoot
247{
248public:
249 virtual IRuntime* getPImpl() noexcept = 0;
250 virtual nvinfer1::ICudaEngine* deserializeCudaEngine(void const* blob, std::size_t size) noexcept = 0;
251 virtual nvinfer1::ICudaEngine* deserializeCudaEngine(IStreamReader& streamReader) noexcept = 0;
252 virtual void setDLACore(int32_t dlaCore) noexcept = 0;
253 virtual int32_t getDLACore() const noexcept = 0;
254 virtual int32_t getNbDLACores() const noexcept = 0;
255 virtual void setGpuAllocator(IGpuAllocator* allocator) noexcept = 0;
256 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
257 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
258 virtual ILogger* getLogger() const noexcept = 0;
259 virtual bool setMaxThreads(int32_t maxThreads) noexcept = 0;
260 virtual int32_t getMaxThreads() const noexcept = 0;
261 virtual void setTemporaryDirectory(char const*) noexcept = 0;
262 virtual char const* getTemporaryDirectory() const noexcept = 0;
263 virtual void setTempfileControlFlags(TempfileControlFlags) noexcept = 0;
264 virtual TempfileControlFlags getTempfileControlFlags() const noexcept = 0;
265 virtual IPluginRegistry& getPluginRegistry() noexcept = 0;
266 virtual void setPluginRegistryParent(IPluginRegistry* parent) noexcept = 0;
267 virtual IRuntime* loadRuntime(char const* path) noexcept = 0;
268 virtual void setEngineHostCodeAllowed(bool allowed) noexcept = 0;
269 virtual bool getEngineHostCodeAllowed() const noexcept = 0;
270 // Added in TensorRT version 10.7
271 virtual nvinfer1::ICudaEngine* deserializeCudaEngineV2(IStreamReaderV2& streamReader) noexcept = 0;
272};
273
274class VRefitter : public VRoot
275{
276public:
277 virtual IRefitter* getPImpl() noexcept = 0;
278 virtual bool setWeights(char const* layerName, WeightsRole role, const Weights weights) noexcept = 0;
279 virtual bool refitCudaEngine() noexcept = 0;
280 virtual int32_t getMissing(int32_t size, char const** layerNames, WeightsRole* roles) noexcept = 0;
281 virtual int32_t getAll(int32_t size, char const** layerNames, WeightsRole* roles) noexcept = 0;
282 virtual bool setDynamicRange(char const* tensorName, float min, float max) noexcept = 0;
283 virtual float getDynamicRangeMin(char const* tensorName) const noexcept = 0;
284 virtual float getDynamicRangeMax(char const* tensorName) const noexcept = 0;
285 virtual int32_t getTensorsWithDynamicRange(int32_t size, char const** tensorNames) const noexcept = 0;
286 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
287 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
288 virtual bool setNamedWeights(char const* name, Weights weights) noexcept = 0;
289 virtual int32_t getMissingWeights(int32_t size, char const** weightsNames) noexcept = 0;
290 virtual int32_t getAllWeights(int32_t size, char const** weightsNames) noexcept = 0;
291 virtual ILogger* getLogger() const noexcept = 0;
292 virtual bool setMaxThreads(int32_t maxThreads) noexcept = 0;
293 virtual int32_t getMaxThreads() const noexcept = 0;
294 virtual bool setNamedWeightsWithLocation(char const* name, Weights weights, TensorLocation location) noexcept = 0;
295 virtual Weights getNamedWeights(char const* weightsName) const noexcept = 0;
296 virtual TensorLocation getWeightsLocation(char const* weightsName) const noexcept = 0;
297 virtual bool unsetNamedWeights(char const* weightsName) noexcept = 0;
298 virtual void setWeightsValidation(bool weightsValidation) noexcept = 0;
299 virtual bool getWeightsValidation() const noexcept = 0;
300 virtual bool refitCudaEngineAsync(cudaStream_t stream) noexcept = 0;
301 virtual Weights getWeightsPrototype(char const* weightsName) const noexcept = 0;
302};
303
304class VOptimizationProfile : public VRoot
305{
306public:
307 virtual bool setDimensions(char const* inputName, OptProfileSelector select, Dims const& dims) noexcept = 0;
308 virtual Dims getDimensions(char const* inputName, OptProfileSelector select) const noexcept = 0;
309 virtual bool setShapeValues(
310 char const* inputName, OptProfileSelector select, int32_t const* values, int32_t nbValues) noexcept = 0;
311 virtual int32_t getNbShapeValues(char const* inputName) const noexcept = 0;
312 virtual int32_t const* getShapeValues(char const* inputName, OptProfileSelector select) const noexcept = 0;
313 virtual bool setExtraMemoryTarget(float target) noexcept = 0;
314 virtual float getExtraMemoryTarget() const noexcept = 0;
315 virtual bool isValid() const noexcept = 0;
316 // Added in TensorRT 10.11
317 TRT_NODISCARD virtual bool setShapeValuesV2(
318 char const* inputName, OptProfileSelector select, int64_t const* values, int32_t nbValues) noexcept = 0;
319 TRT_NODISCARD virtual int64_t const* getShapeValuesV2(
320 char const* inputName, OptProfileSelector select) const noexcept = 0;
321};
322
323class VCudaEngine : public VRoot
324{
325public:
326 virtual ICudaEngine* getPImpl() noexcept = 0;
327 virtual int32_t getNbLayers() const noexcept = 0;
328 virtual IHostMemory* serialize() const noexcept = 0;
329 virtual IExecutionContext* createExecutionContext(ExecutionContextAllocationStrategy strategy) noexcept = 0;
330 virtual bool isRefittable() const noexcept = 0;
331 virtual char const* getName() const noexcept = 0;
332 virtual int32_t getNbOptimizationProfiles() const noexcept = 0;
333 virtual int32_t const* getProfileTensorValues(
334 char const* tensorName, int32_t profileIndex, OptProfileSelector select) const noexcept = 0;
335 virtual EngineCapability getEngineCapability() const noexcept = 0;
336 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
337 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
338 virtual bool hasImplicitBatchDimension() const noexcept = 0;
339 virtual TacticSources getTacticSources() const noexcept = 0;
340 virtual ProfilingVerbosity getProfilingVerbosity() const noexcept = 0;
341 virtual IEngineInspector* createEngineInspector() const noexcept = 0;
342 virtual Dims getTensorShape(char const* tensorName) const noexcept = 0;
343 virtual DataType getTensorDataType(char const* tensorName) const noexcept = 0;
344 virtual TensorLocation getTensorLocation(char const* tensorName) const noexcept = 0;
345 virtual bool isShapeInferenceIO(char const* tensorName) const noexcept = 0;
346 virtual TensorIOMode getTensorIOMode(char const* tensorName) const noexcept = 0;
347 virtual int32_t getTensorBytesPerComponent(char const* tensorName) const noexcept = 0;
348 virtual int32_t getTensorComponentsPerElement(char const* tensorName) const noexcept = 0;
349 virtual TensorFormat getTensorFormat(char const* tensorName) const noexcept = 0;
350 virtual char const* getTensorFormatDesc(char const* tensorName) const noexcept = 0;
351 virtual int32_t getTensorVectorizedDim(char const* tensorName) const noexcept = 0;
352 virtual Dims getProfileShape(
353 char const* tensorName, int32_t profileIndex, OptProfileSelector select) const noexcept = 0;
354 virtual int32_t getNbIOTensors() const noexcept = 0;
355 virtual char const* getIOTensorName(int32_t index) const noexcept = 0;
356 virtual HardwareCompatibilityLevel getHardwareCompatibilityLevel() const noexcept = 0;
357 virtual int32_t getNbAuxStreams() const noexcept = 0;
358
359 virtual int32_t getTensorBytesPerComponentV2(char const* tensorName, int32_t profileIndex) const noexcept = 0;
360 virtual int32_t getTensorComponentsPerElementV2(char const* tensorName, int32_t profileIndex) const noexcept = 0;
361 virtual TensorFormat getTensorFormatV2(char const* tensorName, int32_t profileIndex) const noexcept = 0;
362 virtual char const* getTensorFormatDescV2(char const* tensorName, int32_t profileIndex) const noexcept = 0;
363 virtual int32_t getTensorVectorizedDimV2(char const* tensorName, int32_t profileIndex) const noexcept = 0;
364
365 virtual ISerializationConfig* createSerializationConfig() noexcept = 0;
366 virtual IHostMemory* serializeWithConfig(ISerializationConfig& config) const noexcept = 0;
367
368 virtual size_t getDeviceMemorySizeForProfile(int32_t profileIndex) const noexcept = 0;
369 virtual IRefitter* createRefitter(ILogger& logger) noexcept = 0;
370
371 virtual bool setWeightStreamingBudget(int64_t gpuMemoryBudget) noexcept = 0;
372 virtual int64_t getWeightStreamingBudget() const noexcept = 0;
373 virtual int64_t getMinimumWeightStreamingBudget() const noexcept = 0;
374 virtual int64_t getStreamableWeightsSize() const noexcept = 0;
375
376 virtual bool isDebugTensor(char const* name) const noexcept = 0;
377
378 // Added in TensorRT 10.1
379 virtual bool setWeightStreamingBudgetV2(int64_t gpuMemoryBudget) noexcept = 0;
380 virtual int64_t getWeightStreamingBudgetV2() const noexcept = 0;
381 virtual int64_t getWeightStreamingAutomaticBudget() const noexcept = 0;
382 virtual int64_t getWeightStreamingScratchMemorySize() const noexcept = 0;
383 virtual int64_t getDeviceMemorySizeV2() const noexcept = 0;
384 virtual int64_t getDeviceMemorySizeForProfileV2(int32_t profileIndex) const noexcept = 0;
385 // Added in TensorRT 10.11
386 TRT_NODISCARD virtual int64_t const* getProfileTensorValuesV2(
387 char const* tensorName, int32_t profileIndex, OptProfileSelector select) const noexcept = 0;
388 TRT_NODISCARD virtual IExecutionContext* createExecutionContextWithRuntimeConfig(
389 IRuntimeConfig* runtimeConfig) noexcept = 0;
390 TRT_NODISCARD virtual IRuntimeConfig* createRuntimeConfig() noexcept = 0;
391};
392
393class VExecutionContext : public VRoot
394{
395public:
396 virtual IExecutionContext* getPImpl() noexcept = 0;
397 virtual void setDebugSync(bool sync) noexcept = 0;
398 virtual bool getDebugSync() const noexcept = 0;
399 virtual void setProfiler(IProfiler*) noexcept = 0;
400 virtual IProfiler* getProfiler() const noexcept = 0;
401 virtual ICudaEngine const& getEngine() const noexcept = 0;
402 virtual void setName(char const* name) noexcept = 0;
403 virtual char const* getName() const noexcept = 0;
404 virtual void setDeviceMemory(void* memory) noexcept = 0;
405 virtual int32_t getOptimizationProfile() const noexcept = 0;
406 virtual bool allInputDimensionsSpecified() const noexcept = 0;
407 virtual bool allInputShapesSpecified() const noexcept = 0;
408 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
409 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
410 virtual bool executeV2(void* const* bindings) noexcept = 0;
411 virtual bool setOptimizationProfileAsync(int32_t profileIndex, cudaStream_t stream) noexcept = 0;
412 virtual void setEnqueueEmitsProfile(bool enqueueEmitsProfile) noexcept = 0;
413 virtual bool getEnqueueEmitsProfile() const noexcept = 0;
414 virtual bool reportToProfiler() const noexcept = 0;
415 virtual bool setInputShape(char const* tensorName, Dims const& dims) noexcept = 0;
416 virtual Dims getTensorShape(char const* tensorName) const noexcept = 0;
417 virtual Dims getTensorStrides(char const* tensorName) const noexcept = 0;
418 virtual bool setTensorAddress(char const* tensorName, void* data) noexcept = 0;
419 virtual void const* getTensorAddress(char const* tensorName) const noexcept = 0;
420 virtual bool setInputTensorAddress(char const* tensorName, void const* data) noexcept = 0;
421 virtual bool setOutputTensorAddress(char const* tensorName, void* data) noexcept = 0;
422 virtual int32_t inferShapes(int32_t nbMaxNames, char const** tensorNames) noexcept = 0;
423 virtual bool setInputConsumedEvent(cudaEvent_t event) noexcept = 0;
424 virtual cudaEvent_t getInputConsumedEvent() const noexcept = 0;
425 virtual void* getOutputTensorAddress(char const* tensorName) const noexcept = 0;
426 virtual bool setOutputAllocator(char const* tensorName, IOutputAllocator* outputAllocator) noexcept = 0;
427 virtual IOutputAllocator* getOutputAllocator(char const* name) noexcept = 0;
428 virtual int64_t getMaxOutputSize(char const* tensorName) const noexcept = 0;
429 virtual bool setTemporaryStorageAllocator(IGpuAllocator* allocator) noexcept = 0;
430 virtual IGpuAllocator* getTemporaryStorageAllocator() const noexcept = 0;
431 virtual bool enqueueV3(cudaStream_t stream) noexcept = 0;
432 virtual void setPersistentCacheLimit(size_t size) noexcept = 0;
433 virtual size_t getPersistentCacheLimit() const noexcept = 0;
434 virtual bool setNvtxVerbosity(ProfilingVerbosity verbosity) noexcept = 0;
435 virtual ProfilingVerbosity getNvtxVerbosity() const noexcept = 0;
436 virtual void setAuxStreams(cudaStream_t* auxStreams, int32_t nbStreams) noexcept = 0;
437 virtual bool setDebugListener(IDebugListener* listener) noexcept = 0;
438 virtual IDebugListener* getDebugListener() noexcept = 0;
439 virtual bool setTensorDebugState(char const* name, bool flag) noexcept = 0;
440 virtual bool getDebugState(char const* name) const noexcept = 0;
441 virtual bool setAllTensorsDebugState(bool flag) noexcept = 0;
442 virtual size_t updateDeviceMemorySizeForShapes() noexcept = 0;
443 virtual void setDeviceMemoryV2(void* memory, int64_t size) noexcept = 0;
444 TRT_NODISCARD virtual IRuntimeConfig* getRuntimeConfig() const noexcept = 0;
445 virtual bool setUnfusedTensorsDebugState(bool flag) noexcept = 0;
446 virtual bool getUnfusedTensorsDebugState() const noexcept = 0;
447};
448
449class VEngineInspector : public VRoot
450{
451public:
452 virtual IEngineInspector* getPImpl() noexcept = 0;
453 virtual bool setExecutionContext(IExecutionContext const* context) noexcept = 0;
454 virtual IExecutionContext const* getExecutionContext() const noexcept = 0;
455 virtual char const* getLayerInformation(int32_t layerIndex, LayerInformationFormat format) const noexcept = 0;
456 virtual char const* getEngineInformation(LayerInformationFormat format) const noexcept = 0;
457 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
458 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
459};
460
461class VTensor : public VRoot
462{
463public:
464 virtual void setName(char const* name) noexcept = 0;
465 virtual char const* getName() const noexcept = 0;
466 virtual void setDimensions(Dims const& dimensions) noexcept = 0;
467 virtual Dims getDimensions() const noexcept = 0;
468 virtual void setType(DataType type) noexcept = 0;
469 virtual DataType getType() const noexcept = 0;
470 virtual bool isNetworkInput() const noexcept = 0;
471 virtual bool isNetworkOutput() const noexcept = 0;
472 virtual void setAllowedFormats(TensorFormats formats) noexcept = 0;
473 virtual TensorFormats getAllowedFormats() const noexcept = 0;
474 virtual bool isShapeTensor() const noexcept = 0;
475 virtual bool isExecutionTensor() const noexcept = 0;
476 virtual void setDimensionName(int32_t index, char const* name) noexcept = 0;
477 virtual char const* getDimensionName(int32_t index) const noexcept = 0;
478};
479
480class VLayer : public VRoot
481{
482public:
483 virtual LayerType getType() const noexcept = 0;
484 virtual void setName(char const* name) noexcept = 0;
485 virtual char const* getName() const noexcept = 0;
486 virtual int32_t getNbInputs() const noexcept = 0;
487 virtual ITensor* getInput(int32_t index) const noexcept = 0;
488 virtual int32_t getNbOutputs() const noexcept = 0;
489 virtual ITensor* getOutput(int32_t index) const noexcept = 0;
490 virtual void setInput(int32_t index, ITensor& tensor) noexcept = 0;
491 virtual void setPrecision(DataType dataType) noexcept = 0;
492 virtual DataType getPrecision() const noexcept = 0;
493 virtual bool precisionIsSet() const noexcept = 0;
494 virtual void resetPrecision() noexcept = 0;
495 virtual void setOutputType(int32_t index, DataType dataType) noexcept = 0;
496 virtual DataType getOutputType(int32_t index) const noexcept = 0;
497 virtual bool outputTypeIsSet(int32_t index) const noexcept = 0;
498 virtual void resetOutputType(int32_t index) noexcept = 0;
499 virtual void setMetadata(char const* docString) noexcept = 0;
500 virtual char const* getMetadata() const noexcept = 0;
501};
502
503class VConvolutionLayer : public VRoot
504{
505public:
506 virtual void setNbOutputMaps(int64_t nbOutputMaps) noexcept = 0;
507 virtual int64_t getNbOutputMaps() const noexcept = 0;
508 virtual void setNbGroups(int64_t nbGroups) noexcept = 0;
509 virtual int64_t getNbGroups() const noexcept = 0;
510 virtual void setKernelWeights(Weights weights) noexcept = 0;
511 virtual Weights getKernelWeights() const noexcept = 0;
512 virtual void setBiasWeights(Weights weights) noexcept = 0;
513 virtual Weights getBiasWeights() const noexcept = 0;
514 virtual void setPrePadding(Dims const& padding) noexcept = 0;
515 virtual Dims getPrePadding() const noexcept = 0;
516 virtual void setPostPadding(Dims const& padding) noexcept = 0;
517 virtual Dims getPostPadding() const noexcept = 0;
518 virtual void setPaddingMode(PaddingMode paddingMode) noexcept = 0;
519 virtual PaddingMode getPaddingMode() const noexcept = 0;
520 virtual void setKernelSizeNd(Dims const& kernelSize) noexcept = 0;
521 virtual Dims getKernelSizeNd() const noexcept = 0;
522 virtual void setStrideNd(Dims const& stride) noexcept = 0;
523 virtual Dims getStrideNd() const noexcept = 0;
524 virtual void setPaddingNd(Dims const& padding) noexcept = 0;
525 virtual Dims getPaddingNd() const noexcept = 0;
526 virtual void setDilationNd(Dims const& dilation) noexcept = 0;
527 virtual Dims getDilationNd() const noexcept = 0;
528};
529
530class VActivationLayer : public VRoot
531{
532public:
533 virtual void setActivationType(ActivationType type) noexcept = 0;
534 virtual ActivationType getActivationType() const noexcept = 0;
535 virtual void setAlpha(float alpha) noexcept = 0;
536 virtual void setBeta(float beta) noexcept = 0;
537 virtual float getAlpha() const noexcept = 0;
538 virtual float getBeta() const noexcept = 0;
539};
540
541class VPoolingLayer : public VRoot
542{
543public:
544 virtual void setPoolingType(PoolingType type) noexcept = 0;
545 virtual PoolingType getPoolingType() const noexcept = 0;
546 virtual void setBlendFactor(float blendFactor) noexcept = 0;
547 virtual float getBlendFactor() const noexcept = 0;
548 virtual void setAverageCountExcludesPadding(bool exclusive) noexcept = 0;
549 virtual bool getAverageCountExcludesPadding() const noexcept = 0;
550 virtual void setPrePadding(Dims const& padding) noexcept = 0;
551 virtual Dims getPrePadding() const noexcept = 0;
552 virtual void setPostPadding(Dims const& padding) noexcept = 0;
553 virtual Dims getPostPadding() const noexcept = 0;
554 virtual void setPaddingMode(PaddingMode paddingMode) noexcept = 0;
555 virtual PaddingMode getPaddingMode() const noexcept = 0;
556 virtual void setWindowSizeNd(Dims const& windowSize) noexcept = 0;
557 virtual Dims getWindowSizeNd() const noexcept = 0;
558 virtual void setStrideNd(Dims const& stride) noexcept = 0;
559 virtual Dims getStrideNd() const noexcept = 0;
560 virtual void setPaddingNd(Dims const& padding) noexcept = 0;
561 virtual Dims getPaddingNd() const noexcept = 0;
562};
563
564class VLRNLayer : public VRoot
565{
566public:
567 virtual void setWindowSize(int64_t windowSize) noexcept = 0;
568 virtual int64_t getWindowSize() const noexcept = 0;
569 virtual void setAlpha(float alpha) noexcept = 0;
570 virtual float getAlpha() const noexcept = 0;
571 virtual void setBeta(float beta) noexcept = 0;
572 virtual float getBeta() const noexcept = 0;
573 virtual void setK(float k) noexcept = 0;
574 virtual float getK() const noexcept = 0;
575};
576
577class VScaleLayer : public VRoot
578{
579public:
580 virtual void setMode(ScaleMode mode) noexcept = 0;
581 virtual ScaleMode getMode() const noexcept = 0;
582 virtual void setShift(Weights shift) noexcept = 0;
583 virtual Weights getShift() const noexcept = 0;
584 virtual void setScale(Weights scale) noexcept = 0;
585 virtual Weights getScale() const noexcept = 0;
586 virtual void setPower(Weights power) noexcept = 0;
587 virtual Weights getPower() const noexcept = 0;
588 virtual int32_t getChannelAxis() const noexcept = 0;
589 virtual void setChannelAxis(int32_t channelAxis) noexcept = 0;
590};
591
592class VSoftMaxLayer : public VRoot
593{
594public:
595 virtual void setAxes(uint32_t axes) noexcept = 0;
596 virtual uint32_t getAxes() const noexcept = 0;
597};
598
599class VConcatenationLayer : public VRoot
600{
601public:
602 virtual void setAxis(int32_t axis) noexcept = 0;
603 virtual int32_t getAxis() const noexcept = 0;
604};
605
606class VDeconvolutionLayer : public VRoot
607{
608public:
609 virtual void setNbOutputMaps(int64_t nbOutputMaps) noexcept = 0;
610 virtual int64_t getNbOutputMaps() const noexcept = 0;
611 virtual void setNbGroups(int64_t nbGroups) noexcept = 0;
612 virtual int64_t getNbGroups() const noexcept = 0;
613 virtual void setKernelWeights(Weights weights) noexcept = 0;
614 virtual Weights getKernelWeights() const noexcept = 0;
615 virtual void setBiasWeights(Weights weights) noexcept = 0;
616 virtual Weights getBiasWeights() const noexcept = 0;
617 virtual void setPrePadding(Dims const& padding) noexcept = 0;
618 virtual Dims getPrePadding() const noexcept = 0;
619 virtual void setPostPadding(Dims const& padding) noexcept = 0;
620 virtual Dims getPostPadding() const noexcept = 0;
621 virtual void setPaddingMode(PaddingMode paddingMode) noexcept = 0;
622 virtual PaddingMode getPaddingMode() const noexcept = 0;
623 virtual void setKernelSizeNd(Dims const& kernelSize) noexcept = 0;
624 virtual Dims getKernelSizeNd() const noexcept = 0;
625 virtual void setStrideNd(Dims const& stride) noexcept = 0;
626 virtual Dims getStrideNd() const noexcept = 0;
627 virtual void setPaddingNd(Dims const& padding) noexcept = 0;
628 virtual Dims getPaddingNd() const noexcept = 0;
629 virtual void setDilationNd(Dims const& dilation) noexcept = 0;
630 virtual Dims getDilationNd() const noexcept = 0;
631};
632
633class VElementWiseLayer : public VRoot
634{
635public:
636 virtual void setOperation(ElementWiseOperation op) noexcept = 0;
637 virtual ElementWiseOperation getOperation() const noexcept = 0;
638};
639
640class VGatherLayer : public VRoot
641{
642public:
643 virtual void setGatherAxis(int32_t axis) noexcept = 0;
644 virtual int32_t getGatherAxis() const noexcept = 0;
645 virtual void setNbElementWiseDims(int32_t k) noexcept = 0;
646 virtual int32_t getNbElementWiseDims() const noexcept = 0;
647 virtual void setMode(GatherMode mode) noexcept = 0;
648 virtual GatherMode getMode() const noexcept = 0;
649};
650
651class VUnaryLayer : public VRoot
652{
653public:
654 virtual void setOperation(UnaryOperation op) noexcept = 0;
655 virtual UnaryOperation getOperation() const noexcept = 0;
656};
657
658class VReduceLayer : public VRoot
659{
660public:
661 virtual void setOperation(ReduceOperation op) noexcept = 0;
662 virtual ReduceOperation getOperation() const noexcept = 0;
663 virtual void setReduceAxes(uint32_t reduceAxes) noexcept = 0;
664 virtual uint32_t getReduceAxes() const noexcept = 0;
665 virtual void setKeepDimensions(bool keepDimensions) noexcept = 0;
666 virtual bool getKeepDimensions() const noexcept = 0;
667};
668
669class VPaddingLayer : public VRoot
670{
671public:
672 virtual void setPrePaddingNd(Dims const& padding) noexcept = 0;
673 virtual Dims getPrePaddingNd() const noexcept = 0;
674 virtual void setPostPaddingNd(Dims const& padding) noexcept = 0;
675 virtual Dims getPostPaddingNd() const noexcept = 0;
676};
677
678class VShuffleLayer : public VRoot
679{
680public:
681 virtual void setFirstTranspose(Permutation const& permutation) noexcept = 0;
682 virtual Permutation const& getFirstTranspose() const noexcept = 0;
683 virtual void setReshapeDimensions(Dims const& dimensions) noexcept = 0;
684 virtual Dims getReshapeDimensions() const noexcept = 0;
685 virtual void setSecondTranspose(Permutation const& permutation) noexcept = 0;
686 virtual Permutation const& getSecondTranspose() const noexcept = 0;
687 virtual void setZeroIsPlaceholder(bool zeroIsPlaceholder) noexcept = 0;
688 virtual bool getZeroIsPlaceholder() const noexcept = 0;
689};
690
691class VSliceLayer : public VRoot
692{
693public:
694 virtual void setStart(Dims const& start) noexcept = 0;
695 virtual Dims getStart() const noexcept = 0;
696 virtual void setSize(Dims const& size) noexcept = 0;
697 virtual Dims getSize() const noexcept = 0;
698 virtual void setStride(Dims const& stride) noexcept = 0;
699 virtual Dims getStride() const noexcept = 0;
700 virtual void setMode(SampleMode mode) noexcept = 0;
701 virtual SampleMode getMode() const noexcept = 0;
702 virtual void setAxes(Dims const& axes) noexcept = 0;
703 virtual Dims getAxes() const noexcept = 0;
704};
705
706class VShapeLayer : public VRoot
707{
708public:
709};
710
711class VTopKLayer : public VRoot
712{
713public:
714 virtual void setOperation(TopKOperation op) noexcept = 0;
715 virtual TopKOperation getOperation() const noexcept = 0;
716 virtual void setK(int32_t k) noexcept = 0;
717 virtual int32_t getK() const noexcept = 0;
718 virtual void setReduceAxes(uint32_t reduceAxes) noexcept = 0;
719 virtual uint32_t getReduceAxes() const noexcept = 0;
720};
721
722class VMatrixMultiplyLayer : public VRoot
723{
724public:
725 virtual void setOperation(int32_t index, MatrixOperation op) noexcept = 0;
726 virtual MatrixOperation getOperation(int32_t index) const noexcept = 0;
727};
728
729class VNonZeroLayer : public VRoot
730{
731public:
732};
733
734class VRaggedSoftMaxLayer : public VRoot
735{
736public:
737};
738
739class VIdentityLayer : public VRoot
740{
741public:
742};
743
744class VCastLayer : public VRoot
745{
746public:
747 virtual void setToType(DataType toType) noexcept = 0;
748 virtual DataType getToType() const noexcept = 0;
749};
750
751class VConstantLayer : public VRoot
752{
753public:
754 virtual void setWeights(Weights weights) noexcept = 0;
755 virtual Weights getWeights() const noexcept = 0;
756 virtual void setDimensions(Dims const& dimensions) noexcept = 0;
757 virtual Dims getDimensions() const noexcept = 0;
758};
759
760class VParametricReLULayer : public VRoot
761{
762public:
763};
764
765class VResizeLayer : public VRoot
766{
767public:
768 virtual void setOutputDimensions(Dims const& dimensions) noexcept = 0;
769 virtual Dims getOutputDimensions() const noexcept = 0;
770 virtual void setScales(float const* scales, int32_t nbScales) noexcept = 0;
771 virtual int32_t getScales(int32_t size, float* scales) const noexcept = 0;
772 virtual void setResizeMode(InterpolationMode interpolationMode) noexcept = 0;
773 virtual InterpolationMode getResizeMode() const noexcept = 0;
774 virtual void setCoordinateTransformation(ResizeCoordinateTransformation coordTransform) noexcept = 0;
775 virtual ResizeCoordinateTransformation getCoordinateTransformation() const noexcept = 0;
776 virtual void setSelectorForSinglePixel(ResizeSelector selector) noexcept = 0;
777 virtual ResizeSelector getSelectorForSinglePixel() const noexcept = 0;
778 virtual void setNearestRounding(ResizeRoundMode value) noexcept = 0;
779 virtual ResizeRoundMode getNearestRounding() const noexcept = 0;
780 virtual void setCubicCoeff(float value) noexcept = 0;
781 virtual float getCubicCoeff() const noexcept = 0;
782 virtual void setExcludeOutside(bool value) noexcept = 0;
783 virtual bool getExcludeOutside() const noexcept = 0;
784};
785
786class VLoopBoundaryLayer : public VRoot
787{
788public:
789 virtual ILoop* getLoop() const noexcept = 0;
790};
791
792class VRecurrenceLayer : public VRoot
793{
794public:
795};
796
797class VLoopOutputLayer : public VRoot
798{
799public:
800 virtual LoopOutput getLoopOutput() const noexcept = 0;
801 virtual void setAxis(int32_t axis) noexcept = 0;
802 virtual int32_t getAxis() const noexcept = 0;
803};
804
805class VTripLimitLayer : public VRoot
806{
807public:
808 virtual TripLimit getTripLimit() const noexcept = 0;
809};
810
811class VIteratorLayer : public VRoot
812{
813public:
814 virtual void setAxis(int32_t axis) noexcept = 0;
815 virtual int32_t getAxis() const noexcept = 0;
816 virtual void setReverse(bool reverse) noexcept = 0;
817 virtual bool getReverse() const noexcept = 0;
818};
819class VLoop : public VRoot
820{
821public:
822 virtual IRecurrenceLayer* addRecurrence(ITensor& initialValue) noexcept = 0;
823 virtual ITripLimitLayer* addTripLimit(ITensor& tensor, TripLimit limit) noexcept = 0;
824 virtual IIteratorLayer* addIterator(ITensor& tensor, int32_t axis = 0, bool reverse = false) noexcept = 0;
825 virtual ILoopOutputLayer* addLoopOutput(ITensor& tensor, LoopOutput outputKind, int32_t axis = 0) noexcept = 0;
826 virtual void setName(char const* name) noexcept = 0;
827 virtual char const* getName() const noexcept = 0;
828};
829
830class VConditionalBoundaryLayer : public VRoot
831{
832public:
833 virtual IIfConditional* getConditional() const noexcept = 0;
834};
835
836class VConditionLayer : public VRoot
837{
838public:
839};
840
841class VConditionalInputLayer : public VRoot
842{
843public:
844};
845
846class VConditionalOutputLayer : public VRoot
847{
848public:
849};
850
851class VIfConditional : public VRoot
852{
853public:
854 virtual IConditionLayer* setCondition(ITensor& tensor) noexcept = 0;
855 virtual IIfConditionalInputLayer* addInput(ITensor& tensor) noexcept = 0;
856 virtual IIfConditionalOutputLayer* addOutput(ITensor& trueTensor, ITensor& falseTensor) noexcept = 0;
857 virtual void setName(char const* name) noexcept = 0;
858 virtual char const* getName() const noexcept = 0;
859};
860
861class VSelectLayer : public VRoot
862{
863};
864
865class VAssertionLayer : public VRoot
866{
867public:
868 virtual void setMessage(char const* message) noexcept = 0;
869 virtual char const* getMessage() const noexcept = 0;
870};
871
872class VFillLayer : public VRoot
873{
874public:
875 virtual void setDimensions(Dims const& dimensions) noexcept = 0;
876 virtual Dims getDimensions() const noexcept = 0;
877 virtual void setOperation(FillOperation op) noexcept = 0;
878 virtual FillOperation getOperation() const noexcept = 0;
879 virtual void setAlpha(double alpha) noexcept = 0;
880 virtual double getAlpha() const noexcept = 0;
881 virtual void setBeta(double beta) noexcept = 0;
882 virtual double getBeta() const noexcept = 0;
883 virtual void setAlphaInt64(int64_t alpha) noexcept = 0;
884 virtual int64_t getAlphaInt64() const noexcept = 0;
885 virtual void setBetaInt64(int64_t beta) noexcept = 0;
886 virtual int64_t getBetaInt64() const noexcept = 0;
887 virtual bool isAlphaBetaInt64() const noexcept = 0;
888 virtual DataType getToType() const noexcept = 0;
889 virtual void setToType(DataType toType) noexcept = 0;
890};
891
892class VQuantizeLayer : public VRoot
893{
894public:
895 virtual int32_t getAxis() const noexcept = 0;
896 virtual void setAxis(int32_t axis) noexcept = 0;
897 virtual DataType getToType() const noexcept = 0;
898 virtual void setToType(DataType toType) noexcept = 0;
899};
900
901class VDequantizeLayer : public VRoot
902{
903public:
904 virtual int32_t getAxis() const noexcept = 0;
905 virtual void setAxis(int32_t axis) noexcept = 0;
906 virtual DataType getToType() const noexcept = 0;
907 virtual void setToType(DataType toType) noexcept = 0;
908};
909
910class VDynamicQuantizeLayer : public VRoot
911{
912public:
913 virtual int32_t getAxis() const noexcept = 0;
914 virtual void setAxis(int32_t axis) noexcept = 0;
915 virtual int32_t getBlockSize() const noexcept = 0;
916 virtual void setBlockSize(int32_t axis) noexcept = 0;
917 virtual DataType getScaleType() const noexcept = 0;
918 virtual void setScaleType(DataType axis) noexcept = 0;
919 virtual DataType getToType() const noexcept = 0;
920 virtual void setToType(DataType toType) noexcept = 0;
921};
922
923class VScatterLayer : public VRoot
924{
925public:
926 virtual void setMode(ScatterMode mode) noexcept = 0;
927 virtual ScatterMode getMode() const noexcept = 0;
928 virtual void setAxis(int32_t axis) noexcept = 0;
929 virtual int32_t getAxis() const noexcept = 0;
930}; // class VScatterLayer
931
932class VEinsumLayer : public VRoot
933{
934public:
935 virtual bool setEquation(char const* equation) noexcept = 0;
936 virtual char const* getEquation() const noexcept = 0;
937};
938
939class VOneHotLayer : public VRoot
940{
941public:
942 virtual int32_t getAxis() const noexcept = 0;
943 virtual void setAxis(int32_t axis) noexcept = 0;
944}; // class VOneHotLayer
945
946class VGridSampleLayer : public VRoot
947{
948public:
949 virtual void setInterpolationMode(InterpolationMode mode) noexcept = 0;
950 virtual InterpolationMode getInterpolationMode() const noexcept = 0;
951 virtual void setAlignCorners(bool alignCorners) noexcept = 0;
952 virtual bool getAlignCorners() const noexcept = 0;
953 virtual bool setSampleMode(SampleMode mode) noexcept = 0;
954 virtual SampleMode getSampleMode() const noexcept = 0;
955}; // class VGridSampleLayer
956
957class VNMSLayer : public VRoot
958{
959public:
960 virtual void setBoundingBoxFormat(BoundingBoxFormat fmt) noexcept = 0;
961 virtual BoundingBoxFormat getBoundingBoxFormat() const noexcept = 0;
962 virtual void setTopKBoxLimit(int32_t limit) noexcept = 0;
963 virtual int32_t getTopKBoxLimit() const noexcept = 0;
964}; // class VNMSLayer
965
966class VReverseSequenceLayer : public VRoot
967{
968public:
969 virtual void setBatchAxis(int32_t batchAxis) noexcept = 0;
970 virtual int32_t getBatchAxis() const noexcept = 0;
971
972 virtual void setSequenceAxis(int32_t sequenceAxis) noexcept = 0;
973 virtual int32_t getSequenceAxis() const noexcept = 0;
974}; // class VReverseSequenceLayer
975
976class VNormalizationLayer : public VRoot
977{
978public:
979 virtual void setEpsilon(float eps) noexcept = 0;
980 virtual float getEpsilon() const noexcept = 0;
981 virtual void setAxes(uint32_t axesMask) noexcept = 0;
982 virtual uint32_t getAxes() const noexcept = 0;
983 virtual void setNbGroups(int64_t nbGroups) noexcept = 0;
984 virtual int64_t getNbGroups() const noexcept = 0;
985 virtual void setComputePrecision(DataType type) noexcept = 0;
986 virtual DataType getComputePrecision() const noexcept = 0;
987}; // class VNormalizationLayer
988
989class VSqueezeLayer : public VRoot
990{
991};
992
993class VUnsqueezeLayer : public VRoot
994{
995};
996
997class VCumulativeLayer : public VRoot
998{
999public:
1000 virtual bool setOperation(CumulativeOperation op) noexcept = 0;
1001 virtual CumulativeOperation getOperation() const noexcept = 0;
1002 virtual void setExclusive(bool exclusive) noexcept = 0;
1003 virtual bool getExclusive() const noexcept = 0;
1004 virtual void setReverse(bool reverse) noexcept = 0;
1005 virtual bool getReverse() const noexcept = 0;
1006}; // class VCumulativeLayer
1007
1008
1009class VNetworkDefinition : public VRoot
1010{
1011public:
1012 virtual ITensor* addInput(char const* name, DataType type, Dims const& dimensions) noexcept = 0;
1013 virtual void markOutput(ITensor& tensor) noexcept = 0;
1014 virtual IActivationLayer* addActivation(ITensor& input, ActivationType type) noexcept = 0;
1015 virtual ILRNLayer* addLRN(ITensor& input, int64_t window, float alpha, float beta, float k) noexcept = 0;
1016 virtual IScaleLayer* addScale(
1017 ITensor& input, ScaleMode mode, Weights shift, Weights scale, Weights power) noexcept = 0;
1018 virtual ISoftMaxLayer* addSoftMax(ITensor& input) noexcept = 0;
1019 virtual IConcatenationLayer* addConcatenation(ITensor* const* inputs, int32_t nbInputs) noexcept = 0;
1020 virtual IElementWiseLayer* addElementWise(ITensor& input1, ITensor& input2, ElementWiseOperation op) noexcept = 0;
1021 virtual IUnaryLayer* addUnary(ITensor& input, UnaryOperation operation) noexcept = 0;
1022 virtual IShuffleLayer* addShuffle(ITensor& input) noexcept = 0;
1023 virtual int32_t getNbLayers() const noexcept = 0;
1024 virtual ILayer* getLayer(int32_t index) const noexcept = 0;
1025 virtual int32_t getNbInputs() const noexcept = 0;
1026 virtual ITensor* getInput(int32_t index) const noexcept = 0;
1027 virtual int32_t getNbOutputs() const noexcept = 0;
1028 virtual ITensor* getOutput(int32_t index) const noexcept = 0;
1029 virtual IReduceLayer* addReduce(
1030 ITensor& input, ReduceOperation operation, uint32_t reduceAxes, bool keepDimensions) noexcept
1031 = 0;
1032 virtual ITopKLayer* addTopK(ITensor& input, TopKOperation op, int32_t k, uint32_t reduceAxes) noexcept = 0;
1033 virtual IGatherLayer* addGather(ITensor& data, ITensor& indices, int32_t axis) noexcept = 0;
1034 virtual IRaggedSoftMaxLayer* addRaggedSoftMax(ITensor& input, ITensor& bounds) noexcept = 0;
1035 virtual IMatrixMultiplyLayer* addMatrixMultiply(
1036 ITensor& input0, MatrixOperation op0, ITensor& input1, MatrixOperation op1) noexcept = 0;
1037 virtual IConstantLayer* addConstant(Dims const& dimensions, Weights weights) noexcept = 0;
1038 virtual IIdentityLayer* addIdentity(ITensor& input) noexcept = 0;
1039 virtual void removeTensor(ITensor& tensor) noexcept = 0;
1040 virtual void unmarkOutput(ITensor& tensor) noexcept = 0;
1041 virtual ISliceLayer* addSlice(ITensor& input, Dims const& start, Dims const& size, Dims const& stride) noexcept = 0;
1042 virtual void setName(char const* name) noexcept = 0;
1043 virtual char const* getName() const noexcept = 0;
1044 virtual IShapeLayer* addShape(ITensor& input) noexcept = 0;
1045 virtual bool hasImplicitBatchDimension() const noexcept = 0;
1046 virtual bool markOutputForShapes(ITensor& tensor) noexcept = 0;
1047 virtual bool unmarkOutputForShapes(ITensor& tensor) noexcept = 0;
1048 virtual IParametricReLULayer* addParametricReLU(ITensor& input, ITensor& slope) noexcept = 0;
1049 virtual IConvolutionLayer* addConvolutionNd(
1050 ITensor& input, int64_t nbOutputMaps, Dims const& kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
1051 = 0;
1052 virtual IPoolingLayer* addPoolingNd(ITensor& input, PoolingType type, Dims const& windowSize) noexcept = 0;
1053 virtual IDeconvolutionLayer* addDeconvolutionNd(
1054 ITensor& input, int64_t nbOutputMaps, Dims const& kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
1055 = 0;
1056 virtual IScaleLayer* addScaleNd(
1057 ITensor& input, ScaleMode mode, Weights shift, Weights scale, Weights power, int32_t channelAxis) noexcept = 0;
1058 virtual IResizeLayer* addResize(ITensor& input) noexcept = 0;
1059 virtual ILoop* addLoop() noexcept = 0;
1060 virtual ISelectLayer* addSelect(ITensor& condition, ITensor& thenInput, ITensor& elseInput) noexcept = 0;
1061 virtual IFillLayer* addFill(Dims const& dimensions, FillOperation op) noexcept = 0;
1062 virtual IPaddingLayer* addPaddingNd(ITensor& input, Dims const& prePadding, Dims const& postPadding) noexcept = 0;
1063 virtual bool setWeightsName(Weights weights, char const* name) noexcept = 0;
1064 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
1065 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
1066 virtual IGatherLayer* addGatherV2(ITensor& data, ITensor& indices, GatherMode mode) noexcept = 0;
1067 virtual IIfConditional* addIfConditional() noexcept = 0;
1068 virtual IScatterLayer* addScatter(ITensor& data, ITensor& indices, ITensor& updates, ScatterMode mode) noexcept = 0;
1069 virtual IEinsumLayer* addEinsum(ITensor* const* inputs, int32_t nbInputs, char const* equation) noexcept = 0;
1070 virtual IAssertionLayer* addAssertion(ITensor& condition, char const* message) noexcept = 0;
1071 virtual IOneHotLayer* addOneHot(ITensor& indices, ITensor& values, ITensor& depth, int32_t axis) noexcept = 0;
1072 virtual INonZeroLayer* addNonZero(ITensor& input) noexcept = 0;
1073 virtual IGridSampleLayer* addGridSample(ITensor& input, ITensor& grid) noexcept = 0;
1074 virtual INMSLayer* addNMS(ITensor& boxes, ITensor& scores, ITensor& maxOutputBoxesPerClass) noexcept = 0;
1075 virtual IReverseSequenceLayer* addReverseSequence(ITensor& input, ITensor& sequenceLens) noexcept = 0;
1076 virtual INormalizationLayer* addNormalization(
1077 ITensor& input, ITensor& scale, ITensor& bias, uint32_t axesMask) noexcept = 0;
1078 virtual ICastLayer* addCast(ITensor& input, DataType toType) noexcept = 0;
1079 virtual IBuilder& getBuilder() const noexcept = 0;
1080 virtual NetworkDefinitionCreationFlags getFlags() const noexcept = 0;
1081 virtual bool getFlag(NetworkDefinitionCreationFlag networkDefinitionCreationFlag) const noexcept = 0;
1082 virtual IQuantizeLayer* addQuantizeV2(ITensor& input, ITensor& scale, DataType outputType) noexcept = 0;
1083 virtual IDequantizeLayer* addDequantizeV2(ITensor& input, ITensor& scale, DataType outputType) noexcept = 0;
1084 virtual IFillLayer* addFillV2(Dims const& dimensions, FillOperation op, DataType outputType) noexcept = 0;
1085 virtual bool markDebug(ITensor& tensor) noexcept = 0;
1086 virtual bool unmarkDebug(ITensor& tensor) noexcept = 0;
1087 virtual bool isDebugTensor(nvinfer1::ITensor const& tensor) const noexcept = 0;
1088 virtual bool markWeightsRefittable(char const* name) noexcept = 0;
1089 virtual bool unmarkWeightsRefittable(char const* name) noexcept = 0;
1090 virtual bool areWeightsMarkedRefittable(char const* name) const noexcept = 0;
1091 virtual ISqueezeLayer* addSqueeze(ITensor& input, ITensor& axes) noexcept = 0;
1092 virtual IUnsqueezeLayer* addUnsqueeze(ITensor& input, ITensor& axes) noexcept = 0;
1093 virtual IDynamicQuantizeLayer* addDynamicQuantize(
1094 ITensor& input, int32_t axis, int32_t blockSize, DataType toType, DataType scaleType) noexcept = 0;
1095 virtual ICumulativeLayer* addCumulative(
1096 ITensor& input, ITensor& axis, CumulativeOperation operation, bool exclusive, bool reverse) noexcept = 0;
1097 virtual bool markUnfusedTensorsAsDebugTensors() noexcept = 0;
1098 virtual bool unmarkUnfusedTensorsAsDebugTensors() noexcept = 0;
1099};
1100
1101
1102class VTimingCache : public VRoot
1103{
1104public:
1105 virtual nvinfer1::IHostMemory* serialize() const noexcept = 0;
1106 virtual bool combine(ITimingCache const& inputCache, bool ignoreMismatch) noexcept = 0;
1107 virtual bool reset() noexcept = 0;
1108 virtual int64_t queryKeys(TimingCacheKey* keyBuffer, int64_t capacity) const noexcept = 0;
1109 virtual TimingCacheValue query(TimingCacheKey const& key) const noexcept = 0;
1110 virtual bool update(TimingCacheKey const& key, TimingCacheValue const& value) noexcept = 0;
1111};
1112
1113class VBuilderConfig : public VRoot
1114{
1115public:
1116 virtual void setAvgTimingIterations(int32_t avgTiming) noexcept = 0;
1117 virtual int32_t getAvgTimingIterations() const noexcept = 0;
1118 virtual void setEngineCapability(EngineCapability capability) noexcept = 0;
1119 virtual EngineCapability getEngineCapability() const noexcept = 0;
1120 virtual void setFlags(BuilderFlags builderFlags) noexcept = 0;
1121 virtual BuilderFlags getFlags() const noexcept = 0;
1122 virtual void clearFlag(BuilderFlag builderFlag) noexcept = 0;
1123 virtual void setFlag(BuilderFlag builderFlag) noexcept = 0;
1124 virtual bool getFlag(BuilderFlag builderFlag) const noexcept = 0;
1125 virtual void setDeviceType(ILayer const* layer, DeviceType deviceType) noexcept = 0;
1126 virtual DeviceType getDeviceType(ILayer const* layer) const noexcept = 0;
1127 virtual bool isDeviceTypeSet(ILayer const* layer) const noexcept = 0;
1128 virtual void resetDeviceType(ILayer const* layer) noexcept = 0;
1129 virtual bool canRunOnDLA(ILayer const* layer) const noexcept = 0;
1130 virtual void setDLACore(int32_t dlaCore) noexcept = 0;
1131 virtual int32_t getDLACore() const noexcept = 0;
1132 virtual void setDefaultDeviceType(DeviceType deviceType) noexcept = 0;
1133 virtual DeviceType getDefaultDeviceType() const noexcept = 0;
1134 virtual void reset() noexcept = 0;
1135 virtual void setProfileStream(const cudaStream_t stream) noexcept = 0;
1136 virtual cudaStream_t getProfileStream() const noexcept = 0;
1137 virtual int32_t addOptimizationProfile(IOptimizationProfile const* profile) noexcept = 0;
1138 virtual int32_t getNbOptimizationProfiles() const noexcept = 0;
1139 virtual void setProfilingVerbosity(ProfilingVerbosity verbosity) noexcept = 0;
1140 virtual ProfilingVerbosity getProfilingVerbosity() const noexcept = 0;
1141 virtual bool setTacticSources(TacticSources tacticSources) noexcept = 0;
1142 virtual TacticSources getTacticSources() const noexcept = 0;
1143 virtual nvinfer1::ITimingCache* createTimingCache(void const* blob, std::size_t size) const noexcept = 0;
1144 virtual bool setTimingCache(ITimingCache const& cache, bool ignoreMismatch) noexcept = 0;
1145 virtual nvinfer1::ITimingCache const* getTimingCache() const noexcept = 0;
1146 virtual void setMemoryPoolLimit(MemoryPoolType pool, std::size_t poolSize) noexcept = 0;
1147 virtual std::size_t getMemoryPoolLimit(MemoryPoolType pool) const noexcept = 0;
1148 virtual void setPreviewFeature(PreviewFeature feature, bool enable) noexcept = 0;
1149 virtual bool getPreviewFeature(PreviewFeature feature) const noexcept = 0;
1150 virtual void setBuilderOptimizationLevel(int32_t level) noexcept = 0;
1151 virtual int32_t getBuilderOptimizationLevel() const noexcept = 0;
1152 virtual void setHardwareCompatibilityLevel(HardwareCompatibilityLevel hardwareCompatibilityLevel) noexcept = 0;
1153 virtual HardwareCompatibilityLevel getHardwareCompatibilityLevel() const noexcept = 0;
1154 virtual void setPluginsToSerialize(char const* const* paths, int32_t nbPaths) noexcept = 0;
1155 virtual char const* getPluginToSerialize(int32_t index) const noexcept = 0;
1156 virtual int32_t getNbPluginsToSerialize() const noexcept = 0;
1157 virtual void setMaxAuxStreams(int32_t nbStreams) noexcept = 0;
1158 virtual int32_t getMaxAuxStreams() const noexcept = 0;
1159 virtual void setProgressMonitor(IProgressMonitor* monitor) noexcept = 0;
1160 virtual IProgressMonitor* getProgressMonitor() const noexcept = 0;
1161 virtual void setRuntimePlatform(RuntimePlatform runtimePlatform) noexcept = 0;
1162 virtual RuntimePlatform getRuntimePlatform() const noexcept = 0;
1163 virtual void setMaxNbTactics(int32_t maxTactics) noexcept = 0;
1164 virtual int32_t getMaxNbTactics() const noexcept = 0;
1165 virtual bool setTilingOptimizationLevel(TilingOptimizationLevel level) noexcept = 0;
1166 virtual TilingOptimizationLevel getTilingOptimizationLevel() const noexcept = 0;
1167 virtual bool setL2LimitForTiling(int64_t size) noexcept = 0;
1168 virtual int64_t getL2LimitForTiling() const noexcept = 0;
1169 TRT_NODISCARD virtual bool setNbComputeCapabilities(int32_t maxNbComputeCapabilities) noexcept = 0;
1170 TRT_NODISCARD virtual int32_t getNbComputeCapabilities() const noexcept = 0;
1171 TRT_NODISCARD virtual bool setComputeCapability(ComputeCapability computeCapability, int32_t index) noexcept = 0;
1172 TRT_NODISCARD virtual ComputeCapability getComputeCapability(int32_t index) const noexcept = 0;
1173};
1174
1175class VSerializationConfig : public VRoot
1176{
1177public:
1178 virtual bool setFlags(SerializationFlags serializationFlags) noexcept = 0;
1179 virtual SerializationFlags getFlags() const noexcept = 0;
1180 virtual bool clearFlag(SerializationFlag serializationFlag) noexcept = 0;
1181 virtual bool setFlag(SerializationFlag serializationFlag) noexcept = 0;
1182 virtual bool getFlag(SerializationFlag serializationFlag) const noexcept = 0;
1183};
1184
1185class VBuilder : public VRoot
1186{
1187public:
1188 virtual bool platformHasFastFp16() const noexcept = 0;
1189 virtual bool platformHasFastInt8() const noexcept = 0;
1190 virtual int32_t getMaxDLABatchSize() const noexcept = 0;
1191 virtual int32_t getNbDLACores() const noexcept = 0;
1192 virtual void setGpuAllocator(IGpuAllocator* allocator) noexcept = 0;
1193 virtual nvinfer1::IBuilderConfig* createBuilderConfig() noexcept = 0;
1194 virtual nvinfer1::INetworkDefinition* createNetworkV2(NetworkDefinitionCreationFlags flags) noexcept = 0;
1195 virtual nvinfer1::IOptimizationProfile* createOptimizationProfile() noexcept = 0;
1196 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
1197 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
1198 virtual void reset() noexcept = 0;
1199 virtual bool platformHasTf32() const noexcept = 0;
1200 virtual nvinfer1::IHostMemory* buildSerializedNetwork(
1201 INetworkDefinition& network, IBuilderConfig& config) noexcept = 0;
1202 virtual bool isNetworkSupported(INetworkDefinition const& network, IBuilderConfig const& config) const noexcept = 0;
1203 virtual ILogger* getLogger() const noexcept = 0;
1204 virtual bool setMaxThreads(int32_t maxThreads) noexcept = 0;
1205 virtual int32_t getMaxThreads() const noexcept = 0;
1206 virtual IPluginRegistry& getPluginRegistry() noexcept = 0;
1207};
1208
1209class VRuntimeConfig : public VRoot
1210{
1211public:
1212 virtual IRuntimeConfig* getPImpl() noexcept = 0;
1213 virtual void setExecutionContextAllocationStrategy(ExecutionContextAllocationStrategy strategy) noexcept = 0;
1214 virtual ExecutionContextAllocationStrategy getExecutionContextAllocationStrategy() const noexcept = 0;
1215 TRT_NODISCARD virtual nvinfer1::IRuntimeCache* createRuntimeCache() const noexcept = 0;
1216 virtual bool setRuntimeCache(nvinfer1::IRuntimeCache const& cache) noexcept = 0;
1217 TRT_NODISCARD virtual nvinfer1::IRuntimeCache* getRuntimeCache() const noexcept = 0;
1218 virtual void setDynamicShapesKernelSpecializationStrategy(
1219 DynamicShapesKernelSpecializationStrategy dynamicShapesKernelSpecializationStrategy) noexcept = 0;
1220 virtual DynamicShapesKernelSpecializationStrategy getDynamicShapesKernelSpecializationStrategy() const noexcept = 0;
1221};
1222
1223class VRuntimeCache : public VRoot
1224{
1225public:
1226 TRT_NODISCARD virtual nvinfer1::IHostMemory* serialize() const noexcept = 0;
1227 virtual bool deserialize(void const* blob, size_t size) noexcept = 0;
1228 virtual bool reset() noexcept = 0;
1229};
1230
1231} // namespace apiv
1232} // namespace nvinfer1
1233
1234// @endcond
1235
1236#endif // NV_INFER_RUNTIME_IMPL_H
getLogger
nvinfer1::ILogger * getLogger() noexcept
Return the logger object.
getPluginRegistry
nvinfer1::IPluginRegistry * getPluginRegistry() noexcept
Return the plugin registry.
TRT_NODISCARD
#define TRT_NODISCARD
A stand-in for [[nodiscard]] and [[nodiscard(REASON)]] that works with older compilers.
Definition: NvInferRuntimeBase.h:57
Dims
Structure to define the dimensions of a tensor.
IDebugListener
User-implemented callback for notification when value of a debug tensor is updated.
IErrorRecorder
Reference counted application-implemented error reporting interface for TensorRT objects.
IGpuAllocator
Application-implemented class for controlling allocation on the GPU.
IOutputAllocator
Callback from ExecutionContext::enqueueV3()
IProfiler
Application-implemented interface for profiling.
IProgressMonitor
Application-implemented progress reporting interface for TensorRT.
IStreamReader
Application-implemented class for reading data in a stream-based manner.
IStreamReaderV2
Application-implemented class for reading data in a stream-based manner asynchronously....
nvinfer1::IHostMemory
Class to handle library allocated memory that is accessible to the user.
Definition: NvInferRuntime.h:142
nvinfer1
The TensorRT API version 1 namespace.
nvinfer1::TacticSources
uint32_t TacticSources
Represents a collection of one or more TacticSource values combine using bitwise-OR operations.
Definition: NvInferRuntime.h:2327
nvinfer1::IOutputAllocator
v_1_0::IOutputAllocator IOutputAllocator
Definition: NvInferRuntime.h:3513
nvinfer1::ResizeSelector
ResizeSelector
The coordinate selector when resize to single pixel output.
Definition: NvInfer.h:3879
nvinfer1::EngineCapability
EngineCapability
List of supported engine capability flows.
Definition: NvInferRuntime.h:76
nvinfer1::DimensionOperation
DimensionOperation
An operation on two IDimensionExpr, which represent integer expressions used in dimension computation...
Definition: NvInferRuntime.h:179
nvinfer1::MemoryPoolType
MemoryPoolType
The type for memory pools used by TensorRT.
Definition: NvInfer.h:8297
nvinfer1::ScaleMode
ScaleMode
Controls how shift, scale and power are applied in a Scale layer.
Definition: NvInfer.h:1731
nvinfer1::RuntimePlatform
RuntimePlatform
Describes the intended runtime platform (operating system and CPU architecture) for the execution of ...
Definition: NvInfer.h:7897
nvinfer1::TensorIOMode
TensorIOMode
Definition of tensor IO Mode.
Definition: NvInferRuntimeBase.h:656
nvinfer1::HardwareCompatibilityLevel
HardwareCompatibilityLevel
Describes requirements of compatibility with GPU architectures other than that of the GPU on which th...
Definition: NvInfer.h:8416
nvinfer1::CumulativeOperation
CumulativeOperation
Enumerates the cumulative operations that may be performed by a Cumulative layer.
Definition: NvInfer.h:6363
nvinfer1::SerializationFlag
SerializationFlag
List of valid flags that the engine can enable when serializing the bytes.
Definition: NvInferRuntime.h:2368
nvinfer1::BoundingBoxFormat
BoundingBoxFormat
Representation of bounding box data used for the Boxes input tensor in INMSLayer.
Definition: NvInfer.h:5948
nvinfer1::ComputeCapability
ComputeCapability
Describes compute capability that an engine will be built for.
Definition: NvInfer.h:8465
nvinfer1::UnaryOperation
UnaryOperation
Enumerates the unary operations that may be performed by a Unary layer.
Definition: NvInfer.h:2620
nvinfer1::IProfiler
v_1_0::IProfiler IProfiler
Definition: NvInferRuntime.h:782
nvinfer1::DynamicShapesKernelSpecializationStrategy
DynamicShapesKernelSpecializationStrategy
Different kernel specialization strategies for dynamic shapes.
Definition: NvInferRuntime.h:2546
nvinfer1::IStreamReaderV2
v_1_0::IStreamReaderV2 IStreamReaderV2
Definition: NvInferRuntime.h:751
nvinfer1::ActivationType
ActivationType
Enumerates the types of activation to perform in an activation layer.
Definition: NvInfer.h:137
nvinfer1::TempfileControlFlags
uint32_t TempfileControlFlags
Represents a collection of one or more TempfileControlFlag values combined using bitwise-OR operation...
Definition: NvInferRuntime.h:860
nvinfer1::FillOperation
FillOperation
Enumerates the tensor fill operations that may performed by a fill layer.
Definition: NvInfer.h:4860
nvinfer1::ResizeRoundMode
ResizeRoundMode
The rounding mode for nearest neighbor resize.
Definition: NvInfer.h:3909
nvinfer1::PaddingMode
PaddingMode
Enumerates the modes of padding to perform in convolution, deconvolution and pooling layer,...
Definition: NvInfer.h:909
nvinfer1::TripLimit
TripLimit
Enum that describes kinds of trip limits.
Definition: NvInfer.h:4268
nvinfer1::NetworkDefinitionCreationFlags
uint32_t NetworkDefinitionCreationFlags
Represents one or more NetworkDefinitionCreationFlag flags using binary OR operations....
Definition: NvInfer.h:9462
nvinfer1::PreviewFeature
PreviewFeature
Define preview features.
Definition: NvInfer.h:8372
nvinfer1::TilingOptimizationLevel
TilingOptimizationLevel
Define the optimization levels for Tiling.
Definition: NvInfer.h:8491
nvinfer1::DataType
DataType
The type of weights and tensors.
Definition: NvInferRuntimeBase.h:143
nvinfer1::BuilderFlags
uint32_t BuilderFlags
Represents one or more BuilderFlag values using binary OR operations, e.g., 1U << BuilderFlag::kFP16 ...
Definition: NvInfer.h:7927
nvinfer1::DeviceType
DeviceType
The device that this layer/network will execute on.
Definition: NvInferRuntime.h:814
nvinfer1::LayerType
LayerType
The type values of layer classes.
Definition: NvInfer.h:58
nvinfer1::SampleMode
SampleMode
Controls how ISliceLayer and IGridSample handle out-of-bounds coordinates.
Definition: NvInfer.h:3078
nvinfer1::GatherMode
GatherMode
Control form of IGatherLayer.
Definition: NvInfer.h:2423
nvinfer1::IDebugListener
v_1_0::IDebugListener IDebugListener
Definition: NvInferRuntime.h:3554
nvinfer1::WeightsRole
WeightsRole
How a layer uses particular Weights.
Definition: NvInferRuntime.h:792
nvinfer1::TensorFormats
uint32_t TensorFormats
It is capable of representing one or more TensorFormat by binary OR operations, e....
Definition: NvInfer.h:129
nvinfer1::ProfilingVerbosity
ProfilingVerbosity
List of verbosity levels of layer information exposed in NVTX annotations and in IEngineInspector.
Definition: NvInferRuntime.h:2339
nvinfer1::NetworkDefinitionCreationFlag
NetworkDefinitionCreationFlag
List of immutable network properties expressed at network creation time. NetworkDefinitionCreationFla...
Definition: NvInfer.h:9473
nvinfer1::ElementWiseOperation
ElementWiseOperation
Enumerates the binary operations that may be performed by an ElementWise layer.
Definition: NvInfer.h:2333
nvinfer1::SerializationFlags
uint32_t SerializationFlags
Represents one or more SerializationFlag values using binary OR operations, e.g., 1U << Serialization...
Definition: NvInferRuntime.h:2358
nvinfer1::InterpolationMode
InterpolationMode
Enumerates various modes of interpolation.
Definition: NvInfer.h:3797
nvinfer1::BuilderFlag
BuilderFlag
List of valid modes that the builder can enable when creating an engine from a network definition.
Definition: NvInfer.h:7937
nvinfer1::TensorFormat
TensorFormat
Format of the input/output tensors.
Definition: NvInferRuntime.h:894
nvinfer1::ExecutionContextAllocationStrategy
ExecutionContextAllocationStrategy
Different memory allocation behaviors for IExecutionContext.
Definition: NvInferRuntime.h:2472
nvinfer1::TopKOperation
TopKOperation
Enumerates the operations that may be performed by a TopK layer.
Definition: NvInfer.h:3359
nvinfer1::ReduceOperation
ReduceOperation
Enumerates the reduce operations that may be performed by a Reduce layer.
Definition: NvInfer.h:2715
nvinfer1::LayerInformationFormat
LayerInformationFormat
The format in which the IEngineInspector prints the layer information.
Definition: NvInferRuntime.h:4470
nvinfer1::ScatterMode
ScatterMode
Control form of IScatterLayer.
Definition: NvInfer.h:5682
nvinfer1::MatrixOperation
MatrixOperation
Enumerates the operations that may be performed on a tensor by IMatrixMultiplyLayer before multiplica...
Definition: NvInfer.h:3485
nvinfer1::ResizeCoordinateTransformation
ResizeCoordinateTransformation
The resize coordinate transformation function.
Definition: NvInfer.h:3825
nvinfer1::LoopOutput
LoopOutput
Enum that describes kinds of loop outputs.
Definition: NvInfer.h:4240
nvinfer1::IStreamReader
v_1_0::IStreamReader IStreamReader
Definition: NvInferRuntime.h:674
nvinfer1::PoolingType
PoolingType
The type of pooling to perform in a pooling layer.
Definition: NvInfer.h:1345
nvinfer1::IProgressMonitor
v_1_0::IProgressMonitor IProgressMonitor
Definition: NvInfer.h:8607
nvinfer1::TensorLocation
TensorLocation
The location for tensor data storage, device or host.
Definition: NvInferRuntime.h:204
nvinfer1::OptProfileSelector
OptProfileSelector
When setting or querying optimization profile parameters (such as shape tensor inputs or dynamic dime...
Definition: NvInferRuntime.h:2001

  Copyright © 2024 NVIDIA Corporation
  Privacy Policy | Manage My Privacy | Do Not Sell or Share My Data | Terms of Service | Accessibility | Corporate Policies | Product Security | Contact