TensorRT for RTX 1.6.1
NvInferImpl.h
Go to the documentation of this file.
1/*
2 * SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: Apache-2.0
4 *
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
8 *
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 */
17
18#ifndef NV_INFER_IMPL_H
19#define NV_INFER_IMPL_H
20
21#include "NvInferLegacyDims.h"
22#include "NvInferRuntimeCommon.h"
23
24// @cond SuppressDoxyWarnings
25
26namespace nvinfer1
27{
28
29namespace v_1_0
30{
31class ILogger;
32class IProgressMonitor;
33} // namespace v_1_0
34using ILogger = v_1_0::ILogger;
35using IProgressMonitor = v_1_0::IProgressMonitor;
36
37namespace v_1_0
38{
39class IProfiler;
40} // namespace v_1_0
41using IProfiler = v_1_0::IProfiler;
42
43namespace v_1_0
44{
45class IOutputAllocator;
46} // namespace v_1_0
47using IOutputAllocator = v_1_0::IOutputAllocator;
48
49namespace v_1_0
50{
51class IDebugListener;
52} // namespace v_1_0
53using IDebugListener = v_1_0::IDebugListener;
54
55class IActivationLayer;
56class IAssertionLayer;
57class IAttention;
58class IBuilder;
59class IBuilderConfig;
60class IConcatenationLayer;
61class IConditionLayer;
62class IConstantLayer;
63class IConvolutionLayer;
64class ICudaEngine;
65class ICumulativeLayer;
66class IDeconvolutionLayer;
67class IDequantizeLayer;
68class IDimensionExpr;
69class IDynamicQuantizeLayer;
70class IEinsumLayer;
71class IElementWiseLayer;
72class IEngineInspector;
73class IExecutionContext;
74class IFillLayer;
75class IGatherLayer;
76class IGridSampleLayer;
77class IHostMemory;
78class IIdentityLayer;
79class ICastLayer;
80class IIfConditional;
81class IIfConditionalInputLayer;
82class IIfConditionalOutputLayer;
83class IIteratorLayer;
84class IKVCacheUpdateLayer;
85class ILayer;
86class ILoop;
87class ILoopOutputLayer;
88class ILRNLayer;
89class IMatrixMultiplyLayer;
90class IMoELayer;
91class IDistCollectiveLayer;
92class INetworkDefinition;
93class INormalizationLayer;
94class INMSLayer;
95class INonZeroLayer;
96class IOneHotLayer;
97class IOptimizationProfile;
98class IPaddingLayer;
99class IParametricReLULayer;
100class IPlugin;
101class IPluginExt;
102class IPluginFactory;
103class IPluginLayer;
104class IPluginRegistry;
105class IPluginV2Layer;
106class IRotaryEmbeddingLayer;
107class IRuntimeCache;
108class IRuntimeConfig;
109
110namespace v_1_0
111{
112class IPluginV3;
113} // namespace v_1_0
114using IPluginV3 = v_1_0::IPluginV3;
115
116namespace v_1_0
117{
118class IStreamReader;
119class IStreamWriter;
120} // namespace v_1_0
121using IStreamReader = v_1_0::IStreamReader;
122using IStreamWriter = v_1_0::IStreamWriter;
123namespace v_1_0
124{
125class IStreamReaderV2;
126} // namespace v_1_0
127using IStreamReaderV2 = v_1_0::IStreamReaderV2;
128
129class IPluginV3Layer;
130class IPoolingLayer;
131class IQuantizeLayer;
132class IRaggedSoftMaxLayer;
133class IRecurrenceLayer;
134class IReduceLayer;
135class IRefitter;
136class IResizeLayer;
137class IReverseSequenceLayer;
138class IRuntime;
139class IScaleLayer;
140class IScatterLayer;
141class ISelectLayer;
142class ISerializationConfig;
143class IShapeLayer;
144class IShuffleLayer;
145class ISliceLayer;
146class ISoftMaxLayer;
147class ISqueezeLayer;
148class ITensor;
149
150namespace v_1_0
151{
152struct TimingCacheKey;
153struct TimingCacheValue;
154} // namespace v_1_0
155using TimingCacheKey = v_1_0::TimingCacheKey;
156using TimingCacheValue = v_1_0::TimingCacheValue;
157
158class ITimingCache;
159class ITopKLayer;
160class ITripLimitLayer;
161class IUnaryLayer;
162class IUnsqueezeLayer;
163struct Permutation;
164class Weights;
165
166enum class ActivationType : int32_t;
167enum class AttentionIOForm : int32_t;
168enum class AttentionNormalizationOp : int32_t;
169enum class BoundingBoxFormat : int32_t;
170enum class CausalMaskKind : int32_t;
171enum class BuilderFlag : int32_t;
172enum class CumulativeOperation : int32_t;
173enum class DeviceType : int32_t;
174enum class DimensionOperation : int32_t;
175enum class ElementWiseOperation : int32_t;
176enum class EngineCapability : int32_t;
177enum class FillOperation : int32_t;
178enum class GatherMode : int32_t;
179enum class KVCacheMode : int32_t;
180enum class LayerInformationFormat : int32_t;
181enum class LayerType : int32_t;
182enum class LoopOutput : int32_t;
183enum class MatrixOperation : int32_t;
184enum class MemoryPoolType : int32_t;
185enum class MoEActType : int32_t;
186enum class NetworkDefinitionCreationFlag : int32_t;
187enum class OptProfileSelector : int32_t;
188enum class PaddingMode : int32_t;
189enum class PoolingType : int32_t;
190enum class ProfilingVerbosity : int32_t;
191enum class ReduceOperation : int32_t;
192enum class CollectiveOperation : int32_t;
193enum class ResizeCoordinateTransformation : int32_t;
194enum class InterpolationMode : int32_t;
195enum class ResizeRoundMode : int32_t;
196enum class ResizeSelector : int32_t;
197enum class ScaleMode : int32_t;
198enum class ScatterMode : int32_t;
199enum class SampleMode : int32_t;
200enum class SerializationFlag : int32_t;
201enum class TensorIOMode : int32_t;
202enum class TensorLocation : int32_t;
203enum class TopKOperation : int32_t;
204enum class TripLimit : int32_t;
205enum class UnaryOperation : int32_t;
206enum class WeightsRole : int32_t;
207enum class PreviewFeature : int32_t;
208enum class HardwareCompatibilityLevel : int32_t;
209enum class ExecutionContextAllocationStrategy : int32_t;
210enum class RuntimePlatform : int32_t;
211enum class TilingOptimizationLevel : int32_t;
212enum class ComputeCapability : int32_t;
213enum class EngineValidity : int32_t;
214enum class EngineStat : int32_t;
215
216enum class DynamicShapesKernelSpecializationStrategy : int32_t;
217enum class CudaGraphStrategy : int32_t;
218
219using TacticSources = uint32_t;
220using TensorFormats = uint32_t;
221using BuilderFlags = uint32_t;
222using NetworkDefinitionCreationFlags = uint32_t;
223using TempfileControlFlags = uint32_t;
224using SerializationFlags = uint32_t;
225
233
234namespace apiv
235{
236
237class VRoot
238{
239public:
240 virtual ~VRoot() noexcept = default;
241};
242
243class VHostMemory : public VRoot
244{
245public:
246 virtual void* data() const noexcept = 0;
247 virtual std::size_t size() const noexcept = 0;
248 virtual DataType type() const noexcept = 0;
249};
250
251class VDimensionExpr : public VRoot
252{
253public:
254 virtual bool isConstant() const = 0;
255 virtual int64_t getConstantValue() const = 0;
256 virtual bool isSizeTensor() const = 0;
257};
258
259class VExprBuilder : public VRoot
260{
261public:
262 virtual IDimensionExpr const* constant(int64_t value) = 0;
263 virtual IDimensionExpr const* operation(
264 DimensionOperation op, IDimensionExpr const& first, IDimensionExpr const& second)
265 = 0;
266 virtual IDimensionExpr const* declareSizeTensor(
267 int32_t outputIndex, IDimensionExpr const& opt, IDimensionExpr const& upper)
268 = 0;
269};
270
271class VRuntime : public VRoot
272{
273public:
274 virtual IRuntime* getPImpl() noexcept = 0;
275 virtual nvinfer1::ICudaEngine* deserializeCudaEngine(void const* blob, std::size_t size) noexcept = 0;
276 virtual void setDLACore(int32_t dlaCore) noexcept = 0;
277 virtual int32_t getDLACore() const noexcept = 0;
278 virtual int32_t getNbDLACores() const noexcept = 0;
279 virtual void setGpuAllocator(IGpuAllocator* allocator) noexcept = 0;
280 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
281 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
282 virtual ILogger* getLogger() const noexcept = 0;
283 virtual bool setMaxThreads(int32_t maxThreads) noexcept = 0;
284 virtual int32_t getMaxThreads() const noexcept = 0;
285 virtual void setTemporaryDirectory(char const*) noexcept = 0;
286 virtual char const* getTemporaryDirectory() const noexcept = 0;
287 virtual void setTempfileControlFlags(TempfileControlFlags) noexcept = 0;
288 virtual TempfileControlFlags getTempfileControlFlags() const noexcept = 0;
289 virtual IPluginRegistry& getPluginRegistry() noexcept = 0;
290 virtual void setPluginRegistryParent(IPluginRegistry* parent) noexcept = 0;
291 virtual IRuntime* loadRuntime(char const* path) noexcept = 0;
292 virtual void setEngineHostCodeAllowed(bool allowed) noexcept = 0;
293 virtual bool getEngineHostCodeAllowed() const noexcept = 0;
294 // Added in TensorRT version 10.7
295 virtual nvinfer1::ICudaEngine* deserializeCudaEngineV2(IStreamReaderV2& streamReader) noexcept = 0;
296 virtual int64_t getEngineHeaderSize() const noexcept = 0;
297 virtual EngineValidity getEngineValidity(void const* blob, int64_t blobSize, uint64_t* diagnostics) const noexcept
298 = 0;
299 virtual void setDeferredWeightsLoading(bool defer) noexcept = 0;
300 virtual bool getDeferredWeightsLoading() const noexcept = 0;
301};
302
303class VRefitter : public VRoot
304{
305public:
306 virtual IRefitter* getPImpl() noexcept = 0;
307 virtual bool setWeights(char const* layerName, WeightsRole role, const Weights weights) noexcept = 0;
308 virtual bool refitCudaEngine() noexcept = 0;
309 virtual int32_t getMissing(int32_t size, char const** layerNames, WeightsRole* roles) noexcept = 0;
310 virtual int32_t getAll(int32_t size, char const** layerNames, WeightsRole* roles) noexcept = 0;
311 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
312 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
313 virtual bool setNamedWeights(char const* name, Weights weights) noexcept = 0;
314 virtual int32_t getMissingWeights(int32_t size, char const** weightsNames) noexcept = 0;
315 virtual int32_t getAllWeights(int32_t size, char const** weightsNames) noexcept = 0;
316 virtual ILogger* getLogger() const noexcept = 0;
317 virtual bool setMaxThreads(int32_t maxThreads) noexcept = 0;
318 virtual int32_t getMaxThreads() const noexcept = 0;
319 virtual bool setNamedWeightsWithLocation(char const* name, Weights weights, TensorLocation location) noexcept = 0;
320 virtual Weights getNamedWeights(char const* weightsName) const noexcept = 0;
321 virtual TensorLocation getWeightsLocation(char const* weightsName) const noexcept = 0;
322 virtual bool unsetNamedWeights(char const* weightsName) noexcept = 0;
323 virtual void setWeightsValidation(bool weightsValidation) noexcept = 0;
324 virtual bool getWeightsValidation() const noexcept = 0;
325 virtual bool refitCudaEngineAsync(cudaStream_t stream) noexcept = 0;
326 virtual Weights getWeightsPrototype(char const* weightsName) const noexcept = 0;
327};
328
329class VOptimizationProfile : public VRoot
330{
331public:
332 virtual bool setDimensions(char const* inputName, OptProfileSelector select, Dims const& dims) noexcept = 0;
333 virtual Dims getDimensions(char const* inputName, OptProfileSelector select) const noexcept = 0;
334 virtual int32_t getNbShapeValues(char const* inputName) const noexcept = 0;
335 virtual bool setExtraMemoryTarget(float target) noexcept = 0;
336 virtual float getExtraMemoryTarget() const noexcept = 0;
337 virtual bool isValid() const noexcept = 0;
338 // Added in TensorRT 10.11
339 TRT_NODISCARD virtual bool setShapeValuesV2(
340 char const* inputName, OptProfileSelector select, int64_t const* values, int32_t nbValues) noexcept = 0;
341 TRT_NODISCARD virtual int64_t const* getShapeValuesV2(
342 char const* inputName, OptProfileSelector select) const noexcept = 0;
343};
344
345class VCudaEngine : public VRoot
346{
347public:
348 virtual ICudaEngine* getPImpl() noexcept = 0;
349 virtual int32_t getNbLayers() const noexcept = 0;
350 virtual IHostMemory* serialize() const noexcept = 0;
351 virtual IExecutionContext* createExecutionContext(ExecutionContextAllocationStrategy strategy) noexcept = 0;
352 virtual bool isRefittable() const noexcept = 0;
353 virtual char const* getName() const noexcept = 0;
354 virtual int32_t getNbOptimizationProfiles() const noexcept = 0;
355 virtual EngineCapability getEngineCapability() const noexcept = 0;
356 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
357 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
358 virtual TacticSources getTacticSources() const noexcept = 0;
359 virtual ProfilingVerbosity getProfilingVerbosity() const noexcept = 0;
360 virtual IEngineInspector* createEngineInspector() const noexcept = 0;
361 virtual Dims getTensorShape(char const* tensorName) const noexcept = 0;
362 virtual DataType getTensorDataType(char const* tensorName) const noexcept = 0;
363 virtual TensorLocation getTensorLocation(char const* tensorName) const noexcept = 0;
364 virtual bool isShapeInferenceIO(char const* tensorName) const noexcept = 0;
365 virtual TensorIOMode getTensorIOMode(char const* tensorName) const noexcept = 0;
366 virtual int32_t getTensorBytesPerComponent(char const* tensorName) const noexcept = 0;
367 virtual int32_t getTensorComponentsPerElement(char const* tensorName) const noexcept = 0;
368 virtual TensorFormat getTensorFormat(char const* tensorName) const noexcept = 0;
369 virtual char const* getTensorFormatDesc(char const* tensorName) const noexcept = 0;
370 virtual int32_t getTensorVectorizedDim(char const* tensorName) const noexcept = 0;
371 virtual Dims getProfileShape(
372 char const* tensorName, int32_t profileIndex, OptProfileSelector select) const noexcept = 0;
373 virtual int32_t getNbIOTensors() const noexcept = 0;
374 virtual char const* getIOTensorName(int32_t index) const noexcept = 0;
375 virtual HardwareCompatibilityLevel getHardwareCompatibilityLevel() const noexcept = 0;
376 virtual int32_t getNbAuxStreams() const noexcept = 0;
377
378 virtual int32_t getTensorBytesPerComponentV2(char const* tensorName, int32_t profileIndex) const noexcept = 0;
379 virtual int32_t getTensorComponentsPerElementV2(char const* tensorName, int32_t profileIndex) const noexcept = 0;
380 virtual TensorFormat getTensorFormatV2(char const* tensorName, int32_t profileIndex) const noexcept = 0;
381 virtual char const* getTensorFormatDescV2(char const* tensorName, int32_t profileIndex) const noexcept = 0;
382 virtual int32_t getTensorVectorizedDimV2(char const* tensorName, int32_t profileIndex) const noexcept = 0;
383
384 virtual ISerializationConfig* createSerializationConfig() noexcept = 0;
385 virtual IHostMemory* serializeWithConfig(ISerializationConfig& config) const noexcept = 0;
386
387 virtual IRefitter* createRefitter(ILogger& logger) noexcept = 0;
388
389 virtual int64_t getStreamableWeightsSize() const noexcept = 0;
390
391 virtual bool isDebugTensor(char const* name) const noexcept = 0;
392
393 // Added in TensorRT 10.1
394 virtual bool setWeightStreamingBudgetV2(int64_t gpuMemoryBudget) noexcept = 0;
395 virtual int64_t getWeightStreamingBudgetV2() const noexcept = 0;
396 virtual int64_t getWeightStreamingAutomaticBudget() const noexcept = 0;
397 virtual int64_t getWeightStreamingScratchMemorySize() const noexcept = 0;
398 virtual int64_t getDeviceMemorySizeV2() const noexcept = 0;
399 virtual int64_t getDeviceMemorySizeForProfileV2(int32_t profileIndex) const noexcept = 0;
400 // Added in TensorRT 10.11
401 TRT_NODISCARD virtual int64_t const* getProfileTensorValuesV2(
402 char const* tensorName, int32_t profileIndex, OptProfileSelector select) const noexcept = 0;
403 TRT_NODISCARD virtual IExecutionContext* createExecutionContextWithRuntimeConfig(
404 IRuntimeConfig* runtimeConfig) noexcept = 0;
405 TRT_NODISCARD virtual IRuntimeConfig* createRuntimeConfig() noexcept = 0;
406 TRT_NODISCARD virtual int64_t getEngineStat(EngineStat stat) const noexcept = 0;
407 // Added in TensorRT 10.15
408 TRT_NODISCARD virtual char const* getAliasedInputTensor(char const* tensorName) const noexcept = 0;
409 TRT_NODISCARD virtual bool areWeightsLoaded() const noexcept = 0;
410 TRT_NODISCARD virtual bool loadWeights(void const* blob, int64_t size) noexcept = 0;
411 TRT_NODISCARD virtual bool loadWeightsAsync(IStreamReaderV2& streamReader, cudaStream_t stream) noexcept = 0;
412};
413
414class VExecutionContext : public VRoot
415{
416public:
417 virtual IExecutionContext* getPImpl() noexcept = 0;
418 virtual void setDebugSync(bool sync) noexcept = 0;
419 virtual bool getDebugSync() const noexcept = 0;
420 virtual void setProfiler(IProfiler*) noexcept = 0;
421 virtual IProfiler* getProfiler() const noexcept = 0;
422 virtual ICudaEngine const& getEngine() const noexcept = 0;
423 virtual void setName(char const* name) noexcept = 0;
424 virtual char const* getName() const noexcept = 0;
425 virtual void setDeviceMemory(void* memory) noexcept = 0;
426 virtual int32_t getOptimizationProfile() const noexcept = 0;
427 virtual bool allInputDimensionsSpecified() const noexcept = 0;
428 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
429 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
430 virtual bool executeV2(void* const* bindings) noexcept = 0;
431 virtual bool setOptimizationProfileAsync(int32_t profileIndex, cudaStream_t stream) noexcept = 0;
432 virtual void setEnqueueEmitsProfile(bool enqueueEmitsProfile) noexcept = 0;
433 virtual bool getEnqueueEmitsProfile() const noexcept = 0;
434 virtual bool reportToProfiler() const noexcept = 0;
435 virtual bool setInputShape(char const* tensorName, Dims const& dims) noexcept = 0;
436 virtual Dims getTensorShape(char const* tensorName) const noexcept = 0;
437 virtual Dims getTensorStrides(char const* tensorName) const noexcept = 0;
438 virtual bool setTensorAddress(char const* tensorName, void* data) noexcept = 0;
439 virtual void const* getTensorAddress(char const* tensorName) const noexcept = 0;
440 virtual bool setInputTensorAddress(char const* tensorName, void const* data) noexcept = 0;
441 virtual bool setOutputTensorAddress(char const* tensorName, void* data) noexcept = 0;
442 virtual int32_t inferShapes(int32_t nbMaxNames, char const** tensorNames) noexcept = 0;
443 virtual bool setInputConsumedEvent(cudaEvent_t event) noexcept = 0;
444 virtual cudaEvent_t getInputConsumedEvent() const noexcept = 0;
445 virtual void* getOutputTensorAddress(char const* tensorName) const noexcept = 0;
446 virtual bool setOutputAllocator(char const* tensorName, IOutputAllocator* outputAllocator) noexcept = 0;
447 virtual IOutputAllocator* getOutputAllocator(char const* name) noexcept = 0;
448 virtual int64_t getMaxOutputSize(char const* tensorName) const noexcept = 0;
449 virtual bool setTemporaryStorageAllocator(IGpuAllocator* allocator) noexcept = 0;
450 virtual IGpuAllocator* getTemporaryStorageAllocator() const noexcept = 0;
451 virtual bool enqueueV3(cudaStream_t stream) noexcept = 0;
452 virtual void setPersistentCacheLimit(size_t size) noexcept = 0;
453 virtual size_t getPersistentCacheLimit() const noexcept = 0;
454 virtual bool setNvtxVerbosity(ProfilingVerbosity verbosity) noexcept = 0;
455 virtual ProfilingVerbosity getNvtxVerbosity() const noexcept = 0;
456 virtual void setAuxStreams(cudaStream_t* auxStreams, int32_t nbStreams) noexcept = 0;
457 virtual bool setDebugListener(IDebugListener* listener) noexcept = 0;
458 virtual IDebugListener* getDebugListener() noexcept = 0;
459 virtual bool setTensorDebugState(char const* name, bool flag) noexcept = 0;
460 virtual bool getDebugState(char const* name) const noexcept = 0;
461 virtual bool setAllTensorsDebugState(bool flag) noexcept = 0;
462 virtual size_t updateDeviceMemorySizeForShapes() noexcept = 0;
463 virtual void setDeviceMemoryV2(void* memory, int64_t size) noexcept = 0;
464 TRT_NODISCARD virtual IRuntimeConfig* getRuntimeConfig() const noexcept = 0;
465 virtual bool setUnfusedTensorsDebugState(bool flag) noexcept = 0;
466 virtual bool getUnfusedTensorsDebugState() const noexcept = 0;
467 virtual bool isStreamCapturable(cudaStream_t stream) const noexcept = 0;
468 virtual bool setCommunicator(void* communicator) noexcept = 0;
469};
470
471class VEngineInspector : public VRoot
472{
473public:
474 virtual IEngineInspector* getPImpl() noexcept = 0;
475 virtual bool setExecutionContext(IExecutionContext const* context) noexcept = 0;
476 virtual IExecutionContext const* getExecutionContext() const noexcept = 0;
477 virtual char const* getLayerInformation(int32_t layerIndex, LayerInformationFormat format) const noexcept = 0;
478 virtual char const* getEngineInformation(LayerInformationFormat format) const noexcept = 0;
479 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
480 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
481};
482
483class VTensor : public VRoot
484{
485public:
486 virtual void setName(char const* name) noexcept = 0;
487 virtual char const* getName() const noexcept = 0;
488 virtual void setDimensions(Dims const& dimensions) noexcept = 0;
489 virtual Dims getDimensions() const noexcept = 0;
490 virtual DataType getType() const noexcept = 0;
491 virtual bool isNetworkInput() const noexcept = 0;
492 virtual bool isNetworkOutput() const noexcept = 0;
493 virtual void setAllowedFormats(TensorFormats formats) noexcept = 0;
494 virtual TensorFormats getAllowedFormats() const noexcept = 0;
495 virtual bool isShapeTensor() const noexcept = 0;
496 virtual bool isExecutionTensor() const noexcept = 0;
497 virtual void setDimensionName(int32_t index, char const* name) noexcept = 0;
498 virtual char const* getDimensionName(int32_t index) const noexcept = 0;
499};
500
501class VLayer : public VRoot
502{
503public:
504 virtual LayerType getType() const noexcept = 0;
505 virtual void setName(char const* name) noexcept = 0;
506 virtual char const* getName() const noexcept = 0;
507 virtual int32_t getNbInputs() const noexcept = 0;
508 virtual ITensor* getInput(int32_t index) const noexcept = 0;
509 virtual int32_t getNbOutputs() const noexcept = 0;
510 virtual ITensor* getOutput(int32_t index) const noexcept = 0;
511 virtual void setInput(int32_t index, ITensor& tensor) noexcept = 0;
512 virtual DataType getOutputType(int32_t index) const noexcept = 0;
513 virtual void setMetadata(char const* docString) noexcept = 0;
514 virtual char const* getMetadata() const noexcept = 0;
515 virtual bool setNbRanks(int32_t nbRanks) noexcept = 0;
516 virtual int32_t getNbRanks() const noexcept = 0;
517};
518
519class VConvolutionLayer : public VRoot
520{
521public:
522 virtual void setNbOutputMaps(int64_t nbOutputMaps) noexcept = 0;
523 virtual int64_t getNbOutputMaps() const noexcept = 0;
524 virtual void setNbGroups(int64_t nbGroups) noexcept = 0;
525 virtual int64_t getNbGroups() const noexcept = 0;
526 virtual void setKernelWeights(Weights weights) noexcept = 0;
527 virtual Weights getKernelWeights() const noexcept = 0;
528 virtual void setBiasWeights(Weights weights) noexcept = 0;
529 virtual Weights getBiasWeights() const noexcept = 0;
530 virtual void setPrePadding(Dims const& padding) noexcept = 0;
531 virtual Dims getPrePadding() const noexcept = 0;
532 virtual void setPostPadding(Dims const& padding) noexcept = 0;
533 virtual Dims getPostPadding() const noexcept = 0;
534 virtual void setPaddingMode(PaddingMode paddingMode) noexcept = 0;
535 virtual PaddingMode getPaddingMode() const noexcept = 0;
536 virtual void setKernelSizeNd(Dims const& kernelSize) noexcept = 0;
537 virtual Dims getKernelSizeNd() const noexcept = 0;
538 virtual void setStrideNd(Dims const& stride) noexcept = 0;
539 virtual Dims getStrideNd() const noexcept = 0;
540 virtual void setPaddingNd(Dims const& padding) noexcept = 0;
541 virtual Dims getPaddingNd() const noexcept = 0;
542 virtual void setDilationNd(Dims const& dilation) noexcept = 0;
543 virtual Dims getDilationNd() const noexcept = 0;
544};
545
546class VActivationLayer : public VRoot
547{
548public:
549 virtual void setActivationType(ActivationType type) noexcept = 0;
550 virtual ActivationType getActivationType() const noexcept = 0;
551 virtual void setAlpha(float alpha) noexcept = 0;
552 virtual void setBeta(float beta) noexcept = 0;
553 virtual float getAlpha() const noexcept = 0;
554 virtual float getBeta() const noexcept = 0;
555};
556
557class VPoolingLayer : public VRoot
558{
559public:
560 virtual void setPoolingType(PoolingType type) noexcept = 0;
561 virtual PoolingType getPoolingType() const noexcept = 0;
562 virtual void setBlendFactor(float blendFactor) noexcept = 0;
563 virtual float getBlendFactor() const noexcept = 0;
564 virtual void setAverageCountExcludesPadding(bool exclusive) noexcept = 0;
565 virtual bool getAverageCountExcludesPadding() const noexcept = 0;
566 virtual void setPrePadding(Dims const& padding) noexcept = 0;
567 virtual Dims getPrePadding() const noexcept = 0;
568 virtual void setPostPadding(Dims const& padding) noexcept = 0;
569 virtual Dims getPostPadding() const noexcept = 0;
570 virtual void setPaddingMode(PaddingMode paddingMode) noexcept = 0;
571 virtual PaddingMode getPaddingMode() const noexcept = 0;
572 virtual void setWindowSizeNd(Dims const& windowSize) noexcept = 0;
573 virtual Dims getWindowSizeNd() const noexcept = 0;
574 virtual void setStrideNd(Dims const& stride) noexcept = 0;
575 virtual Dims getStrideNd() const noexcept = 0;
576 virtual void setPaddingNd(Dims const& padding) noexcept = 0;
577 virtual Dims getPaddingNd() const noexcept = 0;
578};
579
580class VLRNLayer : public VRoot
581{
582public:
583 virtual void setWindowSize(int64_t windowSize) noexcept = 0;
584 virtual int64_t getWindowSize() const noexcept = 0;
585 virtual void setAlpha(float alpha) noexcept = 0;
586 virtual float getAlpha() const noexcept = 0;
587 virtual void setBeta(float beta) noexcept = 0;
588 virtual float getBeta() const noexcept = 0;
589 virtual void setK(float k) noexcept = 0;
590 virtual float getK() const noexcept = 0;
591};
592
593class VScaleLayer : public VRoot
594{
595public:
596 virtual void setMode(ScaleMode mode) noexcept = 0;
597 virtual ScaleMode getMode() const noexcept = 0;
598 virtual void setShift(Weights shift) noexcept = 0;
599 virtual Weights getShift() const noexcept = 0;
600 virtual void setScale(Weights scale) noexcept = 0;
601 virtual Weights getScale() const noexcept = 0;
602 virtual void setPower(Weights power) noexcept = 0;
603 virtual Weights getPower() const noexcept = 0;
604 virtual int32_t getChannelAxis() const noexcept = 0;
605 virtual void setChannelAxis(int32_t channelAxis) noexcept = 0;
606};
607
608class VSoftMaxLayer : public VRoot
609{
610public:
611 virtual void setAxes(uint32_t axes) noexcept = 0;
612 virtual uint32_t getAxes() const noexcept = 0;
613};
614
615class VConcatenationLayer : public VRoot
616{
617public:
618 virtual void setAxis(int32_t axis) noexcept = 0;
619 virtual int32_t getAxis() const noexcept = 0;
620};
621
622class VDeconvolutionLayer : public VRoot
623{
624public:
625 virtual void setNbOutputMaps(int64_t nbOutputMaps) noexcept = 0;
626 virtual int64_t getNbOutputMaps() const noexcept = 0;
627 virtual void setNbGroups(int64_t nbGroups) noexcept = 0;
628 virtual int64_t getNbGroups() const noexcept = 0;
629 virtual void setKernelWeights(Weights weights) noexcept = 0;
630 virtual Weights getKernelWeights() const noexcept = 0;
631 virtual void setBiasWeights(Weights weights) noexcept = 0;
632 virtual Weights getBiasWeights() const noexcept = 0;
633 virtual void setPrePadding(Dims const& padding) noexcept = 0;
634 virtual Dims getPrePadding() const noexcept = 0;
635 virtual void setPostPadding(Dims const& padding) noexcept = 0;
636 virtual Dims getPostPadding() const noexcept = 0;
637 virtual void setPaddingMode(PaddingMode paddingMode) noexcept = 0;
638 virtual PaddingMode getPaddingMode() const noexcept = 0;
639 virtual void setKernelSizeNd(Dims const& kernelSize) noexcept = 0;
640 virtual Dims getKernelSizeNd() const noexcept = 0;
641 virtual void setStrideNd(Dims const& stride) noexcept = 0;
642 virtual Dims getStrideNd() const noexcept = 0;
643 virtual void setPaddingNd(Dims const& padding) noexcept = 0;
644 virtual Dims getPaddingNd() const noexcept = 0;
645 virtual void setDilationNd(Dims const& dilation) noexcept = 0;
646 virtual Dims getDilationNd() const noexcept = 0;
647};
648
649class VElementWiseLayer : public VRoot
650{
651public:
652 virtual void setOperation(ElementWiseOperation op) noexcept = 0;
653 virtual ElementWiseOperation getOperation() const noexcept = 0;
654};
655
656class VGatherLayer : public VRoot
657{
658public:
659 virtual void setGatherAxis(int32_t axis) noexcept = 0;
660 virtual int32_t getGatherAxis() const noexcept = 0;
661 virtual void setNbElementWiseDims(int32_t k) noexcept = 0;
662 virtual int32_t getNbElementWiseDims() const noexcept = 0;
663 virtual void setMode(GatherMode mode) noexcept = 0;
664 virtual GatherMode getMode() const noexcept = 0;
665};
666
667class VPluginLayer : public VRoot
668{
669public:
670 virtual IPlugin& getPlugin() noexcept = 0;
671};
672
673class VPluginV2Layer : public VRoot
674{
675public:
676 virtual IPluginV2& getPlugin() noexcept = 0;
677};
678
679class VPluginV3Layer : public VRoot
680{
681public:
682 virtual IPluginV3& getPlugin() noexcept = 0;
683};
684
685class VUnaryLayer : public VRoot
686{
687public:
688 virtual void setOperation(UnaryOperation op) noexcept = 0;
689 virtual UnaryOperation getOperation() const noexcept = 0;
690};
691
692class VReduceLayer : public VRoot
693{
694public:
695 virtual void setOperation(ReduceOperation op) noexcept = 0;
696 virtual ReduceOperation getOperation() const noexcept = 0;
697 virtual void setReduceAxes(uint32_t reduceAxes) noexcept = 0;
698 virtual uint32_t getReduceAxes() const noexcept = 0;
699 virtual void setKeepDimensions(bool keepDimensions) noexcept = 0;
700 virtual bool getKeepDimensions() const noexcept = 0;
701};
702
703class VPaddingLayer : public VRoot
704{
705public:
706 virtual void setPrePaddingNd(Dims const& padding) noexcept = 0;
707 virtual Dims getPrePaddingNd() const noexcept = 0;
708 virtual void setPostPaddingNd(Dims const& padding) noexcept = 0;
709 virtual Dims getPostPaddingNd() const noexcept = 0;
710};
711
712class VShuffleLayer : public VRoot
713{
714public:
715 virtual void setFirstTranspose(Permutation const& permutation) noexcept = 0;
716 virtual Permutation const& getFirstTranspose() const noexcept = 0;
717 virtual void setReshapeDimensions(Dims const& dimensions) noexcept = 0;
718 virtual Dims getReshapeDimensions() const noexcept = 0;
719 virtual void setSecondTranspose(Permutation const& permutation) noexcept = 0;
720 virtual Permutation const& getSecondTranspose() const noexcept = 0;
721 virtual void setZeroIsPlaceholder(bool zeroIsPlaceholder) noexcept = 0;
722 virtual bool getZeroIsPlaceholder() const noexcept = 0;
723};
724
725class VSliceLayer : public VRoot
726{
727public:
728 virtual void setStart(Dims const& start) noexcept = 0;
729 virtual Dims getStart() const noexcept = 0;
730 virtual void setSize(Dims const& size) noexcept = 0;
731 virtual Dims getSize() const noexcept = 0;
732 virtual void setStride(Dims const& stride) noexcept = 0;
733 virtual Dims getStride() const noexcept = 0;
734 virtual void setMode(SampleMode mode) noexcept = 0;
735 virtual SampleMode getMode() const noexcept = 0;
736 virtual void setAxes(Dims const& axes) noexcept = 0;
737 virtual Dims getAxes() const noexcept = 0;
738};
739
740class VShapeLayer : public VRoot
741{
742public:
743};
744
745class VTopKLayer : public VRoot
746{
747public:
748 virtual void setOperation(TopKOperation op) noexcept = 0;
749 virtual TopKOperation getOperation() const noexcept = 0;
750 virtual void setK(int32_t k) noexcept = 0;
751 virtual int32_t getK() const noexcept = 0;
752 virtual void setReduceAxes(uint32_t reduceAxes) noexcept = 0;
753 virtual uint32_t getReduceAxes() const noexcept = 0;
754 virtual bool setIndicesType(DataType type) noexcept = 0;
755 virtual DataType getIndicesType() const noexcept = 0;
756};
757
758class VMatrixMultiplyLayer : public VRoot
759{
760public:
761 virtual void setOperation(int32_t index, MatrixOperation op) noexcept = 0;
762 virtual MatrixOperation getOperation(int32_t index) const noexcept = 0;
763};
764
765class VNonZeroLayer : public VRoot
766{
767public:
768 virtual bool setIndicesType(DataType type) noexcept = 0;
769 virtual DataType getIndicesType() const noexcept = 0;
770};
771
772class VRaggedSoftMaxLayer : public VRoot
773{
774public:
775};
776
777class VIdentityLayer : public VRoot
778{
779public:
780};
781
782class VCastLayer : public VRoot
783{
784public:
785 virtual void setToType(DataType toType) noexcept = 0;
786 virtual DataType getToType() const noexcept = 0;
787};
788
789class VConstantLayer : public VRoot
790{
791public:
792 virtual void setWeights(Weights weights) noexcept = 0;
793 virtual Weights getWeights() const noexcept = 0;
794 virtual void setDimensions(Dims const& dimensions) noexcept = 0;
795 virtual Dims getDimensions() const noexcept = 0;
796};
797
798class VParametricReLULayer : public VRoot
799{
800public:
801};
802
803class VResizeLayer : public VRoot
804{
805public:
806 virtual void setOutputDimensions(Dims const& dimensions) noexcept = 0;
807 virtual Dims getOutputDimensions() const noexcept = 0;
808 virtual void setScales(float const* scales, int32_t nbScales) noexcept = 0;
809 virtual int32_t getScales(int32_t size, float* scales) const noexcept = 0;
810 virtual void setResizeMode(InterpolationMode interpolationMode) noexcept = 0;
811 virtual InterpolationMode getResizeMode() const noexcept = 0;
812 virtual void setCoordinateTransformation(ResizeCoordinateTransformation coordTransform) noexcept = 0;
813 virtual ResizeCoordinateTransformation getCoordinateTransformation() const noexcept = 0;
814 virtual void setSelectorForSinglePixel(ResizeSelector selector) noexcept = 0;
815 virtual ResizeSelector getSelectorForSinglePixel() const noexcept = 0;
816 virtual void setNearestRounding(ResizeRoundMode value) noexcept = 0;
817 virtual ResizeRoundMode getNearestRounding() const noexcept = 0;
818 virtual void setCubicCoeff(float value) noexcept = 0;
819 virtual float getCubicCoeff() const noexcept = 0;
820 virtual void setExcludeOutside(bool value) noexcept = 0;
821 virtual bool getExcludeOutside() const noexcept = 0;
822};
823
824class VLoopBoundaryLayer : public VRoot
825{
826public:
827 virtual ILoop* getLoop() const noexcept = 0;
828};
829
830class VRecurrenceLayer : public VRoot
831{
832public:
833};
834
835class VLoopOutputLayer : public VRoot
836{
837public:
838 virtual LoopOutput getLoopOutput() const noexcept = 0;
839 virtual void setAxis(int32_t axis) noexcept = 0;
840 virtual int32_t getAxis() const noexcept = 0;
841};
842
843class VTripLimitLayer : public VRoot
844{
845public:
846 virtual TripLimit getTripLimit() const noexcept = 0;
847};
848
849class VIteratorLayer : public VRoot
850{
851public:
852 virtual void setAxis(int32_t axis) noexcept = 0;
853 virtual int32_t getAxis() const noexcept = 0;
854 virtual void setReverse(bool reverse) noexcept = 0;
855 virtual bool getReverse() const noexcept = 0;
856};
857class VLoop : public VRoot
858{
859public:
860 virtual IRecurrenceLayer* addRecurrence(ITensor& initialValue) noexcept = 0;
861 virtual ITripLimitLayer* addTripLimit(ITensor& tensor, TripLimit limit) noexcept = 0;
862 virtual IIteratorLayer* addIterator(ITensor& tensor, int32_t axis = 0, bool reverse = false) noexcept = 0;
863 virtual ILoopOutputLayer* addLoopOutput(ITensor& tensor, LoopOutput outputKind, int32_t axis = 0) noexcept = 0;
864 virtual void setName(char const* name) noexcept = 0;
865 virtual char const* getName() const noexcept = 0;
866};
867
868class VConditionalBoundaryLayer : public VRoot
869{
870public:
871 virtual IIfConditional* getConditional() const noexcept = 0;
872};
873
874class VConditionLayer : public VRoot
875{
876public:
877};
878
879class VConditionalInputLayer : public VRoot
880{
881public:
882};
883
884class VConditionalOutputLayer : public VRoot
885{
886public:
887};
888
889class VIfConditional : public VRoot
890{
891public:
892 virtual IConditionLayer* setCondition(ITensor& tensor) noexcept = 0;
893 virtual IIfConditionalInputLayer* addInput(ITensor& tensor) noexcept = 0;
894 virtual IIfConditionalOutputLayer* addOutput(ITensor& trueTensor, ITensor& falseTensor) noexcept = 0;
895 virtual void setName(char const* name) noexcept = 0;
896 virtual char const* getName() const noexcept = 0;
897};
898
899class VAttentionBoundaryLayer : public VRoot
900{
901public:
902 virtual IAttention* getAttention() const noexcept = 0;
903};
904
905class VAttentionInputLayer : public VRoot
906{
907public:
908};
909
910class VAttentionOutputLayer : public VRoot
911{
912public:
913};
914
915class VAttention : public VRoot
916{
917public:
918 TRT_NODISCARD virtual bool setInput(int32_t index, ITensor& input) noexcept = 0;
919 TRT_NODISCARD virtual int32_t getNbInputs() const noexcept = 0;
920 TRT_NODISCARD virtual ITensor* getInput(int32_t index) const noexcept = 0;
921 TRT_NODISCARD virtual int32_t getNbOutputs() const noexcept = 0;
922 TRT_NODISCARD virtual ITensor* getOutput(int32_t index) const noexcept = 0;
923 TRT_NODISCARD virtual bool setName(char const* name) noexcept = 0;
924 TRT_NODISCARD virtual char const* getName() const noexcept = 0;
925 TRT_NODISCARD virtual bool setNormalizationOperation(AttentionNormalizationOp op) noexcept = 0;
926 TRT_NODISCARD virtual AttentionNormalizationOp getNormalizationOperation() const noexcept = 0;
927 TRT_DEPRECATED virtual bool setCausal(bool isCausal) noexcept = 0;
928 TRT_DEPRECATED virtual bool getCausal() const noexcept = 0;
929 TRT_NODISCARD virtual bool setMask(ITensor& mask) noexcept = 0;
930 TRT_NODISCARD virtual ITensor* getMask() const noexcept = 0;
931 TRT_NODISCARD virtual bool setDecomposable(bool decomposable) noexcept = 0;
932 TRT_NODISCARD virtual bool getDecomposable() const noexcept = 0;
933 TRT_NODISCARD virtual bool setNormalizationQuantizeScale(ITensor& tensor) noexcept = 0;
934 TRT_NODISCARD virtual ITensor* getNormalizationQuantizeScale() const noexcept = 0;
935 TRT_NODISCARD virtual bool setNormalizationQuantizeToType(DataType type) noexcept = 0;
936 TRT_NODISCARD virtual DataType getNormalizationQuantizeToType() const noexcept = 0;
937 TRT_NODISCARD virtual bool setMetadata(char const* docString) noexcept = 0;
938 TRT_NODISCARD virtual char const* getMetadata() const noexcept = 0;
939 TRT_NODISCARD virtual bool setNbRanks(int32_t nbRanks) noexcept = 0;
940 TRT_NODISCARD virtual int32_t getNbRanks() const noexcept = 0;
941 TRT_NODISCARD virtual bool setCausalKind(CausalMaskKind kind) noexcept = 0;
942 TRT_NODISCARD virtual CausalMaskKind getCausalKind() const noexcept = 0;
943 TRT_NODISCARD virtual bool setQueryForm(AttentionIOForm form) noexcept = 0;
944 TRT_NODISCARD virtual AttentionIOForm getQueryForm() const noexcept = 0;
945 TRT_NODISCARD virtual bool setKeyValueForm(AttentionIOForm form) noexcept = 0;
946 TRT_NODISCARD virtual AttentionIOForm getKeyValueForm() const noexcept = 0;
947 TRT_NODISCARD virtual bool setQueryLengths(ITensor* lengths) noexcept = 0;
948 TRT_NODISCARD virtual ITensor* getQueryLengths() const noexcept = 0;
949 TRT_NODISCARD virtual bool setKeyValueLengths(ITensor* lengths) noexcept = 0;
950 TRT_NODISCARD virtual ITensor* getKeyValueLengths() const noexcept = 0;
951}; // class VAttention
952
953class VSelectLayer : public VRoot
954{
955};
956
957class VAssertionLayer : public VRoot
958{
959public:
960 virtual void setMessage(char const* message) noexcept = 0;
961 virtual char const* getMessage() const noexcept = 0;
962};
963
964class VFillLayer : public VRoot
965{
966public:
967 virtual void setDimensions(Dims const& dimensions) noexcept = 0;
968 virtual Dims getDimensions() const noexcept = 0;
969 virtual void setOperation(FillOperation op) noexcept = 0;
970 virtual FillOperation getOperation() const noexcept = 0;
971 virtual void setAlpha(double alpha) noexcept = 0;
972 virtual double getAlpha() const noexcept = 0;
973 virtual void setBeta(double beta) noexcept = 0;
974 virtual double getBeta() const noexcept = 0;
975 virtual void setAlphaInt64(int64_t alpha) noexcept = 0;
976 virtual int64_t getAlphaInt64() const noexcept = 0;
977 virtual void setBetaInt64(int64_t beta) noexcept = 0;
978 virtual int64_t getBetaInt64() const noexcept = 0;
979 virtual bool isAlphaBetaInt64() const noexcept = 0;
980 virtual DataType getToType() const noexcept = 0;
981 virtual void setToType(DataType toType) noexcept = 0;
982};
983
984class VQuantizeLayer : public VRoot
985{
986public:
987 virtual int32_t getAxis() const noexcept = 0;
988 virtual void setAxis(int32_t axis) noexcept = 0;
989 virtual DataType getToType() const noexcept = 0;
990 virtual void setToType(DataType toType) noexcept = 0;
991 virtual Dims getBlockShape() const noexcept = 0;
992 virtual bool setBlockShape(Dims const& blockShape) noexcept = 0;
993};
994
995class VDequantizeLayer : public VRoot
996{
997public:
998 virtual int32_t getAxis() const noexcept = 0;
999 virtual void setAxis(int32_t axis) noexcept = 0;
1000 virtual DataType getToType() const noexcept = 0;
1001 virtual void setToType(DataType toType) noexcept = 0;
1002 virtual Dims getBlockShape() const noexcept = 0;
1003 virtual bool setBlockShape(Dims const& blockShape) noexcept = 0;
1004};
1005
1006class VDynamicQuantizeLayer : public VRoot
1007{
1008public:
1009 TRT_DEPRECATED virtual int32_t getAxis() const noexcept = 0;
1010 TRT_DEPRECATED virtual void setAxis(int32_t axis) noexcept = 0;
1011 TRT_DEPRECATED virtual int32_t getBlockSize() const noexcept = 0;
1012 TRT_DEPRECATED virtual void setBlockSize(int32_t axis) noexcept = 0;
1013 virtual DataType getScaleType() const noexcept = 0;
1014 virtual void setScaleType(DataType axis) noexcept = 0;
1015 virtual DataType getToType() const noexcept = 0;
1016 virtual void setToType(DataType toType) noexcept = 0;
1017 virtual Dims getBlockShape() const noexcept = 0;
1018 virtual void setBlockShape(Dims const& blockShape) noexcept = 0;
1019};
1020
1021class VScatterLayer : public VRoot
1022{
1023public:
1024 virtual void setMode(ScatterMode mode) noexcept = 0;
1025 virtual ScatterMode getMode() const noexcept = 0;
1026 virtual void setAxis(int32_t axis) noexcept = 0;
1027 virtual int32_t getAxis() const noexcept = 0;
1028}; // class VScatterLayer
1029
1030class VEinsumLayer : public VRoot
1031{
1032public:
1033 virtual bool setEquation(char const* equation) noexcept = 0;
1034 virtual char const* getEquation() const noexcept = 0;
1035};
1036
1037class VOneHotLayer : public VRoot
1038{
1039public:
1040 virtual int32_t getAxis() const noexcept = 0;
1041 virtual void setAxis(int32_t axis) noexcept = 0;
1042}; // class VOneHotLayer
1043
1044class VGridSampleLayer : public VRoot
1045{
1046public:
1047 virtual void setInterpolationMode(InterpolationMode mode) noexcept = 0;
1048 virtual InterpolationMode getInterpolationMode() const noexcept = 0;
1049 virtual void setAlignCorners(bool alignCorners) noexcept = 0;
1050 virtual bool getAlignCorners() const noexcept = 0;
1051 virtual bool setSampleMode(SampleMode mode) noexcept = 0;
1052 virtual SampleMode getSampleMode() const noexcept = 0;
1053}; // class VGridSampleLayer
1054
1055class VNMSLayer : public VRoot
1056{
1057public:
1058 virtual void setBoundingBoxFormat(BoundingBoxFormat fmt) noexcept = 0;
1059 virtual BoundingBoxFormat getBoundingBoxFormat() const noexcept = 0;
1060 virtual void setTopKBoxLimit(int32_t limit) noexcept = 0;
1061 virtual int32_t getTopKBoxLimit() const noexcept = 0;
1062 virtual bool setIndicesType(DataType type) noexcept = 0;
1063 virtual DataType getIndicesType() const noexcept = 0;
1064}; // class VNMSLayer
1065
1066class VReverseSequenceLayer : public VRoot
1067{
1068public:
1069 virtual void setBatchAxis(int32_t batchAxis) noexcept = 0;
1070 virtual int32_t getBatchAxis() const noexcept = 0;
1071
1072 virtual void setSequenceAxis(int32_t sequenceAxis) noexcept = 0;
1073 virtual int32_t getSequenceAxis() const noexcept = 0;
1074}; // class VReverseSequenceLayer
1075
1076class VNormalizationLayer : public VRoot
1077{
1078public:
1079 virtual void setEpsilon(float eps) noexcept = 0;
1080 virtual float getEpsilon() const noexcept = 0;
1081 virtual void setAxes(uint32_t axesMask) noexcept = 0;
1082 virtual uint32_t getAxes() const noexcept = 0;
1083 virtual void setNbGroups(int64_t nbGroups) noexcept = 0;
1084 virtual int64_t getNbGroups() const noexcept = 0;
1085 virtual bool isV2() const noexcept = 0;
1086}; // class VNormalizationLayer
1087
1088class VSqueezeLayer : public VRoot
1089{
1090};
1091
1092class VUnsqueezeLayer : public VRoot
1093{
1094};
1095
1096class VCumulativeLayer : public VRoot
1097{
1098public:
1099 virtual bool setOperation(CumulativeOperation op) noexcept = 0;
1100 virtual CumulativeOperation getOperation() const noexcept = 0;
1101 virtual void setExclusive(bool exclusive) noexcept = 0;
1102 virtual bool getExclusive() const noexcept = 0;
1103 virtual void setReverse(bool reverse) noexcept = 0;
1104 virtual bool getReverse() const noexcept = 0;
1105}; // class VCumulativeLayer
1106
1107class VRotaryEmbeddingLayer : public VRoot
1108{
1109public:
1110 virtual void setInterleaved(bool interleaved) noexcept = 0;
1111 virtual bool getInterleaved() const noexcept = 0;
1112 virtual bool setRotaryEmbeddingDim(int32_t rotaryEmbeddingDim) noexcept = 0;
1113 virtual int32_t getRotaryEmbeddingDim() const noexcept = 0;
1114 virtual void setInput(int32_t index, ITensor& input) noexcept = 0;
1115}; // class VRotaryEmbeddingLayer
1116
1117class VKVCacheUpdateLayer : public VRoot
1118{
1119public:
1120 TRT_NODISCARD virtual bool setCacheMode(KVCacheMode cacheMode) noexcept = 0;
1121 TRT_NODISCARD virtual KVCacheMode getCacheMode() const noexcept = 0;
1122 TRT_NODISCARD virtual bool setUpdateForm(AttentionIOForm form) noexcept = 0;
1123 TRT_NODISCARD virtual AttentionIOForm getUpdateForm() const noexcept = 0;
1124 TRT_NODISCARD virtual bool setUpdateLengths(ITensor* lengths) noexcept = 0;
1125 TRT_NODISCARD virtual ITensor* getUpdateLengths() const noexcept = 0;
1126}; // class VKVCacheUpdateLayer
1127
1128class VMoELayer : public VRoot
1129{
1130public:
1131 virtual void setGatedWeights(ITensor& fcGateWeights, ITensor& fcUpWeights, ITensor& fcDownWeights, MoEActType activationType) noexcept = 0;
1132 virtual void setGatedBiases(ITensor& fcGateBiases, ITensor& fcUpBiases, ITensor& fcDownBiases) noexcept = 0;
1133 virtual void setActivationType(MoEActType activationType) noexcept = 0;
1134 virtual MoEActType getActivationType() const noexcept = 0;
1135 virtual void setQuantizationStatic(ITensor& fcDownActivationScale, DataType dataType) noexcept = 0;
1136 virtual void setQuantizationDynamicDblQ(ITensor& fcDownActivationDblQScale, DataType dataType, Dims const& blockShape, DataType dynQOutputScaleType) noexcept = 0;
1137 virtual void setQuantizationToType(DataType type) noexcept = 0;
1138 virtual DataType getQuantizationToType() const noexcept = 0;
1139 virtual void setQuantizationBlockShape(Dims const& blockShape) noexcept = 0;
1140 virtual Dims getQuantizationBlockShape() const noexcept = 0;
1141 virtual void setDynQOutputScaleType(DataType type) noexcept = 0;
1142 virtual DataType getDynQOutputScaleType() const noexcept = 0;
1143 virtual void setSwigluParams(float limit, float alpha, float beta) noexcept = 0;
1144 virtual void setSwigluParamLimit(float limit) noexcept = 0;
1145 virtual float getSwigluParamLimit() const noexcept = 0;
1146 virtual void setSwigluParamAlpha(float alpha) noexcept = 0;
1147 virtual float getSwigluParamAlpha() const noexcept = 0;
1148 virtual void setSwigluParamBeta(float beta) noexcept = 0;
1149 virtual float getSwigluParamBeta() const noexcept = 0;
1150 virtual void setInput(int32_t index, ITensor& tensor) noexcept = 0;
1151}; // class VMoELayer
1152
1153
1154class VNetworkDefinition : public VRoot
1155{
1156public:
1157 virtual ITensor* addInput(char const* name, DataType type, Dims const& dimensions) noexcept = 0;
1158 virtual void markOutput(ITensor& tensor) noexcept = 0;
1159 virtual IActivationLayer* addActivation(ITensor& input, ActivationType type) noexcept = 0;
1160 virtual ILRNLayer* addLRN(ITensor& input, int64_t window, float alpha, float beta, float k) noexcept = 0;
1161 virtual IScaleLayer* addScale(
1162 ITensor& input, ScaleMode mode, Weights shift, Weights scale, Weights power) noexcept = 0;
1163 virtual ISoftMaxLayer* addSoftMax(ITensor& input) noexcept = 0;
1164 virtual IConcatenationLayer* addConcatenation(ITensor* const* inputs, int32_t nbInputs) noexcept = 0;
1165 virtual IElementWiseLayer* addElementWise(ITensor& input1, ITensor& input2, ElementWiseOperation op) noexcept = 0;
1166 virtual IUnaryLayer* addUnary(ITensor& input, UnaryOperation operation) noexcept = 0;
1167 virtual IShuffleLayer* addShuffle(ITensor& input) noexcept = 0;
1168 virtual int32_t getNbLayers() const noexcept = 0;
1169 virtual ILayer* getLayer(int32_t index) const noexcept = 0;
1170 virtual int32_t getNbInputs() const noexcept = 0;
1171 virtual ITensor* getInput(int32_t index) const noexcept = 0;
1172 virtual int32_t getNbOutputs() const noexcept = 0;
1173 virtual ITensor* getOutput(int32_t index) const noexcept = 0;
1174 virtual IReduceLayer* addReduce(
1175 ITensor& input, ReduceOperation operation, uint32_t reduceAxes, bool keepDimensions) noexcept
1176 = 0;
1177 virtual ITopKLayer* addTopK(ITensor& input, TopKOperation op, int32_t k, uint32_t reduceAxes) noexcept = 0;
1178 virtual IGatherLayer* addGather(ITensor& data, ITensor& indices, int32_t axis) noexcept = 0;
1179 virtual IRaggedSoftMaxLayer* addRaggedSoftMax(ITensor& input, ITensor& bounds) noexcept = 0;
1180 virtual IMatrixMultiplyLayer* addMatrixMultiply(
1181 ITensor& input0, MatrixOperation op0, ITensor& input1, MatrixOperation op1) noexcept = 0;
1182 virtual IConstantLayer* addConstant(Dims const& dimensions, Weights weights) noexcept = 0;
1183 virtual IIdentityLayer* addIdentity(ITensor& input) noexcept = 0;
1184 virtual void removeTensor(ITensor& tensor) noexcept = 0;
1185 virtual void unmarkOutput(ITensor& tensor) noexcept = 0;
1186 virtual ISliceLayer* addSlice(ITensor& input, Dims const& start, Dims const& size, Dims const& stride) noexcept = 0;
1187 virtual void setName(char const* name) noexcept = 0;
1188 virtual char const* getName() const noexcept = 0;
1189 virtual IShapeLayer* addShape(ITensor& input) noexcept = 0;
1190 virtual bool hasImplicitBatchDimension() const noexcept = 0;
1191 virtual bool markOutputForShapes(ITensor& tensor) noexcept = 0;
1192 virtual bool unmarkOutputForShapes(ITensor& tensor) noexcept = 0;
1193 virtual IParametricReLULayer* addParametricReLU(ITensor& input, ITensor& slope) noexcept = 0;
1194 virtual IConvolutionLayer* addConvolutionNd(
1195 ITensor& input, int64_t nbOutputMaps, Dims const& kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
1196 = 0;
1197 virtual IPoolingLayer* addPoolingNd(ITensor& input, PoolingType type, Dims const& windowSize) noexcept = 0;
1198 virtual IDeconvolutionLayer* addDeconvolutionNd(
1199 ITensor& input, int64_t nbOutputMaps, Dims const& kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
1200 = 0;
1201 virtual IScaleLayer* addScaleNd(
1202 ITensor& input, ScaleMode mode, Weights shift, Weights scale, Weights power, int32_t channelAxis) noexcept = 0;
1203 virtual IResizeLayer* addResize(ITensor& input) noexcept = 0;
1204 virtual ILoop* addLoop() noexcept = 0;
1205 virtual ISelectLayer* addSelect(ITensor& condition, ITensor& thenInput, ITensor& elseInput) noexcept = 0;
1206 virtual IPaddingLayer* addPaddingNd(ITensor& input, Dims const& prePadding, Dims const& postPadding) noexcept = 0;
1207 virtual bool setWeightsName(Weights weights, char const* name) noexcept = 0;
1208 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
1209 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
1210 virtual IGatherLayer* addGatherV2(ITensor& data, ITensor& indices, GatherMode mode) noexcept = 0;
1211 virtual IIfConditional* addIfConditional() noexcept = 0;
1212 virtual IScatterLayer* addScatter(ITensor& data, ITensor& indices, ITensor& updates, ScatterMode mode) noexcept = 0;
1213 virtual IEinsumLayer* addEinsum(ITensor* const* inputs, int32_t nbInputs, char const* equation) noexcept = 0;
1214 virtual IAssertionLayer* addAssertion(ITensor& condition, char const* message) noexcept = 0;
1215 virtual IOneHotLayer* addOneHot(ITensor& indices, ITensor& values, ITensor& depth, int32_t axis) noexcept = 0;
1216 virtual INonZeroLayer* addNonZero(ITensor& input) noexcept = 0;
1217 virtual IGridSampleLayer* addGridSample(ITensor& input, ITensor& grid) noexcept = 0;
1218 virtual INMSLayer* addNMS(ITensor& boxes, ITensor& scores, ITensor& maxOutputBoxesPerClass) noexcept = 0;
1219 virtual IReverseSequenceLayer* addReverseSequence(ITensor& input, ITensor& sequenceLens) noexcept = 0;
1220 virtual INormalizationLayer* addNormalization(
1221 ITensor& input, ITensor& scale, ITensor& bias, uint32_t axesMask) noexcept = 0;
1222 virtual ICastLayer* addCast(ITensor& input, DataType toType) noexcept = 0;
1223 virtual IBuilder& getBuilder() const noexcept = 0;
1224 virtual NetworkDefinitionCreationFlags getFlags() const noexcept = 0;
1225 virtual bool getFlag(NetworkDefinitionCreationFlag networkDefinitionCreationFlag) const noexcept = 0;
1226 virtual IQuantizeLayer* addQuantizeV2(ITensor& input, ITensor& scale, DataType outputType) noexcept = 0;
1227 virtual IDequantizeLayer* addDequantizeV2(ITensor& input, ITensor& scale, DataType outputType) noexcept = 0;
1228 virtual IFillLayer* addFillV2(Dims const& dimensions, FillOperation op, DataType outputType) noexcept = 0;
1229 virtual bool markDebug(ITensor& tensor) noexcept = 0;
1230 virtual bool unmarkDebug(ITensor& tensor) noexcept = 0;
1231 virtual bool isDebugTensor(ITensor const& tensor) const noexcept = 0;
1232 virtual bool markWeightsRefittable(char const* name) noexcept = 0;
1233 virtual bool unmarkWeightsRefittable(char const* name) noexcept = 0;
1234 virtual bool areWeightsMarkedRefittable(char const* name) const noexcept = 0;
1235 virtual ISqueezeLayer* addSqueeze(ITensor& input, ITensor& axes) noexcept = 0;
1236 virtual IUnsqueezeLayer* addUnsqueeze(ITensor& input, ITensor& axes) noexcept = 0;
1237 virtual IDynamicQuantizeLayer* addDynamicQuantize(
1238 ITensor& input, int32_t axis, int32_t blockSize, DataType toType, DataType scaleType) noexcept = 0;
1239 virtual ICumulativeLayer* addCumulative(
1240 ITensor& input, ITensor& axis, CumulativeOperation operation, bool exclusive, bool reverse) noexcept = 0;
1241 virtual bool markUnfusedTensorsAsDebugTensors() noexcept = 0;
1242 virtual bool unmarkUnfusedTensorsAsDebugTensors() noexcept = 0;
1243 virtual ITopKLayer* addTopKV2(
1244 ITensor& input, TopKOperation op, int32_t k, uint32_t reduceAxes, DataType indicesType) noexcept = 0;
1245 virtual INonZeroLayer* addNonZeroV2(ITensor& input, DataType indicesType) noexcept = 0;
1246 virtual INMSLayer* addNMSV2(
1247 ITensor& boxes, ITensor& scores, ITensor& maxOutputBoxesPerClass, DataType indicesType) noexcept = 0;
1248 TRT_DEPRECATED virtual IAttention* addAttention(
1249 ITensor& query, ITensor& key, ITensor& value, AttentionNormalizationOp normOp, bool isCausal) noexcept = 0;
1250 virtual IRotaryEmbeddingLayer* addRotaryEmbedding(ITensor& input, ITensor& cosCache, ITensor& sinCache,
1251 bool interleaved, int32_t rotaryEmbeddingDim) noexcept = 0;
1252 virtual IDynamicQuantizeLayer* addDynamicQuantizeV2(
1253 ITensor& input, Dims const& blockShape, DataType toType, DataType scaleType) noexcept = 0;
1254 virtual IKVCacheUpdateLayer* addKVCacheUpdate(
1255 ITensor& cache, ITensor& update, ITensor& writeIndices, KVCacheMode cacheMode) noexcept = 0;
1256 virtual INormalizationLayer* addNormalizationV2(
1257 ITensor& input, ITensor& scale, ITensor& bias, uint32_t axesMask) noexcept = 0;
1258 virtual IMoELayer* addMoE(
1259 ITensor& hiddenStates, ITensor& selectedExpertsForTokens, ITensor& scoresForSelectedExperts) noexcept = 0;
1260 virtual IDistCollectiveLayer* addDistCollective(ITensor& input, CollectiveOperation distCollectiveOp,
1261 ReduceOperation reduceOp, int64_t root, int64_t* groups, int64_t groupSize) noexcept = 0;
1262 virtual IAttention* addAttentionV2(ITensor& query, ITensor& key, ITensor& value, AttentionNormalizationOp normOp,
1263 CausalMaskKind causalKind) noexcept = 0;
1264};
1265
1266class VTimingCache : public VRoot
1267{
1268public:
1269 virtual nvinfer1::IHostMemory* serialize() const noexcept = 0;
1270 virtual bool combine(ITimingCache const& inputCache, bool ignoreMismatch) noexcept = 0;
1271 virtual bool reset() noexcept = 0;
1272 virtual int64_t queryKeys(TimingCacheKey* keyBuffer, int64_t capacity) const noexcept = 0;
1273 virtual TimingCacheValue query(TimingCacheKey const& key) const noexcept = 0;
1274 virtual bool update(TimingCacheKey const& key, TimingCacheValue const& value) noexcept = 0;
1275};
1276
1277
1278class VBuilderConfig : public VRoot
1279{
1280public:
1281 virtual void setAvgTimingIterations(int32_t avgTiming) noexcept = 0;
1282 virtual int32_t getAvgTimingIterations() const noexcept = 0;
1283 virtual void setEngineCapability(EngineCapability capability) noexcept = 0;
1284 virtual EngineCapability getEngineCapability() const noexcept = 0;
1285 virtual void setFlags(BuilderFlags builderFlags) noexcept = 0;
1286 virtual BuilderFlags getFlags() const noexcept = 0;
1287 virtual void clearFlag(BuilderFlag builderFlag) noexcept = 0;
1288 virtual void setFlag(BuilderFlag builderFlag) noexcept = 0;
1289 virtual bool getFlag(BuilderFlag builderFlag) const noexcept = 0;
1290 virtual void setDeviceType(ILayer const* layer, DeviceType deviceType) noexcept = 0;
1291 virtual DeviceType getDeviceType(ILayer const* layer) const noexcept = 0;
1292 virtual bool isDeviceTypeSet(ILayer const* layer) const noexcept = 0;
1293 virtual void resetDeviceType(ILayer const* layer) noexcept = 0;
1294 virtual bool canRunOnDLA(ILayer const* layer) const noexcept = 0;
1295 virtual void setDLACore(int32_t dlaCore) noexcept = 0;
1296 virtual int32_t getDLACore() const noexcept = 0;
1297 virtual void setDefaultDeviceType(DeviceType deviceType) noexcept = 0;
1298 virtual DeviceType getDefaultDeviceType() const noexcept = 0;
1299 virtual void reset() noexcept = 0;
1300 virtual void setProfileStream(const cudaStream_t stream) noexcept = 0;
1301 virtual cudaStream_t getProfileStream() const noexcept = 0;
1302 virtual int32_t addOptimizationProfile(IOptimizationProfile const* profile) noexcept = 0;
1303 virtual int32_t getNbOptimizationProfiles() const noexcept = 0;
1304 virtual void setProfilingVerbosity(ProfilingVerbosity verbosity) noexcept = 0;
1305 virtual ProfilingVerbosity getProfilingVerbosity() const noexcept = 0;
1306 virtual bool setTacticSources(TacticSources tacticSources) noexcept = 0;
1307 virtual TacticSources getTacticSources() const noexcept = 0;
1308 virtual nvinfer1::ITimingCache* createTimingCache(void const* blob, std::size_t size) const noexcept = 0;
1309 virtual bool setTimingCache(ITimingCache const& cache, bool ignoreMismatch) noexcept = 0;
1310 virtual nvinfer1::ITimingCache const* getTimingCache() const noexcept = 0;
1311 virtual void setMemoryPoolLimit(MemoryPoolType pool, std::size_t poolSize) noexcept = 0;
1312 virtual std::size_t getMemoryPoolLimit(MemoryPoolType pool) const noexcept = 0;
1313 virtual void setPreviewFeature(PreviewFeature feature, bool enable) noexcept = 0;
1314 virtual bool getPreviewFeature(PreviewFeature feature) const noexcept = 0;
1315 virtual void setBuilderOptimizationLevel(int32_t level) noexcept = 0;
1316 virtual int32_t getBuilderOptimizationLevel() const noexcept = 0;
1317 virtual void setHardwareCompatibilityLevel(HardwareCompatibilityLevel hardwareCompatibilityLevel) noexcept = 0;
1318 virtual HardwareCompatibilityLevel getHardwareCompatibilityLevel() const noexcept = 0;
1319 virtual void setPluginsToSerialize(char const* const* paths, int32_t nbPaths) noexcept = 0;
1320 virtual char const* getPluginToSerialize(int32_t index) const noexcept = 0;
1321 virtual int32_t getNbPluginsToSerialize() const noexcept = 0;
1322 virtual bool setMaxAuxStreams(int32_t nbStreams) noexcept = 0;
1323 virtual int32_t getMaxAuxStreams() const noexcept = 0;
1324 virtual void setProgressMonitor(IProgressMonitor* monitor) noexcept = 0;
1325 virtual IProgressMonitor* getProgressMonitor() const noexcept = 0;
1326 virtual void setRuntimePlatform(RuntimePlatform runtimePlatform) noexcept = 0;
1327 virtual RuntimePlatform getRuntimePlatform() const noexcept = 0;
1328 virtual void setMaxNbTactics(int32_t maxTactics) noexcept = 0;
1329 virtual int32_t getMaxNbTactics() const noexcept = 0;
1330 virtual bool setTilingOptimizationLevel(TilingOptimizationLevel level) noexcept = 0;
1331 virtual TilingOptimizationLevel getTilingOptimizationLevel() const noexcept = 0;
1332 virtual bool setL2LimitForTiling(int64_t size) noexcept = 0;
1333 virtual int64_t getL2LimitForTiling() const noexcept = 0;
1334 TRT_NODISCARD virtual bool setNbComputeCapabilities(int32_t maxNbComputeCapabilities) noexcept = 0;
1335 TRT_NODISCARD virtual int32_t getNbComputeCapabilities() const noexcept = 0;
1336 TRT_NODISCARD virtual bool setComputeCapability(ComputeCapability computeCapability, int32_t index) noexcept = 0;
1337 TRT_NODISCARD virtual ComputeCapability getComputeCapability(int32_t index) const noexcept = 0;
1338 virtual bool setBuildRoute(char const* buildRoute) noexcept = 0;
1339 virtual char const* getBuildRoute() const noexcept = 0;
1340 virtual char const* getAllBuildRoutes() const noexcept = 0;
1341};
1342
1343class VSerializationConfig : public VRoot
1344{
1345public:
1346 virtual bool setFlags(SerializationFlags serializationFlags) noexcept = 0;
1347 virtual SerializationFlags getFlags() const noexcept = 0;
1348 virtual bool clearFlag(SerializationFlag serializationFlag) noexcept = 0;
1349 virtual bool setFlag(SerializationFlag serializationFlag) noexcept = 0;
1350 virtual bool getFlag(SerializationFlag serializationFlag) const noexcept = 0;
1351};
1352
1353class VBuilder : public VRoot
1354{
1355public:
1356 virtual int32_t getMaxDLABatchSize() const noexcept = 0;
1357 virtual int32_t getNbDLACores() const noexcept = 0;
1358 virtual void setGpuAllocator(IGpuAllocator* allocator) noexcept = 0;
1359 virtual nvinfer1::IBuilderConfig* createBuilderConfig() noexcept = 0;
1360 virtual nvinfer1::INetworkDefinition* createNetworkV2(NetworkDefinitionCreationFlags flags) noexcept = 0;
1361 virtual nvinfer1::IOptimizationProfile* createOptimizationProfile() noexcept = 0;
1362 virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;
1363 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
1364 virtual void reset() noexcept = 0;
1365 virtual nvinfer1::IHostMemory* buildSerializedNetwork(
1366 INetworkDefinition& network, IBuilderConfig& config) noexcept = 0;
1367 virtual bool isNetworkSupported(INetworkDefinition const& network, IBuilderConfig const& config) const noexcept = 0;
1368 virtual ILogger* getLogger() const noexcept = 0;
1369 virtual bool setMaxThreads(int32_t maxThreads) noexcept = 0;
1370 virtual int32_t getMaxThreads() const noexcept = 0;
1371 virtual IPluginRegistry& getPluginRegistry() noexcept = 0;
1372 virtual bool buildSerializedNetworkToStream(
1373 INetworkDefinition& network, IBuilderConfig& config, IStreamWriter& writer) noexcept = 0;
1374};
1375
1376class VRuntimeConfig : public VRoot
1377{
1378public:
1379 virtual IRuntimeConfig* getPImpl() noexcept = 0;
1380 virtual void setExecutionContextAllocationStrategy(ExecutionContextAllocationStrategy strategy) noexcept = 0;
1381 virtual ExecutionContextAllocationStrategy getExecutionContextAllocationStrategy() const noexcept = 0;
1382 TRT_NODISCARD virtual nvinfer1::IRuntimeCache* createRuntimeCache() const noexcept = 0;
1383 virtual bool setRuntimeCache(nvinfer1::IRuntimeCache const& cache) noexcept = 0;
1384 TRT_NODISCARD virtual nvinfer1::IRuntimeCache* getRuntimeCache() const noexcept = 0;
1385 virtual void setDynamicShapesKernelSpecializationStrategy(
1386 DynamicShapesKernelSpecializationStrategy dynamicShapesKernelSpecializationStrategy) noexcept = 0;
1387 virtual DynamicShapesKernelSpecializationStrategy getDynamicShapesKernelSpecializationStrategy() const noexcept = 0;
1388 virtual bool setCudaGraphStrategy(CudaGraphStrategy strategy) noexcept = 0;
1389 virtual CudaGraphStrategy getCudaGraphStrategy() const noexcept = 0;
1390};
1391
1392class VRuntimeCache : public VRoot
1393{
1394public:
1395 TRT_NODISCARD virtual nvinfer1::IHostMemory* serialize() const noexcept = 0;
1396 virtual bool deserialize(void const* blob, size_t size) noexcept = 0;
1397 virtual bool reset() noexcept = 0;
1398};
1399
1400class VDistCollectiveLayer : public VRoot
1401{
1402}; // VDistCollectiveLayer
1403
1404} // namespace apiv
1405} // namespace nvinfer1
1406
1407// @endcond
1408
1409#endif // NV_INFER_RUNTIME_IMPL_H
getPluginRegistry
TENSORRTAPI nvinfer1::IPluginRegistry * getPluginRegistry() noexcept
Return the plugin registry.
getLogger
TENSORRTAPI nvinfer1::ILogger * getLogger() noexcept
Return the logger object.
TRT_NODISCARD
#define TRT_NODISCARD
A stand-in for [[nodiscard]] and [[nodiscard(REASON)]] that works with older compilers.
Definition: NvInferRuntimeBase.h:57
TRT_DEPRECATED
#define TRT_DEPRECATED
Definition: NvInferRuntimeBase.h:42
Dims
Structure to define the dimensions of a tensor.
IDebugListener
User-implemented callback for notification when value of a debug tensor is updated.
IErrorRecorder
Reference counted application-implemented error reporting interface for TensorRT objects.
IGpuAllocator
Application-implemented class for controlling allocation on the GPU.
IOutputAllocator
Callback from ExecutionContext::enqueueV3()
IPluginV3
Plugin class for the V3 generation of user-implemented layers.
IProfiler
Application-implemented interface for profiling.
IProgressMonitor
Application-implemented progress reporting interface for TensorRT.
IStreamReader
Application-implemented class for reading data in a stream-based manner.
IStreamReaderV2
Application-implemented class for reading data in a stream-based manner asynchronously....
IStreamWriter
Application-implemented class for writing data in a stream-based manner.
nvinfer1::IHostMemory
Class to handle library allocated memory that is accessible to the user.
Definition: NvInferRuntime.h:139
nvinfer1
The TensorRT API version 1 namespace.
Definition: NvInferPluginBase.h:29
nvinfer1::TacticSources
uint32_t TacticSources
Represents a collection of one or more TacticSource values combine using bitwise-OR operations.
Definition: NvInferRuntime.h:2900
nvinfer1::IOutputAllocator
v_1_0::IOutputAllocator IOutputAllocator
Definition: NvInferRuntime.h:4303
nvinfer1::ResizeSelector
ResizeSelector
The coordinate selector when resize to single pixel output.
Definition: NvInfer.h:3917
nvinfer1::EngineCapability
EngineCapability
List of supported engine capability flows.
Definition: NvInferRuntime.h:76
nvinfer1::DimensionOperation
DimensionOperation
An operation on two IDimensionExpr, which represent integer expressions used in dimension computation...
Definition: NvInferRuntime.h:178
nvinfer1::MemoryPoolType
MemoryPoolType
The type for memory pools used by TensorRT.
Definition: NvInfer.h:10190
nvinfer1::AttentionIOForm
AttentionIOForm
Enumerates the layout of the input/output tensors in an Attention layer.
Definition: NvInfer.h:6728
nvinfer1::ScaleMode
ScaleMode
Controls how shift, scale and power are applied in a Scale layer.
Definition: NvInfer.h:1587
nvinfer1::RuntimePlatform
RuntimePlatform
Describes the intended runtime platform (operating system and CPU architecture) for the execution of ...
Definition: NvInfer.h:9803
nvinfer1::TensorIOMode
TensorIOMode
Definition of tensor IO Mode.
Definition: NvInferRuntimeBase.h:662
nvinfer1::HardwareCompatibilityLevel
HardwareCompatibilityLevel
Describes requirements of compatibility with GPU architectures other than that of the GPU on which th...
Definition: NvInfer.h:10298
nvinfer1::CumulativeOperation
CumulativeOperation
Enumerates the cumulative operations that may be performed by a Cumulative layer.
Definition: NvInfer.h:6536
nvinfer1::SerializationFlag
SerializationFlag
List of valid flags that the engine can enable when serializing the bytes.
Definition: NvInferRuntime.h:2941
nvinfer1::BoundingBoxFormat
BoundingBoxFormat
Representation of bounding box data used for the Boxes input tensor in INMSLayer.
Definition: NvInfer.h:6103
nvinfer1::ComputeCapability
ComputeCapability
Describes compute capability that an engine will be built for.
Definition: NvInfer.h:10344
nvinfer1::IPluginV3
v_1_0::IPluginV3 IPluginV3
Definition: NvInferPluginBase.h:273
nvinfer1::UnaryOperation
UnaryOperation
Enumerates the unary operations that may be performed by a Unary layer.
Definition: NvInfer.h:2545
nvinfer1::IStreamWriter
v_1_0::IStreamWriter IStreamWriter
Definition: NvInferRuntime.h:699
nvinfer1::IProfiler
v_1_0::IProfiler IProfiler
Definition: NvInferRuntime.h:1309
nvinfer1::DynamicShapesKernelSpecializationStrategy
DynamicShapesKernelSpecializationStrategy
Different kernel specialization strategies for dynamic shapes.
Definition: NvInferRuntime.h:3133
nvinfer1::IStreamReaderV2
v_1_0::IStreamReaderV2 IStreamReaderV2
Definition: NvInferRuntime.h:776
nvinfer1::ActivationType
ActivationType
Enumerates the types of activation to perform in an activation layer.
Definition: NvInfer.h:143
nvinfer1::TempfileControlFlags
uint32_t TempfileControlFlags
Represents a collection of one or more TempfileControlFlag values combined using bitwise-OR operation...
Definition: NvInferRuntime.h:1387
nvinfer1::FillOperation
FillOperation
Enumerates the tensor fill operations that may performed by a fill layer.
Definition: NvInfer.h:4920
nvinfer1::EngineStat
EngineStat
The kind of engine statistics that queried from the ICudaEngine.
Definition: NvInferRuntime.h:3324
nvinfer1::CudaGraphStrategy
CudaGraphStrategy
Strategies available for CUDA graphs optimizations for JIT (Just-In-Time) inference.
Definition: NvInferRuntime.h:3169
nvinfer1::ResizeRoundMode
ResizeRoundMode
The rounding mode for nearest neighbor resize.
Definition: NvInfer.h:3944
nvinfer1::EngineValidity
EngineValidity
Whether a TensorRT-RTX engine is likely to be valid on the current system.
Definition: NvInferRuntime.h:1859
nvinfer1::ILogger
v_1_0::ILogger ILogger
Definition: NvInferRuntimeBase.h:125
nvinfer1::CausalMaskKind
CausalMaskKind
Enumerates the causal mask alignment orientation for the attention.
Definition: NvInfer.h:6700
nvinfer1::PaddingMode
PaddingMode
Enumerates the modes of padding to perform in convolution, deconvolution and pooling layer,...
Definition: NvInfer.h:763
nvinfer1::TripLimit
TripLimit
Enum that describes kinds of trip limits.
Definition: NvInfer.h:4302
nvinfer1::NetworkDefinitionCreationFlags
uint32_t NetworkDefinitionCreationFlags
Represents one or more NetworkDefinitionCreationFlag flags using binary OR operations....
Definition: NvInfer.h:11422
nvinfer1::PreviewFeature
PreviewFeature
Define preview features.
Definition: NvInfer.h:10264
nvinfer1::TilingOptimizationLevel
TilingOptimizationLevel
Define the optimization levels for Tiling.
Definition: NvInfer.h:10372
nvinfer1::DataType
DataType
The type of weights and tensors. The datatypes other than kBOOL, kINT32, and kINT64 are "activation d...
Definition: NvInferRuntimeBase.h:149
nvinfer1::BuilderFlags
uint32_t BuilderFlags
Represents one or more BuilderFlag values using binary OR operations, e.g., 1U << BuilderFlag::kDEBUG...
Definition: NvInfer.h:9833
nvinfer1::DeviceType
DeviceType
The device that this layer/network will execute on.
Definition: NvInferRuntime.h:1341
nvinfer1::LayerType
LayerType
The type values of layer classes.
Definition: NvInfer.h:58
nvinfer1::SampleMode
SampleMode
Controls how ISliceLayer and IGridSample handle out-of-bounds coordinates.
Definition: NvInfer.h:3047
nvinfer1::GatherMode
GatherMode
Control form of IGatherLayer.
Definition: NvInfer.h:2286
nvinfer1::IDebugListener
v_1_0::IDebugListener IDebugListener
Definition: NvInferRuntime.h:4344
nvinfer1::MoEActType
MoEActType
Enumerates the activation type for the MoE layer.
Definition: NvInfer.h:7617
nvinfer1::WeightsRole
WeightsRole
How a layer uses particular Weights.
Definition: NvInferRuntime.h:1319
nvinfer1::TensorFormats
uint32_t TensorFormats
It is capable of representing one or more TensorFormat by binary OR operations, e....
Definition: NvInfer.h:135
nvinfer1::ProfilingVerbosity
ProfilingVerbosity
List of verbosity levels of layer information exposed in NVTX annotations and in IEngineInspector.
Definition: NvInferRuntime.h:2912
nvinfer1::NetworkDefinitionCreationFlag
NetworkDefinitionCreationFlag
List of immutable network properties expressed at network creation time. NetworkDefinitionCreationFla...
Definition: NvInfer.h:11433
nvinfer1::ElementWiseOperation
ElementWiseOperation
Enumerates the binary operations that may be performed by an ElementWise layer.
Definition: NvInfer.h:2197
nvinfer1::CollectiveOperation
CollectiveOperation
Enumerates the collective operations that may be performed by a DistCollective layer.
Definition: NvInfer.h:2675
nvinfer1::SerializationFlags
uint32_t SerializationFlags
Represents one or more SerializationFlag values using binary OR operations, e.g., 1U << Serialization...
Definition: NvInferRuntime.h:2931
nvinfer1::InterpolationMode
InterpolationMode
Enumerates various modes of interpolation.
Definition: NvInfer.h:3841
nvinfer1::BuilderFlag
BuilderFlag
List of valid modes that the builder can enable when creating an engine from a network definition.
Definition: NvInfer.h:9843
nvinfer1::TensorFormat
TensorFormat
Format of the input/output tensors.
Definition: NvInferRuntime.h:1421
nvinfer1::ExecutionContextAllocationStrategy
ExecutionContextAllocationStrategy
Different memory allocation behaviors for IExecutionContext.
Definition: NvInferRuntime.h:3048
nvinfer1::TopKOperation
TopKOperation
Enumerates the operations that may be performed by a TopK layer.
Definition: NvInfer.h:3332
nvinfer1::ReduceOperation
ReduceOperation
Enumerates the reduce operations that may be performed by a Reduce layer.
Definition: NvInfer.h:2647
nvinfer1::LayerInformationFormat
LayerInformationFormat
The format in which the IEngineInspector prints the layer information.
Definition: NvInferRuntime.h:5299
nvinfer1::ScatterMode
ScatterMode
Control form of IScatterLayer.
Definition: NvInfer.h:5830
nvinfer1::MatrixOperation
MatrixOperation
Enumerates the operations that may be performed on a tensor by IMatrixMultiplyLayer before multiplica...
Definition: NvInfer.h:3487
nvinfer1::ResizeCoordinateTransformation
ResizeCoordinateTransformation
The resize coordinate transformation function.
Definition: NvInfer.h:3866
nvinfer1::LoopOutput
LoopOutput
Enum that describes kinds of loop outputs.
Definition: NvInfer.h:4274
nvinfer1::KVCacheMode
KVCacheMode
Enumerates the KVCache modes that may be performed by a KVCacheUpdate layer.
Definition: NvInfer.h:7464
nvinfer1::IStreamReader
v_1_0::IStreamReader IStreamReader
Definition: NvInferRuntime.h:689
nvinfer1::PoolingType
PoolingType
The type of pooling to perform in a pooling layer.
Definition: NvInfer.h:1200
nvinfer1::IProgressMonitor
v_1_0::IProgressMonitor IProgressMonitor
Definition: NvInfer.h:10485
nvinfer1::TensorLocation
TensorLocation
The location for tensor data storage, device or host.
Definition: NvInferRuntime.h:203
nvinfer1::OptProfileSelector
OptProfileSelector
When setting or querying optimization profile parameters (such as shape tensor inputs or dynamic dime...
Definition: NvInferRuntime.h:2659
nvinfer1::AttentionNormalizationOp
AttentionNormalizationOp
Enumerates the operations that may be performed by the normalization in the attention subgraph.
Definition: NvInfer.h:6668

  Copyright © 2024 NVIDIA Corporation
  Privacy Policy | Manage My Privacy | Do Not Sell or Share My Data | Terms of Service | Accessibility | Corporate Policies | Product Security | Contact