18#ifndef NV_INFER_IMPL_H
19#define NV_INFER_IMPL_H
59class IActivationLayer;
61class IAlgorithmContext;
62class IAlgorithmIOInfo;
63class IAlgorithmVariant;
67class IConcatenationLayer;
70class IConvolutionLayer;
72class IDeconvolutionLayer;
73class IDequantizeLayer;
76class IElementWiseLayer;
77class IEngineInspector;
78class IExecutionContext;
81class IGridSampleLayer;
86class IIfConditionalInputLayer;
87class IIfConditionalOutputLayer;
92class ILoopOutputLayer;
94class IMatrixMultiplyLayer;
95class INetworkDefinition;
96class INormalizationLayer;
100class IOptimizationProfile;
102class IParametricReLULayer;
107class IPluginRegistry;
119class IRaggedSoftMaxLayer;
120class IRecurrenceLayer;
124class IReverseSequenceLayer;
129class ISerializationConfig;
137class ITripLimitLayer;
205 virtual ~VRoot() noexcept = default;
208class VHostMemory : public VRoot
211 virtual void* data() const noexcept = 0;
212 virtual std::
size_t size() const noexcept = 0;
213 virtual
DataType type() const noexcept = 0;
216class VDimensionExpr : public VRoot
219 virtual bool isConstant()
const = 0;
220 virtual int64_t getConstantValue()
const = 0;
221 virtual bool isSizeTensor()
const = 0;
224class VExprBuilder :
public VRoot
227 virtual IDimensionExpr
const* constant(int64_t value) = 0;
228 virtual IDimensionExpr
const* operation(
231 virtual IDimensionExpr
const* declareSizeTensor(
232 int32_t outputIndex, IDimensionExpr
const& opt, IDimensionExpr
const& upper)
236class VRuntime :
public VRoot
239 virtual IRuntime* getPImpl() noexcept = 0;
240 virtual
nvinfer1::ICudaEngine* deserializeCudaEngine(
void const* blob, std::
size_t size) noexcept = 0;
242 virtual
void setDLACore(int32_t dlaCore) noexcept = 0;
243 virtual int32_t getDLACore() const noexcept = 0;
244 virtual int32_t getNbDLACores() const noexcept = 0;
245 virtual
void setGpuAllocator(
IGpuAllocator* allocator) noexcept = 0;
246 virtual
void setErrorRecorder(
IErrorRecorder* recorder) noexcept = 0;
248 virtual ILogger*
getLogger() const noexcept = 0;
249 virtual
bool setMaxThreads(int32_t maxThreads) noexcept = 0;
250 virtual int32_t getMaxThreads() const noexcept = 0;
251 virtual
void setTemporaryDirectory(
char const*) noexcept = 0;
252 virtual
char const* getTemporaryDirectory() const noexcept = 0;
256 virtual
void setPluginRegistryParent(IPluginRegistry* parent) noexcept = 0;
257 virtual IRuntime* loadRuntime(
char const* path) noexcept = 0;
258 virtual
void setEngineHostCodeAllowed(
bool allowed) noexcept = 0;
259 virtual
bool getEngineHostCodeAllowed() const noexcept = 0;
262class VRefitter : public VRoot
265 virtual IRefitter* getPImpl() noexcept = 0;
266 virtual
bool setWeights(
char const* layerName,
WeightsRole role, const Weights weights) noexcept = 0;
267 virtual
bool refitCudaEngine() noexcept = 0;
268 virtual int32_t getMissing(int32_t size,
char const** layerNames,
WeightsRole* roles) noexcept = 0;
269 virtual int32_t getAll(int32_t size,
char const** layerNames,
WeightsRole* roles) noexcept = 0;
270 virtual
bool setDynamicRange(
char const* tensorName,
float min,
float max) noexcept = 0;
271 virtual
float getDynamicRangeMin(
char const* tensorName) const noexcept = 0;
272 virtual
float getDynamicRangeMax(
char const* tensorName) const noexcept = 0;
273 virtual int32_t getTensorsWithDynamicRange(int32_t size,
char const** tensorNames) const noexcept = 0;
274 virtual
void setErrorRecorder(
IErrorRecorder* recorder) noexcept = 0;
276 virtual
bool setNamedWeights(
char const* name, Weights weights) noexcept = 0;
277 virtual int32_t getMissingWeights(int32_t size,
char const** weightsNames) noexcept = 0;
278 virtual int32_t getAllWeights(int32_t size,
char const** weightsNames) noexcept = 0;
279 virtual ILogger*
getLogger() const noexcept = 0;
280 virtual
bool setMaxThreads(int32_t maxThreads) noexcept = 0;
281 virtual int32_t getMaxThreads() const noexcept = 0;
282 virtual
bool setNamedWeightsWithLocation(
char const* name, Weights weights,
TensorLocation location) noexcept = 0;
283 virtual Weights getNamedWeights(
char const* weightsName) const noexcept = 0;
284 virtual
TensorLocation getWeightsLocation(
char const* weightsName) const noexcept = 0;
285 virtual
bool unsetNamedWeights(
char const* weightsName) noexcept = 0;
286 virtual
void setWeightsValidation(
bool weightsValidation) noexcept = 0;
287 virtual
bool getWeightsValidation() const noexcept = 0;
288 virtual
bool refitCudaEngineAsync(cudaStream_t stream) noexcept = 0;
289 virtual Weights getWeightsPrototype(
char const* weightsName) const noexcept = 0;
292class VOptimizationProfile : public VRoot
295 virtual bool setDimensions(
char const* inputName,
OptProfileSelector select,
Dims const& dims)
noexcept = 0;
297 virtual bool setShapeValues(
298 char const* inputName,
OptProfileSelector select, int32_t
const* values, int32_t nbValues)
noexcept = 0;
299 virtual int32_t getNbShapeValues(
char const* inputName)
const noexcept = 0;
300 virtual int32_t
const* getShapeValues(
char const* inputName,
OptProfileSelector select)
const noexcept = 0;
301 virtual bool setExtraMemoryTarget(
float target)
noexcept = 0;
302 virtual float getExtraMemoryTarget() const noexcept = 0;
303 virtual
bool isValid() const noexcept = 0;
306class VCudaEngine : public VRoot
309 virtual ICudaEngine* getPImpl() noexcept = 0;
310 virtual int32_t getNbLayers() const noexcept = 0;
311 virtual IHostMemory* serialize() const noexcept = 0;
313 virtual IExecutionContext* createExecutionContextWithoutDeviceMemory() noexcept = 0;
314 virtual
size_t getDeviceMemorySize() const noexcept = 0;
315 virtual
bool isRefittable() const noexcept = 0;
316 virtual
char const* getName() const noexcept = 0;
317 virtual int32_t getNbOptimizationProfiles() const noexcept = 0;
318 virtual int32_t const* getProfileTensorValues(
319 char const* tensorName, int32_t profileIndex,
OptProfileSelector select) const noexcept = 0;
321 virtual
void setErrorRecorder(
IErrorRecorder* recorder) noexcept = 0;
323 virtual
bool hasImplicitBatchDimension() const noexcept = 0;
326 virtual IEngineInspector* createEngineInspector() const noexcept = 0;
327 virtual
Dims getTensorShape(
char const* tensorName) const noexcept = 0;
328 virtual
DataType getTensorDataType(
char const* tensorName) const noexcept = 0;
329 virtual
TensorLocation getTensorLocation(
char const* tensorName) const noexcept = 0;
330 virtual
bool isShapeInferenceIO(
char const* tensorName) const noexcept = 0;
331 virtual
TensorIOMode getTensorIOMode(
char const* tensorName) const noexcept = 0;
332 virtual int32_t getTensorBytesPerComponent(
char const* tensorName) const noexcept = 0;
333 virtual int32_t getTensorComponentsPerElement(
char const* tensorName) const noexcept = 0;
334 virtual
TensorFormat getTensorFormat(
char const* tensorName) const noexcept = 0;
335 virtual
char const* getTensorFormatDesc(
char const* tensorName) const noexcept = 0;
336 virtual int32_t getTensorVectorizedDim(
char const* tensorName) const noexcept = 0;
337 virtual
Dims getProfileShape(
338 char const* tensorName, int32_t profileIndex,
OptProfileSelector select) const noexcept = 0;
339 virtual int32_t getNbIOTensors() const noexcept = 0;
340 virtual
char const* getIOTensorName(int32_t index) const noexcept = 0;
342 virtual int32_t getNbAuxStreams() const noexcept = 0;
344 virtual int32_t getTensorBytesPerComponentV2(
char const* tensorName, int32_t profileIndex) const noexcept = 0;
345 virtual int32_t getTensorComponentsPerElementV2(
char const* tensorName, int32_t profileIndex) const noexcept = 0;
346 virtual
TensorFormat getTensorFormatV2(
char const* tensorName, int32_t profileIndex) const noexcept = 0;
347 virtual
char const* getTensorFormatDescV2(
char const* tensorName, int32_t profileIndex) const noexcept = 0;
348 virtual int32_t getTensorVectorizedDimV2(
char const* tensorName, int32_t profileIndex) const noexcept = 0;
350 virtual ISerializationConfig* createSerializationConfig() noexcept = 0;
351 virtual IHostMemory* serializeWithConfig(ISerializationConfig& config) const noexcept = 0;
353 virtual
size_t getDeviceMemorySizeForProfile(int32_t profileIndex) const noexcept = 0;
354 virtual IRefitter* createRefitter(ILogger& logger) noexcept = 0;
356 virtual
bool setWeightStreamingBudget(int64_t gpuMemoryBudget) noexcept = 0;
357 virtual int64_t getWeightStreamingBudget() const noexcept = 0;
358 virtual int64_t getMinimumWeightStreamingBudget() const noexcept = 0;
359 virtual int64_t getStreamableWeightsSize() const noexcept = 0;
361 virtual
bool isDebugTensor(
char const* name) const noexcept = 0;
364 virtual
bool setWeightStreamingBudgetV2(int64_t gpuMemoryBudget) noexcept = 0;
365 virtual int64_t getWeightStreamingBudgetV2() const noexcept = 0;
366 virtual int64_t getWeightStreamingAutomaticBudget() const noexcept = 0;
367 virtual int64_t getWeightStreamingScratchMemorySize() const noexcept = 0;
368 virtual int64_t getDeviceMemorySizeV2() const noexcept = 0;
369 virtual int64_t getDeviceMemorySizeForProfileV2(int32_t profileIndex) const noexcept = 0;
372class VExecutionContext : public VRoot
375 virtual IExecutionContext* getPImpl() noexcept = 0;
376 virtual
void setDebugSync(
bool sync) noexcept = 0;
377 virtual
bool getDebugSync() const noexcept = 0;
378 virtual
void setProfiler(
IProfiler*) noexcept = 0;
379 virtual
IProfiler* getProfiler() const noexcept = 0;
380 virtual ICudaEngine const& getEngine() const noexcept = 0;
381 virtual
void setName(
char const* name) noexcept = 0;
382 virtual
char const* getName() const noexcept = 0;
383 virtual
void setDeviceMemory(
void* memory) noexcept = 0;
384 virtual int32_t getOptimizationProfile() const noexcept = 0;
385 virtual
bool allInputDimensionsSpecified() const noexcept = 0;
386 virtual
bool allInputShapesSpecified() const noexcept = 0;
387 virtual
void setErrorRecorder(
IErrorRecorder* recorder) noexcept = 0;
389 virtual
bool executeV2(
void* const* bindings) noexcept = 0;
390 virtual
bool setOptimizationProfileAsync(int32_t profileIndex, cudaStream_t stream) noexcept = 0;
391 virtual
void setEnqueueEmitsProfile(
bool enqueueEmitsProfile) noexcept = 0;
392 virtual
bool getEnqueueEmitsProfile() const noexcept = 0;
393 virtual
bool reportToProfiler() const noexcept = 0;
394 virtual
bool setInputShape(
char const* tensorName,
Dims const& dims) noexcept = 0;
395 virtual
Dims getTensorShape(
char const* tensorName) const noexcept = 0;
396 virtual
Dims getTensorStrides(
char const* tensorName) const noexcept = 0;
397 virtual
bool setTensorAddress(
char const* tensorName,
void* data) noexcept = 0;
398 virtual
void const* getTensorAddress(
char const* tensorName) const noexcept = 0;
399 virtual
bool setInputTensorAddress(
char const* tensorName,
void const* data) noexcept = 0;
400 virtual
bool setOutputTensorAddress(
char const* tensorName,
void* data) noexcept = 0;
401 virtual int32_t inferShapes(int32_t nbMaxNames,
char const** tensorNames) noexcept = 0;
402 virtual
bool setInputConsumedEvent(cudaEvent_t event) noexcept = 0;
403 virtual cudaEvent_t getInputConsumedEvent() const noexcept = 0;
404 virtual
void* getOutputTensorAddress(
char const* tensorName) const noexcept = 0;
405 virtual
bool setOutputAllocator(
char const* tensorName,
IOutputAllocator* outputAllocator) noexcept = 0;
406 virtual
IOutputAllocator* getOutputAllocator(
char const* name) noexcept = 0;
407 virtual int64_t getMaxOutputSize(
char const* tensorName) const noexcept = 0;
408 virtual
bool setTemporaryStorageAllocator(
IGpuAllocator* allocator) noexcept = 0;
409 virtual
IGpuAllocator* getTemporaryStorageAllocator() const noexcept = 0;
410 virtual
bool enqueueV3(cudaStream_t stream) noexcept = 0;
411 virtual
void setPersistentCacheLimit(
size_t size) noexcept = 0;
412 virtual
size_t getPersistentCacheLimit() const noexcept = 0;
415 virtual
void setAuxStreams(cudaStream_t* auxStreams, int32_t nbStreams) noexcept = 0;
416 virtual
bool setDebugListener(
IDebugListener* listener) noexcept = 0;
418 virtual
bool setTensorDebugState(
char const* name,
bool flag) noexcept = 0;
419 virtual
bool getDebugState(
char const* name) const noexcept = 0;
420 virtual
bool setAllTensorsDebugState(
bool flag) noexcept = 0;
421 virtual
size_t updateDeviceMemorySizeForShapes() noexcept = 0;
424 virtual
void setDeviceMemoryV2(
void* memory, int64_t size) noexcept = 0;
427class VEngineInspector : public VRoot
430 virtual IEngineInspector* getPImpl() noexcept = 0;
431 virtual
bool setExecutionContext(IExecutionContext const* context) noexcept = 0;
432 virtual IExecutionContext const* getExecutionContext() const noexcept = 0;
433 virtual
char const* getLayerInformation(int32_t layerIndex,
LayerInformationFormat format) const noexcept = 0;
435 virtual
void setErrorRecorder(
IErrorRecorder* recorder) noexcept = 0;
439class VTensor : public VRoot
442 virtual void setName(
char const* name)
noexcept = 0;
443 virtual char const* getName() const noexcept = 0;
444 virtual
void setDimensions(
Dims const& dimensions) noexcept = 0;
445 virtual
Dims getDimensions() const noexcept = 0;
446 virtual
void setType(
DataType type) noexcept = 0;
447 virtual
DataType getType() const noexcept = 0;
448 virtual
bool setDynamicRange(
float min,
float max) noexcept = 0;
449 virtual
bool isNetworkInput() const noexcept = 0;
450 virtual
bool isNetworkOutput() const noexcept = 0;
451 virtual
void setBroadcastAcrossBatch(
bool broadcastAcrossBatch) noexcept = 0;
452 virtual
bool getBroadcastAcrossBatch() const noexcept = 0;
455 virtual
bool dynamicRangeIsSet() const noexcept = 0;
456 virtual
void resetDynamicRange() noexcept = 0;
457 virtual
float getDynamicRangeMin() const noexcept = 0;
458 virtual
float getDynamicRangeMax() const noexcept = 0;
459 virtual
void setAllowedFormats(
TensorFormats formats) noexcept = 0;
460 virtual
TensorFormats getAllowedFormats() const noexcept = 0;
461 virtual
bool isShapeTensor() const noexcept = 0;
462 virtual
bool isExecutionTensor() const noexcept = 0;
463 virtual
void setDimensionName(int32_t index,
char const* name) noexcept = 0;
464 virtual
char const* getDimensionName(int32_t index) const noexcept = 0;
466class VLayer : public VRoot
469 virtual LayerType getType() const noexcept = 0;
470 virtual
void setName(
char const* name) noexcept = 0;
471 virtual
char const* getName() const noexcept = 0;
472 virtual int32_t getNbInputs() const noexcept = 0;
473 virtual ITensor* getInput(int32_t index) const noexcept = 0;
474 virtual int32_t getNbOutputs() const noexcept = 0;
475 virtual ITensor* getOutput(int32_t index) const noexcept = 0;
476 virtual
void setInput(int32_t index, ITensor& tensor) noexcept = 0;
477 virtual
void setPrecision(
DataType dataType) noexcept = 0;
478 virtual
DataType getPrecision() const noexcept = 0;
479 virtual
bool precisionIsSet() const noexcept = 0;
480 virtual
void resetPrecision() noexcept = 0;
481 virtual
void setOutputType(int32_t index,
DataType dataType) noexcept = 0;
482 virtual
DataType getOutputType(int32_t index) const noexcept = 0;
483 virtual
bool outputTypeIsSet(int32_t index) const noexcept = 0;
484 virtual
void resetOutputType(int32_t index) noexcept = 0;
485 virtual
void setMetadata(
char const* docString) noexcept = 0;
486 virtual
char const* getMetadata() const noexcept = 0;
489class VConvolutionLayer : public VRoot
492 virtual void setNbOutputMaps(int64_t nbOutputMaps)
noexcept = 0;
493 virtual int64_t getNbOutputMaps() const noexcept = 0;
494 virtual
void setNbGroups(int64_t nbGroups) noexcept = 0;
495 virtual int64_t getNbGroups() const noexcept = 0;
496 virtual
void setKernelWeights(Weights weights) noexcept = 0;
497 virtual Weights getKernelWeights() const noexcept = 0;
498 virtual
void setBiasWeights(Weights weights) noexcept = 0;
499 virtual Weights getBiasWeights() const noexcept = 0;
500 virtual
void setPrePadding(
Dims const& padding) noexcept = 0;
501 virtual
Dims getPrePadding() const noexcept = 0;
502 virtual
void setPostPadding(
Dims const& padding) noexcept = 0;
503 virtual
Dims getPostPadding() const noexcept = 0;
504 virtual
void setPaddingMode(
PaddingMode paddingMode) noexcept = 0;
505 virtual
PaddingMode getPaddingMode() const noexcept = 0;
506 virtual
void setKernelSizeNd(
Dims const& kernelSize) noexcept = 0;
507 virtual
Dims getKernelSizeNd() const noexcept = 0;
508 virtual
void setStrideNd(
Dims const& stride) noexcept = 0;
509 virtual
Dims getStrideNd() const noexcept = 0;
510 virtual
void setPaddingNd(
Dims const& padding) noexcept = 0;
511 virtual
Dims getPaddingNd() const noexcept = 0;
512 virtual
void setDilationNd(
Dims const& dilation) noexcept = 0;
513 virtual
Dims getDilationNd() const noexcept = 0;
516class VActivationLayer : public VRoot
521 virtual
void setAlpha(
float alpha) noexcept = 0;
522 virtual
void setBeta(
float beta) noexcept = 0;
523 virtual
float getAlpha() const noexcept = 0;
524 virtual
float getBeta() const noexcept = 0;
527class VPoolingLayer : public VRoot
530 virtual void setPoolingType(
PoolingType type)
noexcept = 0;
531 virtual PoolingType getPoolingType() const noexcept = 0;
532 virtual
void setBlendFactor(
float blendFactor) noexcept = 0;
533 virtual
float getBlendFactor() const noexcept = 0;
534 virtual
void setAverageCountExcludesPadding(
bool exclusive) noexcept = 0;
535 virtual
bool getAverageCountExcludesPadding() const noexcept = 0;
536 virtual
void setPrePadding(
Dims const& padding) noexcept = 0;
537 virtual
Dims getPrePadding() const noexcept = 0;
538 virtual
void setPostPadding(
Dims const& padding) noexcept = 0;
539 virtual
Dims getPostPadding() const noexcept = 0;
540 virtual
void setPaddingMode(
PaddingMode paddingMode) noexcept = 0;
541 virtual
PaddingMode getPaddingMode() const noexcept = 0;
542 virtual
void setWindowSizeNd(
Dims const& windowSize) noexcept = 0;
543 virtual
Dims getWindowSizeNd() const noexcept = 0;
544 virtual
void setStrideNd(
Dims const& stride) noexcept = 0;
545 virtual
Dims getStrideNd() const noexcept = 0;
546 virtual
void setPaddingNd(
Dims const& padding) noexcept = 0;
547 virtual
Dims getPaddingNd() const noexcept = 0;
550class VLRNLayer : public VRoot
553 virtual void setWindowSize(int64_t windowSize)
noexcept = 0;
554 virtual int64_t getWindowSize() const noexcept = 0;
555 virtual
void setAlpha(
float alpha) noexcept = 0;
556 virtual
float getAlpha() const noexcept = 0;
557 virtual
void setBeta(
float beta) noexcept = 0;
558 virtual
float getBeta() const noexcept = 0;
559 virtual
void setK(
float k) noexcept = 0;
560 virtual
float getK() const noexcept = 0;
563class VScaleLayer : public VRoot
566 virtual void setMode(
ScaleMode mode)
noexcept = 0;
567 virtual ScaleMode getMode() const noexcept = 0;
568 virtual
void setShift(Weights shift) noexcept = 0;
569 virtual Weights getShift() const noexcept = 0;
570 virtual
void setScale(Weights scale) noexcept = 0;
571 virtual Weights getScale() const noexcept = 0;
572 virtual
void setPower(Weights power) noexcept = 0;
573 virtual Weights getPower() const noexcept = 0;
574 virtual int32_t getChannelAxis() const noexcept = 0;
575 virtual
void setChannelAxis(int32_t channelAxis) noexcept = 0;
578class VSoftMaxLayer : public VRoot
581 virtual void setAxes(uint32_t axes)
noexcept = 0;
582 virtual uint32_t getAxes() const noexcept = 0;
585class VConcatenationLayer : public VRoot
588 virtual void setAxis(int32_t axis)
noexcept = 0;
589 virtual int32_t getAxis() const noexcept = 0;
592class VDeconvolutionLayer : public VRoot
595 virtual void setNbOutputMaps(int64_t nbOutputMaps)
noexcept = 0;
596 virtual int64_t getNbOutputMaps() const noexcept = 0;
597 virtual
void setNbGroups(int64_t nbGroups) noexcept = 0;
598 virtual int64_t getNbGroups() const noexcept = 0;
599 virtual
void setKernelWeights(Weights weights) noexcept = 0;
600 virtual Weights getKernelWeights() const noexcept = 0;
601 virtual
void setBiasWeights(Weights weights) noexcept = 0;
602 virtual Weights getBiasWeights() const noexcept = 0;
603 virtual
void setPrePadding(
Dims const& padding) noexcept = 0;
604 virtual
Dims getPrePadding() const noexcept = 0;
605 virtual
void setPostPadding(
Dims const& padding) noexcept = 0;
606 virtual
Dims getPostPadding() const noexcept = 0;
607 virtual
void setPaddingMode(
PaddingMode paddingMode) noexcept = 0;
608 virtual
PaddingMode getPaddingMode() const noexcept = 0;
609 virtual
void setKernelSizeNd(
Dims const& kernelSize) noexcept = 0;
610 virtual
Dims getKernelSizeNd() const noexcept = 0;
611 virtual
void setStrideNd(
Dims const& stride) noexcept = 0;
612 virtual
Dims getStrideNd() const noexcept = 0;
613 virtual
void setPaddingNd(
Dims const& padding) noexcept = 0;
614 virtual
Dims getPaddingNd() const noexcept = 0;
615 virtual
void setDilationNd(
Dims const& dilation) noexcept = 0;
616 virtual
Dims getDilationNd() const noexcept = 0;
619class VElementWiseLayer : public VRoot
626class VGatherLayer : public VRoot
629 virtual void setGatherAxis(int32_t axis)
noexcept = 0;
630 virtual int32_t getGatherAxis() const noexcept = 0;
631 virtual
void setNbElementWiseDims(int32_t k) noexcept = 0;
632 virtual int32_t getNbElementWiseDims() const noexcept = 0;
633 virtual
void setMode(
GatherMode mode) noexcept = 0;
634 virtual
GatherMode getMode() const noexcept = 0;
637class VPluginLayer : public VRoot
640 virtual IPlugin& getPlugin() noexcept = 0;
643class VPluginV2Layer : public VRoot
646 virtual IPluginV2& getPlugin() noexcept = 0;
649class VPluginV3Layer : public VRoot
652 virtual IPluginV3& getPlugin() noexcept = 0;
655class VUnaryLayer : public VRoot
662class VReduceLayer : public VRoot
667 virtual
void setReduceAxes(uint32_t reduceAxes) noexcept = 0;
668 virtual uint32_t getReduceAxes() const noexcept = 0;
669 virtual
void setKeepDimensions(
bool keepDimensions) noexcept = 0;
670 virtual
bool getKeepDimensions() const noexcept = 0;
673class VPaddingLayer : public VRoot
676 virtual void setPrePaddingNd(
Dims const& padding)
noexcept = 0;
677 virtual Dims getPrePaddingNd() const noexcept = 0;
678 virtual
void setPostPaddingNd(
Dims const& padding) noexcept = 0;
679 virtual
Dims getPostPaddingNd() const noexcept = 0;
682class VShuffleLayer : public VRoot
685 virtual void setFirstTranspose(Permutation
const& permutation)
noexcept = 0;
686 virtual Permutation
const& getFirstTranspose() const noexcept = 0;
687 virtual
void setReshapeDimensions(
Dims const& dimensions) noexcept = 0;
688 virtual
Dims getReshapeDimensions() const noexcept = 0;
689 virtual
void setSecondTranspose(Permutation const& permutation) noexcept = 0;
690 virtual Permutation const& getSecondTranspose() const noexcept = 0;
691 virtual
void setZeroIsPlaceholder(
bool zeroIsPlaceholder) noexcept = 0;
692 virtual
bool getZeroIsPlaceholder() const noexcept = 0;
695class VSliceLayer : public VRoot
698 virtual void setStart(
Dims const& start)
noexcept = 0;
699 virtual Dims getStart() const noexcept = 0;
700 virtual
void setSize(
Dims const& size) noexcept = 0;
701 virtual
Dims getSize() const noexcept = 0;
702 virtual
void setStride(
Dims const& stride) noexcept = 0;
703 virtual
Dims getStride() const noexcept = 0;
704 virtual
void setMode(
SampleMode mode) noexcept = 0;
705 virtual
SampleMode getMode() const noexcept = 0;
706 virtual
void setAxes(
Dims const& axes) noexcept = 0;
707 virtual
Dims getAxes() const noexcept = 0;
710class VShapeLayer : public VRoot
715class VTopKLayer :
public VRoot
720 virtual
void setK(int32_t k) noexcept = 0;
721 virtual int32_t getK() const noexcept = 0;
722 virtual
void setReduceAxes(uint32_t reduceAxes) noexcept = 0;
723 virtual uint32_t getReduceAxes() const noexcept = 0;
726class VMatrixMultiplyLayer : public VRoot
729 virtual void setOperation(int32_t index,
MatrixOperation op)
noexcept = 0;
730 virtual MatrixOperation getOperation(int32_t index)
const noexcept = 0;
733class VNonZeroLayer :
public VRoot
738class VRaggedSoftMaxLayer :
public VRoot
743class VIdentityLayer :
public VRoot
748class VCastLayer :
public VRoot
751 virtual void setToType(
DataType toType)
noexcept = 0;
752 virtual DataType getToType() const noexcept = 0;
755class VConstantLayer : public VRoot
758 virtual void setWeights(Weights weights)
noexcept = 0;
759 virtual Weights getWeights() const noexcept = 0;
760 virtual
void setDimensions(
Dims const& dimensions) noexcept = 0;
761 virtual
Dims getDimensions() const noexcept = 0;
764class VParametricReLULayer : public VRoot
769class VResizeLayer :
public VRoot
772 virtual void setOutputDimensions(
Dims const& dimensions)
noexcept = 0;
773 virtual Dims getOutputDimensions() const noexcept = 0;
774 virtual
void setScales(
float const* scales, int32_t nbScales) noexcept = 0;
775 virtual int32_t getScales(int32_t size,
float* scales) const noexcept = 0;
780 virtual
void setSelectorForSinglePixel(
ResizeSelector selector) noexcept = 0;
781 virtual
ResizeSelector getSelectorForSinglePixel() const noexcept = 0;
784 virtual
void setCubicCoeff(
float value) noexcept = 0;
785 virtual
float getCubicCoeff() const noexcept = 0;
786 virtual
void setExcludeOutside(
bool value) noexcept = 0;
787 virtual
bool getExcludeOutside() const noexcept = 0;
790class VLoopBoundaryLayer : public VRoot
793 virtual ILoop* getLoop() const noexcept = 0;
796class VRecurrenceLayer : public VRoot
801class VLoopOutputLayer :
public VRoot
804 virtual LoopOutput getLoopOutput() const noexcept = 0;
805 virtual
void setAxis(int32_t axis) noexcept = 0;
806 virtual int32_t getAxis() const noexcept = 0;
809class VTripLimitLayer : public VRoot
812 virtual TripLimit getTripLimit() const noexcept = 0;
815class VIteratorLayer : public VRoot
818 virtual void setAxis(int32_t axis)
noexcept = 0;
819 virtual int32_t getAxis() const noexcept = 0;
820 virtual
void setReverse(
bool reverse) noexcept = 0;
821 virtual
bool getReverse() const noexcept = 0;
823class VLoop : public VRoot
826 virtual IRecurrenceLayer* addRecurrence(ITensor& initialValue)
noexcept = 0;
827 virtual ITripLimitLayer* addTripLimit(ITensor& tensor,
TripLimit limit)
noexcept = 0;
828 virtual IIteratorLayer* addIterator(ITensor& tensor, int32_t axis = 0,
bool reverse =
false) noexcept = 0;
829 virtual ILoopOutputLayer* addLoopOutput(ITensor& tensor,
LoopOutput outputKind, int32_t axis = 0) noexcept = 0;
830 virtual
void setName(
char const* name) noexcept = 0;
831 virtual
char const* getName() const noexcept = 0;
834class VConditionalBoundaryLayer : public VRoot
837 virtual IIfConditional* getConditional() const noexcept = 0;
840class VConditionLayer : public VRoot
845class VConditionalInputLayer :
public VRoot
850class VConditionalOutputLayer :
public VRoot
855class VIfConditional :
public VRoot
858 virtual IConditionLayer* setCondition(ITensor& tensor)
noexcept = 0;
859 virtual IIfConditionalInputLayer* addInput(ITensor& tensor)
noexcept = 0;
860 virtual IIfConditionalOutputLayer* addOutput(ITensor& trueTensor, ITensor& falseTensor)
noexcept = 0;
861 virtual void setName(
char const* name)
noexcept = 0;
862 virtual char const* getName() const noexcept = 0;
865class VSelectLayer : public VRoot
869class VAssertionLayer :
public VRoot
872 virtual void setMessage(
char const* message)
noexcept = 0;
873 virtual char const* getMessage() const noexcept = 0;
876class VFillLayer : public VRoot
879 virtual void setDimensions(
Dims const& dimensions)
noexcept = 0;
880 virtual Dims getDimensions() const noexcept = 0;
883 virtual
void setAlpha(
double alpha) noexcept = 0;
884 virtual
double getAlpha() const noexcept = 0;
885 virtual
void setBeta(
double beta) noexcept = 0;
886 virtual
double getBeta() const noexcept = 0;
887 virtual
void setAlphaInt64(int64_t alpha) noexcept = 0;
888 virtual int64_t getAlphaInt64() const noexcept = 0;
889 virtual
void setBetaInt64(int64_t beta) noexcept = 0;
890 virtual int64_t getBetaInt64() const noexcept = 0;
891 virtual
bool isAlphaBetaInt64() const noexcept = 0;
892 virtual
DataType getToType() const noexcept = 0;
893 virtual
void setToType(
DataType toType) noexcept = 0;
896class VQuantizeLayer : public VRoot
899 virtual int32_t getAxis() const noexcept = 0;
900 virtual
void setAxis(int32_t axis) noexcept = 0;
901 virtual
DataType getToType() const noexcept = 0;
902 virtual
void setToType(
DataType toType) noexcept = 0;
905class VDequantizeLayer : public VRoot
908 virtual int32_t getAxis() const noexcept = 0;
909 virtual
void setAxis(int32_t axis) noexcept = 0;
910 virtual
DataType getToType() const noexcept = 0;
911 virtual
void setToType(
DataType toType) noexcept = 0;
915class VScatterLayer : public VRoot
918 virtual void setMode(
ScatterMode mode)
noexcept = 0;
920 virtual
void setAxis(int32_t axis) noexcept = 0;
921 virtual int32_t getAxis() const noexcept = 0;
924class VEinsumLayer : public VRoot
927 virtual bool setEquation(
char const* equation)
noexcept = 0;
928 virtual char const* getEquation() const noexcept = 0;
931class VOneHotLayer : public VRoot
934 virtual int32_t getAxis() const noexcept = 0;
935 virtual
void setAxis(int32_t axis) noexcept = 0;
938class VGridSampleLayer : public VRoot
943 virtual
void setAlignCorners(
bool alignCorners) noexcept = 0;
944 virtual
bool getAlignCorners() const noexcept = 0;
945 virtual
bool setSampleMode(
SampleMode mode) noexcept = 0;
946 virtual
SampleMode getSampleMode() const noexcept = 0;
949class VNMSLayer : public VRoot
954 virtual
void setTopKBoxLimit(int32_t limit) noexcept = 0;
955 virtual int32_t getTopKBoxLimit() const noexcept = 0;
958class VReverseSequenceLayer : public VRoot
961 virtual void setBatchAxis(int32_t batchAxis)
noexcept = 0;
962 virtual int32_t getBatchAxis() const noexcept = 0;
964 virtual
void setSequenceAxis(int32_t sequenceAxis) noexcept = 0;
965 virtual int32_t getSequenceAxis() const noexcept = 0;
968class VNormalizationLayer : public VRoot
971 virtual void setEpsilon(
float eps)
noexcept = 0;
972 virtual float getEpsilon() const noexcept = 0;
973 virtual
void setAxes(uint32_t axesMask) noexcept = 0;
974 virtual uint32_t getAxes() const noexcept = 0;
975 virtual
void setNbGroups(int64_t nbGroups) noexcept = 0;
976 virtual int64_t getNbGroups() const noexcept = 0;
977 virtual
void setComputePrecision(
DataType type) noexcept = 0;
978 virtual
DataType getComputePrecision() const noexcept = 0;
981class VNetworkDefinition : public VRoot
984 virtual ITensor* addInput(
char const* name,
DataType type,
Dims const& dimensions)
noexcept = 0;
985 virtual void markOutput(ITensor& tensor)
noexcept = 0;
986 virtual IActivationLayer* addActivation(ITensor& input,
ActivationType type)
noexcept = 0;
987 virtual ILRNLayer* addLRN(ITensor& input, int64_t window,
float alpha,
float beta,
float k)
noexcept = 0;
988 virtual IScaleLayer* addScale(
989 ITensor& input,
ScaleMode mode, Weights shift, Weights scale, Weights power)
noexcept = 0;
990 virtual ISoftMaxLayer* addSoftMax(ITensor& input)
noexcept = 0;
991 virtual IConcatenationLayer* addConcatenation(ITensor*
const* inputs, int32_t nbInputs)
noexcept = 0;
992 virtual IElementWiseLayer* addElementWise(ITensor& input1, ITensor& input2,
ElementWiseOperation op)
noexcept = 0;
993 virtual IUnaryLayer* addUnary(ITensor& input,
UnaryOperation operation)
noexcept = 0;
994 virtual IShuffleLayer* addShuffle(ITensor& input)
noexcept = 0;
995 virtual int32_t getNbLayers() const noexcept = 0;
996 virtual ILayer* getLayer(int32_t index) const noexcept = 0;
997 virtual int32_t getNbInputs() const noexcept = 0;
998 virtual ITensor* getInput(int32_t index) const noexcept = 0;
999 virtual int32_t getNbOutputs() const noexcept = 0;
1000 virtual ITensor* getOutput(int32_t index) const noexcept = 0;
1001 virtual IReduceLayer* addReduce(
1002 ITensor& input,
ReduceOperation operation, uint32_t reduceAxes,
bool keepDimensions) noexcept
1004 virtual ITopKLayer* addTopK(ITensor& input,
TopKOperation op, int32_t k, uint32_t reduceAxes) noexcept = 0;
1005 virtual IGatherLayer* addGather(ITensor& data, ITensor& indices, int32_t axis) noexcept = 0;
1006 virtual IRaggedSoftMaxLayer* addRaggedSoftMax(ITensor& input, ITensor& bounds) noexcept = 0;
1007 virtual IMatrixMultiplyLayer* addMatrixMultiply(
1009 virtual IConstantLayer* addConstant(
Dims const& dimensions, Weights weights) noexcept = 0;
1010 virtual IIdentityLayer* addIdentity(ITensor& input) noexcept = 0;
1011 virtual
void removeTensor(ITensor& tensor) noexcept = 0;
1012 virtual
void unmarkOutput(ITensor& tensor) noexcept = 0;
1013 virtual IPluginV2Layer* addPluginV2(ITensor* const* inputs, int32_t nbInputs, IPluginV2& plugin) noexcept = 0;
1014 virtual IPluginV3Layer* addPluginV3(ITensor* const* inputs, int32_t nbInputs, ITensor* const* shapeInputs,
1015 int32_t nbShapeInputs,
IPluginV3& plugin) noexcept = 0;
1016 virtual ISliceLayer* addSlice(ITensor& input,
Dims const& start,
Dims const& size,
Dims const& stride) noexcept = 0;
1017 virtual
void setName(
char const* name) noexcept = 0;
1018 virtual
char const* getName() const noexcept = 0;
1019 virtual IShapeLayer* addShape(ITensor& input) noexcept = 0;
1020 virtual
bool hasImplicitBatchDimension() const noexcept = 0;
1021 virtual
bool markOutputForShapes(ITensor& tensor) noexcept = 0;
1022 virtual
bool unmarkOutputForShapes(ITensor& tensor) noexcept = 0;
1023 virtual IParametricReLULayer* addParametricReLU(ITensor& input, ITensor& slope) noexcept = 0;
1024 virtual IConvolutionLayer* addConvolutionNd(
1025 ITensor& input, int64_t nbOutputMaps,
Dims const& kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
1027 virtual IPoolingLayer* addPoolingNd(ITensor& input,
PoolingType type,
Dims const& windowSize) noexcept = 0;
1028 virtual IDeconvolutionLayer* addDeconvolutionNd(
1029 ITensor& input, int64_t nbOutputMaps,
Dims const& kernelSize, Weights kernelWeights, Weights biasWeights) noexcept
1031 virtual IScaleLayer* addScaleNd(
1032 ITensor& input,
ScaleMode mode, Weights shift, Weights scale, Weights power, int32_t channelAxis) noexcept = 0;
1033 virtual IResizeLayer* addResize(ITensor& input) noexcept = 0;
1034 virtual ILoop* addLoop() noexcept = 0;
1035 virtual ISelectLayer* addSelect(ITensor& condition, ITensor& thenInput, ITensor& elseInput) noexcept = 0;
1036 virtual IFillLayer* addFill(
Dims const& dimensions,
FillOperation op) noexcept = 0;
1037 virtual IPaddingLayer* addPaddingNd(ITensor& input,
Dims const& prePadding,
Dims const& postPadding) noexcept = 0;
1038 virtual
bool setWeightsName(Weights weights,
char const* name) noexcept = 0;
1039 virtual
void setErrorRecorder(
IErrorRecorder* recorder) noexcept = 0;
1041 virtual IDequantizeLayer* addDequantize(ITensor& input, ITensor& scale) noexcept = 0;
1042 virtual IQuantizeLayer* addQuantize(ITensor& input, ITensor& scale) noexcept = 0;
1043 virtual IGatherLayer* addGatherV2(ITensor& data, ITensor& indices,
GatherMode mode) noexcept = 0;
1044 virtual IIfConditional* addIfConditional() noexcept = 0;
1045 virtual IScatterLayer* addScatter(ITensor& data, ITensor& indices, ITensor& updates,
ScatterMode mode) noexcept = 0;
1046 virtual IEinsumLayer* addEinsum(ITensor* const* inputs, int32_t nbInputs,
char const* equation) noexcept = 0;
1047 virtual IAssertionLayer* addAssertion(ITensor& condition,
char const* message) noexcept = 0;
1048 virtual IOneHotLayer* addOneHot(ITensor& indices, ITensor& values, ITensor& depth, int32_t axis) noexcept = 0;
1049 virtual INonZeroLayer* addNonZero(ITensor& input) noexcept = 0;
1050 virtual IGridSampleLayer* addGridSample(ITensor& input, ITensor& grid) noexcept = 0;
1051 virtual INMSLayer* addNMS(ITensor& boxes, ITensor& scores, ITensor& maxOutputBoxesPerClass) noexcept = 0;
1052 virtual IReverseSequenceLayer* addReverseSequence(ITensor& input, ITensor& sequenceLens) noexcept = 0;
1053 virtual INormalizationLayer* addNormalization(
1054 ITensor& input, ITensor& scale, ITensor& bias, uint32_t axesMask) noexcept = 0;
1055 virtual ICastLayer* addCast(ITensor& input,
DataType toType) noexcept = 0;
1056 virtual IBuilder& getBuilder() const noexcept = 0;
1059 virtual IQuantizeLayer* addQuantizeV2(ITensor& input, ITensor& scale,
DataType outputType) noexcept = 0;
1060 virtual IDequantizeLayer* addDequantizeV2(ITensor& input, ITensor& scale,
DataType outputType) noexcept = 0;
1062 virtual
bool markDebug(ITensor& tensor) noexcept = 0;
1063 virtual
bool unmarkDebug(ITensor& tensor) noexcept = 0;
1064 virtual
bool isDebugTensor(
nvinfer1::ITensor const& tensor) const noexcept = 0;
1065 virtual
bool markWeightsRefittable(
char const* name) noexcept = 0;
1066 virtual
bool unmarkWeightsRefittable(
char const* name) noexcept = 0;
1067 virtual
bool areWeightsMarkedRefittable(
char const* name) const noexcept = 0;
1070class VAlgorithmIOInfo : public VRoot
1073 virtual DataType getDataType() const noexcept = 0;
1074 virtual
Dims getStrides() const noexcept = 0;
1075 virtual int64_t getVectorizedDim() const noexcept = 0;
1076 virtual int64_t getComponentsPerElement() const noexcept = 0;
1079class VAlgorithmVariant : public VRoot
1082 virtual int64_t getImplementation() const noexcept = 0;
1083 virtual int64_t getTactic() const noexcept = 0;
1086class VAlgorithmContext : public VRoot
1089 virtual char const* getName() const noexcept = 0;
1091 virtual int32_t getNbInputs() const noexcept = 0;
1092 virtual int32_t getNbOutputs() const noexcept = 0;
1095class VAlgorithm : public VRoot
1098 virtual IAlgorithmVariant
const& getAlgorithmVariant() const noexcept = 0;
1099 virtual
float getTimingMSec() const noexcept = 0;
1100 virtual std::
size_t getWorkspaceSize() const noexcept = 0;
1101 virtual IAlgorithmIOInfo const* getAlgorithmIOInfoByIndex(int32_t index) const noexcept = 0;
1104class VTimingCache : public VRoot
1108 virtual
bool combine(ITimingCache const& inputCache,
bool ignoreMismatch) noexcept = 0;
1109 virtual
bool reset() noexcept = 0;
1112class VBuilderConfig : public VRoot
1115 virtual void setAvgTimingIterations(int32_t avgTiming)
noexcept = 0;
1116 virtual int32_t getAvgTimingIterations() const noexcept = 0;
1117 virtual
void setEngineCapability(
EngineCapability capability) noexcept = 0;
1119 virtual
void setInt8Calibrator(IInt8Calibrator* calibrator) noexcept = 0;
1120 virtual IInt8Calibrator* getInt8Calibrator() const noexcept = 0;
1121 virtual
void setFlags(
BuilderFlags builderFlags) noexcept = 0;
1123 virtual
void clearFlag(
BuilderFlag builderFlag) noexcept = 0;
1124 virtual
void setFlag(
BuilderFlag builderFlag) noexcept = 0;
1125 virtual
bool getFlag(
BuilderFlag builderFlag) const noexcept = 0;
1126 virtual
void setDeviceType(ILayer const* layer,
DeviceType deviceType) noexcept = 0;
1127 virtual
DeviceType getDeviceType(ILayer const* layer) const noexcept = 0;
1128 virtual
bool isDeviceTypeSet(ILayer const* layer) const noexcept = 0;
1129 virtual
void resetDeviceType(ILayer const* layer) noexcept = 0;
1130 virtual
bool canRunOnDLA(ILayer const* layer) const noexcept = 0;
1131 virtual
void setDLACore(int32_t dlaCore) noexcept = 0;
1132 virtual int32_t getDLACore() const noexcept = 0;
1133 virtual
void setDefaultDeviceType(
DeviceType deviceType) noexcept = 0;
1134 virtual
DeviceType getDefaultDeviceType() const noexcept = 0;
1135 virtual
void reset() noexcept = 0;
1136 virtual
void setProfileStream(const cudaStream_t stream) noexcept = 0;
1137 virtual cudaStream_t getProfileStream() const noexcept = 0;
1138 virtual int32_t addOptimizationProfile(IOptimizationProfile const* profile) noexcept = 0;
1139 virtual int32_t getNbOptimizationProfiles() const noexcept = 0;
1144 virtual
bool setCalibrationProfile(IOptimizationProfile const* profile) noexcept = 0;
1145 virtual IOptimizationProfile const* getCalibrationProfile() noexcept = 0;
1150 virtual
bool getQuantizationFlag(
QuantizationFlag flag) const noexcept = 0;
1151 virtual
bool setTacticSources(
TacticSources tacticSources) noexcept = 0;
1152 virtual
TacticSources getTacticSources() const noexcept = 0;
1153 virtual
nvinfer1::ITimingCache* createTimingCache(
void const* blob, std::
size_t size) const noexcept = 0;
1154 virtual
bool setTimingCache(ITimingCache const& cache,
bool ignoreMismatch) noexcept = 0;
1155 virtual
nvinfer1::ITimingCache const* getTimingCache() const noexcept = 0;
1156 virtual
void setMemoryPoolLimit(
MemoryPoolType pool, std::
size_t poolSize) noexcept = 0;
1157 virtual std::
size_t getMemoryPoolLimit(
MemoryPoolType pool) const noexcept = 0;
1158 virtual
void setPreviewFeature(
PreviewFeature feature,
bool enable) noexcept = 0;
1159 virtual
bool getPreviewFeature(
PreviewFeature feature) const noexcept = 0;
1160 virtual
void setBuilderOptimizationLevel(int32_t level) noexcept = 0;
1161 virtual int32_t getBuilderOptimizationLevel() const noexcept = 0;
1164 virtual
void setPluginsToSerialize(
char const* const* paths, int32_t nbPaths) noexcept = 0;
1165 virtual
char const* getPluginToSerialize(int32_t index) const noexcept = 0;
1166 virtual int32_t getNbPluginsToSerialize() const noexcept = 0;
1167 virtual
void setMaxAuxStreams(int32_t nbStreams) noexcept = 0;
1168 virtual int32_t getMaxAuxStreams() const noexcept = 0;
1171 virtual
void setRuntimePlatform(
RuntimePlatform runtimePlatform) noexcept = 0;
1173 virtual
void setMaxNbTactics(int32_t maxTactics) noexcept = 0;
1174 virtual int32_t getMaxNbTactics() const noexcept = 0;
1177class VSerializationConfig : public VRoot
1187class VBuilder : public VRoot
1190 virtual bool platformHasFastFp16() const noexcept = 0;
1191 virtual
bool platformHasFastInt8() const noexcept = 0;
1192 virtual int32_t getMaxDLABatchSize() const noexcept = 0;
1193 virtual int32_t getNbDLACores() const noexcept = 0;
1194 virtual
void setGpuAllocator(
IGpuAllocator* allocator) noexcept = 0;
1195 virtual
nvinfer1::IBuilderConfig* createBuilderConfig() noexcept = 0;
1197 virtual
nvinfer1::IOptimizationProfile* createOptimizationProfile() noexcept = 0;
1198 virtual
void setErrorRecorder(
IErrorRecorder* recorder) noexcept = 0;
1200 virtual
void reset() noexcept = 0;
1201 virtual
bool platformHasTf32() const noexcept = 0;
1202 virtual
nvinfer1::IHostMemory* buildSerializedNetwork(INetworkDefinition& network, IBuilderConfig& config) noexcept
1204 virtual
bool isNetworkSupported(INetworkDefinition const& network, IBuilderConfig const& config) const noexcept = 0;
1205 virtual ILogger*
getLogger() const noexcept = 0;
1206 virtual
bool setMaxThreads(int32_t maxThreads) noexcept = 0;
1207 virtual int32_t getMaxThreads() const noexcept = 0;
nvinfer1::ILogger * getLogger() noexcept
Return the logger object.
nvinfer1::IPluginRegistry * getPluginRegistry() noexcept
Return the plugin registry.
Structure to define the dimensions of a tensor.
Interface implemented by application for selecting and reporting algorithms of a layer provided by th...
User-implemented callback for notification when value of a debug tensor is updated.
Reference counted application-implemented error reporting interface for TensorRT objects.
Application-implemented class for controlling allocation on the GPU.
Callback from ExecutionContext::enqueueV3()
Plugin class for the V3 generation of user-implemented layers.
Application-implemented interface for profiling.
Application-implemented progress reporting interface for TensorRT.
Application-implemented class for reading data in a stream-based manner.
Class to handle library allocated memory that is accessible to the user.
Definition: NvInferRuntime.h:139
The TensorRT API version 1 namespace.
uint32_t TacticSources
Represents a collection of one or more TacticSource values combine using bitwise-OR operations.
Definition: NvInferRuntime.h:2377
v_1_0::IOutputAllocator IOutputAllocator
Definition: NvInferRuntime.h:3512
ResizeSelector
The coordinate selector when resize to single pixel output.
Definition: NvInfer.h:3997
EngineCapability
List of supported engine capability flows.
Definition: NvInferRuntime.h:73
DimensionOperation
An operation on two IDimensionExpr, which represent integer expressions used in dimension computation...
Definition: NvInferRuntime.h:176
MemoryPoolType
The type for memory pools used by TensorRT.
Definition: NvInfer.h:8528
ScaleMode
Controls how shift, scale and power are applied in a Scale layer.
Definition: NvInfer.h:1810
RuntimePlatform
Describes the intended runtime platform (operating system and CPU architecture) for the execution of ...
Definition: NvInfer.h:8263
uint32_t QuantizationFlags
Represents one or more QuantizationFlag values using binary OR operations.
Definition: NvInfer.h:8215
TensorIOMode
Definition of tensor IO Mode.
Definition: NvInferRuntimeBase.h:1100
HardwareCompatibilityLevel
Describes requirements of compatibility with GPU architectures other than that of the GPU on which th...
Definition: NvInfer.h:8642
SerializationFlag
List of valid flags that the engine can enable when serializing the bytes.
Definition: NvInferRuntime.h:2418
BoundingBoxFormat
Representation of bounding box data used for the Boxes input tensor in INMSLayer.
Definition: NvInfer.h:5920
v_1_0::IPluginV3 IPluginV3
Definition: NvInferRuntime.h:736
UnaryOperation
Enumerates the unary operations that may be performed by a Unary layer.
Definition: NvInfer.h:2755
v_1_0::IProfiler IProfiler
Definition: NvInferRuntime.h:1298
v_1_0::IAlgorithmSelector IAlgorithmSelector
Definition: NvInfer.h:8207
ActivationType
Enumerates the types of activation to perform in an activation layer.
Definition: NvInfer.h:133
uint32_t TempfileControlFlags
Represents a collection of one or more TempfileControlFlag values combined using bitwise-OR operation...
Definition: NvInferRuntime.h:1376
FillOperation
Enumerates the tensor fill operations that may performed by a fill layer.
Definition: NvInfer.h:4972
ResizeRoundMode
The rounding mode for nearest neighbor resize.
Definition: NvInfer.h:4027
PaddingMode
Enumerates the modes of padding to perform in convolution, deconvolution and pooling layer,...
Definition: NvInfer.h:988
TripLimit
Definition: NvInfer.h:4382
uint32_t NetworkDefinitionCreationFlags
Represents one or more NetworkDefinitionCreationFlag flags using binary OR operations....
Definition: NvInfer.h:9628
PreviewFeature
Define preview features.
Definition: NvInfer.h:8603
DataType
The type of weights and tensors.
Definition: NvInferRuntimeBase.h:135
uint32_t BuilderFlags
Represents one or more BuilderFlag values using binary OR operations, e.g., 1U << BuilderFlag::kFP16 ...
Definition: NvInfer.h:8293
DeviceType
The device that this layer/network will execute on.
Definition: NvInferRuntime.h:1330
CalibrationAlgoType
Version of calibration algorithm to use.
Definition: NvInfer.h:7651
LayerType
The type values of layer classes.
Definition: NvInfer.h:58
SampleMode
Controls how ISliceLayer and IGridSample handle out-of-bounds coordinates.
Definition: NvInfer.h:3213
GatherMode
Control form of IGatherLayer.
Definition: NvInfer.h:2504
v_1_0::IDebugListener IDebugListener
Definition: NvInferRuntime.h:3553
WeightsRole
How a layer uses particular Weights.
Definition: NvInferRuntime.h:1308
uint32_t TensorFormats
It is capable of representing one or more TensorFormat by binary OR operations, e....
Definition: NvInfer.h:125
ProfilingVerbosity
List of verbosity levels of layer information exposed in NVTX annotations and in IEngineInspector.
Definition: NvInferRuntime.h:2389
NetworkDefinitionCreationFlag
List of immutable network properties expressed at network creation time. NetworkDefinitionCreationFla...
Definition: NvInfer.h:9639
ElementWiseOperation
Enumerates the binary operations that may be performed by an ElementWise layer.
Definition: NvInfer.h:2414
QuantizationFlag
List of valid flags for quantizing the network to int8.
Definition: NvInfer.h:8227
uint32_t SerializationFlags
Represents one or more SerializationFlag values using binary OR operations, e.g., 1U << Serialization...
Definition: NvInferRuntime.h:2408
InterpolationMode
Enumerates various modes of interpolation.
Definition: NvInfer.h:3915
BuilderFlag
List of valid modes that the builder can enable when creating an engine from a network definition.
Definition: NvInfer.h:8303
TensorFormat
Format of the input/output tensors.
Definition: NvInferRuntimeBase.h:250
ExecutionContextAllocationStrategy
Different memory allocation behaviors for IExecutionContext.
Definition: NvInferRuntime.h:2522
TopKOperation
Enumerates the operations that may be performed by a TopK layer.
Definition: NvInfer.h:3494
ReduceOperation
Enumerates the reduce operations that may be performed by a Reduce layer.
Definition: NvInfer.h:2850
LayerInformationFormat
The format in which the IEngineInspector prints the layer information.
Definition: NvInferRuntime.h:4445
ScatterMode
Control form of IScatterLayer.
Definition: NvInfer.h:5655
MatrixOperation
Enumerates the operations that may be performed on a tensor by IMatrixMultiplyLayer before multiplica...
Definition: NvInfer.h:3620
ResizeCoordinateTransformation
The resize coordinate transformation function.
Definition: NvInfer.h:3943
LoopOutput
Definition: NvInfer.h:4356
PoolingType
The type of pooling to perform in a pooling layer.
Definition: NvInfer.h:1424
v_1_0::IProgressMonitor IProgressMonitor
Definition: NvInfer.h:8759
TensorLocation
The location for tensor data storage, device or host.
Definition: NvInferRuntime.h:201
OptProfileSelector
When setting or querying optimization profile parameters (such as shape tensor inputs or dynamic dime...
Definition: NvInferRuntime.h:2121