NvInferRuntime.h File Reference

#include "NvInferImpl.h"
#include "NvInferPluginBase.h"
#include "NvInferRuntimeCommon.h"

Go to the source code of this file.

Classes
class	nvinfer1::INoCopy
	Forward declaration of IEngineInspector for use by other interfaces. More...
struct	nvinfer1::impl::EnumMaxImpl< EngineCapability >
	Maximum number of elements in EngineCapability enum. More...
class	nvinfer1::Weights
	An array of weights used as a layer parameter. More...
class	nvinfer1::IHostMemory
	Class to handle library allocated memory that is accessible to the user. More...
struct	nvinfer1::impl::EnumMaxImpl< TensorLocation >
	Maximum number of elements in TensorLocation enum. More...
class	nvinfer1::IDimensionExpr
	An IDimensionExpr represents an integer expression constructed from constants, input dimensions, and binary operations. These expressions are can be used in overrides of IPluginV2DynamicExt::getOutputDimensions or IPluginV3OneBuild::getOutputShapes() to define output dimensions in terms of input dimensions. More...
class	nvinfer1::IExprBuilder
	Object for constructing IDimensionExpr. More...
class	nvinfer1::DimsExprs
	Analog of class Dims with expressions instead of constants for the dimensions. More...
struct	nvinfer1::DynamicPluginTensorDesc
	Summarizes tensors that a plugin might see for an input or output. More...
class	nvinfer1::IPluginV2DynamicExt
	Similar to IPluginV2Ext, but with support for dynamic shapes. More...
class	nvinfer1::v_1_0::IStreamReader
class	nvinfer1::v_1_0::IStreamWriter
class	nvinfer1::v_1_0::IStreamReaderV2
class	nvinfer1::IPluginResourceContext
	Interface for plugins to access per context resources provided by TensorRT. More...
class	nvinfer1::v_1_0::IPluginV3OneCore
class	nvinfer1::v_1_0::IPluginV3OneBuild
class	nvinfer1::v_1_0::IPluginV3OneRuntime
class	nvinfer1::v_2_0::IPluginV3OneBuild
class	nvinfer1::v_1_0::IProfiler
struct	nvinfer1::impl::EnumMaxImpl< TensorFormat >
	Maximum number of elements in TensorFormat enum. More...
struct	nvinfer1::impl::EnumMaxImpl< AllocatorFlag >
	Maximum number of elements in AllocatorFlag enum. More...
class	nvinfer1::ILogger
	Application-implemented logging interface for the builder, refitter and runtime. More...
struct	nvinfer1::impl::EnumMaxImpl< ILogger::Severity >
	Maximum number of elements in ILogger::Severity enum. More...
class	nvinfer1::v_1_0::IGpuAllocator
struct	nvinfer1::impl::EnumMaxImpl< EngineValidity >
class	nvinfer1::IRuntime
	Allows a serialized functionally unsafe engine to be deserialized. More...
class	nvinfer1::IRefitter
	Updates weights in an engine. More...
class	nvinfer1::IOptimizationProfile
	Optimization profile for dynamic input dimensions and shape tensors. More...
class	nvinfer1::ISerializationConfig
	Holds properties for configuring an engine to serialize the binary. More...
class	nvinfer1::IRuntimeCache
	A class for Runtime cache currently used for TensorRT JIT compilation. This cache can be serialized and deserialized to support persistence in disk files. More...
class	nvinfer1::IRuntimeConfig
	A class for runtime configuration. This class is used during execution context creation. More...
class	nvinfer1::ICudaEngine
	An engine for executing inference on a built network, with functionally unsafe features. More...
class	nvinfer1::v_1_0::IOutputAllocator
class	nvinfer1::v_1_0::IDebugListener
class	nvinfer1::IExecutionContext
	Context for executing inference using an engine, with functionally unsafe features. More...
class	nvinfer1::IEngineInspector
	An engine inspector which prints out the layer information of an engine or an execution context. More...
class	nvinfer1::PluginRegistrar< T >
	Register the plugin creator to the registry The static registry object will be instantiated when the plugin library is loaded. This static object will register all creators available in the library to the registry. More...
class	nvinfer1::ILoggerFinder
	A virtual base class to find a logger. Allows a plugin to find an instance of a logger if it needs to emit a log message. A pointer to an instance of this class is passed to a plugin shared library on initialization when that plugin is serialized as part of a version-compatible plan. See the plugin chapter in the developer guide for details. More...
class	nvinfer1::v_1_0::IGpuAsyncAllocator
class	nvinfer1::v_1_0::IPluginCreatorV3One

Namespaces
namespace	nvinfer1
	The TensorRT-RTX API version 1 namespace.
namespace	nvinfer1::impl
	Maximum number of elements in EngineValidity enum.
namespace	nvinfer1::v_1_0
	Forward declare IErrorRecorder for use in other interfaces.
namespace	nvinfer1::v_2_0
namespace	nvinfer1::anonymous_namespace{NvInferRuntime.h}

Macros
#define	NV_INFER_INTERNAL_INCLUDE   1
#define	REGISTER_TENSORRT_PLUGIN(name)    static nvinfer1::PluginRegistrar<name> pluginRegistrar##name {}

Typedefs
using	nvinfer1::IStreamReader = v_1_0::IStreamReader
using	nvinfer1::IStreamWriter = v_1_0::IStreamWriter
using	nvinfer1::IStreamReaderV2 = v_1_0::IStreamReaderV2
using	nvinfer1::IPluginV3OneCore = v_1_0::IPluginV3OneCore
using	nvinfer1::IPluginV3OneBuild = v_1_0::IPluginV3OneBuild
using	nvinfer1::IPluginV3OneRuntime = v_1_0::IPluginV3OneRuntime
using	nvinfer1::IPluginV3OneBuildV2 = v_2_0::IPluginV3OneBuild
using	nvinfer1::IProfiler = v_1_0::IProfiler
using	nvinfer1::TempfileControlFlags = uint32_t
	Represents a collection of one or more TempfileControlFlag values combined using bitwise-OR operations. More...
using	nvinfer1::AllocatorFlags = uint32_t
using	nvinfer1::IGpuAllocator = v_1_0::IGpuAllocator
using	nvinfer1::TacticSources = uint32_t
	Represents a collection of one or more TacticSource values combine using bitwise-OR operations. More...
using	nvinfer1::SerializationFlags = uint32_t
	Represents one or more SerializationFlag values using binary OR operations, e.g., 1U << SerializationFlag::kEXCLUDE_LEAN_RUNTIME. More...
using	nvinfer1::IOutputAllocator = v_1_0::IOutputAllocator
using	nvinfer1::IDebugListener = v_1_0::IDebugListener
using	nvinfer1::IGpuAsyncAllocator = v_1_0::IGpuAsyncAllocator
using	nvinfer1::IPluginCreatorV3One = v_1_0::IPluginCreatorV3One

Enumerations
enum class	nvinfer1::EngineCapability : int32_t { nvinfer1::kSTANDARD = 0 , nvinfer1::kSAFETY = 1 , nvinfer1::kDLA_STANDALONE = 2 }
	List of supported engine capability flows. More...
enum class	nvinfer1::DimensionOperation : int32_t {   nvinfer1::kSUM = 0 , nvinfer1::kPROD = 1 , nvinfer1::kMAX = 2 , nvinfer1::kMIN = 3 ,   nvinfer1::kSUB = 4 , nvinfer1::kEQUAL = 5 , nvinfer1::kLESS = 6 , nvinfer1::kFLOOR_DIV = 7 ,   nvinfer1::kCEIL_DIV = 8 }
	An operation on two IDimensionExpr, which represent integer expressions used in dimension computations. More...
enum class	nvinfer1::TensorLocation : int32_t { nvinfer1::kDEVICE = 0 , nvinfer1::kHOST = 1 }
	The location for tensor data storage, device or host. More...
enum class	nvinfer1::SeekPosition : int32_t { nvinfer1::kSET = 0 , nvinfer1::kCUR = 1 , nvinfer1::kEND = 2 }
	Controls the seek mode of IStreamReaderV2. More...
enum class	nvinfer1::WeightsRole : int32_t {   nvinfer1::kKERNEL = 0 , nvinfer1::kBIAS = 1 , nvinfer1::kSHIFT = 2 , nvinfer1::kSCALE = 3 ,   nvinfer1::kCONSTANT = 4 , nvinfer1::kANY = 5 }
	How a layer uses particular Weights. More...
enum class	nvinfer1::DeviceType : int32_t { nvinfer1::kGPU = 0 , nvinfer1::kDLA = 1 }
	The device that this layer/network will execute on. More...
enum class	nvinfer1::TempfileControlFlag : int32_t { nvinfer1::kALLOW_IN_MEMORY_FILES = 0 , nvinfer1::kALLOW_TEMPORARY_FILES = 1 }
	Flags used to control TensorRT's behavior when creating executable temporary files. More...
enum class	nvinfer1::TensorFormat : int32_t {   nvinfer1::kLINEAR = 0 , nvinfer1::kCHW2 = 1 , nvinfer1::kHWC8 = 2 , nvinfer1::kCHW4 = 3 ,   nvinfer1::kCHW16 = 4 , nvinfer1::kCHW32 = 5 , nvinfer1::kDHWC8 = 6 , nvinfer1::kCDHW32 = 7 ,   nvinfer1::kHWC = 8 , nvinfer1::kDLA_LINEAR = 9 , nvinfer1::kDLA_HWC4 = 10 , nvinfer1::kHWC16 = 11 ,   nvinfer1::kDHWC = 12 }
	Format of the input/output tensors. More...
enum class	nvinfer1::AllocatorFlag : int32_t { nvinfer1::kRESIZABLE = 0 }
	Allowed type of memory allocation. More...
enum class	nvinfer1::EngineValidity : int32_t { nvinfer1::kVALID = 0 , nvinfer1::kSUBOPTIMAL = 1 , nvinfer1::kINVALID = 2 }
	Whether a TensorRT-RTX engine is likely to be valid on the current system. More...
enum class	nvinfer1::EngineInvalidityDiagnostics : uint64_t {   nvinfer1::kVERSION_MISMATCH = 1ULL << 0 , nvinfer1::kUNSUPPORTED_CC = 1ULL << 1 , nvinfer1::kOLD_CUDA_DRIVER = 1ULL << 2 , nvinfer1::kOLD_CUDA_RUNTIME = 1ULL << 3 ,   nvinfer1::kINSUFFICIENT_GPU_MEMORY = 1ULL << 4 , nvinfer1::kMALFORMED_ENGINE = 1ULL << 5 , nvinfer1::kCUDA_ERROR = 1ULL << 6 }
	Bitmask indicating the reason(s) why an engine is invalid. More...
enum class	nvinfer1::OptProfileSelector : int32_t { nvinfer1::kMIN = 0 , nvinfer1::kOPT = 1 , nvinfer1::kMAX = 2 }
	When setting or querying optimization profile parameters (such as shape tensor inputs or dynamic dimensions), select whether we are interested in the minimum, optimum, or maximum values for these parameters. The minimum and maximum specify the permitted range that is supported at runtime, while the optimum value is used for the kernel selection. This should be the "typical" value that is expected to occur at runtime. More...
enum class	nvinfer1::TacticSource : int32_t {   nvinfer1::kCUBLAS = 0 , nvinfer1::kCUBLAS_LT = 1 , nvinfer1::kCUDNN = 2 , nvinfer1::kEDGE_MASK_CONVOLUTIONS = 3 ,   nvinfer1::kJIT_CONVOLUTIONS = 4 }
	List of tactic sources for TensorRT. More...
enum class	nvinfer1::ProfilingVerbosity : int32_t { nvinfer1::kLAYER_NAMES_ONLY = 0 , nvinfer1::kNONE = 1 , nvinfer1::kDETAILED = 2 }
	List of verbosity levels of layer information exposed in NVTX annotations and in IEngineInspector. More...
enum class	nvinfer1::SerializationFlag : int32_t { nvinfer1::kEXCLUDE_WEIGHTS = 0 , nvinfer1::kEXCLUDE_LEAN_RUNTIME = 1 , nvinfer1::kINCLUDE_REFIT = 2 }
	List of valid flags that the engine can enable when serializing the bytes. More...
enum class	nvinfer1::ExecutionContextAllocationStrategy : int32_t { nvinfer1::kSTATIC = 0 , nvinfer1::kON_PROFILE_CHANGE = 1 , nvinfer1::kUSER_MANAGED = 2 }
	Different memory allocation behaviors for IExecutionContext. More...
enum class	nvinfer1::DynamicShapesKernelSpecializationStrategy : int32_t { nvinfer1::kLAZY = 0 , nvinfer1::kEAGER = 1 , nvinfer1::kNONE = 2 }
	Different kernel specialization strategies for dynamic shapes. More...
enum class	nvinfer1::CudaGraphStrategy : int32_t { nvinfer1::kDISABLED = 0 , nvinfer1::kWHOLE_GRAPH_CAPTURE = 1 }
	Strategies available for CUDA graphs optimizations for JIT (Just-In-Time) inference. More...
enum class	nvinfer1::EngineStat : int32_t { nvinfer1::kTOTAL_WEIGHTS_SIZE = 0 , nvinfer1::kSTRIPPED_WEIGHTS_SIZE = 1 }
	The kind of engine statistics that queried from the ICudaEngine. More...
enum class	nvinfer1::LayerInformationFormat : int32_t { nvinfer1::kONELINE = 0 , nvinfer1::kJSON = 1 }
	The format in which the IEngineInspector prints the layer information. More...

Functions
template<>
constexpr int32_t	nvinfer1::EnumMax< DimensionOperation > () noexcept
	Maximum number of elements in DimensionOperation enum. More...
template<>
constexpr int32_t	nvinfer1::EnumMax< WeightsRole > () noexcept
	Maximum number of elements in WeightsRole enum. More...
template<>
constexpr int32_t	nvinfer1::EnumMax< DeviceType > () noexcept
	Maximum number of elements in DeviceType enum. More...
template<>
constexpr int32_t	nvinfer1::EnumMax< TempfileControlFlag > () noexcept
	Maximum number of elements in TempfileControlFlag enum. More...
template<>
constexpr int32_t	nvinfer1::EnumMax< OptProfileSelector > () noexcept
	Number of different values of OptProfileSelector enum. More...
template<>
constexpr int32_t	nvinfer1::EnumMax< TacticSource > () noexcept
	Maximum number of tactic sources in TacticSource enum. More...
template<>
constexpr int32_t	nvinfer1::EnumMax< ProfilingVerbosity > () noexcept
	Maximum number of profile verbosity levels in ProfilingVerbosity enum. More...
template<>
constexpr int32_t	nvinfer1::EnumMax< SerializationFlag > () noexcept
	Maximum number of serialization flags in SerializationFlag enum. More...
template<>
constexpr int32_t	nvinfer1::EnumMax< ExecutionContextAllocationStrategy > () noexcept
	Maximum number of memory allocation strategies in ExecutionContextAllocationStrategy enum. More...
template<>
constexpr int32_t	nvinfer1::EnumMax< DynamicShapesKernelSpecializationStrategy > () noexcept
	Maximum number of dynamic shape specialization strategies in DynamicShapesKernelSpecializationStrategy enum. More...
template<>
constexpr int32_t	nvinfer1::EnumMax< CudaGraphStrategy > () noexcept
	Maximum number of CUDA graph strategies in CudaGraphStrategy enum. More...
template<>
constexpr int32_t	nvinfer1::EnumMax< EngineStat > () noexcept
	Maximum number of engine statistic kinds in EngineStat enum. More...
template<>
constexpr int32_t	nvinfer1::EnumMax< LayerInformationFormat > () noexcept
TENSORRTAPI nvinfer1::IPluginRegistry *	getPluginRegistry () noexcept
	Return the plugin registry. More...
TENSORRTAPI nvinfer1::ILogger *	getLogger () noexcept
	Return the logger object. More...
IRuntime *	nvinfer1::anonymous_namespace{NvInferRuntime.h}::createInferRuntime (ILogger &logger) noexcept
	Create an instance of an IRuntime class. More...
IRefitter *	nvinfer1::anonymous_namespace{NvInferRuntime.h}::createInferRefitter (ICudaEngine &engine, ILogger &logger) noexcept
	Create an instance of an IRefitter class. More...
TENSORRTAPI int32_t	getInferLibMajorVersion () noexcept
	Return the library major version number. More...
TENSORRTAPI int32_t	getInferLibMinorVersion () noexcept
	Return the library minor version number. More...
TENSORRTAPI int32_t	getInferLibPatchVersion () noexcept
	Return the library patch version number. More...
TENSORRTAPI int32_t	getInferLibBuildVersion () noexcept
	Return the library build version number. More...

Detailed Description

This is the top-level API file for TensorRT extended runtime library.

Macro Definition Documentation

NV_INFER_INTERNAL_INCLUDE

#define NV_INFER_INTERNAL_INCLUDE   1

REGISTER_TENSORRT_PLUGIN

#define REGISTER_TENSORRT_PLUGIN

(

name

)

static nvinfer1::PluginRegistrar<name> pluginRegistrar##name {}

Function Documentation

getInferLibBuildVersion()

TENSORRTAPI int32_t getInferLibBuildVersion ( )

noexcept

Return the library build version number.

getInferLibMajorVersion()

TENSORRTAPI int32_t getInferLibMajorVersion ( )

noexcept

Return the library major version number.

getInferLibMinorVersion()

TENSORRTAPI int32_t getInferLibMinorVersion ( )

noexcept

Return the library minor version number.

getInferLibPatchVersion()

TENSORRTAPI int32_t getInferLibPatchVersion ( )

noexcept

Return the library patch version number.

getLogger()

TENSORRTAPI nvinfer1::ILogger * getLogger ( )

noexcept

Return the logger object.

Note

the global logger is used only by standalone functions which have no associated builder, runtime or refitter.

getPluginRegistry()

TENSORRTAPI nvinfer1::IPluginRegistry * getPluginRegistry ( )

noexcept

Return the plugin registry.