An engine for executing inference on a built network, with functionally unsafe features. More...

#include <NvInferRuntime.h>

Inheritance diagram for nvinfer1::ICudaEngine:

Public Member Functions
virtual	~ICudaEngine () noexcept=default

TRT_DEPRECATED int32_t	getNbBindings () const noexcept
	Get the number of binding indices. More...

TRT_DEPRECATED int32_t	getBindingIndex (char const *name) const noexcept
	Retrieve the binding index for a named tensor. More...

TRT_DEPRECATED char const *	getBindingName (int32_t bindingIndex) const noexcept
	Retrieve the name corresponding to a binding index. More...

TRT_DEPRECATED bool	bindingIsInput (int32_t bindingIndex) const noexcept
	Determine whether a binding is an input binding. More...

TRT_DEPRECATED Dims	getBindingDimensions (int32_t bindingIndex) const noexcept
	Get the dimensions of a binding. More...

Dims	getTensorShape (char const *tensorName) const noexcept
	Get shape of an input or output tensor. More...

TRT_DEPRECATED DataType	getBindingDataType (int32_t bindingIndex) const noexcept
	Determine the required data type for a buffer from its binding index. More...

DataType	getTensorDataType (char const *tensorName) const noexcept
	Determine the required data type for a buffer from its tensor name. More...

TRT_DEPRECATED int32_t	getMaxBatchSize () const noexcept
	Get the maximum batch size which can be used for inference. Should only be called if the engine is built from an INetworkDefinition with implicit batch dimension mode. More...

int32_t	getNbLayers () const noexcept
	Get the number of layers in the network. More...

IHostMemory *	serialize () const noexcept
	Serialize the network to a stream. More...

IExecutionContext *	createExecutionContext () noexcept
	Create an execution context. More...

TRT_DEPRECATED void	destroy () noexcept
	Destroy this object;. More...

TRT_DEPRECATED TensorLocation	getLocation (int32_t bindingIndex) const noexcept
	Get location of binding. More...

TensorLocation	getTensorLocation (char const *tensorName) const noexcept
	Get whether an input or output tensor must be on GPU or CPU. More...

bool	isShapeInferenceIO (char const *tensorName) const noexcept
	True if tensor is required as input for shape calculations or is output from shape calculations. More...

TensorIOMode	getTensorIOMode (char const *tensorName) const noexcept
	Determine whether a tensor is an input or output tensor. More...

IExecutionContext *	createExecutionContextWithoutDeviceMemory () noexcept
	create an execution context without any device memory allocated More...

size_t	getDeviceMemorySize () const noexcept
	Return the amount of device memory required by an execution context. More...

bool	isRefittable () const noexcept
	Return true if an engine can be refit. More...

TRT_DEPRECATED int32_t	getBindingBytesPerComponent (int32_t bindingIndex) const noexcept
	Return the number of bytes per component of an element. More...

int32_t	getTensorBytesPerComponent (char const *tensorName) const noexcept
	Return the number of bytes per component of an element, or -1 if the provided name does not map to an input or output tensor. More...

int32_t	getTensorBytesPerComponent (char const *tensorName, int32_t profileIndex) const noexcept
	Return the number of bytes per component of an element of given profile, or -1 if the provided name does not map to an input or output tensor. More...

TRT_DEPRECATED int32_t	getBindingComponentsPerElement (int32_t bindingIndex) const noexcept
	Return the number of components included in one element. More...

int32_t	getTensorComponentsPerElement (char const *tensorName) const noexcept
	Return the number of components included in one element, or -1 if the provided name does not map to an input or output tensor. More...

int32_t	getTensorComponentsPerElement (char const *tensorName, int32_t profileIndex) const noexcept
	Return the number of components included in one element of given profile, or -1 if the provided name does not map to an input or output tensor. More...

TRT_DEPRECATED TensorFormat	getBindingFormat (int32_t bindingIndex) const noexcept
	Return the binding format. More...

TensorFormat	getTensorFormat (char const *tensorName) const noexcept
	Return the tensor format, or TensorFormat::kLINEAR if the provided name does not map to an input or output tensor. More...

TensorFormat	getTensorFormat (char const *tensorName, int32_t profileIndex) const noexcept
	Return the tensor format of given profile, or TensorFormat::kLINEAR if the provided name does not map to an input or output tensor. More...

TRT_DEPRECATED char const *	getBindingFormatDesc (int32_t bindingIndex) const noexcept
	Return the human readable description of the tensor format, or nullptr if the provided name does not map to an input or output tensor. More...

char const *	getTensorFormatDesc (char const *tensorName) const noexcept
	Return the human readable description of the tensor format, or empty string if the provided name does not map to an input or output tensor. More...

char const *	getTensorFormatDesc (char const *tensorName, int32_t profileIndex) const noexcept
	Return the human readable description of the tensor format of given profile, or empty string if the provided name does not map to an input or output tensor. More...

TRT_DEPRECATED int32_t	getBindingVectorizedDim (int32_t bindingIndex) const noexcept
	Return the dimension index that the buffer is vectorized, or -1 is the name is not found. More...

int32_t	getTensorVectorizedDim (char const *tensorName) const noexcept
	Return the dimension index that the buffer is vectorized, or -1 if the provided name does not map to an input or output tensor. More...

int32_t	getTensorVectorizedDim (char const *tensorName, int32_t profileIndex) const noexcept
	Return the dimension index that the buffer is vectorized of given profile, or -1 if the provided name does not map to an input or output tensor. More...

char const *	getName () const noexcept
	Returns the name of the network associated with the engine. More...

int32_t	getNbOptimizationProfiles () const noexcept
	Get the number of optimization profiles defined for this engine. More...

TRT_DEPRECATED Dims	getProfileDimensions (int32_t bindingIndex, int32_t profileIndex, OptProfileSelector select) const noexcept
	Get the minimum / optimum / maximum dimensions for a particular input binding under an optimization profile. More...

Dims	getProfileShape (char const *tensorName, int32_t profileIndex, OptProfileSelector select) const noexcept
	Get the minimum / optimum / maximum dimensions for an input tensor given its name under an optimization profile. More...

TRT_DEPRECATED int32_t const *	getProfileShapeValues (int32_t profileIndex, int32_t inputIndex, OptProfileSelector select) const noexcept
	Get minimum / optimum / maximum values for an input shape binding under an optimization profile. More...

TRT_DEPRECATED bool	isShapeBinding (int32_t bindingIndex) const noexcept
	True if tensor is required as input for shape calculations or output from them. More...

TRT_DEPRECATED bool	isExecutionBinding (int32_t bindingIndex) const noexcept
	True if pointer to tensor data is required for execution phase, false if nullptr can be supplied. More...

EngineCapability	getEngineCapability () const noexcept
	Determine what execution capability this engine has. More...

void	setErrorRecorder (IErrorRecorder *recorder) noexcept
	Set the ErrorRecorder for this interface. More...

IErrorRecorder *	getErrorRecorder () const noexcept
	Get the ErrorRecorder assigned to this interface. More...

bool	hasImplicitBatchDimension () const noexcept
	Query whether the engine was built with an implicit batch dimension. More...

TacticSources	getTacticSources () const noexcept
	return the tactic sources required by this engine. More...

ProfilingVerbosity	getProfilingVerbosity () const noexcept
	Return the ProfilingVerbosity the builder config was set to when the engine was built. More...

IEngineInspector *	createEngineInspector () const noexcept
	Create a new engine inspector which prints the layer information in an engine or an execution context. More...

int32_t	getNbIOTensors () const noexcept
	Return number of IO tensors. More...

char const *	getIOTensorName (int32_t index) const noexcept
	Return name of an IO tensor. More...

HardwareCompatibilityLevel	getHardwareCompatibilityLevel () const noexcept
	Return the hardware compatibility level of this engine. More...

int32_t	getNbAuxStreams () const noexcept
	Return the number of auxiliary streams used by this engine. More...

Protected Attributes
apiv::VCudaEngine *	mImpl

Additional Inherited Members
Protected Member Functions inherited from nvinfer1::INoCopy
	INoCopy ()=default

virtual	~INoCopy ()=default

	INoCopy (INoCopy const &other)=delete

INoCopy &	operator= (INoCopy const &other)=delete

	INoCopy (INoCopy &&other)=delete

INoCopy &	operator= (INoCopy &&other)=delete

Detailed Description

An engine for executing inference on a built network, with functionally unsafe features.

Warning: Do not inherit from this class, as doing so will break forward-compatibility of the API and ABI.

Constructor & Destructor Documentation

◆ ~ICudaEngine()

virtual nvinfer1::ICudaEngine::~ICudaEngine ( )

virtualdefaultnoexcept

Member Function Documentation

◆ bindingIsInput()

TRT_DEPRECATED bool nvinfer1::ICudaEngine::bindingIsInput ( int32_t bindingIndex ) const

inlinenoexcept

Determine whether a binding is an input binding.

Parameters

bindingIndex The binding index.

Returns: True if the index corresponds to an input binding and the index is in range.

Deprecated:: Deprecated in TensorRT 8.5. Superseded by getTensorIOMode().

See also: getTensorIOMode()

◆ createEngineInspector()

IEngineInspector * nvinfer1::ICudaEngine::createEngineInspector ( ) const

inlinenoexcept

Create a new engine inspector which prints the layer information in an engine or an execution context.

See also: IEngineInspector.

◆ createExecutionContext()

IExecutionContext * nvinfer1::ICudaEngine::createExecutionContext ( )

inlinenoexcept

Create an execution context.

The execution context created will call setOptimizationProfile(0) implicitly if there are no other execution contexts assigned to optimization profile 0. This functionality is deprecated in TensorRT 8.6 and will instead default all optimization profiles to 0 starting in TensorRT 9.0. If an error recorder has been set for the engine, it will also be passed to the execution context.

See also: IExecutionContext.; IExecutionContext::setOptimizationProfile()

◆ createExecutionContextWithoutDeviceMemory()

IExecutionContext * nvinfer1::ICudaEngine::createExecutionContextWithoutDeviceMemory ( )

inlinenoexcept

create an execution context without any device memory allocated

The memory for execution of this device context must be supplied by the application.

◆ destroy()

TRT_DEPRECATED void nvinfer1::ICudaEngine::destroy ( )

inlinenoexcept

Destroy this object;.

Deprecated:: Deprecated in TRT 8.0. Superseded by delete.

Warning: Calling destroy on a managed pointer will result in a double-free error.

◆ getBindingBytesPerComponent()

TRT_DEPRECATED int32_t nvinfer1::ICudaEngine::getBindingBytesPerComponent ( int32_t bindingIndex ) const

inlinenoexcept

Return the number of bytes per component of an element.

The vector component size is returned if getBindingVectorizedDim() != -1.

Parameters

bindingIndex The binding Index.

Deprecated:: Deprecated in TensorRT 8.5. Superseded by getTensorBytesPerComponent().

See also: getBindingVectorizedDim(); getTensorBytesPerComponent()

◆ getBindingComponentsPerElement()

TRT_DEPRECATED int32_t nvinfer1::ICudaEngine::getBindingComponentsPerElement ( int32_t bindingIndex ) const

inlinenoexcept

Return the number of components included in one element.

The number of elements in the vectors is returned if getBindingVectorizedDim() != -1.

Parameters

bindingIndex The binding Index.

Deprecated:: Deprecated in TensorRT 8.5. Superseded by getTensorComponentsPerElement().

See also: getBindingVectorizedDim()

◆ getBindingDataType()

TRT_DEPRECATED DataType nvinfer1::ICudaEngine::getBindingDataType ( int32_t bindingIndex ) const

inlinenoexcept

Determine the required data type for a buffer from its binding index.

Parameters

bindingIndex The binding index.

Returns: The type of the data in the buffer.

Deprecated:: Deprecated in TensorRT 8.5. Superseded by getTensorDataType().

See also: getTensorDataType()

◆ getBindingDimensions()

TRT_DEPRECATED Dims nvinfer1::ICudaEngine::getBindingDimensions ( int32_t bindingIndex ) const

inlinenoexcept

Get the dimensions of a binding.

Parameters

bindingIndex The binding index.

Returns: The dimensions of the binding if the index is in range, otherwise Dims(). Has -1 for any dimension that varies within the optimization profile.

For example, suppose an INetworkDefinition has an input with shape [-1,-1] that becomes a binding b in the engine. If the associated optimization profile specifies that b has minimum dimensions as [6,9] and maximum dimensions [7,9], getBindingDimensions(b) returns [-1,9], despite the second dimension being dynamic in the INetworkDefinition.

Because each optimization profile has separate bindings, the returned value can differ across profiles. Consider another binding b' for the same network input, but for another optimization profile. If that other profile specifies minimum dimensions [5,8] and maximum dimensions [5,9], getBindingDimensions(b') returns [5,-1].

Deprecated:: Deprecated in TensorRT 8.5. Superseded by getTensorShape().

See also: getTensorShape()

◆ getBindingFormat()

TRT_DEPRECATED TensorFormat nvinfer1::ICudaEngine::getBindingFormat ( int32_t bindingIndex ) const

inlinenoexcept

Return the binding format.

Parameters

bindingIndex The binding Index.

Deprecated:: Deprecated in TensorRT 8.5. Superseded by getTensorFormat().

See also: getTensorFormat()

◆ getBindingFormatDesc()

TRT_DEPRECATED char const * nvinfer1::ICudaEngine::getBindingFormatDesc ( int32_t bindingIndex ) const

inlinenoexcept

Return the human readable description of the tensor format, or nullptr if the provided name does not map to an input or output tensor.

The description includes the order, vectorization, data type, and strides. Examples are shown as follows: Example 1: kCHW + FP32 "Row major linear FP32 format" Example 2: kCHW2 + FP16 "Two wide channel vectorized row major FP16 format" Example 3: kHWC8 + FP16 + Line Stride = 32 "Channel major FP16 format where C % 8 == 0 and H Stride % 32 == 0"

Parameters

bindingIndex The binding Index.

Deprecated:: Deprecated in TensorRT 8.5. Superseded by getTensorFormatDesc().

See also: getTensorFormatDesc()

◆ getBindingIndex()

TRT_DEPRECATED int32_t nvinfer1::ICudaEngine::getBindingIndex ( char const * name ) const

inlinenoexcept

Retrieve the binding index for a named tensor.

IExecutionContext::enqueueV2() and IExecutionContext::executeV2() require an array of buffers.

Engine bindings map from tensor names to indices in this array. Binding indices are assigned at engine build time, and take values in the range [0 ... n-1] where n is the total number of inputs and outputs.

To get the binding index of the name in an optimization profile with index k > 0, mangle the name by appending " [profile k]", as described for method getBindingName().

Parameters

name	The tensor name.

Returns: The binding index for the named tensor, or -1 if the provided name does not map to an input or output tensor.

Warning: The string name must be null-terminated, and be at most 4096 bytes including the terminator.

Deprecated:: Deprecated in TensorRT 8.5. Superseded by name-based methods. Use them instead of binding-index based methods.

See also: getNbBindings() getBindingName()

◆ getBindingName()

TRT_DEPRECATED char const * nvinfer1::ICudaEngine::getBindingName ( int32_t bindingIndex ) const

inlinenoexcept

Retrieve the name corresponding to a binding index.

This is the reverse mapping to that provided by getBindingIndex().

For optimization profiles with an index k > 0, the name is mangled by appending " [profile k]", with k written in decimal. For example, if the tensor in the INetworkDefinition had the name "foo", and bindingIndex refers to that tensor in the optimization profile with index 3, getBindingName returns "foo [profile 3]".

Parameters

bindingIndex The binding index.

Returns: The name corresponding to the index, or nullptr if the index is out of range.

Deprecated:: Deprecated in TensorRT 8.5. Superseded by name-based methods. Use them instead of binding-index based methods.

See also: getBindingIndex()

◆ getBindingVectorizedDim()

TRT_DEPRECATED int32_t nvinfer1::ICudaEngine::getBindingVectorizedDim ( int32_t bindingIndex ) const

inlinenoexcept

Return the dimension index that the buffer is vectorized, or -1 is the name is not found.

Specifically -1 is returned if scalars per vector is 1.

Parameters

bindingIndex The binding Index.

Deprecated:: Deprecated in TensorRT 8.5. Superseded by getTensorVectorizedDim().

See also: getTensorVectorizedDim()

◆ getDeviceMemorySize()

size_t nvinfer1::ICudaEngine::getDeviceMemorySize ( ) const

inlinenoexcept

Return the amount of device memory required by an execution context.

See also: IExecutionContext::setDeviceMemory()

◆ getEngineCapability()

EngineCapability nvinfer1::ICudaEngine::getEngineCapability ( ) const

inlinenoexcept

Determine what execution capability this engine has.

If the engine has EngineCapability::kSTANDARD, then all engine functionality is valid. If the engine has EngineCapability::kSAFETY, then only the functionality in safe engine is valid. If the engine has EngineCapability::kDLA_STANDALONE, then only serialize, destroy, and const-accessor functions are valid.

Returns: The EngineCapability flag that the engine was built for.

◆ getErrorRecorder()

IErrorRecorder * nvinfer1::ICudaEngine::getErrorRecorder ( ) const

inlinenoexcept

Get the ErrorRecorder assigned to this interface.

Retrieves the assigned error recorder object for the given class. A nullptr will be returned if an error handler has not been set.

Returns: A pointer to the IErrorRecorder object that has been registered.

See also: setErrorRecorder()

◆ getHardwareCompatibilityLevel()

HardwareCompatibilityLevel nvinfer1::ICudaEngine::getHardwareCompatibilityLevel ( ) const

inlinenoexcept

Return the hardware compatibility level of this engine.

Returns: hardwareCompatibilityLevel The level of hardware compatibility.

This is only supported for Ampere and newer architectures.

◆ getIOTensorName()

char const * nvinfer1::ICudaEngine::getIOTensorName ( int32_t index ) const

inlinenoexcept

Return name of an IO tensor.

Parameters

index value between 0 and getNbIOTensors()-1

See also: getNbIOTensors()

◆ getLocation()

TRT_DEPRECATED TensorLocation nvinfer1::ICudaEngine::getLocation ( int32_t bindingIndex ) const

inlinenoexcept

Get location of binding.

This lets you know whether the binding should be a pointer to device or host memory.

Parameters

bindingIndex The binding index.

Returns: The location of the bound tensor with given index.

Deprecated:: Deprecated in TensorRT 8.5. Superseded by getTensorLocation().

See also: ITensor::setLocation() ITensor::getLocation(); getTensorLocation()

◆ getMaxBatchSize()

TRT_DEPRECATED int32_t nvinfer1::ICudaEngine::getMaxBatchSize ( ) const

inlinenoexcept

Get the maximum batch size which can be used for inference. Should only be called if the engine is built from an INetworkDefinition with implicit batch dimension mode.

Returns: The maximum batch size for this engine.

Warning: For an engine built from an INetworkDefinition with explicit batch dimension mode, this will always return 1.

Deprecated:: Deprecated in TensorRT 8.4.

◆ getName()

char const * nvinfer1::ICudaEngine::getName ( ) const

inlinenoexcept

Returns the name of the network associated with the engine.

The name is set during network creation and is retrieved after building or deserialization.

See also: INetworkDefinition::setName(), INetworkDefinition::getName()

Returns: A null-terminated C-style string representing the name of the network.

◆ getNbAuxStreams()

int32_t nvinfer1::ICudaEngine::getNbAuxStreams ( ) const

inlinenoexcept

Return the number of auxiliary streams used by this engine.

This number will be less than or equal to the maximum allowed number of auxiliary streams set by IBuilderConfig::setMaxAuxStreams() API call when the engine was built.

Returns: the number of auxiliary streams used by this engine.

See also: IBuilderConfig::setMaxAuxStreams(), IExecutionContext::setAuxStreams()

◆ getNbBindings()

TRT_DEPRECATED int32_t nvinfer1::ICudaEngine::getNbBindings ( ) const

inlinenoexcept

Get the number of binding indices.

There are separate binding indices for each optimization profile. This method returns the total over all profiles. If the engine has been built for K profiles, the first getNbBindings() / K bindings are used by profile number 0, the following getNbBindings() / K bindings are used by profile number 1 etc.

Deprecated:: Deprecated in TensorRT 8.5. Superseded by getNbIOTensors.

See also: getBindingIndex()

◆ getNbIOTensors()

int32_t nvinfer1::ICudaEngine::getNbIOTensors ( ) const

inlinenoexcept

Return number of IO tensors.

It is the number of input and output tensors for the network from which the engine was built. The names of the IO tensors can be discovered by calling getIOTensorName(i) for i in 0 to getNbIOTensors()-1.

See also: getIOTensorName()

◆ getNbLayers()

int32_t nvinfer1::ICudaEngine::getNbLayers ( ) const

inlinenoexcept

Get the number of layers in the network.

The number of layers in the network is not necessarily the number in the original network definition, as layers may be combined or eliminated as the engine is optimized. This value can be useful when building per-layer tables, such as when aggregating profiling data over a number of executions.

Returns: The number of layers in the network.

◆ getNbOptimizationProfiles()

int32_t nvinfer1::ICudaEngine::getNbOptimizationProfiles ( ) const

inlinenoexcept

Get the number of optimization profiles defined for this engine.

Returns: Number of optimization profiles. It is always at least 1.

See also: IExecutionContext::setOptimizationProfile()

◆ getProfileDimensions()

TRT_DEPRECATED Dims nvinfer1::ICudaEngine::getProfileDimensions	(	int32_t	bindingIndex,
		int32_t	profileIndex,
		OptProfileSelector	select
	)		const

inlinenoexcept

Get the minimum / optimum / maximum dimensions for a particular input binding under an optimization profile.

Parameters

bindingIndex	The input binding index, which must belong to the given profile, or be between 0 and bindingsPerProfile-1 as described below.
profileIndex	The profile index, which must be between 0 and getNbOptimizationProfiles()-1.
select	Whether to query the minimum, optimum, or maximum dimensions for this binding.

Returns: The minimum / optimum / maximum dimensions for this binding in this profile. If the profileIndex or bindingIndex are invalid, return Dims with nbDims=-1.

For backwards compatibility with earlier versions of TensorRT, if the bindingIndex does not belong to the current optimization profile, but is between 0 and bindingsPerProfile-1, where bindingsPerProfile = getNbBindings()/getNbOptimizationProfiles, then a corrected bindingIndex is used instead, computed by:

profileIndex * bindingsPerProfile + bindingIndex % bindingsPerProfile

Otherwise the bindingIndex is considered invalid.

Deprecated:: Deprecated in TensorRT 8.5. Superseded by getProfileShape().

See also: getProfileShape()

◆ getProfileShape()

Dims nvinfer1::ICudaEngine::getProfileShape	(	char const *	tensorName,
		int32_t	profileIndex,
		OptProfileSelector	select
	)		const

inlinenoexcept

Get the minimum / optimum / maximum dimensions for an input tensor given its name under an optimization profile.

Parameters

tensorName	The name of an input tensor.
profileIndex	The profile index, which must be between 0 and getNbOptimizationProfiles()-1.
select	Whether to query the minimum, optimum, or maximum dimensions for this input tensor.

Returns: The minimum / optimum / maximum dimensions for an input tensor in this profile. If the profileIndex is invalid or provided name does not map to an input tensor, return Dims{-1, {}}

Warning: The string tensorName must be null-terminated, and be at most 4096 bytes including the terminator.

◆ getProfileShapeValues()

TRT_DEPRECATED int32_t const * nvinfer1::ICudaEngine::getProfileShapeValues	(	int32_t	profileIndex,
		int32_t	inputIndex,
		OptProfileSelector	select
	)		const

inlinenoexcept

Get minimum / optimum / maximum values for an input shape binding under an optimization profile.

Parameters

profileIndex	The profile index (must be between 0 and getNbOptimizationProfiles()-1)
inputIndex	The input index (must be between 0 and getNbBindings() - 1)
select	Whether to query the minimum, optimum, or maximum shape values for this binding.

Returns: If the binding is an input shape binding, return a pointer to an array that has the same number of elements as the corresponding tensor, i.e. 1 if dims.nbDims == 0, or dims.d[0] if dims.nbDims == 1, where dims = getBindingDimensions(inputIndex). The array contains the elementwise minimum / optimum / maximum values for this shape binding under the profile. If either of the indices is out of range, or if the binding is not an input shape binding, return nullptr.

For backwards compatibility with earlier versions of TensorRT, a bindingIndex that does not belong to the profile is corrected as described for getProfileDimensions().

Deprecated:: Deprecated in TensorRT 8.5. Superseded by getShapeValues(). Difference between Execution and shape tensor is superficial since TensorRT 8.5.

See also: getProfileDimensions() getShapeValues()

◆ getProfilingVerbosity()

ProfilingVerbosity nvinfer1::ICudaEngine::getProfilingVerbosity ( ) const

inlinenoexcept

Return the ProfilingVerbosity the builder config was set to when the engine was built.

Returns: the profiling verbosity the builder config was set to when the engine was built.

See also: IBuilderConfig::setProfilingVerbosity()

◆ getTacticSources()

TacticSources nvinfer1::ICudaEngine::getTacticSources ( ) const

inlinenoexcept

return the tactic sources required by this engine.

The value returned is equal to zero or more tactics sources set at build time via setTacticSources() in IBuilderConfig. Sources set by the latter but not returned by ICudaEngine::getTacticSources do not reduce overall engine execution time, and can be removed from future builds to reduce build time.

See also: IBuilderConfig::setTacticSources()

◆ getTensorBytesPerComponent() [1/2]

int32_t nvinfer1::ICudaEngine::getTensorBytesPerComponent ( char const * tensorName ) const

inlinenoexcept

Return the number of bytes per component of an element, or -1 if the provided name does not map to an input or output tensor.

The vector component size is returned if getTensorVectorizedDim() != -1.

Parameters

tensorName The name of an input or output tensor.

Warning: The string tensorName must be null-terminated, and be at most 4096 bytes including the terminator.; The function can only return the result of profile 0, and issues a warning message when there are multiple profiles in the engine, use getTensorBytesPerComponent with profileIndex when there are multiple profiles.

See also: getTensorVectorizedDim(); getTensorBytesPerComponent(tensorName, profileIndex)

◆ getTensorBytesPerComponent() [2/2]

int32_t nvinfer1::ICudaEngine::getTensorBytesPerComponent	(	char const *	tensorName,
		int32_t	profileIndex
	)		const

inlinenoexcept

Return the number of bytes per component of an element of given profile, or -1 if the provided name does not map to an input or output tensor.

The vector component size is returned if getTensorVectorizedDim(tensorName, profileIndex) != -1.

Parameters

tensorName	The name of an input or output tensor.
profileIndex	The profile index to query

Warning: The string tensorName must be null-terminated, and be at most 4096 bytes including the terminator.

See also: getTensorVectorizedDim(tensorName, profileIndex)

◆ getTensorComponentsPerElement() [1/2]

int32_t nvinfer1::ICudaEngine::getTensorComponentsPerElement ( char const * tensorName ) const

inlinenoexcept

Return the number of components included in one element, or -1 if the provided name does not map to an input or output tensor.

The number of elements in the vectors is returned if getTensorVectorizedDim() != -1.

Parameters

tensorName The name of an input or output tensor.

Warning: The string tensorName must be null-terminated, and be at most 4096 bytes including the terminator.; The function can only return the result of profile 0, and issues a warning message when there are multiple profiles in the engine, use getTensorComponentsPerElement with profileIndex when there are multiple profiles.

See also: getTensorVectorizedDim(); getTensorComponentsPerElement(tensorName, profileIndex)

◆ getTensorComponentsPerElement() [2/2]

int32_t nvinfer1::ICudaEngine::getTensorComponentsPerElement	(	char const *	tensorName,
		int32_t	profileIndex
	)		const

inlinenoexcept

Return the number of components included in one element of given profile, or -1 if the provided name does not map to an input or output tensor.

The number of elements in the vectors is returned if getTensorVectorizedDim(tensorName, profileIndex) != -1.

Parameters

tensorName	The name of an input or output tensor.
profileIndex	The profile index to query

Warning: The string tensorName must be null-terminated, and be at most 4096 bytes including the terminator.

See also: getTensorVectorizedDim(tensorName, profileIndex)

◆ getTensorDataType()

DataType nvinfer1::ICudaEngine::getTensorDataType ( char const * tensorName ) const

inlinenoexcept

Determine the required data type for a buffer from its tensor name.

Parameters

tensorName The name of an input or output tensor.

Returns: The type of the data in the buffer, or DataType::kFLOAT if the provided name does not map to an input or output tensor.

Warning: The string tensorName must be null-terminated, and be at most 4096 bytes including the terminator.

◆ getTensorFormat() [1/2]

TensorFormat nvinfer1::ICudaEngine::getTensorFormat ( char const * tensorName ) const

inlinenoexcept

Return the tensor format, or TensorFormat::kLINEAR if the provided name does not map to an input or output tensor.

Warning: The string tensorName must be null-terminated, and be at most 4096 bytes including the terminator.; This API can only return the tensor format of profile 0, and issues a warning message when there are multiple profiles in the engine, use getTensorFormat with profileIndex when there are multiple profiles.

See also: getTensorFormat(tensorName, profileIndex)

◆ getTensorFormat() [2/2]

TensorFormat nvinfer1::ICudaEngine::getTensorFormat	(	char const *	tensorName,
		int32_t	profileIndex
	)		const

inlinenoexcept

Return the tensor format of given profile, or TensorFormat::kLINEAR if the provided name does not map to an input or output tensor.

Parameters

tensorName	The name of an input or output tensor.
profileIndex	The profile index to query the format for.

Warning: The string tensorName must be null-terminated, and be at most 4096 bytes including the terminator.

◆ getTensorFormatDesc() [1/2]

char const * nvinfer1::ICudaEngine::getTensorFormatDesc ( char const * tensorName ) const

inlinenoexcept

Return the human readable description of the tensor format, or empty string if the provided name does not map to an input or output tensor.

The description includes the order, vectorization, data type, and strides. Examples are shown as follows: Example 1: kCHW + FP32 "Row major linear FP32 format" Example 2: kCHW2 + FP16 "Two wide channel vectorized row major FP16 format" Example 3: kHWC8 + FP16 + Line Stride = 32 "Channel major FP16 format where C % 8 == 0 and H Stride % 32 == 0"

Parameters

tensorName The name of an input or output tensor.

Warning: The string tensorName must be null-terminated, and be at most 4096 bytes including the terminator.; The function can only return the result of profile 0, and issues a warning message when there are multiple profiles in the engine, use getTensorFormatDesc with profileIndex when there are multiple profiles.

◆ getTensorFormatDesc() [2/2]

char const * nvinfer1::ICudaEngine::getTensorFormatDesc	(	char const *	tensorName,
		int32_t	profileIndex
	)		const

inlinenoexcept

Return the human readable description of the tensor format of given profile, or empty string if the provided name does not map to an input or output tensor.

The description includes the order, vectorization, data type, and strides. Examples are shown as follows: Example 1: kCHW + FP32 "Row major linear FP32 format" Example 2: kCHW2 + FP16 "Two wide channel vectorized row major FP16 format" Example 3: kHWC8 + FP16 + Line Stride = 32 "Channel major FP16 format where C % 8 == 0 and H Stride % 32 == 0"

Parameters

tensorName	The name of an input or output tensor.
profileIndex	The profile index to query the format for.

Warning: The string tensorName must be null-terminated, and be at most 4096 bytes including the terminator.

◆ getTensorIOMode()

TensorIOMode nvinfer1::ICudaEngine::getTensorIOMode ( char const * tensorName ) const

inlinenoexcept

Determine whether a tensor is an input or output tensor.

Parameters

tensorName The name of an input or output tensor.

Returns: kINPUT if tensorName is an input, kOUTPUT if tensorName is an output, or kNONE if neither.

Warning: The string tensorName must be null-terminated, and be at most 4096 bytes including the terminator.

◆ getTensorLocation()

TensorLocation nvinfer1::ICudaEngine::getTensorLocation ( char const * tensorName ) const

inlinenoexcept

Get whether an input or output tensor must be on GPU or CPU.

Parameters

tensorName The name of an input or output tensor.

Returns: TensorLocation::kDEVICE if tensorName must be on GPU, or TensorLocation::kHOST if on CPU, or TensorLocation::kDEVICE if the provided name does not map to an input or output tensor.

The location is established at build time. E.g. shape tensors inputs are typically required to be on the CPU.

Warning: The string tensorName must be null-terminated, and be at most 4096 bytes including the terminator.

◆ getTensorShape()

Dims nvinfer1::ICudaEngine::getTensorShape ( char const * tensorName ) const

inlinenoexcept

Get shape of an input or output tensor.

Parameters

tensorName The name of an input or output tensor.

Returns: shape of the tensor, with -1 in place of each dynamic runtime dimension, or Dims{-1, {}} if the provided name does not map to an input or output tensor.

Warning: The string tensorName must be null-terminated, and be at most 4096 bytes including the terminator.

◆ getTensorVectorizedDim() [1/2]

int32_t nvinfer1::ICudaEngine::getTensorVectorizedDim ( char const * tensorName ) const

inlinenoexcept

Return the dimension index that the buffer is vectorized, or -1 if the provided name does not map to an input or output tensor.

Specifically -1 is returned if scalars per vector is 1.

Parameters

tensorName The name of an input or output tensor.

Warning: The string tensorName must be null-terminated, and be at most 4096 bytes including the terminator.; The function can only return the result of profile 0, and issues a warning message when there are multiple profiles in the engine, use getTensorVectorizedDim with profileIndex when there are multiple profiles.

◆ getTensorVectorizedDim() [2/2]

int32_t nvinfer1::ICudaEngine::getTensorVectorizedDim	(	char const *	tensorName,
		int32_t	profileIndex
	)		const

inlinenoexcept

Return the dimension index that the buffer is vectorized of given profile, or -1 if the provided name does not map to an input or output tensor.

Specifically -1 is returned if scalars per vector is 1.

Parameters

tensorName	The name of an input.
profileIndex	The profile index to query the format for.

Warning: The string tensorName must be null-terminated, and be at most 4096 bytes including the terminator.

◆ hasImplicitBatchDimension()

bool nvinfer1::ICudaEngine::hasImplicitBatchDimension ( ) const

inlinenoexcept

Query whether the engine was built with an implicit batch dimension.

Returns: True if tensors have implicit batch dimension, false otherwise.

This is an engine-wide property. Either all tensors in the engine have an implicit batch dimension or none of them do.

hasImplicitBatchDimension() is true if and only if the INetworkDefinition from which this engine was built was created with createNetworkV2() without NetworkDefinitionCreationFlag::kEXPLICIT_BATCH flag.

See also: createNetworkV2

◆ isExecutionBinding()

TRT_DEPRECATED bool nvinfer1::ICudaEngine::isExecutionBinding ( int32_t bindingIndex ) const

inlinenoexcept

True if pointer to tensor data is required for execution phase, false if nullptr can be supplied.

For example, if a network uses an input tensor with binding i ONLY as the "reshape dimensions" input of IShuffleLayer, then isExecutionBinding(i) is false, and a nullptr can be supplied for it when calling IExecutionContext::execute or IExecutionContext::enqueue.

Deprecated:: No name-based equivalent replacement. Use getTensorLocation() instead to know the location of tensor data. Distinction between execution binding and shape binding is superficial since TensorRT 8.5.

See also: isShapeBinding() getTensorLocation()

◆ isRefittable()

bool nvinfer1::ICudaEngine::isRefittable ( ) const

inlinenoexcept

Return true if an engine can be refit.

See also: nvinfer1::createInferRefitter()

◆ isShapeBinding()

TRT_DEPRECATED bool nvinfer1::ICudaEngine::isShapeBinding ( int32_t bindingIndex ) const

inlinenoexcept

True if tensor is required as input for shape calculations or output from them.

TensorRT evaluates a network in two phases:

Compute shape information required to determine memory allocation requirements and validate that runtime sizes make sense.
Process tensors on the device.

Some tensors are required in phase 1. These tensors are called "shape tensors", and always have type Int32 and no more than one dimension. These tensors are not always shapes themselves, but might be used to calculate tensor shapes for phase 2.

isShapeBinding(i) returns true if the tensor is a required input or an output computed in phase 1. isExecutionBinding(i) returns true if the tensor is a required input or an output computed in phase 2.

For example, if a network uses an input tensor with binding i as an addend to an IElementWiseLayer that computes the "reshape dimensions" for IShuffleLayer, then isShapeBinding(i) == true.

It's possible to have a tensor be required by both phases. For instance, a tensor can be used for the "reshape dimensions" and as the indices for an IGatherLayer collecting floating-point data.

It's also possible to have a tensor be required by neither phase, but nonetheless shows up in the engine's inputs. For example, if an input tensor is used only as an input to IShapeLayer, only its shape matters and its values are irrelevant.

Deprecated:: Use name-based isShapeInferenceIO() instead to know whether a tensor is a shape tensor.

See also: isExecutionBinding() isShapeInferenceIO()

◆ isShapeInferenceIO()

bool nvinfer1::ICudaEngine::isShapeInferenceIO ( char const * tensorName ) const

inlinenoexcept

True if tensor is required as input for shape calculations or is output from shape calculations.

Return true for either of the following conditions:

The tensor is a network input, and its value is required for IExecutionContext::getTensorShape() to return the shape of a network output.
The tensor is a network output, and inferShape() will compute its values.

For example, if a network uses an input tensor "foo" as an addend to an IElementWiseLayer that computes the "reshape dimensions" for IShuffleLayer, then isShapeInferenceIO("foo") == true. If the network copies said input tensor "foo" to an output "bar", then isShapeInferenceIO("bar") == true and IExecutionContext::inferShapes() will write to "bar".

◆ serialize()

IHostMemory * nvinfer1::ICudaEngine::serialize ( ) const

inlinenoexcept

Serialize the network to a stream.

Returns: A IHostMemory object that contains the serialized engine.

The network may be deserialized with IRuntime::deserializeCudaEngine().

See also: IRuntime::deserializeCudaEngine()

◆ setErrorRecorder()

void nvinfer1::ICudaEngine::setErrorRecorder ( IErrorRecorder * recorder )

inlinenoexcept

Set the ErrorRecorder for this interface.

Assigns the ErrorRecorder to this interface. The ErrorRecorder will track all errors during execution. This function will call incRefCount of the registered ErrorRecorder at least once. Setting recorder to nullptr unregisters the recorder with the interface, resulting in a call to decRefCount if a recorder has been registered.

If an error recorder is not set, messages will be sent to the global log stream.

Parameters

recorder The error recorder to register with this interface.

See also: getErrorRecorder()

Member Data Documentation

◆ mImpl

apiv::VCudaEngine* nvinfer1::ICudaEngine::mImpl

protected

The documentation for this class was generated from the following file:

NvInferRuntime.h

Public Member Functions

Protected Attributes

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

◆ ~ICudaEngine()

Member Function Documentation

◆ bindingIsInput()

◆ createEngineInspector()

◆ createExecutionContext()

◆ createExecutionContextWithoutDeviceMemory()

◆ destroy()

◆ getBindingBytesPerComponent()

◆ getBindingComponentsPerElement()

◆ getBindingDataType()

◆ getBindingDimensions()

◆ getBindingFormat()

◆ getBindingFormatDesc()

◆ getBindingIndex()

◆ getBindingName()

◆ getBindingVectorizedDim()

◆ getDeviceMemorySize()

◆ getEngineCapability()

◆ getErrorRecorder()

◆ getHardwareCompatibilityLevel()

◆ getIOTensorName()

◆ getLocation()

◆ getMaxBatchSize()

◆ getName()

◆ getNbAuxStreams()

◆ getNbBindings()

◆ getNbIOTensors()

◆ getNbLayers()

◆ getNbOptimizationProfiles()

◆ getProfileDimensions()

◆ getProfileShape()

◆ getProfileShapeValues()

◆ getProfilingVerbosity()

◆ getTacticSources()

◆ getTensorBytesPerComponent() [1/2]

◆ getTensorBytesPerComponent() [2/2]

◆ getTensorComponentsPerElement() [1/2]

◆ getTensorComponentsPerElement() [2/2]

◆ getTensorDataType()

◆ getTensorFormat() [1/2]

◆ getTensorFormat() [2/2]

◆ getTensorFormatDesc() [1/2]

◆ getTensorFormatDesc() [2/2]

◆ getTensorIOMode()

◆ getTensorLocation()

◆ getTensorShape()

◆ getTensorVectorizedDim() [1/2]

◆ getTensorVectorizedDim() [2/2]

◆ hasImplicitBatchDimension()

◆ isExecutionBinding()

◆ isRefittable()

◆ isShapeBinding()

◆ isShapeInferenceIO()

◆ serialize()

◆ setErrorRecorder()

Member Data Documentation

◆ mImpl