Holds properties for configuring a builder to produce an engine. More...

#include <NvInfer.h>

Inheritance diagram for nvinfer1::IBuilderConfig:

Public Member Functions
virtual void	setMinTimingIterations (int32_t minTiming) noexcept
	Set the number of minimization iterations used when timing layers. More...

virtual int32_t	getMinTimingIterations () const noexcept
	Query the number of minimization iterations. More...

virtual void	setAvgTimingIterations (int32_t avgTiming) noexcept
	Set the number of averaging iterations used when timing layers. More...

int32_t	getAvgTimingIterations () const noexcept
	Query the number of averaging iterations. More...

void	setEngineCapability (EngineCapability capability) noexcept
	Configure the builder to target specified EngineCapability flow. More...

EngineCapability	getEngineCapability () const noexcept
	Query EngineCapability flow configured for the builder. More...

void	setInt8Calibrator (IInt8Calibrator *calibrator) noexcept
	Set Int8 Calibration interface. More...

IInt8Calibrator *	getInt8Calibrator () const noexcept
	Get Int8 Calibration interface.

void	setMaxWorkspaceSize (std::size_t workspaceSize) noexcept
	Set the maximum workspace size. More...

std::size_t	getMaxWorkspaceSize () const noexcept
	Get the maximum workspace size. More...

void	setFlags (BuilderFlags builderFlags) noexcept
	Set the build mode flags to turn on builder options for this network. More...

BuilderFlags	getFlags () const noexcept
	Get the build mode flags for this builder config. Defaults to 0. More...

void	clearFlag (BuilderFlag builderFlag) noexcept
	clear a single build mode flag. More...

void	setFlag (BuilderFlag builderFlag) noexcept
	Set a single build mode flag. More...

bool	getFlag (BuilderFlag builderFlag) const noexcept
	Returns true if the build mode flag is set. More...

void	setDeviceType (const ILayer *layer, DeviceType deviceType) noexcept
	Set the device that this layer must execute on. More...

DeviceType	getDeviceType (const ILayer *layer) const noexcept
	Get the device that this layer executes on. More...

bool	isDeviceTypeSet (const ILayer *layer) const noexcept
	whether the DeviceType has been explicitly set for this layer More...

void	resetDeviceType (const ILayer *layer) noexcept
	reset the DeviceType for this layer More...

bool	canRunOnDLA (const ILayer *layer) const noexcept
	Checks if a layer can run on DLA. More...

void	setDLACore (int32_t dlaCore) noexcept
	Sets the DLA core used by the network. More...

int32_t	getDLACore () const noexcept
	Get the DLA core that the engine executes on. More...

void	setDefaultDeviceType (DeviceType deviceType) noexcept
	Sets the default DeviceType to be used by the builder. It ensures that all the layers that can run on this device will run on it, unless setDeviceType is used to override the default DeviceType for a layer. More...

DeviceType	getDefaultDeviceType () const noexcept
	Get the default DeviceType which was set by setDefaultDeviceType. More...

void	reset () noexcept
	Resets the builder configuration to defaults. More...

TRT_DEPRECATED void	destroy () noexcept
	De-allocates any internally allocated memory. More...

void	setProfileStream (const cudaStream_t stream) noexcept
	Set the cuda stream that is used to profile this network. More...

cudaStream_t	getProfileStream () const noexcept
	Get the cuda stream that is used to profile this network. More...

int32_t	addOptimizationProfile (const IOptimizationProfile *profile) noexcept
	Add an optimization profile. More...

int32_t	getNbOptimizationProfiles () const noexcept
	Get number of optimization profiles. More...

void	setProfilingVerbosity (ProfilingVerbosity verbosity) noexcept
	Set verbosity level of layer information exposed in NVTX annotations. More...

ProfilingVerbosity	getProfilingVerbosity () const noexcept
	Get verbosity level of layer information exposed in NVTX annotations. More...

void	setAlgorithmSelector (IAlgorithmSelector *selector) noexcept
	Set Algorithm Selector. More...

IAlgorithmSelector *	getAlgorithmSelector () const noexcept
	Get Algorithm Selector.

bool	setCalibrationProfile (const IOptimizationProfile *profile) noexcept
	Add a calibration profile. More...

const IOptimizationProfile *	getCalibrationProfile () noexcept
	Get the current calibration profile. More...

void	setQuantizationFlags (QuantizationFlags flags) noexcept
	Set the quantization flags. More...

QuantizationFlags	getQuantizationFlags () const noexcept
	Get the quantization flags. More...

void	clearQuantizationFlag (QuantizationFlag flag) noexcept
	clear a quantization flag. More...

void	setQuantizationFlag (QuantizationFlag flag) noexcept
	Set a single quantization flag. More...

bool	getQuantizationFlag (QuantizationFlag flag) const noexcept
	Returns true if the quantization flag is set. More...

bool	setTacticSources (TacticSources tacticSources) noexcept
	Set tactic sources. More...

TacticSources	getTacticSources () const noexcept
	Get tactic sources. More...

nvinfer1::ITimingCache *	createTimingCache (const void *blob, std::size_t size) const noexcept
	Create timing cache. More...

bool	setTimingCache (const ITimingCache &cache, bool ignoreMismatch) noexcept
	Attach a timing cache to IBuilderConfig. More...

const nvinfer1::ITimingCache *	getTimingCache () const noexcept
	Get the pointer to the timing cache from current IBuilderConfig. More...

Protected Attributes
apiv::VBuilderConfig *	mImpl

Additional Inherited Members
Protected Member Functions inherited from nvinfer1::INoCopy
	INoCopy (const INoCopy &other)=delete

INoCopy &	operator= (const INoCopy &other)=delete

	INoCopy (INoCopy &&other)=delete

INoCopy &	operator= (INoCopy &&other)=delete

Detailed Description

Holds properties for configuring a builder to produce an engine.

See also: BuilderFlags

Member Function Documentation

◆ addOptimizationProfile()

int32_t nvinfer1::IBuilderConfig::addOptimizationProfile ( const IOptimizationProfile * profile )

inlinenoexcept

Add an optimization profile.

This function must be called at least once if the network has dynamic or shape input tensors. This function may be called at most once when building a refittable engine, as more than a single optimization profile are not supported for refittable engines.

Parameters

profile The new optimization profile, which must satisfy profile->isValid() == true

Returns: The index of the optimization profile (starting from 0) if the input is valid, or -1 if the input is not valid.

◆ canRunOnDLA()

bool nvinfer1::IBuilderConfig::canRunOnDLA ( const ILayer * layer ) const

inlinenoexcept

Checks if a layer can run on DLA.

Returns: status true if the layer can on DLA else returns false.

◆ clearFlag()

void nvinfer1::IBuilderConfig::clearFlag ( BuilderFlag builderFlag )

inlinenoexcept

clear a single build mode flag.

clears the builder mode flag from the enabled flags.

See also: setFlags()

◆ clearQuantizationFlag()

void nvinfer1::IBuilderConfig::clearQuantizationFlag ( QuantizationFlag flag )

inlinenoexcept

clear a quantization flag.

Clears the quantization flag from the enabled quantization flags.

See also: setQuantizationFlags()

◆ createTimingCache()

nvinfer1::ITimingCache* nvinfer1::IBuilderConfig::createTimingCache	(	const void *	blob,
		std::size_t	size
	)		const

inlinenoexcept

Create timing cache.

Create ITimingCache instance from serialized raw data. The created timing cache doesn’t belong to a specific IBuilderConfig. It can be shared by multiple builder instances. Call setTimingCache() before launching a builder to attach cache to builder instance.

Parameters

blob	A pointer to the raw data that contains serialized timing cache
size	The size in bytes of the serialized timing cache. Size 0 means create a new cache from scratch

See also: setTimingCache

Returns: the pointer to ITimingCache created

◆ destroy()

TRT_DEPRECATED void nvinfer1::IBuilderConfig::destroy ( )

inlinenoexcept

De-allocates any internally allocated memory.

When destroying a builder config object, we can call this function.

Deprecated:: Deprecated interface will be removed in TensorRT 10.0.

Warning: Calling destroy on a managed pointer will result in a double-free error.

◆ getAvgTimingIterations()

int32_t nvinfer1::IBuilderConfig::getAvgTimingIterations ( ) const

inlinenoexcept

Query the number of averaging iterations.

By default the number of averaging iterations is 1.

See also: setAvgTimingIterations()

◆ getCalibrationProfile()

const IOptimizationProfile* nvinfer1::IBuilderConfig::getCalibrationProfile ( )

inlinenoexcept

Get the current calibration profile.

Returns: A pointer to the current calibration profile or nullptr if calibration profile is unset.

◆ getDefaultDeviceType()

DeviceType nvinfer1::IBuilderConfig::getDefaultDeviceType ( ) const

inlinenoexcept

Get the default DeviceType which was set by setDefaultDeviceType.

By default it returns DeviceType::kGPU.

◆ getDeviceType()

DeviceType nvinfer1::IBuilderConfig::getDeviceType ( const ILayer * layer ) const

inlinenoexcept

Get the device that this layer executes on.

Returns: Returns DeviceType of the layer.

◆ getDLACore()

int32_t nvinfer1::IBuilderConfig::getDLACore ( ) const

inlinenoexcept

Get the DLA core that the engine executes on.

Returns: If setDLACore is called, returns DLA core from 0 to N-1, else returns 0.

Warning: Starting with TensorRT 8, the default value will be -1 if the DLA is not specified or unused.

◆ getEngineCapability()

EngineCapability nvinfer1::IBuilderConfig::getEngineCapability ( ) const

inlinenoexcept

Query EngineCapability flow configured for the builder.

By default it returns EngineCapability::kDEFAULT.

See also: setEngineCapability()

◆ getFlag()

bool nvinfer1::IBuilderConfig::getFlag ( BuilderFlag builderFlag ) const

inlinenoexcept

Returns true if the build mode flag is set.

See also: getFlags()

Returns: True if flag is set, false if unset.

◆ getFlags()

BuilderFlags nvinfer1::IBuilderConfig::getFlags ( ) const

inlinenoexcept

Get the build mode flags for this builder config. Defaults to 0.

Returns: The build options as a bitmask.

See also: setFlags()

◆ getMaxWorkspaceSize()

std::size_t nvinfer1::IBuilderConfig::getMaxWorkspaceSize ( ) const

inlinenoexcept

Get the maximum workspace size.

By default the workspace size is 0, which means there is no temporary memory.

Returns: The maximum workspace size.

See also: setMaxWorkspaceSize()

◆ getMinTimingIterations()

virtual int32_t nvinfer1::IBuilderConfig::getMinTimingIterations ( ) const

inlinevirtualnoexcept

Query the number of minimization iterations.

By default the minimum number of iterations is 2.

See also: setMinTimingIterations()

◆ getNbOptimizationProfiles()

int32_t nvinfer1::IBuilderConfig::getNbOptimizationProfiles ( ) const

inlinenoexcept

Get number of optimization profiles.

This is one higher than the index of the last optimization profile that has be defined (or zero, if none has been defined yet).

Returns: The number of the optimization profiles.

◆ getProfileStream()

cudaStream_t nvinfer1::IBuilderConfig::getProfileStream ( ) const

inlinenoexcept

Get the cuda stream that is used to profile this network.

Returns: The cuda stream set by setProfileStream, nullptr if setProfileStream has not been called.

See also: setProfileStream()

◆ getProfilingVerbosity()

ProfilingVerbosity nvinfer1::IBuilderConfig::getProfilingVerbosity ( ) const

inlinenoexcept

Get verbosity level of layer information exposed in NVTX annotations.

Get the current setting of verbosity level of layer information exposed in NVTX annotations. Default value is ProfilingVerbosity::kDEFAULT.

See also: ProfilingVerbosity, setProfilingVerbosity()

◆ getQuantizationFlag()

bool nvinfer1::IBuilderConfig::getQuantizationFlag ( QuantizationFlag flag ) const

inlinenoexcept

Returns true if the quantization flag is set.

See also: getQuantizationFlags()

Returns: True if quantization flag is set, false if unset.

◆ getQuantizationFlags()

QuantizationFlags nvinfer1::IBuilderConfig::getQuantizationFlags ( ) const

inlinenoexcept

Get the quantization flags.

Returns: The quantization flags as a bitmask.

See also: setQuantizationFlag()

◆ getTacticSources()

TacticSources nvinfer1::IBuilderConfig::getTacticSources ( ) const

inlinenoexcept

Get tactic sources.

Get the tactic sources currently set in the engine build configuration.

See also: setTacticSources

Returns: tactic sources

◆ getTimingCache()

const nvinfer1::ITimingCache* nvinfer1::IBuilderConfig::getTimingCache ( ) const

inlinenoexcept

Get the pointer to the timing cache from current IBuilderConfig.

Returns: pointer to the timing cache used in current IBuilderConfig

◆ isDeviceTypeSet()

bool nvinfer1::IBuilderConfig::isDeviceTypeSet ( const ILayer * layer ) const

inlinenoexcept

whether the DeviceType has been explicitly set for this layer

Returns: true if device type is not default

See also: setDeviceType() getDeviceType() resetDeviceType()

◆ reset()

void nvinfer1::IBuilderConfig::reset ( )

inlinenoexcept

Resets the builder configuration to defaults.

When initializing a builder config object, we can call this function.

◆ resetDeviceType()

void nvinfer1::IBuilderConfig::resetDeviceType ( const ILayer * layer )

inlinenoexcept

reset the DeviceType for this layer

See also: setDeviceType() getDeviceType() isDeviceTypeSet()

◆ setAlgorithmSelector()

void nvinfer1::IBuilderConfig::setAlgorithmSelector ( IAlgorithmSelector * selector )

inlinenoexcept

Set Algorithm Selector.

Parameters

selector The algorithm selector to be set in the build config.

◆ setAvgTimingIterations()

virtual void nvinfer1::IBuilderConfig::setAvgTimingIterations ( int32_t avgTiming )

inlinevirtualnoexcept

Set the number of averaging iterations used when timing layers.

When timing layers, the builder minimizes over a set of average times for layer execution. This parameter controls the number of iterations used in averaging.

See also: getAvgTimingIterations()

◆ setCalibrationProfile()

bool nvinfer1::IBuilderConfig::setCalibrationProfile ( const IOptimizationProfile * profile )

inlinenoexcept

Add a calibration profile.

Calibration optimization profile must be set if int8 calibration is used to set scales for a network with runtime dimensions.

Parameters

profile The new calibration profile, which must satisfy profile->isValid() == true or be nullptr. MIN and MAX values will be overwritten by kOPT.

Returns: True if the calibration profile was set correctly.

◆ setDefaultDeviceType()

void nvinfer1::IBuilderConfig::setDefaultDeviceType ( DeviceType deviceType )

inlinenoexcept

Sets the default DeviceType to be used by the builder. It ensures that all the layers that can run on this device will run on it, unless setDeviceType is used to override the default DeviceType for a layer.

See also: getDefaultDeviceType()

◆ setDeviceType()

void nvinfer1::IBuilderConfig::setDeviceType	(	const ILayer *	layer,
		DeviceType	deviceType
	)

inlinenoexcept

Set the device that this layer must execute on.

Parameters

deviceType that this layer must execute on. If DeviceType is not set or is reset, TensorRT will use the default DeviceType set in the builder.

Note: The device type for a layer must be compatible with the safety flow (if specified). For example a layer cannot be marked for DLA execution while the builder is configured for kSAFE_GPU.

See also: getDeviceType()

◆ setDLACore()

void nvinfer1::IBuilderConfig::setDLACore ( int32_t dlaCore )

inlinenoexcept

Sets the DLA core used by the network.

Parameters

dlaCore The DLA core to execute the engine on (0 to N-1). Default value is 0.

It can be used to specify which DLA core to use via indexing, if multiple DLA cores are available.

See also: IRuntime::setDLACore() getDLACore()

Warning: Starting with TensorRT 8, the default value will be -1 if the DLA is not specified or unused.

◆ setEngineCapability()

void nvinfer1::IBuilderConfig::setEngineCapability ( EngineCapability capability )

inlinenoexcept

Configure the builder to target specified EngineCapability flow.

The flow means a sequence of API calls that allow an application to set up a runtime, engine, and execution context in order to run inference.

The supported flows are specified in the EngineCapability enum.

◆ setFlag()

void nvinfer1::IBuilderConfig::setFlag ( BuilderFlag builderFlag )

inlinenoexcept

Set a single build mode flag.

Add the input builder mode flag to the already enabled flags.

See also: setFlags()

◆ setFlags()

void nvinfer1::IBuilderConfig::setFlags ( BuilderFlags builderFlags )

inlinenoexcept

Set the build mode flags to turn on builder options for this network.

The flags are listed in the BuilderFlags enum. The flags set configuration options to build the network.

Parameters

builderFlags The build option for an engine.

Note: This function will override the previous set flags, rather than bitwise ORing the new flag.

See also: getFlags()

◆ setInt8Calibrator()

void nvinfer1::IBuilderConfig::setInt8Calibrator ( IInt8Calibrator * calibrator )

inlinenoexcept

Set Int8 Calibration interface.

The calibrator is to minimize the information loss during the INT8 quantization process.

◆ setMaxWorkspaceSize()

void nvinfer1::IBuilderConfig::setMaxWorkspaceSize ( std::size_t workspaceSize )

inlinenoexcept

Set the maximum workspace size.

Parameters

workspaceSize The maximum GPU temporary memory which the engine can use at execution time.

See also: getMaxWorkspaceSize()

◆ setMinTimingIterations()

virtual void nvinfer1::IBuilderConfig::setMinTimingIterations ( int32_t minTiming )

inlinevirtualnoexcept

Set the number of minimization iterations used when timing layers.

When timing layers, the builder minimizes over a set of average times for layer execution. This parameter controls the number of iterations used in minimization. The builder may sometimes run layers for more iterations to improve timing accuracy if this parameter is set to a small value and the runtime of the layer is short.

See also: getMinTimingIterations()

◆ setProfileStream()

void nvinfer1::IBuilderConfig::setProfileStream ( const cudaStream_t stream )

inlinenoexcept

Set the cuda stream that is used to profile this network.

Parameters

stream The cuda stream used for profiling by the builder.

See also: getProfileStream()

◆ setProfilingVerbosity()

void nvinfer1::IBuilderConfig::setProfilingVerbosity ( ProfilingVerbosity verbosity )

inlinenoexcept

Set verbosity level of layer information exposed in NVTX annotations.

Control how much layer information will be exposed in NVTX annotations.

See also: ProfilingVerbosity, getProfilingVerbosity()

◆ setQuantizationFlag()

void nvinfer1::IBuilderConfig::setQuantizationFlag ( QuantizationFlag flag )

inlinenoexcept

Set a single quantization flag.

Add the input quantization flag to the already enabled quantization flags.

See also: setQuantizationFlags()

◆ setQuantizationFlags()

void nvinfer1::IBuilderConfig::setQuantizationFlags ( QuantizationFlags flags )

inlinenoexcept

Set the quantization flags.

The flags are listed in the QuantizationFlag enum. The flags set configuration options to quantize the network in int8.

Parameters

flags The quantization flags.

Note: This function will override the previous set flags, rather than bitwise ORing the new flag.

See also: getQuantizationFlags()

◆ setTacticSources()

bool nvinfer1::IBuilderConfig::setTacticSources ( TacticSources tacticSources )

inlinenoexcept

Set tactic sources.

This bitset controls which tactic sources TensorRT is allowed to use for tactic selection.

By default, kCUBLAS is always enabled. kCUBLAS_LT is enabled for x86 platforms, as well as non-x86 platforms if CUDA >= 11.0

Multiple tactic sources may be combined with a bitwise OR operation. For example, to enable cublas and cublasLt as tactic sources, use a value of:

1U << static_cast<uint32_t>(TacticSource::kCUBLAS) | 1U << static_cast<uint32_t>(TacticSource::kCUBLAS_LT)

See also: getTacticSources

Returns: true if the tactic sources in the build configuration were updated. The tactic sources in the build configuration will not be updated if the provided value is invalid.

◆ setTimingCache()

bool nvinfer1::IBuilderConfig::setTimingCache	(	const ITimingCache &	cache,
		bool	ignoreMismatch
	)

inlinenoexcept

Attach a timing cache to IBuilderConfig.

The timing cache has verification header to make sure the provided cache can be used in current environment. A failure will be reported if the CUDA device property in the provided cache is different from current environment. ignoreMismatch = true skips strict verification and allows loading cache created from a different device.

The cache must not be destroyed until after the engine is built.

Parameters

cache	the timing cache to be used
ignoreMismatch	whether or not allow using a cache that contains different CUDA device property

Returns: true if set successfully, false otherwise

Warning: Using cache generated from devices with different CUDA device properties may lead to functional/performance bugs.

The documentation for this class was generated from the following file:

NvInfer.h

Public Member Functions

Protected Attributes

Additional Inherited Members

Detailed Description

Member Function Documentation

◆ addOptimizationProfile()

◆ canRunOnDLA()

◆ clearFlag()

◆ clearQuantizationFlag()

◆ createTimingCache()

◆ destroy()

◆ getAvgTimingIterations()

◆ getCalibrationProfile()

◆ getDefaultDeviceType()

◆ getDeviceType()

◆ getDLACore()

◆ getEngineCapability()

◆ getFlag()

◆ getFlags()

◆ getMaxWorkspaceSize()

◆ getMinTimingIterations()

◆ getNbOptimizationProfiles()

◆ getProfileStream()

◆ getProfilingVerbosity()

◆ getQuantizationFlag()

◆ getQuantizationFlags()

◆ getTacticSources()

◆ getTimingCache()

◆ isDeviceTypeSet()

◆ reset()

◆ resetDeviceType()

◆ setAlgorithmSelector()

◆ setAvgTimingIterations()

◆ setCalibrationProfile()

◆ setDefaultDeviceType()

◆ setDeviceType()

◆ setDLACore()

◆ setEngineCapability()

◆ setFlag()

◆ setFlags()

◆ setInt8Calibrator()

◆ setMaxWorkspaceSize()

◆ setMinTimingIterations()

◆ setProfileStream()

◆ setProfilingVerbosity()

◆ setQuantizationFlag()

◆ setQuantizationFlags()

◆ setTacticSources()

◆ setTimingCache()