Holds properties for configuring a builder to produce an engine. More...

#include <NvInfer.h>

Public Member Functions
virtual void	setMinTimingIterations (int32_t minTiming)=0
	Set the number of minimization iterations used when timing layers. More...

virtual int32_t	getMinTimingIterations () const =0
	Query the number of minimization iterations. More...

virtual void	setAvgTimingIterations (int32_t avgTiming)=0
	Set the number of averaging iterations used when timing layers. More...

virtual int32_t	getAvgTimingIterations () const =0
	Query the number of averaging iterations. More...

virtual void	setEngineCapability (EngineCapability capability)=0
	Configure the builder to target specified EngineCapability flow. More...

virtual EngineCapability	getEngineCapability () const =0
	Query EngineCapability flow configured for the builder. More...

virtual void	setInt8Calibrator (IInt8Calibrator *calibrator)=0
	Set Int8 Calibration interface. More...

virtual IInt8Calibrator *	getInt8Calibrator () const =0
	Get Int8 Calibration interface.

virtual void	setMaxWorkspaceSize (std::size_t workspaceSize)=0
	Set the maximum workspace size. More...

virtual std::size_t	getMaxWorkspaceSize () const =0
	Get the maximum workspace size. More...

virtual void	setFlags (BuilderFlags builderFlags)=0
	Set the build mode flags to turn on builder options for this network. More...

virtual BuilderFlags	getFlags () const =0
	Get the build mode flags for this builder config. Defaults to 0. More...

virtual void	clearFlag (BuilderFlag builderFlag)=0
	clear a single build mode flag. More...

virtual void	setFlag (BuilderFlag builderFlag)=0
	Set a single build mode flag. More...

virtual bool	getFlag (BuilderFlag builderFlag) const =0
	Returns true if the build mode flag is set. More...

virtual void	setDeviceType (const ILayer *layer, DeviceType deviceType)=0
	Set the device that this layer must execute on. More...

virtual DeviceType	getDeviceType (const ILayer *layer) const =0
	Get the device that this layer executes on. More...

virtual bool	isDeviceTypeSet (const ILayer *layer) const =0
	whether the DeviceType has been explicitly set for this layer More...

virtual void	resetDeviceType (const ILayer *layer)=0
	reset the DeviceType for this layer More...

virtual bool	canRunOnDLA (const ILayer *layer) const =0
	Checks if a layer can run on DLA. More...

virtual void	setDLACore (int32_t dlaCore)=0
	Sets the DLA core used by the network. More...

virtual int32_t	getDLACore () const =0
	Get the DLA core that the engine executes on. More...

virtual void	setDefaultDeviceType (DeviceType deviceType)=0
	Sets the default DeviceType to be used by the builder. It ensures that all the layers that can run on this device will run on it, unless setDeviceType is used to override the default DeviceType for a layer. More...

virtual DeviceType	getDefaultDeviceType () const =0
	Get the default DeviceType which was set by setDefaultDeviceType. More...

virtual void	reset ()=0
	Resets the builder configuration to defaults. More...

virtual void	destroy ()=0
	De-allocates any internally allocated memory. More...

virtual void	setProfileStream (const cudaStream_t stream)=0
	Set the cudaStream that is used to profile this network. More...

virtual cudaStream_t	getProfileStream () const =0
	Get the cudaStream that is used to profile this network. More...

virtual int32_t	addOptimizationProfile (const IOptimizationProfile *profile) noexcept=0
	Add an optimization profile. More...

virtual int32_t	getNbOptimizationProfiles () const noexcept=0
	Get number of optimization profiles. More...

virtual void	setProfilingVerbosity (ProfilingVerbosity verbosity)=0
	Set verbosity level of layer information exposed in NVTX annotations. More...

virtual ProfilingVerbosity	getProfilingVerbosity () const =0
	Get verbosity level of layer information exposed in NVTX annotations. More...

virtual void	setAlgorithmSelector (IAlgorithmSelector *selector)=0
	Set Algorithm Selector. More...

virtual IAlgorithmSelector *	getAlgorithmSelector () const =0
	Get Algorithm Selector.

virtual bool	setCalibrationProfile (const IOptimizationProfile *profile) noexcept=0
	Add a calibration profile. More...

virtual const IOptimizationProfile *	getCalibrationProfile () noexcept=0
	Get the current calibration profile. More...

virtual void	setQuantizationFlags (QuantizationFlags flags)=0
	Set the quantization flags. More...

virtual QuantizationFlags	getQuantizationFlags () const =0
	Get the quantization flags. More...

virtual void	clearQuantizationFlag (QuantizationFlag flag)=0
	clear a quantization flag. More...

virtual void	setQuantizationFlag (QuantizationFlag flag)=0
	Set a single quantization flag. More...

virtual bool	getQuantizationFlag (QuantizationFlag flag) const =0
	Returns true if the quantization flag is set. More...

virtual bool	setTacticSources (TacticSources tacticSources)=0
	Set tactic sources. More...

virtual TacticSources	getTacticSources () const =0
	Get tactic sources. More...

Detailed Description

Holds properties for configuring a builder to produce an engine.

See also: BuilderFlags

Member Function Documentation

◆ addOptimizationProfile()

virtual int32_t nvinfer1::IBuilderConfig::addOptimizationProfile ( const IOptimizationProfile * profile )

pure virtualnoexcept

Add an optimization profile.

This function must be called at least once if the network has dynamic or shape input tensors. This function may be called at most once when building a refittable engine, as more than a single optimization profile are not supported for refittable engines.

Parameters

profile The new optimization profile, which must satisfy profile->isValid() == true

Returns: The index of the optimization profile (starting from 0) if the input is valid, or -1 if the input is not valid.

◆ canRunOnDLA()

virtual bool nvinfer1::IBuilderConfig::canRunOnDLA ( const ILayer * layer ) const

pure virtual

Checks if a layer can run on DLA.

Returns: status true if the layer can on DLA else returns false.

◆ clearFlag()

virtual void nvinfer1::IBuilderConfig::clearFlag ( BuilderFlag builderFlag )

pure virtual

clear a single build mode flag.

clears the builder mode flag from the enabled flags.

See also: setFlags()

◆ clearQuantizationFlag()

virtual void nvinfer1::IBuilderConfig::clearQuantizationFlag ( QuantizationFlag flag )

pure virtual

clear a quantization flag.

Clears the quantization flag from the enabled quantization flags.

See also: setQuantizationFlags()

◆ destroy()

virtual void nvinfer1::IBuilderConfig::destroy ( )

pure virtual

De-allocates any internally allocated memory.

When destroying a builder config object, we can call this function.

◆ getAvgTimingIterations()

virtual int32_t nvinfer1::IBuilderConfig::getAvgTimingIterations ( ) const

pure virtual

Query the number of averaging iterations.

By default the number of averaging iterations is 1.

See also: setAvgTimingIterations()

◆ getCalibrationProfile()

virtual const IOptimizationProfile* nvinfer1::IBuilderConfig::getCalibrationProfile ( )

pure virtualnoexcept

Get the current calibration profile.

Returns: A pointer to the current calibration profile or nullptr if calibration profile is unset.

◆ getDefaultDeviceType()

virtual DeviceType nvinfer1::IBuilderConfig::getDefaultDeviceType ( ) const

pure virtual

Get the default DeviceType which was set by setDefaultDeviceType.

By default it returns DeviceType::kGPU.

◆ getDeviceType()

virtual DeviceType nvinfer1::IBuilderConfig::getDeviceType ( const ILayer * layer ) const

pure virtual

Get the device that this layer executes on.

Returns: Returns DeviceType of the layer.

◆ getDLACore()

virtual int32_t nvinfer1::IBuilderConfig::getDLACore ( ) const

pure virtual

Get the DLA core that the engine executes on.

Returns: If setDLACore is called, returns DLA core from 0 to N-1, else returns 0.

Warning: Starting with TensorRT 8, the default value will be -1 if the DLA is not specified or unused.

◆ getEngineCapability()

virtual EngineCapability nvinfer1::IBuilderConfig::getEngineCapability ( ) const

pure virtual

Query EngineCapability flow configured for the builder.

By default it returns EngineCapability::kDEFAULT.

See also: setEngineCapability()

◆ getFlag()

virtual bool nvinfer1::IBuilderConfig::getFlag ( BuilderFlag builderFlag ) const

pure virtual

Returns true if the build mode flag is set.

See also: getFlags()

Returns: True if flag is set, false if unset.

◆ getFlags()

virtual BuilderFlags nvinfer1::IBuilderConfig::getFlags ( ) const

pure virtual

Get the build mode flags for this builder config. Defaults to 0.

Returns: The build options as a bitmask.

See also: setFlags()

◆ getMaxWorkspaceSize()

virtual std::size_t nvinfer1::IBuilderConfig::getMaxWorkspaceSize ( ) const

pure virtual

Get the maximum workspace size.

By default the workspace size is 0, which means there is no temporary memory.

Returns: The maximum workspace size.

See also: setMaxWorkspaceSize()

◆ getMinTimingIterations()

virtual int32_t nvinfer1::IBuilderConfig::getMinTimingIterations ( ) const

pure virtual

Query the number of minimization iterations.

By default the minimum number of iterations is 2.

See also: setMinTimingIterations()

◆ getNbOptimizationProfiles()

virtual int32_t nvinfer1::IBuilderConfig::getNbOptimizationProfiles ( ) const

pure virtualnoexcept

Get number of optimization profiles.

This is one higher than the index of the last optimization profile that has be defined (or zero, if none has been defined yet).

Returns: The number of the optimization profiles.

◆ getProfileStream()

virtual cudaStream_t nvinfer1::IBuilderConfig::getProfileStream ( ) const

pure virtual

Get the cudaStream that is used to profile this network.

Returns: The cuda stream used for profiling by the builder.

See also: setProfileStream()

◆ getProfilingVerbosity()

virtual ProfilingVerbosity nvinfer1::IBuilderConfig::getProfilingVerbosity ( ) const

pure virtual

Get verbosity level of layer information exposed in NVTX annotations.

Get the current setting of verbosity level of layer information exposed in NVTX annotations. Default value is ProfilingVerbosity::kDEFAULT.

See also: ProfilingVerbosity, setProfilingVerbosity()

◆ getQuantizationFlag()

virtual bool nvinfer1::IBuilderConfig::getQuantizationFlag ( QuantizationFlag flag ) const

pure virtual

Returns true if the quantization flag is set.

See also: getQuantizationFlags()

Returns: True if quantization flag is set, false if unset.

◆ getQuantizationFlags()

virtual QuantizationFlags nvinfer1::IBuilderConfig::getQuantizationFlags ( ) const

pure virtual

Get the quantization flags.

Returns: The quantization flags as a bitmask.

See also: setQuantizationFlag()

◆ getTacticSources()

virtual TacticSources nvinfer1::IBuilderConfig::getTacticSources ( ) const

pure virtual

Get tactic sources.

Get the tactic sources currently set in the engine build configuration.

See also: setTacticSources

Returns: tactic sources

◆ isDeviceTypeSet()

virtual bool nvinfer1::IBuilderConfig::isDeviceTypeSet ( const ILayer * layer ) const

pure virtual

whether the DeviceType has been explicitly set for this layer

Returns: true if device type is not default

See also: setDeviceType() getDeviceType() resetDeviceType()

◆ reset()

virtual void nvinfer1::IBuilderConfig::reset ( )

pure virtual

Resets the builder configuration to defaults.

When initializing a builder config object, we can call this function.

◆ resetDeviceType()

virtual void nvinfer1::IBuilderConfig::resetDeviceType ( const ILayer * layer )

pure virtual

reset the DeviceType for this layer

See also: setDeviceType() getDeviceType() isDeviceTypeSet()

◆ setAlgorithmSelector()

virtual void nvinfer1::IBuilderConfig::setAlgorithmSelector ( IAlgorithmSelector * selector )

pure virtual

Set Algorithm Selector.

Parameters

selector The algorithm selector to be set in the build config.

◆ setAvgTimingIterations()

virtual void nvinfer1::IBuilderConfig::setAvgTimingIterations ( int32_t avgTiming )

pure virtual

Set the number of averaging iterations used when timing layers.

When timing layers, the builder minimizes over a set of average times for layer execution. This parameter controls the number of iterations used in averaging.

See also: getAvgTimingIterations()

◆ setCalibrationProfile()

virtual bool nvinfer1::IBuilderConfig::setCalibrationProfile ( const IOptimizationProfile * profile )

pure virtualnoexcept

Add a calibration profile.

Calibration optimization profile must be set if int8 calibration is used to set scales for a network with runtime dimensions.

Parameters

profile The new calibration profile, which must satisfy profile->isValid() == true or be nullptr. MIN and MAX values will be overwritten by kOPT.

Returns: True if the calibration profile was set correctly.

◆ setDefaultDeviceType()

virtual void nvinfer1::IBuilderConfig::setDefaultDeviceType ( DeviceType deviceType )

pure virtual

Sets the default DeviceType to be used by the builder. It ensures that all the layers that can run on this device will run on it, unless setDeviceType is used to override the default DeviceType for a layer.

See also: getDefaultDeviceType()

◆ setDeviceType()

virtual void nvinfer1::IBuilderConfig::setDeviceType	(	const ILayer *	layer,
		DeviceType	deviceType
	)

pure virtual

Set the device that this layer must execute on.

Parameters

deviceType that this layer must execute on. If DeviceType is not set or is reset, TensorRT will use the default DeviceType set in the builder.

Note: The device type for a layer must be compatible with the safety flow (if specified). For example a layer cannot be marked for DLA execution while the builder is configured for kSAFE_GPU.

See also: getDeviceType()

◆ setDLACore()

virtual void nvinfer1::IBuilderConfig::setDLACore ( int32_t dlaCore )

pure virtual

Sets the DLA core used by the network.

Parameters

dlaCore The DLA core to execute the engine on (0 to N-1). Default value is 0.

It can be used to specify which DLA core to use via indexing, if multiple DLA cores are available.

See also: IRuntime::setDLACore() getDLACore()

Warning: Starting with TensorRT 8, the default value will be -1 if the DLA is not specified or unused.

◆ setEngineCapability()

virtual void nvinfer1::IBuilderConfig::setEngineCapability ( EngineCapability capability )

pure virtual

Configure the builder to target specified EngineCapability flow.

The flow means a sequence of API calls that allow an application to set up a runtime, engine, and execution context in order to run inference.

The supported flows are specified in the EngineCapability enum.

◆ setFlag()

virtual void nvinfer1::IBuilderConfig::setFlag ( BuilderFlag builderFlag )

pure virtual

Set a single build mode flag.

Add the input builder mode flag to the already enabled flags.

See also: setFlags()

◆ setFlags()

virtual void nvinfer1::IBuilderConfig::setFlags ( BuilderFlags builderFlags )

pure virtual

Set the build mode flags to turn on builder options for this network.

The flags are listed in the BuilderFlags enum. The flags set configuration options to build the network.

Parameters

builderFlags The build option for an engine.

Note: This function will override the previous set flags, rather than bitwise ORing the new flag.

See also: getFlags()

◆ setInt8Calibrator()

virtual void nvinfer1::IBuilderConfig::setInt8Calibrator ( IInt8Calibrator * calibrator )

pure virtual

Set Int8 Calibration interface.

The calibrator is to minimize the information loss during the INT8 quantization process.

◆ setMaxWorkspaceSize()

virtual void nvinfer1::IBuilderConfig::setMaxWorkspaceSize ( std::size_t workspaceSize )

pure virtual

Set the maximum workspace size.

Parameters

workspaceSize The maximum GPU temporary memory which the engine can use at execution time.

See also: getMaxWorkspaceSize()

◆ setMinTimingIterations()

virtual void nvinfer1::IBuilderConfig::setMinTimingIterations ( int32_t minTiming )

pure virtual

Set the number of minimization iterations used when timing layers.

When timing layers, the builder minimizes over a set of average times for layer execution. This parameter controls the number of iterations used in minimization. The builder may sometimes run layers for more iterations to improve timing accuracy if this parameter is set to a small value and the runtime of the layer is short.

See also: getMinTimingIterations()

◆ setProfileStream()

virtual void nvinfer1::IBuilderConfig::setProfileStream ( const cudaStream_t stream )

pure virtual

Set the cudaStream that is used to profile this network.

Parameters

stream The cuda stream used for profiling by the builder.

See also: getProfileStream()

◆ setProfilingVerbosity()

virtual void nvinfer1::IBuilderConfig::setProfilingVerbosity ( ProfilingVerbosity verbosity )

pure virtual

Set verbosity level of layer information exposed in NVTX annotations.

Control how much layer information will be exposed in NVTX annotations.

See also: ProfilingVerbosity, getProfilingVerbosity()

◆ setQuantizationFlag()

virtual void nvinfer1::IBuilderConfig::setQuantizationFlag ( QuantizationFlag flag )

pure virtual

Set a single quantization flag.

Add the input quantization flag to the already enabled quantization flags.

See also: setQuantizationFlags()

◆ setQuantizationFlags()

virtual void nvinfer1::IBuilderConfig::setQuantizationFlags ( QuantizationFlags flags )

pure virtual

Set the quantization flags.

The flags are listed in the QuantizationFlag enum. The flags set configuration options to quantize the network in int8.

Parameters

flags The quantization flags.

Note: This function will override the previous set flags, rather than bitwise ORing the new flag.

See also: getQuantizationFlags()

◆ setTacticSources()

virtual bool nvinfer1::IBuilderConfig::setTacticSources ( TacticSources tacticSources )

pure virtual

Set tactic sources.

This bitset controls which tactic sources TensorRT is allowed to use for tactic selection.

By default, kCUBLAS is always enabled. kCUBLAS_LT is enabled for x86 platforms, as well as non-x86 platforms if CUDA >= 11.0

Multiple tactic sources may be combined with a bitwise OR operation. For example, to enable cublas and cublasLt as tactic sources, use a value of:

1U << static_cast<uint32_t>(TacticSource::kCUBLAS) | 1U << static_cast<uint32_t>(TacticSource::kCUBLAS_LT)

See also: getTacticSources

Returns: true if the tactic sources in the build configuration were updated. The tactic sources in the build configuration will not be updated if the provided value is invalid.

The documentation for this class was generated from the following file:

NvInfer.h

Public Member Functions

Detailed Description

Member Function Documentation

◆ addOptimizationProfile()

◆ canRunOnDLA()

◆ clearFlag()

◆ clearQuantizationFlag()

◆ destroy()

◆ getAvgTimingIterations()

◆ getCalibrationProfile()

◆ getDefaultDeviceType()

◆ getDeviceType()

◆ getDLACore()

◆ getEngineCapability()

◆ getFlag()

◆ getFlags()

◆ getMaxWorkspaceSize()

◆ getMinTimingIterations()

◆ getNbOptimizationProfiles()

◆ getProfileStream()

◆ getProfilingVerbosity()

◆ getQuantizationFlag()

◆ getQuantizationFlags()

◆ getTacticSources()

◆ isDeviceTypeSet()

◆ reset()

◆ resetDeviceType()

◆ setAlgorithmSelector()

◆ setAvgTimingIterations()

◆ setCalibrationProfile()

◆ setDefaultDeviceType()

◆ setDeviceType()

◆ setDLACore()

◆ setEngineCapability()

◆ setFlag()

◆ setFlags()

◆ setInt8Calibrator()

◆ setMaxWorkspaceSize()

◆ setMinTimingIterations()

◆ setProfileStream()

◆ setProfilingVerbosity()

◆ setQuantizationFlag()

◆ setQuantizationFlags()

◆ setTacticSources()