TensorRT 8.6.1
|
Holds properties for configuring a builder to produce an engine. More...
#include <NvInfer.h>
Public Member Functions | |
virtual | ~IBuilderConfig () noexcept=default |
virtual TRT_DEPRECATED void | setMinTimingIterations (int32_t minTiming) noexcept |
Set the number of minimization iterations used when timing layers. More... | |
virtual TRT_DEPRECATED int32_t | getMinTimingIterations () const noexcept |
Query the number of minimization iterations. More... | |
virtual void | setAvgTimingIterations (int32_t avgTiming) noexcept |
Set the number of averaging iterations used when timing layers. More... | |
int32_t | getAvgTimingIterations () const noexcept |
Query the number of averaging iterations. More... | |
void | setEngineCapability (EngineCapability capability) noexcept |
Configure the builder to target specified EngineCapability flow. More... | |
EngineCapability | getEngineCapability () const noexcept |
Query EngineCapability flow configured for the builder. More... | |
void | setInt8Calibrator (IInt8Calibrator *calibrator) noexcept |
Set Int8 Calibration interface. More... | |
IInt8Calibrator * | getInt8Calibrator () const noexcept |
Get Int8 Calibration interface. More... | |
TRT_DEPRECATED void | setMaxWorkspaceSize (std::size_t workspaceSize) noexcept |
Set the maximum workspace size. More... | |
TRT_DEPRECATED std::size_t | getMaxWorkspaceSize () const noexcept |
Get the maximum workspace size. More... | |
void | setFlags (BuilderFlags builderFlags) noexcept |
Set the build mode flags to turn on builder options for this network. More... | |
BuilderFlags | getFlags () const noexcept |
Get the build mode flags for this builder config. Defaults to 0. More... | |
void | clearFlag (BuilderFlag builderFlag) noexcept |
clear a single build mode flag. More... | |
void | setFlag (BuilderFlag builderFlag) noexcept |
Set a single build mode flag. More... | |
bool | getFlag (BuilderFlag builderFlag) const noexcept |
Returns true if the build mode flag is set. More... | |
void | setDeviceType (ILayer const *layer, DeviceType deviceType) noexcept |
Set the device that this layer must execute on. More... | |
DeviceType | getDeviceType (ILayer const *layer) const noexcept |
Get the device that this layer executes on. More... | |
bool | isDeviceTypeSet (ILayer const *layer) const noexcept |
whether the DeviceType has been explicitly set for this layer More... | |
void | resetDeviceType (ILayer const *layer) noexcept |
reset the DeviceType for this layer More... | |
bool | canRunOnDLA (ILayer const *layer) const noexcept |
Checks if a layer can run on DLA. More... | |
void | setDLACore (int32_t dlaCore) noexcept |
Sets the DLA core used by the network. Defaults to -1. More... | |
int32_t | getDLACore () const noexcept |
Get the DLA core that the engine executes on. More... | |
void | setDefaultDeviceType (DeviceType deviceType) noexcept |
Sets the default DeviceType to be used by the builder. It ensures that all the layers that can run on this device will run on it, unless setDeviceType is used to override the default DeviceType for a layer. More... | |
DeviceType | getDefaultDeviceType () const noexcept |
Get the default DeviceType which was set by setDefaultDeviceType. More... | |
void | reset () noexcept |
Resets the builder configuration to defaults. More... | |
TRT_DEPRECATED void | destroy () noexcept |
Delete this IBuilderConfig. More... | |
void | setProfileStream (const cudaStream_t stream) noexcept |
Set the cuda stream that is used to profile this network. More... | |
cudaStream_t | getProfileStream () const noexcept |
Get the cuda stream that is used to profile this network. More... | |
int32_t | addOptimizationProfile (IOptimizationProfile const *profile) noexcept |
Add an optimization profile. More... | |
int32_t | getNbOptimizationProfiles () const noexcept |
Get number of optimization profiles. More... | |
void | setProfilingVerbosity (ProfilingVerbosity verbosity) noexcept |
Set verbosity level of layer information exposed in NVTX annotations and IEngineInspector. More... | |
ProfilingVerbosity | getProfilingVerbosity () const noexcept |
Get verbosity level of layer information exposed in NVTX annotations and IEngineInspector. More... | |
void | setAlgorithmSelector (IAlgorithmSelector *selector) noexcept |
Set Algorithm Selector. More... | |
IAlgorithmSelector * | getAlgorithmSelector () const noexcept |
Get Algorithm Selector. More... | |
bool | setCalibrationProfile (IOptimizationProfile const *profile) noexcept |
Add a calibration profile. More... | |
IOptimizationProfile const * | getCalibrationProfile () noexcept |
Get the current calibration profile. More... | |
void | setQuantizationFlags (QuantizationFlags flags) noexcept |
Set the quantization flags. More... | |
QuantizationFlags | getQuantizationFlags () const noexcept |
Get the quantization flags. More... | |
void | clearQuantizationFlag (QuantizationFlag flag) noexcept |
clear a quantization flag. More... | |
void | setQuantizationFlag (QuantizationFlag flag) noexcept |
Set a single quantization flag. More... | |
bool | getQuantizationFlag (QuantizationFlag flag) const noexcept |
Returns true if the quantization flag is set. More... | |
bool | setTacticSources (TacticSources tacticSources) noexcept |
Set tactic sources. More... | |
TacticSources | getTacticSources () const noexcept |
Get tactic sources. More... | |
nvinfer1::ITimingCache * | createTimingCache (void const *blob, std::size_t size) const noexcept |
Create timing cache. More... | |
bool | setTimingCache (ITimingCache const &cache, bool ignoreMismatch) noexcept |
Attach a timing cache to IBuilderConfig. More... | |
nvinfer1::ITimingCache const * | getTimingCache () const noexcept |
Get the pointer to the timing cache from current IBuilderConfig. More... | |
void | setMemoryPoolLimit (MemoryPoolType pool, std::size_t poolSize) noexcept |
Set the memory size for the memory pool. More... | |
std::size_t | getMemoryPoolLimit (MemoryPoolType pool) const noexcept |
Get the memory size limit of the memory pool. More... | |
void | setPreviewFeature (PreviewFeature feature, bool enable) noexcept |
Enable or disable a specific preview feature. More... | |
bool | getPreviewFeature (PreviewFeature feature) const noexcept |
Get status of preview feature. More... | |
void | setBuilderOptimizationLevel (int32_t level) noexcept |
Set builder optimization level. More... | |
int32_t | getBuilderOptimizationLevel () noexcept |
Get builder optimization level. More... | |
void | setHardwareCompatibilityLevel (HardwareCompatibilityLevel hardwareCompatibilityLevel) noexcept |
Set the hardware compatibility level. More... | |
HardwareCompatibilityLevel | getHardwareCompatibilityLevel () const noexcept |
Get the hardware compatibility level. More... | |
void | setPluginsToSerialize (char const *const *paths, int32_t nbPaths) noexcept |
Set the plugin libraries to be serialized with version-compatible engines. More... | |
char const * | getPluginToSerialize (int32_t index) const noexcept |
Get the plugin library path to be serialized with version-compatible engines. More... | |
int32_t | getNbPluginsToSerialize () const noexcept |
Get the number of plugin library paths to be serialized with version-compatible engines. More... | |
void | setMaxAuxStreams (int32_t nbStreams) noexcept |
Set the maximum number of auxiliary streams that TRT is allowed to use. More... | |
int32_t | getMaxAuxStreams () const noexcept |
Get the maximum number of auxiliary streams that TRT is allowed to use. More... | |
Protected Attributes | |
apiv::VBuilderConfig * | mImpl |
Additional Inherited Members | |
![]() | |
INoCopy ()=default | |
virtual | ~INoCopy ()=default |
INoCopy (INoCopy const &other)=delete | |
INoCopy & | operator= (INoCopy const &other)=delete |
INoCopy (INoCopy &&other)=delete | |
INoCopy & | operator= (INoCopy &&other)=delete |
Holds properties for configuring a builder to produce an engine.
|
virtualdefaultnoexcept |
|
inlinenoexcept |
Add an optimization profile.
This function must be called at least once if the network has dynamic or shape input tensors. This function may be called at most once when building a refittable engine, as more than a single optimization profile are not supported for refittable engines.
profile | The new optimization profile, which must satisfy profile->isValid() == true |
|
inlinenoexcept |
Checks if a layer can run on DLA.
|
inlinenoexcept |
clear a single build mode flag.
clears the builder mode flag from the enabled flags.
|
inlinenoexcept |
clear a quantization flag.
Clears the quantization flag from the enabled quantization flags.
|
inlinenoexcept |
Create timing cache.
Create ITimingCache instance from serialized raw data. The created timing cache doesn’t belong to a specific IBuilderConfig. It can be shared by multiple builder instances. Call setTimingCache() before launching a builder to attach cache to builder instance.
blob | A pointer to the raw data that contains serialized timing cache |
size | The size in bytes of the serialized timing cache. Size 0 means create a new cache from scratch |
|
inlinenoexcept |
Delete this IBuilderConfig.
De-allocates any internally allocated memory.
delete
.
|
inlinenoexcept |
Get Algorithm Selector.
|
inlinenoexcept |
Query the number of averaging iterations.
By default the number of averaging iterations is 1.
|
inlinenoexcept |
Get builder optimization level.
|
inlinenoexcept |
Get the current calibration profile.
|
inlinenoexcept |
Get the default DeviceType which was set by setDefaultDeviceType.
By default it returns DeviceType::kGPU.
|
inlinenoexcept |
Get the device that this layer executes on.
|
inlinenoexcept |
Get the DLA core that the engine executes on.
|
inlinenoexcept |
Query EngineCapability flow configured for the builder.
By default it returns EngineCapability::kSTANDARD.
|
inlinenoexcept |
Returns true if the build mode flag is set.
|
inlinenoexcept |
Get the build mode flags for this builder config. Defaults to 0.
|
inlinenoexcept |
Get the hardware compatibility level.
|
inlinenoexcept |
Get Int8 Calibration interface.
|
inlinenoexcept |
Get the maximum number of auxiliary streams that TRT is allowed to use.
|
inlinenoexcept |
Get the maximum workspace size.
By default the workspace size is the size of total global memory in the device.
|
inlinenoexcept |
Get the memory size limit of the memory pool.
Retrieve the memory size limit of the corresponding pool in bytes. If setMemoryPoolLimit for the pool has not been called, this returns the default value used by TensorRT. This default value is not necessarily the maximum possible value for that configuration.
pool | The memory pool to get the limit for. |
|
inlinevirtualnoexcept |
Query the number of minimization iterations.
By default the minimum number of iterations is 1.
|
inlinenoexcept |
Get number of optimization profiles.
This is one higher than the index of the last optimization profile that has be defined (or zero, if none has been defined yet).
|
inlinenoexcept |
Get the number of plugin library paths to be serialized with version-compatible engines.
|
inlinenoexcept |
Get the plugin library path to be serialized with version-compatible engines.
index | Index of the plugin library path in the list. Should be in the range [0, getNbPluginsToSerialize()) . |
|
inlinenoexcept |
Get status of preview feature.
feature | the feature to query |
feature
is enabled, false otherwise
|
inlinenoexcept |
Get the cuda stream that is used to profile this network.
|
inlinenoexcept |
Get verbosity level of layer information exposed in NVTX annotations and IEngineInspector.
Get the current setting of verbosity level of layer information exposed in NVTX annotations and IEngineInspector. Default value is ProfilingVerbosity::kLAYER_NAMES_ONLY.
|
inlinenoexcept |
Returns true if the quantization flag is set.
|
inlinenoexcept |
Get the quantization flags.
|
inlinenoexcept |
Get tactic sources.
Get the tactic sources currently set in the engine build configuration.
|
inlinenoexcept |
Get the pointer to the timing cache from current IBuilderConfig.
|
inlinenoexcept |
whether the DeviceType has been explicitly set for this layer
|
inlinenoexcept |
Resets the builder configuration to defaults.
Useful for initializing a builder config object to its original state.
|
inlinenoexcept |
reset the DeviceType for this layer
|
inlinenoexcept |
Set Algorithm Selector.
selector | The algorithm selector to be set in the build config. |
|
inlinevirtualnoexcept |
Set the number of averaging iterations used when timing layers.
When timing layers, the builder minimizes over a set of average times for layer execution. This parameter controls the number of iterations used in averaging.
|
inlinenoexcept |
Set builder optimization level.
Set the builder optimization level. Setting a higher optimization level allows the optimizer to spend more time searching for optimization opportunities. The resulting engine may have better performance compared to an engine built with a lower optimization level.
The default optimization level is 3. Valid values include integers from 0 to the maximum optimization level, which is currently 5. Setting it to greater than the maximum level results in behavior identical to the maximum level.
level | The optimization level to set to. Must be non-negative. |
|
inlinenoexcept |
Add a calibration profile.
Calibration optimization profile must be set if int8 calibration is used to set scales for a network with runtime dimensions.
profile | The new calibration profile, which must satisfy profile->isValid() == true or be nullptr. MIN and MAX values will be overwritten by kOPT. |
|
inlinenoexcept |
Sets the default DeviceType to be used by the builder. It ensures that all the layers that can run on this device will run on it, unless setDeviceType is used to override the default DeviceType for a layer.
|
inlinenoexcept |
Set the device that this layer must execute on.
layer | which layer to execute. |
deviceType | that this layer must execute on. If DeviceType is not set or is reset, TensorRT will use the default DeviceType set in the builder. |
|
inlinenoexcept |
Sets the DLA core used by the network. Defaults to -1.
dlaCore | The DLA core to execute the engine on, in the range [0,getNbDlaCores()). |
This function is used to specify which DLA core to use via indexing, if multiple DLA cores are available.
|
inlinenoexcept |
Configure the builder to target specified EngineCapability flow.
The flow means a sequence of API calls that allow an application to set up a runtime, engine, and execution context in order to run inference.
The supported flows are specified in the EngineCapability enum.
|
inlinenoexcept |
Set a single build mode flag.
Add the input builder mode flag to the already enabled flags.
|
inlinenoexcept |
Set the build mode flags to turn on builder options for this network.
The flags are listed in the BuilderFlags enum. The flags set configuration options to build the network.
builderFlags | The build option for an engine. |
|
inlinenoexcept |
Set the hardware compatibility level.
Hardware compatibility allows an engine to run on GPU architectures other than that of the GPU where the engine was built.
The default hardware compatibility level is HardwareCompatibilityLevel::kNONE.
hardwareCompatibilityLevel | The level of hardware compatibility. |
|
inlinenoexcept |
Set Int8 Calibration interface.
The calibrator is to minimize the information loss during the INT8 quantization process.
|
inlinenoexcept |
Set the maximum number of auxiliary streams that TRT is allowed to use.
If the network contains operators that can run in parallel, TRT can execute them using auxiliary streams in addition to the one provided to the IExecutionContext::enqueueV3() call.
The default maximum number of auxiliary streams is determined by the heuristics in TensorRT on whether enabling multi-stream would improve the performance. This behavior can be overridden by calling this API to set the maximum number of auxiliary streams explicitly. Set this to 0 to enforce single-stream inference.
The resulting engine may use fewer auxiliary streams than the maximum if the network does not contain enough parallelism or if TensorRT determines that using more auxiliary streams does not help improve the performance.
nbStreams | The maximum number of auxiliary streams that TRT is allowed to use. |
|
inlinenoexcept |
Set the maximum workspace size.
workspaceSize | The maximum GPU temporary memory which the engine can use at execution time. |
|
inlinenoexcept |
Set the memory size for the memory pool.
TensorRT layers access different memory pools depending on the operation. This function sets in the IBuilderConfig the size limit, specified by poolSize
, for the corresponding memory pool, specified by pool
. TensorRT will build a plan file that is constrained by these limits or report which constraint caused the failure.
If the size of the pool, specified by poolSize
, fails to meet the size requirements for the pool, this function does nothing and emits the recoverable error, ErrorCode::kINVALID_ARGUMENT, to the registered IErrorRecorder.
If the size of the pool is larger than the maximum possible value for the configuration, this function does nothing and emits ErrorCode::kUNSUPPORTED_STATE.
If the pool does not exist on the requested device type when building the network, a warning is emitted to the logger, and the memory pool value is ignored.
Refer to MemoryPoolType to see the size requirements for each pool.
pool | The memory pool to limit the available memory for. |
poolSize | The size of the pool in bytes. |
|
inlinevirtualnoexcept |
Set the number of minimization iterations used when timing layers.
When timing layers, the builder minimizes over a set of average times for layer execution. This parameter controls the number of iterations used in minimization. The builder may sometimes run layers for more iterations to improve timing accuracy if this parameter is set to a small value and the runtime of the layer is short.
|
inlinenoexcept |
Set the plugin libraries to be serialized with version-compatible engines.
Each entry in the list of libraries must be unique.
paths | The paths of plugin libraries. |
nbPaths | The number of paths. |
|
inlinenoexcept |
Enable or disable a specific preview feature.
Allows enabling or disabling experimental features, which are not enabled by default in the current release.
Refer to PreviewFeature for additional information, and a list of the available features.
feature | the feature to enable / disable |
enable | true for enable, false for disable |
|
inlinenoexcept |
Set the cuda stream that is used to profile this network.
stream | The cuda stream used for profiling by the builder. |
|
inlinenoexcept |
Set verbosity level of layer information exposed in NVTX annotations and IEngineInspector.
Control how much layer information will be exposed in NVTX annotations and IEngineInspector.
|
inlinenoexcept |
Set a single quantization flag.
Add the input quantization flag to the already enabled quantization flags.
|
inlinenoexcept |
Set the quantization flags.
The flags are listed in the QuantizationFlag enum. The flags set configuration options to quantize the network in int8.
flags | The quantization flags. |
|
inlinenoexcept |
Set tactic sources.
This bitset controls which tactic sources TensorRT is allowed to use for tactic selection.
Multiple tactic sources may be combined with a bitwise OR operation. For example, to enable cublas and cublasLt as tactic sources, use a value of:
1U << static_cast<uint32_t>(TacticSource::kCUBLAS) | 1U << static_cast<uint32_t>(TacticSource::kCUBLAS_LT)
|
inlinenoexcept |
Attach a timing cache to IBuilderConfig.
The timing cache has verification header to make sure the provided cache can be used in current environment. A failure will be reported if the CUDA device property in the provided cache is different from current environment. ignoreMismatch = true skips strict verification and allows loading cache created from a different device.
The cache must not be destroyed until after the engine is built.
cache | the timing cache to be used |
ignoreMismatch | whether or not allow using a cache that contains different CUDA device property |
|
protected |