Go to the documentation of this file.
50 #ifndef NV_INFER_RUNTIME_H
51 #define NV_INFER_RUNTIME_H
65 class IExecutionContext;
119 kSAFE_GPU TRT_DEPRECATED_ENUM =
kSAFETY,
136 static constexpr int32_t kVALUE = 3;
178 return mImpl->data();
182 std::size_t
size() const noexcept
184 return mImpl->size();
190 return mImpl->type();
254 static constexpr int32_t kVALUE = 2;
276 return mImpl->isConstant();
283 return mImpl->getConstantValue();
314 return mImpl->constant(value);
322 return mImpl->operation(op, first, second);
413 static constexpr int32_t kFORMAT_COMBINATION_LIMIT = 100;
447 virtual bool supportsFormatCombination(
448 int32_t pos,
const PluginTensorDesc* inOut, int32_t nbInputs, int32_t nbOutputs) noexcept
503 int32_t nbOutputs)
const noexcept
519 const void*
const* inputs,
void*
const* outputs,
void* workspace,
cudaStream_t stream) noexcept
541 bool const*,
PluginFormat, int32_t) noexcept
override final
550 Dims getOutputDimensions(int32_t,
Dims const*, int32_t) noexcept
override final
555 bool isOutputBroadcastAcrossBatch(int32_t,
bool const*, int32_t)
const noexcept
override final
560 bool canBroadcastInputAcrossBatch(int32_t)
const noexcept
override final
565 size_t getWorkspaceSize(int32_t)
const noexcept
override final
570 int32_t enqueue(int32_t,
const void*
const*,
void*
const*,
void*,
cudaStream_t) noexcept
override final
595 virtual void reportLayerTime(
const char* layerName,
float ms) noexcept = 0;
651 virtual ~
IRuntime() noexcept =
default;
669 const void* blob, std::size_t size, IPluginFactory* pluginFactory) noexcept
671 return mImpl->deserializeCudaEngine(blob, size,
nullptr);
683 mImpl->setDLACore(dlaCore);
694 return mImpl->getDLACore();
702 return mImpl->getNbDLACores();
728 mImpl->setGpuAllocator(allocator);
747 mImpl->setErrorRecorder(recorder);
762 return mImpl->getErrorRecorder();
775 return mImpl->deserializeCudaEngine(blob, size,
nullptr);
806 return mImpl->setWeights(layerName, role, weights);
821 return mImpl->refitCudaEngine();
842 return mImpl->getMissing(size, layerNames, roles);
859 return mImpl->getAll(size, layerNames, roles);
886 return mImpl->setDynamicRange(tensorName, min, max);
898 return mImpl->getDynamicRangeMin(tensorName);
910 return mImpl->getDynamicRangeMax(tensorName);
926 return mImpl->getTensorsWithDynamicRange(size, tensorNames);
945 mImpl->setErrorRecorder(recorder);
960 return mImpl->getErrorRecorder();
978 return mImpl->setNamedWeights(name, weights);
998 return mImpl->getMissingWeights(size, weightsNames);
1014 return mImpl->getAllWeights(size, weightsNames);
1040 constexpr
inline int32_t EnumMax<OptProfileSelector>() noexcept
1097 return mImpl->setDimensions(inputName, select, dims);
1107 return mImpl->getDimensions(inputName, select);
1149 const char* inputName,
OptProfileSelector select,
const int32_t* values, int32_t nbValues) noexcept
1151 return mImpl->setShapeValues(inputName, select, values, nbValues);
1162 return mImpl->getNbShapeValues(inputName);
1172 return mImpl->getShapeValues(inputName, select);
1190 return mImpl->setExtraMemoryTarget(target);
1198 return mImpl->getExtraMemoryTarget();
1214 return mImpl->isValid();
1275 return mImpl->getNbBindings();
1297 return mImpl->getBindingIndex(name);
1317 return mImpl->getBindingName(bindingIndex);
1330 return mImpl->bindingIsInput(bindingIndex);
1355 return mImpl->getBindingDimensions(bindingIndex);
1368 return mImpl->getBindingDataType(bindingIndex);
1380 return mImpl->getMaxBatchSize();
1394 return mImpl->getNbLayers();
1408 return mImpl->serialize();
1424 return mImpl->createExecutionContext();
1451 return mImpl->getLocation(bindingIndex);
1460 return mImpl->createExecutionContextWithoutDeviceMemory();
1470 return mImpl->getDeviceMemorySize();
1480 return mImpl->isRefittable();
1494 return mImpl->getBindingBytesPerComponent(bindingIndex);
1508 return mImpl->getBindingComponentsPerElement(bindingIndex);
1518 return mImpl->getBindingFormat(bindingIndex);
1537 return mImpl->getBindingFormatDesc(bindingIndex);
1549 return mImpl->getBindingVectorizedDim(bindingIndex);
1564 return mImpl->getName();
1575 return mImpl->getNbOptimizationProfiles();
1602 return mImpl->getProfileDimensions(bindingIndex, profileIndex, select);
1629 return mImpl->getProfileShapeValues(profileIndex, inputIndex, select);
1665 return mImpl->isShapeBinding(bindingIndex);
1679 return mImpl->isExecutionBinding(bindingIndex);
1694 return mImpl->getEngineCapability();
1712 return mImpl->setErrorRecorder(recorder);
1727 return mImpl->getErrorRecorder();
1746 return mImpl->hasImplicitBatchDimension();
1755 return mImpl->getTacticSources();
1794 bool execute(int32_t batchSize,
void*
const* bindings) noexcept
1796 return mImpl->execute(batchSize, bindings);
1825 return mImpl->enqueue(batchSize, bindings, stream, inputConsumed);
1838 mImpl->setDebugSync(sync);
1848 return mImpl->getDebugSync();
1858 mImpl->setProfiler(profiler);
1868 return mImpl->getProfiler();
1878 return mImpl->getEngine();
1902 mImpl->setName(name);
1912 return mImpl->getName();
1930 mImpl->setDeviceMemory(memory);
1951 return mImpl->getStrides(bindingIndex);
1993 return mImpl->setOptimizationProfile(profileIndex);
2005 return mImpl->getOptimizationProfile();
2042 return mImpl->setBindingDimensions(bindingIndex, dimensions);
2072 return mImpl->getBindingDimensions(bindingIndex);
2102 return mImpl->setInputShapeBinding(bindingIndex, data);
2124 return mImpl->getShapeBinding(bindingIndex, data);
2139 return mImpl->allInputDimensionsSpecified();
2154 return mImpl->allInputShapesSpecified();
2173 mImpl->setErrorRecorder(recorder);
2188 return mImpl->getErrorRecorder();
2205 return mImpl->executeV2(bindings);
2233 return mImpl->enqueueV2(bindings, stream, inputConsumed);
2281 return mImpl->setOptimizationProfileAsync(profileIndex, stream);
2293 extern "C" TENSORRTAPI
void* createInferRuntime_INTERNAL(
void* logger, int32_t version) noexcept;
2299 extern "C" TENSORRTAPI
void* createInferRefitter_INTERNAL(
void* engine,
void* logger, int32_t version) noexcept;
2323 return static_cast<IRuntime*>(createInferRuntime_INTERNAL(&logger, NV_TENSORRT_VERSION));
2333 return static_cast<IRefitter*>(createInferRefitter_INTERNAL(&engine, &logger, NV_TENSORRT_VERSION));
2349 template <
typename T>
2365 #define REGISTER_TENSORRT_PLUGIN(name) \
2366 static nvinfer1::PluginRegistrar<name> pluginRegistrar##name {}
2367 #endif // NV_INFER_RUNTIME_H
int32_t getNbShapeValues(const char *inputName) const noexcept
Get the number of values for an input shape tensor.
Definition: NvInferRuntime.h:1160
bool allInputDimensionsSpecified() const noexcept
Whether all dynamic dimensions of input tensors have been specified.
Definition: NvInferRuntime.h:2137
struct CUevent_st * cudaEvent_t
Forward declaration of cudaEvent_t.
Definition: NvInferRuntimeCommon.h:112
int32_t getConstantValue() const noexcept
Definition: NvInferRuntime.h:281
Structure to define the dimensions of a tensor.
Definition: NvInferRuntime.h:347
IErrorRecorder * getErrorRecorder() const noexcept
get the ErrorRecorder assigned to this interface.
Definition: NvInferRuntime.h:760
DimensionOperation
An operation on two IDimensionExpr, which represent integer expressions used in dimension computation...
Definition: NvInferRuntime.h:218
Minimum of the two elements.
TRT_DEPRECATED void setDeviceMemory(void *memory) noexcept
Set the device memory for use by this execution context.
Definition: NvInferRuntime.h:1928
nvinfer1::ILogger * getLogger() noexcept
Return the logger object.
bool getShapeBinding(int32_t bindingIndex, int32_t *data) const noexcept
Get values of an input tensor required for shape calculations or an output tensor produced by shape c...
Definition: NvInferRuntime.h:2122
void setName(const char *name) noexcept
Set the name of the execution context.
Definition: NvInferRuntime.h:1900
int32_t getTensorRTVersion() const noexcept override
Return the API version with which this plugin was built. The upper byte reserved by TensorRT and is u...
Definition: NvInferRuntime.h:530
An array of weights used as a layer parameter.
Definition: NvInferRuntime.h:152
size_t getDeviceMemorySize() const noexcept
Return the amount of device memory required by an execution context.
Definition: NvInferRuntime.h:1468
Register the plugin creator to the registry The static registry object will be instantiated when the ...
Definition: NvInferRuntime.h:2350
nvinfer1::IPluginRegistry * getPluginRegistry() noexcept
Return the plugin registry.
bool isConstant() const noexcept
Return true if expression is a build-time constant.
Definition: NvInferRuntime.h:274
Class to handle library allocated memory that is accessible to the user.
Definition: NvInferRuntime.h:170
void * data() const noexcept
A pointer to the raw data that is owned by the library.
Definition: NvInferRuntime.h:176
TRT_DEPRECATED void destroy() noexcept
Definition: NvInferRuntime.h:199
DeviceType
The device that this layer/network will execute on.
Definition: NvInferRuntime.h:628
float getDynamicRangeMin(const char *tensorName) const noexcept
Get minimum of dynamic range.
Definition: NvInferRuntime.h:896
IErrorRecorder * getErrorRecorder() const noexcept
Get the ErrorRecorder assigned to this interface.
Definition: NvInferRuntime.h:2186
float getDynamicRangeMax(const char *tensorName) const noexcept
Get maximum of dynamic range.
Definition: NvInferRuntime.h:908
IHostMemory * serialize() const noexcept
Serialize the network to a stream.
Definition: NvInferRuntime.h:1406
Declaration of EnumMaxImpl struct to store maximum number of elements in an enumeration type.
Definition: NvInferRuntimeCommon.h:141
constexpr int32_t EnumMax< WeightsRole >() noexcept
Maximum number of elements in WeightsRole enum.
Definition: NvInferRuntime.h:618
Dims getBindingDimensions(int32_t bindingIndex) const noexcept
Get the dimensions of a binding.
Definition: NvInferRuntime.h:1353
Updates weights in an engine.
Definition: NvInferRuntime.h:789
Definition: NvInferRuntimeCommon.h:194
TacticSource
List of tactic sources for TensorRT.
Definition: NvInferRuntime.h:1229
int32_t const * getShapeValues(const char *inputName, OptProfileSelector select) const noexcept
Get the minimum / optimum / maximum values for an input shape tensor.
Definition: NvInferRuntime.h:1170
kernel for IConvolutionLayer, IDeconvolutionLayer, or IFullyConnectedLayer
Fail with error when the coordinates are out of bounds. This is the default.
Dims getProfileDimensions(int32_t bindingIndex, int32_t profileIndex, OptProfileSelector select) const noexcept
Get the minimum / optimum / maximum dimensions for a particular binding under an optimization profile...
Definition: NvInferRuntime.h:1600
TensorLocation getLocation(int32_t bindingIndex) const noexcept
Get location of binding.
Definition: NvInferRuntime.h:1449
const int32_t * getProfileShapeValues(int32_t profileIndex, int32_t inputIndex, OptProfileSelector select) const noexcept
Get minimum / optimum / maximum values for an input shape binding under an optimization profile.
Definition: NvInferRuntime.h:1626
TRT_DEPRECATED void destroy() noexcept
Destroy this object;.
Definition: NvInferRuntime.h:1434
bool setDimensions(const char *inputName, OptProfileSelector select, Dims dims) noexcept
Set the minimum / optimum / maximum dimensions for a dynamic input tensor.
Definition: NvInferRuntime.h:1095
bool setDynamicRange(const char *tensorName, float min, float max) noexcept
Definition: NvInferRuntime.h:884
TensorFormat
Format of the input/output tensors.
Definition: NvInferRuntimeCommon.h:225
TRT_DEPRECATED void destroy() noexcept
Destroy this object.
Definition: NvInferRuntime.h:712
int64_t count
The number of weights in the array.
Definition: NvInferRuntime.h:157
Single registration point for all plugins in an application. It is used to find plugin implementation...
Definition: NvInferRuntimeCommon.h:1007
bool enqueueV2(void *const *bindings, cudaStream_t stream, cudaEvent_t *inputConsumed) noexcept
Asynchronously execute inference.
Definition: NvInferRuntime.h:2231
Application-implemented logging interface for the builder, engine and runtime.
Definition: NvInferRuntimeCommon.h:1194
Definition: NvInferImpl.h:246
An engine for executing inference on a built network, with functionally unsafe features.
Definition: NvInferRuntime.h:1258
int32_t getBindingComponentsPerElement(int32_t bindingIndex) const noexcept
Return the number of components included in one element.
Definition: NvInferRuntime.h:1506
DataType type() const noexcept
The type of the memory that was allocated.
Definition: NvInferRuntime.h:188
bool refitCudaEngine() noexcept
Updates associated engine. Return true if successful.
Definition: NvInferRuntime.h:819
IRuntime * createInferRuntime(ILogger &logger) noexcept
Create an instance of an IRuntime class.
Definition: NvInferRuntime.h:2321
nvinfer1::ICudaEngine * deserializeCudaEngine(const void *blob, std::size_t size) noexcept
Deserialize an engine from a stream.
Definition: NvInferRuntime.h:773
Definition: NvInferImpl.h:190
int32_t uint32_t TacticSources
Represents a collection of one or more TacticSource values combine using bitwise-OR operations.
Definition: NvInferImpl.h:152
bool isValid() const noexcept
Check whether the optimization profile can be passed to an IBuilderConfig object.
Definition: NvInferRuntime.h:1212
bool enqueue(int32_t batchSize, void *const *bindings, cudaStream_t stream, cudaEvent_t *inputConsumed) noexcept
Asynchronously execute inference on a batch.
Definition: NvInferRuntime.h:1823
bool getDebugSync() const noexcept
Get the debug sync flag.
Definition: NvInferRuntime.h:1846
int32_t getAll(int32_t size, const char **layerNames, WeightsRole *roles) noexcept
Get description of all weights that could be refit.
Definition: NvInferRuntime.h:857
bool execute(int32_t batchSize, void *const *bindings) noexcept
Synchronously execute inference on a batch.
Definition: NvInferRuntime.h:1794
TRT_DEPRECATED nvinfer1::ICudaEngine * deserializeCudaEngine(const void *blob, std::size_t size, IPluginFactory *pluginFactory) noexcept
Deserialize an engine from a stream.
Definition: NvInferRuntime.h:668
void setErrorRecorder(IErrorRecorder *recorder) noexcept
Set the ErrorRecorder for this interface.
Definition: NvInferRuntime.h:1710
std::size_t size() const noexcept
The size in bytes of the data that was allocated.
Definition: NvInferRuntime.h:182
constexpr int32_t EnumMax< TacticSource >() noexcept
Maximum number of tactic sources in TacticSource enum.
Definition: NvInferRuntime.h:1238
int32_t nbDims
The number of dimensions.
Definition: NvInferRuntime.h:338
bool isExecutionBinding(int32_t bindingIndex) const noexcept
True if pointer to tensor data is required for execution phase, false if nullptr can be supplied.
Definition: NvInferRuntime.h:1677
The TensorRT API version 1 namespace.
bool setShapeValues(const char *inputName, OptProfileSelector select, const int32_t *values, int32_t nbValues) noexcept
Set the minimum / optimum / maximum values for an input shape tensor.
Definition: NvInferRuntime.h:1148
constexpr int32_t EnumMax< DeviceType >() noexcept
Maximum number of elements in DeviceType enum.
Definition: NvInferRuntime.h:636
constexpr int32_t EnumMax< DimensionOperation >() noexcept
Maximum number of elements in DimensionOperation enum.
Definition: NvInferRuntime.h:233
bool executeV2(void *const *bindings) noexcept
Synchronously execute inference a network.
Definition: NvInferRuntime.h:2203
int32_t getTensorsWithDynamicRange(int32_t size, const char **tensorNames) const noexcept
Get names of all tensors that have refittable dynamic ranges.
Definition: NvInferRuntime.h:924
Definition: NvInferRuntime.h:377
void setGpuAllocator(IGpuAllocator *allocator) noexcept
Set the GPU allocator.
Definition: NvInferRuntime.h:726
int32_t getNbDLACores() const noexcept
Returns number of DLA hardware cores accessible.
Definition: NvInferRuntime.h:700
float getExtraMemoryTarget() const noexcept
Get the extra memory target that has been defined for this profile.
Definition: NvInferRuntime.h:1196
int32_t getNbOptimizationProfiles() const noexcept
Get the number of optimization profiles defined for this engine.
Definition: NvInferRuntime.h:1573
This is used to set or get the value that is used in the optimization (kernel selection).
const IDimensionExpr * constant(int32_t value) noexcept
Return pointer to IDimensionExp for given value.
Definition: NvInferRuntime.h:312
IErrorRecorder * getErrorRecorder() const noexcept
Get the ErrorRecorder assigned to this interface.
Definition: NvInferRuntime.h:1725
DataType getBindingDataType(int32_t bindingIndex) const noexcept
Determine the required data type for a buffer from its binding index.
Definition: NvInferRuntime.h:1366
Dims max
Upper bounds on tensor’s dimensions.
Definition: NvInferRuntime.h:356
Check if two elements are equal.
void setErrorRecorder(IErrorRecorder *recorder) noexcept
Set the ErrorRecorder for this interface.
Definition: NvInferRuntime.h:2171
TRT_DEPRECATED bool setOptimizationProfile(int32_t profileIndex) noexcept
Select an optimization profile for the current context.
Definition: NvInferRuntime.h:1991
void setProfiler(IProfiler *profiler) noexcept
Set the profiler.
Definition: NvInferRuntime.h:1856
int32_t getNbLayers() const noexcept
Get the number of layers in the network.
Definition: NvInferRuntime.h:1392
DataType type
The type of the weights.
Definition: NvInferRuntime.h:155
bias for IConvolutionLayer, IDeconvolutionLayer, or IFullyConnectedLayer
int32_t getBindingBytesPerComponent(int32_t bindingIndex) const noexcept
Return the number of bytes per component of an element.
Definition: NvInferRuntime.h:1492
Definition: NvInferImpl.h:199
Dims min
Lower bounds on tensor’s dimensions.
Definition: NvInferRuntime.h:353
int32_t getBindingIndex(const char *name) const noexcept
Retrieve the binding index for a named tensor.
Definition: NvInferRuntime.h:1295
int32_t getNbBindings() const noexcept
Get the number of binding indices.
Definition: NvInferRuntime.h:1273
void setDLACore(int32_t dlaCore) noexcept
Set the DLA core that the deserialized engine must execute on.
Definition: NvInferRuntime.h:681
const ICudaEngine & getEngine() const noexcept
Get the associated engine.
Definition: NvInferRuntime.h:1876
IErrorRecorder * getErrorRecorder() const noexcept
Get the ErrorRecorder assigned to this interface.
Definition: NvInferRuntime.h:958
shift part of IScaleLayer
Optimization profile for dynamic input dimensions and shape tensors.
Definition: NvInferRuntime.h:1067
IExecutionContext * createExecutionContextWithoutDeviceMemory() noexcept
create an execution context without any device memory allocated
Definition: NvInferRuntime.h:1458
Definition: NvInferImpl.h:231
DataType
The type of weights and tensors.
Definition: NvInferRuntimeCommon.h:155
TacticSources getTacticSources() const noexcept
return the tactic sources required by this engine
Definition: NvInferRuntime.h:1753
int32_t getBindingVectorizedDim(int32_t bindingIndex) const noexcept
Return the dimension index that the buffer is vectorized.
Definition: NvInferRuntime.h:1547
Definition: NvInferRuntime.h:335
bool setOptimizationProfileAsync(int32_t profileIndex, cudaStream_t stream) noexcept
Select an optimization profile for the current context with async semantics.
Definition: NvInferRuntime.h:2279
Floor division of the first element by the second.
PluginTensorDesc desc
Information required to interpret a pointer to tensor data, except that desc.dims has -1 in place of ...
Definition: NvInferRuntime.h:350
Definition: NvInferImpl.h:175
Substract the second element from the first.
Forward declaration of IPluginFactory for use by other interfaces.
Definition: NvInferRuntime.h:78
const char * getName() const noexcept
Returns the name of the network associated with the engine.
Definition: NvInferRuntime.h:1562
TensorFormat getBindingFormat(int32_t bindingIndex) const noexcept
Return the binding format.
Definition: NvInferRuntime.h:1516
Reference counted application-implemented error reporting interface for TensorRT objects.
Definition: NvInferRuntimeCommon.h:1373
IProfiler * getProfiler() const noexcept
Get the profiler.
Definition: NvInferRuntime.h:1866
int32_t getAllWeights(int32_t size, const char **weightsNames) noexcept
Get names of all weights that could be refit.
Definition: NvInferRuntime.h:1012
OptProfileSelector
When setting or querying optimization profile parameters (such as shape tensor inputs or dynamic dime...
Definition: NvInferRuntime.h:1031
WeightsRole
How a layer uses particular Weights.
Definition: NvInferRuntime.h:606
virtual bool registerCreator(IPluginCreator &creator, AsciiChar const *const pluginNamespace) noexcept=0
Register a plugin creator. Returns false if one with same type is already registered.
bool isShapeBinding(int32_t bindingIndex) const noexcept
True if tensor is required as input for shape calculations or output from them.
Definition: NvInferRuntime.h:1663
int32_t getMaxBatchSize() const noexcept
Get the maximum batch size which can be used for inference.
Definition: NvInferRuntime.h:1378
const char * getBindingFormatDesc(int32_t bindingIndex) const noexcept
Return the human readable description of the tensor format.
Definition: NvInferRuntime.h:1535
void setErrorRecorder(IErrorRecorder *recorder) noexcept
Set the ErrorRecorder for this interface.
Definition: NvInferRuntime.h:745
Plugin class for user-implemented layers.
Definition: NvInferRuntimeCommon.h:608
bool setInputShapeBinding(int32_t bindingIndex, int32_t const *data) noexcept
Set values of input tensor required by shape calculations.
Definition: NvInferRuntime.h:2100
static constexpr int32_t MAX_DIMS
The maximum number of dimensions supported for a tensor.
Definition: NvInferRuntimeCommon.h:198
Definition: NvInferRuntime.h:270
Dims getBindingDimensions(int32_t bindingIndex) const noexcept
Get the dynamic dimensions of a binding.
Definition: NvInferRuntime.h:2070
Product of the two elements.
EngineCapability
List of supported engine capability flows.
Definition: NvInferRuntime.h:104
bool hasImplicitBatchDimension() const noexcept
Query whether the engine was built with an implicit batch dimension.
Definition: NvInferRuntime.h:1744
Application-implemented interface for profiling.
Definition: NvInferRuntime.h:586
Check if element in first tensor is less than corresponding element in second tensor.
bool bindingIsInput(int32_t bindingIndex) const noexcept
Determine whether a binding is an input binding.
Definition: NvInferRuntime.h:1328
struct CUstream_st * cudaStream_t
Forward declaration of cudaStream_t.
Definition: NvInferRuntimeCommon.h:109
bool setExtraMemoryTarget(float target) noexcept
Set a target for extra GPU memory that may be used by this profile.
Definition: NvInferRuntime.h:1188
bool allInputShapesSpecified() const noexcept
Whether all input shape bindings have been specified.
Definition: NvInferRuntime.h:2151
Fields that a plugin might see for an input or output.
Definition: NvInferRuntimeCommon.h:373
TRT_DEPRECATED void destroy() noexcept
Definition: NvInferRuntime.h:867
const void * values
The weight values, in a contiguous array.
Definition: NvInferRuntime.h:156
Definition: NvInferImpl.h:183
TensorFormat PluginFormat
PluginFormat is reserved for backward compatibility.
Definition: NvInferRuntimeCommon.h:350
int32_t getDLACore() const noexcept
Get the DLA core that the engine executes on.
Definition: NvInferRuntime.h:692
Definition: NvInferImpl.h:285
const char * getBindingName(int32_t bindingIndex) const noexcept
Retrieve the name corresponding to a binding index.
Definition: NvInferRuntime.h:1315
bool setNamedWeights(const char *name, Weights weights) noexcept
Specify new weights of given name.
Definition: NvInferRuntime.h:976
const char * getName() const noexcept
Return the name of the execution context.
Definition: NvInferRuntime.h:1910
Application-implemented class for controlling allocation on the GPU.
Definition: NvInferRuntimeCommon.h:1103
Dims getDimensions(const char *inputName, OptProfileSelector select) const noexcept
Get the minimum / optimum / maximum dimensions for a dynamic input tensor.
Definition: NvInferRuntime.h:1105
IRefitter * createInferRefitter(ICudaEngine &engine, ILogger &logger) noexcept
Create an instance of an IRefitter class.
Definition: NvInferRuntime.h:2331
bool isRefittable() const noexcept
Return true if an engine can be refit.
Definition: NvInferRuntime.h:1478
int32_t getMissing(int32_t size, const char **layerNames, WeightsRole *roles) noexcept
Get description of missing weights.
Definition: NvInferRuntime.h:840
Allows a serialized functionally unsafe engine to be deserialized.
Definition: NvInferRuntime.h:648
void setDebugSync(bool sync) noexcept
Set the debug sync flag.
Definition: NvInferRuntime.h:1836
EngineCapability getEngineCapability() const noexcept
Determine what execution capability this engine has.
Definition: NvInferRuntime.h:1692
Definition: NvInferImpl.h:213
TensorLocation
The location for tensor data storage, device or host.
Definition: NvInferRuntime.h:242
Context for executing inference using an engine, with functionally unsafe features.
Definition: NvInferRuntime.h:1772
int32_t getMissingWeights(int32_t size, const char **weightsNames) noexcept
Get names of missing weights.
Definition: NvInferRuntime.h:996
TRT_DEPRECATED void destroy() noexcept
Destroy this object.
Definition: NvInferRuntime.h:1888
const IDimensionExpr * operation(DimensionOperation op, const IDimensionExpr &first, const IDimensionExpr &second) noexcept
Definition: NvInferRuntime.h:319
Dims getStrides(int32_t bindingIndex) const noexcept
Return the strides of the buffer for the given binding.
Definition: NvInferRuntime.h:1949
#define TRT_DEPRECATED
< Items that are marked as deprecated will be removed in a future release.
Definition: NvInferRuntimeCommon.h:76
bool setWeights(const char *layerName, WeightsRole role, Weights weights) noexcept
Specify new weights for a layer of given name. Returns true on success, or false if new weights are r...
Definition: NvInferRuntime.h:804
int32_t getOptimizationProfile() const noexcept
Get the index of the currently selected optimization profile.
Definition: NvInferRuntime.h:2003
void setErrorRecorder(IErrorRecorder *recorder) noexcept
Set the ErrorRecorder for this interface.
Definition: NvInferRuntime.h:943
Definition: NvInferRuntime.h:308
IExecutionContext * createExecutionContext() noexcept
Create an execution context.
Definition: NvInferRuntime.h:1422
bool setBindingDimensions(int32_t bindingIndex, Dims dimensions) noexcept
Set the dynamic dimensions of a binding.
Definition: NvInferRuntime.h:2040
Copyright © 2024 NVIDIA Corporation
Privacy Policy |
Manage My Privacy |
Do Not Sell or Share My Data |
Terms of Service |
Accessibility |
Corporate Policies |
Product Security |
Contact