18#ifndef NV_INFER_RUNTIME_BASE_H
19#define NV_INFER_RUNTIME_BASE_H
24#include <cuda_runtime_api.h>
27#if __cplusplus >= 201402L
28#define TRT_DEPRECATED [[deprecated]]
30#define TRT_DEPRECATED_ENUM
32#define TRT_DEPRECATED_ENUM TRT_DEPRECATED
35#define TRT_DEPRECATED_API __declspec(dllexport)
37#define TRT_DEPRECATED_API [[deprecated]] __attribute__((visibility("default")))
42#define TRT_DEPRECATED_ENUM
43#define TRT_DEPRECATED_API __declspec(dllexport)
45#define TRT_DEPRECATED __attribute__((deprecated))
46#define TRT_DEPRECATED_ENUM
47#define TRT_DEPRECATED_API __attribute__((deprecated, visibility("default")))
52#ifdef TENSORRT_BUILD_LIB
54#define TENSORRTAPI __declspec(dllexport)
56#define TENSORRTAPI __attribute__((visibility("default")))
73#if !defined(NV_INFER_INTERNAL_INCLUDE_RUNTIME_BASE)
74static_assert(
false,
"Do not directly include this file. Include NvInferRuntime.h or NvInferSafeRuntime.h or NvInferConsistency.h or NvInferPluginUtils.h");
89#define NV_TENSORRT_VERSION_INT(major, minor, patch) ((major) *10000L + (minor) *100L + (patch) *1L)
93#define NV_TENSORRT_VERSION NV_TENSORRT_VERSION_INT(NV_TENSORRT_MAJOR, NV_TENSORRT_MINOR, NV_TENSORRT_PATCH)
187 static constexpr int32_t kVALUE = 10;
390 static constexpr int32_t kVALUE = 2;
434 static constexpr int32_t kVALUE = 13;
457 static constexpr int32_t kVALUE = 1;
496 uint64_t
const size, uint64_t
const alignment,
AllocatorFlags const flags)
noexcept = 0;
538 virtual void*
reallocate(
void*
const , uint64_t , uint64_t )
noexcept
592 uint64_t
const size, uint64_t
const alignment,
AllocatorFlags const flags, cudaStream_t )
noexcept
594 return allocate(size, alignment, flags);
626 return deallocate(memory);
634 return {
"IGpuAllocator", 1, 0};
743 static constexpr int32_t kVALUE = 5;
850 static constexpr int32_t kVALUE = 11;
877 static constexpr size_t kMAX_DESC_LENGTH{127U};
912 virtual int32_t getNbErrors() const noexcept = 0;
932 virtual
ErrorCode getErrorCode(int32_t errorIdx) const noexcept = 0;
955 virtual
ErrorDesc getErrorDesc(int32_t errorIdx) const noexcept = 0;
971 virtual
bool hasOverflowed() const noexcept = 0;
987 virtual
void clear() noexcept = 0;
1139 virtual int64_t
read(
void* destination, int64_t nbBytes) = 0;
1228 static constexpr int32_t kVALUE = 3;
#define TENSORRTAPI
Definition: NvInferRuntimeBase.h:59
int32_t getInferLibVersion() noexcept
Return the library version number.
#define TRT_DEPRECATED
Definition: NvInferRuntimeBase.h:45
Definition: NvInferRuntimeBase.h:202
static constexpr int32_t MAX_DIMS
The maximum rank (number of dimensions) supported for a tensor.
Definition: NvInferRuntimeBase.h:205
int64_t d[MAX_DIMS]
The extent of each dimension.
Definition: NvInferRuntimeBase.h:211
int32_t nbDims
The rank (number of dimensions).
Definition: NvInferRuntimeBase.h:208
Application-implemented logging interface for the builder, refitter and runtime.
Definition: NvInferRuntimeBase.h:683
virtual ~ILogger()=default
Severity
The severity corresponding to a log message.
Definition: NvInferRuntimeBase.h:691
virtual void log(Severity severity, AsciiChar const *msg) noexcept=0
A callback implemented by the application to handle logging messages;.
An Interface class for version control.
Definition: NvInferRuntimeBase.h:400
virtual InterfaceInfo getInterfaceInfo() const noexcept=0
Return version information associated with this interface. Applications must not override this method...
virtual APILanguage getAPILanguage() const noexcept
The language used to build the implementation of this Interface.
Definition: NvInferRuntimeBase.h:407
IVersionedInterface & operator=(IVersionedInterface const &) &=default
Version information associated with a TRT interface.
Definition: NvInferRuntimeBase.h:365
InterfaceKind kind
Definition: NvInferRuntimeBase.h:367
int32_t major
Definition: NvInferRuntimeBase.h:368
int32_t minor
Definition: NvInferRuntimeBase.h:369
Definition: NvInferRuntimeBase.h:857
char const * ErrorDesc
A typedef of a C-style string for reporting error descriptions.
Definition: NvInferRuntimeBase.h:870
~IErrorRecorder() noexcept override=default
int32_t RefCount
A typedef of a 32-bit integer for reference counting.
Definition: NvInferRuntimeBase.h:882
InterfaceInfo getInterfaceInfo() const noexcept override
Return version information associated with this interface. Applications must not override this method...
Definition: NvInferRuntimeBase.h:862
Definition: NvInferRuntimeBase.h:469
virtual void * allocateAsync(uint64_t const size, uint64_t const alignment, AllocatorFlags const flags, cudaStream_t) noexcept
A thread-safe callback implemented by the application to handle stream-ordered acquisition of GPU mem...
Definition: NvInferRuntimeBase.h:591
InterfaceInfo getInterfaceInfo() const noexcept override
Return version information associated with this interface. Applications must not override this method...
Definition: NvInferRuntimeBase.h:632
virtual TRT_DEPRECATED bool deallocate(void *const memory) noexcept=0
A thread-safe callback implemented by the application to handle release of GPU memory.
~IGpuAllocator() override=default
virtual void * reallocate(void *const, uint64_t, uint64_t) noexcept
A thread-safe callback implemented by the application to resize an existing allocation.
Definition: NvInferRuntimeBase.h:538
virtual TRT_DEPRECATED void * allocate(uint64_t const size, uint64_t const alignment, AllocatorFlags const flags) noexcept=0
A thread-safe callback implemented by the application to handle acquisition of GPU memory.
virtual bool deallocateAsync(void *const memory, cudaStream_t) noexcept
A thread-safe callback implemented by the application to handle stream-ordered release of GPU memory.
Definition: NvInferRuntimeBase.h:624
Definition: NvInferRuntimeBase.h:1163
virtual int32_t release() noexcept=0
Free the underlying resource.
InterfaceInfo getInterfaceInfo() const noexcept override
Return version information associated with this interface. Applications must not override this method...
Definition: NvInferRuntimeBase.h:1168
Definition: NvInferRuntimeBase.h:1114
~IStreamReader() override=default
IStreamReader & operator=(IStreamReader const &) &=default
IStreamReader & operator=(IStreamReader &&) &=default
virtual int64_t read(void *destination, int64_t nbBytes)=0
Read the next number of bytes in the stream.
IStreamReader(IStreamReader &&)=default
IStreamReader(IStreamReader const &)=default
InterfaceInfo getInterfaceInfo() const noexcept override
Return version information associated with this interface. Applications must not override this method...
Definition: NvInferRuntimeBase.h:1126
The TensorRT API version 1 namespace.
ErrorCode
Error codes that can be returned by TensorRT during execution.
Definition: NvInferRuntimeBase.h:753
TensorIOMode
Definition of tensor IO Mode.
Definition: NvInferRuntimeBase.h:1100
@ kOUTPUT
Tensor is output by the engine.
@ kINPUT
Tensor is input to the engine.
APILanguage
Programming language used in the implementation of a TRT interface.
Definition: NvInferRuntimeBase.h:378
char_t AsciiChar
Definition: NvInferRuntimeBase.h:107
char char_t
char_t is the type used by TensorRT to represent all valid characters.
Definition: NvInferRuntimeBase.h:103
DataType
The type of weights and tensors.
Definition: NvInferRuntimeBase.h:135
@ kINT64
Signed 64-bit integer type.
@ kFLOAT
32-bit floating point format.
@ kBOOL
8-bit boolean. 0 = false, 1 = true, other values undefined.
@ kHALF
IEEE 16-bit floating-point format – has a 5 bit exponent and 11 bit significand.
@ kINT8
Signed 8-bit integer representing a quantized floating-point value.
@ kBF16
Brain float – has an 8 bit exponent and 8 bit significand.
@ kINT4
Signed 4-bit integer type.
@ kINT32
Signed 32-bit integer format.
char const * InterfaceKind
Definition: NvInferRuntimeBase.h:357
v_1_0::IGpuAllocator IGpuAllocator
Definition: NvInferRuntimeBase.h:668
TensorFormat
Format of the input/output tensors.
Definition: NvInferRuntimeBase.h:250
constexpr int32_t EnumMax() noexcept
Maximum number of elements in an enumeration type.
Definition: NvInferRuntimeBase.h:125
v_1_0::IStreamReader IStreamReader
Definition: NvInferRuntimeBase.h:1157
AllocatorFlag
Allowed type of memory allocation.
Definition: NvInferRuntimeBase.h:445
@ kRESIZABLE
TensorRT may call realloc() on this allocation.
v_1_0::IErrorRecorder IErrorRecorder
Definition: NvInferRuntimeBase.h:114
uint32_t AllocatorFlags
Definition: NvInferRuntimeBase.h:461
Declaration of EnumMaxImpl struct to store maximum number of elements in an enumeration type.
Definition: NvInferRuntimeBase.h:120