18#ifndef NV_INFER_RUNTIME_BASE_H
19#define NV_INFER_RUNTIME_BASE_H
24#include <cuda_runtime_api.h>
27#if __cplusplus >= 201402L
28#define TRT_DEPRECATED [[deprecated]]
30#define TRT_DEPRECATED_ENUM
32#define TRT_DEPRECATED_ENUM TRT_DEPRECATED
35#define TRT_DEPRECATED_API __declspec(dllexport)
37#define TRT_DEPRECATED_API [[deprecated]] __attribute__((visibility("default")))
42#define TRT_DEPRECATED_ENUM
43#define TRT_DEPRECATED_API __declspec(dllexport)
45#define TRT_DEPRECATED __attribute__((deprecated))
46#define TRT_DEPRECATED_ENUM
47#define TRT_DEPRECATED_API __attribute__((deprecated, visibility("default")))
52#ifdef TENSORRT_BUILD_LIB
54#define TENSORRTAPI __declspec(dllexport)
56#define TENSORRTAPI __attribute__((visibility("default")))
73#if !defined(NV_INFER_INTERNAL_INCLUDE_RUNTIME_BASE) && !defined(TRT_VCAST_SAFE)
74static_assert(
false,
"Do not directly include this file. Include NvInferRuntime.h or NvInferSafeRuntime.h or NvInferConsistency.h or NvInferPluginUtils.h");
89#define NV_TENSORRT_VERSION_INT(major, minor, patch) ((major) *10000L + (minor) *100L + (patch) *1L)
93#define NV_TENSORRT_VERSION NV_TENSORRT_VERSION_INT(NV_TENSORRT_MAJOR, NV_TENSORRT_MINOR, NV_TENSORRT_PATCH)
186 static constexpr int32_t kVALUE = 10;
389 static constexpr int32_t kVALUE = 2;
433 static constexpr int32_t kVALUE = 13;
456 static constexpr int32_t kVALUE = 1;
495 uint64_t
const size, uint64_t
const alignment,
AllocatorFlags const flags)
noexcept = 0;
537 virtual void*
reallocate(
void*
const , uint64_t , uint64_t )
noexcept
591 uint64_t
const size, uint64_t
const alignment,
AllocatorFlags const flags, cudaStream_t )
noexcept
593 return allocate(size, alignment, flags);
625 return deallocate(memory);
633 return {
"IGpuAllocator", 1, 0};
742 static constexpr int32_t kVALUE = 5;
849 static constexpr int32_t kVALUE = 11;
876 static constexpr size_t kMAX_DESC_LENGTH{127U};
911 virtual int32_t getNbErrors() const noexcept = 0;
931 virtual
ErrorCode getErrorCode(int32_t errorIdx) const noexcept = 0;
954 virtual
ErrorDesc getErrorDesc(int32_t errorIdx) const noexcept = 0;
970 virtual
bool hasOverflowed() const noexcept = 0;
986 virtual
void clear() noexcept = 0;
1138 virtual int64_t
read(
void* destination, int64_t nbBytes) = 0;
1227 static constexpr int32_t kVALUE = 3;
#define TENSORRTAPI
Definition: NvInferRuntimeBase.h:59
int32_t getInferLibVersion() noexcept
Return the library version number.
#define TRT_DEPRECATED
Definition: NvInferRuntimeBase.h:45
Definition: NvInferRuntimeBase.h:201
static constexpr int32_t MAX_DIMS
The maximum rank (number of dimensions) supported for a tensor.
Definition: NvInferRuntimeBase.h:204
int64_t d[MAX_DIMS]
The extent of each dimension.
Definition: NvInferRuntimeBase.h:210
int32_t nbDims
The rank (number of dimensions).
Definition: NvInferRuntimeBase.h:207
Application-implemented logging interface for the builder, refitter and runtime.
Definition: NvInferRuntimeBase.h:682
virtual ~ILogger()=default
Severity
The severity corresponding to a log message.
Definition: NvInferRuntimeBase.h:690
virtual void log(Severity severity, AsciiChar const *msg) noexcept=0
A callback implemented by the application to handle logging messages;.
An Interface class for version control.
Definition: NvInferRuntimeBase.h:399
virtual InterfaceInfo getInterfaceInfo() const noexcept=0
Return version information associated with this interface. Applications must not override this method...
virtual APILanguage getAPILanguage() const noexcept
The language used to build the implementation of this Interface.
Definition: NvInferRuntimeBase.h:406
IVersionedInterface & operator=(IVersionedInterface const &) &=default
Version information associated with a TRT interface.
Definition: NvInferRuntimeBase.h:364
InterfaceKind kind
Definition: NvInferRuntimeBase.h:366
int32_t major
Definition: NvInferRuntimeBase.h:367
int32_t minor
Definition: NvInferRuntimeBase.h:368
Definition: NvInferRuntimeBase.h:856
char const * ErrorDesc
A typedef of a C-style string for reporting error descriptions.
Definition: NvInferRuntimeBase.h:869
~IErrorRecorder() noexcept override=default
int32_t RefCount
A typedef of a 32-bit integer for reference counting.
Definition: NvInferRuntimeBase.h:881
InterfaceInfo getInterfaceInfo() const noexcept override
Return version information associated with this interface. Applications must not override this method...
Definition: NvInferRuntimeBase.h:861
Definition: NvInferRuntimeBase.h:468
virtual void * allocateAsync(uint64_t const size, uint64_t const alignment, AllocatorFlags const flags, cudaStream_t) noexcept
A thread-safe callback implemented by the application to handle stream-ordered acquisition of GPU mem...
Definition: NvInferRuntimeBase.h:590
InterfaceInfo getInterfaceInfo() const noexcept override
Return version information associated with this interface. Applications must not override this method...
Definition: NvInferRuntimeBase.h:631
virtual TRT_DEPRECATED bool deallocate(void *const memory) noexcept=0
A thread-safe callback implemented by the application to handle release of GPU memory.
~IGpuAllocator() override=default
virtual void * reallocate(void *const, uint64_t, uint64_t) noexcept
A thread-safe callback implemented by the application to resize an existing allocation.
Definition: NvInferRuntimeBase.h:537
virtual TRT_DEPRECATED void * allocate(uint64_t const size, uint64_t const alignment, AllocatorFlags const flags) noexcept=0
A thread-safe callback implemented by the application to handle acquisition of GPU memory.
virtual bool deallocateAsync(void *const memory, cudaStream_t) noexcept
A thread-safe callback implemented by the application to handle stream-ordered release of GPU memory.
Definition: NvInferRuntimeBase.h:623
Definition: NvInferRuntimeBase.h:1162
virtual int32_t release() noexcept=0
Free the underlying resource.
InterfaceInfo getInterfaceInfo() const noexcept override
Return version information associated with this interface. Applications must not override this method...
Definition: NvInferRuntimeBase.h:1167
Definition: NvInferRuntimeBase.h:1113
~IStreamReader() override=default
IStreamReader & operator=(IStreamReader const &) &=default
IStreamReader & operator=(IStreamReader &&) &=default
virtual int64_t read(void *destination, int64_t nbBytes)=0
Read the next number of bytes in the stream.
IStreamReader(IStreamReader &&)=default
IStreamReader(IStreamReader const &)=default
InterfaceInfo getInterfaceInfo() const noexcept override
Return version information associated with this interface. Applications must not override this method...
Definition: NvInferRuntimeBase.h:1125
The TensorRT API version 1 namespace.
ErrorCode
Error codes that can be returned by TensorRT during execution.
Definition: NvInferRuntimeBase.h:752
TensorIOMode
Definition of tensor IO Mode.
Definition: NvInferRuntimeBase.h:1099
@ kOUTPUT
Tensor is output by the engine.
@ kINPUT
Tensor is input to the engine.
APILanguage
Programming language used in the implementation of a TRT interface.
Definition: NvInferRuntimeBase.h:377
char_t AsciiChar
Definition: NvInferRuntimeBase.h:107
char char_t
char_t is the type used by TensorRT to represent all valid characters.
Definition: NvInferRuntimeBase.h:103
DataType
The type of weights and tensors.
Definition: NvInferRuntimeBase.h:135
@ kINT64
Signed 64-bit integer type.
@ kFLOAT
32-bit floating point format.
@ kBOOL
8-bit boolean. 0 = false, 1 = true, other values undefined.
@ kHALF
IEEE 16-bit floating-point format – has a 5 bit exponent and 11 bit significand.
@ kINT8
Signed 8-bit integer representing a quantized floating-point value.
@ kBF16
Brain float – has an 8 bit exponent and 8 bit significand.
@ kINT4
Signed 4-bit integer type.
@ kINT32
Signed 32-bit integer format.
char const * InterfaceKind
Definition: NvInferRuntimeBase.h:356
v_1_0::IGpuAllocator IGpuAllocator
Definition: NvInferRuntimeBase.h:667
TensorFormat
Format of the input/output tensors.
Definition: NvInferRuntimeBase.h:249
constexpr int32_t EnumMax() noexcept
Maximum number of elements in an enumeration type.
Definition: NvInferRuntimeBase.h:125
v_1_0::IStreamReader IStreamReader
Definition: NvInferRuntimeBase.h:1156
AllocatorFlag
Allowed type of memory allocation.
Definition: NvInferRuntimeBase.h:444
@ kRESIZABLE
TensorRT may call realloc() on this allocation.
v_1_0::IErrorRecorder IErrorRecorder
Definition: NvInferRuntimeBase.h:114
uint32_t AllocatorFlags
Definition: NvInferRuntimeBase.h:460
Declaration of EnumMaxImpl struct to store maximum number of elements in an enumeration type.
Definition: NvInferRuntimeBase.h:120