NvInferRuntimeBase.h

Go to the documentation of this file.

/*
 * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 * SPDX-License-Identifier: Apache-2.0
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
 
#ifndef NV_INFER_RUNTIME_BASE_H
#define NV_INFER_RUNTIME_BASE_H
 
#include "NvInferVersion.h"
#include <cstddef>
#include <cstdint>
#include <cuda_runtime_api.h>
 
// Items that are marked as deprecated will be removed in a future release.
#if __cplusplus >= 201402L
#define TRT_DEPRECATED [[deprecated]]
#if __GNUC__ < 6
#define TRT_DEPRECATED_ENUM
#else
#define TRT_DEPRECATED_ENUM TRT_DEPRECATED
#endif
#ifdef _MSC_VER
#define TRT_DEPRECATED_API __declspec(dllexport)
#else
#define TRT_DEPRECATED_API [[deprecated]] __attribute__((visibility("default")))
#endif
#else
#ifdef _MSC_VER
#define TRT_DEPRECATED
#define TRT_DEPRECATED_ENUM
#define TRT_DEPRECATED_API __declspec(dllexport)
#else
#define TRT_DEPRECATED __attribute__((deprecated))
#define TRT_DEPRECATED_ENUM
#define TRT_DEPRECATED_API __attribute__((deprecated, visibility("default")))
#endif
#endif
 
// Defines which symbols are exported
#ifdef TENSORRT_BUILD_LIB
#ifdef _MSC_VER
#define TENSORRTAPI __declspec(dllexport)
#else
#define TENSORRTAPI __attribute__((visibility("default")))
#endif
#else
#define TENSORRTAPI
#endif
#define TRTNOEXCEPT
#if !defined(NV_INFER_INTERNAL_INCLUDE_RUNTIME_BASE)
static_assert(false, "Do not directly include this file. Include NvInferRuntime.h or NvInferSafeRuntime.h or NvInferConsistency.h or NvInferPluginUtils.h");
#endif
 
 
extern "C"
{
    struct cublasContext;
    struct cudnnContext;
}
 
#define NV_TENSORRT_VERSION_INT(major, minor, patch) ((major) *10000L + (minor) *100L + (patch) *1L)
 
#define NV_TENSORRT_VERSION NV_TENSORRT_VERSION_INT(NV_TENSORRT_MAJOR, NV_TENSORRT_MINOR, NV_TENSORRT_PATCH)
 
namespace nvinfer1
{
using char_t = char;
 
using AsciiChar = char_t;
 
namespace v_1_0
{
class IErrorRecorder;
}
using IErrorRecorder = v_1_0::IErrorRecorder;
 
namespace impl
{
template <typename T>
struct EnumMaxImpl;
} // namespace impl
 
template <typename T>
constexpr int32_t EnumMax() noexcept
{
    return impl::EnumMaxImpl<T>::kVALUE;
}
 
enum class DataType : int32_t
{
    kFLOAT = 0,
 
    kHALF = 1,
 
    kINT8 = 2,
 
    kINT32 = 3,
 
    kBOOL = 4,
 
    kUINT8 = 5,
 
    kFP8 = 6,
 
    kBF16 = 7,
 
    kINT64 = 8,
 
    kINT4 = 9,
 
};
 
namespace impl
{
template <>
struct EnumMaxImpl<DataType>
{
    static constexpr int32_t kVALUE = 10;
};
} // namespace impl
 
class Dims64
{
public:
    static constexpr int32_t MAX_DIMS{8};
 
    int32_t nbDims;
 
    int64_t d[MAX_DIMS];
};
 
using Dims = Dims64;
 
enum class TensorFormat : int32_t
{
    kLINEAR = 0,
 
    kCHW2 = 1,
 
    kHWC8 = 2,
 
    kCHW4 = 3,
 
    kCHW16 = 4,
 
    kCHW32 = 5,
 
    kDHWC8 = 6,
 
    kCDHW32 = 7,
 
    kHWC = 8,
 
    kDLA_LINEAR = 9,
 
    kDLA_HWC4 = 10,
 
    kHWC16 = 11,
 
    kDHWC = 12
};
 
using InterfaceKind = char const*;
 
class InterfaceInfo
{
public:
    InterfaceKind kind;
    int32_t major;
    int32_t minor;
};
 
enum class APILanguage : int32_t
{
    kCPP = 0,
    kPYTHON = 1
};
 
namespace impl
{
template <>
struct EnumMaxImpl<APILanguage>
{
    static constexpr int32_t kVALUE = 2;
};
} // namespace impl
 
class IVersionedInterface
{
public:
    virtual APILanguage getAPILanguage() const noexcept
    {
        return APILanguage::kCPP;
    }
 
    virtual InterfaceInfo getInterfaceInfo() const noexcept = 0;
 
    virtual ~IVersionedInterface() noexcept = default;
 
protected:
    IVersionedInterface() = default;
    IVersionedInterface(IVersionedInterface const&) = default;
    IVersionedInterface(IVersionedInterface&&) = default;
    IVersionedInterface& operator=(IVersionedInterface const&) & = default;
    IVersionedInterface& operator=(IVersionedInterface&&) & = default;
};
 
namespace impl
{
template <>
struct EnumMaxImpl<TensorFormat>
{
    static constexpr int32_t kVALUE = 13;
};
} // namespace impl
 
 
enum class AllocatorFlag : int32_t
{
    kRESIZABLE = 0,
};
 
namespace impl
{
template <>
struct EnumMaxImpl<AllocatorFlag>
{
    static constexpr int32_t kVALUE = 1;
};
} // namespace impl
 
using AllocatorFlags = uint32_t;
 
namespace v_1_0
{
 
class IGpuAllocator : public IVersionedInterface
{
public:
    TRT_DEPRECATED virtual void* allocate(
        uint64_t const size, uint64_t const alignment, AllocatorFlags const flags) noexcept = 0;
 
    ~IGpuAllocator() override = default;
    IGpuAllocator() = default;
 
    virtual void* reallocate(void* const /*baseAddr*/, uint64_t /*alignment*/, uint64_t /*newSize*/) noexcept
    {
        return nullptr;
    }
 
    TRT_DEPRECATED virtual bool deallocate(void* const memory) noexcept = 0;
 
    virtual void* allocateAsync(
        uint64_t const size, uint64_t const alignment, AllocatorFlags const flags, cudaStream_t /*stream*/) noexcept
    {
        return allocate(size, alignment, flags);
    }
    virtual bool deallocateAsync(void* const memory, cudaStream_t /*stream*/) noexcept
    {
        return deallocate(memory);
    }
 
    InterfaceInfo getInterfaceInfo() const noexcept override
    {
        return {"IGpuAllocator", 1, 0};
    }
 
protected:
    // @cond SuppressDoxyWarnings
    IGpuAllocator(IGpuAllocator const&) = default;
    IGpuAllocator(IGpuAllocator&&) = default;
    IGpuAllocator& operator=(IGpuAllocator const&) & = default;
    IGpuAllocator& operator=(IGpuAllocator&&) & = default;
    // @endcond
};
 
} // namespace v_1_0
 
using IGpuAllocator = v_1_0::IGpuAllocator;
 
class ILogger
{
public:
    enum class Severity : int32_t
    {
        kINTERNAL_ERROR = 0,
        kERROR = 1,
        kWARNING = 2,
        kINFO = 3,
        kVERBOSE = 4,
    };
 
    virtual void log(Severity severity, AsciiChar const* msg) noexcept = 0;
 
    ILogger() = default;
    virtual ~ILogger() = default;
 
protected:
// @cond SuppressDoxyWarnings
    ILogger(ILogger const&) = default;
    ILogger(ILogger&&) = default;
    ILogger& operator=(ILogger const&) & = default;
    ILogger& operator=(ILogger&&) & = default;
// @endcond
};
 
namespace impl
{
template <>
struct EnumMaxImpl<ILogger::Severity>
{
    static constexpr int32_t kVALUE = 5;
};
} // namespace impl
 
enum class ErrorCode : int32_t
{
    kSUCCESS = 0,
 
    kUNSPECIFIED_ERROR = 1,
 
    kINTERNAL_ERROR = 2,
 
    kINVALID_ARGUMENT = 3,
 
    kINVALID_CONFIG = 4,
 
    kFAILED_ALLOCATION = 5,
 
    kFAILED_INITIALIZATION = 6,
 
    kFAILED_EXECUTION = 7,
 
    kFAILED_COMPUTATION = 8,
 
    kINVALID_STATE = 9,
 
    kUNSUPPORTED_STATE = 10,
 
};
 
namespace impl
{
template <>
struct EnumMaxImpl<ErrorCode>
{
    static constexpr int32_t kVALUE = 11;
};
} // namespace impl
 
namespace v_1_0
{
class IErrorRecorder : public IVersionedInterface
{
public:
    InterfaceInfo getInterfaceInfo() const noexcept override
    {
        return InterfaceInfo{"IErrorRecorder", 1, 0};
    }
 
    using ErrorDesc = char const*;
 
    static constexpr size_t kMAX_DESC_LENGTH{127U};
 
    using RefCount = int32_t;
 
    IErrorRecorder() = default;
    ~IErrorRecorder() noexcept override = default;
 
    // Public API used to retrieve information from the error recorder.
 
    virtual int32_t getNbErrors() const noexcept = 0;
 
    virtual ErrorCode getErrorCode(int32_t errorIdx) const noexcept = 0;
 
    virtual ErrorDesc getErrorDesc(int32_t errorIdx) const noexcept = 0;
 
    virtual bool hasOverflowed() const noexcept = 0;
 
    virtual void clear() noexcept = 0;
 
    // API used by TensorRT to report Error information to the application.
 
    virtual bool reportError(ErrorCode val, ErrorDesc desc) noexcept = 0;
 
    virtual RefCount incRefCount() noexcept = 0;
 
    virtual RefCount decRefCount() noexcept = 0;
 
protected:
    // @cond SuppressDoxyWarnings
    IErrorRecorder(IErrorRecorder const&) = default;
    IErrorRecorder(IErrorRecorder&&) = default;
    IErrorRecorder& operator=(IErrorRecorder const&) & = default;
    IErrorRecorder& operator=(IErrorRecorder&&) & = default;
    // @endcond
}; // class IErrorRecorder
} // namespace v_1_0
 
using IErrorRecorder = v_1_0::IErrorRecorder;
 
enum class TensorIOMode : int32_t
{
    kNONE = 0,
 
    kINPUT = 1,
 
    kOUTPUT = 2
};
 
namespace v_1_0
{
class IStreamReader : public IVersionedInterface
{
public:
    ~IStreamReader() override = default;
    IStreamReader() = default;
 
    InterfaceInfo getInterfaceInfo() const noexcept override
    {
        return InterfaceInfo{"IStreamReader", 1, 0};
    }
 
    virtual int64_t read(void* destination, int64_t nbBytes) = 0;
 
protected:
    IStreamReader(IStreamReader const&) = default;
    IStreamReader(IStreamReader&&) = default;
    IStreamReader& operator=(IStreamReader const&) & = default;
    IStreamReader& operator=(IStreamReader&&) & = default;
};
} // namespace v_1_0
 
using IStreamReader = v_1_0::IStreamReader;
 
namespace v_1_0
{
 
class IPluginResource : public IVersionedInterface
{
public:
    InterfaceInfo getInterfaceInfo() const noexcept override
    {
        return InterfaceInfo{"IPluginResource", 1, 0};
    }
    virtual int32_t release() noexcept = 0;
 
    virtual IPluginResource* clone() noexcept = 0;
 
    ~IPluginResource() noexcept override = default;
 
    IPluginResource() = default;
    IPluginResource(IPluginResource const&) = default;
    IPluginResource(IPluginResource&&) = default;
    IPluginResource& operator=(IPluginResource const&) & = default;
    IPluginResource& operator=(IPluginResource&&) & = default;
}; // class IPluginResource
} // namespace v_1_0
 
using IPluginResource = v_1_0::IPluginResource;
 
namespace impl
{
template <>
struct EnumMaxImpl<TensorIOMode>
{
    // Declaration of kVALUE that represents maximum number of elements in TensorIOMode enum
    static constexpr int32_t kVALUE = 3;
};
} // namespace impl
} // namespace nvinfer1
 
extern "C" TENSORRTAPI int32_t getInferLibVersion() noexcept;
 
#endif // NV_INFER_RUNTIME_BASE_H