NvInferRuntimeCommon.h

Go to the documentation of this file.

/*
 * Copyright (c) 1993-2021 NVIDIA Corporation. All rights reserved.
 *
 * NOTICE TO LICENSEE:
 *
 * This source code and/or documentation ("Licensed Deliverables") are
 * subject to NVIDIA intellectual property rights under U.S. and
 * international Copyright laws.
 *
 * These Licensed Deliverables contained herein is PROPRIETARY and
 * CONFIDENTIAL to NVIDIA and is being provided under the terms and
 * conditions of a form of NVIDIA software license agreement by and
 * between NVIDIA and Licensee ("License Agreement") or electronically
 * accepted by Licensee.  Notwithstanding any terms or conditions to
 * the contrary in the License Agreement, reproduction or disclosure
 * of the Licensed Deliverables to any third party without the express
 * written consent of NVIDIA is prohibited.
 *
 * NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
 * LICENSE AGREEMENT, NVIDIA MAKES NO REPRESENTATION ABOUT THE
 * SUITABILITY OF THESE LICENSED DELIVERABLES FOR ANY PURPOSE.  IT IS
 * PROVIDED "AS IS" WITHOUT EXPRESS OR IMPLIED WARRANTY OF ANY KIND.
 * NVIDIA DISCLAIMS ALL WARRANTIES WITH REGARD TO THESE LICENSED
 * DELIVERABLES, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY,
 * NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE.
 * NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
 * LICENSE AGREEMENT, IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY
 * SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, OR ANY
 * DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
 * WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
 * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
 * OF THESE LICENSED DELIVERABLES.
 *
 * U.S. Government End Users.  These Licensed Deliverables are a
 * "commercial item" as that term is defined at 48 C.F.R. 2.101 (OCT
 * 1995), consisting of "commercial computer software" and "commercial
 * computer software documentation" as such terms are used in 48
 * C.F.R. 12.212 (SEPT 1995) and is provided to the U.S. Government
 * only as a commercial end item.  Consistent with 48 C.F.R.12.212 and
 * 48 C.F.R. 227.7202-1 through 227.7202-4 (JUNE 1995), all
 * U.S. Government End Users acquire the Licensed Deliverables with
 * only those rights set forth herein.
 *
 * Any use of the Licensed Deliverables in individual and commercial
 * software must include, in the user documentation and internal
 * comments to the code, the above Disclaimer and U.S. Government End
 * Users Notice.
 */
 
#ifndef NV_INFER_RUNTIME_COMMON_H
#define NV_INFER_RUNTIME_COMMON_H
 
#include "NvInferVersion.h"
#include <cstddef>
#include <cstdint>
#include <cuda_runtime_api.h>
 
#if __cplusplus >= 201402L
#define TRT_DEPRECATED [[deprecated]]
#if __GNUC__ < 6
#define TRT_DEPRECATED_ENUM
#else
#define TRT_DEPRECATED_ENUM TRT_DEPRECATED
#endif
#ifdef _MSC_VER
#define TRT_DEPRECATED_API __declspec(dllexport)
#else
#define TRT_DEPRECATED_API [[deprecated]] __attribute__((visibility("default")))
#endif
#else
#ifdef _MSC_VER
#define TRT_DEPRECATED
#define TRT_DEPRECATED_ENUM
#define TRT_DEPRECATED_API __declspec(dllexport)
#else
#define TRT_DEPRECATED __attribute__((deprecated))
#define TRT_DEPRECATED_ENUM
#define TRT_DEPRECATED_API __attribute__((deprecated, visibility("default")))
#endif
#endif
 
#ifdef TENSORRT_BUILD_LIB
#ifdef _MSC_VER
#define TENSORRTAPI __declspec(dllexport)
#else
#define TENSORRTAPI __attribute__((visibility("default")))
#endif
#else
#define TENSORRTAPI
#endif
#define TRTNOEXCEPT
 
// forward declare some CUDA types to avoid an include dependency
 
extern "C"
{
    struct cublasContext;
    struct cudnnContext;
}
 
#define NV_TENSORRT_VERSION nvinfer1::kNV_TENSORRT_VERSION_IMPL
namespace nvinfer1
{
 
static constexpr int32_t kNV_TENSORRT_VERSION_IMPL
    = (NV_TENSORRT_MAJOR * 1000) + (NV_TENSORRT_MINOR * 100) + NV_TENSORRT_PATCH; // major, minor, patch
 
using char_t = char;
using AsciiChar = char_t;
 
class IErrorRecorder;
class IGpuAllocator;
 
namespace impl
{
template <typename T>
struct EnumMaxImpl;
} // namespace impl
 
template <typename T>
constexpr int32_t EnumMax() noexcept
{
    return impl::EnumMaxImpl<T>::kVALUE;
}
 
enum class DataType : int32_t
{
    kFLOAT = 0,
 
    kHALF = 1,
 
    kINT8 = 2,
 
    kINT32 = 3,
 
    kBOOL = 4
};
 
namespace impl
{
template <>
struct EnumMaxImpl<DataType>
{
    // Declaration of kVALUE that represents maximum number of elements in DataType enum
    static constexpr int32_t kVALUE = 5;
};
} // namespace impl
 
class Dims32
{
public:
    static constexpr int32_t MAX_DIMS{8};
    int32_t nbDims;
    int32_t d[MAX_DIMS];
};
 
using Dims = Dims32;
 
enum class TensorFormat : int32_t
{
    kLINEAR = 0,
 
    kCHW2 = 1,
 
    kHWC8 = 2,
 
    kCHW4 = 3,
 
    kCHW16 = 4,
 
    kCHW32 = 5,
 
    kDHWC8 = 6,
 
    kCDHW32 = 7,
 
    kHWC = 8,
 
    kDLA_LINEAR = 9,
 
    kDLA_HWC4 = 10,
 
    kHWC16 = 11
};
 
using PluginFormat = TensorFormat;
 
namespace impl
{
template <>
struct EnumMaxImpl<TensorFormat>
{
    // coverity[autosar_cpp14_m0_1_4_violation] Approved RFD: https://jirasw.nvidia.com/browse/TID-489
    static constexpr int32_t kVALUE = 12;
};
} // namespace impl
 
struct PluginTensorDesc
{
    Dims dims;
    DataType type;
    TensorFormat format;
    float scale;
};
 
enum class PluginVersion : uint8_t
{
    kV2 = 0,
    kV2_EXT = 1,
    kV2_IOEXT = 2,
    kV2_DYNAMICEXT = 3,
};
 
class IPluginV2
{
public:
    virtual int32_t getTensorRTVersion() const noexcept
    {
        return NV_TENSORRT_VERSION;
    }
 
    virtual AsciiChar const* getPluginType() const noexcept = 0;
 
    virtual AsciiChar const* getPluginVersion() const noexcept = 0;
 
    virtual int32_t getNbOutputs() const noexcept = 0;
 
    virtual Dims getOutputDimensions(int32_t index, Dims const* inputs, int32_t nbInputDims) noexcept = 0;
 
    virtual bool supportsFormat(DataType type, PluginFormat format) const noexcept = 0;
 
    virtual void configureWithFormat(Dims const* inputDims, int32_t nbInputs, Dims const* outputDims, int32_t nbOutputs,
        DataType type, PluginFormat format, int32_t maxBatchSize) noexcept
        = 0;
 
    virtual int32_t initialize() noexcept = 0;
 
    virtual void terminate() noexcept = 0;
 
    virtual size_t getWorkspaceSize(int32_t maxBatchSize) const noexcept = 0;
 
    virtual int32_t enqueue(int32_t batchSize, void const* const* inputs, void* const* outputs, void* workspace,
        cudaStream_t stream) noexcept
        = 0;
 
    virtual size_t getSerializationSize() const noexcept = 0;
 
    virtual void serialize(void* buffer) const noexcept = 0;
 
    virtual void destroy() noexcept = 0;
 
    virtual IPluginV2* clone() const noexcept = 0;
 
    virtual void setPluginNamespace(AsciiChar const* pluginNamespace) noexcept = 0;
 
    virtual AsciiChar const* getPluginNamespace() const noexcept = 0;
 
    // @cond SuppressDoxyWarnings
    IPluginV2() = default;
    virtual ~IPluginV2() noexcept = default;
// @endcond
 
protected:
// @cond SuppressDoxyWarnings
    IPluginV2(IPluginV2 const&) = default;
    IPluginV2(IPluginV2&&) = default;
    IPluginV2& operator=(IPluginV2 const&) & = default;
    IPluginV2& operator=(IPluginV2&&) & = default;
// @endcond
};
 
class IPluginV2Ext : public IPluginV2
{
public:
    virtual nvinfer1::DataType getOutputDataType(
        int32_t index, nvinfer1::DataType const* inputTypes, int32_t nbInputs) const noexcept
        = 0;
 
    virtual bool isOutputBroadcastAcrossBatch(
        int32_t outputIndex, bool const* inputIsBroadcasted, int32_t nbInputs) const noexcept
        = 0;
 
    virtual bool canBroadcastInputAcrossBatch(int32_t inputIndex) const noexcept = 0;
 
    virtual void configurePlugin(Dims const* inputDims, int32_t nbInputs, Dims const* outputDims, int32_t nbOutputs,
        DataType const* inputTypes, DataType const* outputTypes, bool const* inputIsBroadcast,
        bool const* outputIsBroadcast, PluginFormat floatFormat, int32_t maxBatchSize) noexcept
        = 0;
 
    IPluginV2Ext() = default;
    ~IPluginV2Ext() override = default;
 
    virtual void attachToContext(
        cudnnContext* /*cudnn*/, cublasContext* /*cublas*/, IGpuAllocator* /*allocator*/) noexcept
    {
    }
 
    virtual void detachFromContext() noexcept {}
 
    IPluginV2Ext* clone() const noexcept override = 0;
 
protected:
    // @cond SuppressDoxyWarnings
    IPluginV2Ext(IPluginV2Ext const&) = default;
    IPluginV2Ext(IPluginV2Ext&&) = default;
    IPluginV2Ext& operator=(IPluginV2Ext const&) & = default;
    IPluginV2Ext& operator=(IPluginV2Ext&&) & = default;
// @endcond
 
    int32_t getTensorRTVersion() const noexcept override
    {
        return static_cast<int32_t>((static_cast<uint32_t>(PluginVersion::kV2_EXT) << 24U)
            | (static_cast<uint32_t>(NV_TENSORRT_VERSION) & 0xFFFFFFU));
    }
 
    void configureWithFormat(Dims const* /*inputDims*/, int32_t /*nbInputs*/, Dims const* /*outputDims*/,
        int32_t /*nbOutputs*/, DataType /*type*/, PluginFormat /*format*/, int32_t /*maxBatchSize*/) noexcept override
    {
    }
};
 
class IPluginV2IOExt : public IPluginV2Ext
{
public:
    virtual void configurePlugin(
        PluginTensorDesc const* in, int32_t nbInput, PluginTensorDesc const* out, int32_t nbOutput) noexcept
        = 0;
 
    virtual bool supportsFormatCombination(
        int32_t pos, PluginTensorDesc const* inOut, int32_t nbInputs, int32_t nbOutputs) const noexcept
        = 0;
 
    // @cond SuppressDoxyWarnings
    IPluginV2IOExt() = default;
    ~IPluginV2IOExt() override = default;
// @endcond
 
protected:
// @cond SuppressDoxyWarnings
    IPluginV2IOExt(IPluginV2IOExt const&) = default;
    IPluginV2IOExt(IPluginV2IOExt&&) = default;
    IPluginV2IOExt& operator=(IPluginV2IOExt const&) & = default;
    IPluginV2IOExt& operator=(IPluginV2IOExt&&) & = default;
// @endcond
 
    int32_t getTensorRTVersion() const noexcept override
    {
        return static_cast<int32_t>((static_cast<uint32_t>(PluginVersion::kV2_IOEXT) << 24U)
            | (static_cast<uint32_t>(NV_TENSORRT_VERSION) & 0xFFFFFFU));
    }
 
private:
    // Following are obsolete base class methods, and must not be implemented or used.
 
    void configurePlugin(Dims const*, int32_t, Dims const*, int32_t, DataType const*, DataType const*, bool const*,
        bool const*, PluginFormat, int32_t) noexcept final
    {
    }
 
    bool supportsFormat(DataType, PluginFormat) const noexcept final
    {
        return false;
    }
};
 
 
enum class PluginFieldType : int32_t
{
    kFLOAT16 = 0,
    kFLOAT32 = 1,
    kFLOAT64 = 2,
    kINT8 = 3,
    kINT16 = 4,
    kINT32 = 5,
    kCHAR = 6,
    kDIMS = 7,
    kUNKNOWN = 8
};
 
class PluginField
{
public:
    AsciiChar const* name;
    void const* data;
    PluginFieldType type;
    int32_t length;
 
    PluginField(AsciiChar const* const name_ = nullptr, void const* const data_ = nullptr,
        PluginFieldType const type_ = PluginFieldType::kUNKNOWN, int32_t const length_ = 0) noexcept
        : name(name_)
        , data(data_)
        , type(type_)
        , length(length_)
    {
    }
};
 
struct PluginFieldCollection
{
    int32_t nbFields;
    PluginField const* fields;
};
 
 
class IPluginCreator
{
public:
    virtual int32_t getTensorRTVersion() const noexcept
    {
        return NV_TENSORRT_VERSION;
    }
 
    virtual AsciiChar const* getPluginName() const noexcept = 0;
 
    virtual AsciiChar const* getPluginVersion() const noexcept = 0;
 
    virtual PluginFieldCollection const* getFieldNames() noexcept = 0;
 
    virtual IPluginV2* createPlugin(AsciiChar const* name, PluginFieldCollection const* fc) noexcept = 0;
 
    virtual IPluginV2* deserializePlugin(AsciiChar const* name, void const* serialData, size_t serialLength) noexcept
        = 0;
 
    virtual void setPluginNamespace(AsciiChar const* pluginNamespace) noexcept = 0;
 
    virtual AsciiChar const* getPluginNamespace() const noexcept = 0;
 
    IPluginCreator() = default;
    virtual ~IPluginCreator() = default;
 
protected:
// @cond SuppressDoxyWarnings
    IPluginCreator(IPluginCreator const&) = default;
    IPluginCreator(IPluginCreator&&) = default;
    IPluginCreator& operator=(IPluginCreator const&) & = default;
    IPluginCreator& operator=(IPluginCreator&&) & = default;
// @endcond
};
 
 
class IPluginRegistry
{
public:
    virtual bool registerCreator(IPluginCreator& creator, AsciiChar const* const pluginNamespace) noexcept = 0;
 
    virtual IPluginCreator* const* getPluginCreatorList(int32_t* const numCreators) const noexcept = 0;
 
    virtual IPluginCreator* getPluginCreator(AsciiChar const* const pluginName, AsciiChar const* const pluginVersion,
        AsciiChar const* const pluginNamespace = "") noexcept
        = 0;
 
    // @cond SuppressDoxyWarnings
    IPluginRegistry() = default;
    IPluginRegistry(IPluginRegistry const&) = delete;
    IPluginRegistry(IPluginRegistry&&) = delete;
    IPluginRegistry& operator=(IPluginRegistry const&) & = delete;
    IPluginRegistry& operator=(IPluginRegistry&&) & = delete;
// @endcond
 
protected:
    virtual ~IPluginRegistry() noexcept = default;
 
public:
    //
    virtual void setErrorRecorder(IErrorRecorder* const recorder) noexcept = 0;
 
    virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
 
    virtual bool deregisterCreator(IPluginCreator const& creator) noexcept = 0;
};
 
enum class AllocatorFlag : int32_t
{
    kRESIZABLE = 0, 
};
 
namespace impl
{
template <>
struct EnumMaxImpl<AllocatorFlag>
{
    static constexpr int32_t kVALUE = 1;        
};
} // namespace impl
 
using AllocatorFlags = uint32_t;
 
class IGpuAllocator
{
public:
    virtual void* allocate(uint64_t const size, uint64_t const alignment, AllocatorFlags const flags) noexcept = 0;
 
    TRT_DEPRECATED virtual void free(void* const memory) noexcept = 0;
 
    virtual ~IGpuAllocator() = default;
    IGpuAllocator() = default;
 
    virtual void* reallocate(void* /*baseAddr*/, uint64_t /*alignment*/, uint64_t /*newSize*/) noexcept
    {
        return nullptr;
    }
 
    virtual bool deallocate(void* const memory) noexcept
    {
        this->free(memory);
        return true;
    }
 
protected:
// @cond SuppressDoxyWarnings
    IGpuAllocator(IGpuAllocator const&) = default;
    IGpuAllocator(IGpuAllocator&&) = default;
    IGpuAllocator& operator=(IGpuAllocator const&) & = default;
    IGpuAllocator& operator=(IGpuAllocator&&) & = default;
// @endcond
};
 
class ILogger
{
public:
    enum class Severity : int32_t
    {
        kINTERNAL_ERROR = 0,
        kERROR = 1,
        kWARNING = 2,
        kINFO = 3,
        kVERBOSE = 4,
    };
 
    virtual void log(Severity severity, AsciiChar const* msg) noexcept = 0;
 
    ILogger() = default;
    virtual ~ILogger() = default;
 
protected:
// @cond SuppressDoxyWarnings
    ILogger(ILogger const&) = default;
    ILogger(ILogger&&) = default;
    ILogger& operator=(ILogger const&) & = default;
    ILogger& operator=(ILogger&&) & = default;
// @endcond
};
 
namespace impl
{
template <>
struct EnumMaxImpl<ILogger::Severity>
{
    static constexpr int32_t kVALUE = 5;
};
} // namespace impl
 
enum class ErrorCode : int32_t
{
    kSUCCESS = 0,
 
    kUNSPECIFIED_ERROR = 1,
 
    kINTERNAL_ERROR = 2,
 
    kINVALID_ARGUMENT = 3,
 
    kINVALID_CONFIG = 4,
 
    kFAILED_ALLOCATION = 5,
 
    kFAILED_INITIALIZATION = 6,
 
    kFAILED_EXECUTION = 7,
 
    kFAILED_COMPUTATION = 8,
 
    kINVALID_STATE = 9,
 
    kUNSUPPORTED_STATE = 10,
 
};
 
namespace impl
{
template <>
struct EnumMaxImpl<ErrorCode>
{
    static constexpr int32_t kVALUE = 11;
};
} // namespace impl
 
class IErrorRecorder
{
public:
    using ErrorDesc = char const*;
 
    // coverity[autosar_cpp14_m0_1_4_violation] Approved RFD: https://jirasw.nvidia.com/browse/TID-489
    static constexpr size_t kMAX_DESC_LENGTH{127U};
 
    using RefCount = int32_t;
 
    IErrorRecorder() = default;
    virtual ~IErrorRecorder() noexcept = default;
 
    // Public API used to retrieve information from the error recorder.
 
    virtual int32_t getNbErrors() const noexcept = 0;
 
    virtual ErrorCode getErrorCode(int32_t errorIdx) const noexcept = 0;
 
    virtual ErrorDesc getErrorDesc(int32_t errorIdx) const noexcept = 0;
 
    virtual bool hasOverflowed() const noexcept = 0;
 
    virtual void clear() noexcept = 0;
 
    // API used by TensorRT to report Error information to the application.
 
    virtual bool reportError(ErrorCode val, ErrorDesc desc) noexcept = 0;
 
    virtual RefCount incRefCount() noexcept = 0;
 
    virtual RefCount decRefCount() noexcept = 0;
 
protected:
    // @cond SuppressDoxyWarnings
    IErrorRecorder(IErrorRecorder const&) = default;
    IErrorRecorder(IErrorRecorder&&) = default;
    IErrorRecorder& operator=(IErrorRecorder const&) & = default;
    IErrorRecorder& operator=(IErrorRecorder&&) & = default;
    // @endcond
}; // class IErrorRecorder
} // namespace nvinfer1
 
extern "C" TENSORRTAPI int32_t getInferLibVersion() noexcept;
 
#endif // NV_INFER_RUNTIME_COMMON_H