NvInferRuntimeCommon.h

Go to the documentation of this file.

/*
 * Copyright 1993-2019 NVIDIA Corporation.  All rights reserved.
 *
 * NOTICE TO LICENSEE:
 *
 * This source code and/or documentation ("Licensed Deliverables") are
 * subject to NVIDIA intellectual property rights under U.S. and
 * international Copyright laws.
 *
 * These Licensed Deliverables contained herein is PROPRIETARY and
 * CONFIDENTIAL to NVIDIA and is being provided under the terms and
 * conditions of a form of NVIDIA software license agreement by and
 * between NVIDIA and Licensee ("License Agreement") or electronically
 * accepted by Licensee.  Notwithstanding any terms or conditions to
 * the contrary in the License Agreement, reproduction or disclosure
 * of the Licensed Deliverables to any third party without the express
 * written consent of NVIDIA is prohibited.
 *
 * NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
 * LICENSE AGREEMENT, NVIDIA MAKES NO REPRESENTATION ABOUT THE
 * SUITABILITY OF THESE LICENSED DELIVERABLES FOR ANY PURPOSE.  IT IS
 * PROVIDED "AS IS" WITHOUT EXPRESS OR IMPLIED WARRANTY OF ANY KIND.
 * NVIDIA DISCLAIMS ALL WARRANTIES WITH REGARD TO THESE LICENSED
 * DELIVERABLES, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY,
 * NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE.
 * NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
 * LICENSE AGREEMENT, IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY
 * SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, OR ANY
 * DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
 * WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
 * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
 * OF THESE LICENSED DELIVERABLES.
 *
 * U.S. Government End Users.  These Licensed Deliverables are a
 * "commercial item" as that term is defined at 48 C.F.R. 2.101 (OCT
 * 1995), consisting of "commercial computer software" and "commercial
 * computer software documentation" as such terms are used in 48
 * C.F.R. 12.212 (SEPT 1995) and is provided to the U.S. Government
 * only as a commercial end item.  Consistent with 48 C.F.R.12.212 and
 * 48 C.F.R. 227.7202-1 through 227.7202-4 (JUNE 1995), all
 * U.S. Government End Users acquire the Licensed Deliverables with
 * only those rights set forth herein.
 *
 * Any use of the Licensed Deliverables in individual and commercial
 * software must include, in the user documentation and internal
 * comments to the code, the above Disclaimer and U.S. Government End
 * Users Notice.
 */

#ifndef NV_INFER_RUNTIME_COMMON_H
#define NV_INFER_RUNTIME_COMMON_H

#include <cstddef>
#include <cstdint>
#include "NvInferVersion.h"

#if __cplusplus >= 201103L
#define _TENSORRT_FINAL final
#define _TENSORRT_OVERRIDE override
#else
#define _TENSORRT_FINAL
#define _TENSORRT_OVERRIDE
#endif

#if __cplusplus >= 201402L
#define TRT_DEPRECATED [[deprecated]]
#if __GNUC__ < 6
#define TRT_DEPRECATED_ENUM
#else
#define TRT_DEPRECATED_ENUM TRT_DEPRECATED
#endif
#ifdef _MSC_VER
#define TRT_DEPRECATED_API __declspec(dllexport)
#else
#define TRT_DEPRECATED_API [[deprecated]] __attribute__((visibility("default")))
#endif
#else
#ifdef _MSC_VER
#define TRT_DEPRECATED
#define TRT_DEPRECATED_ENUM
#define TRT_DEPRECATED_API __declspec(dllexport)
#else
#define TRT_DEPRECATED __attribute__((deprecated))
#define TRT_DEPRECATED_ENUM
#define TRT_DEPRECATED_API __attribute__((deprecated, visibility("default")))
#endif
#endif

#ifdef TENSORRT_BUILD_LIB
#ifdef _MSC_VER
#define TENSORRTAPI __declspec(dllexport)
#else
#define TENSORRTAPI __attribute__((visibility("default")))
#endif
#else
#define TENSORRTAPI
#endif
#define TRTNOEXCEPT

// forward declare some CUDA types to avoid an include dependency

struct cublasContext;
struct cudnnContext;

typedef struct CUstream_st* cudaStream_t; 
typedef struct CUevent_st* cudaEvent_t;   

static const int NV_TENSORRT_VERSION = (NV_TENSORRT_MAJOR * 1000) + (NV_TENSORRT_MINOR * 100) + NV_TENSORRT_PATCH; // major, minor, patch

namespace nvinfer1
{

class IErrorRecorder; 
class IGpuAllocator; 

enum class ActivationType : int
{
    kRELU = 0,             
    kSIGMOID = 1,          
    kTANH = 2,             
    kLEAKY_RELU = 3,       
    kELU = 4,              
    kSELU = 5,             
    kSOFTSIGN = 6,         
    kSOFTPLUS = 7,         
    kCLIP = 8,             
    kHARD_SIGMOID = 9,     
    kSCALED_TANH = 10,     
    kTHRESHOLDED_RELU = 11 
};

template <typename T>
constexpr inline int EnumMax(); 

template <>
constexpr inline int EnumMax<ActivationType>()
{
    return 12;
} 

enum class DataType : int
{
    kFLOAT = 0, 
    kHALF = 1,  
    kINT8 = 2,  
    kINT32 = 3, 
    kBOOL = 4   
};

template <>
constexpr inline int EnumMax<DataType>()
{
    return 5;
} 

enum class DimensionType : int
{
    kSPATIAL = 0, 
    kCHANNEL = 1, 
    kINDEX = 2,   
    kSEQUENCE = 3 
};

template <>
constexpr inline int EnumMax<DimensionType>()
{
    return 4;
} 

class Dims
{
public:
    static const int MAX_DIMS = 8; 
    int nbDims;                    
    int d[MAX_DIMS];               
    TRT_DEPRECATED DimensionType type[MAX_DIMS];  
};

typedef uint32_t TensorFormats;

enum class TensorFormat : int
{
    kLINEAR = 0,
    kNCHW TRT_DEPRECATED_ENUM = kLINEAR, 

    kCHW2 = 1,
    kNC2HW2 TRT_DEPRECATED_ENUM = kCHW2, 

    kHWC8 = 2,
    kNHWC8 TRT_DEPRECATED_ENUM = kHWC8, 

    kCHW4 = 3,

    kCHW16 = 4,

    kCHW32 = 5
};

using PluginFormat = TensorFormat;

template <>
constexpr inline int EnumMax<TensorFormat>()
{
    return 6;
} 

struct PluginTensorDesc
{
    Dims dims;
    DataType type;
    TensorFormat format;
    float scale;
};

enum class PluginVersion : uint8_t
{
    kV2 = 0,            
    kV2_EXT = 1,        
    kV2_IOEXT = 2,      
    kV2_DYNAMICEXT = 3, 
};

class IPluginV2
{
public:
    virtual int getTensorRTVersion() const TRTNOEXCEPT
    {
        return NV_TENSORRT_VERSION;
    }

    // \see IPluginCreator::getPluginName()
    virtual const char* getPluginType() const TRTNOEXCEPT = 0;

    // \see IPluginCreator::getPluginVersion()
    virtual const char* getPluginVersion() const TRTNOEXCEPT = 0;

    virtual int getNbOutputs() const TRTNOEXCEPT = 0;

    virtual Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) TRTNOEXCEPT = 0;

    virtual bool supportsFormat(DataType type, PluginFormat format) const TRTNOEXCEPT = 0;

    virtual void configureWithFormat(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, DataType type, PluginFormat format, int maxBatchSize) TRTNOEXCEPT = 0;

    virtual int initialize() TRTNOEXCEPT = 0;

    virtual void terminate() TRTNOEXCEPT = 0;

    virtual size_t getWorkspaceSize(int maxBatchSize) const TRTNOEXCEPT = 0;

    virtual int enqueue(int batchSize, const void* const* inputs, void** outputs, void* workspace, cudaStream_t stream) TRTNOEXCEPT = 0;

    virtual size_t getSerializationSize() const TRTNOEXCEPT = 0;

    virtual void serialize(void* buffer) const TRTNOEXCEPT = 0;

    virtual void destroy() TRTNOEXCEPT = 0;

    virtual IPluginV2* clone() const TRTNOEXCEPT = 0;

    virtual void setPluginNamespace(const char* pluginNamespace) TRTNOEXCEPT = 0;

    virtual const char* getPluginNamespace() const TRTNOEXCEPT = 0;

protected:
    virtual ~IPluginV2() {}
};

class IPluginV2Ext : public IPluginV2
{
public:
    virtual nvinfer1::DataType getOutputDataType(int index, const nvinfer1::DataType* inputTypes, int nbInputs) const TRTNOEXCEPT = 0;

    virtual bool isOutputBroadcastAcrossBatch(int outputIndex, const bool* inputIsBroadcasted, int nbInputs) const TRTNOEXCEPT = 0;

    virtual bool canBroadcastInputAcrossBatch(int inputIndex) const TRTNOEXCEPT = 0;


    virtual void configurePlugin(const Dims* inputDims, int nbInputs, const Dims* outputDims,
                                 int nbOutputs, const DataType* inputTypes, const DataType* outputTypes,
                                 const bool* inputIsBroadcast, const bool* outputIsBroadcast, PluginFormat floatFormat, int maxBatchSize) TRTNOEXCEPT = 0;

    virtual ~IPluginV2Ext() {}

    virtual void attachToContext(cudnnContext* /*cudnn*/, cublasContext* /*cublas*/, IGpuAllocator* /*allocator*/) TRTNOEXCEPT {}

    virtual void detachFromContext() TRTNOEXCEPT {}

    virtual IPluginV2Ext* clone() const _TENSORRT_OVERRIDE TRTNOEXCEPT = 0;

protected:
    int getTensorRTVersion() const _TENSORRT_OVERRIDE TRTNOEXCEPT
    {
        return (static_cast<int>(PluginVersion::kV2_EXT) << 24 | (NV_TENSORRT_VERSION & 0xFFFFFF));
    }

    void configureWithFormat(const Dims* /*inputDims*/, int /*nbInputs*/, const Dims* /*outputDims*/,
                             int /*nbOutputs*/, DataType /*type*/, PluginFormat /*format*/, int /*maxBatchSize*/) _TENSORRT_OVERRIDE TRTNOEXCEPT {}
};

class IPluginV2IOExt : public IPluginV2Ext
{
public:
    virtual void configurePlugin(const PluginTensorDesc* in, int nbInput, const PluginTensorDesc* out, int nbOutput) TRTNOEXCEPT = 0;

    virtual bool supportsFormatCombination(int pos, const PluginTensorDesc* inOut, int nbInputs, int nbOutputs) const TRTNOEXCEPT = 0;

protected:
    TRT_DEPRECATED
    int getTensorRTVersion() const _TENSORRT_OVERRIDE
    {
        return (static_cast<int>(PluginVersion::kV2_IOEXT) << 24 | (NV_TENSORRT_VERSION & 0xFFFFFF));
    }

    TRT_DEPRECATED
    void configureWithFormat(
        const Dims*, int, const Dims*, int, DataType, PluginFormat, int) _TENSORRT_OVERRIDE _TENSORRT_FINAL
    {
    }

    TRT_DEPRECATED
    void configurePlugin(const Dims*, int, const Dims*, int, const DataType*, const DataType*, const bool*, const bool*,
        PluginFormat, int) _TENSORRT_OVERRIDE _TENSORRT_FINAL
    {
    }

    TRT_DEPRECATED
    bool supportsFormat(DataType, PluginFormat) const _TENSORRT_OVERRIDE _TENSORRT_FINAL
    {
        return false;
    }
};


enum class PluginFieldType : int
{
    kFLOAT16 = 0, 
    kFLOAT32 = 1, 
    kFLOAT64 = 2, 
    kINT8 = 3,    
    kINT16 = 4,   
    kINT32 = 5,   
    kCHAR = 6,    
    kDIMS = 7,    
    kUNKNOWN = 8
};

struct PluginField
{
    const char* name{nullptr};
    const void* data{nullptr};
    PluginFieldType type{PluginFieldType::kUNKNOWN};
    int32_t length{0};

    PluginField(const char* name_ = nullptr, const void* data_ = nullptr, const PluginFieldType type_ = PluginFieldType::kUNKNOWN, int32_t length_ = 0)
        : name(name_)
        , data(data_)
        , type(type_)
        , length(length_)
    {
    }
};

struct PluginFieldCollection
{
    int nbFields;              
    const PluginField* fields; 
};


class IPluginCreator
{
public:
    virtual int getTensorRTVersion() const TRTNOEXCEPT { return NV_TENSORRT_VERSION; }

    virtual const char* getPluginName() const TRTNOEXCEPT = 0;

    virtual const char* getPluginVersion() const TRTNOEXCEPT = 0;

    virtual const PluginFieldCollection* getFieldNames() TRTNOEXCEPT = 0;

    virtual IPluginV2* createPlugin(const char* name, const PluginFieldCollection* fc) TRTNOEXCEPT = 0;

    virtual IPluginV2* deserializePlugin(const char* name, const void* serialData, size_t serialLength) TRTNOEXCEPT = 0;

    virtual void setPluginNamespace(const char* pluginNamespace) TRTNOEXCEPT = 0;

    virtual const char* getPluginNamespace() const TRTNOEXCEPT = 0;

    virtual ~IPluginCreator() {}
};


class IPluginRegistry
{
public:
    virtual bool registerCreator(IPluginCreator& creator, const char* pluginNamespace) noexcept = 0;

    virtual IPluginCreator* const* getPluginCreatorList(int* numCreators) const noexcept = 0;

    virtual IPluginCreator* getPluginCreator(const char* pluginType, const char* pluginVersion, const char* pluginNamespace = "") noexcept = 0;

protected:
    virtual ~IPluginRegistry() noexcept {}

public:
    //
    virtual void setErrorRecorder(IErrorRecorder* recorder) noexcept = 0;

    virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
};


enum class TensorLocation : int
{
    kDEVICE = 0, 
    kHOST = 1,   
};

template <>
constexpr inline int EnumMax<TensorLocation>()
{
    return 2;
} 

class IGpuAllocator
{
public:
    virtual void* allocate(uint64_t size, uint64_t alignment, uint32_t flags) TRTNOEXCEPT = 0;

    virtual void free(void* memory) TRTNOEXCEPT = 0;

    virtual ~IGpuAllocator() {}
};

class ILogger
{
public:
    enum class Severity : int
    {
        kINTERNAL_ERROR = 0, 
        kERROR = 1,          
        kWARNING = 2,        
        kINFO = 3,           
        kVERBOSE = 4,        
    };

    virtual void log(Severity severity, const char* msg) TRTNOEXCEPT = 0;

    virtual ~ILogger() {}
};

template <>
constexpr inline int EnumMax<ILogger::Severity>()
{
    return 5;
} 

enum class ErrorCode : int
{
    kSUCCESS = 0,

    kUNSPECIFIED_ERROR = 1,

    kINTERNAL_ERROR = 2,

    kINVALID_ARGUMENT = 3,

    kINVALID_CONFIG = 4,

    kFAILED_ALLOCATION = 5,

    kFAILED_INITIALIZATION = 6,

    kFAILED_EXECUTION = 7,

    kFAILED_COMPUTATION = 8,

    kINVALID_STATE = 9,

    kUNSUPPORTED_STATE = 10,

};

template <>
constexpr inline int EnumMax<ErrorCode>()
{
    return 11;
} 


class IErrorRecorder
{
public:
    using ErrorDesc = const char*;

    using RefCount = int32_t;

    virtual ~IErrorRecorder() noexcept {};

    // Public API’s used to retrieve information from the error recorder.

    virtual int32_t getNbErrors() const noexcept = 0;

    virtual ErrorCode getErrorCode(int32_t errorIdx) const noexcept = 0;

    virtual ErrorDesc getErrorDesc(int32_t errorIdx) const noexcept = 0;

    virtual bool hasOverflowed() const noexcept = 0;

    virtual void clear() noexcept = 0;

    // API’s used by TensorRT to report Error information to the application.

    virtual bool reportError(ErrorCode val, ErrorDesc desc) noexcept = 0;

    virtual RefCount incRefCount() noexcept = 0;

    virtual RefCount decRefCount() noexcept = 0;

}; // class IErrorRecorder

} // namespace nvinfer1

extern "C" TENSORRTAPI void* createSafeInferRuntime_INTERNAL(void* logger, int version);

extern "C" TENSORRTAPI nvinfer1::ILogger* getLogger();

extern "C" TENSORRTAPI int getInferLibVersion();

extern "C" TENSORRTAPI nvinfer1::IPluginRegistry* getPluginRegistry();

namespace nvinfer1
{

template <typename T>
class PluginRegistrar
{
public:
    PluginRegistrar() { getPluginRegistry()->registerCreator(instance, ""); }
private:
    T instance{};
};

#define REGISTER_TENSORRT_PLUGIN(name) \
    static nvinfer1::PluginRegistrar<name> pluginRegistrar##name {}

} // namespace nvinfer1

#endif // NV_INFER_RUNTIME_COMMON_H