NvInfer.h

Go to the documentation of this file.

/*
 * Copyright 1993-2020 NVIDIA Corporation.  All rights reserved.
 *
 * NOTICE TO LICENSEE:
 *
 * This source code and/or documentation ("Licensed Deliverables") are
 * subject to NVIDIA intellectual property rights under U.S. and
 * international Copyright laws.
 *
 * These Licensed Deliverables contained herein is PROPRIETARY and
 * CONFIDENTIAL to NVIDIA and is being provided under the terms and
 * conditions of a form of NVIDIA software license agreement by and
 * between NVIDIA and Licensee ("License Agreement") or electronically
 * accepted by Licensee.  Notwithstanding any terms or conditions to
 * the contrary in the License Agreement, reproduction or disclosure
 * of the Licensed Deliverables to any third party without the express
 * written consent of NVIDIA is prohibited.
 *
 * NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
 * LICENSE AGREEMENT, NVIDIA MAKES NO REPRESENTATION ABOUT THE
 * SUITABILITY OF THESE LICENSED DELIVERABLES FOR ANY PURPOSE.  IT IS
 * PROVIDED "AS IS" WITHOUT EXPRESS OR IMPLIED WARRANTY OF ANY KIND.
 * NVIDIA DISCLAIMS ALL WARRANTIES WITH REGARD TO THESE LICENSED
 * DELIVERABLES, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY,
 * NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE.
 * NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE
 * LICENSE AGREEMENT, IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY
 * SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, OR ANY
 * DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
 * WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
 * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
 * OF THESE LICENSED DELIVERABLES.
 *
 * U.S. Government End Users.  These Licensed Deliverables are a
 * "commercial item" as that term is defined at 48 C.F.R. 2.101 (OCT
 * 1995), consisting of "commercial computer software" and "commercial
 * computer software documentation" as such terms are used in 48
 * C.F.R. 12.212 (SEPT 1995) and is provided to the U.S. Government
 * only as a commercial end item.  Consistent with 48 C.F.R.12.212 and
 * 48 C.F.R. 227.7202-1 through 227.7202-4 (JUNE 1995), all
 * U.S. Government End Users acquire the Licensed Deliverables with
 * only those rights set forth herein.
 *
 * Any use of the Licensed Deliverables in individual and commercial
 * software must include, in the user documentation and internal
 * comments to the code, the above Disclaimer and U.S. Government End
 * Users Notice.
 */
 
#ifndef NV_INFER_H
#define NV_INFER_H
 
#include "NvInferRuntime.h"
 
//
 
 
namespace nvinfer1
{
 
class Dims2 : public Dims
{
public:
    Dims2()
    {
        nbDims = 2;
        d[0] = d[1] = 0;
    }
 
    Dims2(int32_t d0, int32_t d1)
    {
        nbDims = 2;
        d[0] = d0;
        d[1] = d1;
    }
};
 
class DimsHW : public Dims2
{
public:
    DimsHW()
        : Dims2()
    {
        type[0] = type[1] = DimensionType::kSPATIAL;
    }
 
    DimsHW(int32_t height, int32_t width)
        : Dims2(height, width)
    {
        type[0] = type[1] = DimensionType::kSPATIAL;
    }
 
    int32_t& h()
    {
        return d[0];
    }
 
    int32_t h() const
    {
        return d[0];
    }
 
    int32_t& w()
    {
        return d[1];
    }
 
    int32_t w() const
    {
        return d[1];
    }
};
 
class Dims3 : public Dims
{
public:
    Dims3()
    {
        nbDims = 3;
        d[0] = d[1] = d[2] = 0;
    }
 
    Dims3(int32_t d0, int32_t d1, int32_t d2)
    {
        nbDims = 3;
        d[0] = d0;
        d[1] = d1;
        d[2] = d2;
    }
};
 
class TRT_DEPRECATED DimsCHW : public Dims3
{
public:
    DimsCHW()
        : Dims3()
    {
        type[0] = DimensionType::kCHANNEL;
        type[1] = type[2] = DimensionType::kSPATIAL;
    }
 
    DimsCHW(int32_t channels, int32_t height, int32_t width)
        : Dims3(channels, height, width)
    {
        type[0] = DimensionType::kCHANNEL;
        type[1] = type[2] = DimensionType::kSPATIAL;
    }
 
    int32_t& c()
    {
        return d[0];
    }
 
    int32_t c() const
    {
        return d[0];
    }
 
    int32_t& h()
    {
        return d[1];
    }
 
    int32_t h() const
    {
        return d[1];
    }
 
    int32_t& w()
    {
        return d[2];
    }
 
    int32_t w() const
    {
        return d[2];
    }
};
 
class Dims4 : public Dims
{
public:
    Dims4()
    {
        nbDims = 4;
        d[0] = d[1] = d[2] = d[3] = 0;
    }
 
    Dims4(int32_t d0, int32_t d1, int32_t d2, int32_t d3)
    {
        nbDims = 4;
        d[0] = d0;
        d[1] = d1;
        d[2] = d2;
        d[3] = d3;
    }
};
 
class TRT_DEPRECATED DimsNCHW : public Dims4
{
public:
    DimsNCHW()
        : Dims4()
    {
        type[0] = DimensionType::kINDEX;
        type[1] = DimensionType::kCHANNEL;
        type[2] = type[3] = DimensionType::kSPATIAL;
    }
 
    DimsNCHW(int32_t batchSize, int32_t channels, int32_t height, int32_t width)
        : Dims4(batchSize, channels, height, width)
    {
        type[0] = DimensionType::kINDEX;
        type[1] = DimensionType::kCHANNEL;
        type[2] = type[3] = DimensionType::kSPATIAL;
    }
 
    int32_t& n()
    {
        return d[0];
    }
 
    int32_t n() const
    {
        return d[0];
    }
 
    int32_t& c()
    {
        return d[1];
    }
 
    int32_t c() const
    {
        return d[1];
    }
 
    int32_t& h()
    {
        return d[2];
    }
 
    int32_t h() const
    {
        return d[2];
    }
 
    int32_t& w()
    {
        return d[3];
    }
 
    int32_t w() const
    {
        return d[3];
    }
};
 
enum class LayerType : int32_t
{
    kCONVOLUTION = 0,      
    kFULLY_CONNECTED = 1,  
    kACTIVATION = 2,       
    kPOOLING = 3,          
    kLRN = 4,              
    kSCALE = 5,            
    kSOFTMAX = 6,          
    kDECONVOLUTION = 7,    
    kCONCATENATION = 8,    
    kELEMENTWISE = 9,      
    kPLUGIN = 10,          
    kRNN = 11,             
    kUNARY = 12,           
    kPADDING = 13,         
    kSHUFFLE = 14,         
    kREDUCE = 15,          
    kTOPK = 16,            
    kGATHER = 17,          
    kMATRIX_MULTIPLY = 18, 
    kRAGGED_SOFTMAX = 19,  
    kCONSTANT = 20,        
    kRNN_V2 = 21,          
    kIDENTITY = 22,        
    kPLUGIN_V2 = 23,       
    kSLICE = 24,           
    kSHAPE = 25,           
    kPARAMETRIC_RELU = 26, 
    kRESIZE = 27,          
    kTRIP_LIMIT = 28,      
    kRECURRENCE = 29,      
    kITERATOR = 30,        
    kLOOP_OUTPUT = 31,     
    kSELECT = 32,          
    kFILL = 33             
};
 
template <>
constexpr inline int32_t EnumMax<LayerType>()
{
    return 34;
}
 
class ITensor
{
public:
    virtual void setName(const char* name) TRTNOEXCEPT = 0;
 
    virtual const char* getName() const TRTNOEXCEPT = 0;
 
    virtual void setDimensions(Dims dimensions) TRTNOEXCEPT = 0; // only valid for input tensors
 
    virtual Dims getDimensions() const TRTNOEXCEPT = 0;
 
    virtual void setType(DataType type) TRTNOEXCEPT = 0;
 
    virtual DataType getType() const TRTNOEXCEPT = 0;
 
    virtual bool setDynamicRange(float min, float max) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual float getDynamicRange() const TRTNOEXCEPT = 0;
 
    virtual bool isNetworkInput() const TRTNOEXCEPT = 0;
 
    virtual bool isNetworkOutput() const TRTNOEXCEPT = 0;
 
protected:
    virtual ~ITensor() {}
 
public:
    virtual void setBroadcastAcrossBatch(bool broadcastAcrossBatch) TRTNOEXCEPT = 0;
 
    virtual bool getBroadcastAcrossBatch() const TRTNOEXCEPT = 0;
 
    virtual TensorLocation getLocation() const TRTNOEXCEPT = 0;
 
    virtual void setLocation(TensorLocation location) TRTNOEXCEPT = 0;
 
    virtual bool dynamicRangeIsSet() const TRTNOEXCEPT = 0;
 
    virtual void resetDynamicRange() TRTNOEXCEPT = 0;
 
    virtual float getDynamicRangeMin() const TRTNOEXCEPT = 0;
 
    virtual float getDynamicRangeMax() const TRTNOEXCEPT = 0;
 
    virtual void setAllowedFormats(TensorFormats formats) TRTNOEXCEPT = 0;
 
    virtual TensorFormats getAllowedFormats() const TRTNOEXCEPT = 0;
 
    virtual bool isShapeTensor() const TRTNOEXCEPT = 0;
 
    virtual bool isExecutionTensor() const TRTNOEXCEPT = 0;
};
 
class ILayer
{
public:
    virtual LayerType getType() const TRTNOEXCEPT = 0;
 
    virtual void setName(const char* name) TRTNOEXCEPT = 0;
 
 
    virtual const char* getName() const TRTNOEXCEPT = 0;
 
    virtual int32_t getNbInputs() const TRTNOEXCEPT = 0;
 
    virtual ITensor* getInput(int32_t index) const TRTNOEXCEPT = 0;
 
    virtual int32_t getNbOutputs() const TRTNOEXCEPT = 0;
 
    virtual ITensor* getOutput(int32_t index) const TRTNOEXCEPT = 0;
 
    //
    virtual void setInput(int32_t index, ITensor& tensor) TRTNOEXCEPT = 0;
 
 
    virtual void setPrecision(DataType dataType) TRTNOEXCEPT = 0;
 
 
    virtual DataType getPrecision() const TRTNOEXCEPT = 0;
 
 
    virtual bool precisionIsSet() const TRTNOEXCEPT = 0;
 
 
    virtual void resetPrecision() TRTNOEXCEPT = 0;
 
 
    virtual void setOutputType(int32_t index, DataType dataType) TRTNOEXCEPT = 0;
 
 
    virtual DataType getOutputType(int32_t index) const TRTNOEXCEPT = 0;
 
 
    virtual bool outputTypeIsSet(int32_t index) const TRTNOEXCEPT = 0;
 
 
    virtual void resetOutputType(int32_t index) TRTNOEXCEPT = 0;
 
protected:
    virtual ~ILayer() {}
};
 
enum class PaddingMode : int32_t
{
    kEXPLICIT_ROUND_DOWN = 0, 
    kEXPLICIT_ROUND_UP = 1,   
    kSAME_UPPER = 2,          
    kSAME_LOWER = 3,          
    kCAFFE_ROUND_DOWN = 4,    
    kCAFFE_ROUND_UP = 5       
};
 
template <>
constexpr inline int32_t EnumMax<PaddingMode>()
{
    return 6;
}
 
class IConvolutionLayer : public ILayer
{
public:
    TRT_DEPRECATED virtual void setKernelSize(DimsHW kernelSize) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual DimsHW getKernelSize() const TRTNOEXCEPT = 0;
 
    virtual void setNbOutputMaps(int32_t nbOutputMaps) TRTNOEXCEPT = 0;
 
    virtual int32_t getNbOutputMaps() const TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual void setStride(DimsHW stride) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual DimsHW getStride() const TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual void setPadding(DimsHW padding) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual DimsHW getPadding() const TRTNOEXCEPT = 0;
 
    virtual void setNbGroups(int32_t nbGroups) TRTNOEXCEPT = 0;
 
    virtual int32_t getNbGroups() const TRTNOEXCEPT = 0;
 
    virtual void setKernelWeights(Weights weights) TRTNOEXCEPT = 0;
 
    virtual Weights getKernelWeights() const TRTNOEXCEPT = 0;
 
    virtual void setBiasWeights(Weights weights) TRTNOEXCEPT = 0;
 
    virtual Weights getBiasWeights() const TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual void setDilation(DimsHW dilation) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual DimsHW getDilation() const TRTNOEXCEPT = 0;
 
protected:
    virtual ~IConvolutionLayer() {}
 
public:
    virtual void setPrePadding(Dims padding) TRTNOEXCEPT = 0;
 
    virtual Dims getPrePadding() const TRTNOEXCEPT = 0;
 
    virtual void setPostPadding(Dims padding) TRTNOEXCEPT = 0;
 
    virtual Dims getPostPadding() const TRTNOEXCEPT = 0;
 
    virtual void setPaddingMode(PaddingMode paddingMode) TRTNOEXCEPT = 0;
 
    virtual PaddingMode getPaddingMode() const TRTNOEXCEPT = 0;
 
    virtual void setKernelSizeNd(Dims kernelSize) TRTNOEXCEPT = 0;
 
    virtual Dims getKernelSizeNd() const TRTNOEXCEPT = 0;
 
    virtual void setStrideNd(Dims stride) TRTNOEXCEPT = 0;
 
    virtual Dims getStrideNd() const TRTNOEXCEPT = 0;
 
    virtual void setPaddingNd(Dims padding) TRTNOEXCEPT = 0;
 
    virtual Dims getPaddingNd() const TRTNOEXCEPT = 0;
 
    virtual void setDilationNd(Dims dilation) TRTNOEXCEPT = 0;
 
    virtual Dims getDilationNd() const TRTNOEXCEPT = 0;
 
    void setInput(int32_t index, ITensor& tensor) _TENSORRT_OVERRIDE TRTNOEXCEPT = 0;
};
 
class IFullyConnectedLayer : public ILayer
{
public:
    virtual void setNbOutputChannels(int32_t nbOutputs) TRTNOEXCEPT = 0;
 
    virtual int32_t getNbOutputChannels() const TRTNOEXCEPT = 0;
 
    virtual void setKernelWeights(Weights weights) TRTNOEXCEPT = 0;
 
    virtual Weights getKernelWeights() const TRTNOEXCEPT = 0;
 
    virtual void setBiasWeights(Weights weights) TRTNOEXCEPT = 0;
 
    virtual Weights getBiasWeights() const TRTNOEXCEPT = 0;
 
protected:
    virtual ~IFullyConnectedLayer() {}
 
public:
    void setInput(int32_t index, ITensor& tensor) _TENSORRT_OVERRIDE TRTNOEXCEPT = 0;
};
 
class IActivationLayer : public ILayer
{
public:
    virtual void setActivationType(ActivationType type) TRTNOEXCEPT = 0;
 
    virtual ActivationType getActivationType() const TRTNOEXCEPT = 0;
 
protected:
    virtual ~IActivationLayer() {}
public:
    virtual void setAlpha(float alpha) TRTNOEXCEPT = 0;
 
    virtual void setBeta(float beta) TRTNOEXCEPT = 0;
 
    virtual float getAlpha() const TRTNOEXCEPT = 0;
 
    virtual float getBeta() const TRTNOEXCEPT = 0;
};
 
enum class PoolingType : int32_t
{
    kMAX = 0,              // Maximum over elements
    kAVERAGE = 1,          // Average over elements. If the tensor is padded, the count includes the padding
    kMAX_AVERAGE_BLEND = 2 // Blending between max and average pooling: (1-blendFactor)*maxPool + blendFactor*avgPool
};
 
template <>
constexpr inline int32_t EnumMax<PoolingType>()
{
    return 3;
}
 
class IPoolingLayer : public ILayer
{
public:
    virtual void setPoolingType(PoolingType type) TRTNOEXCEPT = 0;
 
    virtual PoolingType getPoolingType() const TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual void setWindowSize(DimsHW windowSize) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual DimsHW getWindowSize() const TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual void setStride(DimsHW stride) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual DimsHW getStride() const TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual void setPadding(DimsHW padding) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual DimsHW getPadding() const TRTNOEXCEPT = 0;
 
    virtual void setBlendFactor(float blendFactor) TRTNOEXCEPT = 0;
 
    virtual float getBlendFactor() const TRTNOEXCEPT = 0;
 
    virtual void setAverageCountExcludesPadding(bool exclusive) TRTNOEXCEPT = 0;
 
    virtual bool getAverageCountExcludesPadding() const TRTNOEXCEPT = 0;
 
protected:
    virtual ~IPoolingLayer() {}
 
public:
    virtual void setPrePadding(Dims padding) TRTNOEXCEPT = 0;
 
    virtual Dims getPrePadding() const TRTNOEXCEPT = 0;
 
    virtual void setPostPadding(Dims padding) TRTNOEXCEPT = 0;
 
    virtual Dims getPostPadding() const TRTNOEXCEPT = 0;
 
    virtual void setPaddingMode(PaddingMode paddingMode) TRTNOEXCEPT = 0;
 
    virtual PaddingMode getPaddingMode() const TRTNOEXCEPT = 0;
 
    virtual void setWindowSizeNd(Dims windowSize) TRTNOEXCEPT = 0;
 
    virtual Dims getWindowSizeNd() const TRTNOEXCEPT = 0;
 
    virtual void setStrideNd(Dims stride) TRTNOEXCEPT = 0;
 
    virtual Dims getStrideNd() const TRTNOEXCEPT = 0;
 
    virtual void setPaddingNd(Dims padding) TRTNOEXCEPT = 0;
 
    virtual Dims getPaddingNd() const TRTNOEXCEPT = 0;
};
 
class ILRNLayer : public ILayer
{
public:
    virtual void setWindowSize(int32_t windowSize) TRTNOEXCEPT = 0;
 
    virtual int32_t getWindowSize() const TRTNOEXCEPT = 0;
 
    virtual void setAlpha(float alpha) TRTNOEXCEPT = 0;
 
    virtual float getAlpha() const TRTNOEXCEPT = 0;
 
    virtual void setBeta(float beta) TRTNOEXCEPT = 0;
 
    virtual float getBeta() const TRTNOEXCEPT = 0;
 
    virtual void setK(float k) TRTNOEXCEPT = 0;
 
    virtual float getK() const TRTNOEXCEPT = 0;
 
protected:
    virtual ~ILRNLayer() {}
};
 
enum class ScaleMode : int32_t
{
    kUNIFORM = 0,    
    kCHANNEL = 1,    
    kELEMENTWISE = 2 
};
 
template <>
constexpr inline int32_t EnumMax<ScaleMode>()
{
    return 3;
}
 
class IScaleLayer : public ILayer
{
public:
    virtual void setMode(ScaleMode mode) TRTNOEXCEPT = 0;
 
    virtual ScaleMode getMode() const TRTNOEXCEPT = 0;
 
    virtual void setShift(Weights shift) TRTNOEXCEPT = 0;
 
    virtual Weights getShift() const TRTNOEXCEPT = 0;
 
    virtual void setScale(Weights scale) TRTNOEXCEPT = 0;
 
    virtual Weights getScale() const TRTNOEXCEPT = 0;
 
    virtual void setPower(Weights power) TRTNOEXCEPT = 0;
 
    virtual Weights getPower() const TRTNOEXCEPT = 0;
 
protected:
    virtual ~IScaleLayer() {}
 
public:
    virtual int32_t getChannelAxis() const TRTNOEXCEPT = 0;
};
 
class ISoftMaxLayer : public ILayer
{
protected:
    virtual ~ISoftMaxLayer() {}
public:
    virtual void setAxes(uint32_t axes) TRTNOEXCEPT = 0;
 
    virtual uint32_t getAxes() const TRTNOEXCEPT = 0;
};
 
class IConcatenationLayer : public ILayer
{
protected:
    virtual ~IConcatenationLayer() {}
 
public:
    virtual void setAxis(int32_t axis) TRTNOEXCEPT = 0;
 
    virtual int32_t getAxis() const TRTNOEXCEPT = 0;
};
 
class IDeconvolutionLayer : public ILayer
{
public:
    TRT_DEPRECATED virtual void setKernelSize(DimsHW kernelSize) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual DimsHW getKernelSize() const TRTNOEXCEPT = 0;
 
    virtual void setNbOutputMaps(int32_t nbOutputMaps) TRTNOEXCEPT = 0;
 
    virtual int32_t getNbOutputMaps() const TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual void setStride(DimsHW stride) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual DimsHW getStride() const TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual void setPadding(DimsHW padding) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual DimsHW getPadding() const TRTNOEXCEPT = 0;
 
    virtual void setNbGroups(int32_t nbGroups) TRTNOEXCEPT = 0;
 
    virtual int32_t getNbGroups() const TRTNOEXCEPT = 0;
 
    virtual void setKernelWeights(Weights weights) TRTNOEXCEPT = 0;
 
    virtual Weights getKernelWeights() const TRTNOEXCEPT = 0;
 
    virtual void setBiasWeights(Weights weights) TRTNOEXCEPT = 0;
 
    virtual Weights getBiasWeights() const TRTNOEXCEPT = 0;
 
protected:
    virtual ~IDeconvolutionLayer() {}
 
public:
    virtual void setPrePadding(Dims padding) TRTNOEXCEPT = 0;
 
    virtual Dims getPrePadding() const TRTNOEXCEPT = 0;
 
    virtual void setPostPadding(Dims padding) TRTNOEXCEPT = 0;
 
    virtual Dims getPostPadding() const TRTNOEXCEPT = 0;
 
    virtual void setPaddingMode(PaddingMode paddingMode) TRTNOEXCEPT = 0;
 
    virtual PaddingMode getPaddingMode() const TRTNOEXCEPT = 0;
 
    virtual void setKernelSizeNd(Dims kernelSize) TRTNOEXCEPT = 0;
 
    virtual Dims getKernelSizeNd() const TRTNOEXCEPT = 0;
 
    virtual void setStrideNd(Dims stride) TRTNOEXCEPT = 0;
 
    virtual Dims getStrideNd() const TRTNOEXCEPT = 0;
 
    virtual void setPaddingNd(Dims padding) TRTNOEXCEPT = 0;
 
    virtual Dims getPaddingNd() const TRTNOEXCEPT = 0;
 
    void setInput(int32_t index, ITensor& tensor) _TENSORRT_OVERRIDE TRTNOEXCEPT = 0;
 
    virtual void setDilationNd(Dims dilation) TRTNOEXCEPT = 0;
 
    virtual Dims getDilationNd() const TRTNOEXCEPT = 0;
};
 
enum class ElementWiseOperation : int32_t
{
    kSUM = 0,       
    kPROD = 1,      
    kMAX = 2,       
    kMIN = 3,       
    kSUB = 4,       
    kDIV = 5,       
    kPOW = 6,       
    kFLOOR_DIV = 7, 
    kAND = 8,       
    kOR = 9,        
    kXOR = 10,      
    kEQUAL = 11,    
    kGREATER = 12,  
    kLESS = 13      
};
 
template <>
constexpr inline int32_t EnumMax<ElementWiseOperation>()
{
    return 14;
}
 
//！
class IElementWiseLayer : public ILayer
{
public:
    virtual void setOperation(ElementWiseOperation op) TRTNOEXCEPT = 0;
 
    virtual ElementWiseOperation getOperation() const TRTNOEXCEPT = 0;
 
protected:
    virtual ~IElementWiseLayer() {}
};
 
class IGatherLayer : public ILayer
{
public:
    virtual void setGatherAxis(int32_t axis) TRTNOEXCEPT = 0;
 
    virtual int32_t getGatherAxis() const TRTNOEXCEPT = 0;
 
    virtual void setNbElementWiseDims(int32_t k) TRTNOEXCEPT = 0;
 
    virtual int32_t getNbElementWiseDims() const TRTNOEXCEPT = 0;
 
protected:
    virtual ~IGatherLayer() {}
};
 
enum class RNNOperation : int32_t
{
    kRELU = 0, 
    kTANH = 1, 
    kLSTM = 2, 
    kGRU = 3   
};
 
template <>
constexpr inline int32_t EnumMax<RNNOperation>()
{
    return 4;
}
 
enum class RNNDirection : int32_t
{
    kUNIDIRECTION = 0, 
    kBIDIRECTION = 1   
};
 
template <>
constexpr inline int32_t EnumMax<RNNDirection>()
{
    return 2;
}
 
enum class RNNInputMode : int32_t
{
    kLINEAR = 0, 
    kSKIP = 1    
};
 
template <>
constexpr inline int32_t EnumMax<RNNInputMode>()
{
    return 2;
}
 
class TRT_DEPRECATED IRNNLayer : public ILayer
{
public:
    virtual uint32_t getLayerCount() const TRTNOEXCEPT = 0;
 
    virtual std::size_t getHiddenSize() const TRTNOEXCEPT = 0;
 
    virtual int32_t getSeqLength() const TRTNOEXCEPT = 0;
 
    virtual void setOperation(RNNOperation op) TRTNOEXCEPT = 0;
 
    virtual RNNOperation getOperation() const TRTNOEXCEPT = 0;
 
    virtual void setInputMode(RNNInputMode op) TRTNOEXCEPT = 0;
 
    virtual RNNInputMode getInputMode() const TRTNOEXCEPT = 0;
 
    virtual void setDirection(RNNDirection op) TRTNOEXCEPT = 0;
 
    virtual RNNDirection getDirection() const TRTNOEXCEPT = 0;
 
    virtual void setWeights(Weights weights) TRTNOEXCEPT = 0;
 
    virtual Weights getWeights() const TRTNOEXCEPT = 0;
 
    virtual void setBias(Weights bias) TRTNOEXCEPT = 0;
 
    virtual Weights getBias() const TRTNOEXCEPT = 0;
 
    virtual int32_t getDataLength() const TRTNOEXCEPT = 0;
 
    virtual void setHiddenState(ITensor& hidden) TRTNOEXCEPT = 0;
 
    virtual ITensor* getHiddenState() const TRTNOEXCEPT = 0;
 
    virtual void setCellState(ITensor& cell) TRTNOEXCEPT = 0;
 
    virtual ITensor* getCellState() const TRTNOEXCEPT = 0;
 
protected:
    virtual ~IRNNLayer() {}
};
 
enum class RNNGateType : int32_t
{
    kINPUT = 0,  
    kOUTPUT = 1, 
    kFORGET = 2, 
    kUPDATE = 3, 
    kRESET = 4,  
    kCELL = 5,   
    kHIDDEN = 6  
};
 
template <>
constexpr inline int32_t EnumMax<RNNGateType>()
{
    return 7;
}
 
class IRNNv2Layer : public ILayer
{
public:
    virtual int32_t getLayerCount() const TRTNOEXCEPT = 0;   //< Get the layer count of the RNN
    virtual int32_t getHiddenSize() const TRTNOEXCEPT = 0;   //< Get the hidden size of the RNN
    virtual int32_t getMaxSeqLength() const TRTNOEXCEPT = 0; //< Get the maximum sequence length of the RNN
    virtual int32_t getDataLength() const TRTNOEXCEPT = 0;   //< Get the maximum data length of the RNN
 
    virtual void setSequenceLengths(ITensor& seqLengths) TRTNOEXCEPT = 0;
 
    virtual ITensor* getSequenceLengths() const TRTNOEXCEPT = 0;
 
    virtual void setOperation(RNNOperation op) TRTNOEXCEPT = 0;
 
    virtual RNNOperation getOperation() const TRTNOEXCEPT = 0;
 
    virtual void setInputMode(RNNInputMode op) TRTNOEXCEPT = 0;
 
    virtual RNNInputMode getInputMode() const TRTNOEXCEPT = 0;
 
    virtual void setDirection(RNNDirection op) TRTNOEXCEPT = 0;
 
    virtual RNNDirection getDirection() const TRTNOEXCEPT = 0;
 
    virtual void setWeightsForGate(int32_t layerIndex, RNNGateType gate, bool isW, Weights weights) TRTNOEXCEPT = 0;
 
    virtual Weights getWeightsForGate(int32_t layerIndex, RNNGateType gate, bool isW) const TRTNOEXCEPT = 0;
 
    virtual void setBiasForGate(int32_t layerIndex, RNNGateType gate, bool isW, Weights bias) TRTNOEXCEPT = 0;
 
    virtual Weights getBiasForGate(int32_t layerIndex, RNNGateType gate, bool isW) const TRTNOEXCEPT = 0;
 
    virtual void setHiddenState(ITensor& hidden) TRTNOEXCEPT = 0;
 
    virtual ITensor* getHiddenState() const TRTNOEXCEPT = 0;
 
    virtual void setCellState(ITensor& cell) TRTNOEXCEPT = 0;
 
    virtual ITensor* getCellState() const TRTNOEXCEPT = 0;
 
protected:
    virtual ~IRNNv2Layer() {}
};
 
class TRT_DEPRECATED IOutputDimensionsFormula
{
public:
    virtual DimsHW compute(DimsHW inputDims, DimsHW kernelSize, DimsHW stride, DimsHW padding, DimsHW dilation, const char* layerName) const TRTNOEXCEPT = 0;
 
    virtual ~IOutputDimensionsFormula() {}
};
 
class TRT_DEPRECATED IPluginLayer : public ILayer
{
public:
    virtual IPlugin& getPlugin() TRTNOEXCEPT = 0;
 
protected:
    virtual ~IPluginLayer() {}
};
 
class IPluginV2Layer : public ILayer
{
public:
    virtual IPluginV2& getPlugin() TRTNOEXCEPT = 0;
 
protected:
    virtual ~IPluginV2Layer() {}
};
 
enum class UnaryOperation : int32_t
{
    kEXP = 0,    
    kLOG = 1,    
    kSQRT = 2,   
    kRECIP = 3,  
    kABS = 4,    
    kNEG = 5,    
    kSIN = 6,    
    kCOS = 7,    
    kTAN = 8,    
    kSINH = 9,   
    kCOSH = 10,  
    kASIN = 11,  
    kACOS = 12,  
    kATAN = 13,  
    kASINH = 14, 
    kACOSH = 15, 
    kATANH = 16, 
    kCEIL = 17,  
    kFLOOR = 18, 
    kERF = 19,   
    kNOT = 20    
};
 
template <>
constexpr inline int32_t EnumMax<UnaryOperation>()
{
    return 21;
}
 
class IUnaryLayer : public ILayer
{
public:
    virtual void setOperation(UnaryOperation op) TRTNOEXCEPT = 0;
 
    virtual UnaryOperation getOperation() const TRTNOEXCEPT = 0;
 
protected:
    virtual ~IUnaryLayer() {}
};
 
enum class ReduceOperation : int32_t
{
    kSUM = 0,
    kPROD = 1,
    kMAX = 2,
    kMIN = 3,
    kAVG = 4
};
 
template <>
constexpr inline int32_t EnumMax<ReduceOperation>()
{
    return 5;
}
 
class IReduceLayer : public ILayer
{
public:
    virtual void setOperation(ReduceOperation op) TRTNOEXCEPT = 0;
 
    virtual ReduceOperation getOperation() const TRTNOEXCEPT = 0;
 
    virtual void setReduceAxes(uint32_t reduceAxes) TRTNOEXCEPT = 0;
 
    virtual uint32_t getReduceAxes() const TRTNOEXCEPT = 0;
 
    virtual void setKeepDimensions(bool keepDimensions) TRTNOEXCEPT = 0;
 
    virtual bool getKeepDimensions() const TRTNOEXCEPT = 0;
 
protected:
    virtual ~IReduceLayer() {}
};
 
class IPaddingLayer : public ILayer
{
public:
    TRT_DEPRECATED virtual void setPrePadding(DimsHW padding) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual DimsHW getPrePadding() const TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual void setPostPadding(DimsHW padding) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual DimsHW getPostPadding() const TRTNOEXCEPT = 0;
 
protected:
    virtual ~IPaddingLayer() {}
 
public:
    virtual void setPrePaddingNd(Dims padding) TRTNOEXCEPT = 0;
 
    virtual Dims getPrePaddingNd() const TRTNOEXCEPT = 0;
 
    virtual void setPostPaddingNd(Dims padding) TRTNOEXCEPT = 0;
 
    virtual Dims getPostPaddingNd() const TRTNOEXCEPT = 0;
};
 
struct Permutation
{
    int32_t order[Dims::MAX_DIMS];
};
 
class IShuffleLayer : public ILayer
{
public:
    virtual void setFirstTranspose(Permutation permutation) TRTNOEXCEPT = 0;
 
    virtual Permutation getFirstTranspose() const TRTNOEXCEPT = 0;
 
    virtual void setReshapeDimensions(Dims dimensions) TRTNOEXCEPT = 0;
 
    virtual Dims getReshapeDimensions() const TRTNOEXCEPT = 0;
 
    //
    void setInput(int32_t index, ITensor& tensor) _TENSORRT_OVERRIDE TRTNOEXCEPT = 0;
 
    virtual void setSecondTranspose(Permutation permutation) TRTNOEXCEPT = 0;
 
    virtual Permutation getSecondTranspose() const TRTNOEXCEPT = 0;
 
protected:
    virtual ~IShuffleLayer() {}
 
public:
    virtual void setZeroIsPlaceholder(bool zeroIsPlaceholder) = 0;
 
    virtual bool getZeroIsPlaceholder() const = 0;
};
 
enum class SliceMode : int32_t
{
    kDEFAULT = 0, 
    kWRAP = 1,    
};
 
template <>
constexpr inline int32_t EnumMax<SliceMode>()
{
    return 2;
}
 
class ISliceLayer : public ILayer
{
public:
    virtual void setStart(Dims start) TRTNOEXCEPT = 0;
 
    virtual Dims getStart() const TRTNOEXCEPT = 0;
 
    virtual void setSize(Dims size) TRTNOEXCEPT = 0;
 
    virtual Dims getSize() const TRTNOEXCEPT = 0;
 
    virtual void setStride(Dims stride) TRTNOEXCEPT = 0;
 
    virtual Dims getStride() const TRTNOEXCEPT = 0;
 
    virtual void setMode(SliceMode mode) TRTNOEXCEPT = 0;
 
    virtual SliceMode getMode() const TRTNOEXCEPT = 0;
 
    void setInput(int32_t index, ITensor& tensor) _TENSORRT_OVERRIDE TRTNOEXCEPT = 0;
 
protected:
    virtual ~ISliceLayer() {}
};
 
class IShapeLayer : public ILayer
{
protected:
    virtual ~IShapeLayer() {}
};
 
enum class TopKOperation : int32_t
{
    kMAX = 0, 
    kMIN = 1, 
};
 
template <>
constexpr inline int32_t EnumMax<TopKOperation>()
{
    return 2;
}
 
class ITopKLayer : public ILayer
{
public:
    virtual void setOperation(TopKOperation op) TRTNOEXCEPT = 0;
 
    virtual TopKOperation getOperation() const TRTNOEXCEPT = 0;
 
    virtual void setK(int32_t k) TRTNOEXCEPT = 0;
 
    virtual int32_t getK() const TRTNOEXCEPT = 0;
 
    virtual void setReduceAxes(uint32_t reduceAxes) TRTNOEXCEPT = 0;
 
    virtual uint32_t getReduceAxes() const TRTNOEXCEPT = 0;
 
protected:
    virtual ~ITopKLayer() {}
};
 
enum class MatrixOperation : int32_t
{
    kNONE,
 
    kTRANSPOSE,
 
    kVECTOR
};
 
template <>
constexpr inline int32_t EnumMax<MatrixOperation>()
{
    return 3;
}
 
class IMatrixMultiplyLayer : public ILayer
{
public:
    virtual void setOperation(int32_t index, MatrixOperation op) TRTNOEXCEPT = 0;
 
    virtual MatrixOperation getOperation(int32_t index) const TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual void setTranspose(int32_t index, bool val) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual bool getTranspose(int32_t index) const TRTNOEXCEPT = 0;
 
protected:
    virtual ~IMatrixMultiplyLayer() {}
};
 
class IRaggedSoftMaxLayer : public ILayer
{
protected:
    virtual ~IRaggedSoftMaxLayer() {}
};
 
class IIdentityLayer : public ILayer
{
protected:
    virtual ~IIdentityLayer() {}
};
 
class IConstantLayer : public ILayer
{
public:
    virtual void setWeights(Weights weights) TRTNOEXCEPT = 0;
 
    virtual Weights getWeights() const TRTNOEXCEPT = 0;
 
    virtual void setDimensions(Dims dimensions) TRTNOEXCEPT = 0;
 
    virtual Dims getDimensions() const TRTNOEXCEPT = 0;
 
protected:
    virtual ~IConstantLayer() {}
};
 
class IParametricReLULayer : public ILayer
{
protected:
    virtual ~IParametricReLULayer() noexcept {}
};
 
enum class ResizeMode : int32_t
{
    kNEAREST = 0, // ND (0 < N <= 8) nearest neighbor resizing.
    kLINEAR = 1   // Can handle linear (1D), bilinear (2D), and trilinear (3D) resizing.
};
 
template <>
constexpr inline int32_t EnumMax<ResizeMode>()
{
    return 2;
}
 
class IResizeLayer : public ILayer
{
public:
    virtual void setOutputDimensions(Dims dimensions) TRTNOEXCEPT = 0;
 
    virtual Dims getOutputDimensions() const TRTNOEXCEPT = 0;
 
    virtual void setScales(const float* scales, int32_t nbScales) TRTNOEXCEPT = 0;
 
    virtual int32_t getScales(int32_t size, float* scales) const TRTNOEXCEPT = 0;
 
    virtual void setResizeMode(ResizeMode resizeMode) TRTNOEXCEPT = 0;
 
    virtual ResizeMode getResizeMode() const TRTNOEXCEPT = 0;
 
    virtual void setAlignCorners(bool alignCorners) TRTNOEXCEPT = 0;
 
    virtual bool getAlignCorners() const TRTNOEXCEPT = 0;
 
    void setInput(int32_t index, ITensor& tensor) _TENSORRT_OVERRIDE TRTNOEXCEPT = 0;
 
protected:
    virtual ~IResizeLayer() {}
};
 
enum class LoopOutput : int32_t
{
    kLAST_VALUE = 0,
 
    kCONCATENATE = 1,
 
    kREVERSE = 2
};
 
template <>
constexpr inline int32_t EnumMax<LoopOutput>()
{
    return 3;
}
 
enum class TripLimit : int32_t
{
    // Tensor is scalar of type kINT32 that contains the trip count.
    kCOUNT = 0,
 
    // Tensor is a scalar of type kBOOL. Loop terminates when value is false.
    kWHILE = 1
};
 
template <>
constexpr inline int32_t EnumMax<TripLimit>()
{
    return 2;
}
 
class ILoop;
 
class ILoopBoundaryLayer : public ILayer
{
public:
    virtual ILoop* getLoop() const noexcept = 0;
};
 
class IRecurrenceLayer : public ILoopBoundaryLayer
{
public:
    //
    void setInput(int32_t index, ITensor& tensor) _TENSORRT_OVERRIDE TRTNOEXCEPT = 0;
};
 
class ILoopOutputLayer : public ILoopBoundaryLayer
{
public:
    virtual LoopOutput getLoopOutput() const noexcept = 0;
 
    virtual void setAxis(int32_t axis) noexcept = 0;
 
    virtual int32_t getAxis() const noexcept = 0;
 
    //
    void setInput(int32_t index, ITensor& tensor) _TENSORRT_OVERRIDE TRTNOEXCEPT = 0;
};
 
class ITripLimitLayer : public ILoopBoundaryLayer
{
public:
    virtual TripLimit getTripLimit() const noexcept = 0;
};
 
class IIteratorLayer : public ILoopBoundaryLayer
{
public:
    virtual void setAxis(int32_t axis) noexcept = 0;
 
    virtual int32_t getAxis() const noexcept = 0;
 
    virtual void setReverse(bool reverse) noexcept = 0;
 
    virtual bool getReverse() const noexcept = 0;
};
 
class ILoop
{
public:
    //
    virtual IRecurrenceLayer* addRecurrence(ITensor& initialValue) noexcept = 0;
 
    virtual ITripLimitLayer* addTripLimit(ITensor& tensor, TripLimit limit) noexcept = 0;
 
    virtual IIteratorLayer* addIterator(ITensor& tensor, int32_t axis = 0, bool reverse = false) noexcept = 0;
 
    virtual ILoopOutputLayer* addLoopOutput(ITensor& tensor, LoopOutput outputKind, int32_t axis = 0) noexcept = 0;
 
    virtual void setName(const char* name) noexcept = 0;
 
    virtual const char* getName() const noexcept = 0;
 
protected:
    virtual ~ILoop() {}
};
 
class ISelectLayer : public ILayer
{
protected:
    virtual ~ISelectLayer() {}
};
 
enum class FillOperation : int32_t
{
    kLINSPACE = 0,      
    kRANDOM_UNIFORM = 1 
};
 
template <>
constexpr inline int32_t EnumMax<FillOperation>()
{
    return 2;
}
 
class IFillLayer : public ILayer
{
public:
    //
    virtual void setDimensions(Dims dimensions) noexcept = 0;
 
    virtual Dims getDimensions() const noexcept = 0;
 
    virtual void setOperation(FillOperation op) noexcept = 0;
 
    virtual FillOperation getOperation() const noexcept = 0;
 
    //
    virtual void setAlpha(double alpha) noexcept = 0;
 
    virtual double getAlpha() const noexcept = 0;
 
    virtual void setBeta(double beta) noexcept = 0;
 
    virtual double getBeta() const noexcept = 0;
 
    void setInput(int32_t index, ITensor& tensor) _TENSORRT_OVERRIDE TRTNOEXCEPT = 0;
 
protected:
    virtual ~IFillLayer() {}
};
 
class INetworkDefinition
{
public:
 
    virtual ITensor* addInput(const char* name, DataType type, Dims dimensions) TRTNOEXCEPT = 0;
 
    virtual void markOutput(ITensor& tensor) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual IConvolutionLayer* addConvolution(ITensor& input, int32_t nbOutputMaps, DimsHW kernelSize,
        Weights kernelWeights, Weights biasWeights) TRTNOEXCEPT = 0;
 
    virtual IFullyConnectedLayer* addFullyConnected(
        ITensor& input, int32_t nbOutputs, Weights kernelWeights, Weights biasWeights) TRTNOEXCEPT = 0;
 
    virtual IActivationLayer* addActivation(ITensor& input, ActivationType type) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual IPoolingLayer* addPooling(
        ITensor& input, PoolingType type, DimsHW windowSize) TRTNOEXCEPT = 0;
 
    virtual ILRNLayer* addLRN(ITensor& input, int32_t window, float alpha, float beta, float k) TRTNOEXCEPT = 0;
 
    virtual IScaleLayer* addScale(ITensor& input, ScaleMode mode, Weights shift, Weights scale, Weights power) TRTNOEXCEPT = 0;
 
    virtual ISoftMaxLayer* addSoftMax(ITensor& input) TRTNOEXCEPT = 0;
 
    virtual IConcatenationLayer* addConcatenation(ITensor* const* inputs, int32_t nbInputs) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual IDeconvolutionLayer* addDeconvolution(ITensor& input, int32_t nbOutputMaps,
        DimsHW kernelSize, Weights kernelWeights, Weights biasWeights) TRTNOEXCEPT = 0;
 
    virtual IElementWiseLayer* addElementWise(ITensor& input1, ITensor& input2, ElementWiseOperation op) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual IRNNLayer* addRNN(ITensor& inputs, int32_t layerCount, std::size_t hiddenSize,
        int32_t maxSeqLen, RNNOperation op, RNNInputMode mode, RNNDirection dir, Weights weights,
        Weights bias) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual IPluginLayer* addPlugin(
        ITensor* const* inputs, int32_t nbInputs, IPlugin& plugin) TRTNOEXCEPT = 0;
 
    virtual IUnaryLayer* addUnary(ITensor& input, UnaryOperation operation) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual IPaddingLayer* addPadding(
        ITensor& input, DimsHW prePadding, DimsHW postPadding) TRTNOEXCEPT = 0;
 
    virtual IShuffleLayer* addShuffle(ITensor& input) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual void setPoolingOutputDimensionsFormula(IOutputDimensionsFormula* formula) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual IOutputDimensionsFormula& getPoolingOutputDimensionsFormula() const TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual void setConvolutionOutputDimensionsFormula(
        IOutputDimensionsFormula* formula) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual IOutputDimensionsFormula& getConvolutionOutputDimensionsFormula() const TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual void setDeconvolutionOutputDimensionsFormula(
        IOutputDimensionsFormula* formula) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual IOutputDimensionsFormula& getDeconvolutionOutputDimensionsFormula() const TRTNOEXCEPT = 0;
 
    virtual int32_t getNbLayers() const TRTNOEXCEPT = 0;
 
    virtual ILayer* getLayer(int32_t index) const TRTNOEXCEPT = 0;
 
    virtual int32_t getNbInputs() const TRTNOEXCEPT = 0;
 
    virtual ITensor* getInput(int32_t index) const TRTNOEXCEPT = 0; // adding inputs invalidates indexing here
 
    virtual int32_t getNbOutputs() const TRTNOEXCEPT = 0;
 
    virtual ITensor* getOutput(int32_t index) const TRTNOEXCEPT = 0; // adding outputs invalidates indexing here
 
    virtual void destroy() TRTNOEXCEPT = 0;
 
protected:
    virtual ~INetworkDefinition() {}
 
public:
    virtual IReduceLayer* addReduce(ITensor& input, ReduceOperation operation, uint32_t reduceAxes, bool keepDimensions) TRTNOEXCEPT = 0;
 
    virtual ITopKLayer* addTopK(ITensor& input, TopKOperation op, int32_t k, uint32_t reduceAxes) TRTNOEXCEPT = 0;
 
    virtual IGatherLayer* addGather(ITensor& data, ITensor& indices, int32_t axis) TRTNOEXCEPT = 0;
 
    virtual IRaggedSoftMaxLayer* addRaggedSoftMax(ITensor& input, ITensor& bounds) TRTNOEXCEPT = 0;
 
    virtual IMatrixMultiplyLayer* addMatrixMultiply(
        ITensor& input0, MatrixOperation op0, ITensor& input1, MatrixOperation op1) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual IMatrixMultiplyLayer* addMatrixMultiply(
        ITensor& input0, bool transpose0, ITensor& input1, bool transpose1) TRTNOEXCEPT = 0;
 
    virtual IConstantLayer* addConstant(Dims dimensions, Weights weights) TRTNOEXCEPT = 0;
 
    virtual IRNNv2Layer* addRNNv2(
        ITensor& input, int32_t layerCount, int32_t hiddenSize, int32_t maxSeqLen, RNNOperation op) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual IPluginLayer* addPluginExt(
        ITensor* const* inputs, int32_t nbInputs, IPluginExt& plugin) TRTNOEXCEPT = 0;
 
    virtual IIdentityLayer* addIdentity(ITensor& input) TRTNOEXCEPT = 0;
 
    virtual void removeTensor(ITensor& tensor) TRTNOEXCEPT = 0;
 
    virtual void unmarkOutput(ITensor& tensor) TRTNOEXCEPT = 0;
 
    virtual IPluginV2Layer* addPluginV2(ITensor* const* inputs, int32_t nbInputs, IPluginV2& plugin) TRTNOEXCEPT = 0;
 
    virtual ISliceLayer* addSlice(ITensor& input, Dims start, Dims size, Dims stride) TRTNOEXCEPT = 0;
 
    virtual void setName(const char* name) TRTNOEXCEPT = 0;
 
    virtual const char* getName() const TRTNOEXCEPT = 0;
 
    virtual IShapeLayer* addShape(ITensor& input) TRTNOEXCEPT = 0;
 
    virtual bool hasImplicitBatchDimension() const TRTNOEXCEPT = 0;
 
    virtual bool markOutputForShapes(ITensor& tensor) TRTNOEXCEPT = 0;
 
    virtual bool unmarkOutputForShapes(ITensor& tensor) TRTNOEXCEPT = 0;
 
    virtual IParametricReLULayer* addParametricReLU(ITensor& input, ITensor& slope) noexcept = 0;
 
    virtual IConvolutionLayer* addConvolutionNd(ITensor& input, int32_t nbOutputMaps, Dims kernelSize,
        Weights kernelWeights, Weights biasWeights) TRTNOEXCEPT = 0;
 
    virtual IPoolingLayer* addPoolingNd(ITensor& input, PoolingType type, Dims windowSize) TRTNOEXCEPT = 0;
 
    //
    virtual IDeconvolutionLayer* addDeconvolutionNd(ITensor& input, int32_t nbOutputMaps, Dims kernelSize,
        Weights kernelWeights, Weights biasWeights) TRTNOEXCEPT = 0;
 
    virtual IScaleLayer* addScaleNd(ITensor& input, ScaleMode mode, Weights shift, Weights scale, Weights power,
        int32_t channelAxis) TRTNOEXCEPT = 0;
 
    virtual IResizeLayer* addResize(ITensor& input) TRTNOEXCEPT = 0;
 
    virtual bool hasExplicitPrecision() const TRTNOEXCEPT = 0;
 
    virtual ILoop* addLoop() noexcept = 0;
 
    virtual ISelectLayer* addSelect(ITensor& condition, ITensor& thenInput, ITensor& elseInput) TRTNOEXCEPT = 0;
 
    virtual IFillLayer* addFill(Dims dimensions, FillOperation op) noexcept = 0;
 
    virtual IPaddingLayer* addPaddingNd(
        ITensor& input, Dims prePadding, Dims postPadding) TRTNOEXCEPT = 0;
};
 
enum class CalibrationAlgoType : int32_t
{
    kLEGACY_CALIBRATION = 0,
    kENTROPY_CALIBRATION = 1,
    kENTROPY_CALIBRATION_2 = 2,
    kMINMAX_CALIBRATION = 3,
};
 
template <>
constexpr inline int32_t EnumMax<CalibrationAlgoType>()
{
    return 4;
}
 
class IInt8Calibrator
{
public:
    virtual int32_t getBatchSize() const TRTNOEXCEPT = 0;
 
    virtual bool getBatch(void* bindings[], const char* names[], int32_t nbBindings) TRTNOEXCEPT = 0;
 
    virtual const void* readCalibrationCache(std::size_t& length) TRTNOEXCEPT = 0;
 
    virtual void writeCalibrationCache(const void* ptr, std::size_t length) TRTNOEXCEPT = 0;
 
    virtual CalibrationAlgoType getAlgorithm() TRTNOEXCEPT = 0;
 
    virtual ~IInt8Calibrator() {}
};
 
class IInt8EntropyCalibrator : public IInt8Calibrator
{
public:
    CalibrationAlgoType getAlgorithm() TRTNOEXCEPT override { return CalibrationAlgoType::kENTROPY_CALIBRATION; }
 
    virtual ~IInt8EntropyCalibrator() {}
};
 
class IInt8EntropyCalibrator2 : public IInt8Calibrator
{
public:
    CalibrationAlgoType getAlgorithm() TRTNOEXCEPT override { return CalibrationAlgoType::kENTROPY_CALIBRATION_2; }
 
    virtual ~IInt8EntropyCalibrator2() {}
};
 
class IInt8MinMaxCalibrator : public IInt8Calibrator
{
public:
    CalibrationAlgoType getAlgorithm() TRTNOEXCEPT override { return CalibrationAlgoType::kMINMAX_CALIBRATION; }
 
    virtual ~IInt8MinMaxCalibrator() {}
};
 
class IInt8LegacyCalibrator : public IInt8Calibrator
{
public:
    CalibrationAlgoType getAlgorithm() TRTNOEXCEPT override { return CalibrationAlgoType::kLEGACY_CALIBRATION; }
 
    virtual double getQuantile() const TRTNOEXCEPT = 0;
 
    virtual double getRegressionCutoff() const TRTNOEXCEPT = 0;
 
    virtual const void* readHistogramCache(std::size_t& length) TRTNOEXCEPT = 0;
 
    virtual void writeHistogramCache(const void* ptr, std::size_t length) TRTNOEXCEPT = 0;
 
    virtual ~IInt8LegacyCalibrator() {}
};
 
class IAlgorithmIOInfo
{
public:
    virtual TensorFormat getTensorFormat() const TRTNOEXCEPT = 0;
 
    virtual DataType getDataType() const TRTNOEXCEPT = 0;
 
    virtual Dims getStrides() const TRTNOEXCEPT = 0;
 
protected:
    virtual ~IAlgorithmIOInfo() {}
};
 
class IAlgorithmVariant
{
public:
    virtual int64_t getImplementation() const TRTNOEXCEPT = 0;
 
    virtual int64_t getTactic() const TRTNOEXCEPT = 0;
 
protected:
    virtual ~IAlgorithmVariant() {}
};
 
class IAlgorithmContext
{
public:
    virtual const char* getName() const TRTNOEXCEPT = 0;
 
    virtual Dims getDimensions(int32_t index, OptProfileSelector select) const TRTNOEXCEPT = 0;
 
    virtual int32_t getNbInputs() const TRTNOEXCEPT = 0;
 
    virtual int32_t getNbOutputs() const TRTNOEXCEPT = 0;
 
protected:
    virtual ~IAlgorithmContext() {}
};
 
class IAlgorithm
{
public:
    virtual const IAlgorithmIOInfo& getAlgorithmIOInfo(int32_t index) const TRTNOEXCEPT = 0;
 
    virtual const IAlgorithmVariant& getAlgorithmVariant() const TRTNOEXCEPT = 0;
 
    virtual float getTimingMSec() const TRTNOEXCEPT = 0;
 
    virtual std::size_t getWorkspaceSize() const TRTNOEXCEPT = 0;
 
protected:
    virtual ~IAlgorithm() {}
};
 
class  IAlgorithmSelector
{
public:
    virtual int32_t selectAlgorithms(const IAlgorithmContext& context, const IAlgorithm* const* choices,
                                     int32_t nbChoices, int32_t* selection) TRTNOEXCEPT = 0;
    virtual void reportAlgorithms(const IAlgorithmContext* const* algoContexts, const IAlgorithm* const* algoChoices,
        int32_t nbAlgorithms) TRTNOEXCEPT = 0;
 
    virtual ~IAlgorithmSelector() {}
};
 
typedef uint32_t QuantizationFlags;
 
enum class QuantizationFlag : int32_t
{
    kCALIBRATE_BEFORE_FUSION = 0
};
 
template <>
constexpr inline int32_t EnumMax<QuantizationFlag>()
{
    return 1;
}
 
typedef uint32_t BuilderFlags;
 
enum class BuilderFlag : int32_t
{
    kFP16 = 0,         
    kINT8 = 1,         
    kDEBUG = 2,        
    kGPU_FALLBACK = 3, 
    kSTRICT_TYPES = 4, 
    kREFIT = 5,        
    kDISABLE_TIMING_CACHE = 6, 
 
    kTF32 = 7
};
 
template <>
constexpr inline int32_t EnumMax<BuilderFlag>()
{
    return 8;
}
 
enum class ProfilingVerbosity : int32_t
{
    kDEFAULT = 0, 
    kNONE = 1,    
    kVERBOSE = 2, 
};
 
template <>
constexpr inline int32_t EnumMax<ProfilingVerbosity>()
{
    return 3;
}
 
 
enum class TacticSource : int32_t
{
    kCUBLAS = 0,   
    kCUBLAS_LT = 1 
};
 
template <>
constexpr inline int32_t EnumMax<TacticSource>()
{
    return 2;
}
 
using TacticSources = uint32_t;
 
class IBuilderConfig
{
public:
    virtual void setMinTimingIterations(int32_t minTiming) TRTNOEXCEPT = 0;
 
    virtual int32_t getMinTimingIterations() const TRTNOEXCEPT = 0;
 
    virtual void setAvgTimingIterations(int32_t avgTiming) TRTNOEXCEPT = 0;
 
    virtual int32_t getAvgTimingIterations() const TRTNOEXCEPT = 0;
 
    virtual void setEngineCapability(EngineCapability capability) TRTNOEXCEPT = 0;
 
    virtual EngineCapability getEngineCapability() const TRTNOEXCEPT = 0;
 
    virtual void setInt8Calibrator(IInt8Calibrator* calibrator) TRTNOEXCEPT = 0;
 
    virtual IInt8Calibrator* getInt8Calibrator() const TRTNOEXCEPT = 0;
 
    virtual void setMaxWorkspaceSize(std::size_t workspaceSize) TRTNOEXCEPT = 0;
 
    virtual std::size_t getMaxWorkspaceSize() const TRTNOEXCEPT = 0;
 
    virtual void setFlags(BuilderFlags builderFlags) TRTNOEXCEPT = 0;
 
    virtual BuilderFlags getFlags() const TRTNOEXCEPT = 0;
 
    virtual void clearFlag(BuilderFlag builderFlag) TRTNOEXCEPT = 0;
 
    virtual void setFlag(BuilderFlag builderFlag) TRTNOEXCEPT = 0;
 
    virtual bool getFlag(BuilderFlag builderFlag) const TRTNOEXCEPT = 0;
 
    virtual void setDeviceType(const ILayer* layer, DeviceType deviceType) TRTNOEXCEPT = 0;
 
    virtual DeviceType getDeviceType(const ILayer* layer) const TRTNOEXCEPT = 0;
 
    virtual bool isDeviceTypeSet(const ILayer* layer) const TRTNOEXCEPT = 0;
 
    virtual void resetDeviceType(const ILayer* layer) TRTNOEXCEPT = 0;
 
    virtual bool canRunOnDLA(const ILayer* layer) const TRTNOEXCEPT = 0;
 
    virtual void setDLACore(int32_t dlaCore) TRTNOEXCEPT = 0;
 
    virtual int32_t getDLACore() const TRTNOEXCEPT = 0;
 
    virtual void setDefaultDeviceType(DeviceType deviceType) TRTNOEXCEPT = 0;
 
    virtual DeviceType getDefaultDeviceType() const TRTNOEXCEPT = 0;
 
    virtual void reset() TRTNOEXCEPT = 0;
 
    virtual void destroy() TRTNOEXCEPT = 0;
 
    virtual void setProfileStream(const cudaStream_t stream) TRTNOEXCEPT = 0;
 
    virtual cudaStream_t getProfileStream() const TRTNOEXCEPT = 0;
 
    virtual int32_t addOptimizationProfile(const IOptimizationProfile* profile) noexcept = 0;
 
    virtual int32_t getNbOptimizationProfiles() const noexcept = 0;
 
protected:
    virtual ~IBuilderConfig() {}
 
public:
    virtual void setProfilingVerbosity(ProfilingVerbosity verbosity) TRTNOEXCEPT = 0;
 
    virtual ProfilingVerbosity getProfilingVerbosity() const TRTNOEXCEPT = 0;
 
    virtual void setAlgorithmSelector(IAlgorithmSelector* selector) TRTNOEXCEPT = 0;
 
    virtual IAlgorithmSelector* getAlgorithmSelector() const TRTNOEXCEPT = 0;
 
    virtual bool setCalibrationProfile(const IOptimizationProfile* profile) noexcept = 0;
 
    virtual const IOptimizationProfile* getCalibrationProfile() noexcept = 0;
 
    virtual void setQuantizationFlags(QuantizationFlags flags) TRTNOEXCEPT = 0;
 
    virtual QuantizationFlags getQuantizationFlags() const TRTNOEXCEPT = 0;
 
    virtual void clearQuantizationFlag(QuantizationFlag flag) TRTNOEXCEPT = 0;
 
    virtual void setQuantizationFlag(QuantizationFlag flag) TRTNOEXCEPT = 0;
 
    virtual bool getQuantizationFlag(QuantizationFlag flag) const TRTNOEXCEPT = 0;
 
    virtual bool setTacticSources(TacticSources tacticSources) TRTNOEXCEPT = 0;
 
    virtual TacticSources getTacticSources() const TRTNOEXCEPT = 0;
};
 
typedef uint32_t NetworkDefinitionCreationFlags;
 
enum class NetworkDefinitionCreationFlag : int32_t
{
    kEXPLICIT_BATCH = 0, 
 
    kEXPLICIT_PRECISION = 1, 
};
 
template <>
constexpr inline int32_t EnumMax<NetworkDefinitionCreationFlag>()
{
    return 2;
}
 
class IBuilder
{
public:
    TRT_DEPRECATED virtual nvinfer1::INetworkDefinition* createNetwork() TRTNOEXCEPT = 0;
 
    virtual void setMaxBatchSize(int32_t batchSize) TRTNOEXCEPT = 0;
 
    virtual int32_t getMaxBatchSize() const TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual void setMaxWorkspaceSize(std::size_t workspaceSize) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual std::size_t getMaxWorkspaceSize() const TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual void setHalf2Mode(bool mode) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual bool getHalf2Mode() const TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual void setDebugSync(bool sync) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual bool getDebugSync() const TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual void setMinFindIterations(int32_t minFind) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual int32_t getMinFindIterations() const TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual void setAverageFindIterations(int32_t avgFind) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual int32_t getAverageFindIterations() const TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual nvinfer1::ICudaEngine* buildCudaEngine(
        nvinfer1::INetworkDefinition& network) TRTNOEXCEPT = 0;
 
    virtual bool platformHasFastFp16() const TRTNOEXCEPT = 0;
 
    virtual bool platformHasFastInt8() const TRTNOEXCEPT = 0;
 
    virtual void destroy() TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual void setInt8Mode(bool mode) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual bool getInt8Mode() const TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual void setInt8Calibrator(IInt8Calibrator* calibrator) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual void setDeviceType(ILayer* layer, DeviceType deviceType) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual DeviceType getDeviceType(const ILayer* layer) const TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual bool isDeviceTypeSet(const ILayer* layer) const TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual void resetDeviceType(ILayer* layer) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual bool canRunOnDLA(const ILayer* layer) const TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual void setDefaultDeviceType(DeviceType deviceType) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual DeviceType getDefaultDeviceType() const TRTNOEXCEPT = 0;
 
    virtual int32_t getMaxDLABatchSize() const TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual void allowGPUFallback(bool setFallBackMode) TRTNOEXCEPT = 0;
 
    virtual int32_t getNbDLACores() const TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual void setDLACore(int32_t dlaCore) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual int32_t getDLACore() const TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual void reset(nvinfer1::INetworkDefinition& network) TRTNOEXCEPT = 0;
 
protected:
    virtual ~IBuilder()
    {
    }
 
public:
    virtual void setGpuAllocator(IGpuAllocator* allocator) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual void setFp16Mode(bool mode) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual bool getFp16Mode() const TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual void setStrictTypeConstraints(bool mode) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual bool getStrictTypeConstraints() const TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual void setRefittable(bool canRefit) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual bool getRefittable() const TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual void setEngineCapability(EngineCapability capability) TRTNOEXCEPT = 0;
 
    TRT_DEPRECATED virtual EngineCapability getEngineCapability() const TRTNOEXCEPT = 0;
 
    virtual nvinfer1::IBuilderConfig* createBuilderConfig() TRTNOEXCEPT = 0;
 
    virtual nvinfer1::ICudaEngine* buildEngineWithConfig(
        INetworkDefinition& network, IBuilderConfig& config) TRTNOEXCEPT = 0;
 
 
    virtual nvinfer1::INetworkDefinition* createNetworkV2(NetworkDefinitionCreationFlags flags) TRTNOEXCEPT = 0;
 
 
    virtual nvinfer1::IOptimizationProfile* createOptimizationProfile() noexcept = 0;
 
    //
    virtual void setErrorRecorder(IErrorRecorder* recorder) TRTNOEXCEPT = 0;
 
    virtual IErrorRecorder* getErrorRecorder() const TRTNOEXCEPT = 0;
 
    virtual void reset() TRTNOEXCEPT = 0;
 
    virtual bool platformHasTf32() const TRTNOEXCEPT = 0;
};
 
} // namespace nvinfer1
 
extern "C" TENSORRTAPI void* createInferBuilder_INTERNAL(void* logger, int32_t version);
 
namespace nvinfer1
{
namespace
{
inline IBuilder* createInferBuilder(ILogger& logger)
{
    return static_cast<IBuilder*>(createInferBuilder_INTERNAL(&logger, NV_TENSORRT_VERSION));
}
}
}
 
#endif