NvInferRuntime.h

Go to the documentation of this file.

/*
 * SPDX-FileCopyrightText: Copyright (c) 1993-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 * SPDX-License-Identifier: LicenseRef-NvidiaProprietary
 *
 * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual
 * property and proprietary rights in and to this material, related
 * documentation and any modifications thereto. Any use, reproduction,
 * disclosure or distribution of this material and related documentation
 * without an express license agreement from NVIDIA CORPORATION or
 * its affiliates is strictly prohibited.
 */
 
#ifndef NV_INFER_RUNTIME_H
#define NV_INFER_RUNTIME_H
 
 
#include "NvInferImpl.h"
#include "NvInferRuntimeCommon.h"
 
namespace nvinfer1
{
 
class IExecutionContext; 
class ICudaEngine;       
class IPluginFactory;    
class IEngineInspector;  
 
 
class INoCopy
{
protected:
    INoCopy() = default;
    virtual ~INoCopy() = default;
    INoCopy(INoCopy const& other) = delete;
    INoCopy& operator=(INoCopy const& other) = delete;
    INoCopy(INoCopy&& other) = delete;
    INoCopy& operator=(INoCopy&& other) = delete;
};
 
 
enum class EngineCapability : int32_t
{
    kSTANDARD = 0,
 
    kDEFAULT TRT_DEPRECATED_ENUM = kSTANDARD,
 
    kSAFETY = 1,
 
    kSAFE_GPU TRT_DEPRECATED_ENUM = kSAFETY,
 
    kDLA_STANDALONE = 2,
 
    kSAFE_DLA TRT_DEPRECATED_ENUM = kDLA_STANDALONE,
};
 
namespace impl
{
template <>
struct EnumMaxImpl<EngineCapability>
{
    static constexpr int32_t kVALUE = 3;
};
} // namespace impl
 
class Weights
{
public:
    DataType type;      
    void const* values; 
    int64_t count;      
};
 
class IHostMemory : public INoCopy
{
public:
    virtual ~IHostMemory() noexcept = default;
 
    void* data() const noexcept
    {
        return mImpl->data();
    }
 
    std::size_t size() const noexcept
    {
        return mImpl->size();
    }
 
    DataType type() const noexcept
    {
        return mImpl->type();
    }
    TRT_DEPRECATED void destroy() noexcept
    {
        delete this;
    }
 
protected:
    apiv::VHostMemory* mImpl;
};
 
enum class DimensionOperation : int32_t
{
    kSUM = 0,       
    kPROD = 1,      
    kMAX = 2,       
    kMIN = 3,       
    kSUB = 4,       
    kEQUAL = 5,     
    kLESS = 6,      
    kFLOOR_DIV = 7, 
    kCEIL_DIV = 8   
};
 
template <>
constexpr inline int32_t EnumMax<DimensionOperation>() noexcept
{
    return 9;
}
 
enum class TensorLocation : int32_t
{
    kDEVICE = 0, 
    kHOST = 1,   
};
 
namespace impl
{
template <>
struct EnumMaxImpl<TensorLocation>
{
    static constexpr int32_t kVALUE = 2;
};
} // namespace impl
 
class IDimensionExpr : public INoCopy
{
public:
    bool isConstant() const noexcept
    {
        return mImpl->isConstant();
    }
 
    int32_t getConstantValue() const noexcept
    {
        return mImpl->getConstantValue();
    }
 
protected:
    apiv::VDimensionExpr* mImpl;
    virtual ~IDimensionExpr() noexcept = default;
};
 
class IExprBuilder : public INoCopy
{
public:
    IDimensionExpr const* constant(int32_t value) noexcept
    {
        return mImpl->constant(value);
    }
 
    IDimensionExpr const* operation(
        DimensionOperation op, IDimensionExpr const& first, IDimensionExpr const& second) noexcept
    {
        return mImpl->operation(op, first, second);
    }
 
protected:
    apiv::VExprBuilder* mImpl;
    virtual ~IExprBuilder() noexcept = default;
};
 
class DimsExprs
{
public:
    int32_t nbDims;                          
    IDimensionExpr const* d[Dims::MAX_DIMS]; 
};
 
struct DynamicPluginTensorDesc
{
    PluginTensorDesc desc;
 
    Dims min;
 
    Dims max;
};
 
class IPluginV2DynamicExt : public nvinfer1::IPluginV2Ext
{
public:
    IPluginV2DynamicExt* clone() const noexcept override = 0;
 
    virtual DimsExprs getOutputDimensions(
        int32_t outputIndex, DimsExprs const* inputs, int32_t nbInputs, IExprBuilder& exprBuilder) noexcept = 0;
 
    static constexpr int32_t kFORMAT_COMBINATION_LIMIT = 100;
 
    virtual bool supportsFormatCombination(
        int32_t pos, PluginTensorDesc const* inOut, int32_t nbInputs, int32_t nbOutputs) noexcept = 0;
 
    virtual void configurePlugin(DynamicPluginTensorDesc const* in, int32_t nbInputs,
        DynamicPluginTensorDesc const* out, int32_t nbOutputs) noexcept = 0;
 
    virtual size_t getWorkspaceSize(PluginTensorDesc const* inputs, int32_t nbInputs, PluginTensorDesc const* outputs,
        int32_t nbOutputs) const noexcept = 0;
 
    virtual int32_t enqueue(PluginTensorDesc const* inputDesc, PluginTensorDesc const* outputDesc,
        void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept = 0;
 
protected:
    int32_t getTensorRTVersion() const noexcept override
    {
        return (static_cast<int32_t>(PluginVersion::kV2_DYNAMICEXT) << 24 | (NV_TENSORRT_VERSION & 0xFFFFFF));
    }
 
    virtual ~IPluginV2DynamicExt() noexcept {}
 
private:
    // Following are obsolete base class methods, and must not be implemented or used.
 
    void configurePlugin(Dims const*, int32_t, Dims const*, int32_t, DataType const*, DataType const*, bool const*,
        bool const*, PluginFormat, int32_t) noexcept override final
    {
    }
 
    bool supportsFormat(DataType, PluginFormat) const noexcept override final
    {
        return false;
    }
 
    Dims getOutputDimensions(int32_t, Dims const*, int32_t) noexcept override final
    {
        return Dims{-1, {}};
    }
 
    bool isOutputBroadcastAcrossBatch(int32_t, bool const*, int32_t) const noexcept override final
    {
        return false;
    }
 
    bool canBroadcastInputAcrossBatch(int32_t) const noexcept override final
    {
        return true;
    }
 
    size_t getWorkspaceSize(int32_t) const noexcept override final
    {
        return 0;
    }
 
    int32_t enqueue(int32_t, void const* const*, void* const*, void*, cudaStream_t) noexcept override final
    {
        return 1;
    }
};
 
class IProfiler
{
public:
    virtual void reportLayerTime(char const* layerName, float ms) noexcept = 0;
 
    virtual ~IProfiler() noexcept {}
};
 
enum class WeightsRole : int32_t
{
    kKERNEL = 0,   
    kBIAS = 1,     
    kSHIFT = 2,    
    kSCALE = 3,    
    kCONSTANT = 4, 
    kANY = 5,      
};
 
template <>
constexpr inline int32_t EnumMax<WeightsRole>() noexcept
{
    return 6;
}
 
enum class DeviceType : int32_t
{
    kGPU, 
    kDLA, 
};
 
template <>
constexpr inline int32_t EnumMax<DeviceType>() noexcept
{
    return 2;
}
 
enum class TempfileControlFlag : int32_t
{
    kALLOW_IN_MEMORY_FILES = 0,
 
    kALLOW_TEMPORARY_FILES = 1,
};
 
template <>
constexpr inline int32_t EnumMax<TempfileControlFlag>() noexcept
{
    return 2;
}
 
using TempfileControlFlags = uint32_t;
 
class IRuntime : public INoCopy
{
public:
    virtual ~IRuntime() noexcept = default;
 
    TRT_DEPRECATED nvinfer1::ICudaEngine* deserializeCudaEngine(
        void const* blob, std::size_t size, IPluginFactory* pluginFactory) noexcept
    {
        return mImpl->deserializeCudaEngine(blob, size, nullptr);
    }
 
    void setDLACore(int32_t dlaCore) noexcept
    {
        mImpl->setDLACore(dlaCore);
    }
 
    int32_t getDLACore() const noexcept
    {
        return mImpl->getDLACore();
    }
 
    int32_t getNbDLACores() const noexcept
    {
        return mImpl->getNbDLACores();
    }
 
    TRT_DEPRECATED void destroy() noexcept
    {
        delete this;
    }
 
    void setGpuAllocator(IGpuAllocator* allocator) noexcept
    {
        mImpl->setGpuAllocator(allocator);
    }
 
    //
    void setErrorRecorder(IErrorRecorder* recorder) noexcept
    {
        mImpl->setErrorRecorder(recorder);
    }
 
    IErrorRecorder* getErrorRecorder() const noexcept
    {
        return mImpl->getErrorRecorder();
    }
 
    ICudaEngine* deserializeCudaEngine(void const* blob, std::size_t size) noexcept
    {
        return mImpl->deserializeCudaEngine(blob, size, nullptr);
    }
 
    ILogger* getLogger() const noexcept
    {
        return mImpl->getLogger();
    }
 
    bool setMaxThreads(int32_t maxThreads) noexcept
    {
        return mImpl->setMaxThreads(maxThreads);
    }
 
    int32_t getMaxThreads() const noexcept
    {
        return mImpl->getMaxThreads();
    }
 
    void setTemporaryDirectory(char const* path) noexcept
    {
        return mImpl->setTemporaryDirectory(path);
    }
 
    char const* getTemporaryDirectory() const noexcept
    {
        return mImpl->getTemporaryDirectory();
    }
 
    void setTempfileControlFlags(TempfileControlFlags flags) noexcept
    {
        return mImpl->setTempfileControlFlags(flags);
    }
 
    TempfileControlFlags getTempfileControlFlags() const noexcept
    {
        return mImpl->getTempfileControlFlags();
    }
 
    IPluginRegistry& getPluginRegistry() noexcept
    {
        return mImpl->getPluginRegistry();
    }
 
    IRuntime* loadRuntime(char const* path) noexcept
    {
        return mImpl->loadRuntime(path);
    }
 
    void setEngineHostCodeAllowed(bool allowed) noexcept
    {
        return mImpl->setEngineHostCodeAllowed(allowed);
    }
 
    bool getEngineHostCodeAllowed() const noexcept
    {
        return mImpl->getEngineHostCodeAllowed();
    }
 
protected:
    apiv::VRuntime* mImpl;
};
 
class IRefitter : public INoCopy
{
public:
    virtual ~IRefitter() noexcept = default;
 
    bool setWeights(char const* layerName, WeightsRole role, Weights weights) noexcept
    {
        return mImpl->setWeights(layerName, role, weights);
    }
 
    bool refitCudaEngine() noexcept
    {
        return mImpl->refitCudaEngine();
    }
 
    int32_t getMissing(int32_t size, char const** layerNames, WeightsRole* roles) noexcept
    {
        return mImpl->getMissing(size, layerNames, roles);
    }
 
    int32_t getAll(int32_t size, char const** layerNames, WeightsRole* roles) noexcept
    {
        return mImpl->getAll(size, layerNames, roles);
    }
 
    TRT_DEPRECATED void destroy() noexcept
    {
        delete this;
    }
 
    bool setDynamicRange(char const* tensorName, float min, float max) noexcept
    {
        return mImpl->setDynamicRange(tensorName, min, max);
    }
 
    float getDynamicRangeMin(char const* tensorName) const noexcept
    {
        return mImpl->getDynamicRangeMin(tensorName);
    }
 
    float getDynamicRangeMax(char const* tensorName) const noexcept
    {
        return mImpl->getDynamicRangeMax(tensorName);
    }
 
    int32_t getTensorsWithDynamicRange(int32_t size, char const** tensorNames) const noexcept
    {
        return mImpl->getTensorsWithDynamicRange(size, tensorNames);
    }
 
    //
    void setErrorRecorder(IErrorRecorder* recorder) noexcept
    {
        mImpl->setErrorRecorder(recorder);
    }
 
    IErrorRecorder* getErrorRecorder() const noexcept
    {
        return mImpl->getErrorRecorder();
    }
 
    bool setNamedWeights(char const* name, Weights weights) noexcept
    {
        return mImpl->setNamedWeights(name, weights);
    }
 
    int32_t getMissingWeights(int32_t size, char const** weightsNames) noexcept
    {
        return mImpl->getMissingWeights(size, weightsNames);
    }
 
    int32_t getAllWeights(int32_t size, char const** weightsNames) noexcept
    {
        return mImpl->getAllWeights(size, weightsNames);
    }
 
    ILogger* getLogger() const noexcept
    {
        return mImpl->getLogger();
    }
 
    bool setMaxThreads(int32_t maxThreads) noexcept
    {
        return mImpl->setMaxThreads(maxThreads);
    }
 
    int32_t getMaxThreads() const noexcept
    {
        return mImpl->getMaxThreads();
    }
 
protected:
    apiv::VRefitter* mImpl;
};
 
enum class OptProfileSelector : int32_t
{
    kMIN = 0, 
    kOPT = 1, 
    kMAX = 2  
};
 
template <>
constexpr inline int32_t EnumMax<OptProfileSelector>() noexcept
{
    return 3;
}
 
class IOptimizationProfile : public INoCopy
{
public:
    bool setDimensions(char const* inputName, OptProfileSelector select, Dims dims) noexcept
    {
        return mImpl->setDimensions(inputName, select, dims);
    }
 
    Dims getDimensions(char const* inputName, OptProfileSelector select) const noexcept
    {
        return mImpl->getDimensions(inputName, select);
    }
 
    bool setShapeValues(
        char const* inputName, OptProfileSelector select, int32_t const* values, int32_t nbValues) noexcept
    {
        return mImpl->setShapeValues(inputName, select, values, nbValues);
    }
 
    int32_t getNbShapeValues(char const* inputName) const noexcept
    {
        return mImpl->getNbShapeValues(inputName);
    }
 
    int32_t const* getShapeValues(char const* inputName, OptProfileSelector select) const noexcept
    {
        return mImpl->getShapeValues(inputName, select);
    }
 
    bool setExtraMemoryTarget(float target) noexcept
    {
        return mImpl->setExtraMemoryTarget(target);
    }
 
    float getExtraMemoryTarget() const noexcept
    {
        return mImpl->getExtraMemoryTarget();
    }
 
    bool isValid() const noexcept
    {
        return mImpl->isValid();
    }
 
protected:
    apiv::VOptimizationProfile* mImpl;
    virtual ~IOptimizationProfile() noexcept = default;
};
 
enum class TacticSource : int32_t
{
    kCUBLAS = 0,
    kCUBLAS_LT = 1,
    kCUDNN = 2,
 
    kEDGE_MASK_CONVOLUTIONS = 3,
 
    kJIT_CONVOLUTIONS = 4,
};
 
template <>
constexpr inline int32_t EnumMax<TacticSource>() noexcept
{
    return 5;
} 
 
using TacticSources = uint32_t;
 
enum class ProfilingVerbosity : int32_t
{
    kLAYER_NAMES_ONLY = 0, 
    kNONE = 1,             
    kDETAILED = 2,         
 
    kDEFAULT TRT_DEPRECATED_ENUM = kLAYER_NAMES_ONLY,
    kVERBOSE TRT_DEPRECATED_ENUM = kDETAILED
};
 
template <>
constexpr inline int32_t EnumMax<ProfilingVerbosity>() noexcept
{
    return 3;
}
 
class ICudaEngine : public INoCopy
{
public:
    virtual ~ICudaEngine() noexcept = default;
 
    TRT_DEPRECATED int32_t getNbBindings() const noexcept
    {
        return mImpl->getNbBindings();
    }
 
    TRT_DEPRECATED int32_t getBindingIndex(char const* name) const noexcept
    {
        return mImpl->getBindingIndex(name);
    }
 
    TRT_DEPRECATED char const* getBindingName(int32_t bindingIndex) const noexcept
    {
        return mImpl->getBindingName(bindingIndex);
    }
 
    TRT_DEPRECATED bool bindingIsInput(int32_t bindingIndex) const noexcept
    {
        return mImpl->bindingIsInput(bindingIndex);
    }
 
    TRT_DEPRECATED Dims getBindingDimensions(int32_t bindingIndex) const noexcept
    {
        return mImpl->getBindingDimensions(bindingIndex);
    }
 
    Dims getTensorShape(char const* tensorName) const noexcept
    {
        return mImpl->getTensorShape(tensorName);
    }
 
    TRT_DEPRECATED DataType getBindingDataType(int32_t bindingIndex) const noexcept
    {
        return mImpl->getBindingDataType(bindingIndex);
    }
 
    DataType getTensorDataType(char const* tensorName) const noexcept
    {
        return mImpl->getTensorDataType(tensorName);
    }
 
    TRT_DEPRECATED int32_t getMaxBatchSize() const noexcept
    {
        return mImpl->getMaxBatchSize();
    }
 
    int32_t getNbLayers() const noexcept
    {
        return mImpl->getNbLayers();
    }
 
    IHostMemory* serialize() const noexcept
    {
        return mImpl->serialize();
    }
 
    IExecutionContext* createExecutionContext() noexcept
    {
        return mImpl->createExecutionContext();
    }
 
    TRT_DEPRECATED void destroy() noexcept
    {
        delete this;
    }
 
    TRT_DEPRECATED TensorLocation getLocation(int32_t bindingIndex) const noexcept
    {
        return mImpl->getLocation(bindingIndex);
    }
 
    TensorLocation getTensorLocation(char const* tensorName) const noexcept
    {
        return mImpl->getTensorLocation(tensorName);
    }
 
    bool isShapeInferenceIO(char const* tensorName) const noexcept
    {
        return mImpl->isShapeInferenceIO(tensorName);
    }
 
    TensorIOMode getTensorIOMode(char const* tensorName) const noexcept
    {
        return mImpl->getTensorIOMode(tensorName);
    }
 
    IExecutionContext* createExecutionContextWithoutDeviceMemory() noexcept
    {
        return mImpl->createExecutionContextWithoutDeviceMemory();
    }
 
    size_t getDeviceMemorySize() const noexcept
    {
        return mImpl->getDeviceMemorySize();
    }
 
    bool isRefittable() const noexcept
    {
        return mImpl->isRefittable();
    }
 
    TRT_DEPRECATED int32_t getBindingBytesPerComponent(int32_t bindingIndex) const noexcept
    {
        return mImpl->getBindingBytesPerComponent(bindingIndex);
    }
 
    int32_t getTensorBytesPerComponent(char const* tensorName) const noexcept
    {
        return mImpl->getTensorBytesPerComponent(tensorName);
    }
 
    TRT_DEPRECATED int32_t getBindingComponentsPerElement(int32_t bindingIndex) const noexcept
    {
        return mImpl->getBindingComponentsPerElement(bindingIndex);
    }
 
    int32_t getTensorComponentsPerElement(char const* tensorName) const noexcept
    {
        return mImpl->getTensorComponentsPerElement(tensorName);
    }
 
    TRT_DEPRECATED TensorFormat getBindingFormat(int32_t bindingIndex) const noexcept
    {
        return mImpl->getBindingFormat(bindingIndex);
    }
 
    TensorFormat getTensorFormat(char const* tensorName) const noexcept
    {
        return mImpl->getTensorFormat(tensorName);
    }
 
    TRT_DEPRECATED char const* getBindingFormatDesc(int32_t bindingIndex) const noexcept
    {
        return mImpl->getBindingFormatDesc(bindingIndex);
    }
 
    char const* getTensorFormatDesc(char const* tensorName) const noexcept
    {
        return mImpl->getTensorFormatDesc(tensorName);
    }
 
    TRT_DEPRECATED int32_t getBindingVectorizedDim(int32_t bindingIndex) const noexcept
    {
        return mImpl->getBindingVectorizedDim(bindingIndex);
    }
 
    int32_t getTensorVectorizedDim(char const* tensorName) const noexcept
    {
        return mImpl->getTensorVectorizedDim(tensorName);
    }
 
    char const* getName() const noexcept
    {
        return mImpl->getName();
    }
 
    int32_t getNbOptimizationProfiles() const noexcept
    {
        return mImpl->getNbOptimizationProfiles();
    }
 
    TRT_DEPRECATED Dims getProfileDimensions(
        int32_t bindingIndex, int32_t profileIndex, OptProfileSelector select) const noexcept
    {
        return mImpl->getProfileDimensions(bindingIndex, profileIndex, select);
    }
 
    Dims getProfileShape(char const* tensorName, int32_t profileIndex, OptProfileSelector select) const noexcept
    {
        return mImpl->getProfileShape(tensorName, profileIndex, select);
    }
 
    TRT_DEPRECATED int32_t const* getProfileShapeValues(
        int32_t profileIndex, int32_t inputIndex, OptProfileSelector select) const noexcept
    {
        return mImpl->getProfileShapeValues(profileIndex, inputIndex, select);
    }
 
    TRT_DEPRECATED bool isShapeBinding(int32_t bindingIndex) const noexcept
    {
        return mImpl->isShapeBinding(bindingIndex);
    }
 
    TRT_DEPRECATED bool isExecutionBinding(int32_t bindingIndex) const noexcept
    {
        return mImpl->isExecutionBinding(bindingIndex);
    }
 
    EngineCapability getEngineCapability() const noexcept
    {
        return mImpl->getEngineCapability();
    }
 
    //
    void setErrorRecorder(IErrorRecorder* recorder) noexcept
    {
        return mImpl->setErrorRecorder(recorder);
    }
 
    IErrorRecorder* getErrorRecorder() const noexcept
    {
        return mImpl->getErrorRecorder();
    }
 
    bool hasImplicitBatchDimension() const noexcept
    {
        return mImpl->hasImplicitBatchDimension();
    }
 
    TacticSources getTacticSources() const noexcept
    {
        return mImpl->getTacticSources();
    }
 
    ProfilingVerbosity getProfilingVerbosity() const noexcept
    {
        return mImpl->getProfilingVerbosity();
    }
 
    IEngineInspector* createEngineInspector() const noexcept
    {
        return mImpl->createEngineInspector();
    }
 
    int32_t getNbIOTensors() const noexcept
    {
        return mImpl->getNbIOTensors();
    }
 
    char const* getIOTensorName(int32_t index) const noexcept
    {
        return mImpl->getIOTensorName(index);
    }
 
    HardwareCompatibilityLevel getHardwareCompatibilityLevel() const noexcept
    {
        return mImpl->getHardwareCompatibilityLevel();
    }
 
    int32_t getNbAuxStreams() const noexcept
    {
        return mImpl->getNbAuxStreams();
    }
 
protected:
    apiv::VCudaEngine* mImpl;
};
 
class IOutputAllocator
{
public:
    virtual int32_t getInterfaceVersion() const noexcept
    {
        return 1;
    }
 
    virtual void* reallocateOutput(char const* tensorName, void* currentMemory, uint64_t size, uint64_t alignment) noexcept = 0;
 
    virtual void notifyShape(char const* tensorName, Dims const& dims) noexcept = 0;
 
    virtual ~IOutputAllocator() = default;
};
 
class IExecutionContext : public INoCopy
{
public:
    virtual ~IExecutionContext() noexcept = default;
 
    TRT_DEPRECATED bool execute(int32_t batchSize, void* const* bindings) noexcept
    {
        return mImpl->execute(batchSize, bindings);
    }
 
    TRT_DEPRECATED bool enqueue(
        int32_t batchSize, void* const* bindings, cudaStream_t stream, cudaEvent_t* inputConsumed) noexcept
    {
        return mImpl->enqueue(batchSize, bindings, stream, inputConsumed);
    }
 
    void setDebugSync(bool sync) noexcept
    {
        mImpl->setDebugSync(sync);
    }
 
    bool getDebugSync() const noexcept
    {
        return mImpl->getDebugSync();
    }
 
    void setProfiler(IProfiler* profiler) noexcept
    {
        mImpl->setProfiler(profiler);
    }
 
    IProfiler* getProfiler() const noexcept
    {
        return mImpl->getProfiler();
    }
 
    ICudaEngine const& getEngine() const noexcept
    {
        return mImpl->getEngine();
    }
 
    TRT_DEPRECATED void destroy() noexcept
    {
        delete this;
    }
 
    void setName(char const* name) noexcept
    {
        mImpl->setName(name);
    }
 
    char const* getName() const noexcept
    {
        return mImpl->getName();
    }
 
    void setDeviceMemory(void* memory) noexcept
    {
        mImpl->setDeviceMemory(memory);
    }
 
    TRT_DEPRECATED Dims getStrides(int32_t bindingIndex) const noexcept
    {
        return mImpl->getStrides(bindingIndex);
    }
 
    Dims getTensorStrides(char const* tensorName) const noexcept
    {
        return mImpl->getTensorStrides(tensorName);
    }
 
public:
    TRT_DEPRECATED
    bool setOptimizationProfile(int32_t profileIndex) noexcept
    {
        return mImpl->setOptimizationProfile(profileIndex);
    }
 
    int32_t getOptimizationProfile() const noexcept
    {
        return mImpl->getOptimizationProfile();
    }
 
    TRT_DEPRECATED bool setBindingDimensions(int32_t bindingIndex, Dims dimensions) noexcept
    {
        return mImpl->setBindingDimensions(bindingIndex, dimensions);
    }
 
    bool setInputShape(char const* tensorName, Dims const& dims) noexcept
    {
        return mImpl->setInputShape(tensorName, dims);
    }
 
    TRT_DEPRECATED Dims getBindingDimensions(int32_t bindingIndex) const noexcept
    {
        return mImpl->getBindingDimensions(bindingIndex);
    }
 
    Dims getTensorShape(char const* tensorName) const noexcept
    {
        return mImpl->getTensorShape(tensorName);
    }
 
    TRT_DEPRECATED bool setInputShapeBinding(int32_t bindingIndex, int32_t const* data) noexcept
    {
        return mImpl->setInputShapeBinding(bindingIndex, data);
    }
 
    TRT_DEPRECATED bool getShapeBinding(int32_t bindingIndex, int32_t* data) const noexcept
    {
        return mImpl->getShapeBinding(bindingIndex, data);
    }
 
    bool allInputDimensionsSpecified() const noexcept
    {
        return mImpl->allInputDimensionsSpecified();
    }
 
    bool allInputShapesSpecified() const noexcept
    {
        return mImpl->allInputShapesSpecified();
    }
 
    //
    void setErrorRecorder(IErrorRecorder* recorder) noexcept
    {
        mImpl->setErrorRecorder(recorder);
    }
 
    IErrorRecorder* getErrorRecorder() const noexcept
    {
        return mImpl->getErrorRecorder();
    }
 
    bool executeV2(void* const* bindings) noexcept
    {
        return mImpl->executeV2(bindings);
    }
 
    TRT_DEPRECATED bool enqueueV2(void* const* bindings, cudaStream_t stream, cudaEvent_t* inputConsumed) noexcept
    {
        return mImpl->enqueueV2(bindings, stream, inputConsumed);
    }
 
    bool setOptimizationProfileAsync(int32_t profileIndex, cudaStream_t stream) noexcept
    {
        return mImpl->setOptimizationProfileAsync(profileIndex, stream);
    }
 
    void setEnqueueEmitsProfile(bool enqueueEmitsProfile) noexcept
    {
        mImpl->setEnqueueEmitsProfile(enqueueEmitsProfile);
    }
 
    bool getEnqueueEmitsProfile() const noexcept
    {
        return mImpl->getEnqueueEmitsProfile();
    }
 
    bool reportToProfiler() const noexcept
    {
        return mImpl->reportToProfiler();
    }
 
    bool setTensorAddress(char const* tensorName, void* data) noexcept
    {
        return mImpl->setTensorAddress(tensorName, data);
    }
 
    void const* getTensorAddress(char const* tensorName) const noexcept
    {
        return mImpl->getTensorAddress(tensorName);
    }
 
    bool setInputTensorAddress(char const* tensorName, void const* data) noexcept
    {
        return mImpl->setInputTensorAddress(tensorName, data);
    }
 
    void* getOutputTensorAddress(char const* tensorName) const noexcept
    {
        return mImpl->getOutputTensorAddress(tensorName);
    }
 
    int32_t inferShapes(int32_t nbMaxNames, char const** tensorNames) noexcept
    {
        return mImpl->inferShapes(nbMaxNames, tensorNames);
    }
 
    bool setInputConsumedEvent(cudaEvent_t event) noexcept
    {
        return mImpl->setInputConsumedEvent(event);
    }
 
    cudaEvent_t getInputConsumedEvent() const noexcept
    {
        return mImpl->getInputConsumedEvent();
    }
 
    bool setOutputAllocator(char const* tensorName, IOutputAllocator* outputAllocator) noexcept
    {
        return mImpl->setOutputAllocator(tensorName, outputAllocator);
    }
 
    IOutputAllocator* getOutputAllocator(char const* tensorName) const noexcept
    {
        return mImpl->getOutputAllocator(tensorName);
    }
 
    int64_t getMaxOutputSize(char const* tensorName) const noexcept
    {
        return mImpl->getMaxOutputSize(tensorName);
    }
 
    bool setTemporaryStorageAllocator(IGpuAllocator* allocator) noexcept
    {
        return mImpl->setTemporaryStorageAllocator(allocator);
    }
 
    IGpuAllocator* getTemporaryStorageAllocator() const noexcept
    {
        return mImpl->getTemporaryStorageAllocator();
    }
 
    bool enqueueV3(cudaStream_t stream) noexcept
    {
        return mImpl->enqueueV3(stream);
    }
 
    void setPersistentCacheLimit(size_t size) noexcept
    {
        mImpl->setPersistentCacheLimit(size);
    }
 
    size_t getPersistentCacheLimit() const noexcept
    {
        return mImpl->getPersistentCacheLimit();
    }
 
    bool setNvtxVerbosity(ProfilingVerbosity verbosity) noexcept
    {
        return mImpl->setNvtxVerbosity(verbosity);
    }
 
    ProfilingVerbosity getNvtxVerbosity() const noexcept
    {
        return mImpl->getNvtxVerbosity();
    }
 
    void setAuxStreams(cudaStream_t* auxStreams, int32_t nbStreams) noexcept
    {
        mImpl->setAuxStreams(auxStreams, nbStreams);
    }
 
protected:
    apiv::VExecutionContext* mImpl;
}; // class IExecutionContext
 
enum class LayerInformationFormat : int32_t
{
    kONELINE = 0, 
    kJSON = 1,    
};
 
template <>
constexpr inline int32_t EnumMax<LayerInformationFormat>() noexcept
{
    return 2;
}
 
class IEngineInspector : public INoCopy
{
public:
    virtual ~IEngineInspector() noexcept = default;
 
    bool setExecutionContext(IExecutionContext const* context) noexcept
    {
        return mImpl->setExecutionContext(context);
    }
 
    IExecutionContext const* getExecutionContext() const noexcept
    {
        return mImpl->getExecutionContext();
    }
 
    char const* getLayerInformation(int32_t layerIndex, LayerInformationFormat format) const noexcept
    {
        return mImpl->getLayerInformation(layerIndex, format);
    }
 
    char const* getEngineInformation(LayerInformationFormat format) const noexcept
    {
        return mImpl->getEngineInformation(format);
    }
 
    //
    void setErrorRecorder(IErrorRecorder* recorder) noexcept
    {
        mImpl->setErrorRecorder(recorder);
    }
 
    IErrorRecorder* getErrorRecorder() const noexcept
    {
        return mImpl->getErrorRecorder();
    }
 
protected:
    apiv::VEngineInspector* mImpl;
}; // class IEngineInspector
 
} // namespace nvinfer1
 
extern "C" TENSORRTAPI void* createInferRuntime_INTERNAL(void* logger, int32_t version) noexcept;
 
extern "C" TENSORRTAPI void* createInferRefitter_INTERNAL(void* engine, void* logger, int32_t version) noexcept;
 
extern "C" TENSORRTAPI nvinfer1::IPluginRegistry* getPluginRegistry() noexcept;
 
extern "C" TENSORRTAPI nvinfer1::ILogger* getLogger() noexcept;
 
namespace nvinfer1
{
namespace // unnamed namespace avoids linkage surprises when linking objects built with different versions of this
          // header.
{
inline IRuntime* createInferRuntime(ILogger& logger) noexcept
{
    return static_cast<IRuntime*>(createInferRuntime_INTERNAL(&logger, NV_TENSORRT_VERSION));
}
 
inline IRefitter* createInferRefitter(ICudaEngine& engine, ILogger& logger) noexcept
{
    return static_cast<IRefitter*>(createInferRefitter_INTERNAL(&engine, &logger, NV_TENSORRT_VERSION));
}
 
} // namespace
 
template <typename T>
class PluginRegistrar
{
public:
    PluginRegistrar()
    {
        getPluginRegistry()->registerCreator(instance, "");
    }
 
private:
    T instance{};
};
 
} // namespace nvinfer1
 
#define REGISTER_TENSORRT_PLUGIN(name)                                                                                 \
    static nvinfer1::PluginRegistrar<name> pluginRegistrar##name {}
 
namespace nvinfer1
{
class ILoggerFinder
{
public:
    virtual ILogger* findLogger() = 0;
 
protected:
    virtual ~ILoggerFinder() = default;
};
 
} // namespace nvinfer1
 
#endif // NV_INFER_RUNTIME_H