NvInferRuntimePlugin.h

Go to the documentation of this file.

/*
 * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 * SPDX-License-Identifier: Apache-2.0
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
 
#ifndef NV_INFER_RUNTIME_PLUGIN_H
#define NV_INFER_RUNTIME_PLUGIN_H
 
#define NV_INFER_INTERNAL_INCLUDE_RUNTIME_BASE 1
#include "NvInferRuntimeBase.h"
#undef NV_INFER_INTERNAL_INCLUDE_RUNTIME_BASE
 
 
namespace nvinfer1
{
 
using PluginFormat = TensorFormat;
 
static constexpr int32_t kPLUGIN_VERSION_PYTHON_BIT = 0x40;
 
struct PluginTensorDesc
{
    Dims dims;
    DataType type;
    TensorFormat format;
    float scale;
};
 
enum class PluginVersion : uint8_t
{
    kV2 = 0,
    kV2_EXT = 1,
    kV2_IOEXT = 2,
    kV2_DYNAMICEXT = 3,
    kV2_DYNAMICEXT_PYTHON = kPLUGIN_VERSION_PYTHON_BIT | 3
};
 
enum class PluginCreatorVersion : int32_t
{
    kV1 = 0,
    kV1_PYTHON = kPLUGIN_VERSION_PYTHON_BIT
};
 
class TRT_DEPRECATED IPluginV2
{
public:
    virtual int32_t getTensorRTVersion() const noexcept
    {
        return NV_TENSORRT_VERSION;
    }
 
    virtual AsciiChar const* getPluginType() const noexcept = 0;
 
    virtual AsciiChar const* getPluginVersion() const noexcept = 0;
 
    virtual int32_t getNbOutputs() const noexcept = 0;
 
    virtual Dims getOutputDimensions(int32_t index, Dims const* inputs, int32_t nbInputDims) noexcept = 0;
 
    virtual bool supportsFormat(DataType type, PluginFormat format) const noexcept = 0;
 
    virtual void configureWithFormat(Dims const* inputDims, int32_t nbInputs, Dims const* outputDims, int32_t nbOutputs,
        DataType type, PluginFormat format, int32_t maxBatchSize) noexcept
        = 0;
 
    virtual int32_t initialize() noexcept = 0;
 
    virtual void terminate() noexcept = 0;
 
    virtual size_t getWorkspaceSize(int32_t maxBatchSize) const noexcept = 0;
 
    virtual int32_t enqueue(int32_t batchSize, void const* const* inputs, void* const* outputs, void* workspace,
        cudaStream_t stream) noexcept
        = 0;
 
    virtual size_t getSerializationSize() const noexcept = 0;
 
    virtual void serialize(void* buffer) const noexcept = 0;
 
    virtual void destroy() noexcept = 0;
 
    virtual IPluginV2* clone() const noexcept = 0;
 
    virtual void setPluginNamespace(AsciiChar const* pluginNamespace) noexcept = 0;
 
    virtual AsciiChar const* getPluginNamespace() const noexcept = 0;
 
    // @cond SuppressDoxyWarnings
    IPluginV2() = default;
    virtual ~IPluginV2() noexcept = default;
// @endcond
 
protected:
// @cond SuppressDoxyWarnings
    IPluginV2(IPluginV2 const&) = default;
    IPluginV2(IPluginV2&&) = default;
    IPluginV2& operator=(IPluginV2 const&) & = default;
    IPluginV2& operator=(IPluginV2&&) & = default;
// @endcond
};
 
class TRT_DEPRECATED IPluginV2Ext : public IPluginV2
{
public:
    virtual nvinfer1::DataType getOutputDataType(
        int32_t index, nvinfer1::DataType const* inputTypes, int32_t nbInputs) const noexcept
        = 0;
 
    TRT_DEPRECATED virtual bool isOutputBroadcastAcrossBatch(
        int32_t outputIndex, bool const* inputIsBroadcasted, int32_t nbInputs) const noexcept
        = 0;
 
    TRT_DEPRECATED virtual bool canBroadcastInputAcrossBatch(int32_t inputIndex) const noexcept = 0;
 
    virtual void configurePlugin(Dims const* inputDims, int32_t nbInputs, Dims const* outputDims, int32_t nbOutputs,
        DataType const* inputTypes, DataType const* outputTypes, bool const* inputIsBroadcast,
        bool const* outputIsBroadcast, PluginFormat floatFormat, int32_t maxBatchSize) noexcept
        = 0;
 
    IPluginV2Ext() = default;
    ~IPluginV2Ext() override = default;
 
    virtual void attachToContext(
        cudnnContext* /*cudnn*/, cublasContext* /*cublas*/, IGpuAllocator* /*allocator*/) noexcept
    {
    }
 
    virtual void detachFromContext() noexcept {}
 
    IPluginV2Ext* clone() const noexcept override = 0;
 
protected:
    // @cond SuppressDoxyWarnings
    IPluginV2Ext(IPluginV2Ext const&) = default;
    IPluginV2Ext(IPluginV2Ext&&) = default;
    IPluginV2Ext& operator=(IPluginV2Ext const&) & = default;
    IPluginV2Ext& operator=(IPluginV2Ext&&) & = default;
// @endcond
 
    int32_t getTensorRTVersion() const noexcept override
    {
        return static_cast<int32_t>((static_cast<uint32_t>(PluginVersion::kV2_EXT) << 24U)
            | (static_cast<uint32_t>(NV_TENSORRT_VERSION) & 0xFFFFFFU));
    }
 
    void configureWithFormat(Dims const* /*inputDims*/, int32_t /*nbInputs*/, Dims const* /*outputDims*/,
        int32_t /*nbOutputs*/, DataType /*type*/, PluginFormat /*format*/, int32_t /*maxBatchSize*/) noexcept override
    {
    }
};
 
class TRT_DEPRECATED IPluginV2IOExt : public IPluginV2Ext
{
public:
    virtual void configurePlugin(
        PluginTensorDesc const* in, int32_t nbInput, PluginTensorDesc const* out, int32_t nbOutput) noexcept
        = 0;
 
    virtual bool supportsFormatCombination(
        int32_t pos, PluginTensorDesc const* inOut, int32_t nbInputs, int32_t nbOutputs) const noexcept
        = 0;
 
    // @cond SuppressDoxyWarnings
    IPluginV2IOExt() = default;
    ~IPluginV2IOExt() override = default;
// @endcond
 
protected:
// @cond SuppressDoxyWarnings
    IPluginV2IOExt(IPluginV2IOExt const&) = default;
    IPluginV2IOExt(IPluginV2IOExt&&) = default;
    IPluginV2IOExt& operator=(IPluginV2IOExt const&) & = default;
    IPluginV2IOExt& operator=(IPluginV2IOExt&&) & = default;
// @endcond
 
    int32_t getTensorRTVersion() const noexcept override
    {
        return static_cast<int32_t>((static_cast<uint32_t>(PluginVersion::kV2_IOEXT) << 24U)
            | (static_cast<uint32_t>(NV_TENSORRT_VERSION) & 0xFFFFFFU));
    }
 
private:
    // Following are obsolete base class methods, and must not be implemented or used.
 
    void configurePlugin(Dims const*, int32_t, Dims const*, int32_t, DataType const*, DataType const*, bool const*,
        bool const*, PluginFormat, int32_t) noexcept final
    {
    }
 
    bool supportsFormat(DataType, PluginFormat) const noexcept final
    {
        return false;
    }
};
 
enum class PluginFieldType : int32_t
{
    kFLOAT16 = 0,
    kFLOAT32 = 1,
    kFLOAT64 = 2,
    kINT8 = 3,
    kINT16 = 4,
    kINT32 = 5,
    kCHAR = 6,
    kDIMS = 7,
    kUNKNOWN = 8,
    kBF16 = 9,
    kINT64 = 10,
    kFP8 = 11,
};
 
class PluginField
{
public:
    AsciiChar const* name;
    void const* data;
    PluginFieldType type;
    int32_t length;
 
    PluginField(AsciiChar const* const name_ = nullptr, void const* const data_ = nullptr,
        PluginFieldType const type_ = PluginFieldType::kUNKNOWN, int32_t const length_ = 0) noexcept
        : name(name_)
        , data(data_)
        , type(type_)
        , length(length_)
    {
    }
};
 
struct PluginFieldCollection
{
    int32_t nbFields;
    PluginField const* fields;
};
 
enum class PluginCapabilityType : int32_t
{
    kCORE = 0,
    kBUILD = 1,
    kRUNTIME = 2
};
 
enum class TensorRTPhase : int32_t
{
    kBUILD = 0,
    kRUNTIME = 1
};
 
namespace v_1_0
{
class IPluginCreatorInterface : public IVersionedInterface
{
public:
    ~IPluginCreatorInterface() noexcept override = default;
 
protected:
    IPluginCreatorInterface() = default;
    IPluginCreatorInterface(IPluginCreatorInterface const&) = default;
    IPluginCreatorInterface(IPluginCreatorInterface&&) = default;
    IPluginCreatorInterface& operator=(IPluginCreatorInterface const&) & = default;
    IPluginCreatorInterface& operator=(IPluginCreatorInterface&&) & = default;
};
 
class TRT_DEPRECATED IPluginCreator : public IPluginCreatorInterface
{
public:
    virtual AsciiChar const* getPluginName() const noexcept = 0;
 
    virtual AsciiChar const* getPluginVersion() const noexcept = 0;
 
    virtual PluginFieldCollection const* getFieldNames() noexcept = 0;
 
    virtual IPluginV2* createPlugin(AsciiChar const* name, PluginFieldCollection const* fc) noexcept = 0;
 
    virtual IPluginV2* deserializePlugin(AsciiChar const* name, void const* serialData, size_t serialLength) noexcept
        = 0;
 
    virtual void setPluginNamespace(AsciiChar const* pluginNamespace) noexcept = 0;
 
    virtual AsciiChar const* getPluginNamespace() const noexcept = 0;
 
    IPluginCreator() = default;
    ~IPluginCreator() override = default;
 
protected:
    // @cond SuppressDoxyWarnings
    IPluginCreator(IPluginCreator const&) = default;
    IPluginCreator(IPluginCreator&&) = default;
    IPluginCreator& operator=(IPluginCreator const&) & = default;
    IPluginCreator& operator=(IPluginCreator&&) & = default;
    // @endcond
public:
    InterfaceInfo getInterfaceInfo() const noexcept override
    {
        return InterfaceInfo{"PLUGIN CREATOR_V1", 1, 0};
    }
};
} // namespace v_1_0
 
using IPluginCreatorInterface = v_1_0::IPluginCreatorInterface;
 
using IPluginCreator = v_1_0::IPluginCreator;
 
} // namespace nvinfer1
 
#endif // NV_INFER_RUNTIME_PLUGIN_H