NvInferSafeRuntime.h

Go to the documentation of this file.

/*
 * SPDX-FileCopyrightText: Copyright (c) 1993-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 * SPDX-License-Identifier: LicenseRef-NvidiaProprietary
 *
 * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual
 * property and proprietary rights in and to this material, related
 * documentation and any modifications thereto. Any use, reproduction,
 * disclosure or distribution of this material and related documentation
 * without an express license agreement from NVIDIA CORPORATION or
 * its affiliates is strictly prohibited.
 */
 
#ifndef NV_INFER_SAFE_RUNTIME_H
#define NV_INFER_SAFE_RUNTIME_H
 
#include "NvInferRuntimeBase.h"
#include "NvInferRuntimePlugin.h"
#include <cstddef>
#include <cstdint>
 
 
namespace nvinfer1
{
namespace safe
{
class ICudaEngine;
class IExecutionContext;
 
class IRuntime
{
public:
    virtual ICudaEngine* deserializeCudaEngine(void const* const blob, std::size_t const size) noexcept = 0;
 
    virtual void setGpuAllocator(IGpuAllocator* const allocator) noexcept = 0;
 
    //
    virtual void setErrorRecorder(IErrorRecorder* const recorder) noexcept = 0;
 
    virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
 
    IRuntime() = default;
    virtual ~IRuntime() noexcept = default;
    IRuntime(IRuntime const&) = delete;
    IRuntime(IRuntime&&) = delete;
    IRuntime& operator=(IRuntime const&) & = delete;
    IRuntime& operator=(IRuntime&&) & = delete;
};
 
class ICudaEngine
{
public:
    TRT_DEPRECATED virtual std::int32_t getNbBindings() const noexcept = 0;
 
    TRT_DEPRECATED virtual std::int32_t getBindingIndex(AsciiChar const* const name) const noexcept = 0;
 
    TRT_DEPRECATED virtual AsciiChar const* getBindingName(std::int32_t const bindingIndex) const noexcept = 0;
 
    TRT_DEPRECATED virtual bool bindingIsInput(std::int32_t const bindingIndex) const noexcept = 0;
 
    TRT_DEPRECATED virtual Dims getBindingDimensions(std::int32_t const bindingIndex) const noexcept = 0;
 
    TRT_DEPRECATED virtual DataType getBindingDataType(std::int32_t const bindingIndex) const noexcept = 0;
 
    virtual IExecutionContext* createExecutionContext() noexcept = 0;
 
    virtual IExecutionContext* createExecutionContextWithoutDeviceMemory() noexcept = 0;
 
    virtual size_t getDeviceMemorySize() const noexcept = 0;
 
    TRT_DEPRECATED virtual std::int32_t getBindingBytesPerComponent(std::int32_t const bindingIndex) const noexcept = 0;
 
    TRT_DEPRECATED virtual std::int32_t getBindingComponentsPerElement(std::int32_t const bindingIndex) const noexcept = 0;
 
    TRT_DEPRECATED virtual TensorFormat getBindingFormat(std::int32_t const bindingIndex) const noexcept = 0;
 
    TRT_DEPRECATED virtual std::int32_t getBindingVectorizedDim(std::int32_t const bindingIndex) const noexcept = 0;
 
    virtual AsciiChar const* getName() const noexcept = 0;
 
    //
    virtual void setErrorRecorder(IErrorRecorder* const recorder) noexcept = 0;
 
    virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
 
    ICudaEngine() = default;
    virtual ~ICudaEngine() noexcept = default;
    ICudaEngine(ICudaEngine const&) = delete;
    ICudaEngine(ICudaEngine&&) = delete;
    ICudaEngine& operator=(ICudaEngine const&) & = delete;
    ICudaEngine& operator=(ICudaEngine&&) & = delete;
 
    virtual Dims getTensorShape(AsciiChar const* tensorName) const noexcept = 0;
 
    virtual DataType getTensorDataType(AsciiChar const* tensorName) const noexcept = 0;
 
    virtual TensorIOMode getTensorIOMode(AsciiChar const* tensorName) const noexcept = 0;
 
    virtual std::int32_t getTensorBytesPerComponent(AsciiChar const* tensorName) const noexcept = 0;
 
    virtual std::int32_t getTensorComponentsPerElement(AsciiChar const* tensorName) const noexcept = 0;
 
    virtual TensorFormat getTensorFormat(AsciiChar const* tensorName) const noexcept = 0;
 
    virtual std::int32_t getTensorVectorizedDim(AsciiChar const* tensorName) const noexcept = 0;
 
    virtual std::int32_t getNbIOTensors() const noexcept = 0;
 
    virtual AsciiChar const* getIOTensorName(std::int32_t const index) const noexcept = 0;
};
 
struct FloatingPointErrorInformation
{
    int32_t nbNanErrors;
    int32_t nbInfErrors;
};
 
class IExecutionContext
{
public:
    virtual ICudaEngine const& getEngine() const noexcept = 0;
 
    virtual void setName(AsciiChar const* const name) noexcept = 0;
 
    virtual AsciiChar const* getName() const noexcept = 0;
 
    virtual void setDeviceMemory(void* const memory) noexcept = 0;
 
    TRT_DEPRECATED virtual Dims getStrides(std::int32_t const bindingIndex) const noexcept = 0;
 
    //
    virtual void setErrorRecorder(IErrorRecorder* const recorder) noexcept = 0;
 
    virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
 
    TRT_DEPRECATED virtual bool enqueueV2(
        void* const* const bindings, cudaStream_t const stream, cudaEvent_t const* const inputConsumed) noexcept = 0;
 
    IExecutionContext() = default;
    virtual ~IExecutionContext() noexcept = default;
    IExecutionContext(IExecutionContext const&) = delete;
    IExecutionContext(IExecutionContext&&) = delete;
    IExecutionContext& operator=(IExecutionContext const&) & = delete;
    IExecutionContext& operator=(IExecutionContext&&) & = delete;
 
    virtual void setErrorBuffer(FloatingPointErrorInformation* const buffer) noexcept = 0;
 
    virtual FloatingPointErrorInformation* getErrorBuffer() const noexcept = 0;
 
    virtual Dims getTensorStrides(AsciiChar const* tensorName) const noexcept = 0;
 
    virtual bool setInputTensorAddress(AsciiChar const* tensorName, void const* data) noexcept = 0;
 
    virtual bool setOutputTensorAddress(AsciiChar const* tensorName, void* data) noexcept = 0;
 
    virtual bool setInputConsumedEvent(cudaEvent_t event) noexcept = 0;
 
    virtual cudaEvent_t getInputConsumedEvent() const noexcept = 0;
 
    virtual void const* getInputTensorAddress(AsciiChar const* tensorName) const noexcept = 0;
 
    virtual void* getOutputTensorAddress(AsciiChar const* tensorName) const noexcept = 0;
 
    virtual bool enqueueV3(cudaStream_t stream) noexcept = 0;
};
 
 
class IPluginRegistry
{
public:
    virtual bool registerCreator(IPluginCreator& creator, AsciiChar const* const pluginNamespace) noexcept = 0;
 
    virtual IPluginCreator* const* getPluginCreatorList(int32_t* const numCreators) const noexcept = 0;
 
    virtual IPluginCreator* getPluginCreator(AsciiChar const* const pluginName, AsciiChar const* const pluginVersion,
        AsciiChar const* const pluginNamespace = "") noexcept
        = 0;
 
    //
    virtual void setErrorRecorder(IErrorRecorder* const recorder) noexcept = 0;
 
    virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
 
    virtual bool deregisterCreator(IPluginCreator const& creator) noexcept = 0;
 
    // @cond SuppressDoxyWarnings
    IPluginRegistry() = default;
    IPluginRegistry(IPluginRegistry const&) = delete;
    IPluginRegistry(IPluginRegistry&&) = delete;
    IPluginRegistry& operator=(IPluginRegistry const&) & = delete;
    IPluginRegistry& operator=(IPluginRegistry&&) & = delete;
    // @endcond
 
protected:
    virtual ~IPluginRegistry() noexcept = default;
};
 
IRuntime* createInferRuntime(ILogger& logger) noexcept;
 
extern "C" TENSORRTAPI IPluginRegistry* getSafePluginRegistry() noexcept;
 
template <typename T>
class PluginRegistrar
{
public:
    PluginRegistrar()
    {
        getSafePluginRegistry()->registerCreator(instance, "");
    }
 
private:
    T instance{};
};
 
} // namespace safe
 
} // namespace nvinfer1
 
#define REGISTER_SAFE_TENSORRT_PLUGIN(name)                                                                            \
    static nvinfer1::safe::PluginRegistrar<name> pluginRegistrar##name {}
#endif // NV_INFER_SAFE_RUNTIME_H