NvInferSafeRuntime.h

Go to the documentation of this file.

/*
 * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 * SPDX-License-Identifier: Apache-2.0
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
 
#ifndef NV_INFER_SAFE_RUNTIME_H
#define NV_INFER_SAFE_RUNTIME_H
 
#define NV_INFER_INTERNAL_INCLUDE_RUNTIME_BASE 1
#include "NvInferRuntimeBase.h"
#undef NV_INFER_INTERNAL_INCLUDE_RUNTIME_BASE
#include "NvInferRuntimePlugin.h"
#include <cstddef>
#include <cstdint>
 
 
namespace nvinfer1
{
namespace safe
{
class ICudaEngine;
class IExecutionContext;
 
class IRuntime
{
public:
    virtual ICudaEngine* deserializeCudaEngine(void const* const blob, std::size_t const size) noexcept = 0;
 
    virtual void setGpuAllocator(IGpuAllocator* const allocator) noexcept = 0;
 
    virtual void setErrorRecorder(IErrorRecorder* const recorder) noexcept = 0;
 
    virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
 
    IRuntime() = default;
    virtual ~IRuntime() noexcept = default;
    IRuntime(IRuntime const&) = delete;
    IRuntime(IRuntime&&) = delete;
    IRuntime& operator=(IRuntime const&) & = delete;
    IRuntime& operator=(IRuntime&&) & = delete;
};
 
class ICudaEngine
{
public:
    virtual IExecutionContext* createExecutionContext() noexcept = 0;
 
    virtual IExecutionContext* createExecutionContextWithoutDeviceMemory() noexcept = 0;
 
    virtual size_t getDeviceMemorySize() const noexcept = 0;
 
    virtual AsciiChar const* getName() const noexcept = 0;
 
    virtual void setErrorRecorder(IErrorRecorder* const recorder) noexcept = 0;
 
    virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
 
    ICudaEngine() = default;
    virtual ~ICudaEngine() noexcept = default;
    ICudaEngine(ICudaEngine const&) = delete;
    ICudaEngine(ICudaEngine&&) = delete;
    ICudaEngine& operator=(ICudaEngine const&) & = delete;
    ICudaEngine& operator=(ICudaEngine&&) & = delete;
 
    virtual Dims getTensorShape(AsciiChar const* const tensorName) const noexcept = 0;
 
    virtual DataType getTensorDataType(AsciiChar const* const tensorName) const noexcept = 0;
 
    virtual TensorIOMode getTensorIOMode(AsciiChar const* const tensorName) const noexcept = 0;
 
    virtual std::int32_t getTensorBytesPerComponent(AsciiChar const* const tensorName) const noexcept = 0;
 
    virtual std::int32_t getTensorComponentsPerElement(AsciiChar const* const tensorName) const noexcept = 0;
 
    virtual TensorFormat getTensorFormat(AsciiChar const* const tensorName) const noexcept = 0;
 
    virtual std::int32_t getTensorVectorizedDim(AsciiChar const* const tensorName) const noexcept = 0;
 
    virtual std::int32_t getNbIOTensors() const noexcept = 0;
 
    virtual AsciiChar const* getIOTensorName(std::int32_t const index) const noexcept = 0;
};
 
struct RuntimeErrorInformation
{
    uint64_t bitMask;
};
 
enum class RuntimeErrorType : uint64_t
{
    kNAN_CONSUMED = 1ULL << 0,
    kINF_CONSUMED = 1ULL << 1,
    kGATHER_OOB = 1ULL << 2,
    kSCATTER_OOB = 1ULL << 3,
    kSCATTER_RACE = 1ULL << 4,
};
 
class IExecutionContext
{
public:
    virtual ICudaEngine const& getEngine() const noexcept = 0;
 
    virtual void setName(AsciiChar const* const name) noexcept = 0;
 
    virtual AsciiChar const* getName() const noexcept = 0;
 
    virtual void setDeviceMemory(void* const memory) noexcept = 0;
 
    virtual void setErrorRecorder(IErrorRecorder* const recorder) noexcept = 0;
 
    virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
 
    IExecutionContext() = default;
    virtual ~IExecutionContext() noexcept = default;
    IExecutionContext(IExecutionContext const&) = delete;
    IExecutionContext(IExecutionContext&&) = delete;
    IExecutionContext& operator=(IExecutionContext const&) & = delete;
    IExecutionContext& operator=(IExecutionContext&&) & = delete;
 
    virtual void setErrorBuffer(RuntimeErrorInformation* const buffer) noexcept = 0;
 
    virtual RuntimeErrorInformation* getErrorBuffer() const noexcept = 0;
 
    virtual Dims getTensorStrides(AsciiChar const* const tensorName) const noexcept = 0;
 
    virtual bool setInputTensorAddress(AsciiChar const* const tensorName, void const* const data) noexcept = 0;
 
    virtual bool setOutputTensorAddress(AsciiChar const* const tensorName, void* const data) noexcept = 0;
 
    virtual bool setInputConsumedEvent(cudaEvent_t const event) noexcept = 0;
 
    virtual cudaEvent_t getInputConsumedEvent() const noexcept = 0;
 
    virtual void const* getInputTensorAddress(AsciiChar const* const tensorName) const noexcept = 0;
 
    virtual void* getOutputTensorAddress(AsciiChar const* const tensorName) const noexcept = 0;
 
    virtual bool enqueueV3(cudaStream_t const stream) noexcept = 0;
};
 
 
class IPluginRegistry
{
public:
    virtual bool registerCreator(IPluginCreator& creator, AsciiChar const* const pluginNamespace) noexcept = 0;
 
    virtual IPluginCreator* const* getPluginCreatorList(int32_t* const numCreators) const noexcept = 0;
 
    virtual IPluginCreator* getPluginCreator(AsciiChar const* const pluginName, AsciiChar const* const pluginVersion,
        AsciiChar const* const pluginNamespace = "") noexcept = 0;
 
    virtual void setErrorRecorder(IErrorRecorder* const recorder) noexcept = 0;
 
    virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
 
    virtual bool deregisterCreator(IPluginCreator const& creator) noexcept = 0;
 
    // @cond SuppressDoxyWarnings
    IPluginRegistry() = default;
    IPluginRegistry(IPluginRegistry const&) = delete;
    IPluginRegistry(IPluginRegistry&&) = delete;
    IPluginRegistry& operator=(IPluginRegistry const&) & = delete;
    IPluginRegistry& operator=(IPluginRegistry&&) & = delete;
    // @endcond
 
protected:
    virtual ~IPluginRegistry() noexcept = default;
};
 
IRuntime* createInferRuntime(ILogger& logger) noexcept;
 
extern "C" TENSORRTAPI IPluginRegistry* getSafePluginRegistry() noexcept;
 
template <typename T>
class PluginRegistrar
{
public:
    PluginRegistrar()
    {
        getSafePluginRegistry()->registerCreator(instance, "");
    }
 
private:
    T instance{};
};
 
} // namespace safe
 
} // namespace nvinfer1
 
#define REGISTER_SAFE_TENSORRT_PLUGIN(name)                                                                            \
    static nvinfer1::safe::PluginRegistrar<name> pluginRegistrar##name {}
#endif // NV_INFER_SAFE_RUNTIME_H