TensorRT 8.6.1
NvInferSafeRuntime.h
Go to the documentation of this file.
1/*
2 * SPDX-FileCopyrightText: Copyright (c) 1993-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: LicenseRef-NvidiaProprietary
4 *
5 * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual
6 * property and proprietary rights in and to this material, related
7 * documentation and any modifications thereto. Any use, reproduction,
8 * disclosure or distribution of this material and related documentation
9 * without an express license agreement from NVIDIA CORPORATION or
10 * its affiliates is strictly prohibited.
11 */
12
13#ifndef NV_INFER_SAFE_RUNTIME_H
14#define NV_INFER_SAFE_RUNTIME_H
15
16#include "NvInferRuntimeBase.h"
18#include <cstddef>
19#include <cstdint>
20
25
31namespace nvinfer1
32{
38namespace safe
39{
41class ICudaEngine;
43class IExecutionContext;
44
56{
57public:
77 virtual ICudaEngine* deserializeCudaEngine(void const* const blob, std::size_t const size) noexcept = 0;
78
92 virtual void setGpuAllocator(IGpuAllocator* const allocator) noexcept = 0;
93
103 //
110 virtual void setErrorRecorder(IErrorRecorder* const recorder) noexcept = 0;
111
126 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
127
128 IRuntime() = default;
129 virtual ~IRuntime() noexcept = default;
130 IRuntime(IRuntime const&) = delete;
131 IRuntime(IRuntime&&) = delete;
132 IRuntime& operator=(IRuntime const&) & = delete;
133 IRuntime& operator=(IRuntime&&) & = delete;
134};
135
144{
145public:
159 TRT_DEPRECATED virtual std::int32_t getNbBindings() const noexcept = 0;
160
182 TRT_DEPRECATED virtual std::int32_t getBindingIndex(AsciiChar const* const name) const noexcept = 0;
183
201 TRT_DEPRECATED virtual AsciiChar const* getBindingName(std::int32_t const bindingIndex) const noexcept = 0;
202
217 TRT_DEPRECATED virtual bool bindingIsInput(std::int32_t const bindingIndex) const noexcept = 0;
218
233 TRT_DEPRECATED virtual Dims getBindingDimensions(std::int32_t const bindingIndex) const noexcept = 0;
234
249 TRT_DEPRECATED virtual DataType getBindingDataType(std::int32_t const bindingIndex) const noexcept = 0;
250
262 virtual IExecutionContext* createExecutionContext() noexcept = 0;
263
277 virtual IExecutionContext* createExecutionContextWithoutDeviceMemory() noexcept = 0;
278
288 virtual size_t getDeviceMemorySize() const noexcept = 0;
289
305 TRT_DEPRECATED virtual std::int32_t getBindingBytesPerComponent(std::int32_t const bindingIndex) const noexcept = 0;
306
322 TRT_DEPRECATED virtual std::int32_t getBindingComponentsPerElement(std::int32_t const bindingIndex) const noexcept = 0;
323
337 TRT_DEPRECATED virtual TensorFormat getBindingFormat(std::int32_t const bindingIndex) const noexcept = 0;
338
354 TRT_DEPRECATED virtual std::int32_t getBindingVectorizedDim(std::int32_t const bindingIndex) const noexcept = 0;
355
370 virtual AsciiChar const* getName() const noexcept = 0;
371
381 //
388 virtual void setErrorRecorder(IErrorRecorder* const recorder) noexcept = 0;
389
405 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
406
407 ICudaEngine() = default;
408 virtual ~ICudaEngine() noexcept = default;
409 ICudaEngine(ICudaEngine const&) = delete;
411 ICudaEngine& operator=(ICudaEngine const&) & = delete;
412 ICudaEngine& operator=(ICudaEngine&&) & = delete;
413
430 virtual Dims getTensorShape(AsciiChar const* tensorName) const noexcept = 0;
431
449 virtual DataType getTensorDataType(AsciiChar const* tensorName) const noexcept = 0;
450
465 virtual TensorIOMode getTensorIOMode(AsciiChar const* tensorName) const noexcept = 0;
466
487 virtual std::int32_t getTensorBytesPerComponent(AsciiChar const* tensorName) const noexcept = 0;
488
509 virtual std::int32_t getTensorComponentsPerElement(AsciiChar const* tensorName) const noexcept = 0;
510
528 virtual TensorFormat getTensorFormat(AsciiChar const* tensorName) const noexcept = 0;
529
550 virtual std::int32_t getTensorVectorizedDim(AsciiChar const* tensorName) const noexcept = 0;
551
563 virtual std::int32_t getNbIOTensors() const noexcept = 0;
564
582 virtual AsciiChar const* getIOTensorName(std::int32_t const index) const noexcept = 0;
583};
584
592{
594 int32_t nbNanErrors;
596 int32_t nbInfErrors;
597};
598
613{
614public:
624 virtual ICudaEngine const& getEngine() const noexcept = 0;
625
640 virtual void setName(AsciiChar const* const name) noexcept = 0;
641
651 virtual AsciiChar const* getName() const noexcept = 0;
652
668 virtual void setDeviceMemory(void* const memory) noexcept = 0;
669
683 TRT_DEPRECATED virtual Dims getStrides(std::int32_t const bindingIndex) const noexcept = 0;
684
694 //
701 virtual void setErrorRecorder(IErrorRecorder* const recorder) noexcept = 0;
702
717 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
718
740 TRT_DEPRECATED virtual bool enqueueV2(
741 void* const* const bindings, cudaStream_t const stream, cudaEvent_t const* const inputConsumed) noexcept = 0;
742
743 IExecutionContext() = default;
744 virtual ~IExecutionContext() noexcept = default;
747 IExecutionContext& operator=(IExecutionContext const&) & = delete;
748 IExecutionContext& operator=(IExecutionContext&&) & = delete;
749
769 virtual void setErrorBuffer(FloatingPointErrorInformation* const buffer) noexcept = 0;
770
782 virtual FloatingPointErrorInformation* getErrorBuffer() const noexcept = 0;
783
804 virtual Dims getTensorStrides(AsciiChar const* tensorName) const noexcept = 0;
805
830 virtual bool setInputTensorAddress(AsciiChar const* tensorName, void const* data) noexcept = 0;
831
856 virtual bool setOutputTensorAddress(AsciiChar const* tensorName, void* data) noexcept = 0;
857
871 virtual bool setInputConsumedEvent(cudaEvent_t event) noexcept = 0;
872
882 virtual cudaEvent_t getInputConsumedEvent() const noexcept = 0;
883
902 virtual void const* getInputTensorAddress(AsciiChar const* tensorName) const noexcept = 0;
903
922 virtual void* getOutputTensorAddress(AsciiChar const* tensorName) const noexcept = 0;
923
939 virtual bool enqueueV3(cudaStream_t stream) noexcept = 0;
940};
941
959
961{
962public:
974 virtual bool registerCreator(IPluginCreator& creator, AsciiChar const* const pluginNamespace) noexcept = 0;
975
984 virtual IPluginCreator* const* getPluginCreatorList(int32_t* const numCreators) const noexcept = 0;
985
997 virtual IPluginCreator* getPluginCreator(AsciiChar const* const pluginName, AsciiChar const* const pluginVersion,
998 AsciiChar const* const pluginNamespace = "") noexcept
999 = 0;
1000
1010 //
1017 virtual void setErrorRecorder(IErrorRecorder* const recorder) noexcept = 0;
1018
1034 virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
1035
1051 virtual bool deregisterCreator(IPluginCreator const& creator) noexcept = 0;
1052
1053 // @cond SuppressDoxyWarnings
1054 IPluginRegistry() = default;
1055 IPluginRegistry(IPluginRegistry const&) = delete;
1056 IPluginRegistry(IPluginRegistry&&) = delete;
1057 IPluginRegistry& operator=(IPluginRegistry const&) & = delete;
1058 IPluginRegistry& operator=(IPluginRegistry&&) & = delete;
1059 // @endcond
1060
1061protected:
1062 virtual ~IPluginRegistry() noexcept = default;
1063};
1064
1075
1084
1096template <typename T>
1098{
1099public:
1101 {
1102 getSafePluginRegistry()->registerCreator(instance, "");
1103 }
1104
1105private:
1107 T instance{};
1108};
1109
1110} // namespace safe
1111
1112} // namespace nvinfer1
1113
1114#define REGISTER_SAFE_TENSORRT_PLUGIN(name) \
1115 static nvinfer1::safe::PluginRegistrar<name> pluginRegistrar##name {}
1116#endif // NV_INFER_SAFE_RUNTIME_H
#define TENSORRTAPI
Definition: NvInferRuntimeBase.h:54
#define TRT_DEPRECATED
Definition: NvInferRuntimeBase.h:40
Definition: NvInferRuntimeBase.h:179
Reference counted application-implemented error reporting interface for TensorRT objects.
Definition: NvInferRuntimeBase.h:694
Application-implemented class for controlling allocation on the GPU.
Definition: NvInferRuntimeBase.h:367
Application-implemented logging interface for the builder, refitter and runtime.
Definition: NvInferRuntimeBase.h:505
Plugin creator class for user implemented layers.
Definition: NvInferRuntimePlugin.h:803
A functionally safe engine for executing inference on a built network.
Definition: NvInferSafeRuntime.h:144
virtual TRT_DEPRECATED std::int32_t getNbBindings() const noexcept=0
Get the number of binding indices.
Functionally safe context for executing inference using an engine.
Definition: NvInferSafeRuntime.h:613
virtual ICudaEngine const & getEngine() const noexcept=0
Get the associated engine.
Single registration point for all plugins in an application. It is used to find plugin implementation...
Definition: NvInferSafeRuntime.h:961
virtual IPluginCreator * getPluginCreator(AsciiChar const *const pluginName, AsciiChar const *const pluginVersion, AsciiChar const *const pluginNamespace="") noexcept=0
Return plugin creator based on plugin name, version, and namespace associated with plugin during netw...
virtual IPluginCreator *const * getPluginCreatorList(int32_t *const numCreators) const noexcept=0
Return all the registered plugin creators and the number of registered plugin creators....
virtual bool registerCreator(IPluginCreator &creator, AsciiChar const *const pluginNamespace) noexcept=0
Register a plugin creator. Returns false if one with same type is already registered.
Allows a serialized functionally safe engine to be deserialized.
Definition: NvInferSafeRuntime.h:56
virtual ICudaEngine * deserializeCudaEngine(void const *const blob, std::size_t const size) noexcept=0
Deserialize an engine from a stream.
virtual IErrorRecorder * getErrorRecorder() const noexcept=0
Get the ErrorRecorder assigned to this interface.
virtual void setGpuAllocator(IGpuAllocator *const allocator) noexcept=0
Set the GPU allocator.
virtual void setErrorRecorder(IErrorRecorder *const recorder) noexcept=0
Set the ErrorRecorder for this interface.
Register the plugin creator to the registry The static registry object will be instantiated when the ...
Definition: NvInferSafeRuntime.h:1098
PluginRegistrar()
Definition: NvInferSafeRuntime.h:1100
IPluginRegistry * getSafePluginRegistry() noexcept
Return the safe plugin registry.
IRuntime * createInferRuntime(ILogger &logger) noexcept
Create an instance of an safe::IRuntime class.
The TensorRT API version 1 namespace.
TensorIOMode
Definition of tensor IO Mode.
Definition: NvInferRuntimeBase.h:883
char_t AsciiChar
Definition: NvInferRuntimeBase.h:94
DataType
The type of weights and tensors.
Definition: NvInferRuntimeBase.h:120
TensorFormat
Format of the input/output tensors.
Definition: NvInferRuntimeBase.h:209
Space to record information about floating point runtime errors.
Definition: NvInferSafeRuntime.h:592
int32_t nbInfErrors
Total count of errors relating to INF values (0 if none)
Definition: NvInferSafeRuntime.h:596
int32_t nbNanErrors
Total count of errors relating to NAN values (0 if none)
Definition: NvInferSafeRuntime.h:594

  Copyright © 2024 NVIDIA Corporation
  Privacy Policy | Manage My Privacy | Do Not Sell or Share My Data | Terms of Service | Accessibility | Corporate Policies | Product Security | Contact