Program Listing for File core.hpp

Clara Holoscan v0.4.0

Return to documentation for file (modules/holoinfer/src/infer/trt/core.hpp)


/* * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include <cstring> #include <iostream> #include <memory> #include <string> #include <vector> #include <NvInfer.h> #include <NvInferPlugin.h> #include <holoinfer_constants.hpp> #include <infer/infer.hpp> #include "utils.hpp" namespace holoscan { namespace inference { class TrtInfer : public InferBase { public: TrtInfer(const std::string& model_path, const std::string& model_name, bool enable_fp16, bool is_engine_path, bool cuda_buf_in, bool cuda_buf_out); ~TrtInfer(); InferStatus do_inference(std::shared_ptr<DataBuffer>& input_data, std::shared_ptr<DataBuffer>& output_buffer); std::vector<int64_t> get_input_dims() const; std::vector<int64_t> get_output_dims() const; void cleanup() {} private: std::string model_path_; std::string model_name_; std::vector<int64_t> input_dims_; std::vector<int64_t> output_dims_; std::shared_ptr<std::vector<void*>> inference_bindings; bool enable_fp16_; bool cuda_buf_in_; bool cuda_buf_out_; bool is_engine_path_; bool initialize_parameters(); bool load_engine(); std::unique_ptr<nvinfer1::ICudaEngine> engine_ = nullptr; std::unique_ptr<nvinfer1::IExecutionContext> context_ = nullptr; NetworkOptions network_options_; Logger logger_; std::shared_ptr<DataBuffer> input_buffer_; std::shared_ptr<DataBuffer> output_buffer_; std::string engine_path_; cudaStream_t cuda_stream_ = nullptr; }; } // namespace inference } // namespace holoscan

© Copyright 2022, NVIDIA. Last updated on Jun 28, 2023.