What can I help you with?
NVIDIA Holoscan SDK v3.2.0

Program Listing for File holoviz.hpp

Return to documentation for file (include/holoscan/operators/holoviz/holoviz.hpp)

Copy
Copied!
            

/* * SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef HOLOSCAN_OPERATORS_HOLOVIZ_HOLOVIZ_HPP #define HOLOSCAN_OPERATORS_HOLOVIZ_HOLOVIZ_HPP #include <array> #include <memory> #include <optional> #include <string> #include <utility> #include <vector> #include "holoscan/core/conditions/gxf/boolean.hpp" #include "holoscan/core/io_context.hpp" #include "holoscan/core/io_spec.hpp" #include "holoscan/core/operator.hpp" #include "holoscan/core/operator_spec.hpp" #include "holoscan/core/resources/gxf/allocator.hpp" #include "holoscan/core/resources/gxf/cuda_stream_pool.hpp" #include <holoviz/callbacks.hpp> namespace holoscan::viz { typedef void* InstanceHandle; } // namespace holoscan::viz namespace holoscan::ops { // forward declaration struct BufferInfo; class HolovizOp : public Operator { public: HOLOSCAN_OPERATOR_FORWARD_ARGS(HolovizOp) HolovizOp() = default; void setup(OperatorSpec& spec) override; void initialize() override; void start() override; void compute(InputContext& op_input, OutputContext& op_output, ExecutionContext& context) override; void stop() override; enum class InputType { UNKNOWN, COLOR, COLOR_LUT, POINTS, LINES, LINE_STRIP, TRIANGLES, CROSSES, RECTANGLES, OVALS, TEXT, DEPTH_MAP, DEPTH_MAP_COLOR, POINTS_3D, LINES_3D, LINE_STRIP_3D, TRIANGLES_3D, }; enum class ImageFormat { R8_UINT, R8_SINT, R8_UNORM, R8_SNORM, R8_SRGB, R16_UINT, R16_SINT, R16_UNORM, R16_SNORM, R16_SFLOAT, R32_UINT, R32_SINT, R32_SFLOAT, R8G8B8_UNORM, R8G8B8_SNORM, R8G8B8_SRGB, R8G8B8A8_UNORM, R8G8B8A8_SNORM, R8G8B8A8_SRGB, R16G16B16A16_UNORM, R16G16B16A16_SNORM, R16G16B16A16_SFLOAT, R32G32B32A32_SFLOAT, D16_UNORM, X8_D24_UNORM, D32_SFLOAT, A2B10G10R10_UNORM_PACK32, A2R10G10B10_UNORM_PACK32, B8G8R8A8_UNORM, B8G8R8A8_SRGB, A8B8G8R8_UNORM_PACK32, A8B8G8R8_SRGB_PACK32, Y8U8Y8V8_422_UNORM, U8Y8V8Y8_422_UNORM, Y8_U8V8_2PLANE_420_UNORM, Y8_U8V8_2PLANE_422_UNORM, Y8_U8_V8_3PLANE_420_UNORM, Y8_U8_V8_3PLANE_422_UNORM, Y16_U16V16_2PLANE_420_UNORM, Y16_U16V16_2PLANE_422_UNORM, Y16_U16_V16_3PLANE_420_UNORM, Y16_U16_V16_3PLANE_422_UNORM, AUTO_DETECT = -1 }; enum class YuvModelConversion { YUV_601, YUV_709, YUV_2020, }; enum class YuvRange { ITU_FULL, ITU_NARROW, }; enum class ChromaLocation { COSITED_EVEN, MIDPOINT, }; enum class DepthMapRenderMode { POINTS, LINES, TRIANGLES }; enum class ColorSpace { SRGB_NONLINEAR, EXTENDED_SRGB_LINEAR, BT2020_LINEAR, HDR10_ST2084, PASS_THROUGH, BT709_LINEAR, AUTO = -1, }; struct InputSpec { InputSpec() = default; InputSpec(const std::string& tensor_name, InputType type) : tensor_name_(tensor_name), type_(type) {} InputSpec(const std::string& tensor_name, const std::string& type_str); explicit InputSpec(const std::string& yaml_description); explicit operator bool() const noexcept { return !tensor_name_.empty(); } std::string description() const; std::string tensor_name_; InputType type_ = InputType::UNKNOWN; float opacity_ = 1.F; int32_t priority_ = 0; ImageFormat image_format_ = ImageFormat::AUTO_DETECT; YuvModelConversion yuv_model_conversion_ = YuvModelConversion::YUV_601; YuvRange yuv_range_ = YuvRange::ITU_FULL; ChromaLocation x_chroma_location_ = ChromaLocation::COSITED_EVEN; ChromaLocation y_chroma_location_ = ChromaLocation::COSITED_EVEN; std::vector<float> color_{1.F, 1.F, 1.F, 1.F}; float line_width_ = 1.F; float point_size_ = 1.F; std::vector<std::string> text_; DepthMapRenderMode depth_map_render_mode_ = DepthMapRenderMode::POINTS; struct View { float offset_x_ = 0.F, offset_y_ = 0.F; float width_ = 1.F, height_ = 1.F; std::optional<std::array<float, 16>> matrix_; }; std::vector<View> views_; }; using Key = viz::Key; using KeyAndButtonAction = viz::KeyAndButtonAction; using KeyModifiers = viz::KeyModifiers; using MouseButton = viz::MouseButton; using KeyCallbackFunction = std::function<void(Key key, KeyAndButtonAction action, KeyModifiers modifiers)>; using UnicodeCharCallbackFunction = std::function<void(uint32_t code_point)>; using MouseButtonCallbackFunction = std::function<void(MouseButton button, KeyAndButtonAction action, KeyModifiers modifiers)>; using ScrollCallbackFunction = std::function<void(double x_offset, double y_offset)>; using CursorPosCallbackFunction = std::function<void(double x_pos, double y_pos)>; using FramebufferSizeCallbackFunction = std::function<void(int width, int height)>; using WindowSizeCallbackFunction = std::function<void(int width, int height)>; using LayerCallbackFunction = std::function<void(const std::vector<holoscan::gxf::Entity>& inputs)>; static const std::array<std::pair<InputType, std::string>, 17> kInputTypeToStr; static nvidia::gxf::Expected<holoscan::ops::HolovizOp::InputType> inputTypeFromString( const std::string& string); static std::string inputTypeToString(holoscan::ops::HolovizOp::InputType input_type); static const std::array<std::pair<holoscan::ops::HolovizOp::ImageFormat, std::string>, 41> kImageFormatToStr; static nvidia::gxf::Expected<holoscan::ops::HolovizOp::ImageFormat> imageFormatFromString( const std::string& string); static std::string imageFormatToString(holoscan::ops::HolovizOp::ImageFormat image_format); static const std::array<std::pair<holoscan::ops::HolovizOp::DepthMapRenderMode, std::string>, 3> kDepthMapRenderModeToStr; static nvidia::gxf::Expected<holoscan::ops::HolovizOp::DepthMapRenderMode> depthMapRenderModeFromString(const std::string& string); static std::string depthMapRenderModeToString( holoscan::ops::HolovizOp::DepthMapRenderMode depth_map_render_mode); static const std::array<std::pair<holoscan::ops::HolovizOp::YuvModelConversion, std::string>, 3> kYuvModelConversionToStr; static nvidia::gxf::Expected<holoscan::ops::HolovizOp::YuvModelConversion> yuvModelConversionFromString(const std::string& string); static std::string yuvModelConversionToString( holoscan::ops::HolovizOp::YuvModelConversion yuv_model_conversion); static const std::array<std::pair<holoscan::ops::HolovizOp::YuvRange, std::string>, 2> kYuvRangeToStr; static nvidia::gxf::Expected<holoscan::ops::HolovizOp::YuvRange> yuvRangeFromString( const std::string& string); static std::string yuvRangeToString(holoscan::ops::HolovizOp::YuvRange yuv_range); static const std::array<std::pair<holoscan::ops::HolovizOp::ChromaLocation, std::string>, 2> kChromaLoactionToStr; static nvidia::gxf::Expected<holoscan::ops::HolovizOp::ChromaLocation> chromaLocationFromString( const std::string& string); static std::string chromaLocationToString( holoscan::ops::HolovizOp::ChromaLocation chroma_location); static const std::array<std::pair<holoscan::ops::HolovizOp::ColorSpace, std::string>, 7> kColorSpaceToStr; static nvidia::gxf::Expected<holoscan::ops::HolovizOp::ColorSpace> colorSpaceFromString( const std::string& string); static std::string colorSpaceToString(holoscan::ops::HolovizOp::ColorSpace color_space); protected: void disable_via_window_close(); private: bool enable_conditional_port(const std::string& name, bool set_none_condition_on_disabled = false); void set_input_spec(const InputSpec& input_spec); void set_input_spec_geometry(const InputSpec& input_spec); void read_frame_buffer(InputContext& op_input, OutputContext& op_output, ExecutionContext& context); void render_color_image(const InputSpec& input_spec, BufferInfo& buffer_info); void render_geometry(const InputSpec& input_spec, BufferInfo& buffer_info, cudaStream_t stream); void render_depth_map(InputSpec* const input_spec_depth_map, const BufferInfo& buffer_info_depth_map, InputSpec* const input_spec_depth_map_color, const BufferInfo& buffer_info_depth_map_color); Parameter<holoscan::IOSpec*> render_buffer_input_; Parameter<holoscan::IOSpec*> render_buffer_output_; Parameter<holoscan::IOSpec*> camera_pose_output_; Parameter<std::vector<InputSpec>> tensors_; Parameter<std::vector<std::vector<float>>> color_lut_; Parameter<std::string> window_title_; Parameter<std::string> display_name_; Parameter<uint32_t> width_; Parameter<uint32_t> height_; Parameter<float> framerate_; Parameter<bool> use_exclusive_display_; Parameter<bool> fullscreen_; Parameter<bool> headless_; Parameter<bool> framebuffer_srgb_; Parameter<bool> vsync_; Parameter<ColorSpace> display_color_space_; Parameter<std::shared_ptr<BooleanCondition>> window_close_condition_; Parameter<std::shared_ptr<BooleanCondition>> window_close_scheduling_term_; Parameter<std::shared_ptr<Allocator>> allocator_; Parameter<std::shared_ptr<CudaStreamPool>> cuda_stream_pool_; Parameter<std::string> font_path_; Parameter<std::string> camera_pose_output_type_; Parameter<std::array<float, 3>> camera_eye_; Parameter<std::array<float, 3>> camera_look_at_; Parameter<std::array<float, 3>> camera_up_; std::vector<ImageFormat> supported_image_formats_; holoscan::Parameter<KeyCallbackFunction> key_callback_; holoscan::Parameter<UnicodeCharCallbackFunction> unicode_char_callback_; holoscan::Parameter<MouseButtonCallbackFunction> mouse_button_callback_; holoscan::Parameter<ScrollCallbackFunction> scroll_callback_; holoscan::Parameter<CursorPosCallbackFunction> cursor_pos_callback_; holoscan::Parameter<FramebufferSizeCallbackFunction> framebuffer_size_callback_; holoscan::Parameter<WindowSizeCallbackFunction> window_size_callback_; holoscan::Parameter<LayerCallbackFunction> layer_callback_; // internal state viz::InstanceHandle instance_ = nullptr; std::vector<float> lut_; std::vector<InputSpec> initial_input_spec_; bool render_buffer_input_enabled_ = false; bool render_buffer_output_enabled_ = false; bool camera_pose_output_enabled_ = false; bool is_first_tick_ = true; static std::remove_pointer_t<viz::KeyCallbackFunction> key_callback_handler; static std::remove_pointer_t<viz::UnicodeCharCallbackFunction> unicode_char_callback_handler; static std::remove_pointer_t<viz::MouseButtonCallbackFunction> mouse_button_callback_handler; static std::remove_pointer_t<viz::ScrollCallbackFunction> scroll_callback_handler; static std::remove_pointer_t<viz::CursorPosCallbackFunction> cursor_pos_callback_handler; static std::remove_pointer_t<viz::FramebufferSizeCallbackFunction> framebuffer_size_callback_handler; static std::remove_pointer_t<viz::WindowSizeCallbackFunction> window_size_callback_handler; std::array<float, 3> camera_eye_cur_; //< current camera eye position std::array<float, 3> camera_look_at_cur_; //< current camera look at position std::array<float, 3> camera_up_cur_; //< current camera up vector }; } // namespace holoscan::ops template <> struct YAML::convert<holoscan::ops::HolovizOp::InputSpec> { static Node encode(const holoscan::ops::HolovizOp::InputSpec& input_spec) { Node node; node["type"] = holoscan::ops::HolovizOp::inputTypeToString(input_spec.type_); node["name"] = input_spec.tensor_name_; node["opacity"] = std::to_string(input_spec.opacity_); node["priority"] = std::to_string(input_spec.priority_); switch (input_spec.type_) { case holoscan::ops::HolovizOp::InputType::COLOR: case holoscan::ops::HolovizOp::InputType::COLOR_LUT: case holoscan::ops::HolovizOp::InputType::DEPTH_MAP_COLOR: node["image_format"] = holoscan::ops::HolovizOp::imageFormatToString(input_spec.image_format_); switch (input_spec.image_format_) { case holoscan::ops::HolovizOp::ImageFormat::Y8U8Y8V8_422_UNORM: case holoscan::ops::HolovizOp::ImageFormat::U8Y8V8Y8_422_UNORM: case holoscan::ops::HolovizOp::ImageFormat::Y8_U8V8_2PLANE_420_UNORM: case holoscan::ops::HolovizOp::ImageFormat::Y8_U8V8_2PLANE_422_UNORM: case holoscan::ops::HolovizOp::ImageFormat::Y8_U8_V8_3PLANE_420_UNORM: case holoscan::ops::HolovizOp::ImageFormat::Y8_U8_V8_3PLANE_422_UNORM: case holoscan::ops::HolovizOp::ImageFormat::Y16_U16V16_2PLANE_420_UNORM: case holoscan::ops::HolovizOp::ImageFormat::Y16_U16V16_2PLANE_422_UNORM: case holoscan::ops::HolovizOp::ImageFormat::Y16_U16_V16_3PLANE_420_UNORM: case holoscan::ops::HolovizOp::ImageFormat::Y16_U16_V16_3PLANE_422_UNORM: node["yuv_model_conversion"] = holoscan::ops::HolovizOp::yuvModelConversionToString( input_spec.yuv_model_conversion_); node["yuv_range"] = holoscan::ops::HolovizOp::yuvRangeToString(input_spec.yuv_range_); node["x_chroma_location"] = holoscan::ops::HolovizOp::chromaLocationToString(input_spec.x_chroma_location_); node["y_chroma_location"] = holoscan::ops::HolovizOp::chromaLocationToString(input_spec.y_chroma_location_); break; default: break; } break; case holoscan::ops::HolovizOp::InputType::POINTS: case holoscan::ops::HolovizOp::InputType::LINES: case holoscan::ops::HolovizOp::InputType::LINE_STRIP: case holoscan::ops::HolovizOp::InputType::TRIANGLES: case holoscan::ops::HolovizOp::InputType::CROSSES: case holoscan::ops::HolovizOp::InputType::RECTANGLES: case holoscan::ops::HolovizOp::InputType::OVALS: case holoscan::ops::HolovizOp::InputType::POINTS_3D: case holoscan::ops::HolovizOp::InputType::LINES_3D: case holoscan::ops::HolovizOp::InputType::LINE_STRIP_3D: case holoscan::ops::HolovizOp::InputType::TRIANGLES_3D: node["color"] = input_spec.color_; node["line_width"] = std::to_string(input_spec.line_width_); node["point_size"] = std::to_string(input_spec.point_size_); break; case holoscan::ops::HolovizOp::InputType::TEXT: node["color"] = input_spec.color_; node["text"] = input_spec.text_; break; case holoscan::ops::HolovizOp::InputType::DEPTH_MAP: node["depth_map_render_mode"] = holoscan::ops::HolovizOp::depthMapRenderModeToString(input_spec.depth_map_render_mode_); break; default: break; } for (auto&& view : input_spec.views_) { node["views"].push_back(view); } return node; } static bool decode(const Node& node, holoscan::ops::HolovizOp::InputSpec& input_spec) { if (!node.IsMap()) { HOLOSCAN_LOG_ERROR("InputSpec: expected a map"); return false; } // YAML is using exceptions, catch them try { const auto maybe_input_type = holoscan::ops::HolovizOp::inputTypeFromString(node["type"].as<std::string>()); if (!maybe_input_type) { return false; } input_spec.tensor_name_ = node["name"].as<std::string>(); input_spec.type_ = maybe_input_type.value(); input_spec.opacity_ = node["opacity"].as<float>(input_spec.opacity_); input_spec.priority_ = node["priority"].as<int32_t>(input_spec.priority_); switch (input_spec.type_) { case holoscan::ops::HolovizOp::InputType::COLOR: case holoscan::ops::HolovizOp::InputType::COLOR_LUT: case holoscan::ops::HolovizOp::InputType::DEPTH_MAP_COLOR: if (node["image_format"]) { const auto maybe_image_format = holoscan::ops::HolovizOp::imageFormatFromString( node["image_format"].as<std::string>()); if (maybe_image_format) { input_spec.image_format_ = maybe_image_format.value(); } } if (node["yuv_model_conversion"]) { const auto maybe_yuv_model_conversion = holoscan::ops::HolovizOp::yuvModelConversionFromString( node["yuv_model_conversion"].as<std::string>()); if (maybe_yuv_model_conversion) { input_spec.yuv_model_conversion_ = maybe_yuv_model_conversion.value(); } } if (node["yuv_range"]) { const auto maybe_yuv_range = holoscan::ops::HolovizOp::yuvRangeFromString(node["yuv_range"].as<std::string>()); if (maybe_yuv_range) { input_spec.yuv_range_ = maybe_yuv_range.value(); } } if (node["x_chroma_location"]) { const auto maybe_x_chroma_location = holoscan::ops::HolovizOp::chromaLocationFromString( node["x_chroma_location"].as<std::string>()); if (maybe_x_chroma_location) { input_spec.x_chroma_location_ = maybe_x_chroma_location.value(); } } if (node["chroma_y_location"]) { const auto maybe_y_chroma_location = holoscan::ops::HolovizOp::chromaLocationFromString( node["y_chroma_location"].as<std::string>()); if (maybe_y_chroma_location) { input_spec.y_chroma_location_ = maybe_y_chroma_location.value(); } } break; case holoscan::ops::HolovizOp::InputType::LINES: case holoscan::ops::HolovizOp::InputType::LINE_STRIP: case holoscan::ops::HolovizOp::InputType::TRIANGLES: case holoscan::ops::HolovizOp::InputType::CROSSES: case holoscan::ops::HolovizOp::InputType::RECTANGLES: case holoscan::ops::HolovizOp::InputType::OVALS: case holoscan::ops::HolovizOp::InputType::POINTS_3D: case holoscan::ops::HolovizOp::InputType::LINES_3D: case holoscan::ops::HolovizOp::InputType::LINE_STRIP_3D: case holoscan::ops::HolovizOp::InputType::TRIANGLES_3D: input_spec.color_ = node["color"].as<std::vector<float>>(input_spec.color_); input_spec.line_width_ = node["line_width"].as<float>(input_spec.line_width_); input_spec.point_size_ = node["point_size"].as<float>(input_spec.point_size_); break; case holoscan::ops::HolovizOp::InputType::TEXT: input_spec.color_ = node["color"].as<std::vector<float>>(input_spec.color_); input_spec.text_ = node["text"].as<std::vector<std::string>>(input_spec.text_); break; case holoscan::ops::HolovizOp::InputType::DEPTH_MAP: if (node["depth_map_render_mode"]) { const auto maybe_depth_map_render_mode = holoscan::ops::HolovizOp::depthMapRenderModeFromString( node["depth_map_render_mode"].as<std::string>()); if (maybe_depth_map_render_mode) { input_spec.depth_map_render_mode_ = maybe_depth_map_render_mode.value(); } } break; default: break; } if (node["views"]) { input_spec.views_ = node["views"].as<std::vector<holoscan::ops::HolovizOp::InputSpec::View>>(); } return true; } catch (const std::exception& e) { HOLOSCAN_LOG_ERROR(e.what()); return false; } } }; template <> struct YAML::convert<holoscan::ops::HolovizOp::InputSpec::View> { static Node encode(const holoscan::ops::HolovizOp::InputSpec::View& view) { Node node; node["offset_x"] = view.offset_x_; node["offset_y"] = view.offset_y_; node["width"] = view.width_; node["height"] = view.height_; if (view.matrix_.has_value()) { node["matrix"] = view.matrix_.value(); } return node; } static bool decode(const Node& node, holoscan::ops::HolovizOp::InputSpec::View& view) { if (!node.IsMap()) { HOLOSCAN_LOG_ERROR("InputSpec: expected a map"); return false; } // YAML is using exceptions, catch them try { view.offset_x_ = node["offset_x"].as<float>(view.offset_x_); view.offset_y_ = node["offset_y"].as<float>(view.offset_y_); view.width_ = node["width"].as<float>(view.width_); view.height_ = node["height"].as<float>(view.height_); if (node["matrix"]) { view.matrix_ = node["matrix"].as<std::array<float, 16>>(); } return true; } catch (const std::exception& e) { HOLOSCAN_LOG_ERROR(e.what()); return false; } } }; template <> struct YAML::convert<holoscan::ops::HolovizOp::ColorSpace> { static Node encode(const holoscan::ops::HolovizOp::ColorSpace& color_space) { Node node; node.push_back(holoscan::ops::HolovizOp::colorSpaceToString(color_space)); return node; } static bool decode(const Node& node, holoscan::ops::HolovizOp::ColorSpace& color_space) { if (!node.IsScalar()) { return false; } // YAML is using exceptions, catch them try { const auto maybe_color_space = holoscan::ops::HolovizOp::colorSpaceFromString(node.Scalar()); if (maybe_color_space) { color_space = maybe_color_space.value(); return true; } return false; } catch (const std::exception& e) { HOLOSCAN_LOG_ERROR(e.what()); return false; } } }; #define HOLOVIZ_YAML_CONVERTER(TYPE) \ template <> \ struct YAML::convert<TYPE> { \ static Node encode(TYPE&) { \ throw std::runtime_error(#TYPE " is unsupported in YAML"); \ } \ \ static bool decode(const Node&, TYPE&) { \ throw std::runtime_error(#TYPE " is unsupported in YAML"); \ } \ }; HOLOVIZ_YAML_CONVERTER(holoscan::ops::HolovizOp::KeyCallbackFunction); HOLOVIZ_YAML_CONVERTER(holoscan::ops::HolovizOp::UnicodeCharCallbackFunction); HOLOVIZ_YAML_CONVERTER(holoscan::ops::HolovizOp::MouseButtonCallbackFunction); HOLOVIZ_YAML_CONVERTER(holoscan::ops::HolovizOp::ScrollCallbackFunction); // don't need CursorPosCallbackFunction since it has the same signature as ScrollCallbackFunction HOLOVIZ_YAML_CONVERTER(holoscan::ops::HolovizOp::FramebufferSizeCallbackFunction); // don't need WindowSizeCallbackFunction since it has the same signature as // FramebufferSizeCallbackFunction HOLOVIZ_YAML_CONVERTER(holoscan::ops::HolovizOp::LayerCallbackFunction); #endif/* HOLOSCAN_OPERATORS_HOLOVIZ_HOLOVIZ_HPP */

© Copyright 2022-2025, NVIDIA. Last updated on Apr 30, 2025.