Skip to navigation
Isaac LabHow-to guide

Streaming a camera view in a visualizer

View as Markdown

Display ground-truth camera frames from many environments in one live visualizer panel. For how the view works and which visualizers support it, see Streaming Camera View.

Quick Start

Run the run_tiled_camera_visualizer.py script in IsaacLab/scripts/tutorials/07_visualizers:

uv run python scripts/tutorials/07_visualizers/run_tiled_camera_visualizer.py \
    --task Isaac-Velocity-Rough-AnymalD --num_envs 256 --viz kit
# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
# All rights reserved.
#
# SPDX-License-Identifier: BSD-3-Clause
"""
This script demonstrates the visualizer tiled camera panel.
.. code-block:: bash
    # Kit visualizer tiled camera panel
    uv run python scripts/tutorials/07_visualizers/run_tiled_camera_visualizer.py \
 --task Isaac-Velocity-Rough-AnymalD --num_envs 256 --viz kit
    # Newton visualizer tiled camera panel
    uv run python scripts/tutorials/07_visualizers/run_tiled_camera_visualizer.py \
        --task IsaacContrib-Stack-Cube-Galbot-Left-Arm-Gripper-Visuomotor --num_envs 25 --viz newton_gl
"""
from __future__ import annotations
import argparse
import contextlib
import sys
import gymnasium as gym
import torch
import isaaclab_tasks  # noqa: F401
with contextlib.suppress(ImportError):
    import isaaclab_tasks_experimental  # noqa: F401
from isaaclab.app import add_launcher_args, launch_simulation
from isaaclab_tasks.utils import resolve_task_config, setup_preset_cli
KIT_DEFAULT_TASK = "Isaac-Velocity-Rough-AnymalD"
NEWTON_DEFAULT_TASK = "IsaacContrib-Stack-Cube-Galbot-Left-Arm-Gripper-Visuomotor"
SUPPORTED_TILED_VISUALIZERS = {"kit", "newton", "newton_gl", "newton_rtx"}
UNSUPPORTED_TILED_VISUALIZERS = {"rerun", "viser"}
def _resolve_env_regex_path(prim_path: str) -> str:
    """Resolve scene config env namespace macros to the cloned-env regex."""
    return prim_path.format(ENV_REGEX_NS="/World/envs/env_.*")
def _requested_visualizers(args_cli: argparse.Namespace) -> list[str]:
    """Return requested visualizers, defaulting to Kit for this tutorial."""
    visualizers = args_cli.visualizer or ["kit"]
    visualizers = [str(visualizer).lower() for visualizer in visualizers]
    if "none" in visualizers:
        raise ValueError("This demo requires a tiled-camera visualizer. Use '--viz kit' or '--viz newton_gl'.")
    unsupported = sorted(set(visualizers) & UNSUPPORTED_TILED_VISUALIZERS)
    if unsupported:
        raise ValueError(
            "The visualizer tiled camera panel is only implemented for Kit and Newton. "
            f"Unsupported selection: {unsupported}."
        )
    unknown = sorted(set(visualizers) - SUPPORTED_TILED_VISUALIZERS)
    if unknown:
        raise ValueError(f"Unknown visualizer selection for this demo: {unknown}.")
    return visualizers
def _make_kit_visualizer_cfg(env_cfg):
    """Create the Kit streaming-camera visualizer for the selected task."""
    from isaaclab_visualizers.kit import KitVisualizerCfg
    visualizer_cfg = KitVisualizerCfg()
    visualizer_cfg.streaming_view = True
    visualizer_cfg.streaming_envs = 36
    ego_cam_cfg = getattr(env_cfg.scene, "ego_cam", None)
    if ego_cam_cfg is not None:
        visualizer_cfg.streaming_sensor_prim_path = _resolve_env_regex_path(ego_cam_cfg.prim_path)
        return visualizer_cfg
    visualizer_cfg.streaming_sensor_prim_path = None
    visualizer_cfg.streaming_cam_eye = (3.0, 3.0, 3.0)
    visualizer_cfg.streaming_cam_target_prim_path = "/World/envs/*/Robot/base"
    # Here is an alternative eye position for a top down view
    # visualizer_cfg.streaming_cam_eye = (0.0, 0.0, 5.0)
    return visualizer_cfg
def _make_newton_visualizer_cfg(env_cfg):
    """Create the Newton streaming-camera visualizer for the selected task."""
    from isaaclab_visualizers.newton import NewtonGLVisualizerCfg
    visualizer_cfg = NewtonGLVisualizerCfg()
    visualizer_cfg.streaming_view = True
    visualizer_cfg.streaming_envs = 12
    ego_cam_cfg = getattr(env_cfg.scene, "ego_cam", None)
    if ego_cam_cfg is not None:
        visualizer_cfg.streaming_sensor_prim_path = _resolve_env_regex_path(ego_cam_cfg.prim_path)
        return visualizer_cfg
    # Here are other robot mounted camera options for this environment
    # visualizer_cfg.streaming_sensor_prim_path = "/World/envs/env_.*/Robot/left_arm_camera_sim_view_frame/left_camera"
    # visualizer_cfg.streaming_sensor_prim_path = (
    #     "/World/envs/env_.*/Robot/right_arm_camera_sim_view_frame/right_camera"
    # )
    visualizer_cfg.streaming_sensor_prim_path = None
    visualizer_cfg.streaming_cam_eye = (3.0, 3.0, 3.0)
    visualizer_cfg.streaming_cam_target_prim_path = "/World/envs/*/Robot/base"
    return visualizer_cfg
def _configure_visualizers(env_cfg, args_cli: argparse.Namespace) -> None:
    """Attach tiled camera visualizer configs to the environment simulation config."""
    visualizers = _requested_visualizers(args_cli)
    args_cli.visualizer = visualizers
    env_cfg.sim.visualizer_cfgs = [
        _make_kit_visualizer_cfg(env_cfg) if visualizer == "kit" else _make_newton_visualizer_cfg(env_cfg)
        for visualizer in visualizers
    ]
def _resolve_task(args_cli: argparse.Namespace) -> str:
    """Resolve the task for the selected visualizer."""
    if args_cli.task is not None:
        return args_cli.task
    if "newton" in _requested_visualizers(args_cli):
        return NEWTON_DEFAULT_TASK
    return KIT_DEFAULT_TASK
# add argparse arguments
parser = argparse.ArgumentParser(description="Showcase the Kit/Newton visualizer tiled camera panel.")
parser.add_argument("--num_envs", type=int, default=None, help="Number of environments to simulate.")
parser.add_argument("--task", type=str, default=None, help="Name of the task.")
# append simulation launcher cli args
add_launcher_args(parser)
args_cli, hydra_args = setup_preset_cli(parser)
args_cli.task = _resolve_task(args_cli)
sys.argv = [sys.argv[0]] + hydra_args
def main():
    """Run a random-action environment with a tiled camera visualizer."""
    # parse configuration via Hydra (supports preset selection, e.g. presets=newton_mjwarp)
    env_cfg, _ = resolve_task_config(args_cli.task, "")
    _configure_visualizers(env_cfg, args_cli)
    with launch_simulation(env_cfg, args_cli):
        # override with CLI arguments
        env_cfg.scene.num_envs = args_cli.num_envs if args_cli.num_envs is not None else env_cfg.scene.num_envs
        env_cfg.sim.device = args_cli.device if args_cli.device is not None else env_cfg.sim.device
        # create environment
        env = gym.make(args_cli.task, cfg=env_cfg)
        # print info (this is vectorized environment)
        print(f"[INFO]: Gym observation space: {env.observation_space}")
        print(f"[INFO]: Gym action space: {env.action_space}")
        env.reset()
        # keep stepping until all visualizer windows have been closed
        sim = env.unwrapped.sim
        if not sim.visualizers:
            print("[WARN]: No visualizers found. Exiting.")
            env.close()
            return
        while True:
            if sim.visualizers and not any(v.is_running() and not v.is_closed for v in sim.visualizers):
                break
            with torch.inference_mode():
                actions = 2 * torch.rand(env.action_space.shape, device=env.unwrapped.device) - 1
                env.step(actions)
        env.close()
if __name__ == "__main__":
    main()

See Examples_ below for the two ways to run the script. For the VisualizerCfg fields that customize streaming, see Configuration_ and Troubleshooting_.

Examples

Running run_tiled_camera_visualizer.py demonstrates two ways to use the streaming camera view:

  • auto-created cameras pointed at and following moving AnymalD robots, shown in the Kit visualizer
  • streaming from existing wrist-mounted robot cameras, shown in the Newton visualizer

Example 1: Following AnymalD Robots

uv run python scripts/tutorials/07_visualizers/run_tiled_camera_visualizer.py \
    --task Isaac-Velocity-Rough-AnymalD --num_envs 256 --viz kit

The script’s KitVisualizerCfg creates cameras that point at and follow each robot’s base prim, offset by streaming_cam_eye (here (3.0, 3.0, 3.0); try (0, 0, 5) for a top-down view). Of the 256 environments, 36 are randomly sampled for the camera view.

Kit visualizer: interactive viewport Kit visualizer: streaming camera view
Kit visualizer: interactive viewport Kit visualizer: streaming camera view

Example 2: Streaming from Robot-Mounted Cameras

uv run --extra teleop python scripts/tutorials/07_visualizers/run_tiled_camera_visualizer.py \
    --task IsaacContrib-Stack-Cube-Galbot-Left-Arm-Gripper-Visuomotor --num_envs 25 --viz newton_gl

The Galbot cube-stacking environment ships with wrist-mounted cameras giving an egocentric view of the gripper, table, and cubes. The script’s NewtonGLVisualizerCfg streams from the existing sensor at /World/envs/env_.*/Robot/head_camera_sim_view_frame/head_camera; edit streaming_sensor_prim_path to show a different camera. Of the 25 environments, 12 camera feeds are shown by default.

Newton visualizer: interactive viewport Newton visualizer: streaming camera view
Newton visualizer: interactive viewport Newton visualizer: streaming camera view

Configuration

To customize streaming camera behavior, edit the highlighted VisualizerCfg fields in run_tiled_camera_visualizer.py:

  • For auto-created cameras, streaming_cam_target_prim_path chooses the followed prim and streaming_cam_eye sets the camera offset from that prim. Defaults to None, which causes the visualizer to adopt the first scene camera it discovers at init; no explicit path is needed when a TiledCamera sensor is already in the scene.
  • For existing scene cameras, streaming_sensor_prim_path must match an Isaac Lab Camera sensor prim path in the selected task.
  • streaming_envs controls how many environment tiles are shown. Pass an int to randomly sample that many environments, or a list[int] to pin specific environment indices.
  • streaming_gt_types selects which ground-truth types are shown, e.g. ["rgb", "depth", "segmentation", "normals"].
  • streaming_depth_min / streaming_depth_max set the depth colormap range in metres.

Troubleshooting

  • If a generated view fails with a missing prim error, verify that streaming_cam_target_prim_path resolves in each selected environment; common template forms are /World/envs/*/... and /World/envs/env_.*/.... In most cases you can leave it as None and let the visualizer adopt an existing scene camera automatically.
  • If an existing-camera view reports that no Isaac Lab camera owns the prim, check that streaming_sensor_prim_path matches a Camera sensor in the task.
  • If the depth panel shows a flat color, adjust streaming_depth_min and streaming_depth_max to bracket the expected depth range in your scene.
  • If the view is too expensive, reduce streaming_envs, --num_envs, or the camera resolution.

Newton MJWarp with replicate_physics=True and auto-created cameras

With replicate_physics=True, only env_0 has a USD prim after physics initialization. Cameras for the remaining environments (env_1 through env_{N-1}) are dropped, causing initialization to fail:

RuntimeError: Number of camera prims in the view (1) does not match
the number of environments (N).

Workaround: set streaming_sensor_prim_path to a scene camera that was declared in the scene config before physics init (for example, a TiledCamera on a vision-based task).

See also