Skip to navigation
Isaac LabHow-to Guides

Streaming a camera view in a visualizer

View as Markdown

This guide shows how to show ground-truth camera frames from many environments in one live panel of a visualizer. For how the view works and which visualizers support it, see Streaming Camera View.

Quick Start

Run the run_tiled_camera_visualizer.py script in IsaacLab/scripts/tutorials/07_visualizers:

uv run python scripts/tutorials/07_visualizers/run_tiled_camera_visualizer.py \
    --task Isaac-Velocity-Rough-AnymalD --num_envs 256 --viz kit
# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
# All rights reserved.
#
# SPDX-License-Identifier: BSD-3-Clause
"""
This script demonstrates the visualizer tiled camera panel.
~~~bash
   # Kit visualizer tiled camera panel
   uv run python scripts/tutorials/07_visualizers/run_tiled_camera_visualizer.py \
--task Isaac-Velocity-Rough-AnymalD --num_envs 256 --viz kit
   # Newton visualizer tiled camera panel
   uv run python scripts/tutorials/07_visualizers/run_tiled_camera_visualizer.py \
       --task IsaacContrib-Stack-Cube-Galbot-Left-Arm-Gripper-Visuomotor --num_envs 25 --viz newton_gl
~~~
"""
from __future__ import annotations
import argparse
import contextlib
import sys
import gymnasium as gym
import torch
import isaaclab_tasks  # noqa: F401
with contextlib.suppress(ImportError):
    import isaaclab_tasks_experimental  # noqa: F401
from isaaclab.app import add_launcher_args, launch_simulation
from isaaclab_tasks.utils import resolve_task_config, setup_preset_cli
KIT_DEFAULT_TASK = "Isaac-Velocity-Rough-AnymalD"
NEWTON_DEFAULT_TASK = "IsaacContrib-Stack-Cube-Galbot-Left-Arm-Gripper-Visuomotor"
SUPPORTED_TILED_VISUALIZERS = {"kit", "newton", "newton_gl", "newton_rtx"}
UNSUPPORTED_TILED_VISUALIZERS = {"rerun", "viser"}
def _resolve_env_regex_path(prim_path: str) -> str:
    """Resolve scene config env namespace macros to the cloned-env regex."""
    return prim_path.format(ENV_REGEX_NS="/World/envs/env_.*")
def _requested_visualizers(args_cli: argparse.Namespace) -> list[str]:
    """Return requested visualizers, defaulting to Kit for this tutorial."""
    visualizers = args_cli.visualizer or ["kit"]
    visualizers = [str(visualizer).lower() for visualizer in visualizers]
    if "none" in visualizers:
        raise ValueError("This demo requires a tiled-camera visualizer. Use '--viz kit' or '--viz newton_gl'.")
    unsupported = sorted(set(visualizers) & UNSUPPORTED_TILED_VISUALIZERS)
    if unsupported:
        raise ValueError(
            "The visualizer tiled camera panel is only implemented for Kit and Newton. "
            f"Unsupported selection: {unsupported}."
        )
    unknown = sorted(set(visualizers) - SUPPORTED_TILED_VISUALIZERS)
    if unknown:
        raise ValueError(f"Unknown visualizer selection for this demo: {unknown}.")
    return visualizers
def _make_kit_visualizer_cfg(env_cfg):
    """Create the Kit streaming-camera visualizer for the selected task."""
    from isaaclab_visualizers.kit import KitVisualizerCfg
    visualizer_cfg = KitVisualizerCfg()
    visualizer_cfg.streaming_view = True
    visualizer_cfg.streaming_envs = 36
    ego_cam_cfg = getattr(env_cfg.scene, "ego_cam", None)
    if ego_cam_cfg is not None:
        visualizer_cfg.streaming_sensor_prim_path = _resolve_env_regex_path(ego_cam_cfg.prim_path)
        return visualizer_cfg
    visualizer_cfg.streaming_sensor_prim_path = None
    visualizer_cfg.streaming_cam_eye = (3.0, 3.0, 3.0)
    visualizer_cfg.streaming_cam_target_prim_path = "/World/envs/*/Robot/base"
    # Here is an alternative eye position for a top down view
    # visualizer_cfg.streaming_cam_eye = (0.0, 0.0, 5.0)
    return visualizer_cfg
def _make_newton_visualizer_cfg(env_cfg):
    """Create the Newton streaming-camera visualizer for the selected task."""
    from isaaclab_visualizers.newton import NewtonGLVisualizerCfg
    visualizer_cfg = NewtonGLVisualizerCfg()
    visualizer_cfg.streaming_view = True
    visualizer_cfg.streaming_envs = 12
    ego_cam_cfg = getattr(env_cfg.scene, "ego_cam", None)
    if ego_cam_cfg is not None:
        visualizer_cfg.streaming_sensor_prim_path = _resolve_env_regex_path(ego_cam_cfg.prim_path)
        return visualizer_cfg
    # Here are other robot mounted camera options for this environment
    # visualizer_cfg.streaming_sensor_prim_path = "/World/envs/env_.*/Robot/left_arm_camera_sim_view_frame/left_camera"
    # visualizer_cfg.streaming_sensor_prim_path = (
    #     "/World/envs/env_.*/Robot/right_arm_camera_sim_view_frame/right_camera"
    # )
    visualizer_cfg.streaming_sensor_prim_path = None
    visualizer_cfg.streaming_cam_eye = (3.0, 3.0, 3.0)
    visualizer_cfg.streaming_cam_target_prim_path = "/World/envs/*/Robot/base"
    return visualizer_cfg
def _configure_visualizers(env_cfg, args_cli: argparse.Namespace) -> None:
    """Attach tiled camera visualizer configs to the environment simulation config."""
    visualizers = _requested_visualizers(args_cli)
    args_cli.visualizer = visualizers
    env_cfg.sim.visualizer_cfgs = [
        _make_kit_visualizer_cfg(env_cfg) if visualizer == "kit" else _make_newton_visualizer_cfg(env_cfg)
        for visualizer in visualizers
    ]
def _resolve_task(args_cli: argparse.Namespace) -> str:
    """Resolve the task for the selected visualizer."""
    if args_cli.task is not None:
        return args_cli.task
    if "newton" in _requested_visualizers(args_cli):
        return NEWTON_DEFAULT_TASK
    return KIT_DEFAULT_TASK
# add argparse arguments
parser = argparse.ArgumentParser(description="Showcase the Kit/Newton visualizer tiled camera panel.")
parser.add_argument("--num_envs", type=int, default=None, help="Number of environments to simulate.")
parser.add_argument("--task", type=str, default=None, help="Name of the task.")
# append simulation launcher cli args
add_launcher_args(parser)
args_cli, hydra_args = setup_preset_cli(parser)
args_cli.task = _resolve_task(args_cli)
sys.argv = [sys.argv[0]] + hydra_args
def main():
    """Run a random-action environment with a tiled camera visualizer."""
    # parse configuration via Hydra (supports preset selection, e.g. presets=newton_mjwarp)
    env_cfg, _ = resolve_task_config(args_cli.task, "")
    _configure_visualizers(env_cfg, args_cli)
    with launch_simulation(env_cfg, args_cli):
        # override with CLI arguments
        env_cfg.scene.num_envs = args_cli.num_envs if args_cli.num_envs is not None else env_cfg.scene.num_envs
        env_cfg.sim.device = args_cli.device if args_cli.device is not None else env_cfg.sim.device
        # create environment
        env = gym.make(args_cli.task, cfg=env_cfg)
        # print info (this is vectorized environment)
        print(f"[INFO]: Gym observation space: {env.observation_space}")
        print(f"[INFO]: Gym action space: {env.action_space}")
        env.reset()
        # keep stepping until all visualizer windows have been closed
        sim = env.unwrapped.sim
        if not sim.visualizers:
            print("[WARN]: No visualizers found. Exiting.")
            env.close()
            return
        while True:
            if sim.visualizers and not any(v.is_running() and not v.is_closed for v in sim.visualizers):
                break
            with torch.inference_mode():
                actions = 2 * torch.rand(env.action_space.shape, device=env.unwrapped.device) - 1
                env.step(actions)
        env.close()
if __name__ == "__main__":
    main()

See Examples_ below for the two ways to run the script. For the VisualizerCfg fields that customize streaming, see Configuration_ and Troubleshooting_.

Examples

Running run_tiled_camera_visualizer.py demonstrates two ways to use the streaming camera view:

  • auto-created cameras pointed at and following moving AnymalD robots, shown in the Kit visualizer
  • streaming from existing wrist-mounted robot cameras, shown in the Newton visualizer

Example 1: Following AnymalD Robots

uv run python scripts/tutorials/07_visualizers/run_tiled_camera_visualizer.py \
    --task Isaac-Velocity-Rough-AnymalD --num_envs 256 --viz kit

The script’s KitVisualizerCfg creates cameras that point at and follow each robot’s base prim, offset by streaming_cam_eye (here (3.0, 3.0, 3.0); try (0, 0, 5) for a top-down view). Of the 256 environments, 36 are randomly sampled for the camera view.

Watch the embedded video Watch the embedded video

Example 2: Streaming from Robot-Mounted Cameras

uv run --extra teleop python scripts/tutorials/07_visualizers/run_tiled_camera_visualizer.py \
    --task IsaacContrib-Stack-Cube-Galbot-Left-Arm-Gripper-Visuomotor --num_envs 25 --viz newton_gl

The Galbot cube-stacking environment ships with wrist-mounted cameras giving an egocentric view of the gripper, table, and cubes. The script’s NewtonGLVisualizerCfg streams from the existing sensor at /World/envs/env_.*/Robot/head_camera_sim_view_frame/head_camera; edit streaming_sensor_prim_path to show a different camera. Of the 25 environments, 12 camera feeds are shown by default.

Watch the embedded video Watch the embedded video

Configuration

To customize streaming camera behavior, edit the highlighted VisualizerCfg fields in run_tiled_camera_visualizer.py:

  • For auto-created cameras, streaming_cam_target_prim_path chooses the followed prim and streaming_cam_eye sets the camera offset from that prim. Defaults to None, which causes the visualizer to adopt the first scene camera it discovers at init; no explicit path is needed when a TiledCamera sensor is already in the scene.
  • For existing scene cameras, streaming_sensor_prim_path must match an Isaac Lab ~isaaclab.sensors.Camera sensor prim path in the selected task.
  • streaming_envs controls how many environment tiles are shown. Pass an int to randomly sample that many environments, or a list[int] to pin specific environment indices.
  • streaming_gt_types selects which ground-truth types are shown, e.g. ["rgb", "depth", "segmentation", "normals"].
  • streaming_depth_min / streaming_depth_max set the depth colormap range in metres.

Troubleshooting

  • If a generated view fails with a missing prim error, verify that streaming_cam_target_prim_path resolves in each selected environment; common template forms are /World/envs/*/... and /World/envs/env_.*/.... In most cases you can leave it as None and let the visualizer adopt an existing scene camera automatically.
  • If an existing-camera view reports that no Isaac Lab camera owns the prim, check that streaming_sensor_prim_path matches a ~isaaclab.sensors.Camera sensor in the task.
  • If the depth panel shows a flat color, adjust streaming_depth_min and streaming_depth_max to bracket the expected depth range in your scene.
  • If the view is too expensive, reduce streaming_envs, --num_envs, or the camera resolution.

Newton MJWarp with replicate_physics=True and auto-created cameras

With replicate_physics=True, only env_0 has a USD prim after physics initialization. Cameras for the remaining environments (env_1 through env_{N-1}) are dropped, causing initialization to fail:

RuntimeError: Number of camera prims in the view (1) does not match
the number of environments (N).

Workaround: set streaming_sensor_prim_path to a scene camera that was declared in the scene config before physics init (for example, a TiledCamera on a vision-based task).

See also