RecipesImage Generation

Humanoid Robot Scene Understanding

View as Markdown
Download Recipe

Download the complete recipe script

1# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2# SPDX-License-Identifier: Apache-2.0
3# /// script
4# requires-python = ">=3.10"
5# dependencies = [
6#     "data-designer",
7# ]
8# ///
9"""Humanoid Robot Scene Understanding Image Generation Recipe
10
11Generate synthetic egocentric humanoid robot images with controlled variation
12over indoor environment, robot viewpoint, task goal, object set, scene state,
13safety condition, lighting, and adult human presence.
14
15Use the generated images for embodied-AI scene understanding, visual QA,
16reviewer calibration, safety review, and robotics demos where the image should
17look like a frame captured from the robot's own camera in a controlled setting.
18
19Prerequisites:
20    - An image-generation provider key for the selected model. The defaults use
21      OpenRouter, so set OPENROUTER_API_KEY before running.
22
23Run:
24    uv run humanoid_robot_scene_understanding.py --num-records 10
25"""
26
27from __future__ import annotations
28
29import argparse
30from pathlib import Path
31
32import data_designer.config as dd
33from data_designer.interface import DataDesigner, DatasetCreationResults
34
35DEFAULT_MODEL_PROVIDER = "openrouter"
36DEFAULT_MODEL_ID = "google/gemini-3.1-flash-image-preview"
37DEFAULT_MODEL_ALIAS = "humanoid-scene-model"
38
39
40def build_model_configs(
41    *,
42    model_provider: str,
43    model_id: str,
44    model_alias: str,
45    image_size: str,
46    aspect_ratio: str,
47    max_parallel_requests: int,
48) -> list[dd.ModelConfig]:
49    return [
50        dd.ModelConfig(
51            alias=model_alias,
52            model=model_id,
53            provider=model_provider,
54            inference_parameters=dd.ImageInferenceParams(
55                extra_body={
56                    "modalities": ["image", "text"],
57                    "image_config": {
58                        "aspect_ratio": aspect_ratio,
59                        "image_size": image_size,
60                    },
61                },
62                max_parallel_requests=max_parallel_requests,
63            ),
64            skip_health_check=True,
65        )
66    ]
67
68
69def add_category(config_builder: dd.DataDesignerConfigBuilder, name: str, values: list[str]) -> None:
70    config_builder.add_column(
71        dd.SamplerColumnConfig(
72            name=name,
73            sampler_type=dd.SamplerType.CATEGORY,
74            params=dd.CategorySamplerParams(values=values),
75        )
76    )
77
78
79def add_visual_variation_id(config_builder: dd.DataDesignerConfigBuilder) -> None:
80    """Add a unique row-level key that discourages duplicate image generations."""
81    config_builder.add_column(
82        dd.SamplerColumnConfig(
83            name="visual_variation_id",
84            sampler_type=dd.SamplerType.UUID,
85            params=dd.UUIDSamplerParams(prefix="humanoid-", short_form=True),
86        )
87    )
88
89
90def build_config(
91    *,
92    model_provider: str = DEFAULT_MODEL_PROVIDER,
93    model_id: str = DEFAULT_MODEL_ID,
94    model_alias: str = DEFAULT_MODEL_ALIAS,
95    image_size: str = "1K",
96    aspect_ratio: str = "16:9",
97    max_parallel_requests: int = 10,
98) -> dd.DataDesignerConfigBuilder:
99    model_configs = build_model_configs(
100        model_provider=model_provider,
101        model_id=model_id,
102        model_alias=model_alias,
103        image_size=image_size,
104        aspect_ratio=aspect_ratio,
105        max_parallel_requests=max_parallel_requests,
106    )
107    config_builder = dd.DataDesignerConfigBuilder(model_configs=model_configs)
108    add_visual_variation_id(config_builder)
109
110    add_category(
111        config_builder,
112        "environment",
113        [
114            "teaching kitchen with counters, cabinets, and everyday objects",
115            "mock apartment living room arranged for assistive robotics",
116            "assisted living bedroom with bedside table and mobility aids",
117            "robotics lab workbench with tools and calibration objects",
118            "retail stockroom with shelves, totes, and handheld items",
119            "hospital supply room with carts, bins, and sealed supplies",
120            "office break room with appliances, tableware, and waste bins",
121            "laundry room with baskets, detergent, shelves, and folded towels",
122            "tool bench training area with bins, fasteners, and hand tools",
123            "grocery practice aisle with shelves, baskets, and fallen items",
124        ],
125    )
126    add_category(
127        config_builder,
128        "robot_viewpoint",
129        [
130            "head-mounted camera at standing adult height",
131            "chest-mounted camera with both robot hands barely visible at the bottom edge",
132            "slightly downward gaze toward a tabletop work surface",
133            "close manipulation view with one robot hand near the target object",
134            "wide room scan from a doorway before entering the scene",
135            "low crouched inspection angle looking under a table or cart",
136        ],
137    )
138    add_category(
139        config_builder,
140        "task_goal",
141        [
142            "locate the requested object before reaching",
143            "judge whether the path is safe to walk through",
144            "identify which objects are reachable from the current pose",
145            "verify that a cleanup task is complete",
146            "prepare a clear handoff area for an adult user",
147            "find the missing tool or supply item",
148            "inspect a spill or obstacle before moving closer",
149            "decide whether fragile items are too close to an edge",
150        ],
151    )
152    add_category(
153        config_builder,
154        "object_set",
155        [
156            "mug, kettle, sponge, dish towel, and cereal bowl",
157            "water glass, medication organizer, tissue box, and walking cane",
158            "pipette rack, beaker, nitrile gloves, and small screwdriver",
159            "barcode scanner, tote, tape dispenser, folded shirt, and box cutter",
160            "laundry basket, detergent bottle, folded towels, and loose sock",
161            "pliers, hex keys, small bolts, tape measure, and plastic bins",
162            "shopping basket, cereal boxes, soup cans, and fallen fruit",
163            "meal tray, sealed supplies, clipboard, and rolling cart",
164        ],
165    )
166    add_category(
167        config_builder,
168        "scene_state",
169        [
170            "organized and ready for the task",
171            "moderately cluttered but navigable",
172            "target object partly occluded by other items",
173            "target object moved to an unexpected location",
174            "container open with mixed contents visible",
175            "fragile item near the table edge",
176            "object stack unstable but still standing",
177            "task area partly blocked by a chair or cart",
178        ],
179    )
180    add_category(
181        config_builder,
182        "safety_condition",
183        [
184            "no visible hazard",
185            "small liquid spill on the floor",
186            "power cable crossing the walking path",
187            "sharp tool exposed on the work surface",
188            "hot appliance indicator light visible",
189            "glass object on the floor near the path",
190            "drawer left open at knee height",
191            "rolling cart partially blocking the doorway",
192        ],
193    )
194    add_category(
195        config_builder,
196        "human_presence",
197        [
198            "no person visible",
199            "adult worker's gloved hands visible at a safe distance",
200            "adult caregiver standing in the background with face turned away",
201            "adult shopper passing through the background, not identifiable",
202            "adult lab worker partially visible from shoulders down",
203            "adult office worker's arm visible near the handoff area",
204        ],
205    )
206    add_category(
207        config_builder,
208        "lighting",
209        [
210            "bright even lab lighting",
211            "warm apartment lighting",
212            "overcast window light",
213            "mixed overhead and task lighting",
214            "dim hallway light with localized task lamp",
215            "high-contrast backlighting from a nearby window",
216        ],
217    )
218
219    config_builder.add_column(
220        dd.ImageColumnConfig(
221            name="humanoid_scene_image",
222            prompt=HUMANOID_SCENE_PROMPT,
223            model_alias=model_alias,
224        )
225    )
226
227    return config_builder
228
229
230def create_dataset(
231    config_builder: dd.DataDesignerConfigBuilder,
232    *,
233    num_records: int,
234    dataset_name: str,
235    artifact_path: Path | str | None = None,
236) -> DatasetCreationResults:
237    data_designer = DataDesigner(artifact_path=artifact_path)
238    data_designer.validate(config_builder)
239    return data_designer.create(config_builder, num_records=num_records, dataset_name=dataset_name)
240
241
242HUMANOID_SCENE_PROMPT = """\
243Create a realistic egocentric humanoid robot scene-understanding image.
244
245The frame must look like it was captured from the humanoid robot's own camera
246inside a controlled indoor environment. Show the robot's viewpoint clearly:
247camera height, reachable workspace, path geometry, task-relevant objects,
248obstacles, and safety condition should all be visible enough for visual QA or
249embodied-AI scene understanding. If the viewpoint mentions robot hands, show at
250most one or two simple robot hands at the image edge; do not make the robot the
251main subject.
252
253Scene requirements:
254- Visual variation ID, for internal diversity only: {{ visual_variation_id }}
255- Environment: {{ environment }}
256- Robot viewpoint: {{ robot_viewpoint }}
257- Task goal: {{ task_goal }}
258- Object set: {{ object_set }}
259- Scene state: {{ scene_state }}
260- Safety condition: {{ safety_condition }}
261- Human presence: {{ human_presence }}
262- Lighting: {{ lighting }}
263
264Make the requested task goal, object set, scene state, and safety condition
265visually legible without adding labels or annotation graphics. Use realistic
266materials, clutter, occlusion, reachability cues, shadows, and indoor scale.
267
268Do not include children, identifiable faces, readable personal names, real
269company logos, surveillance UI, bounding boxes, arrows, captions, labels,
270watermarks, subtitles, HUD overlays, or diagnostic text. Generate exactly one
271final camera frame for this row. Do not return alternate versions, a grid, a
272pair of examples, before/after panels, or multiple frames. Use the visual
273variation ID only as an internal diversity key; never render it as text.
274"""
275
276
277def parse_args() -> argparse.Namespace:
278    parser = argparse.ArgumentParser(description="Generate synthetic humanoid robot scene-understanding images.")
279    parser.add_argument("--num-records", type=int, default=10, help="Number of humanoid scene images to generate.")
280    parser.add_argument("--dataset-name", default="humanoid-robot-scene-understanding", help="Output dataset name.")
281    parser.add_argument("--artifact-path", type=Path, default=None, help="Optional Data Designer artifact directory.")
282    parser.add_argument("--model-provider", default=DEFAULT_MODEL_PROVIDER, help="Image model provider name.")
283    parser.add_argument("--model-id", default=DEFAULT_MODEL_ID, help="Provider model ID.")
284    parser.add_argument("--model-alias", default=DEFAULT_MODEL_ALIAS, help="Alias used by image columns.")
285    parser.add_argument("--image-size", default="1K", help="OpenRouter image size tier, such as 1K, 2K, or 4K.")
286    parser.add_argument("--aspect-ratio", default="16:9", help="Provider-specific aspect ratio value.")
287    parser.add_argument("--max-parallel-requests", type=int, default=10, help="Maximum parallel image requests.")
288    return parser.parse_args()
289
290
291def main() -> None:
292    args = parse_args()
293    config_builder = build_config(
294        model_provider=args.model_provider,
295        model_id=args.model_id,
296        model_alias=args.model_alias,
297        image_size=args.image_size,
298        aspect_ratio=args.aspect_ratio,
299        max_parallel_requests=args.max_parallel_requests,
300    )
301    results = create_dataset(
302        config_builder,
303        num_records=args.num_records,
304        dataset_name=args.dataset_name,
305        artifact_path=args.artifact_path,
306    )
307    dataset = results.load_dataset()
308    print(f"Generated {len(dataset)} humanoid robot scene-understanding rows.")
309    print(f"Dataset artifacts: {results.artifact_storage.base_dataset_path}")
310
311
312if __name__ == "__main__":
313    main()