Humanoid Robot Scene Understanding

1
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
# SPDX-License-Identifier: Apache-2.0
3
# /// script
4
# requires-python = ">=3.10"
5
# dependencies = [
6
#     "data-designer",
7
# ]
8
# ///
9
"""Humanoid Robot Scene Understanding Image Generation Recipe
10
11
Generate synthetic egocentric humanoid robot images with controlled variation
12
over indoor environment, robot viewpoint, task goal, object set, scene state,
13
safety condition, lighting, and adult human presence.
14
15
Use the generated images for embodied-AI scene understanding, visual QA,
16
reviewer calibration, safety review, and robotics demos where the image should
17
look like a frame captured from the robot's own camera in a controlled setting.
18
19
Prerequisites:
20
    - An image-generation provider key for the selected model. The defaults use
21
      OpenRouter, so set OPENROUTER_API_KEY before running.
22
23
Run:
24
    uv run humanoid_robot_scene_understanding.py --num-records 10
25
"""
26
27
from __future__ import annotations
28
29
import argparse
30
from pathlib import Path
31
32
import data_designer.config as dd
33
from data_designer.interface import DataDesigner, DatasetCreationResults
34
35
DEFAULT_MODEL_PROVIDER = "openrouter"
36
DEFAULT_MODEL_ID = "google/gemini-3.1-flash-image-preview"
37
DEFAULT_MODEL_ALIAS = "humanoid-scene-model"
38
39
40
def build_model_configs(
41
    *,
42
    model_provider: str,
43
    model_id: str,
44
    model_alias: str,
45
    image_size: str,
46
    aspect_ratio: str,
47
    max_parallel_requests: int,
48
) -> list[dd.ModelConfig]:
49
    return [
50
        dd.ModelConfig(
51
            alias=model_alias,
52
            model=model_id,
53
            provider=model_provider,
54
            inference_parameters=dd.ImageInferenceParams(
55
                extra_body={
56
                    "modalities": ["image", "text"],
57
                    "image_config": {
58
                        "aspect_ratio": aspect_ratio,
59
                        "image_size": image_size,
60
                    },
61
                },
62
                max_parallel_requests=max_parallel_requests,
63
            ),
64
            skip_health_check=True,
65
        )
66
    ]
67
68
69
def add_category(config_builder: dd.DataDesignerConfigBuilder, name: str, values: list[str]) -> None:
70
    config_builder.add_column(
71
        dd.SamplerColumnConfig(
72
            name=name,
73
            sampler_type=dd.SamplerType.CATEGORY,
74
            params=dd.CategorySamplerParams(values=values),
75
        )
76
    )
77
78
79
def add_visual_variation_id(config_builder: dd.DataDesignerConfigBuilder) -> None:
80
    """Add a unique row-level key that discourages duplicate image generations."""
81
    config_builder.add_column(
82
        dd.SamplerColumnConfig(
83
            name="visual_variation_id",
84
            sampler_type=dd.SamplerType.UUID,
85
            params=dd.UUIDSamplerParams(prefix="humanoid-", short_form=True),
86
        )
87
    )
88
89
90
def build_config(
91
    *,
92
    model_provider: str = DEFAULT_MODEL_PROVIDER,
93
    model_id: str = DEFAULT_MODEL_ID,
94
    model_alias: str = DEFAULT_MODEL_ALIAS,
95
    image_size: str = "1K",
96
    aspect_ratio: str = "16:9",
97
    max_parallel_requests: int = 10,
98
) -> dd.DataDesignerConfigBuilder:
99
    model_configs = build_model_configs(
100
        model_provider=model_provider,
101
        model_id=model_id,
102
        model_alias=model_alias,
103
        image_size=image_size,
104
        aspect_ratio=aspect_ratio,
105
        max_parallel_requests=max_parallel_requests,
106
    )
107
    config_builder = dd.DataDesignerConfigBuilder(model_configs=model_configs)
108
    add_visual_variation_id(config_builder)
109
110
    add_category(
111
        config_builder,
112
        "environment",
113
        [
114
            "teaching kitchen with counters, cabinets, and everyday objects",
115
            "mock apartment living room arranged for assistive robotics",
116
            "assisted living bedroom with bedside table and mobility aids",
117
            "robotics lab workbench with tools and calibration objects",
118
            "retail stockroom with shelves, totes, and handheld items",
119
            "hospital supply room with carts, bins, and sealed supplies",
120
            "office break room with appliances, tableware, and waste bins",
121
            "laundry room with baskets, detergent, shelves, and folded towels",
122
            "tool bench training area with bins, fasteners, and hand tools",
123
            "grocery practice aisle with shelves, baskets, and fallen items",
124
        ],
125
    )
126
    add_category(
127
        config_builder,
128
        "robot_viewpoint",
129
        [
130
            "head-mounted camera at standing adult height",
131
            "chest-mounted camera with both robot hands barely visible at the bottom edge",
132
            "slightly downward gaze toward a tabletop work surface",
133
            "close manipulation view with one robot hand near the target object",
134
            "wide room scan from a doorway before entering the scene",
135
            "low crouched inspection angle looking under a table or cart",
136
        ],
137
    )
138
    add_category(
139
        config_builder,
140
        "task_goal",
141
        [
142
            "locate the requested object before reaching",
143
            "judge whether the path is safe to walk through",
144
            "identify which objects are reachable from the current pose",
145
            "verify that a cleanup task is complete",
146
            "prepare a clear handoff area for an adult user",
147
            "find the missing tool or supply item",
148
            "inspect a spill or obstacle before moving closer",
149
            "decide whether fragile items are too close to an edge",
150
        ],
151
    )
152
    add_category(
153
        config_builder,
154
        "object_set",
155
        [
156
            "mug, kettle, sponge, dish towel, and cereal bowl",
157
            "water glass, medication organizer, tissue box, and walking cane",
158
            "pipette rack, beaker, nitrile gloves, and small screwdriver",
159
            "barcode scanner, tote, tape dispenser, folded shirt, and box cutter",
160
            "laundry basket, detergent bottle, folded towels, and loose sock",
161
            "pliers, hex keys, small bolts, tape measure, and plastic bins",
162
            "shopping basket, cereal boxes, soup cans, and fallen fruit",
163
            "meal tray, sealed supplies, clipboard, and rolling cart",
164
        ],
165
    )
166
    add_category(
167
        config_builder,
168
        "scene_state",
169
        [
170
            "organized and ready for the task",
171
            "moderately cluttered but navigable",
172
            "target object partly occluded by other items",
173
            "target object moved to an unexpected location",
174
            "container open with mixed contents visible",
175
            "fragile item near the table edge",
176
            "object stack unstable but still standing",
177
            "task area partly blocked by a chair or cart",
178
        ],
179
    )
180
    add_category(
181
        config_builder,
182
        "safety_condition",
183
        [
184
            "no visible hazard",
185
            "small liquid spill on the floor",
186
            "power cable crossing the walking path",
187
            "sharp tool exposed on the work surface",
188
            "hot appliance indicator light visible",
189
            "glass object on the floor near the path",
190
            "drawer left open at knee height",
191
            "rolling cart partially blocking the doorway",
192
        ],
193
    )
194
    add_category(
195
        config_builder,
196
        "human_presence",
197
        [
198
            "no person visible",
199
            "adult worker's gloved hands visible at a safe distance",
200
            "adult caregiver standing in the background with face turned away",
201
            "adult shopper passing through the background, not identifiable",
202
            "adult lab worker partially visible from shoulders down",
203
            "adult office worker's arm visible near the handoff area",
204
        ],
205
    )
206
    add_category(
207
        config_builder,
208
        "lighting",
209
        [
210
            "bright even lab lighting",
211
            "warm apartment lighting",
212
            "overcast window light",
213
            "mixed overhead and task lighting",
214
            "dim hallway light with localized task lamp",
215
            "high-contrast backlighting from a nearby window",
216
        ],
217
    )
218
219
    config_builder.add_column(
220
        dd.ImageColumnConfig(
221
            name="humanoid_scene_image",
222
            prompt=HUMANOID_SCENE_PROMPT,
223
            model_alias=model_alias,
224
        )
225
    )
226
227
    return config_builder
228
229
230
def create_dataset(
231
    config_builder: dd.DataDesignerConfigBuilder,
232
    *,
233
    num_records: int,
234
    dataset_name: str,
235
    artifact_path: Path | str | None = None,
236
) -> DatasetCreationResults:
237
    data_designer = DataDesigner(artifact_path=artifact_path)
238
    data_designer.validate(config_builder)
239
    return data_designer.create(config_builder, num_records=num_records, dataset_name=dataset_name)
240
241
242
HUMANOID_SCENE_PROMPT = """\
243
Create a realistic egocentric humanoid robot scene-understanding image.
244
245
The frame must look like it was captured from the humanoid robot's own camera
246
inside a controlled indoor environment. Show the robot's viewpoint clearly:
247
camera height, reachable workspace, path geometry, task-relevant objects,
248
obstacles, and safety condition should all be visible enough for visual QA or
249
embodied-AI scene understanding. If the viewpoint mentions robot hands, show at
250
most one or two simple robot hands at the image edge; do not make the robot the
251
main subject.
252
253
Scene requirements:
254
- Visual variation ID, for internal diversity only: {{ visual_variation_id }}
255
- Environment: {{ environment }}
256
- Robot viewpoint: {{ robot_viewpoint }}
257
- Task goal: {{ task_goal }}
258
- Object set: {{ object_set }}
259
- Scene state: {{ scene_state }}
260
- Safety condition: {{ safety_condition }}
261
- Human presence: {{ human_presence }}
262
- Lighting: {{ lighting }}
263
264
Make the requested task goal, object set, scene state, and safety condition
265
visually legible without adding labels or annotation graphics. Use realistic
266
materials, clutter, occlusion, reachability cues, shadows, and indoor scale.
267
268
Do not include children, identifiable faces, readable personal names, real
269
company logos, surveillance UI, bounding boxes, arrows, captions, labels,
270
watermarks, subtitles, HUD overlays, or diagnostic text. Generate exactly one
271
final camera frame for this row. Do not return alternate versions, a grid, a
272
pair of examples, before/after panels, or multiple frames. Use the visual
273
variation ID only as an internal diversity key; never render it as text.
274
"""
275
276
277
def parse_args() -> argparse.Namespace:
278
    parser = argparse.ArgumentParser(description="Generate synthetic humanoid robot scene-understanding images.")
279
    parser.add_argument("--num-records", type=int, default=10, help="Number of humanoid scene images to generate.")
280
    parser.add_argument("--dataset-name", default="humanoid-robot-scene-understanding", help="Output dataset name.")
281
    parser.add_argument("--artifact-path", type=Path, default=None, help="Optional Data Designer artifact directory.")
282
    parser.add_argument("--model-provider", default=DEFAULT_MODEL_PROVIDER, help="Image model provider name.")
283
    parser.add_argument("--model-id", default=DEFAULT_MODEL_ID, help="Provider model ID.")
284
    parser.add_argument("--model-alias", default=DEFAULT_MODEL_ALIAS, help="Alias used by image columns.")
285
    parser.add_argument("--image-size", default="1K", help="OpenRouter image size tier, such as 1K, 2K, or 4K.")
286
    parser.add_argument("--aspect-ratio", default="16:9", help="Provider-specific aspect ratio value.")
287
    parser.add_argument("--max-parallel-requests", type=int, default=10, help="Maximum parallel image requests.")
288
    return parser.parse_args()
289
290
291
def main() -> None:
292
    args = parse_args()
293
    config_builder = build_config(
294
        model_provider=args.model_provider,
295
        model_id=args.model_id,
296
        model_alias=args.model_alias,
297
        image_size=args.image_size,
298
        aspect_ratio=args.aspect_ratio,
299
        max_parallel_requests=args.max_parallel_requests,
300
    )
301
    results = create_dataset(
302
        config_builder,
303
        num_records=args.num_records,
304
        dataset_name=args.dataset_name,
305
        artifact_path=args.artifact_path,
306
    )
307
    dataset = results.load_dataset()
308
    print(f"Generated {len(dataset)} humanoid robot scene-understanding rows.")
309
    print(f"Dataset artifacts: {results.artifact_storage.base_dataset_path}")
310
311
312
if __name__ == "__main__":
313
    main()