nemo_gym.web.visual_browser

Backend-neutral headed Chromium/PyAutoGUI mechanics for visual web sessions.

Module Contents

Classes

Name	Description
`VisualBrowserDriver`	One thread-affine Playwright context with visible PyAutoGUI actions.
`VisualBrowserDriverConfig`	-
`VisualBrowserEvaluationContext`	Live process-local state exposed only to a colocated evaluator.

Functions

Name	Description
`_paste_unicode`	-
`_stop_clipboard_owner`	-
`_type_browser_text`	Type text without losing Unicode or interpreting newlines as glyphs.
`_url_origin`	Return a log-safe origin without URL paths, queries, or credentials.
`is_retryable_navigation_transport_error`	Return whether `page.goto` failed with a dropped transport.

Data

CHROME_ARGS

LOG

NAVIGATION_WAIT_UNTIL

RESET_WAIT_UNTIL

RETRYABLE_NAVIGATION_ERRORS

SHIFT_TEXT_KEYS

SPECIAL_TEXT_KEYS

API

class nemo_gym.web.visual_browser.VisualBrowserDriver(
    config: nemo_gym.web.visual_browser.VisualBrowserDriverConfig,
    session_id: str,
    artifacts: nemo_gym.web.artifacts.WebArtifactStore,
    browser_lease: nemo_gym.web.browser_session.BrowserSessionHandle
)

One thread-affine Playwright context with visible PyAutoGUI actions.

_evidence

deque[WebArtifactRef] = deque(maxlen=(config.max_evidence_screenshots))

_last_action

= ''

_last_error

= ''

_observation

WebObservation | None = None

_proxy_origin

= 'none'

_site_access_status

str | None = None

_started_at

= 0.0

_step

= 0

_task

WebTask | None = None

nemo_gym.web.visual_browser.VisualBrowserDriver._access_helpers_enabled() -> bool

nemo_gym.web.visual_browser.VisualBrowserDriver._after_reset() -> None

nemo_gym.web.visual_browser.VisualBrowserDriver._after_start_navigation(
    page: typing.Any,
    url: str
) -> None

nemo_gym.web.visual_browser.VisualBrowserDriver._after_tool_call(
    name: str,
    failure_step: int
) -> None

nemo_gym.web.visual_browser.VisualBrowserDriver._before_post_action_capture(
    failure_step: int
) -> None

nemo_gym.web.visual_browser.VisualBrowserDriver._capture() -> nemo_gym.web.models.WebObservation

nemo_gym.web.visual_browser.VisualBrowserDriver._configure_page(
    page: typing.Any
) -> None

staticmethod

nemo_gym.web.visual_browser.VisualBrowserDriver._configure_proxy_metadata(
    proxy_config: dict[str, str]
) -> None

nemo_gym.web.visual_browser.VisualBrowserDriver._configure_pyautogui() -> None

staticmethod

nemo_gym.web.visual_browser.VisualBrowserDriver._execute_call(
    name: str,
    arguments: dict[str, typing.Any]
) -> None

nemo_gym.web.visual_browser.VisualBrowserDriver._execute_computer(
    spec: dict[str, typing.Any]
) -> None

nemo_gym.web.visual_browser.VisualBrowserDriver._goto(
    page: typing.Any,
    url: str,
    wait_until: str
) -> typing.Any

Navigate using the retry policy supplied by a benchmark subclass.

Only page.goto is retried. History navigation remains a single attempt because retrying go_back or go_forward can move twice.

nemo_gym.web.visual_browser.VisualBrowserDriver._goto_task_start(
    page: typing.Any,
    url: str
) -> typing.Any

nemo_gym.web.visual_browser.VisualBrowserDriver._install_context_scripts(
    task: nemo_gym.web.models.WebTask
) -> None

nemo_gym.web.visual_browser.VisualBrowserDriver._navigation_retry_delays() -> tuple[float, ...]

nemo_gym.web.visual_browser.VisualBrowserDriver._normalize_key(
    key: str
) -> str

staticmethod

nemo_gym.web.visual_browser.VisualBrowserDriver._pixel(
    coordinate: typing.Any
) -> tuple[int, int]

nemo_gym.web.visual_browser.VisualBrowserDriver._playwright_proxy(
    proxy: str
) -> dict[str, str]

staticmethod

nemo_gym.web.visual_browser.VisualBrowserDriver._prepare_task(
    task: nemo_gym.web.models.WebTask
) -> list[str]

nemo_gym.web.visual_browser.VisualBrowserDriver._proxy_for_task(
    task: nemo_gym.web.models.WebTask
) -> str

Return a browser proxy for this task; local Arena sites use none.

nemo_gym.web.visual_browser.VisualBrowserDriver._reset_metadata() -> dict[str, typing.Any]

nemo_gym.web.visual_browser.VisualBrowserDriver._reset_task_state(
    task: nemo_gym.web.models.WebTask
) -> None

nemo_gym.web.visual_browser.VisualBrowserDriver._runtime_should_terminate() -> bool

nemo_gym.web.visual_browser.VisualBrowserDriver._runtime_target_closed() -> bool

nemo_gym.web.visual_browser.VisualBrowserDriver._safe_title(
    page: typing.Any
) -> str

staticmethod

nemo_gym.web.visual_browser.VisualBrowserDriver._select_page(
    tab_id: int | None
) -> None

nemo_gym.web.visual_browser.VisualBrowserDriver._should_retry_navigation(
    exc: Exception
) -> bool

nemo_gym.web.visual_browser.VisualBrowserDriver._step_info(
    action: nemo_gym.web.models.WebAction
) -> dict[str, typing.Any]

nemo_gym.web.visual_browser.VisualBrowserDriver._sync_active_page() -> bool

Follow GUI tab changes without bringing any page to the foreground.

Playwright’s context page order is creation order, not the active tab. A visible document also identifies the selected tab when the address bar owns focus. A closed tab is terminal only when no live page remains or the browser is disconnected.

nemo_gym.web.visual_browser.VisualBrowserDriver._validate_browser_lease() -> None

Fail closed before touching process-affine browser state.

The current visual-control driver requires a local DISPLAY because PyAutoGUI drives the complete headed browser surface. AgentEnv and cloud providers can use the same session lifecycle contract, but must supply a matching visual-control driver instead of silently falling back to page-only CDP mouse events.

nemo_gym.web.visual_browser.VisualBrowserDriver.close() -> None

nemo_gym.web.visual_browser.VisualBrowserDriver.evaluation_context() -> nemo_gym.web.visual_browser.VisualBrowserEvaluationContext

nemo_gym.web.visual_browser.VisualBrowserDriver.observe() -> nemo_gym.web.models.WebObservation

nemo_gym.web.visual_browser.VisualBrowserDriver.reset(
    task: nemo_gym.web.models.WebTask
) -> tuple[nemo_gym.web.models.WebObservation, dict[str, typing.Any]]

nemo_gym.web.visual_browser.VisualBrowserDriver.step(
    action: nemo_gym.web.models.WebAction
) -> nemo_gym.web.models.WebStepResult

class nemo_gym.web.visual_browser.VisualBrowserDriverConfig()

Bases: WebResourcesServerConfig

action_delay_seconds

float = Field(default=2.0, ge=0, le=30)

action_limits

WebActionLimits = Field(default_factory=WebActionLimits)

artifact_dir

str = 'cache/visual-browser/artifacts'

browser_auto_install

bool = True

Configuration shared by headed visual-browser benchmark policies.

browser_channel

str | None = None

default_timeout_ms

int = Field(default=45000, ge=1000, le=600000)

headless

bool = False

max_computer_actions

int = Field(default=20, ge=1, le=100)

max_evidence_screenshots

int = Field(default=200, ge=1, le=500)

max_task_image_bytes

int

max_tool_calls

int | None = Field(default=8, ge=1)

record_video

bool = False

task_image_root

str | None = None

terminate_on_action_error

bool = True

viewport_height

int = Field(default=1080, ge=480)

viewport_width

int = Field(default=1920, ge=640)

nemo_gym.web.visual_browser.VisualBrowserDriverConfig.validate_visual_browser() -> 'VisualBrowserDriverConfig'

class nemo_gym.web.visual_browser.VisualBrowserEvaluationContext(
    page: typing.Any,
    browser_context: typing.Any,
    evidence: tuple[nemo_gym.web.models.WebArtifactRef, ...],
    artifact_dir: pathlib.Path | None = None
)

Dataclass

Live process-local state exposed only to a colocated evaluator.

artifact_dir

Path | None = None

evidence

tuple[WebArtifactRef, ...]

nemo_gym.web.visual_browser._paste_unicode(
    pyautogui: typing.Any,
    text: str
) -> None

nemo_gym.web.visual_browser._stop_clipboard_owner(
    process: subprocess.Popen[bytes] | None
) -> None

nemo_gym.web.visual_browser._type_browser_text(
    pyautogui: typing.Any,
    text: str
) -> None

Type text without losing Unicode or interpreting newlines as glyphs.

nemo_gym.web.visual_browser._url_origin(
    url: str
) -> str

Return a log-safe origin without URL paths, queries, or credentials.

nemo_gym.web.visual_browser.is_retryable_navigation_transport_error(
    exc: Exception
) -> bool

Return whether page.goto failed with a dropped transport.

nemo_gym.web.visual_browser.CHROME_ARGS = ['--window-position=0,0', '--no-first-run', '--no-default-browser-check', '--dis...

nemo_gym.web.visual_browser.LOG = logging.getLogger('nemo_gym.web.visual_browser')

nemo_gym.web.visual_browser.NAVIGATION_WAIT_UNTIL = 'load'

nemo_gym.web.visual_browser.RESET_WAIT_UNTIL = 'domcontentloaded'

nemo_gym.web.visual_browser.RETRYABLE_NAVIGATION_ERRORS = ('net::ERR_EMPTY_RESPONSE', 'net::ERR_PROXY_CONNECTION_FAILED', 'net::ERR_TUNNEL...

nemo_gym.web.visual_browser.SHIFT_TEXT_KEYS = {'<': ','}

nemo_gym.web.visual_browser.SPECIAL_TEXT_KEYS = {'\n': 'enter', '\t': 'tab'}