nemo_curator.core.serve

Subpackages

Submodules

Package Contents

Classes

Name	Description
`BaseModelConfig`	Base public model config shared by inference backends.
`BaseServerConfig`	Base server-level config; subclasses declare which model config types they accept.
`DynamoRoleConfig`	Per-role config for disaggregated Dynamo serving.
`DynamoRouterConfig`	Frontend router config for Dynamo.
`DynamoServerConfig`	Server-level Dynamo config.
`DynamoVLLMModelConfig`	Dynamo vLLM model config.
`InferenceServer`	Serve one or more models behind a typed backend config.
`RayServeModelConfig`	Ray Serve model config.
`RayServeServerConfig`	Server-level Ray Serve config.

Functions

Name	Description
`is_inference_server_active`	Check whether any inference server is currently running in this process.

API

class nemo_curator.core.serve.BaseModelConfig(
    model_identifier: str,
    model_name: str | None = None,
    runtime_env: dict[str, typing.Any] = dict()
)

Dataclass

Base public model config shared by inference backends.

model_identifier

str

model_name

str | None = None

resolved_model_name

str

runtime_env

dict[str, Any] = field(default_factory=dict)

nemo_curator.core.serve.BaseModelConfig._merge_package_runtime_env(
    key: str,
    base: dict[str, typing.Any] | list[str] | None,
    override: dict[str, typing.Any] | list[str] | None
) -> dict[str, typing.Any] | list[str]

staticmethod

nemo_curator.core.serve.BaseModelConfig.merge_runtime_envs(
    base: dict[str, typing.Any],
    override: dict[str, typing.Any] | None
) -> dict[str, typing.Any]

staticmethod

Merge two runtime_env dicts while preserving package lists.

class nemo_curator.core.serve.BaseServerConfig()

Dataclass

Base server-level config; subclasses declare which model config types they accept.

model_configs

tuple[type[BaseModelConfig], ...] = ()

class nemo_curator.core.serve.DynamoRoleConfig(
    num_replicas: int = 1,
    engine_kwargs: dict[str, typing.Any] = dict()
)

Dataclass

Per-role config for disaggregated Dynamo serving.

engine_kwargs

dict[str, Any] = field(default_factory=dict)

num_replicas

int = 1

nemo_curator.core.serve.DynamoRoleConfig.__post_init__() -> None

class nemo_curator.core.serve.DynamoRouterConfig(
    mode: typing.Literal['round_robin', 'random', 'kv', 'direct'] | None = None,
    kv_events: bool = False,
    router_kwargs: dict[str, typing.Any] = dict()
)

Dataclass

Frontend router config for Dynamo.

mode=None means “auto”: Curator picks "kv" if any model uses mode="disagg", else leaves --router-mode unset so the Dynamo frontend falls back to its own round_robin default. kv_events only applies when mode == "kv": pass kv_events=True to opt into exact ZMQ KV-cache event publishing; the default uses the router’s approximate tree-based tracking. Anything else is forwarded to the Dynamo frontend as CLI args via router_kwargs.

_RESERVED_ROUTER_KWARGS

frozenset[str] = frozenset({'router_mode', 'router_kv_events'})

kv_events

bool = False

mode

Literal['round_robin', 'random', 'kv', 'direct'] | None = None

router_kwargs

dict[str, Any] = field(default_factory=dict)

nemo_curator.core.serve.DynamoRouterConfig.__post_init__() -> None

class nemo_curator.core.serve.DynamoServerConfig(
    etcd_endpoint: str | None = None,
    nats_url: str | None = None,
    namespace: str = DEFAULT_DYNAMO_NAMESPACE,
    request_plane: str = DEFAULT_DYNAMO_REQUEST_PLANE,
    event_plane: str = DEFAULT_DYNAMO_EVENT_PLANE,
    router: nemo_curator.core.serve.dynamo.config.DynamoRouterConfig = DynamoRouterConfig(),
    subprocess_env: dict[str, str] = dict()
)

Dataclass

Bases: BaseServerConfig

Server-level Dynamo config.

etcd_endpoint

str | None = None

event_plane

str = DEFAULT_DYNAMO_EVENT_PLANE

model_configs

tuple[type[BaseModelConfig], ...] = (DynamoVLLMModelConfig,)

namespace

str = DEFAULT_DYNAMO_NAMESPACE

nats_url

str | None = None

request_plane

str = DEFAULT_DYNAMO_REQUEST_PLANE

router

DynamoRouterConfig = field(default_factory=DynamoRouterConfig)

subprocess_env

dict[str, str] = field(default_factory=dict)

class nemo_curator.core.serve.DynamoVLLMModelConfig(
    model_identifier: str,
    model_name: str | None = None,
    runtime_env: dict[str, typing.Any] = dict(),
    engine_kwargs: dict[str, typing.Any] = dict(),
    num_replicas: int = 1,
    mode: typing.Literal['aggregated', 'disagg'] = 'aggregated',
    prefill: nemo_curator.core.serve.dynamo.config.DynamoRoleConfig | None = None,
    decode: nemo_curator.core.serve.dynamo.config.DynamoRoleConfig | None = None,
    dynamo_kwargs: dict[str, typing.Any] = dict()
)

Dataclass

Bases: BaseModelConfig

Dynamo vLLM model config.

Typed fields cover deployment/placement knobs Curator branches on; anything else is forwarded to python -m dynamo.vllm via dynamo_kwargs. kv_events_config and kv_transfer_config are Curator-managed (init=False): events are derived from router state + port allocation, transfer defaults to NixlConnector for disagg.

decode

DynamoRoleConfig | None = None

dynamo_kwargs

dict[str, Any] = field(default_factory=dict)

engine_kwargs

dict[str, Any] = field(default_factory=dict)

kv_events_config

dict[str, Any]

kv_transfer_config

dict[str, Any]

mode

Literal['aggregated', 'disagg'] = 'aggregated'

num_replicas

int = 1

prefill

DynamoRoleConfig | None = None

nemo_curator.core.serve.DynamoVLLMModelConfig.__post_init__() -> None

class nemo_curator.core.serve.InferenceServer(
    models: list[nemo_curator.core.serve.base.BaseModelConfig],
    backend: nemo_curator.core.serve.base.BaseServerConfig = RayServeServerConfig(),
    name: str = 'default',
    port: int = DEFAULT_SERVE_PORT,
    health_check_timeout_s: int = DEFAULT_SERVE_HEALTH_TIMEOUT_S,
    verbose: bool = False
)

Dataclass

Serve one or more models behind a typed backend config.

_backend_impl

InferenceBackend | None = field(init=False, default=None, repr=False)

_host

str = field(init=False, default='localhost', repr=False)

_started

bool = field(init=False, default=False, repr=False)

backend

BaseServerConfig = field(default_factory=RayServeServerConfig)

endpoint

str

OpenAI-compatible base URL for the served models.

health_check_timeout_s

int = DEFAULT_SERVE_HEALTH_TIMEOUT_S

models

list[BaseModelConfig]

name

str = 'default'

port

int = DEFAULT_SERVE_PORT

verbose

bool = False

nemo_curator.core.serve.InferenceServer.__enter__()

nemo_curator.core.serve.InferenceServer.__exit__(
    exc = ()
)

nemo_curator.core.serve.InferenceServer.__post_init__() -> None

nemo_curator.core.serve.InferenceServer._create_backend() -> nemo_curator.core.serve.base.InferenceBackend

nemo_curator.core.serve.InferenceServer._validate_model_configs() -> None

Check every model is accepted by the backend and that all models share one concrete type.

nemo_curator.core.serve.InferenceServer._wait_for_healthy() -> None

Poll /v1/models until all expected models appear in the response.

nemo_curator.core.serve.InferenceServer.start() -> None

Deploy all models and wait for them to become healthy.

nemo_curator.core.serve.InferenceServer.stop() -> None

Shut down the active inference backend and release resources.

class nemo_curator.core.serve.RayServeModelConfig(
    model_identifier: str,
    model_name: str | None = None,
    runtime_env: dict[str, typing.Any] = dict(),
    deployment_config: dict[str, typing.Any] = dict(),
    engine_kwargs: dict[str, typing.Any] = dict()
)

Dataclass

Bases: BaseModelConfig

Ray Serve model config.

deployment_config

dict[str, Any] = field(default_factory=dict)

engine_kwargs

dict[str, Any] = field(default_factory=dict)

class nemo_curator.core.serve.RayServeServerConfig()

Dataclass

Bases: BaseServerConfig

Server-level Ray Serve config.

model_configs

tuple[type[BaseModelConfig], ...] = (RayServeModelConfig,)

nemo_curator.core.serve.is_inference_server_active() -> bool

Check whether any inference server is currently running in this process.