`nemo_rl.models.generation.vllm.config`#

Module Contents#

Classes#

`VllmVideoConfig`	Video sampling contract shared by policy preprocessing and vLLM.
`VllmSpecificArgs`
`VllmDeltaCompressionConfig`
`VllmRefitStorageConfig`
`VllmRefitBaselineConfig`
`VllmRefitTuningConfig`
`VllmSparseRefitConfig`
`VllmNixlRefitConfig`
`VllmCheckpointEnginePluginConfig`
`VllmRefitConfig`
`VllmConfig`

Functions#

`resolve_vllm_video_config`	Validate and return the optional vLLM video sampling contract.
`materialize_vllm_video_config`	Apply one video contract to tokenizer, data, and vLLM request config.
`normalize_vllm_refit_config`	Validate the selected refit transport and resolve its scoped defaults.

Data#

`VllmRefitTransportName`
`VllmRefitSelector`
`VLLM_SPARSE_REFIT_TRANSPORTS`

API#

nemo_rl.models.generation.vllm.config.VllmRefitTransportName#: None

nemo_rl.models.generation.vllm.config.VllmRefitSelector#: None

nemo_rl.models.generation.vllm.config.VLLM_SPARSE_REFIT_TRANSPORTS#: ‘frozenset(…)’

class nemo_rl.models.generation.vllm.config.VllmVideoConfig#

Bases: pydantic.BaseModel

Video sampling contract shared by policy preprocessing and vLLM.

model_config#: ‘ConfigDict(…)’

sampling_style: Literal[nemotron_vl]#: None

num_frames: pydantic.PositiveInt#: None

temporal_patch_size: pydantic.PositiveInt#: None

class nemo_rl.models.generation.vllm.config.VllmSpecificArgs#

Bases: typing.TypedDict

tensor_parallel_size: int#: None

pipeline_parallel_size: int#: None

expert_parallel_size: int#: None

gpu_memory_utilization: float#: None

max_model_len: int#: None

skip_tokenizer_init: bool#: None

async_engine: bool#: None

video: NotRequired[nemo_rl.models.generation.vllm.config.VllmVideoConfig]#: None

load_format: NotRequired[str]#: None

precision: NotRequired[str]#: None

logprobs_mode: NotRequired[Literal[processed_logprobs, raw_logprobs]]#: None

cap_max_tokens_to_context: NotRequired[bool]#: None

is_mx: NotRequired[bool]#: None

quantization_ignored_layer_kws: NotRequired[list[str]]#: None

quantization_ignore_patterns: NotRequired[list[str]]#: None

kv_cache_dtype: Literal[auto, fp8, fp8_e4m3]#: None

enforce_eager: NotRequired[bool]#: None

enable_return_routed_experts: NotRequired[bool]#: None

use_tqdm: NotRequired[bool]#: None

expose_http_server: NotRequired[bool]#: None

http_refit_api_key_env_var: NotRequired[str | None]#: None

reset_encoder_cache_after_weight_update: NotRequired[bool]#: None

http_refit_server_port: NotRequired[int | None]#: None

zmq_refit_server_port: NotRequired[int | None]#: None

http_server_serving_chat_kwargs: NotRequired[dict[str, Any]]#: None

tool_parser_plugin: NotRequired[str]#: None

env_vars: NotRequired[dict[str, str]]#: None

reasoning_parser_plugin: NotRequired[str]#: None

class nemo_rl.models.generation.vllm.config.VllmDeltaCompressionConfig#

Bases: pydantic.BaseModel

encoding: Literal[xor, overwrite]#: ‘xor’

sparse_bucket_size_bytes: pydantic.PositiveInt#: None

export_chunk_bytes: dict[str, pydantic.PositiveInt]#: ‘Field(…)’

zstd_threads: dict[str, pydantic.NonNegativeInt]#: ‘Field(…)’

class nemo_rl.models.generation.vllm.config.VllmRefitStorageConfig#

Bases: pydantic.BaseModel

s3_bucket: str | None#: None

s3_region: str#: ‘us-east-1’

s3_prefix: str#: ‘nemo-rl-refit’

staging_dir: str#: ‘/dev/shm’

class nemo_rl.models.generation.vllm.config.VllmRefitBaselineConfig#

Bases: pydantic.BaseModel

in_memory: bool#: False

mmap_dir: str | None#: None

class nemo_rl.models.generation.vllm.config.VllmRefitTuningConfig#

Bases: pydantic.BaseModel

encode_workers: dict[str, pydantic.PositiveInt]#: ‘Field(…)’

transfer_workers: dict[str, pydantic.PositiveInt]#: ‘Field(…)’

zmq_retries: pydantic.NonNegativeInt#: 3

zmq_relay_payload_workers: pydantic.PositiveInt#: 16

zmq_relay_forward_workers: pydantic.PositiveInt#: 8

apply_queue_depth: pydantic.PositiveInt#: 32

apply_batch_size: pydantic.PositiveInt#: 8

partition_workers: pydantic.PositiveInt#: 8

class nemo_rl.models.generation.vllm.config.VllmSparseRefitConfig#

Bases: pydantic.BaseModel

delta_compression: nemo_rl.models.generation.vllm.config.VllmDeltaCompressionConfig#: ‘Field(…)’

storage: nemo_rl.models.generation.vllm.config.VllmRefitStorageConfig#: ‘Field(…)’

baseline: nemo_rl.models.generation.vllm.config.VllmRefitBaselineConfig#: ‘Field(…)’

tuning: nemo_rl.models.generation.vllm.config.VllmRefitTuningConfig#: ‘Field(…)’

verify_samples_per_payload: pydantic.NonNegativeInt#: 0

request_timeout_s: pydantic.PositiveFloat#: 600.0

class nemo_rl.models.generation.vllm.config.VllmNixlRefitConfig#

Bases: pydantic.BaseModel

update_weights_bucket_memory_ratio: Annotated[float, Field(gt=0, lt=1)]#: 0.05

device: str#: ‘cuda’

backend_name: str#: ‘UCX’

backend_init_params: dict[str, Any] | None#: None

release_after_refit: bool#: False

shard_expert_weights: bool#: False

class nemo_rl.models.generation.vllm.config.VllmCheckpointEnginePluginConfig#

Bases: pydantic.BaseModel

update_weights_bucket_memory_ratio: Annotated[float, Field(gt=0, lt=1)]#: 0.05

release_after_refit: bool#: False

class nemo_rl.models.generation.vllm.config.VllmRefitConfig#

Bases: pydantic.BaseModel

sparse: nemo_rl.models.generation.vllm.config.VllmSparseRefitConfig#: ‘Field(…)’

nixl: nemo_rl.models.generation.vllm.config.VllmNixlRefitConfig#: ‘Field(…)’

class nemo_rl.models.generation.vllm.config.VllmConfig#

Bases: nemo_rl.models.generation.interfaces.GenerationConfig

vllm_cfg: nemo_rl.models.generation.vllm.config.VllmSpecificArgs#: None

vllm_kwargs: NotRequired[dict[str, Any]]#: None

refit_transport: NotRequired[nemo_rl.models.generation.vllm.config.VllmRefitSelector | str | None]#: None

refit_cfg: NotRequired[nemo_rl.models.generation.vllm.config.VllmRefitConfig | None]#: None

quant_cfg: NotRequired[str | None]#: None

real_quant: NotRequired[bool]#: None

real_quant_export_cpu_offload: NotRequired[bool]#: None

real_quant_ignore: NotRequired[list[str]]#: None

nemo_rl.models.generation.vllm.config.resolve_vllm_video_config( config: nemo_rl.models.generation.vllm.config.VllmConfig, ) → nemo_rl.models.generation.vllm.config.VllmVideoConfig | None#: Validate and return the optional vLLM video sampling contract.

nemo_rl.models.generation.vllm.config.materialize_vllm_video_config( policy_config: dict[str, Any], data_config: dict[str, Any], ) → None#: Apply one video contract to tokenizer, data, and vLLM request config.

nemo_rl.models.generation.vllm.config.normalize_vllm_refit_config( config: nemo_rl.models.generation.vllm.config.VllmConfig, ) → nemo_rl.models.generation.vllm.config.VllmRefitConfig | None#: Validate the selected refit transport and resolve its scoped defaults.

nemo_rl.models.generation.vllm.config#

Module Contents#

Classes#

Functions#

Data#

API#

`nemo_rl.models.generation.vllm.config`#