`nemo_rl.models.generation.vllm.config`#

Module Contents#

Classes#

`VllmSpecificArgs`
`VllmConfig`

API#

class nemo_rl.models.generation.vllm.config.VllmSpecificArgs#

Bases: typing.TypedDict

tensor_parallel_size: int#: None

pipeline_parallel_size: int#: None

gpu_memory_utilization: float#: None

max_model_len: int#: None

skip_tokenizer_init: bool#: None

async_engine: bool#: None

load_format: NotRequired[str]#: None

precision: NotRequired[str]#: None

enforce_eager: NotRequired[bool]#: None

class nemo_rl.models.generation.vllm.config.VllmConfig#

Bases: nemo_rl.models.generation.interfaces.GenerationConfig

vllm_cfg: nemo_rl.models.generation.vllm.config.VllmSpecificArgs#: None

vllm_kwargs: NotRequired[dict[str, Any]]#: None