nemo_automodel.components.models.llava_onevision

Submodules

Package Contents

Classes

Name	Description
`LLaVAOneVision1_5_ForConditionalGeneration`	LLaVA-OneVision-1.5 for conditional generation (Rice ViT + Qwen3 text).
`LLaVAOneVision1_5_Model`	Combined vision + language backbone. Returns last_hidden_state.
`LlavaOneVisionStateDictAdapter`	-
`Llavaonevision1_5Config`	Top-level config for LLaVA-OneVision-1.5.
`RiceConfig`	Configuration for the Rice ViT vision tower.

API

class nemo_automodel.components.models.llava_onevision.model.LLaVAOneVision1_5_ForConditionalGeneration(
    config,
    attn_implementation: str | None = None,
    kwargs = {}
)

Bases: HFCheckpointingMixin, Module

LLaVA-OneVision-1.5 for conditional generation (Rice ViT + Qwen3 text).

image_token_id

= getattr(config, 'image_token_id', 151655)

lm_head

model

state_dict_adapter

= LlavaOneVisionStateDictAdapter(config)

tie_word_embeddings_support

TieSupport = TieSupport.UNTIED_ONLY

video_token_id

= getattr(config, 'video_token_id', 151656)

vocab_size

= self.model.text_config.vocab_size

nemo_automodel.components.models.llava_onevision.model.LLaVAOneVision1_5_ForConditionalGeneration.forward(
    input_ids: torch.LongTensor | None = None,
    attention_mask: torch.Tensor | None = None,
    position_ids: torch.LongTensor | None = None,
    past_key_values: typing.List[torch.FloatTensor] | None = None,
    inputs_embeds: torch.FloatTensor | None = None,
    labels: torch.LongTensor | None = None,
    use_cache: bool | None = None,
    pixel_values: torch.FloatTensor | None = None,
    pixel_values_videos: torch.FloatTensor | None = None,
    image_grid_thw: torch.LongTensor | None = None,
    video_grid_thw: torch.LongTensor | None = None,
    output_hidden_states: bool | None = None,
    logits_to_keep: typing.Union[int, torch.Tensor] = 0,
    kwargs = {}
) -> typing.Union[typing.Tuple, transformers.modeling_outputs.CausalLMOutputWithPast]

nemo_automodel.components.models.llava_onevision.model.LLaVAOneVision1_5_ForConditionalGeneration.from_config(
    config,
    kwargs = {}
)

classmethod

nemo_automodel.components.models.llava_onevision.model.LLaVAOneVision1_5_ForConditionalGeneration.from_pretrained(
    pretrained_model_name_or_path: str,
    model_args = (),
    kwargs = {}
)

classmethod

nemo_automodel.components.models.llava_onevision.model.LLaVAOneVision1_5_ForConditionalGeneration.get_input_embeddings()

nemo_automodel.components.models.llava_onevision.model.LLaVAOneVision1_5_ForConditionalGeneration.get_output_embeddings()

nemo_automodel.components.models.llava_onevision.model.LLaVAOneVision1_5_ForConditionalGeneration.set_input_embeddings(
    value
)

nemo_automodel.components.models.llava_onevision.model.LLaVAOneVision1_5_ForConditionalGeneration.set_output_embeddings(
    new_embeddings
)

class nemo_automodel.components.models.llava_onevision.model.LLaVAOneVision1_5_Model(
    config: nemo_automodel.components.models.llava_onevision.model.Llavaonevision1_5Config,
    attn_implementation: str = 'eager'
)

Bases: Module

Combined vision + language backbone. Returns last_hidden_state.

language_model

= Qwen3Model(self.text_config)

text_config

= _coerce_text_config(config.text_config)

vision_config

= _coerce_vision_config(config.vision_config)

visual

nemo_automodel.components.models.llava_onevision.model.LLaVAOneVision1_5_Model.forward(
    input_ids: torch.LongTensor | None = None,
    attention_mask: torch.Tensor | None = None,
    position_ids: torch.LongTensor | None = None,
    past_key_values: typing.List[torch.FloatTensor] | None = None,
    inputs_embeds: torch.FloatTensor | None = None,
    pixel_values: torch.FloatTensor | None = None,
    pixel_values_videos: torch.FloatTensor | None = None,
    image_grid_thw: torch.LongTensor | None = None,
    video_grid_thw: torch.LongTensor | None = None,
    use_cache: bool | None = None,
    kwargs = {}
)

nemo_automodel.components.models.llava_onevision.model.LLaVAOneVision1_5_Model.get_image_features(
    pixel_values: torch.FloatTensor,
    image_grid_thw: torch.LongTensor
) -> torch.Tensor

nemo_automodel.components.models.llava_onevision.model.LLaVAOneVision1_5_Model.get_input_embeddings()

nemo_automodel.components.models.llava_onevision.model.LLaVAOneVision1_5_Model.set_input_embeddings(
    value
)

class nemo_automodel.components.models.llava_onevision.state_dict_adapter.LlavaOneVisionStateDictAdapter(
    config: typing.Any = None,
    kwargs = {}
)

Bases: StateDictAdapter

nemo_automodel.components.models.llava_onevision.state_dict_adapter.LlavaOneVisionStateDictAdapter.convert_single_tensor_to_hf(
    fqn: str,
    tensor: typing.Any,
    kwargs = {}
) -> list[tuple[str, typing.Any]]

nemo_automodel.components.models.llava_onevision.state_dict_adapter.LlavaOneVisionStateDictAdapter.from_hf(
    hf_state_dict: dict[str, typing.Any],
    kwargs = {}
) -> dict[str, typing.Any]

nemo_automodel.components.models.llava_onevision.state_dict_adapter.LlavaOneVisionStateDictAdapter.to_hf(
    state_dict: dict[str, typing.Any],
    exclude_key_regex: str | None = None,
    kwargs = {}
) -> dict[str, typing.Any]

class nemo_automodel.components.models.llava_onevision.model.Llavaonevision1_5Config(
    text_config: typing.Union[typing.Dict, transformers.configuration_utils.PretrainedConfig] | None = None,
    vision_config: typing.Union[typing.Dict, nemo_automodel.components.models.llava_onevision.model.RiceConfig] | None = None,
    image_token_id: int = 151655,
    video_token_id: int = 151656,
    vision_start_token_id: int = 151652,
    vision_end_token_id: int = 151653,
    vocab_size: int = 152064,
    architectures: typing.List[str] | None = None,
    kwargs = {}
)

Bases: PretrainedConfig

Top-level config for LLaVA-OneVision-1.5.

model_type matches the on-hub value exactly so AutoConfig.from_pretrained resolves to this class without trust_remote_code once registered.

model_type

= 'llavaonevision1_5'

sub_configs

= {'vision_config': RiceConfig}

text_config

= _build_text_config(text_config)

vision_config

= RiceConfig(**vision_config)

nemo_automodel.components.models.llava_onevision.model.Llavaonevision1_5Config.to_dict() -> typing.Dict[str, typing.Any]

class nemo_automodel.components.models.llava_onevision.model.RiceConfig(
    depth: int = 24,
    embed_dim: int = 1024,
    hidden_size: int = 1024,
    hidden_act: str = 'gelu',
    intermediate_size: int = 4096,
    num_heads: int = 16,
    in_channels: int = 3,
    patch_size: int = 14,
    spatial_merge_size: int = 2,
    temporal_patch_size: int = 1,
    initializer_range: float = 0.02,
    layer_norm_eps: float = 1e-05,
    text_hidden_size: int = 2560,
    kwargs = {}
)

Bases: PretrainedConfig

Configuration for the Rice ViT vision tower.

base_config_key

= 'vision_config'

model_type

= 'rice_vit'