nemo_automodel.components.models.laguna.config

Module Contents

Classes

Name	Description
`LagunaConfig`	Configuration for Poolside Laguna causal language models.

Functions

Name	Description
`_normalize_layer_list`	Return a per-layer list sized for `num_hidden_layers`.

Data

_PRE_INIT_OVERRIDE_FIELDS

__all__

API

class nemo_automodel.components.models.laguna.config.LagunaConfig(
    vocab_size: int = 100352,
    hidden_size: int = 2048,
    intermediate_size: int = 8192,
    num_hidden_layers: int = 48,
    num_attention_heads: int = 32,
    num_key_value_heads: int = 8,
    head_dim: int = 128,
    qkv_bias: bool = False,
    attention_bias: bool = False,
    gating: bool | str = True,
    gating_types: list[str] | None = None,
    hidden_act: str = 'silu',
    max_position_embeddings: int = 4096,
    initializer_range: float = 0.02,
    rms_norm_eps: float = 1e-06,
    use_cache: bool = True,
    tie_word_embeddings: bool = False,
    rope_parameters: dict[str, typing.Any] | None = None,
    partial_rotary_factor: float | None = None,
    attention_dropout: float = 0.0,
    sliding_window: int | None = None,
    layer_types: list[str] | None = None,
    num_attention_heads_per_layer: list[int] | None = None,
    swa_attention_sink_enabled: bool = False,
    swa_rope_parameters: dict[str, typing.Any] | None = None,
    num_experts: int = 256,
    num_experts_per_tok: int = 16,
    moe_intermediate_size: int = 1024,
    shared_expert_intermediate_size: int = 1024,
    norm_topk_prob: bool = True,
    decoder_sparse_step: int = 1,
    mlp_only_layers: list[int] | None = None,
    mlp_layer_types: list[str] | None = None,
    router_aux_loss_coef: float = 0.001,
    moe_routed_scaling_factor: float = 1.0,
    moe_apply_router_weight_on_input: bool = False,
    moe_router_logit_softcapping: float = 0.0,
    output_router_logits: bool = False,
    torch_dtype: str = 'bfloat16',
    kwargs = {}
)

Bases: PretrainedConfig

Configuration for Poolside Laguna causal language models.

bos_token_id

int | None = None

eos_token_id

int | list[int] | None = None

keys_to_ignore_at_inference

= ['past_key_values']

model_type

= 'laguna'

pad_token_id

int | None = None

nemo_automodel.components.models.laguna.config.LagunaConfig.from_dict(
    config_dict: dict[str, typing.Any],
    kwargs = {}
)

classmethod

nemo_automodel.components.models.laguna.config._normalize_layer_list(
    name: str,
    value: list[typing.Any] | None,
    num_hidden_layers: int
) -> list[typing.Any] | None

Return a per-layer list sized for num_hidden_layers.

from_pretrained(..., num_hidden_layers=N) is useful for first-layer parity checks against a large checkpoint whose JSON still carries full-length per-layer lists. Longer lists are clipped; shorter lists remain an error.

nemo_automodel.components.models.laguna.config._PRE_INIT_OVERRIDE_FIELDS = {'num_hidden_layers', 'layer_types', 'num_attention_heads_per_layer', 'gating_ty...

nemo_automodel.components.models.laguna.config.__all__ = ['LagunaConfig']