nemo_automodel.components.models.mimo_v25.config

class nemo_automodel.components.models.mimo_v25.config.MiMoV2Config(
    vocab_size: int = 151936,
    hidden_size: int = 4096,
    intermediate_size: int = 22016,
    num_hidden_layers: int = 32,
    num_attention_heads: int = 32,
    num_key_value_heads: int = 32,
    hidden_act: str = 'silu',
    max_position_embeddings: int = 32768,
    initializer_range: float = 0.02,
    layernorm_epsilon: float = 1e-06,
    rms_norm_eps: float | None = None,
    use_cache: bool = True,
    tie_word_embeddings: bool = False,
    rope_theta: float = 10000.0,
    rope_scaling: dict | None = None,
    attention_dropout: float = 0.0,
    attention_bias: bool = False,
    attention_value_scale: float | None = None,
    head_dim: int | None = None,
    v_head_dim: int | None = None,
    swa_num_attention_heads: int | None = None,
    swa_num_key_value_heads: int | None = None,
    swa_head_dim: int | None = None,
    swa_v_head_dim: int | None = None,
    swa_rope_theta: float | None = None,
    sliding_window: int | None = None,
    sliding_window_size: int | None = None,
    attention_chunk_size: int | None = None,
    add_full_attention_sink_bias: bool = False,
    add_swa_attention_sink_bias: bool = False,
    hybrid_block_size: int | None = None,
    hybrid_layer_pattern: list[int] | None = None,
    partial_rotary_factor: float = 1.0,
    n_routed_experts: int | None = None,
    n_shared_experts: int | None = None,
    moe_intermediate_size: int | None = None,
    num_experts_per_tok: int | None = None,
    routed_scaling_factor: float | None = None,
    scoring_func: str = 'sigmoid',
    topk_method: str = 'noaux_tc',
    n_group: int | None = None,
    topk_group: int | None = None,
    norm_topk_prob: bool = True,
    moe_layer_freq: list[int] | None = None,
    attention_projection_layout: str = 'split',
    torch_dtype: str = 'bfloat16',
    kwargs = {}
)

nemo_automodel.components.models.mimo_v25.config._MIMOV2_ATTENTION_PROJECTION_LAYOUTS = {'split', 'fused_qkv'}