nemo_automodel.components.models.kimi_k3.config

class nemo_automodel.components.models.kimi_k3.config.KimiK3Config(
    text_config: dict[str, typing.Any] | nemo_automodel.components.models.kimi_k3.config.KimiK3TextConfig | None = None,
    vision_config: dict[str, typing.Any] | nemo_automodel.components.models.kimi_k3.config.KimiK3VisionConfig | None = None,
    ignore_index: int = -100,
    media_placeholder_token_id: int = 163605,
    pad_token_id: int = 0,
    architectures: list[str] | None = None,
    kwargs: typing.Any = {}
)

class nemo_automodel.components.models.kimi_k3.config.KimiK3TextConfig(
    vocab_size: int = 163840,
    hidden_size: int = 7168,
    head_dim: int | None = None,
    intermediate_size: int = 18432,
    num_hidden_layers: int = 93,
    num_attention_heads: int = 56,
    num_key_value_heads: int | None = None,
    hidden_act: str = 'situ',
    initializer_range: float = 0.02,
    rms_norm_eps: float = 1e-05,
    use_cache: bool = True,
    pad_token_id: int = 0,
    bos_token_id: int = 1,
    eos_token_id: int = 2,
    architectures: list[str] | None = None,
    rope_theta: float = 10000.0,
    rope_scaling: dict[str, typing.Any] | None = None,
    tie_word_embeddings: bool = False,
    attention_dropout: float = 0.0,
    max_position_embeddings: int = 1048576,
    moe_intermediate_size: int | None = 3072,
    moe_renormalize: bool = True,
    moe_router_activation_func: str = 'sigmoid',
    num_experts: int | None = 896,
    num_experts_per_token: int | None = 16,
    num_shared_experts: int = 2,
    routed_scaling_factor: float = 1.0,
    first_k_dense_replace: int = 1,
    moe_layer_freq: int = 1,
    use_grouped_topk: bool = True,
    num_expert_group: int = 1,
    topk_group: int = 1,
    topk_method: str = 'noaux_tc',
    routed_expert_hidden_size: int | None = 3584,
    latent_moe_use_norm: bool = True,
    q_lora_rank: int | None = 1536,
    kv_lora_rank: int | None = 512,
    qk_nope_head_dim: int | None = 128,
    qk_rope_head_dim: int | None = 64,
    v_head_dim: int | None = 128,
    mla_use_nope: bool = True,
    mla_use_output_gate: bool = True,
    linear_attn_config: dict[str, typing.Any] | None = None,
    kda_mode: str = 'chunk',
    kda_unpad_inputs: bool = True,
    kda_use_fused_gate: bool = True,
    kda_use_qk_l2norm_in_kernel: bool = True,
    attn_res_block_size: int | None = 12,
    activation_situ_beta: float | None = 4.0,
    activation_situ_linear_beta: float | None = 25.0,
    num_nextn_predict_layers: int = 0,
    kwargs: typing.Any = {}
)

class nemo_automodel.components.models.kimi_k3.config.KimiK3VisionConfig(
    patch_size: int = 14,
    init_pos_emb_height: int = 64,
    init_pos_emb_width: int = 64,
    init_pos_emb_time: int = 4,
    pos_emb_type: str = 'divided_fixed',
    vt_num_attention_heads: int = 12,
    vt_num_hidden_layers: int = 27,
    vt_hidden_size: int = 1024,
    vt_intermediate_size: int = 4096,
    merge_kernel_size: tuple[int, int] | list[int] = (2, 2),
    merge_type: str = 'sd2_tpool',
    attn_implementation: str = 'flash_attention_2',
    mm_projector_type: str = 'patchmergerv2',
    mm_hidden_size: int | None = None,
    projector_hidden_act: str = 'gelu',
    projector_ln_eps: float = 1e-05,
    qkv_hidden_size: int = 1536,
    norm_type: str = 'rmsnorm',
    attn_bias: bool = False,
    patch_embed_proj_bias: bool = False,
    mlp_type: str = 'mlp2',
    linear_bias: bool = False,
    activation_func: str = 'gelu_pytorch_tanh',
    pos_emb_interpolation_mode: str = 'bilinear',
    ignore_index: int = -100,
    media_placeholder_token_id: int = 163605,
    pad_token_id: int = 0,
    text_hidden_size: int = 7168,
    kwargs: typing.Any = {}
)