nemo_automodel.components.models.kimi_k3.config

Checkpoint-compatible configuration classes for Moonshot Kimi K3.

Module Contents

Classes

Name	Description
`KimiK3Config`	Top-level Kimi K3 vision-language checkpoint configuration.
`KimiK3TextConfig`	Configuration for the Kimi K3 hybrid KDA/MLA text backbone.
`KimiK3VisionConfig`	Configuration for the Kimi K3 MoonViT3d vision tower and projector.

API

class nemo_automodel.components.models.kimi_k3.config.KimiK3Config(
    text_config: dict[str, typing.Any] | nemo_automodel.components.models.kimi_k3.config.KimiK3TextConfig | None = None,
    vision_config: dict[str, typing.Any] | nemo_automodel.components.models.kimi_k3.config.KimiK3VisionConfig | None = None,
    ignore_index: int = -100,
    media_placeholder_token_id: int = 163605,
    pad_token_id: int = 0,
    architectures: list[str] | None = None,
    kwargs: typing.Any = {}
)

Bases: PretrainedConfig

Top-level Kimi K3 vision-language checkpoint configuration.

architectures

= ['KimiK3ForConditionalGeneration']

hidden_size

= text_config.hidden_size

max_position_embeddings

= text_config.max_position_embeddings

model_type

= 'kimi_k3'

quantization_config

= text_config.quantization_config

sub_configs

vocab_size

= text_config.vocab_size

class nemo_automodel.components.models.kimi_k3.config.KimiK3TextConfig(
    vocab_size: int = 163840,
    hidden_size: int = 7168,
    head_dim: int | None = None,
    intermediate_size: int = 33792,
    num_hidden_layers: int = 93,
    num_attention_heads: int = 96,
    num_key_value_heads: int | None = None,
    hidden_act: str = 'situ',
    initializer_range: float = 0.02,
    rms_norm_eps: float = 1e-05,
    use_cache: bool = True,
    pad_token_id: int = 0,
    bos_token_id: int = 1,
    eos_token_id: int = 2,
    architectures: list[str] | None = None,
    rope_theta: float = 10000.0,
    rope_scaling: dict[str, typing.Any] | None = None,
    tie_word_embeddings: bool = False,
    attention_dropout: float = 0.0,
    max_position_embeddings: int = 1048576,
    moe_intermediate_size: int | None = 3072,
    moe_renormalize: bool = True,
    moe_router_activation_func: str = 'sigmoid',
    num_experts: int | None = 896,
    num_experts_per_token: int | None = 16,
    num_shared_experts: int = 2,
    routed_scaling_factor: float = 1.0,
    first_k_dense_replace: int = 1,
    moe_layer_freq: int = 1,
    use_grouped_topk: bool = True,
    num_expert_group: int = 1,
    topk_group: int = 1,
    topk_method: str = 'noaux_tc',
    routed_expert_hidden_size: int | None = 3584,
    latent_moe_use_norm: bool = True,
    q_lora_rank: int | None = 1536,
    kv_lora_rank: int | None = 512,
    qk_nope_head_dim: int | None = 128,
    qk_rope_head_dim: int | None = 64,
    v_head_dim: int | None = 128,
    mla_use_nope: bool = True,
    mla_use_output_gate: bool = True,
    linear_attn_config: dict[str, typing.Any] | None = None,
    kda_mode: str = 'chunk',
    kda_unpad_inputs: bool = True,
    kda_use_fused_gate: bool = True,
    kda_use_qk_l2norm_in_kernel: bool = True,
    attn_res_block_size: int | None = 12,
    activation_situ_beta: float | None = 4.0,
    activation_situ_linear_beta: float | None = 25.0,
    num_nextn_predict_layers: int = 0,
    kwargs: typing.Any = {}
)

Bases: PretrainedConfig

Configuration for the Kimi K3 hybrid KDA/MLA text backbone.

Defaults reproduce the released moonshotai/Kimi-K3 checkpoint (config.json text_config; tech report Table 1): 93 layers (69 KDA + 24 gated MLA), 96 attention heads, dense FFN 33792, 896 latent-MoE experts (16 active, 2 shared), so from_config builds the same shape that from_pretrained loads.

architectures

= ['KimiK3ForCausalLM']

is_linear_attn

bool

Whether any decoder layer uses Kimi Delta Attention.

is_mla

bool

Whether full-attention layers use Kimi multi-latent attention.

is_moe

bool

Whether the text checkpoint has routed experts.

keys_to_ignore_at_inference

= ['past_key_values']

model_type

= 'kimi_linear'

nemo_automodel.components.models.kimi_k3.config.KimiK3TextConfig._validate() -> None

nemo_automodel.components.models.kimi_k3.config.KimiK3TextConfig.is_kda_layer(
    layer_idx: int
) -> bool

Return whether zero-based layer_idx is a KDA layer.

class nemo_automodel.components.models.kimi_k3.config.KimiK3VisionConfig(
    patch_size: int = 14,
    init_pos_emb_height: int = 64,
    init_pos_emb_width: int = 64,
    init_pos_emb_time: int = 4,
    pos_emb_type: str = 'divided_fixed',
    vt_num_attention_heads: int = 12,
    vt_num_hidden_layers: int = 27,
    vt_hidden_size: int = 1024,
    vt_intermediate_size: int = 4096,
    merge_kernel_size: tuple[int, int] | list[int] = (2, 2),
    merge_type: str = 'sd2_tpool',
    attn_implementation: str = 'flash_attention_2',
    mm_projector_type: str = 'patchmergerv2',
    mm_hidden_size: int | None = None,
    projector_hidden_act: str = 'gelu',
    projector_ln_eps: float = 1e-05,
    qkv_hidden_size: int = 1536,
    norm_type: str = 'rmsnorm',
    attn_bias: bool = False,
    patch_embed_proj_bias: bool = False,
    mlp_type: str = 'mlp2',
    linear_bias: bool = False,
    activation_func: str = 'gelu_pytorch_tanh',
    pos_emb_interpolation_mode: str = 'bilinear',
    ignore_index: int = -100,
    media_placeholder_token_id: int = 163605,
    pad_token_id: int = 0,
    text_hidden_size: int = 7168,
    kwargs: typing.Any = {}
)

Bases: PretrainedConfig

Configuration for the Kimi K3 MoonViT3d vision tower and projector.

merge_kernel_size

= list(merge_kernel_size)

mm_hidden_size

model_type

= 'kimi_k3_vision'