nemo_automodel.components.models.qwen3_8_flash_next.config

Checkpoint-compatible configuration classes for Qwen3.8-Flash-Next.

Module Contents

Classes

Name	Description
`Qwen3_8_FlashNextConfig`	Top-level configuration for Qwen3.8-Flash-Next conditional generation checkpoints.
`Qwen3_8_FlashNextLegacyConfig`	Config alias for checkpoint dumps that predate the model rename.
`Qwen3_8_FlashNextLegacyTextConfig`	Text config alias for checkpoint dumps that predate the model rename.
`Qwen3_8_FlashNextTextConfig`	Configuration for the Qwen3.8-Flash-Next HyperConnection, PLE, hybrid-MoE text backbone.
`Qwen3_8_FlashNextVisionConfig`	Configuration for the Qwen3.8-Flash-Next Qwen3-VL-style vision tower.

API

class nemo_automodel.components.models.qwen3_8_flash_next.config.Qwen3_8_FlashNextConfig(
    text_config: dict[str, typing.Any] | nemo_automodel.components.models.qwen3_8_flash_next.config.Qwen3_8_FlashNextTextConfig | None = None,
    vision_config: dict[str, typing.Any] | nemo_automodel.components.models.qwen3_8_flash_next.config.Qwen3_8_FlashNextVisionConfig | None = None,
    image_token_id: int = 248056,
    video_token_id: int = 248057,
    vision_start_token_id: int = 248053,
    vision_end_token_id: int = 248054,
    language_model_only: bool = False,
    tie_word_embeddings: bool = False,
    rope_parameters: dict[str, typing.Any] | None = None,
    architectures: list[str] | None = None,
    kwargs: typing.Any = {}
)

Bases: PretrainedConfig

Top-level configuration for Qwen3.8-Flash-Next conditional generation checkpoints.

architectures

= ['Qwen3_8_FlashNextForConditionalGeneration']

keys_to_ignore_at_inference

= ['past_key_values']

model_type

= 'qwen3_8_flash_next'

rope_parameters

sub_configs

class nemo_automodel.components.models.qwen3_8_flash_next.config.Qwen3_8_FlashNextLegacyConfig(
    kwargs: typing.Any = {}
)

Bases: Qwen3_8_FlashNextConfig

Config alias for checkpoint dumps that predate the model rename.

Released checkpoints store model_type: qwen4_exp and architecture Qwen4ExpForConditionalGeneration. The dumps are immutable, so the legacy identifiers resolve to the renamed classes here instead.

model_type

= 'qwen4_exp'

class nemo_automodel.components.models.qwen3_8_flash_next.config.Qwen3_8_FlashNextLegacyTextConfig(
    kwargs: typing.Any = {}
)

Bases: Qwen3_8_FlashNextTextConfig

Text config alias for checkpoint dumps that predate the model rename.

model_type

= 'qwen4_exp_text'

class nemo_automodel.components.models.qwen3_8_flash_next.config.Qwen3_8_FlashNextTextConfig(
    vocab_size: int = 248320,
    hidden_size: int = 2560,
    intermediate_size: int = 5632,
    num_hidden_layers: int = 48,
    num_attention_heads: int = 24,
    num_key_value_heads: int = 2,
    head_dim: int = 256,
    hidden_act: str = 'silu',
    initializer_range: float = 0.02,
    rms_norm_eps: float = 1e-06,
    use_cache: bool = True,
    attention_bias: bool = False,
    attention_dropout: float = 0.0,
    max_position_embeddings: int = 262144,
    tie_word_embeddings: bool = False,
    dtype: str = 'bfloat16',
    rope_theta: float | None = None,
    rope_scaling: dict[str, typing.Any] | None = None,
    rope_parameters: dict[str, typing.Any] | None = None,
    partial_rotary_factor: float | None = None,
    full_attention_interval: int = 4,
    layer_types: list[str] | None = None,
    output_gate_type: str = 'sigmoid',
    linear_conv_kernel_dim: int = 4,
    linear_key_head_dim: int = 128,
    linear_value_head_dim: int = 128,
    linear_num_key_heads: int = 16,
    linear_num_value_heads: int = 48,
    mamba_ssm_dtype: str = 'float32',
    decoder_sparse_step: int = 1,
    moe_intermediate_size: int = 640,
    shared_expert_intermediate_size: int = 640,
    num_experts: int = 512,
    num_experts_per_tok: int = 10,
    norm_topk_prob: bool = True,
    output_router_logits: bool = False,
    router_aux_loss_coef: float = 0.001,
    mlp_only_layers: list[int] | None = None,
    hc_count: int = 4,
    hc_lowrank: int = 320,
    ple_layer_ids: list[int] | None = None,
    ple_embed_dim: int | None = None,
    ple_conv_kernel_size: int = 4,
    ngram_size: int = 3,
    heads_per_ngram: int = 8,
    ngram_vocab_size_base: int = 20000000,
    make_ngram_vocab_size_divisible_by: int = 128,
    split_ngram_parts: int = 128,
    indexer_budget: int = 2048,
    indexer_compress_ratio: int = 4,
    indexer_head_dim: int = 128,
    indexer_kv_heads: int = 1,
    indexer_n_heads: int = 4,
    mtp: dict[str, typing.Any] | None = None,
    mtp_num_hidden_layers: int = 1,
    mtp_use_dedicated_embeddings: bool = False,
    pad_token_id: int | None = None,
    bos_token_id: int = 248044,
    eos_token_id: int = 248044,
    kwargs: typing.Any = {}
)

Bases: PretrainedConfig

Configuration for the Qwen3.8-Flash-Next HyperConnection, PLE, hybrid-MoE text backbone.

base_config_key

= 'text_config'

full_attention_layer_ids

list[int]

Return zero-based QSA full-attention decoder-layer indices.

keys_to_ignore_at_inference

= ['past_key_values']

layers_block_type

list[str]

Return SGLang-compatible mixer names for each decoder layer.

linear_layer_ids

list[int]

Return zero-based GatedDeltaNet decoder-layer indices.

model_type

= 'qwen3_8_flash_next_text'

ngram_context_len

int

Return the number of preceding token IDs required by PLE hashing.

ple_embed_dim

= ple_embed_dim or hidden_size

short_conv_layer_ids

list[int]

Return zero-based decoder-layer indices that contain PLE short convolution.

short_conv_state_shape

tuple[int, int] | None

Return PLE convolution cache shape as [channels, history].

class nemo_automodel.components.models.qwen3_8_flash_next.config.Qwen3_8_FlashNextVisionConfig(
    depth: int = 27,
    hidden_act: str = 'gelu_pytorch_tanh',
    hidden_size: int = 1152,
    in_channels: int = 3,
    initializer_range: float = 0.02,
    intermediate_size: int = 4304,
    num_heads: int = 16,
    num_position_embeddings: int = 2304,
    out_hidden_size: int = 2560,
    patch_size: int = 16,
    spatial_merge_size: int = 2,
    temporal_patch_size: int = 2,
    deepstack_visual_indexes: list[int] | None = None,
    kwargs: typing.Any = {}
)

Bases: PretrainedConfig

Configuration for the Qwen3.8-Flash-Next Qwen3-VL-style vision tower.

base_config_key

= 'vision_config'

deepstack_visual_indexes

model_type

= 'qwen3_8_flash_next'