`nemo_automodel.components.models.deepseek_v3.layers`#

Module Contents#

Classes#

API#

class nemo_automodel.components.models.deepseek_v3.layers.MLA( config: transformers.models.deepseek_v3.configuration_deepseek_v3.DeepseekV3Config, backend: nemo_automodel.components.moe.utils.BackendConfig, )#

Bases: torch.nn.Module

Initialization

forward(

x: torch.Tensor,

freqs_cis: torch.Tensor,

attention_mask: torch.Tensor | None = None,

**attn_kwargs: Any,

)#

init_weights(buffer_device: torch.device, init_std: float = 0.02)#