# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import inspect
from typing import Literal
import torch
from absl import logging
from torch.optim.optimizer import ParamsT
from emerging_optimizers import registry, triton_kernels
from emerging_optimizers.mixin import WeightDecayT
from emerging_optimizers.orthogonalized_optimizers import muon_utils
from emerging_optimizers.orthogonalized_optimizers.muon_utils import NSCoeffT
from emerging_optimizers.orthogonalized_optimizers.orthogonalized_optimizer import OrthogonalizedOptimizer, _args_doc
from emerging_optimizers.utils import FP32MatmulPrecT
from emerging_optimizers.weight_update_hooks import WeightUpdateHook
from emerging_optimizers.weight_update_hooks.base import HookStateT
__all__ = ["Muon", "get_muon_scale_factor"]
MuonScaleT = Literal["shape_scaling", "spectral", "unit_rms_norm"]
[docs]
@registry.register_optimizer("muon")
class Muon(OrthogonalizedOptimizer[HookStateT]):
"""Muon: MomentUm Orthogonalized by Newton-schulz
Muon runs EMA-style momentum with optional Nesterov momentum, and then performs an orthogonalization
post-processing step, in which each 2D parameter's update is replaced with the nearest orthogonal
matrix. To efficiently orthogonalize each update, Newton-Schulz iteration is used, which has the
advantage that it may be stably run on tensor cores on GPUs.
Orthogonalization can be viewed as steepest descent in the spectral norm. The theoretical foundation
is based on modular duality and norm-constrained optimization.
This implementation incorporates decoupled weight decay, refer to Scion which views weight decay as constrained
optimization via Frank-Wolfe.
References:
- Jordan, K. *Muon Optimizer Implementation.*
[`GitHub <https://github.com/KellerJordan/Muon/blob/master/muon.py>`_]
- *Modular Duality in Deep Learning.* arXiv:2410.21265 (2024).
[`arXiv:2410.21265 <https://arxiv.org/abs/2410.21265>`_]
- *Training Deep Learning Models with Norm-Constrained LMOs.* arXiv:2502.07529 (2025).
[`arXiv:2502.07529 <https://arxiv.org/abs/2502.07529>`_]
- *SOAP, Muon, and Beyond.* arXiv:2607.20548 (2026). Default hyperparameters follow this recipe,
except for using 8 Polar Express steps instead of 16.
[`arXiv:2607.20548 <https://arxiv.org/abs/2607.20548>`_]
Warning:
- This optimizer requires that all parameters passed in are 2D.
- It should not be used for the embedding layer, the final fully connected layer, or any 1-D
parameters; those can all be optimized by a standard method (e.g., AdamW).
Args:
{_args_doc}
coefficient_type: The type of coefficient set to use for the Newton-Schulz iteration. Can be one of
["simple", "quintic", "polar_express", "cans", "aol", "deepseekv4", "cubic5", "custom"].
num_ns_steps: The number of iteration steps to use in the Newton-Schulz iteration.
scale_mode: The type of scale factor to use for the update. Defaults to "spectral" style scaling.
extra_scale_factor: The additional scale factor to use for the update. Defaults to 0.2, following the recipe
in https://arxiv.org/abs/2607.20548, which closely matches the update RMS norm of AdamW as suggested by
https://arxiv.org/abs/2502.16982.
use_syrk: Whether to use the Triton kernel for the Newton-Schulz iteration.
"""
def __init__(
self,
params: ParamsT,
lr: float = 3e-4,
momentum: float = 0.9,
weight_decay: float = 0.1,
*,
nesterov: bool = False,
weight_decay_method: WeightDecayT = "decoupled",
fp32_matmul_prec: FP32MatmulPrecT = "medium",
coefficient_type: NSCoeffT = "polar_express",
num_ns_steps: int = 8,
scale_mode: MuonScaleT = "spectral",
extra_scale_factor: float = 0.2,
use_syrk: bool = False,
weight_update_hook: WeightUpdateHook[HookStateT] | None = None,
) -> None:
if num_ns_steps < 1:
raise ValueError(f"num_ns_steps must be at least 1, got {num_ns_steps}")
self._num_ns_steps = num_ns_steps
self._coefficient_type = coefficient_type
self._scale_mode = scale_mode
self._extra_scale_factor = extra_scale_factor
if use_syrk:
if torch.cuda.is_available():
sm_version = torch.cuda.get_device_capability()
else:
sm_version = (0, 0)
if not triton_kernels.HAS_TRITON_340: # type: ignore[attr-defined]
logging.error("Triton 3.4.0 or higher is required for use_syrk to be True.")
use_syrk = False
elif sm_version not in ((8, 0), (9, 0), (10, 0), (10, 3)):
logging.error(
"Correctness of Triton kernel on SM %s cannot be guaranteed. Setting use_syrk to False.",
sm_version,
)
use_syrk = False
self._use_syrk = use_syrk
def scaled_orthogonalize_fn(grad: torch.Tensor) -> torch.Tensor:
logging.debug(
"Orthogonalizing grad with %s steps, %s coefficient, %s scale mode, extra_scale_factor=%s",
num_ns_steps,
coefficient_type,
scale_mode,
extra_scale_factor,
)
orth_grad = muon_utils.newton_schulz(
grad,
steps=num_ns_steps,
coefficient_type=coefficient_type,
use_syrk=use_syrk,
)
scale_factor = get_muon_scale_factor(grad.size(-2), grad.size(-1), mode=scale_mode)
return orth_grad * scale_factor * extra_scale_factor
super().__init__(
params,
lr,
momentum,
nesterov=nesterov,
weight_decay=weight_decay,
weight_decay_method=weight_decay_method,
fp32_matmul_prec=fp32_matmul_prec,
scaled_orthogonalize_fn=scaled_orthogonalize_fn,
weight_update_hook=weight_update_hook,
)
@property
def num_ns_steps(self) -> int:
return self._num_ns_steps
@num_ns_steps.setter
def num_ns_steps(self, _value: int) -> None:
raise AttributeError("Cannot set `num_ns_steps` post-initialization.")
@property
def coefficient_type(self) -> NSCoeffT:
return self._coefficient_type
@coefficient_type.setter
def coefficient_type(self, _value: NSCoeffT) -> None:
raise AttributeError("Cannot set `coefficient_type` post-initialization.")
@property
def scale_mode(self) -> MuonScaleT:
return self._scale_mode
@scale_mode.setter
def scale_mode(self, _value: MuonScaleT) -> None:
raise AttributeError("Cannot set `scale_mode` post-initialization.")
@property
def extra_scale_factor(self) -> float:
return self._extra_scale_factor
@extra_scale_factor.setter
def extra_scale_factor(self, _value: float) -> None:
raise AttributeError("Cannot set `extra_scale_factor` post-initialization.")
@property
def use_syrk(self) -> bool:
return self._use_syrk
@use_syrk.setter
def use_syrk(self, _value: bool) -> None:
raise AttributeError("Cannot set `use_syrk` post-initialization.")
Muon.__doc__ = inspect.cleandoc(Muon.__doc__).format(_args_doc=_args_doc) # type: ignore[arg-type]
def get_muon_scale_factor(size_out: int, size_in: int, mode: MuonScaleT = "spectral") -> float:
"""Get the scale for the update.
Default mode is "spectral", which is the mode that allows for learning rate transferability from AdamW.
An extra scale factor is used to match the update RMS norm of AdamW, so that we can transfer hyperparameters
from AdamW to Muon. An extra scale factor of sqrt((1-β₁)/(1+β₁)), where β₁ is AdamW's momentum EMA coefficient,
analytically gives the update RMS norm of AdamW (https://kexue.fm/archives/11267).
Args:
size_out: The size of the output tensor.
size_in: The size of the input tensor.
mode: The mode to use for the scale.
Returns:
The scale factor for the update.
"""
if mode == "shape_scaling":
# Suggested by Muon (https://kellerjordan.github.io/posts/muon/)
return max(1, size_out / size_in) ** 0.5
elif mode == "spectral":
# Suggested by K. Jordan and Kimi (https://arxiv.org/abs/2502.16982)
return max(size_out, size_in) ** 0.5
elif mode == "unit_rms_norm":
# Suggested by Scion (https://arxiv.org/abs/2502.07529) and Bernstein et al.
# (https://jeremybernste.in/writing/deriving-muon)
return (size_out / size_in) ** 0.5
else:
raise ValueError(f"Invalid mode for Muon update scale factor: {mode}")