Source code for emerging_optimizers.legacy_soap.moso

# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TYPE_CHECKING, Callable, override


if TYPE_CHECKING:
    from typing import overload

import torch
from torch import optim
from torch.optim.optimizer import ParamsT

from emerging_optimizers import mixin as opt_mixin
from emerging_optimizers import registry, utils
from emerging_optimizers.legacy_soap import soap_utils
from emerging_optimizers.legacy_soap.soap import _clip_update_rms_in_place
from emerging_optimizers.scalar_optimizers import update_functions


__all__ = ["MOSO"]


[docs] @registry.register_optimizer("moso") class MOSO(opt_mixin.WeightDecayMixin, optim.Optimizer): r"""Momentum One-Sided SOAP. MOSO tracks EMA momentum like Muon, accumulates a SOAP/Shampoo-style covariance of that momentum on the smaller matrix side, and applies an RMSProp update in the covariance eigenbasis. Conceptually, this is one-sided SOAP where ``G_t G_t^T`` is replaced by ``M_t M_t^T`` (or ``M_t^T M_t`` for the right side), and the update is computed by projecting the momentum into the eigenbasis, applying RMSProp there, and projecting back: .. math:: C_t = \beta_s C_{t-1} + (1 - \beta_s) M_t M_t^T,\quad C_t = Q_M \Lambda_M Q_M^T U_t = Q_M \operatorname{RMSprop}(Q_M^T M_t) for the left-preconditioned case where ``M_t.shape[0] <= M_t.shape[1]``; the right-preconditioned case uses ``C_t = M_t^T M_t`` and computes ``U_t = \operatorname{RMSprop}(M_t Q_M) Q_M^T``. Args: params: Iterable of parameters to optimize or dicts defining parameter groups. lr: Learning rate. momentum: EMA coefficient for the Muon-style momentum. rms_beta: EMA coefficient for the second-moment (RMS) normalization in the eigenbasis. shampoo_beta: EMA coefficient for the one-sided momentum covariance. eps: RMSProp epsilon for numerical stability. weight_decay: Weight decay coefficient. max_update_rms: Clip the update RMS to this value (0 means no clipping). """ def __init__( self, params: ParamsT, lr: float = 3e-4, momentum: float = 0.95, rms_beta: float = 0.95, shampoo_beta: float = 0.95, eps: float = 1e-8, weight_decay: float = 0.01, *, max_update_rms: float = 0.0, ) -> None: self.weight_decay_method = "decoupled" self.max_update_rms = max_update_rms defaults = { "lr": lr, "momentum": momentum, "rms_beta": rms_beta, "shampoo_beta": shampoo_beta, "eps": eps, "weight_decay": weight_decay, } super().__init__(params, defaults) @torch.no_grad() # type: ignore[misc] def _init_group( self, group: dict, skip_non_grad_params: bool = True, ) -> None: """Performs lazy state initialization for 2D parameters.""" for p in group["params"]: if skip_non_grad_params and p.grad is None: continue # pragma: no cover if p.dim() != 2: raise TypeError("MOSO is only supported for 2D tensors") state = self.state[p] if len(state) == 0: rows, cols = p.shape preconditioner_size = min(rows, cols) state["step"] = 0 state["momentum_buffer"] = torch.zeros_like(p.data, dtype=torch.float32) state["exp_avg_sq"] = torch.zeros_like(p.data, dtype=torch.float32) state["M"] = torch.zeros( preconditioner_size, preconditioner_size, device=p.device, dtype=torch.float32, ) state["Q_M"] = torch.eye(preconditioner_size, device=p.device, dtype=torch.float32) if TYPE_CHECKING: @overload def step(self, closure: None = ...) -> None: ... @overload def step(self, closure: Callable[[], float]) -> float: ...
[docs] @torch.no_grad() # type: ignore[misc] @override def step(self, closure: Callable[[], float] | None = None) -> float | None: """Performs a single optimization step.""" if closure is not None: raise ValueError("closure is not supported") for group in self.param_groups: self._init_group(group) for p in group["params"]: if p.grad is None: continue # pragma: no cover grad = p.grad.to(torch.float32) state = self.state[p] curr_iter_1_based = state["step"] + 1 self._apply_weight_decay_inplace( p, grad, group["lr"], group["weight_decay"], ) state["momentum_buffer"].lerp_(grad, 1 - group["momentum"]) momentum = state["momentum_buffer"] shampoo_beta = 1 - (1 - group["shampoo_beta"]) / (1 - group["shampoo_beta"] ** curr_iter_1_based) with utils.fp32_matmul_precision("highest"): _update_one_sided_momentum_factor( momentum_factor=state["M"], momentum=momentum, shampoo_beta=shampoo_beta, ) left_preconditioned = momentum.shape[0] <= momentum.shape[1] with utils.fp32_matmul_precision("highest"): state["Q_M"], state["exp_avg_sq"] = _update_eigenbasis_and_exp_avg_sq( momentum_factor=state["M"], eigenbasis=state["Q_M"], exp_avg_sq=state["exp_avg_sq"], left_preconditioned=left_preconditioned, use_eigh=state["step"] == 0, power_iter_steps=1, ) with utils.fp32_matmul_precision("highest"): momentum_projected = _project_to_one_sided_eigenbasis( x=momentum, eigenbasis=state["Q_M"], left_preconditioned=left_preconditioned, ) rmsprop_update = update_functions.calculate_rmsprop_update( momentum_projected, state["exp_avg_sq"], alpha=group["rms_beta"], eps=group["eps"], ) update = _project_from_one_sided_eigenbasis( x=rmsprop_update, eigenbasis=state["Q_M"], left_preconditioned=left_preconditioned, ) _clip_update_rms_in_place(update, self.max_update_rms) p.add_(update, alpha=-group["lr"]) state["step"] += 1 return None
@torch.no_grad() # type: ignore[misc] def _update_one_sided_momentum_factor( momentum_factor: torch.Tensor, momentum: torch.Tensor, shampoo_beta: float, ) -> None: """Update the smaller-side covariance of the Muon momentum.""" left_preconditioned = momentum.shape[0] <= momentum.shape[1] maybe_transposed_momentum = momentum if left_preconditioned else momentum.T momentum_factor.lerp_(maybe_transposed_momentum @ maybe_transposed_momentum.T, 1 - shampoo_beta) @torch.no_grad() # type: ignore[misc] def _update_eigenbasis_and_exp_avg_sq( momentum_factor: torch.Tensor, eigenbasis: torch.Tensor, exp_avg_sq: torch.Tensor, left_preconditioned: bool, *, use_eigh: bool, power_iter_steps: int, ) -> tuple[torch.Tensor, torch.Tensor]: """Update one eigenbasis and keep the RMSProp second moment aligned with that basis.""" if use_eigh: _, (updated_eigenbasis,) = soap_utils.get_eigenbasis_eigh([momentum_factor]) else: x = exp_avg_sq if left_preconditioned else exp_avg_sq.mT exp_avg_sq = x if left_preconditioned else x.mT _, (updated_eigenbasis,) = soap_utils.get_eigenbasis_qr( [momentum_factor], [eigenbasis], power_iter_steps=power_iter_steps, ) return updated_eigenbasis, exp_avg_sq @torch.no_grad() # type: ignore[misc] def _project_to_one_sided_eigenbasis( x: torch.Tensor, eigenbasis: torch.Tensor, left_preconditioned: bool, ) -> torch.Tensor: """Project a matrix into the smaller-side covariance eigenbasis.""" if left_preconditioned: return eigenbasis.T @ x return x @ eigenbasis @torch.no_grad() # type: ignore[misc] def _project_from_one_sided_eigenbasis( x: torch.Tensor, eigenbasis: torch.Tensor, left_preconditioned: bool, ) -> torch.Tensor: """Project a matrix from the smaller-side covariance eigenbasis.""" if left_preconditioned: return eigenbasis @ x return x @ eigenbasis.T