# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TYPE_CHECKING, Callable, ClassVar, Literal, override
if TYPE_CHECKING:
from typing import overload
import torch
from torch import optim
from torch.optim.optimizer import ParamsT
from emerging_optimizers import mixin as opt_mixin
from emerging_optimizers import registry, utils
from emerging_optimizers.legacy_soap import soap
from emerging_optimizers.shampoo import precond_base
from emerging_optimizers.utils import eig as eig_utils
__all__ = [
"KlShampoo",
"KlShampooPreconditioner",
"Shampoo",
"ShampooBase",
"ShampooPreconditioner",
"ShampooScaleT",
"get_shampoo_scale_factor",
]
def _get_root_inverse_from_eigens(
eigvals: torch.Tensor,
eigvecs: torch.Tensor,
p_root_inv: float,
eps: float,
) -> torch.Tensor:
# Eigh can sometime return negative values for numerical 0; clamping to 0 removes them
eigvals = eigvals.clamp_min(0)
# Tikhonov regularization
exp = 1.0 / p_root_inv
inv_root_scale = eigvals**exp / (eigvals ** (2 * exp) + eps ** (2 * exp))
return (eigvecs * inv_root_scale) @ eigvecs.mT
[docs]
class ShampooPreconditioner:
"""Per-parameter Shampoo preconditioner holding the Kronecker factors of one 2D parameter.
Args:
state: Per-parameter optimizer state holding ``L`` and ``R``.
p_root_inv: Inverse root order; each factor is applied as ``A^(-1/p_root_inv)``.
eps: Floor on the eigenvalue magnitudes before inversion.
"""
kl_corrected: ClassVar[bool] = False
def __init__(
self,
state: dict,
p_root_inv: float,
eps: float,
) -> None:
self.kronecker_factor_pair = precond_base.TensorPair(state["L"], state["R"])
self.p_root_inv = p_root_inv
self.eps = eps
[docs]
@staticmethod
def init_state(
shape: tuple[int, ...],
device: torch.device,
) -> dict[str, torch.Tensor]:
"""Creates the Kronecker factors for a parameter shape.
Args:
shape: Shape of the 2D parameter the preconditioner will be attached to.
device: Device to allocate the state tensors on.
Returns:
The state entries owned by this preconditioner, keyed as :meth:`rebind_state` expects them.
Raises:
TypeError: If ``shape`` is not 2D.
"""
if len(shape) != 2:
raise TypeError(f"ShampooPreconditioner is only supported for 2D tensors, got shape {tuple(shape)}")
m, n = shape
return {
"L": torch.zeros(m, m, device=device),
"R": torch.zeros(n, n, device=device),
}
[docs]
def rebind_state(self, state: dict) -> None:
"""Binds the current preconditioner tensors back into the optimizer state dict.
Args:
state: Per-parameter optimizer state, updated in place.
Raises:
KeyError: If ``state`` is missing any of the preconditioner keys.
"""
updates = {
"L": self.kronecker_factor_pair.L,
"R": self.kronecker_factor_pair.R,
}
missing = updates.keys() - state.keys()
if missing:
raise KeyError(f"rebind_state: state missing keys {sorted(missing)}")
state.update(updates)
[docs]
def init_step(self, grad: torch.Tensor, shampoo_beta: float) -> None:
"""Performs the first step's factor update, before any history exists.
Args:
grad: Gradient of the parameter on the first step.
shampoo_beta: EMA coefficient for the Kronecker factor update.
"""
# Changing initial kronecker factors to be epsilon along the diagonal to match the paper
self.kronecker_factor_pair.L += self.eps * torch.eye(grad.shape[-2], device=grad.device)
self.kronecker_factor_pair.R += self.eps * torch.eye(grad.shape[-1], device=grad.device)
self.update_kronecker_factors(grad, shampoo_beta)
[docs]
def update_kronecker_factors(self, grad: torch.Tensor, shampoo_beta: float) -> None:
"""Accumulates the gradient outer products into the Kronecker factors.
Args:
grad: Gradient of the parameter.
shampoo_beta: EMA coefficient for the Kronecker factor update.
"""
self.kronecker_factor_pair.L.lerp_(grad @ grad.T, 1 - shampoo_beta)
self.kronecker_factor_pair.R.lerp_(grad.T @ grad, 1 - shampoo_beta)
[docs]
def step(self, grad: torch.Tensor, shampoo_beta: float) -> None:
"""Updates the Kronecker factors with the latest gradient.
Args:
grad: Gradient of the parameter.
shampoo_beta: EMA coefficient for the Kronecker factor update.
"""
self.update_kronecker_factors(grad, shampoo_beta)
[docs]
def precondition(self, x: torch.Tensor) -> torch.Tensor:
"""Applies both root inverse to a matrix in the parameter basis.
Args:
x: Matrix in the parameter basis.
Returns:
The preconditioned matrix, in the parameter basis.
"""
eigvals_L, eigvecs_L = eig_utils.eigh_with_fallback(self.kronecker_factor_pair.L)
eigvals_R, eigvecs_R = eig_utils.eigh_with_fallback(self.kronecker_factor_pair.R)
root_inv_L = _get_root_inverse_from_eigens(eigvals_L, eigvecs_L, self.p_root_inv, self.eps)
root_inv_R = _get_root_inverse_from_eigens(eigvals_R, eigvecs_R, self.p_root_inv, self.eps)
return root_inv_L @ x @ root_inv_R
[docs]
class KlShampooPreconditioner(ShampooPreconditioner):
"""Shampoo preconditioner with the KL-corrected Kronecker factor update."""
kl_corrected: ClassVar[bool] = True
def __init__(
self,
state: dict,
p_root_inv: float,
eps: float,
) -> None:
super().__init__(state, p_root_inv, eps)
self.eigenbasis_pair = precond_base.TensorPair(state["Q_L"], state["Q_R"])
self.eigvals_pair = precond_base.TensorPair(state["eigvals_L"], state["eigvals_R"])
[docs]
@staticmethod
@override
def init_state(
shape: tuple[int, ...],
device: torch.device,
) -> dict[str, torch.Tensor]:
state = ShampooPreconditioner.init_state(shape, device)
m, n = shape
state.update(
{
"Q_L": torch.eye(m, device=device),
"Q_R": torch.eye(n, device=device),
# Initialize eigenvalues to one to skip KL correction for the first step.
# They will be overwritten immediately afterward.
"eigvals_L": torch.ones(m, device=device),
"eigvals_R": torch.ones(n, device=device),
}
)
return state
[docs]
@override
def rebind_state(self, state: dict) -> None:
updates = {
"L": self.kronecker_factor_pair.L,
"R": self.kronecker_factor_pair.R,
"Q_L": self.eigenbasis_pair.L,
"Q_R": self.eigenbasis_pair.R,
"eigvals_L": self.eigvals_pair.L,
"eigvals_R": self.eigvals_pair.R,
}
missing = updates.keys() - state.keys()
if missing:
raise KeyError(f"rebind_state: state missing keys {sorted(missing)}")
state.update(updates)
[docs]
@override
def init_step(self, grad: torch.Tensor, shampoo_beta: float) -> None:
"""Performs the first step's factor update, before any history exists.
Args:
grad: Gradient of the parameter on the first step.
shampoo_beta: EMA coefficient for the Kronecker factor update.
"""
self.update_kronecker_factors(grad, shampoo_beta)
[docs]
@override
def update_kronecker_factors(self, grad: torch.Tensor, shampoo_beta: float) -> None:
with utils.fp32_matmul_precision("highest"):
soap.update_kronecker_factors_kl_shampoo(
self.kronecker_factor_pair,
grad,
shampoo_beta,
self.eigenbasis_pair,
self.eigvals_pair,
self.eps,
)
[docs]
@override
def precondition(self, x: torch.Tensor) -> torch.Tensor:
eigvals_L, eigvecs_L = eig_utils.eigh_with_fallback(self.kronecker_factor_pair.L)
eigvals_R, eigvecs_R = eig_utils.eigh_with_fallback(self.kronecker_factor_pair.R)
self.eigenbasis_pair = precond_base.TensorPair(eigvecs_L, eigvecs_R)
self.eigvals_pair = precond_base.TensorPair(eigvals_L, eigvals_R)
root_inv_L = _get_root_inverse_from_eigens(eigvals_L, eigvecs_L, self.p_root_inv, self.eps)
root_inv_R = _get_root_inverse_from_eigens(eigvals_R, eigvecs_R, self.p_root_inv, self.eps)
return root_inv_L @ x @ root_inv_R
ShampooScaleT = Literal["shape_scaling", "spectral", "unit_rms_norm"]
[docs]
def get_shampoo_scale_factor(
L: torch.Tensor,
R: torch.Tensor,
p_root_inv: float,
mode: ShampooScaleT = "spectral",
*,
kl_corrected: bool = False,
eps: float,
) -> float | torch.Tensor:
"""Get the scale for the Shampoo update.
Args:
L: Left Kronecker factor, ``size_out x size_out``.
R: Right Kronecker factor, ``size_in x size_in``.
p_root_inv: Inverse root order applied to each Kronecker factor.
mode: The mode to use for the scale.
kl_corrected: Whether the Kronecker factors are KL-corrected.
eps: Floor on the factor traces; use the preconditioner's ``eps``.
Returns:
The scale factor for the update: a float when the trace exponent vanishes, otherwise a 0-d tensor.
"""
size_out, size_in = L.shape[-1], R.shape[-1]
trace_exp = 1 / p_root_inv - (0.5 if kl_corrected else 0.25)
scale: float | torch.Tensor = (size_out * size_in) ** (0.5 - 1 / p_root_inv)
if trace_exp != 0:
scale = scale * (L.trace().clamp_min(eps) ** trace_exp) * (R.trace().clamp_min(eps) ** trace_exp)
if mode == "spectral":
return scale
elif mode == "unit_rms_norm":
return scale * (size_out / size_in) ** 0.5 / (size_out**0.5 + size_in**0.5)
elif mode == "shape_scaling":
return scale * max(1, size_out / size_in) ** 0.5 / (size_out**0.5 + size_in**0.5)
else:
raise ValueError(f"Invalid mode for Shampoo update scale factor: {mode}")
[docs]
class ShampooBase(optim.Optimizer, opt_mixin.WeightDecayMixin):
"""Canonical Shampoo step loop, shared by the Shampoo-family optimizers.
:meth:`step` is the whole algorithm: update the preconditioner from the gradient, run an inner scalar
optimizer in the parameter basis, precondition its update on both sides, and scale it with
:func:`get_shampoo_scale_factor`. Subclasses customize the
two pieces that vary between Shampoo variants and leave the loop alone:
- :attr:`PreconditionerCls` -- how the Kronecker factors and their inverse roots are maintained.
- :meth:`_scalar_update` -- which scalar optimizer produces the update being preconditioned.
Args:
params: Iterable of 2D CUDA parameters to optimize or dicts defining parameter groups.
lr: Learning rate.
momentum: Momentum EMA coefficient.
shampoo_beta: Kronecker factor EMA coefficient.
eps: Numerical epsilon
weight_decay: Decoupled weight decay coefficient.
p_root_inv: Inverse root order applied to each Kronecker factor.
scale_mode: The type of scale factor to use for the update, see :func:`get_shampoo_scale_factor`.
``None`` disables it.
Attributes:
PreconditionerCls: Preconditioner used for every parameter, and the source of the state layout
allocated by :meth:`_init_group`. Subclasses set it to change how the factors are maintained; its
``kl_corrected`` attribute selects the update scale, see :func:`get_shampoo_scale_factor`.
"""
PreconditionerCls: ClassVar[type[precond_base.ShampooPreconditionerProtocol]]
def __init__(
self,
params: ParamsT,
lr: float,
momentum: float = 0.9,
shampoo_beta: float = 0.95,
eps: float = 1e-8,
weight_decay: float = 0.01,
*,
p_root_inv: float = 4,
scale_mode: ShampooScaleT | None = "spectral",
) -> None:
self.eps = eps
self.weight_decay_method = "decoupled"
self.p_root_inv = p_root_inv
self.scale_mode = scale_mode
if lr < 0.0:
raise ValueError(f"Invalid learning rate: {lr}")
if p_root_inv <= 0 or round(p_root_inv) != p_root_inv:
raise ValueError(f"p_root_inv must be positive integer, got {p_root_inv}")
defaults = {
"lr": lr,
"momentum": momentum,
"shampoo_beta": shampoo_beta,
"weight_decay": weight_decay,
}
super().__init__(params, defaults)
def _scalar_update(
self,
grad: torch.Tensor,
exp_avg: torch.Tensor,
*,
momentum: float,
) -> torch.Tensor:
"""Applies the inner scalar optimizer to the gradient, in the parameter basis.
Override this to run a different scalar update ahead of the preconditioner.
Args:
grad: Gradient of the parameter.
exp_avg: Momentum buffer, updated in place.
momentum: Momentum EMA coefficient.
Returns:
The scalar update, in the parameter basis.
Raises:
NotImplementedError: Always; subclasses must provide the inner update.
"""
raise NotImplementedError
@torch.no_grad() # type: ignore[misc]
def _init_group(
self,
group: dict,
skip_non_grad_params: bool = True,
) -> None:
"""Performs lazy state initialization for parameters with gradients.
Args:
group: Parameter group dictionary.
skip_non_grad_params: Whether to skip parameters with no gradients.
Raises:
TypeError: If the parameter is not a 2D tensor.
"""
for p in group["params"]:
if skip_non_grad_params and p.grad is None:
continue # pragma: no cover
if p.dim() != 2:
raise TypeError(f"{type(self).__name__} is only supported for 2D tensors")
state = self.state[p]
if len(state) == 0:
state["step"] = 0
state["exp_avg"] = torch.zeros_like(p, dtype=torch.float32)
state.update(self.PreconditionerCls.init_state(p.shape, p.device))
if TYPE_CHECKING:
@overload
def step(self, closure: None = ...) -> None: ...
@overload
def step(self, closure: Callable[[], float]) -> float: ...
[docs]
@torch.no_grad() # type: ignore[misc]
@override
def step(self, closure: Callable[[], float] | None = None) -> float | None:
"""Performs a single optimization step.
Args:
closure: Unsupported; must be ``None``.
Raises:
ValueError: If ``closure`` is not ``None``.
"""
if closure is not None:
raise ValueError("closure is not supported")
for group in self.param_groups:
self._init_group(group)
for group in self.param_groups:
for p in group["params"]:
if p.grad is None:
continue # pragma: no cover
grad = p.grad.to(torch.float32)
state = self.state[p]
preconditioner = self.PreconditionerCls(state, self.p_root_inv, self.eps)
scalar_update = self._scalar_update(grad, state["exp_avg"], momentum=group["momentum"])
# bias correction on shampoo beta
curr_iter_1_based = state["step"] + 1
shampoo_beta = group["shampoo_beta"]
shampoo_beta = 1 - (1 - shampoo_beta) / (1 - shampoo_beta**curr_iter_1_based)
if state["step"] == 0:
preconditioner.init_step(grad, shampoo_beta)
else:
preconditioner.step(grad, shampoo_beta)
preconditioned_update = preconditioner.precondition(scalar_update)
if self.scale_mode is not None:
preconditioned_update = preconditioned_update * get_shampoo_scale_factor(
state["L"],
state["R"],
self.p_root_inv,
self.scale_mode,
kl_corrected=self.PreconditionerCls.kl_corrected,
eps=self.eps,
)
self._apply_weight_decay_inplace(
p,
grad,
group["lr"],
group["weight_decay"],
)
p.add_(preconditioned_update.to(p.dtype), alpha=-group["lr"])
preconditioner.rebind_state(state)
state["step"] += 1
return None
[docs]
@registry.register_optimizer("shampoo")
class Shampoo(ShampooBase):
"""Shampoo with EMA momentum as the inner scalar optimizer."""
PreconditionerCls: ClassVar[type[precond_base.ShampooPreconditionerProtocol]] = ShampooPreconditioner
@torch.compile
@override
def _scalar_update(
self,
grad: torch.Tensor,
exp_avg: torch.Tensor,
*,
momentum: float,
) -> torch.Tensor:
"""Applies EMA momentum to the gradient, in the parameter basis.
Args:
grad: Gradient of the parameter.
exp_avg: Momentum buffer, updated in place.
momentum: Momentum EMA coefficient.
Returns:
The momentum update, in the parameter basis.
"""
exp_avg.lerp_(grad, 1 - momentum)
return exp_avg
[docs]
@registry.register_optimizer("kl_shampoo")
class KlShampoo(Shampoo):
"""Shampoo with KL-corrected Kronecker factors and EMA momentum."""
PreconditionerCls: ClassVar[type[precond_base.ShampooPreconditionerProtocol]] = KlShampooPreconditioner
def __init__(
self,
params: ParamsT,
lr: float,
momentum: float = 0.9,
shampoo_beta: float = 0.95,
eps: float = 1e-8,
weight_decay: float = 0.01,
*,
p_root_inv: float = 2,
scale_mode: ShampooScaleT | None = "spectral",
) -> None:
super().__init__(
params,
lr,
momentum,
shampoo_beta,
eps,
weight_decay,
p_root_inv=p_root_inv,
scale_mode=scale_mode,
)