Source code for emerging_optimizers.orthogonalized_optimizers.sinkhorn_utils

# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import math

import torch


__all__ = ["sinkhorn_balance"]


[docs] @torch.no_grad() # type: ignore[misc] def sinkhorn_balance( update: torch.Tensor, *, eps: float = 1e-20, num_normalization_steps: int = 11, zero_row_threshold: float = 1e-3, ) -> torch.Tensor: r"""Balance a signed 2D update along its row and column axes. Given an update ``G`` with row norms ``rho_i = ||G[i, :]||_2``, rows satisfying ``rho_i <= zero_row_threshold * mean(rho)`` are first set to zero. Starting from the masked update ``U``, an odd number ``K = num_normalization_steps`` of alternating normalizations applies ``U[i, :] <- U[i, :] / (||U[i, :]||_2 + eps)`` for odd steps, and ``U[:, j] <- U[:, j] / (||U[:, j]||_2 + eps)`` for even steps. The returned update is ``Delta = sqrt(n) * U``, where ``n`` is the number of columns. When no rows are masked, this approximately targets ``(1 / n) * sum_j Delta[i, j]^2 = 1`` for every row and ``(1 / m) * sum_i Delta[i, j]^2 = 1`` for every column, where ``m`` is the number of rows. Args: update: Dense signed update with shape ``(num_rows, num_columns)``. eps: Numerical stability term added to each row or column norm. num_normalization_steps: Total positive odd number of individual axis-normalization steps. zero_row_threshold: Masking threshold relative to the mean pre-balancing row norm. Returns: A new Sinkhorn-balanced tensor with the input shape, dtype, and device. Raises: ValueError: If ``update`` is not a nonempty, tall 2D tensor in BF16, FP16, or FP32, or if a numerical argument is invalid. """ if update.ndim != 2: raise ValueError(f"sinkhorn_balance requires a 2D tensor, got {update.ndim}D") if update.size(0) == 0 or update.size(1) == 0: raise ValueError("sinkhorn_balance requires nonempty matrix dimensions") if update.size(0) < update.size(1): raise ValueError("sinkhorn_balance expects rows to be the larger matrix dimension") if update.dtype not in (torch.bfloat16, torch.float16, torch.float32): raise ValueError(f"sinkhorn_balance only supports bfloat16, float16, and float32 tensors, got {update.dtype}") if eps <= 0.0 or not math.isfinite(eps): raise ValueError(f"eps must be positive and finite, got {eps}") if num_normalization_steps < 1 or num_normalization_steps % 2 == 0: raise ValueError(f"num_normalization_steps must be a positive odd integer, got {num_normalization_steps}") if zero_row_threshold < 0.0 or not math.isfinite(zero_row_threshold): raise ValueError(f"zero_row_threshold must be nonnegative and finite, got {zero_row_threshold}") balanced_update = update.to(dtype=torch.float32, copy=True) # DeepSeek Algorithm 1 measures each row norm before balancing and masks rows whose norm is at most # zero_row_threshold times the mean row norm. This keeps inactive or near-zero token rows at zero # instead of amplifying their numerical noise during normalization. row_norms = torch.linalg.vector_norm(balanced_update, dim=1, keepdim=True) balanced_update.masked_fill_(row_norms <= zero_row_threshold * row_norms.mean(), 0.0) # Apply the first row step separately, then express each remaining iteration as a column/row pair. balanced_update.div_(row_norms.add_(eps)) num_column_row_pairs = (num_normalization_steps - 1) // 2 for _ in range(num_column_row_pairs): column_norms = torch.linalg.vector_norm(balanced_update, dim=0, keepdim=True) balanced_update.div_(column_norms.add_(eps)) row_norms = torch.linalg.vector_norm(balanced_update, dim=1, keepdim=True) balanced_update.div_(row_norms.add_(eps)) # A unit-L2 row has RMS 1 / sqrt(n); this scale converts the balanced update to unit row-wise RMS. balanced_update.mul_(math.sqrt(update.size(1))) return balanced_update.to(update.dtype)