Source code for emerging_optimizers.weight_update_hooks.radial_brake

# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import math

import torch


__all__ = ["RadialBrakeHook"]


[docs] class RadialBrakeHook: """Dampen radial norm changes after an optimizer update. The optimizer first applies its usual update ``w = w_prev + dw``. This hook then rescales ``w`` so that .. math:: \\|w_{brake}\\| = \\|w_{prev}\\| + s(\\|w\\| - \\|w_{prev}\\|) where ``s`` is ``outward_scale`` when the update increases the norm, otherwise ``inward_scale``. Args: outward_scale: Fraction of an outward norm change to retain. inward_scale: Fraction of an inward norm change to retain. eps: Norm threshold below which values are treated as numerical zero. """ def __init__( self, outward_scale: float = 0.5, inward_scale: float = 1.0, eps: float = 1e-15, ) -> None: if not 0.0 <= outward_scale <= 1.0: raise ValueError(f"outward_scale must be in [0, 1], got {outward_scale}") if not 0.0 <= inward_scale <= 1.0: raise ValueError(f"inward_scale must be in [0, 1], got {inward_scale}") if not math.isfinite(eps) or eps <= 0.0: raise ValueError(f"eps must be finite and positive, got {eps}") self.outward_scale = outward_scale self.inward_scale = inward_scale self.eps = eps def _norm_or_zero(self, tensor: torch.Tensor) -> torch.Tensor: norm = torch.linalg.vector_norm(tensor, dtype=torch.float32) return torch.where(norm < self.eps, torch.zeros_like(norm), norm) def pre_weight_update_inplace( self, p: torch.Tensor, update: torch.Tensor, ) -> torch.Tensor: return self._norm_or_zero(p) def post_weight_update_inplace( self, p: torch.Tensor, pre_update_state: torch.Tensor, ) -> None: pre_norm = pre_update_state post_norm = self._norm_or_zero(p) norm_delta = post_norm - pre_norm outward_scale = torch.as_tensor(self.outward_scale, device=p.device, dtype=torch.float32) inward_scale = torch.as_tensor(self.inward_scale, device=p.device, dtype=torch.float32) scale = torch.where( norm_delta >= self.eps, outward_scale, torch.where(norm_delta <= -self.eps, inward_scale, torch.zeros_like(norm_delta)), ) target_norm = pre_norm + scale * norm_delta projection_scale = torch.where( post_norm < self.eps, torch.zeros_like(post_norm), target_norm / post_norm.clamp_min(self.eps), ) p.mul_(projection_scale.to(dtype=p.dtype))