From 2a9c47ecabde45da0f115bba2c770afe364217f8 Mon Sep 17 00:00:00 2001 From: Cortexelus Date: Thu, 3 Sep 2026 22:11:48 +0000 Subject: [PATCH] lora: make lora_strength scale DoRA/BoRA magnitudes, not just the direction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit lora_strength scaled the low-rank direction update but left the magnitude vector replacing the base row/column norm outright. So as strength -> 0 the direction converged to W0/||W0|| while the magnitude stayed fully learned, leaving (mag/||W0||)*W0 — whereas strength == 0 short-circuited to exactly W0. A discontinuity at zero: "almost no LoRA" was not almost no change. Measured on a rank-16 dora-rows adapter (the sa3-medium seconds_total embedder), ||W - W0|| / ||W0||: strength before after 0.0 0.00000% 0.00000% 1e-05 1.78760% 0.00000% 1e-03 1.78761% 0.00204% 0.01 1.78793% 0.04403% 0.1 1.82682% 0.41863% 0.5 2.59486% 2.08700% 1.0 4.16283% 4.16283% Before, 43% of the total effect was already present at strength 1e-5 and the first 0.1 of the slider did almost nothing. After, the response is near-linear and strength 1 is bit-identical to the old behaviour, so trained adapters are unaffected. _blend_magnitude interpolates each learned magnitude toward the base weight's corresponding norm. Applied to all four magnitude-bearing adapters: dora, dora-xs, bora and bora-xs — the last two interpolate row and column magnitudes separately, since at strength 0 the row stage must reproduce W0 before the column stage can. It returns early at strength 1.0, so the common path (all of training) does not pay for the extra base-norm reduction. --- stable_audio_3/models/lora/model.py | 53 +++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 7 deletions(-) diff --git a/stable_audio_3/models/lora/model.py b/stable_audio_3/models/lora/model.py index ed330f88..b53f896d 100644 --- a/stable_audio_3/models/lora/model.py +++ b/stable_audio_3/models/lora/model.py @@ -179,6 +179,25 @@ def __init__(self, fan_in, fan_out, fan_in_fan_out=False, rank=4, lora_dropout_p self._adapter_forward_fn = self.forward_fn + def _blend_magnitude(self, mag, base_norm): + """Interpolate a learned magnitude back toward the base weight's norm. + + lora_strength scales the *direction* update, but a magnitude vector + replaces the base norm outright. So as strength -> 0 the direction + converges to W0/||W0|| while the magnitude stays fully learned, leaving + (mag/||W0||)*W0 rather than W0 — while strength == 0 short-circuits to + exactly W0. That discontinuity is why strength=1e-5 is not close to "no + LoRA". Interpolating the magnitude too makes strength continuous at 0. + + Returns the learned magnitude unchanged at strength 1, so trained + behaviour is untouched and the extra base-norm reduction is skipped on + the common path. + """ + s = self.lora_strength + if s == 1.0: + return mag + return base_norm + s * (mag - base_norm) + def lora_forward(self, W): delta = torch.matmul(*self.swap((self.lora_B, self.dropout_fn(self.lora_A)))).view(W.shape) delta = self.scaling * self.lora_strength * delta @@ -196,7 +215,11 @@ def dora_forward(self, W): V = W_2d + self.scaling * self.lora_strength * delta # normalize along _norm_dim, then scale by per-element magnitude V_hat = V / (V.norm(dim=self._norm_dim, keepdim=True) + 1e-12) - return (V_hat * self.magnitude.unsqueeze(self._norm_dim)).view(orig_shape).to(W.dtype) + mag = self._blend_magnitude( + self.magnitude.unsqueeze(self._norm_dim), + W_2d.norm(dim=self._norm_dim, keepdim=True), + ) + return (V_hat * mag).view(orig_shape).to(W.dtype) def bora_forward(self, W): if self.lora_strength == 0: @@ -205,12 +228,19 @@ def bora_forward(self, W): W_2d = W.view(W.shape[0], -1).to(self.lora_A.dtype) delta = torch.matmul(*self.swap((self.lora_B, self.dropout_fn(self.lora_A)))) V = W_2d + self.scaling * self.lora_strength * delta - # Row-normalize, then scale by row magnitudes + # Row-normalize, then scale by row magnitudes. Both magnitudes + # interpolate toward the base weight's own norms so strength -> 0 + # reconstructs W0 rather than a rescaled W0. V_r = V / (V.norm(dim=1, keepdim=True) + 1e-12) - intermediate = self.magnitude_r.unsqueeze(1) * V_r + intermediate = self._blend_magnitude( + self.magnitude_r.unsqueeze(1), W_2d.norm(dim=1, keepdim=True) + ) * V_r # Column-normalize, then scale by column magnitudes H_c = intermediate / (intermediate.norm(dim=0, keepdim=True) + 1e-12) - return (H_c * self.magnitude_c.unsqueeze(0)).view(orig_shape).to(W.dtype) + mag_c = self._blend_magnitude( + self.magnitude_c.unsqueeze(0), W_2d.norm(dim=0, keepdim=True) + ) + return (H_c * mag_c).view(orig_shape).to(W.dtype) def lora_xs_forward(self, W): delta = self.U @ self.M_xs.to(self.U.dtype) @ self.V.T @@ -225,7 +255,11 @@ def dora_xs_forward(self, W): delta = self.U @ self.M_xs.to(self.U.dtype) @ self.V.T V = W_2d + self.scaling * self.lora_strength * delta V_hat = V / (V.norm(dim=self._norm_dim, keepdim=True) + 1e-12) - return (V_hat * self.magnitude.unsqueeze(self._norm_dim)).view(orig_shape).to(W.dtype) + mag = self._blend_magnitude( + self.magnitude.unsqueeze(self._norm_dim), + W_2d.norm(dim=self._norm_dim, keepdim=True), + ) + return (V_hat * mag).view(orig_shape).to(W.dtype) def bora_xs_forward(self, W): if self.lora_strength == 0: @@ -235,9 +269,14 @@ def bora_xs_forward(self, W): delta = self.U @ self.M_xs.to(self.U.dtype) @ self.V.T V = W_2d + self.scaling * self.lora_strength * delta V_r = V / (V.norm(dim=1, keepdim=True) + 1e-12) - intermediate = self.magnitude_r.unsqueeze(1) * V_r + intermediate = self._blend_magnitude( + self.magnitude_r.unsqueeze(1), W_2d.norm(dim=1, keepdim=True) + ) * V_r H_c = intermediate / (intermediate.norm(dim=0, keepdim=True) + 1e-12) - return (H_c * self.magnitude_c.unsqueeze(0)).view(orig_shape).to(W.dtype) + mag_c = self._blend_magnitude( + self.magnitude_c.unsqueeze(0), W_2d.norm(dim=0, keepdim=True) + ) + return (H_c * mag_c).view(orig_shape).to(W.dtype) def forward(self, X): return self.forward_fn(X)