From abee870ada10589e698c7b3f60877fa7529fcf2b Mon Sep 17 00:00:00 2001 From: Jens Ahrensfeld Date: Sat, 30 May 2026 18:26:06 +0200 Subject: [PATCH] [bugfix] - fix L1 and L2 regularisation gradients in grad_compute MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previous code computed scalar norms and multiplied by W, giving gradients that scaled with total weight magnitude rather than the correct per-element derivatives: L1: gradient is λ·sign(W), not λ·‖W‖₁·W L2: gradient is 2λ·W, not λ·‖W‖₂²·W Also removed the combined (l1+l2)*W term which incorrectly mixed the two. weight_decay (correct L2 form λW) is kept as a separate term. Co-Authored-By: Claude Sonnet 4.6 --- src/rbm/entity.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/rbm/entity.py b/src/rbm/entity.py index b9c98ea..d48f8e1 100644 --- a/src/rbm/entity.py +++ b/src/rbm/entity.py @@ -97,15 +97,11 @@ class Entity: self.grad.b_v = params.momentum * self.grad.b_v + lr * d_bv self.grad.b_h = params.momentum * self.grad.b_h + lr * d_bh - # compute L1 term and penalize cost function (d_whv) - l1_norm = np.sum(np.abs(self.state.w_hv)) - l1_term = params.l1_lambda*l1_norm - - # compute L2 term and penalize cost function (d_whv) - l2_norm = np.sum(np.square(self.state.w_hv)) - l2_term = params.l2_lambda*l2_norm - - self.grad.w_hv = params.momentum * self.grad.w_hv + lr * (d_whv - self.state.w_hv*(l1_term+l2_term)) - lr * params.weight_decay * self.state.w_hv + self.grad.w_hv = (params.momentum * self.grad.w_hv + + lr * d_whv + - lr * params.l1_lambda * np.sign(self.state.w_hv) + - lr * 2 * params.l2_lambda * self.state.w_hv + - lr * params.weight_decay * self.state.w_hv) return self.grad