From d4d3d33931e845cc3703f9c1730ac608d621e04e Mon Sep 17 00:00:00 2001 From: Jens Ahrensfeld Date: Sat, 30 May 2026 19:58:44 +0200 Subject: [PATCH] [bugfix] - fix L1/L2 scaling: move regularisation out of grad_compute into state_adjust Regularisation terms (L1, L2, weight_decay) were being divided by mini_batch_size in state_adjust along with the CD gradient. The CD gradient is a batch sum so 1/N normalisation is correct; regularisation penalties are per-weight and batch-size independent. Separating them makes l1_lambda/l2_lambda directly interpretable regardless of batch size. Also adds --l1_lambda CLI arg to test_faces_sub_image.py and sets num_epochs=1000. Co-Authored-By: Claude Sonnet 4.6 --- src/rbm/entity.py | 21 ++++++++++++--------- src/tests/test_faces_sub_image.py | 11 +++++++---- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/rbm/entity.py b/src/rbm/entity.py index d48f8e1..78bef9c 100644 --- a/src/rbm/entity.py +++ b/src/rbm/entity.py @@ -97,19 +97,22 @@ class Entity: self.grad.b_v = params.momentum * self.grad.b_v + lr * d_bv self.grad.b_h = params.momentum * self.grad.b_h + lr * d_bh - self.grad.w_hv = (params.momentum * self.grad.w_hv - + lr * d_whv - - lr * params.l1_lambda * np.sign(self.state.w_hv) - - lr * 2 * params.l2_lambda * self.state.w_hv - - lr * params.weight_decay * self.state.w_hv) + # CD gradient — scaled by 1/mini_batch_size in state_adjust + self.grad.w_hv = params.momentum * self.grad.w_hv + lr * d_whv return self.grad def state_adjust(self, grad: RbmState, k: float = 1.0): - # Adjust state - self.state.b_v += k*grad.b_v - self.state.b_h += k*grad.b_h - self.state.w_hv += k*grad.w_hv + params = self.training_params + lr = params.learning_rate + + self.state.b_v += k * grad.b_v + self.state.b_h += k * grad.b_h + # CD update scaled by 1/N; regularisation applied at full strength (batch-size independent) + self.state.w_hv += (k * grad.w_hv + - lr * params.l1_lambda * np.sign(self.state.w_hv) + - lr * 2 * params.l2_lambda * self.state.w_hv + - lr * params.weight_decay * self.state.w_hv) def forward(self, v: Mat, num_gibbs: int = 0) -> Mat: num_gibbs = self.params.num_gibbs_samples if num_gibbs == 0 else num_gibbs diff --git a/src/tests/test_faces_sub_image.py b/src/tests/test_faces_sub_image.py index 07d2e1a..e578559 100644 --- a/src/tests/test_faces_sub_image.py +++ b/src/tests/test_faces_sub_image.py @@ -14,17 +14,18 @@ STRIDE = 16 # 50% overlap; set equal to PATCH for non-overlapping GRAYSCALE = False # reassigned in __main__ when --grayscale is set N_CH = 1 if GRAYSCALE else 3 N_VIS = N_CH * PATCH * PATCH # 1024 grayscale / 3072 colour -N_HID = 64 +N_HID = 128 N_IMAGES = 50 class TestModel(Model): - def __init__(self, name: str, work_dir: str = '.'): + def __init__(self, name: str, work_dir: str = '.', l1_lambda: float = 0.0): super().__init__(name, work_dir) self.unit1 = Entity( (N_VIS, N_HID), EntityParams(do_gaussian_visible=True, do_gaussian_hidden=False), - TrainingParams(learning_rate=0.001, momentum=0.9, num_epochs=3000, mini_batch_size=1000) + TrainingParams(learning_rate=0.001, momentum=0.9, num_epochs=1000, mini_batch_size=1000, + l1_lambda=l1_lambda) ) def forward(self, x: Mat) -> Mat: @@ -175,6 +176,8 @@ if __name__ == '__main__': help='Train the model (default: false)') ap.add_argument('--grayscale', action='store_true', default=False, help='Use single-channel grayscale patches (default: false)') + ap.add_argument('--l1_lambda', type=float, default=0.0, + help='L1 regularisation strength (default: 0.0)') args = ap.parse_args() if args.grayscale: @@ -185,7 +188,7 @@ if __name__ == '__main__': prj_name = 'faces_sub_image_gray' if args.grayscale else 'faces_sub_image' work_dir = 'results' - model = TestModel(prj_name, work_dir) + model = TestModel(prj_name, work_dir, l1_lambda=args.l1_lambda) model.init(0.001) if args.load_model: