diff --git a/src/rbm/entity.py b/src/rbm/entity.py index d48f8e1..78bef9c 100644 --- a/src/rbm/entity.py +++ b/src/rbm/entity.py @@ -97,19 +97,22 @@ class Entity: self.grad.b_v = params.momentum * self.grad.b_v + lr * d_bv self.grad.b_h = params.momentum * self.grad.b_h + lr * d_bh - self.grad.w_hv = (params.momentum * self.grad.w_hv - + lr * d_whv - - lr * params.l1_lambda * np.sign(self.state.w_hv) - - lr * 2 * params.l2_lambda * self.state.w_hv - - lr * params.weight_decay * self.state.w_hv) + # CD gradient — scaled by 1/mini_batch_size in state_adjust + self.grad.w_hv = params.momentum * self.grad.w_hv + lr * d_whv return self.grad def state_adjust(self, grad: RbmState, k: float = 1.0): - # Adjust state - self.state.b_v += k*grad.b_v - self.state.b_h += k*grad.b_h - self.state.w_hv += k*grad.w_hv + params = self.training_params + lr = params.learning_rate + + self.state.b_v += k * grad.b_v + self.state.b_h += k * grad.b_h + # CD update scaled by 1/N; regularisation applied at full strength (batch-size independent) + self.state.w_hv += (k * grad.w_hv + - lr * params.l1_lambda * np.sign(self.state.w_hv) + - lr * 2 * params.l2_lambda * self.state.w_hv + - lr * params.weight_decay * self.state.w_hv) def forward(self, v: Mat, num_gibbs: int = 0) -> Mat: num_gibbs = self.params.num_gibbs_samples if num_gibbs == 0 else num_gibbs diff --git a/src/tests/test_faces_sub_image.py b/src/tests/test_faces_sub_image.py index 07d2e1a..e578559 100644 --- a/src/tests/test_faces_sub_image.py +++ b/src/tests/test_faces_sub_image.py @@ -14,17 +14,18 @@ STRIDE = 16 # 50% overlap; set equal to PATCH for non-overlapping GRAYSCALE = False # reassigned in __main__ when --grayscale is set N_CH = 1 if GRAYSCALE else 3 N_VIS = N_CH * PATCH * PATCH # 1024 grayscale / 3072 colour -N_HID = 64 +N_HID = 128 N_IMAGES = 50 class TestModel(Model): - def __init__(self, name: str, work_dir: str = '.'): + def __init__(self, name: str, work_dir: str = '.', l1_lambda: float = 0.0): super().__init__(name, work_dir) self.unit1 = Entity( (N_VIS, N_HID), EntityParams(do_gaussian_visible=True, do_gaussian_hidden=False), - TrainingParams(learning_rate=0.001, momentum=0.9, num_epochs=3000, mini_batch_size=1000) + TrainingParams(learning_rate=0.001, momentum=0.9, num_epochs=1000, mini_batch_size=1000, + l1_lambda=l1_lambda) ) def forward(self, x: Mat) -> Mat: @@ -175,6 +176,8 @@ if __name__ == '__main__': help='Train the model (default: false)') ap.add_argument('--grayscale', action='store_true', default=False, help='Use single-channel grayscale patches (default: false)') + ap.add_argument('--l1_lambda', type=float, default=0.0, + help='L1 regularisation strength (default: 0.0)') args = ap.parse_args() if args.grayscale: @@ -185,7 +188,7 @@ if __name__ == '__main__': prj_name = 'faces_sub_image_gray' if args.grayscale else 'faces_sub_image' work_dir = 'results' - model = TestModel(prj_name, work_dir) + model = TestModel(prj_name, work_dir, l1_lambda=args.l1_lambda) model.init(0.001) if args.load_model: