[bugfix] - fix L1/L2 scaling: move regularisation out of grad_compute into state_adjust

Regularisation terms (L1, L2, weight_decay) were being divided by mini_batch_size
in state_adjust along with the CD gradient. The CD gradient is a batch sum so
1/N normalisation is correct; regularisation penalties are per-weight and
batch-size independent. Separating them makes l1_lambda/l2_lambda directly
interpretable regardless of batch size.

Also adds --l1_lambda CLI arg to test_faces_sub_image.py and sets num_epochs=1000.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-05-30 19:58:44 +02:00
co-authored by Claude Sonnet 4.6
parent 95b114003e
commit d4d3d33931
2 changed files with 19 additions and 13 deletions
+7 -4
View File
@@ -14,17 +14,18 @@ STRIDE = 16 # 50% overlap; set equal to PATCH for non-overlapping
GRAYSCALE = False # reassigned in __main__ when --grayscale is set
N_CH = 1 if GRAYSCALE else 3
N_VIS = N_CH * PATCH * PATCH # 1024 grayscale / 3072 colour
N_HID = 64
N_HID = 128
N_IMAGES = 50
class TestModel(Model):
def __init__(self, name: str, work_dir: str = '.'):
def __init__(self, name: str, work_dir: str = '.', l1_lambda: float = 0.0):
super().__init__(name, work_dir)
self.unit1 = Entity(
(N_VIS, N_HID),
EntityParams(do_gaussian_visible=True, do_gaussian_hidden=False),
TrainingParams(learning_rate=0.001, momentum=0.9, num_epochs=3000, mini_batch_size=1000)
TrainingParams(learning_rate=0.001, momentum=0.9, num_epochs=1000, mini_batch_size=1000,
l1_lambda=l1_lambda)
)
def forward(self, x: Mat) -> Mat:
@@ -175,6 +176,8 @@ if __name__ == '__main__':
help='Train the model (default: false)')
ap.add_argument('--grayscale', action='store_true', default=False,
help='Use single-channel grayscale patches (default: false)')
ap.add_argument('--l1_lambda', type=float, default=0.0,
help='L1 regularisation strength (default: 0.0)')
args = ap.parse_args()
if args.grayscale:
@@ -185,7 +188,7 @@ if __name__ == '__main__':
prj_name = 'faces_sub_image_gray' if args.grayscale else 'faces_sub_image'
work_dir = 'results'
model = TestModel(prj_name, work_dir)
model = TestModel(prj_name, work_dir, l1_lambda=args.l1_lambda)
model.init(0.001)
if args.load_model: