[bugfix] - fix L1/L2 scaling: move regularisation out of grad_compute into state_adjust
Regularisation terms (L1, L2, weight_decay) were being divided by mini_batch_size in state_adjust along with the CD gradient. The CD gradient is a batch sum so 1/N normalisation is correct; regularisation penalties are per-weight and batch-size independent. Separating them makes l1_lambda/l2_lambda directly interpretable regardless of batch size. Also adds --l1_lambda CLI arg to test_faces_sub_image.py and sets num_epochs=1000. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -14,17 +14,18 @@ STRIDE = 16 # 50% overlap; set equal to PATCH for non-overlapping
|
||||
GRAYSCALE = False # reassigned in __main__ when --grayscale is set
|
||||
N_CH = 1 if GRAYSCALE else 3
|
||||
N_VIS = N_CH * PATCH * PATCH # 1024 grayscale / 3072 colour
|
||||
N_HID = 64
|
||||
N_HID = 128
|
||||
N_IMAGES = 50
|
||||
|
||||
|
||||
class TestModel(Model):
|
||||
def __init__(self, name: str, work_dir: str = '.'):
|
||||
def __init__(self, name: str, work_dir: str = '.', l1_lambda: float = 0.0):
|
||||
super().__init__(name, work_dir)
|
||||
self.unit1 = Entity(
|
||||
(N_VIS, N_HID),
|
||||
EntityParams(do_gaussian_visible=True, do_gaussian_hidden=False),
|
||||
TrainingParams(learning_rate=0.001, momentum=0.9, num_epochs=3000, mini_batch_size=1000)
|
||||
TrainingParams(learning_rate=0.001, momentum=0.9, num_epochs=1000, mini_batch_size=1000,
|
||||
l1_lambda=l1_lambda)
|
||||
)
|
||||
|
||||
def forward(self, x: Mat) -> Mat:
|
||||
@@ -175,6 +176,8 @@ if __name__ == '__main__':
|
||||
help='Train the model (default: false)')
|
||||
ap.add_argument('--grayscale', action='store_true', default=False,
|
||||
help='Use single-channel grayscale patches (default: false)')
|
||||
ap.add_argument('--l1_lambda', type=float, default=0.0,
|
||||
help='L1 regularisation strength (default: 0.0)')
|
||||
args = ap.parse_args()
|
||||
|
||||
if args.grayscale:
|
||||
@@ -185,7 +188,7 @@ if __name__ == '__main__':
|
||||
prj_name = 'faces_sub_image_gray' if args.grayscale else 'faces_sub_image'
|
||||
work_dir = 'results'
|
||||
|
||||
model = TestModel(prj_name, work_dir)
|
||||
model = TestModel(prj_name, work_dir, l1_lambda=args.l1_lambda)
|
||||
model.init(0.001)
|
||||
|
||||
if args.load_model:
|
||||
|
||||
Reference in New Issue
Block a user