added L2-Regulation

README.md

@@ -8,4 +8,9 @@ https://github.com/DSL-Lab/GRBM/tree/main
 
 # CRBM 
 ## Paper
-https://www.ee.nthu.edu.tw/hchen/pubs/iee2003.pdf
+https://www.ee.nthu.edu.tw/hchen/pubs/iee2003.pdf
+
+# Regularization
+https://benihime91.github.io/blog/machinelearning/deeplearning/python3.x/tensorflow2.x/2020/10/08/adamW.html
+https://medium.com/analytics-vidhya/l1-vs-l2-regularization-which-is-better-d01068e6658c
+https://jamesmccaffreyblog.com/2019/05/09/the-difference-between-neural-network-l2-regularization-and-weight-decay

src/rbm/entity.py

@@ -1,5 +1,5 @@
 from .state import RbmState
-from .matrix import prob, Mat
+from .matrix import prob, Mat, np
 from enum import Enum
 
 class TrainingParams:
@@ -7,6 +7,7 @@ class TrainingParams:
 				 learning_rate: float = 0.1,
 				 momentum: float = 0.5,
 				 weight_decay: float = 0.0,
+				 l2_lambda: float = 0.0,
 				 num_epochs: int = 1000,
 				 num_gibbs_samples: int = 1,
 				 mini_batch_size: int = 0,
@@ -19,6 +20,7 @@ class TrainingParams:
 		self.learning_rate = learning_rate
 		self.momentum = momentum
 		self.weight_decay = weight_decay
+		self.l2_lambda = l2_lambda
 		self.num_epochs = num_epochs
 		self.num_gibbs_samples = num_gibbs_samples
 		self.mini_batch_size = mini_batch_size
@@ -33,6 +35,7 @@ class TrainingParams:
 		obj.learning_rate = params["learningRate"]
 		obj.momentum = params["momentum"]
 		obj.weight_decay = params["weightDecay"]
+		obj.l2_lambda = params["l2_lambda"]
 		obj.num_epochs = params["numEpochs"]
 		obj.num_gibbs_samples = params["numGibbs"]
 		obj.mini_batch_size = params["miniBatchSize"]
@@ -96,11 +99,14 @@ class Entity:
 	def grad_zero(self):
 		self.grad = RbmState.from_layer_params(self.shape)
 
-	def grad_compute(self, d_bv: Mat, d_bh: Mat, d_whv: Mat, learning_rate: float, momentum: float, weight_decay: float):
+	def grad_compute(self, d_bv: Mat, d_bh: Mat, d_whv: Mat, learning_rate: float, momentum: float, weight_decay: float, l2_lambda: float):
 		# Compute gradient
-		self.grad.b_v = (momentum * self.grad.b_v + learning_rate * d_bv)
+		self.grad.b_v = momentum * self.grad.b_v + learning_rate * d_bv
-		self.grad.b_h = (momentum * self.grad.b_h + learning_rate * d_bh)
+		self.grad.b_h = momentum * self.grad.b_h + learning_rate * d_bh
-		self.grad.w_hv = (momentum * self.grad.w_hv + learning_rate * d_whv - learning_rate * weight_decay * self.state.w_hv)
+		# compute L2 term and penalize cost function (d_whv)
+		l2_norm = np.sum(np.square(self.state.w_hv))/self.state.w_hv.shape[1]
+		l2_term = l2_lambda*l2_norm*self.state.w_hv
+		self.grad.w_hv = momentum * self.grad.w_hv + learning_rate * (d_whv-l2_term) - learning_rate * weight_decay * self.state.w_hv
 
 		return self.grad
 

src/rbm/status.py

@@ -1,4 +1,5 @@
 from .entity import Entity
+from .matrix import np
 
 class Status:
 	def __init__(self, update_interval=10):
@@ -16,6 +17,9 @@ class Status:
 			else:
 				print(f"{entity.name}: {key}              : {value}{unit}")
 
+		l2_norm = np.sum(np.square(entity.state.w_hv))/entity.state.w_hv.shape[1]
+		print(f"{entity.name}: l2_norm            : {l2_norm}")
+
 	def on_change(self, entity: Entity, status: dict|None=None) -> bool:
 		if status is None:
 			self.progress = -1

src/rbm/train.py

@@ -225,7 +225,7 @@ def train(entity: Entity, batch: Mat, status: Status):
 			dwhv, dbv, dbh = cd_func(entity, mini_batch)
 
 			# Adjust weight and biases
-			grad = entity.grad_compute(dbv, dbh, dwhv, learning_rate=params.learning_rate/batch.shape[0], momentum=params.momentum, weight_decay=params.weight_decay)
+			grad = entity.grad_compute(dbv, dbh, dwhv, learning_rate=params.learning_rate/batch.shape[0], momentum=params.momentum, weight_decay=params.weight_decay, l2_lambda=params.l2_lambda)
 			entity.state_adjust(grad)
 
 			# check if status update is needed

src/tests/test_norbs.py

@@ -16,7 +16,7 @@ class TestModel(Model):
 		else:
 			# Hidden binary
 			self.unit1 = Entity((96 * 96, 333), EntityParams(do_gaussian_visible=True, do_gaussian_hidden=False),
-								TrainingParams(learning_rate=0.001, momentum=0.9, num_epochs=1000))
+								TrainingParams(learning_rate=0.001, momentum=0.9, num_epochs=1000, l2_lambda=0.4))
 
 	def forward(self, x: Mat):
 		x = self.unit1.forward(x)
@@ -38,7 +38,7 @@ if __name__ == "__main__":
 	model.init(0.1)
 
 	# load state
-	model.load()
+#	model.load()
 
 	# Load train data
 	train_batch = read_armadillo(os.path.join(prj_root, f"{prj_name}.training.dat"))