added L2-Regulation

src/rbm/entity.py

@@ -1,5 +1,5 @@
 from .state import RbmState
-from .matrix import prob, Mat
+from .matrix import prob, Mat, np
 from enum import Enum
 
 class TrainingParams:
@@ -7,6 +7,7 @@ class TrainingParams:
 				 learning_rate: float = 0.1,
 				 momentum: float = 0.5,
 				 weight_decay: float = 0.0,
+				 l2_lambda: float = 0.0,
 				 num_epochs: int = 1000,
 				 num_gibbs_samples: int = 1,
 				 mini_batch_size: int = 0,
@@ -19,6 +20,7 @@ class TrainingParams:
 		self.learning_rate = learning_rate
 		self.momentum = momentum
 		self.weight_decay = weight_decay
+		self.l2_lambda = l2_lambda
 		self.num_epochs = num_epochs
 		self.num_gibbs_samples = num_gibbs_samples
 		self.mini_batch_size = mini_batch_size
@@ -33,6 +35,7 @@ class TrainingParams:
 		obj.learning_rate = params["learningRate"]
 		obj.momentum = params["momentum"]
 		obj.weight_decay = params["weightDecay"]
+		obj.l2_lambda = params["l2_lambda"]
 		obj.num_epochs = params["numEpochs"]
 		obj.num_gibbs_samples = params["numGibbs"]
 		obj.mini_batch_size = params["miniBatchSize"]
@@ -96,11 +99,14 @@ class Entity:
 	def grad_zero(self):
 		self.grad = RbmState.from_layer_params(self.shape)
 
-	def grad_compute(self, d_bv: Mat, d_bh: Mat, d_whv: Mat, learning_rate: float, momentum: float, weight_decay: float):
+	def grad_compute(self, d_bv: Mat, d_bh: Mat, d_whv: Mat, learning_rate: float, momentum: float, weight_decay: float, l2_lambda: float):
 		# Compute gradient
-		self.grad.b_v = (momentum * self.grad.b_v + learning_rate * d_bv)
-		self.grad.b_h = (momentum * self.grad.b_h + learning_rate * d_bh)
-		self.grad.w_hv = (momentum * self.grad.w_hv + learning_rate * d_whv - learning_rate * weight_decay * self.state.w_hv)
+		self.grad.b_v = momentum * self.grad.b_v + learning_rate * d_bv
+		self.grad.b_h = momentum * self.grad.b_h + learning_rate * d_bh
+		# compute L2 term and penalize cost function (d_whv)
+		l2_norm = np.sum(np.square(self.state.w_hv))/self.state.w_hv.shape[1]
+		l2_term = l2_lambda*l2_norm*self.state.w_hv
+		self.grad.w_hv = momentum * self.grad.w_hv + learning_rate * (d_whv-l2_term) - learning_rate * weight_decay * self.state.w_hv
 
 		return self.grad