fixed dividing l2-norm by length of vector

2026-01-06 18:17:45 +01:00
parent 542b8ae9b0
commit 7d3ab02e26
2 changed files with 2 additions and 2 deletions
@@ -104,7 +104,7 @@ class Entity:
 		self.grad.b_v = momentum * self.grad.b_v + learning_rate * d_bv
 		self.grad.b_h = momentum * self.grad.b_h + learning_rate * d_bh
 		# compute L2 term and penalize cost function (d_whv)
-		l2_norm = np.sum(np.square(self.state.w_hv))/self.state.w_hv.shape[1]
+		l2_norm = 0.5*np.sum(np.square(self.state.w_hv))
 		l2_term = l2_lambda*l2_norm*self.state.w_hv
 		self.grad.w_hv = momentum * self.grad.w_hv + learning_rate * (d_whv-l2_term) - learning_rate * weight_decay * self.state.w_hv