diff --git a/src/rbm/entity.py b/src/rbm/entity.py index b9c98ea..d48f8e1 100644 --- a/src/rbm/entity.py +++ b/src/rbm/entity.py @@ -97,15 +97,11 @@ class Entity: self.grad.b_v = params.momentum * self.grad.b_v + lr * d_bv self.grad.b_h = params.momentum * self.grad.b_h + lr * d_bh - # compute L1 term and penalize cost function (d_whv) - l1_norm = np.sum(np.abs(self.state.w_hv)) - l1_term = params.l1_lambda*l1_norm - - # compute L2 term and penalize cost function (d_whv) - l2_norm = np.sum(np.square(self.state.w_hv)) - l2_term = params.l2_lambda*l2_norm - - self.grad.w_hv = params.momentum * self.grad.w_hv + lr * (d_whv - self.state.w_hv*(l1_term+l2_term)) - lr * params.weight_decay * self.state.w_hv + self.grad.w_hv = (params.momentum * self.grad.w_hv + + lr * d_whv + - lr * params.l1_lambda * np.sign(self.state.w_hv) + - lr * 2 * params.l2_lambda * self.state.w_hv + - lr * params.weight_decay * self.state.w_hv) return self.grad