diff --git a/source/Rbm.cpp b/source/Rbm.cpp index a8f566e..71d8eaa 100644 --- a/source/Rbm.cpp +++ b/source/Rbm.cpp @@ -367,14 +367,20 @@ void Rbm::train(arma::mat const &batch, IListener* pListener) m_bh += inc_bh; m_whv += inc_whv; - // Weight decay applied directly to the weights, outside the - // momentum recursion above -- folding it into inc_whv would let - // momentum geometrically amplify its effective strength to - // roughly weightDecay/(1-momentum), well past the configured value. + // Weight decay and L1 applied directly to the weights, outside the + // momentum recursion above -- folding either into inc_whv would let + // momentum geometrically amplify its effective strength to roughly + // lambda/(1-momentum), well past the configured value. Neither + // touches the biases, matching pyRBM's state_adjust(). if (m_params.weightDecay != 0.0) { m_whv -= m_params.learningRate*m_params.weightDecay*m_whv; } + if (m_params.l1Lambda != 0.0) + { + // Subgradient of lambda*||W||_1 is lambda*sign(W) (sign(0)=0). + m_whv -= m_params.learningRate*m_params.l1Lambda*arma::sign(m_whv); + } progress += dProgress*miniBatchSizeActual; status.progress = (int)(progress + 0.5); @@ -386,6 +392,7 @@ void Rbm::train(arma::mat const &batch, IListener* pListener) // Calculate error status.err = rms_error_accu(miniBatch - toVisibleProbs(toHiddenProbs(v_states))); + status.L1 = arma::accu(arma::abs(m_whv)); if (pListener) { if(!pListener->onProgress(this, status)) @@ -402,6 +409,7 @@ void Rbm::train(arma::mat const &batch, IListener* pListener) // Update final status status.err_total = rms_error_accu(batch - toVisibleProbs(toHiddenProbs(batch))); + status.L1 = arma::accu(arma::abs(m_whv)); if (pListener) { diff --git a/source/Rbm.hpp b/source/Rbm.hpp index 8f93276..f90aa9e 100644 --- a/source/Rbm.hpp +++ b/source/Rbm.hpp @@ -28,6 +28,7 @@ public: Params() : learningRate(0.1) , weightDecay(0.0) + , l1Lambda(0.0) , momentum(0.9) , doGaussianHidden(false) , doGaussianVisible(false) @@ -46,6 +47,7 @@ public: std::cout << "Exporting Rbm::Params" << std::endl; Json::Value params; params["weightDecay"] = weightDecay; + params["l1Lambda"] = l1Lambda; params["learningRate"] = learningRate; params["momentum"] = momentum; params["doGaussianVisible"] = (int)doGaussianVisible; @@ -65,6 +67,7 @@ public: { std::cout << "Importing Rbm::Params" << std::endl; weightDecay = params.get("weightDecay", weightDecay).asDouble(); + l1Lambda = params.get("l1Lambda", l1Lambda).asDouble(); learningRate = params.get("learningRate", learningRate).asDouble(); momentum = params.get("momentum", momentum).asDouble(); doGaussianVisible = params.get("doGaussianVisible", doGaussianVisible) == 1; @@ -79,6 +82,7 @@ public: } double weightDecay; + double l1Lambda; double learningRate; double momentum; bool doGaussianVisible;