diff --git a/source/MainComponent.cpp b/source/MainComponent.cpp index cdace60..03b4ca2 100644 --- a/source/MainComponent.cpp +++ b/source/MainComponent.cpp @@ -784,7 +784,7 @@ void MainComponent::labelTextChanged (Label* labelThatHasChanged) else if (labelThatHasChanged == weightDecayLabel) { //[UserLabelCode_weightDecayLabel] -- add your label text handling code here.. - m_pLayer->params().weightDecay = labelThatHasChanged->getText().getFloatValue(); + m_pLayer->params().l2Lambda = labelThatHasChanged->getText().getFloatValue(); //[/UserLabelCode_weightDecayLabel] } else if (labelThatHasChanged == momentumLabel) @@ -947,7 +947,7 @@ void MainComponent::updateControls() rbmDoSampleVisibleToggleButton->setToggleState(m_pLayer->params().gibbsDoSampleVisible, dontSendNotification); rbmDoSampleHidden->setToggleState(m_pLayer->params().gibbsDoSampleHidden, dontSendNotification); - weightDecayLabel->setText(String(m_pLayer->params().weightDecay), dontSendNotification); + weightDecayLabel->setText(String(m_pLayer->params().l2Lambda), dontSendNotification); learningRateLabel->setText(String(m_pLayer->params().learningRate), dontSendNotification); momentumLabel->setText(String(m_pLayer->params().momentum), dontSendNotification); numVisibleLabel->setText(String(m_pLayer->numVisibleX()), dontSendNotification ); diff --git a/source/Rbm.cpp b/source/Rbm.cpp index 71d8eaa..946b9f3 100644 --- a/source/Rbm.cpp +++ b/source/Rbm.cpp @@ -367,14 +367,14 @@ void Rbm::train(arma::mat const &batch, IListener* pListener) m_bh += inc_bh; m_whv += inc_whv; - // Weight decay and L1 applied directly to the weights, outside the - // momentum recursion above -- folding either into inc_whv would let - // momentum geometrically amplify its effective strength to roughly + // L2 and L1 applied directly to the weights, outside the momentum + // recursion above -- folding either into inc_whv would let momentum + // geometrically amplify its effective strength to roughly // lambda/(1-momentum), well past the configured value. Neither // touches the biases, matching pyRBM's state_adjust(). - if (m_params.weightDecay != 0.0) + if (m_params.l2Lambda != 0.0) { - m_whv -= m_params.learningRate*m_params.weightDecay*m_whv; + m_whv -= m_params.learningRate*m_params.l2Lambda*m_whv; } if (m_params.l1Lambda != 0.0) { diff --git a/source/Rbm.hpp b/source/Rbm.hpp index f90aa9e..7752086 100644 --- a/source/Rbm.hpp +++ b/source/Rbm.hpp @@ -27,7 +27,7 @@ public: { Params() : learningRate(0.1) - , weightDecay(0.0) + , l2Lambda(0.0) , l1Lambda(0.0) , momentum(0.9) , doGaussianHidden(false) @@ -46,7 +46,7 @@ public: { std::cout << "Exporting Rbm::Params" << std::endl; Json::Value params; - params["weightDecay"] = weightDecay; + params["l2Lambda"] = l2Lambda; params["l1Lambda"] = l1Lambda; params["learningRate"] = learningRate; params["momentum"] = momentum; @@ -66,7 +66,7 @@ public: void fromJson(Json::Value params) { std::cout << "Importing Rbm::Params" << std::endl; - weightDecay = params.get("weightDecay", weightDecay).asDouble(); + l2Lambda = params.get("l2Lambda", l2Lambda).asDouble(); l1Lambda = params.get("l1Lambda", l1Lambda).asDouble(); learningRate = params.get("learningRate", learningRate).asDouble(); momentum = params.get("momentum", momentum).asDouble(); @@ -81,7 +81,7 @@ public: numEpochs = params.get("numEpochs", numEpochs).asUInt(); } - double weightDecay; + double l2Lambda; double l1Lambda; double learningRate; double momentum;