From 93fa8c64cebbbc265ab12f34369821563ba33128 Mon Sep 17 00:00:00 2001 From: Jens Ahrensfeld Date: Tue, 28 Oct 2014 21:12:03 +0000 Subject: [PATCH] - bugfix: weightDecay needs to be multiplied by mu_weights - Biases are initialized with zero - developing full matrix calculation in train2() git-svn-id: http://moon:8086/svn/software/trunk/projects/RBM@43 b431acfa-c32f-4a4a-93f1-934dc6c82436 --- Source/MainComponent.cpp | 1 + Source/Rbm.hpp | 147 ++++++++++++++++++++++++++++++++++++++- Source/Weights.hpp | 4 +- 3 files changed, 149 insertions(+), 3 deletions(-) diff --git a/Source/MainComponent.cpp b/Source/MainComponent.cpp index 67717f1..362465e 100644 --- a/Source/MainComponent.cpp +++ b/Source/MainComponent.cpp @@ -863,6 +863,7 @@ void MainComponent::run() { trainButton->setEnabled(false); m_pRbm->train(m_layers, numEpochslabel->getText().getIntValue()); +// m_pRbm->train2(m_layers, numEpochslabel->getText().getIntValue(), 100); trainButton->setEnabled(true); } diff --git a/Source/Rbm.hpp b/Source/Rbm.hpp index e7e8fb0..c250b6d 100644 --- a/Source/Rbm.hpp +++ b/Source/Rbm.hpp @@ -173,7 +173,7 @@ public: } // TrainingSize - deltaWeights = m_momentum*deltaWeights + m_muWeights*kTrain*sumWeights - m_weightDecay*m_w.weights(); + deltaWeights = m_momentum*deltaWeights + m_muWeights*(kTrain*sumWeights - m_weightDecay*m_w.weights()); m_w.weights() += deltaWeights; deltaBiasV = m_momentum*deltaBiasV + m_muWeights*kTrain*sumBiasV; @@ -218,6 +218,151 @@ public: } // Number of epochs } + MatrixXd sample(const MatrixXd &src) + { + uint32_t i; + MatrixXd res(src); + + for (i=0; i < src.array().size(); i++) + { + res.array()(i) = (double) src.array()(i) > Noise_Uniform(&m_noise); + } + + return res; + } + + void train2(const LayerArray &vt, uint32_t numEpochs, uint32_t batchSize, double sigmaMin = 0.05) + { + uint32_t t, i; + uint32_t epoch; + uint32_t gibbs; + double sigma = m_sigma; + double dProgress = 1.0/numEpochs; + double kTrain = 1.0/vt.getSize(); + + if (batchSize > vt.getSize()) + batchSize = vt.getSize(); + + MatrixXd vp(m_w.getNumVisible(), batchSize); + MatrixXd vs(m_w.getNumVisible(), batchSize); + MatrixXd hp(m_w.getNumHidden(), batchSize); + MatrixXd hs(m_w.getNumHidden(), batchSize); + MatrixXd batch(m_w.getNumVisible(), batchSize); + + VectorXd sumBiasV(m_w.getNumVisible()); + VectorXd sumBiasH(m_w.getNumHidden()); + MatrixXd sumWeights(m_w.getNumVisible(), m_w.getNumHidden()); + + VectorXd deltaBiasV(m_w.getNumVisible()); + VectorXd deltaBiasH(m_w.getNumHidden()); + MatrixXd deltaWeights(m_w.getNumVisible(), m_w.getNumHidden()); + + m_progress = 0; + m_doCancel = false; + + for (i=0; i < batchSize; i++) + { + t = (uint32_t)(0.5 + (vt.getSize()-1)*Noise_Uniform(&m_noise)); + batch.col(i) = vt[t].states(); + } + + for (epoch=0; epoch < numEpochs; epoch++) + { + if (m_doCancel) + { + m_doCancel = false; + break; + } + + sumWeights.fill(0); + sumBiasV.fill(0); + sumBiasH.fill(0); + for (i=0; i < batchSize; i++) + { + vs = batch.col(i); + + // h.probsUpdateLogistic(vt[t], m_w, m_lambda, sigma); + hp = -vs.transpose() * m_w.weights(); + hp.array() = hp.array().exp(); + hp.array() += 1; + hp.array() = 1.0/hp.array(); + + // Create hidden layer base on training data + hs = sample(hp); + + // Update weights (positive phase) + sumBiasV += vp.colwise().sum(); + if (m_doRaoBlackwell) + { + sumWeights += vs * hp.transpose(); + sumBiasH += hp.colwise().sum(); + } + else + { + sumWeights += vs * hs.transpose(); + sumBiasH += hs.colwise().sum(); + } + + for (gibbs=0; gibbs < m_numGibbs; gibbs++) + { + // Create visible reconstruction (a fantasy...) + if (m_useProbsForHiddenReconstruction) + { + vp = -hs * m_w.weights().transpose(); + vp.array() = vp.array().exp(); + vp.array() += 1; + vp.array() = 1.0/vp.array(); + } + else + { + vs = sample(vp); + } + + // Create hidden reconstruction + hp = -vs.transpose() * m_w.weights(); + hp.array() = hp.array().exp(); + hp.array() += 1; + hp.array() = 1.0/hp.array(); + } + + // Update weights (negative phase) + sumBiasV -= vp.colwise().sum(); + if (m_doRaoBlackwell) + { + sumWeights -= vs * hp.transpose(); + sumBiasH -= hp.colwise().sum(); + } + else + { + sumWeights -= vs * hs.transpose(); + sumBiasH -= hs.colwise().sum(); + } + + } // TrainingSize + + deltaWeights = m_momentum*deltaWeights + m_muWeights*kTrain*sumWeights - m_weightDecay*m_w.weights(); + m_w.weights() += deltaWeights; + + deltaBiasV = m_momentum*deltaBiasV + m_muWeights*kTrain*sumBiasV; + m_w.visibleBias() += deltaBiasV; + + deltaBiasH = m_momentum*deltaBiasH + m_muWeights*kTrain*sumBiasH; + m_w.hiddenBias() += deltaBiasH; + + if (sigma > sigmaMin) + { + sigma *= m_sigmaDecay; + } + + m_progress += dProgress; + if (m_pListener) + { + m_pListener->onEpochTrained(*this); + } + + } // Number of epochs + } + double getProgress() const { return m_progress; diff --git a/Source/Weights.hpp b/Source/Weights.hpp index 6380ff6..08f8c11 100644 --- a/Source/Weights.hpp +++ b/Source/Weights.hpp @@ -72,12 +72,12 @@ public: for (i=0; i < m_numVisible; i++) { - m_bv(i) = kdev*Noise_Uniform(&m_noise); + m_bv(i) = 0; //kdev*Noise_Uniform(&m_noise); } for (j=0; j < m_numHidden; j++) { - m_bh(j) = kdev*Noise_Uniform(&m_noise); + m_bh(j) = 0; //kdev*Noise_Uniform(&m_noise); } for (i=0; i < m_numVisible; i++)