From 0166b986cbb216542b2c26db687f21df2fd9fe1c Mon Sep 17 00:00:00 2001 From: Jens Ahrensfeld Date: Sun, 12 Oct 2014 18:18:09 +0000 Subject: [PATCH] - vectorized logSigmoid() and gaussProb() - added switch RBM_SPARSE - added some experimental expect functions git-svn-id: http://moon:8086/svn/software/trunk/projects/RBM@24 b431acfa-c32f-4a4a-93f1-934dc6c82436 --- Source/DrawComponent.cpp | 4 +- Source/Layer.hpp | 38 +++++++++-------- Source/MainComponent.cpp | 16 ++++---- Source/Rbm.hpp | 88 ++++++++++++++++++++++++++++------------ 4 files changed, 95 insertions(+), 51 deletions(-) diff --git a/Source/DrawComponent.cpp b/Source/DrawComponent.cpp index 2682a41..dc7a3e0 100644 --- a/Source/DrawComponent.cpp +++ b/Source/DrawComponent.cpp @@ -231,11 +231,13 @@ void DrawComponent::setData (const VectorXd& data) { double a; + m_data = data; + for (int i=0; i < m_height; i++) { for (int j=0; j < m_width; j++) { - a = std::min(std::max((double)data[i*m_width + j], 0), 1); + a = std::min(std::max((double)m_data[i*m_width + j], 0), 1); m_pG->setColour(Colour(Colours::white).greyLevel(a)); m_pG->fillRect(m_scaleX*j, m_scaleY*i, m_scaleX, m_scaleY); } diff --git a/Source/Layer.hpp b/Source/Layer.hpp index 7dee41a..5801509 100644 --- a/Source/Layer.hpp +++ b/Source/Layer.hpp @@ -65,29 +65,24 @@ public: m_states.fill(value); } - void probsUpdate(Layer &layer, Weights &weights, double lambda = 1.0, double variance = 1.0) - { - probsUpdateLogistic(layer, weights, lambda, variance); - } - - void probsUpdateLogistic(Layer &layer, Weights &weights, double lambda, double variance) + void probsUpdateLogistic(Layer &layer, Weights &weights, double lambda = 1.0, double variance = 1.0) { uint32_t i; - for (i=0; i < m_numUnits; i++) { - m_probs(i) = logSigmoid(lambda/variance*accum(layer, weights, i)); + m_probs(i) = lambda/variance*accum(layer, weights, i); } + logSigmoid(m_probs); } void probsUpdateGaussian(Layer &layer, Weights &weights, double lambda, double variance) { uint32_t i; - for (i=0; i < m_numUnits; i++) { - m_probs(i) = gaussProb(lambda*accum(layer, weights, i), variance); + m_probs(i) = lambda*accum(layer, weights, i); } + gaussProb(m_probs, variance); } void statesUpdateStochastic() @@ -128,22 +123,29 @@ protected: VectorXd m_states; virtual double accum(Layer &layer, Weights &weights, uint32_t index) = 0; - inline double logSigmoid(double x) const + void logSigmoid(const VectorXd &x) { - return 1./(1 + exp(-x)); + uint32_t i; + + for (i=0; i < m_numUnits; i++) + { + m_probs[i] = 1./(1 + exp(-(double)x[i])); + } } - inline double gaussProb(double x, double var) const + void gaussProb(const VectorXd &x, double var) { double k = 1.0/sqrt(var*2*3.14159265359); - double mu = 0; - double x2 = (x-mu)*(x-mu); - return k*exp(-x2/(2*var)); + uint32_t i; + + for (i=0; i < m_numUnits; i++) + { + double x2 = ((double)x[i]-mu) * ((double)x[i]-mu); + m_probs[i] = k*exp(-x2/(2*var)); + } } - - }; #endif // LAYER_HPP diff --git a/Source/MainComponent.cpp b/Source/MainComponent.cpp index a56e680..a71850a 100644 --- a/Source/MainComponent.cpp +++ b/Source/MainComponent.cpp @@ -414,16 +414,18 @@ void MainComponent::buttonClicked (Button* buttonThatWasClicked) else if (buttonThatWasClicked == reconstructEquButton) { //[UserButtonCode_reconstructEquButton] -- add your button handler code here.. - uint32_t i; - const VectorXd *pV, *pH; + // Draw2->setData(m_pRbm->expectVisible(Draw->getData(), 100)); - pV = (const VectorXd*)&Draw->getData(); + uint32_t i; + VectorXd V, H; + + V = Draw->getData(); for (i=0; i < 100; i++) { - pH = (const VectorXd*)&m_pRbm->toHidden(*pV); - DrawHidden->setData(*pH); - pV = (const VectorXd*)&m_pRbm->toVisible(*pH); - Draw2->setData(*pV); + H = m_pRbm->toHidden(V); + DrawHidden->setData(H); + V = m_pRbm->toVisible(H); + Draw2->setData(V); } //[/UserButtonCode_reconstructEquButton] } diff --git a/Source/Rbm.hpp b/Source/Rbm.hpp index 96ce223..3a38d04 100644 --- a/Source/Rbm.hpp +++ b/Source/Rbm.hpp @@ -36,8 +36,6 @@ public: Rbm(Weights &weights, RbmListener *pListener = nullptr) : m_w(weights) , m_pListener(pListener) - , m_tv(weights.getNumVisible()) - , m_th(weights.getNumHidden()) , m_progress(0) { Noise_Init(&m_noise, 0x32727155); @@ -65,7 +63,7 @@ public: { m_w.hiddenBias().array() += mu*h.states().array(); } - +//#define RBM_SPARSE void train(LayerArray &vt, uint32_t numEpochs, double mu, uint32_t numGibbs = 1, bool useExpectations = false, bool doRaoBlackwell = false, bool useProbsForHiddenReconstruction = false, bool doRobbinsMonro = false) { uint32_t t, i; @@ -81,9 +79,15 @@ public: double dProgress = 1.0/numEpochs; m_progress = 0; +#ifdef RBM_SPARSE + const double lambda = 0.05; + const double variance = 0.4; + const double penalty = 0.05; +#else const double lambda = 1.0; const double variance = 1.0; const double penalty = 0.0; +#endif if (useExpectations) { @@ -95,7 +99,7 @@ public: for (i=0; i < vt.getSize(); i++) { // Create hidden layer base on training data - ht[i].probsUpdate(vt[i], w, lambda, variance); + ht[i].probsUpdateLogistic(vt[i], w, lambda, variance); } } @@ -104,7 +108,7 @@ public: for (i=0; i < vt.getSize(); i++) { t = (uint32_t)(0.5 + (vt.getSize()-1)*Noise_Uniform(&m_noise, 0.5)); - h.probsUpdate(vt[t], w, lambda, variance); + h.probsUpdateLogistic(vt[t], w, lambda, variance); // Create hidden layer base on training data if (doRobbinsMonro) @@ -134,7 +138,12 @@ public: pH->statesUpdateStochastic(); // Create visible reconstruction (a fantasy...) - v.probsUpdate(*pH, w, lambda, variance); +#ifdef RBM_SPARSE + v.probsUpdateGaussian(*pH, w, lambda, variance); +#else + v.probsUpdateLogistic(*pH, w); +#endif + if (useProbsForHiddenReconstruction) { v.states() = v.probs(); @@ -145,7 +154,7 @@ public: } // Create hidden reconstruction - pH->probsUpdate(v, w, lambda, variance); + pH->probsUpdateLogistic(v, w, lambda, variance); } // Update weights (negative phase) if (doRaoBlackwell) @@ -166,17 +175,18 @@ public: } } -#if 0 +#ifdef RBM_SPARSE { HiddenLayer th(m_w.getNumHidden()); + VectorXd m(th.states()); + m.fill(0); for (i=0; i < vt.getSize(); i++) { - ht[i].probsUpdate(vt[t], w, lambda, variance); - ht[i].statesUpdateStochastic(); - th += ht[i]; + m += expectHidden(vt[i].states(), lambda, variance, 10); } - th *= 1.0/vt.getSize(); - th += -0.02; + m.array() = penalty - m.array(); + m *= 1.0/vt.getSize(); + th.states() = m; hiddenBiasUpdate(th, -mu); } #endif @@ -186,8 +196,6 @@ public: w = m_w; } - getEnergy(v, *pH); - m_progress += dProgress; if (m_pListener) { @@ -233,7 +241,7 @@ public: for (j=0; j < vts.getSize(); j++) { h[j].setNumUnits(m_w.getNumHidden()); - h[j].probsUpdate(vts.getAt(j), m_w); + h[j].probsUpdateLogistic(vts.getAt(j), m_w); // h[j].statesAssignfromProbs(); h[j].statesUpdateStochastic(); } @@ -272,7 +280,7 @@ public: // Reconstruct for (i=0; i < vts.getSize(); i++) { - vts.getAt(i).probsUpdate(h[i], m_w); + vts.getAt(i).probsUpdateLogistic(h[i], m_w); } printf("A fantasy... (v^, t>)\n"); @@ -284,29 +292,59 @@ public: delete [] h; } - const VectorXd& toHidden(const VectorXd& visible) + VectorXd toHidden(const VectorXd& visible, double lambda = 1.0, double variance = 1.0) { + HiddenLayer th(m_w.getNumHidden()); VisibleLayer tv(m_w.getNumVisible(), (const VectorXd*)&visible); - m_th.probsUpdate(tv, m_w); + th.probsUpdateLogistic(tv, m_w, lambda, variance); - return m_th.probs(); + return th.probs(); } - const VectorXd& toVisible(const VectorXd& hidden) + VectorXd toVisible(const VectorXd& hidden, double lambda = 1.0, double variance = 1.0) { HiddenLayer th(m_w.getNumHidden(), (const VectorXd*)&hidden); + VisibleLayer tv(m_w.getNumVisible()); - m_tv.probsUpdate(th, m_w); + tv.probsUpdateLogistic(th, m_w, lambda, variance); - return m_tv.probs(); + return tv.probs(); + } + + VectorXd expectHidden(VectorXd visible, uint32_t numIter, double lambda = 1.0, double variance = 1.0) + { + uint32_t i; + VisibleLayer v(m_w.getNumVisible(), (const VectorXd*)&visible); + HiddenLayer h(m_w.getNumHidden()); + + for (i=0; i < numIter; i++) + { + h.probsUpdateLogistic(v, (Weights&)m_w, lambda, variance); + v.probsUpdateGaussian(h, (Weights&)m_w, lambda, variance); + } + + return h.probs(); + } + + VectorXd expectVisible(VectorXd visible, uint32_t numIter, double lambda = 1.0, double variance = 1.0) + { + uint32_t i; + VisibleLayer v(m_w.getNumVisible(), (const VectorXd*)&visible); + HiddenLayer h(m_w.getNumHidden()); + + for (i=0; i < numIter; i++) + { + h.probsUpdateLogistic(v, (Weights&)m_w, lambda, variance); + v.probsUpdateLogistic(h, (Weights&)m_w, lambda, variance); + } + + return v.probs(); } private: Weights &m_w; RbmListener *m_pListener; - VisibleLayer m_tv; - HiddenLayer m_th; noise_gen_t m_noise; double m_progress;