From 866a0349dbe1a7b9257ff18f6fd5bc56508d5d0c Mon Sep 17 00:00:00 2001 From: Jens Ahrensfeld Date: Wed, 22 Jun 2016 19:11:09 +0000 Subject: [PATCH] [RBM] - DrawComponent: use fixed value scaling - Rbm: fixed weight decay - Rbm: fixed sparsity git-svn-id: http://moon:8086/svn/software/trunk/projects/RBM@298 b431acfa-c32f-4a4a-93f1-934dc6c82436 --- Source/DrawComponent.cpp | 39 ++++++++++-------------------- Source/DrawComponent.h | 4 +++- Source/Rbm.hpp | 51 +++++++++++----------------------------- Source/RbmComponent.cpp | 19 +++++++++++++-- 4 files changed, 46 insertions(+), 67 deletions(-) diff --git a/Source/DrawComponent.cpp b/Source/DrawComponent.cpp index bb8a477..0c5b37f 100644 --- a/Source/DrawComponent.cpp +++ b/Source/DrawComponent.cpp @@ -29,13 +29,15 @@ void mylog(const char* format, ...); //[/MiscUserDefs] //============================================================================== -DrawComponent::DrawComponent (int width, int height) +DrawComponent::DrawComponent (int width, int height, float offset, float scale) : m_pListener(nullptr) { //[UserPreSize] m_width = width; m_height = height; + m_offset = offset; + m_scale = scale; m_scaleX = 1.0; m_scaleY = 1.0; m_pG = nullptr; @@ -231,39 +233,22 @@ RowVectorXd& DrawComponent::getData () void DrawComponent::DrawData () { - double a; RowVectorXd temp = m_data; - double min = +1E12; - double max = -1E12; for (int i=0; i < m_width*m_height; i++) { - min = std::min(min, (double)temp[i]); - max = std::max(max, (double)temp[i]); - } - - if (min < 0) - { - for (int i=0; i < m_width*m_height; i++) - { - temp[i] -= min; - } - max -= min; - } - for (int i=0; i < m_width*m_height; i++) - { - temp[i] /= max; + temp[i] = std::min(1.0, (double)temp[i]); + temp[i] = std::max(-1.0, (double)temp[i]); } for (int i=0; i < m_height; i++) - { - for (int j=0; j < m_width; j++) - { - a = std::min(std::max((double)temp[i*m_width + j], 0), 1); - m_pG->setColour(Colour(Colours::white).greyLevel(a)); - m_pG->fillRect(m_scaleX*j, m_scaleY*i, m_scaleX, m_scaleY); - } - } + { + for (int j=0; j < m_width; j++) + { + m_pG->setColour(Colour(Colours::white).greyLevel(m_offset + m_scale*temp[i*m_width + j])); + m_pG->fillRect(m_scaleX*j, m_scaleY*i, m_scaleX, m_scaleY); + } + } repaint(); } diff --git a/Source/DrawComponent.h b/Source/DrawComponent.h index 7b809de..d8c3114 100644 --- a/Source/DrawComponent.h +++ b/Source/DrawComponent.h @@ -49,7 +49,7 @@ class DrawComponent : public Component { public: //============================================================================== - DrawComponent (int width, int height); + DrawComponent (int width, int height, float offset=0.0, float scale=1.0); ~DrawComponent(); //============================================================================== @@ -79,6 +79,8 @@ private: DrawListener *m_pListener; int m_width; int m_height; + float m_offset; + float m_scale; float m_scaleX; float m_scaleY; ScopedPointerm_pG; diff --git a/Source/Rbm.hpp b/Source/Rbm.hpp index 99a136c..6077548 100644 --- a/Source/Rbm.hpp +++ b/Source/Rbm.hpp @@ -27,7 +27,7 @@ public: Params() : m_constantSigma(1.0) , m_sigmaDecay(1.0) - , m_weightDecay(0.01) + , m_weightDecay(0.00001) , m_lambda(1.0) , m_sparsity(0.05) , m_muWeights(0.1) @@ -69,17 +69,6 @@ public: , m_progress(0) { Noise_Init(&m_noise, 0x32727155); - -#if 1 - VectorXd a(4); - a << 1, 2, 3, 4; - VectorXd b(4); - - b.array() = -a.array().exp(); - - cout << b << endl; - -#endif m_variableSigma.fill(m_params.m_constantSigma); updateHiddenBatch(); } @@ -260,7 +249,7 @@ public: m_v.resize(batchSize, m_w.getNumVisible()); MatrixXd h(batchSize, m_w.getNumHidden()); - + MatrixXd dBiasV_curr(MatrixXd::Zero(1, m_w.getNumVisible())); MatrixXd dBiasH_curr(MatrixXd::Zero(1, m_w.getNumHidden())); MatrixXd dW_curr(MatrixXd::Zero(m_w.getNumVisible(), m_w.getNumHidden())); @@ -289,9 +278,9 @@ public: { onProgressChanged(); - // When the hidden units are being driven by data, always use stochastic binary states if (m_params.m_doSampleBatch) { + // When the hidden units are being driven by data, always use stochastic binary states sample(batch_sampled, batch); // Create hidden layer base on sampled training data @@ -310,10 +299,7 @@ public: // Update weights (positive phase) dBiasV_curr = batch.colwise().sum(); - if (!m_params.m_doSparse) - { - dBiasH_curr = h.colwise().sum(); - } + dBiasH_curr = h.colwise().sum(); dW_curr = batch.transpose() * h; for (gibbs=0; gibbs < m_params.m_numGibbs; gibbs++) @@ -348,37 +334,28 @@ public: // Update weights (negative phase) dBiasV_curr -= m_v.colwise().sum(); - if (!m_params.m_doSparse) - { - dBiasH_curr -= h.colwise().sum(); - } + dBiasH_curr -= h.colwise().sum(); dW_curr -= m_v.transpose() * h; m_w.visibleBias() += mu_biasV*(m_params.m_momentum*dBiasV + (1-m_params.m_momentum)*dBiasV_curr); dBiasV = dBiasV_curr; - m_w.weights() += mu_w*(m_params.m_momentum*dW + (1-m_params.m_momentum)*dW_curr - m_params.m_weightDecay*m_w.weights()); - dW = dW_curr; - if (m_params.m_doSparse) { - // Create hidden representation given v - toHiddenBatch(h, batch); - - dBiasH_curr = m_params.m_sparsity * MatrixXd::Ones(dBiasH.rows(), dBiasH.cols()) - h.colwise().mean(); - - m_w.hiddenBias() += m_params.m_muSparsity*(m_params.m_momentum*dBiasH + (1-m_params.m_momentum)*dBiasH_curr); - dBiasH = dBiasH_curr; - -// cout << "Mean(" << m_sparsity << ") = " << (double)sumBiasH.array().mean() << endl; -// cout << sumBiasH << endl; + MatrixXd h1 = h-MatrixXd::Ones(h.rows(), h.cols())*m_params.m_sparsity; + RowVectorXd hm = h1.colwise().mean(); + m_w.hiddenBias() -= m_params.m_muSparsity * hm; } else { m_w.hiddenBias() += mu_biasH*(m_params.m_momentum*dBiasH + (1-m_params.m_momentum)*dBiasH_curr); - dBiasH = dBiasH_curr; } - + dBiasH = dBiasH_curr; + + m_w.weights() -= m_params.m_weightDecay*m_w.weights(); + m_w.weights() += mu_w*(m_params.m_momentum*dW + (1-m_params.m_momentum)*dW_curr); + dW = dW_curr; + if (m_variableSigma[0] > sigmaMin) { m_variableSigma.array() *= m_params.m_sigmaDecay; diff --git a/Source/RbmComponent.cpp b/Source/RbmComponent.cpp index 10d2072..54c020a 100644 --- a/Source/RbmComponent.cpp +++ b/Source/RbmComponent.cpp @@ -44,7 +44,7 @@ RbmComponent::RbmComponent (Weights &weights, MatrixXd const &batch, RbmComponen DrawTraining->setListener(this); addAndMakeVisible (DrawReconstruction = new DrawComponent (vNumX, vNumY)); - addAndMakeVisible (DrawWeights = new DrawComponent (vNumX, vNumY)); + addAndMakeVisible (DrawWeights = new DrawComponent (vNumX, vNumY, 0.5, 0.5)); addAndMakeVisible (DrawVars = new DrawComponent (vNumX, vNumY)); addAndMakeVisible (DrawHidden = new DrawComponent (hNum, 1)); DrawHidden->setListener(this); @@ -55,7 +55,22 @@ RbmComponent::RbmComponent (Weights &weights, MatrixXd const &batch, RbmComponen setSize (430, 130); resized(); - + + MatrixXd m = MatrixXd::Random(4,3); + MatrixXd m2 = MatrixXd::Zero(4,3); + cout << "Rows = " << m.rows() << ", cols = " << m.cols() << endl; + cout << "m = " << endl << m << endl; + cout << "max = " << endl << m.rowwise().maxCoeff() << endl; + cout << "m.colwise().sum() = " << endl << m.colwise().sum() << endl; + + for (MatrixXf::Index i=0; i < m.rows(); i++) + { + MatrixXf::Index j = 8; + cout << "max = " << endl << m.row(i).maxCoeff(&j); + cout << "at row = " << i << ", col = " << j << endl; + m2(i, j) = m(i, j); + } + cout << "m2 = " << endl << m2 << endl; } RbmComponent::~RbmComponent()