From 3272899888795d6e13bb29e001bd06be8c786b46 Mon Sep 17 00:00:00 2001 From: Jens Ahrensfeld Date: Sat, 8 Jan 2022 13:30:26 +0000 Subject: [PATCH] - train whv and whc separately git-svn-id: http://moon:8086/svn/software/trunk/projects/Rbm@758 b431acfa-c32f-4a4a-93f1-934dc6c82436 --- source/Rbm.cpp | 129 +++++++++++++++++++++++++++++++------------------ source/Rbm.hpp | 7 +-- 2 files changed, 85 insertions(+), 51 deletions(-) diff --git a/source/Rbm.cpp b/source/Rbm.cpp index 3d0f2ea..2419259 100644 --- a/source/Rbm.cpp +++ b/source/Rbm.cpp @@ -53,9 +53,9 @@ Rbm::~Rbm() void Rbm::weightsInit(double stddev, double mu) { uniform(m_whv, stddev, mu); - uniform(m_whc, 0*stddev, mu); + uniform(m_whc, 0.01*stddev, mu); uniform(m_bhv, stddev, mu); - uniform(m_bhc, 0*stddev, mu); + uniform(m_bhc, 0.01*stddev, mu); uniform(m_bv, stddev, mu); uniform(m_bc, stddev, mu); } @@ -74,66 +74,59 @@ Json::Value Rbm::toJson() const return rbm; } -void Rbm::gibbs(arma::mat &h_probs, arma::mat &v_probs) +void Rbm::gibbs_hv(arma::mat &hv_probs, arma::mat &v_probs) { for (int gibbs=0; gibbs < m_params.numGibbs; gibbs++) { // Create visible reconstruction (a fantasy...) given hid if (m_params.gibbsDoSampleHidden) { - v_probs = prob(h_to_v(sample(h_probs))); + v_probs = prob(h_to_v(sample(hv_probs))); } else { - v_probs = prob(h_to_v(h_probs)); + v_probs = prob(h_to_v(hv_probs)); } // Create hidden representation given v if (m_params.gibbsDoSampleVisible) { - h_probs = prob(v_to_h(sample(v_probs))); + hv_probs = prob(v_to_h(sample(v_probs))); } else { - h_probs = prob(v_to_h(v_probs)); + hv_probs = prob(v_to_h(v_probs)); } } } -void Rbm::weightUpdate(arma::mat const &v_states, arma::mat const &c_states, arma::mat &dwhv, arma::mat &dwhc, arma::mat &dbh, arma::mat &dbv, arma::mat &dbc) +void Rbm::gibbs_hc(arma::mat &hc_probs, arma::mat &c_probs) { - arma::mat v_probs(v_states); - arma::mat hv_states = v_to_h(v_states); - arma::mat hc_states = c_to_h(c_states); - - arma::mat h_states = hv_states + hc_states; - arma::mat h_probs = prob(h_states); - - // Sample hidden - if (m_params.doRaoBlackwell) + for (int gibbs=0; gibbs < m_params.numGibbs; gibbs++) { - h_states = h_probs; - } - else - { - h_states = sample(h_probs); - } - - // Update weights (positive phase) - dwhv = v_states.t() * h_states; - dbv = sum(v_states, 0); - dbh = sum(h_states, 0); + // Create visible reconstruction (a fantasy...) given hid + if (m_params.gibbsDoSampleHidden) + { + c_probs = prob(h_to_c(sample(hc_probs))); + } + else + { + c_probs = prob(h_to_c(hc_probs)); + } - gibbs(h_probs, v_probs); - - // Update weights (negative phase) - dwhv -= v_probs.t() * h_probs; - dbv -= sum(v_probs, 0); - dbh -= sum(h_probs, 0); - + // Create hidden representation given v + if (m_params.gibbsDoSampleVisible) + { + hc_probs = prob(c_to_h(sample(c_probs))); + } + else + { + hc_probs = prob(c_to_h(c_probs)); + } + } } -void Rbm::weightUpdate(arma::mat const &v_states, arma::mat &dw, arma::mat &dbh, arma::mat &dbv) +void Rbm::weightUpdate_hv(arma::mat const &v_states, arma::mat &dw, arma::mat &dbhv, arma::mat &dbv) { arma::mat v_probs(v_states); arma::mat h_states = v_to_h(v_states); @@ -152,14 +145,43 @@ void Rbm::weightUpdate(arma::mat const &v_states, arma::mat &dw, arma::mat &dbh, // Update weights (positive phase) dw = v_states.t() * h_states; dbv = sum(v_states, 0); - dbh = sum(h_states, 0); + dbhv = sum(h_states, 0); - gibbs(h_probs, v_probs); + gibbs_hv(h_probs, v_probs); // Update weights (negative phase) dw -= v_probs.t() * h_probs; dbv -= sum(v_probs, 0); - dbh -= sum(h_probs, 0); + dbhv -= sum(h_probs, 0); +} + +void Rbm::weightUpdate_hc(arma::mat const &c_states, arma::mat &dw, arma::mat &dbhc, arma::mat &dbc) +{ + arma::mat c_probs(c_states); + arma::mat h_states = c_to_h(c_states); + arma::mat h_probs = prob(h_states); + + // Sample hidden + if (m_params.doRaoBlackwell) + { + h_states = h_probs; + } + else + { + h_states = sample(h_probs); + } + + // Update weights (positive phase) + dw = c_states.t() * h_states; + dbc = sum(c_states, 0); + dbhc = sum(h_states, 0); + + gibbs_hc(h_probs, c_probs); + + // Update weights (negative phase) + dw -= c_probs.t() * h_probs; + dbc -= sum(c_probs, 0); + dbhc -= sum(h_probs, 0); } void Rbm::train(const arma::mat& batch, IListener* pListener) @@ -171,13 +193,17 @@ void Rbm::train(const arma::mat& batch, IListener* pListener) int batchRowIndex = 0; arma::mat grad_bias_v(arma::zeros(1, m_bv.n_cols)); - arma::mat grad_bias_h(arma::zeros(1, m_bhv.n_cols)); arma::mat grad_bias_c(arma::zeros(1, m_bc.n_cols)); + arma::mat grad_bias_hv(arma::zeros(1, m_bhv.n_cols)); + arma::mat grad_bias_hc(arma::zeros(1, m_bhc.n_cols)); arma::mat grad_weight_hv(arma::zeros(m_whv.n_rows, m_whv.n_cols)); arma::mat grad_weight_hc(arma::zeros(m_whc.n_rows, m_whc.n_cols)); arma::mat momentum_whv = arma::zeros(m_whv.n_rows, m_whv.n_cols); + arma::mat momentum_whc = arma::zeros(m_whc.n_rows, m_whc.n_cols); arma::mat momentum_bias_v(arma::zeros(1, m_bv.n_cols)); - arma::mat momentum_bias_h(arma::zeros(1, m_bhv.n_cols)); + arma::mat momentum_bias_c(arma::zeros(1, m_bc.n_cols)); + arma::mat momentum_bias_hv(arma::zeros(1, m_bhv.n_cols)); + arma::mat momentum_bias_hc(arma::zeros(1, m_bhc.n_cols)); arma::mat penalty_weights = arma::zeros(m_whv.n_rows, m_whv.n_cols); int trainingSizeRemain = batch.n_rows; @@ -225,9 +251,9 @@ void Rbm::train(const arma::mat& batch, IListener* pListener) // Update weights (positive phase) grad_weight = v_states.t() * hid_states; grad_bias_v = sum(v_states, 0); - grad_bias_h = sum(hid_states, 0); + grad_bias_hv = sum(hid_states, 0); - for (int gibbs=0; gibbs < m_params.numGibbs; gibbs++) + for (int gibbs_hv=0; gibbs_hv < m_params.numGibbs; gibbs_hv++) { // Create visible reconstruction (a fantasy...) given hid if (m_params.gibbsDoSampleHidden) @@ -255,15 +281,16 @@ void Rbm::train(const arma::mat& batch, IListener* pListener) // Update weights (negative phase) grad_weight -= v_probs.t() * h_probs; grad_bias_v -= sum(v_probs, 0); - grad_bias_h -= sum(h_probs, 0); + grad_bias_hv -= sum(h_probs, 0); #else if (m_bc.n_cols == 0) { - weightUpdate(v_states, grad_weight_hv, grad_bias_h, grad_bias_v); + weightUpdate_hv(v_states, grad_weight_hv, grad_bias_hv, grad_bias_v); } else { - weightUpdate(v_states, c_states, grad_weight_hv, grad_weight_hc, grad_bias_h, grad_bias_v, grad_bias_c); + weightUpdate_hv(v_states, grad_weight_hv, grad_bias_hv, grad_bias_v); + weightUpdate_hc(c_states, grad_weight_hc, grad_bias_hc, grad_bias_c); } #endif @@ -272,12 +299,18 @@ void Rbm::train(const arma::mat& batch, IListener* pListener) status.L1 = accu(abs(m_whv)); status.L2 = accu(m_whv % m_whv); momentum_bias_v = m_params.momentum*momentum_bias_v + grad_bias_v; - momentum_bias_h = m_params.momentum*momentum_bias_h + grad_bias_h; + momentum_bias_hv = m_params.momentum*momentum_bias_hv + grad_bias_hv; + momentum_bias_c = m_params.momentum*momentum_bias_c + grad_bias_c; + momentum_bias_hc = m_params.momentum*momentum_bias_hc + grad_bias_hc; momentum_whv = m_params.momentum*momentum_whv + grad_weight_hv - status.L2*penalty_weights; + momentum_whc = m_params.momentum*momentum_whc + grad_weight_hc; m_bv += learning_rate*momentum_bias_v; - m_bhv += learning_rate*momentum_bias_h; + m_bc += learning_rate*momentum_bias_c; + m_bhv += learning_rate*momentum_bias_hv; + m_bhc += learning_rate*momentum_bias_hc; m_whv += learning_rate*momentum_whv; + m_whc += learning_rate*momentum_whc; progress += dProgress*miniBatchSizeActual; status.progress = (int)(progress + 0.5); diff --git a/source/Rbm.hpp b/source/Rbm.hpp index d3c1b9d..12b2f28 100644 --- a/source/Rbm.hpp +++ b/source/Rbm.hpp @@ -142,10 +142,11 @@ public: private: Params m_params; arma::mat sample(arma::mat const &src); - void weightUpdate(arma::mat const &v_states, arma::mat &dw, arma::mat &dbh, arma::mat &dbv); - void weightUpdate(arma::mat const &v_states, arma::mat const &c_states, arma::mat &dwhv, arma::mat &dwhc, arma::mat &dbh, arma::mat &dbv, arma::mat &dbc); + void weightUpdate_hv(arma::mat const &v_states, arma::mat &dwhv, arma::mat &dbhv, arma::mat &dbv); + void weightUpdate_hc(arma::mat const &c_states, arma::mat &dwhc, arma::mat &dbhc, arma::mat &dbc); void uniform(arma::mat &srcDst, double stdDev=1.0, double mu=0.5); - void gibbs(arma::mat &h_states, arma::mat &v_states); + void gibbs_hv(arma::mat &hv_states, arma::mat &v_states); + void gibbs_hc(arma::mat &hc_states, arma::mat &c_states); protected: arma::mat m_whv;