From 66fe4a018dbe93520d6920ad0487f14d2bf2761e Mon Sep 17 00:00:00 2001 From: Jens Ahrensfeld Date: Fri, 7 Jan 2022 12:40:57 +0000 Subject: [PATCH] - refactored Rbm git-svn-id: http://moon:8086/svn/software/trunk/projects/Rbm@746 b431acfa-c32f-4a4a-93f1-934dc6c82436 --- source/Rbm.cpp | 90 +++++++++++++++++++++++++++++++++++++++++--------- source/Rbm.hpp | 1 + 2 files changed, 75 insertions(+), 16 deletions(-) diff --git a/source/Rbm.cpp b/source/Rbm.cpp index e94a9f6..b5d2454 100644 --- a/source/Rbm.cpp +++ b/source/Rbm.cpp @@ -14,6 +14,8 @@ #include #include "Rbm.hpp" +#define RBM_TRAIN_FLAT 1 + Rbm::Rbm(size_t numVisible, size_t numHidden, size_t numContext) : m_params() , m_whv(numVisible, numHidden) @@ -69,6 +71,56 @@ Json::Value Rbm::toJson() const return rbm; } +void Rbm::weightUpdate(arma::mat &v_state, arma::mat &dw, arma::mat &dbh, arma::mat &dbv) +{ + arma::mat hid_probs = probsLogistic(toHiddenState(v_state)); + arma::mat vis_probs(dbv.n_rows, dbv.n_cols); + + arma::mat h_state = hid_probs; + + // Sample hidden + if (!m_params.doRaoBlackwell) + { + h_state = sample(hid_probs); + } + + // Update weights (positive phase) + dw = v_state.t() * h_state; + dbv = sum(v_state, 0); + dbh = sum(h_state, 0); + + for (int gibbs=0; gibbs < m_params.numGibbs; gibbs++) + { + // Create visible reconstruction (a fantasy...) given hid + if (m_params.gibbsDoSampleHidden) + { + vis_probs = toVisibleProbs(sample(hid_probs)); + } + else + { + vis_probs = toVisibleProbs(hid_probs); + } + + // Create hidden representation given v + if (m_params.gibbsDoSampleVisible) + { + h_state = toHiddenState(sample(vis_probs)); + } + else + { + h_state = toHiddenState(vis_probs); + } + + hid_probs = probsLogistic(h_state); + + } + + // Update weights (negative phase) + dw -= vis_probs.t() * hid_probs; + dbv -= sum(vis_probs, 0); + dbh -= sum(hid_probs, 0); +} + void Rbm::train(const arma::mat& batch, IListener* pListener) { Status status; @@ -80,11 +132,14 @@ void Rbm::train(const arma::mat& batch, IListener* pListener) arma::mat grad_bias_v(arma::zeros(1, m_bv.n_cols)); arma::mat grad_bias_h(arma::zeros(1, m_bh.n_cols)); arma::mat grad_weight(arma::zeros(m_whv.n_rows, m_whv.n_cols)); - arma::mat momentum_weights = arma::zeros(m_whv.n_rows, m_whv.n_cols); + arma::mat momentum_whv = arma::zeros(m_whv.n_rows, m_whv.n_cols); arma::mat momentum_bias_v(arma::zeros(1, m_bv.n_cols)); arma::mat momentum_bias_h(arma::zeros(1, m_bh.n_cols)); arma::mat penalty_weights = arma::zeros(m_whv.n_rows, m_whv.n_cols); + arma::mat ctx_state(1, m_bc.n_cols); + arma::mat ctx_probs(1, m_bc.n_cols); + int trainingSizeRemain = batch.n_rows; bool shouldAbort = false; @@ -98,10 +153,13 @@ void Rbm::train(const arma::mat& batch, IListener* pListener) double learning_rate = m_params.learningRate/scaler; double weight_decay = m_params.weightDecay/scaler; - arma::mat vis_state(miniBatchSizeActual, m_bv.n_cols); - arma::mat vis_probs(miniBatchSizeActual, m_bv.n_cols); - arma::mat hid_state(miniBatchSizeActual, m_bh.n_cols); +#if RBM_TRAIN_FLAT arma::mat hid_probs(miniBatchSizeActual, m_bh.n_cols); + arma::mat vis_probs(miniBatchSizeActual, m_bv.n_cols); +#endif + + arma::mat hid_state(miniBatchSizeActual, m_bh.n_cols); + arma::mat vis_state(miniBatchSizeActual, m_bv.n_cols); // Create hidden layer base on training data if (m_params.doSampleBatch) @@ -113,13 +171,12 @@ void Rbm::train(const arma::mat& batch, IListener* pListener) { vis_state = miniBatch; } - + for (int epoch=0; epoch < m_params.numEpochs; epoch++) { - - hid_probs = probsLogistic(toHiddenState(vis_state)); - +#if RBM_TRAIN_FLAT // Sample hidden + hid_probs = probsLogistic(toHiddenState(vis_state)); if (m_params.doRaoBlackwell) { hid_state = hid_probs; @@ -128,7 +185,7 @@ void Rbm::train(const arma::mat& batch, IListener* pListener) { hid_state = sample(hid_probs); } - + // Update weights (positive phase) grad_weight = vis_state.t() * hid_state; grad_bias_v = sum(vis_state, 0); @@ -163,18 +220,21 @@ void Rbm::train(const arma::mat& batch, IListener* pListener) grad_weight -= vis_probs.t() * hid_probs; grad_bias_v -= sum(vis_probs, 0); grad_bias_h -= sum(hid_probs, 0); - +#else + weightUpdate(vis_state, grad_weight, grad_bias_h, grad_bias_v); +#endif + penalty_weights = weight_decay*arma::sign(m_whv); status.L1 = accu(abs(m_whv)); status.L2 = accu(m_whv % m_whv); momentum_bias_v = m_params.momentum*momentum_bias_v + grad_bias_v; momentum_bias_h = m_params.momentum*momentum_bias_h + grad_bias_h; - momentum_weights = m_params.momentum*momentum_weights + grad_weight - status.L2*penalty_weights; + momentum_whv = m_params.momentum*momentum_whv + grad_weight - status.L2*penalty_weights; m_bv += learning_rate*momentum_bias_v; m_bh += learning_rate*momentum_bias_h; - m_whv += learning_rate*momentum_weights; + m_whv += learning_rate*momentum_whv; progress += dProgress*miniBatchSizeActual; status.progress = (int)(progress + 0.5); @@ -184,7 +244,7 @@ void Rbm::train(const arma::mat& batch, IListener* pListener) lastProgress = status.progress; // Calculate error - arma::mat diffErr = miniBatch - vis_probs; + arma::mat diffErr = miniBatch - toVisibleProbs(toHiddenProbs(vis_state)); arma::mat diffErr_squared = diffErr % diffErr; status.err = accu(diffErr_squared)/diffErr_squared.n_elem; if (pListener) @@ -201,9 +261,7 @@ void Rbm::train(const arma::mat& batch, IListener* pListener) } // number of mini batches - arma::mat hid_probs = toHiddenProbs(batch); - arma::mat vis_probs = toVisibleProbs(hid_probs); - arma::mat diffErr = batch - vis_probs; + arma::mat diffErr = batch - toVisibleProbs(toHiddenProbs(batch)); arma::mat diffErr_squared = diffErr % diffErr; status.err_total = accu(diffErr_squared)/diffErr_squared.n_elem; diff --git a/source/Rbm.hpp b/source/Rbm.hpp index 432d07c..68e72fb 100644 --- a/source/Rbm.hpp +++ b/source/Rbm.hpp @@ -141,6 +141,7 @@ private: Params m_params; arma::mat sample(arma::mat const &src); static arma::mat probsLogistic(arma::mat const &src); + void weightUpdate(arma::mat &v_state, arma::mat &dw, arma::mat &dbh, arma::mat &dbv); void uniform(arma::mat &srcDst, double stdDev=1.0, double mu=0.5); protected: