From b46e53f207e852999772e59c4df2bcf27b0c814b Mon Sep 17 00:00:00 2001 From: Jens Ahrensfeld Date: Sat, 8 Jan 2022 11:52:01 +0000 Subject: [PATCH] - prepared for context processing git-svn-id: http://moon:8086/svn/software/trunk/projects/Rbm@757 b431acfa-c32f-4a4a-93f1-934dc6c82436 --- source/Layer.cpp | 10 +++++----- source/Layer.hpp | 2 +- source/Rbm.cpp | 46 ++++++++++++++++++++++++++++------------------ source/Rbm.hpp | 3 ++- 4 files changed, 36 insertions(+), 25 deletions(-) diff --git a/source/Layer.cpp b/source/Layer.cpp index b6dbe1b..eed9b57 100644 --- a/source/Layer.cpp +++ b/source/Layer.cpp @@ -15,7 +15,7 @@ using namespace std; Layer::Layer(const string &name, size_t id, size_t numVisibleX, size_t numVisibleY, size_t numHidden) -: Rbm(numVisibleX*numVisibleY, numHidden) +: Rbm(numVisibleX*numVisibleY, numHidden, numHidden) , next(nullptr) , prev(nullptr) , m_name(name) @@ -86,7 +86,7 @@ bool Layer::loadWeights(const string &prjname) result = fscanf(pFile, "%f", &v); if (result > 0) { - m_bh(i) = v; + m_bhv(i) = v; } } for (i=0; i < numVisible; i++) @@ -108,7 +108,7 @@ bool Layer::loadWeights(const string &prjname) bool Layer::saveWeights(const string &prjname) { - int numHidden = m_bh.n_elem; + int numHidden = m_bhv.n_elem; int numVisible = m_bv.n_elem; string filename = m_weightsFile; if (prjname.size() > 0) @@ -134,7 +134,7 @@ bool Layer::saveWeights(const string &prjname) } for (i=0; i < numHidden; i++) { - fprintf(pFile, "%3.6f\n", m_bh(i)); + fprintf(pFile, "%3.6f\n", m_bhv(i)); } for (i=0; i < numVisible; i++) { @@ -158,7 +158,7 @@ Json::Value Layer::toJson() const layer["weights_file"] = m_weightsFile; layer["numVisibleX"] = (int)m_numVisibleX; layer["numVisibleY"] = (int)m_numVisibleY; - layer["numHidden"] = (int)m_bh.n_elem; + layer["numHidden"] = (int)m_bhv.n_elem; layer["rbm"] = Rbm::toJson(); return layer; diff --git a/source/Layer.hpp b/source/Layer.hpp index 1950c60..3d95386 100644 --- a/source/Layer.hpp +++ b/source/Layer.hpp @@ -64,7 +64,7 @@ public: int numHidden() { - return m_bh.n_elem; + return m_bhv.n_elem; } arma::mat toHiddenProbs(const arma::mat &visible) const diff --git a/source/Rbm.cpp b/source/Rbm.cpp index 2637633..3d0f2ea 100644 --- a/source/Rbm.cpp +++ b/source/Rbm.cpp @@ -20,7 +20,8 @@ Rbm::Rbm(size_t numVisible, size_t numHidden, size_t numContext) : m_params() , m_whv(numVisible, numHidden) , m_whc(numContext, numHidden) -, m_bh(1, numHidden) +, m_bhv(1, numHidden) +, m_bhc(1, numHidden) , m_bv(1, numVisible) , m_bc(1, numContext) { @@ -37,7 +38,8 @@ Rbm::Rbm(const Rbm& orig) : m_params(orig.m_params) , m_whv(orig.m_whv) , m_whc(orig.m_whc) -, m_bh(orig.m_bh) +, m_bhv(orig.m_bhv) +, m_bhc(orig.m_bhc) , m_bv(orig.m_bv) , m_bc(orig.m_bc) { @@ -51,8 +53,9 @@ Rbm::~Rbm() void Rbm::weightsInit(double stddev, double mu) { uniform(m_whv, stddev, mu); - uniform(m_whc, stddev, mu); - uniform(m_bh, stddev, mu); + uniform(m_whc, 0*stddev, mu); + uniform(m_bhv, stddev, mu); + uniform(m_bhc, 0*stddev, mu); uniform(m_bv, stddev, mu); uniform(m_bc, stddev, mu); } @@ -168,16 +171,15 @@ void Rbm::train(const arma::mat& batch, IListener* pListener) int batchRowIndex = 0; arma::mat grad_bias_v(arma::zeros(1, m_bv.n_cols)); - arma::mat grad_bias_h(arma::zeros(1, m_bh.n_cols)); - arma::mat grad_weight(arma::zeros(m_whv.n_rows, m_whv.n_cols)); + arma::mat grad_bias_h(arma::zeros(1, m_bhv.n_cols)); + arma::mat grad_bias_c(arma::zeros(1, m_bc.n_cols)); + arma::mat grad_weight_hv(arma::zeros(m_whv.n_rows, m_whv.n_cols)); + arma::mat grad_weight_hc(arma::zeros(m_whc.n_rows, m_whc.n_cols)); arma::mat momentum_whv = arma::zeros(m_whv.n_rows, m_whv.n_cols); arma::mat momentum_bias_v(arma::zeros(1, m_bv.n_cols)); - arma::mat momentum_bias_h(arma::zeros(1, m_bh.n_cols)); + arma::mat momentum_bias_h(arma::zeros(1, m_bhv.n_cols)); arma::mat penalty_weights = arma::zeros(m_whv.n_rows, m_whv.n_cols); - arma::mat ctx_state(1, m_bc.n_cols); - arma::mat ctx_probs(1, m_bc.n_cols); - int trainingSizeRemain = batch.n_rows; bool shouldAbort = false; @@ -192,11 +194,12 @@ void Rbm::train(const arma::mat& batch, IListener* pListener) double weight_decay = m_params.weightDecay/scaler; #if RBM_TRAIN_FLAT - arma::mat h_probs(miniBatchSizeActual, m_bh.n_cols); + arma::mat h_probs(miniBatchSizeActual, m_bhv.n_cols); arma::mat v_probs(miniBatchSizeActual, m_bv.n_cols); - arma::mat hid_states(miniBatchSizeActual, m_bh.n_cols); + arma::mat hid_states(miniBatchSizeActual, m_bhv.n_cols); #endif arma::mat v_states(miniBatch); + arma::mat c_states(arma::zeros(miniBatchSizeActual, m_bc.n_cols)); // Create hidden layer base on training data if (m_params.doSampleBatch) @@ -254,7 +257,14 @@ void Rbm::train(const arma::mat& batch, IListener* pListener) grad_bias_v -= sum(v_probs, 0); grad_bias_h -= sum(h_probs, 0); #else - weightUpdate(v_states, grad_weight, grad_bias_h, grad_bias_v); + if (m_bc.n_cols == 0) + { + weightUpdate(v_states, grad_weight_hv, grad_bias_h, grad_bias_v); + } + else + { + weightUpdate(v_states, c_states, grad_weight_hv, grad_weight_hc, grad_bias_h, grad_bias_v, grad_bias_c); + } #endif penalty_weights = weight_decay*arma::sign(m_whv); @@ -263,10 +273,10 @@ void Rbm::train(const arma::mat& batch, IListener* pListener) status.L2 = accu(m_whv % m_whv); momentum_bias_v = m_params.momentum*momentum_bias_v + grad_bias_v; momentum_bias_h = m_params.momentum*momentum_bias_h + grad_bias_h; - momentum_whv = m_params.momentum*momentum_whv + grad_weight - status.L2*penalty_weights; + momentum_whv = m_params.momentum*momentum_whv + grad_weight_hv - status.L2*penalty_weights; m_bv += learning_rate*momentum_bias_v; - m_bh += learning_rate*momentum_bias_h; + m_bhv += learning_rate*momentum_bias_h; m_whv += learning_rate*momentum_whv; progress += dProgress*miniBatchSizeActual; @@ -326,7 +336,7 @@ arma::mat Rbm::sample(const arma::mat &src) arma::mat Rbm::v_to_h(const arma::mat &visible) const { - return visible * m_whv + arma::repmat(m_bh, visible.n_rows, 1); + return visible * m_whv + arma::repmat(m_bhv, visible.n_rows, 1); } arma::mat Rbm::h_to_v(const arma::mat &hidden) const @@ -336,7 +346,7 @@ arma::mat Rbm::h_to_v(const arma::mat &hidden) const arma::mat Rbm::c_to_h(const arma::mat &context) const { - return context * m_whc + arma::repmat(m_bh, context.n_rows, 1); + return context * m_whc + arma::repmat(m_bhc, context.n_rows, 1); } arma::mat Rbm::h_to_c(const arma::mat &hidden) const @@ -383,7 +393,7 @@ const arma::mat& Rbm::bv() const const arma::mat& Rbm::bh() const { - return m_bh; + return m_bhv; } diff --git a/source/Rbm.hpp b/source/Rbm.hpp index 9aa7279..d3c1b9d 100644 --- a/source/Rbm.hpp +++ b/source/Rbm.hpp @@ -150,7 +150,8 @@ private: protected: arma::mat m_whv; arma::mat m_whc; - arma::mat m_bh; + arma::mat m_bhv; + arma::mat m_bhc; arma::mat m_bv; arma::mat m_bc;