jens/Rbm
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

- refactored Rbm

Browse Source 
git-svn-id: http://moon:8086/svn/software/trunk/projects/Rbm@746 b431acfa-c32f-4a4a-93f1-934dc6c82436
This commit is contained in:
jens
2022-01-07 12:40:57 +00:00
parent 7f3535954b
commit 66fe4a018d
2 changed files with 75 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
source/Rbm.cpp
+74 -16
View File
@@ -14,6 +14,8 @@
#include <cassert>
#include "Rbm.hpp"
#define RBM_TRAIN_FLAT 1
Rbm::Rbm(size_t numVisible, size_t numHidden, size_t numContext)
: m_params()
, m_whv(numVisible, numHidden)
@@ -69,6 +71,56 @@ Json::Value Rbm::toJson() const
return rbm;
}
void Rbm::weightUpdate(arma::mat &v_state, arma::mat &dw, arma::mat &dbh, arma::mat &dbv)
{
arma::mat hid_probs = probsLogistic(toHiddenState(v_state));
arma::mat vis_probs(dbv.n_rows, dbv.n_cols);
arma::mat h_state = hid_probs;
// Sample hidden
if (!m_params.doRaoBlackwell)
{
h_state = sample(hid_probs);
}
// Update weights (positive phase)
dw = v_state.t() * h_state;
dbv = sum(v_state, 0);
dbh = sum(h_state, 0);
for (int gibbs=0; gibbs < m_params.numGibbs; gibbs++)
{
// Create visible reconstruction (a fantasy...) given hid
if (m_params.gibbsDoSampleHidden)
{
vis_probs = toVisibleProbs(sample(hid_probs));
}
else
{
vis_probs = toVisibleProbs(hid_probs);
}
// Create hidden representation given v
if (m_params.gibbsDoSampleVisible)
{
h_state = toHiddenState(sample(vis_probs));
}
else
{
h_state = toHiddenState(vis_probs);
}
hid_probs = probsLogistic(h_state);
}
// Update weights (negative phase)
dw -= vis_probs.t() * hid_probs;
dbv -= sum(vis_probs, 0);
dbh -= sum(hid_probs, 0);
}
void Rbm::train(const arma::mat& batch, IListener* pListener)
{
Status status;
@@ -80,11 +132,14 @@ void Rbm::train(const arma::mat& batch, IListener* pListener)
arma::mat grad_bias_v(arma::zeros(1, m_bv.n_cols));
arma::mat grad_bias_h(arma::zeros(1, m_bh.n_cols));
arma::mat grad_weight(arma::zeros(m_whv.n_rows, m_whv.n_cols));
arma::mat momentum_weights = arma::zeros(m_whv.n_rows, m_whv.n_cols);
arma::mat momentum_whv = arma::zeros(m_whv.n_rows, m_whv.n_cols);
arma::mat momentum_bias_v(arma::zeros(1, m_bv.n_cols));
arma::mat momentum_bias_h(arma::zeros(1, m_bh.n_cols));
arma::mat penalty_weights = arma::zeros(m_whv.n_rows, m_whv.n_cols);
arma::mat ctx_state(1, m_bc.n_cols);
arma::mat ctx_probs(1, m_bc.n_cols);
int trainingSizeRemain = batch.n_rows;
bool shouldAbort = false;
@@ -98,10 +153,13 @@ void Rbm::train(const arma::mat& batch, IListener* pListener)
double learning_rate = m_params.learningRate/scaler;
double weight_decay = m_params.weightDecay/scaler;
arma::mat vis_state(miniBatchSizeActual, m_bv.n_cols);
arma::mat vis_probs(miniBatchSizeActual, m_bv.n_cols);
arma::mat hid_state(miniBatchSizeActual, m_bh.n_cols);
#if RBM_TRAIN_FLAT
arma::mat hid_probs(miniBatchSizeActual, m_bh.n_cols);
arma::mat vis_probs(miniBatchSizeActual, m_bv.n_cols);
#endif
arma::mat hid_state(miniBatchSizeActual, m_bh.n_cols);
arma::mat vis_state(miniBatchSizeActual, m_bv.n_cols);
// Create hidden layer base on training data
if (m_params.doSampleBatch)
@@ -113,13 +171,12 @@ void Rbm::train(const arma::mat& batch, IListener* pListener)
{
vis_state = miniBatch;
}
for (int epoch=0; epoch < m_params.numEpochs; epoch++)
{
hid_probs = probsLogistic(toHiddenState(vis_state));
#if RBM_TRAIN_FLAT
// Sample hidden
hid_probs = probsLogistic(toHiddenState(vis_state));
if (m_params.doRaoBlackwell)
{
hid_state = hid_probs;
@@ -128,7 +185,7 @@ void Rbm::train(const arma::mat& batch, IListener* pListener)
{
hid_state = sample(hid_probs);
}
// Update weights (positive phase)
grad_weight = vis_state.t() * hid_state;
grad_bias_v = sum(vis_state, 0);
@@ -163,18 +220,21 @@ void Rbm::train(const arma::mat& batch, IListener* pListener)
grad_weight -= vis_probs.t() * hid_probs;
grad_bias_v -= sum(vis_probs, 0);
grad_bias_h -= sum(hid_probs, 0);
#else
weightUpdate(vis_state, grad_weight, grad_bias_h, grad_bias_v);
#endif
penalty_weights = weight_decay*arma::sign(m_whv);
status.L1 = accu(abs(m_whv));
status.L2 = accu(m_whv % m_whv);
momentum_bias_v = m_params.momentum*momentum_bias_v + grad_bias_v;
momentum_bias_h = m_params.momentum*momentum_bias_h + grad_bias_h;
momentum_weights = m_params.momentum*momentum_weights + grad_weight - status.L2*penalty_weights;
momentum_whv = m_params.momentum*momentum_whv + grad_weight - status.L2*penalty_weights;
m_bv += learning_rate*momentum_bias_v;
m_bh += learning_rate*momentum_bias_h;
m_whv += learning_rate*momentum_weights;
m_whv += learning_rate*momentum_whv;
progress += dProgress*miniBatchSizeActual;
status.progress = (int)(progress + 0.5);
@@ -184,7 +244,7 @@ void Rbm::train(const arma::mat& batch, IListener* pListener)
lastProgress = status.progress;
// Calculate error
arma::mat diffErr = miniBatch - vis_probs;
arma::mat diffErr = miniBatch - toVisibleProbs(toHiddenProbs(vis_state));
arma::mat diffErr_squared = diffErr % diffErr;
status.err = accu(diffErr_squared)/diffErr_squared.n_elem;
if (pListener)
@@ -201,9 +261,7 @@ void Rbm::train(const arma::mat& batch, IListener* pListener)
} // number of mini batches
arma::mat hid_probs = toHiddenProbs(batch);
arma::mat vis_probs = toVisibleProbs(hid_probs);
arma::mat diffErr = batch - vis_probs;
arma::mat diffErr = batch - toVisibleProbs(toHiddenProbs(batch));
arma::mat diffErr_squared = diffErr % diffErr;
status.err_total = accu(diffErr_squared)/diffErr_squared.n_elem;