jens/Rbm
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

- train whv and whc separately

Browse Source 
git-svn-id: http://moon:8086/svn/software/trunk/projects/Rbm@758 b431acfa-c32f-4a4a-93f1-934dc6c82436
This commit is contained in:
jens
2022-01-08 13:30:26 +00:00
parent b46e53f207
commit 3272899888
2 changed files with 85 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
source/Rbm.cpp
+81 -48
View File
@@ -53,9 +53,9 @@ Rbm::~Rbm()
void Rbm::weightsInit(double stddev, double mu)
{
uniform(m_whv, stddev, mu);
uniform(m_whc, 0*stddev, mu);
uniform(m_whc, 0.01*stddev, mu);
uniform(m_bhv, stddev, mu);
uniform(m_bhc, 0*stddev, mu);
uniform(m_bhc, 0.01*stddev, mu);
uniform(m_bv, stddev, mu);
uniform(m_bc, stddev, mu);
}
@@ -74,66 +74,59 @@ Json::Value Rbm::toJson() const
return rbm;
}
void Rbm::gibbs(arma::mat &h_probs, arma::mat &v_probs)
void Rbm::gibbs_hv(arma::mat &hv_probs, arma::mat &v_probs)
{
for (int gibbs=0; gibbs < m_params.numGibbs; gibbs++)
{
// Create visible reconstruction (a fantasy...) given hid
if (m_params.gibbsDoSampleHidden)
{
v_probs = prob(h_to_v(sample(h_probs)));
v_probs = prob(h_to_v(sample(hv_probs)));
}
else
{
v_probs = prob(h_to_v(h_probs));
v_probs = prob(h_to_v(hv_probs));
}
// Create hidden representation given v
if (m_params.gibbsDoSampleVisible)
{
h_probs = prob(v_to_h(sample(v_probs)));
hv_probs = prob(v_to_h(sample(v_probs)));
}
else
{
h_probs = prob(v_to_h(v_probs));
hv_probs = prob(v_to_h(v_probs));
}
}
}
void Rbm::weightUpdate(arma::mat const &v_states, arma::mat const &c_states, arma::mat &dwhv, arma::mat &dwhc, arma::mat &dbh, arma::mat &dbv, arma::mat &dbc)
void Rbm::gibbs_hc(arma::mat &hc_probs, arma::mat &c_probs)
{
arma::mat v_probs(v_states);
arma::mat hv_states = v_to_h(v_states);
arma::mat hc_states = c_to_h(c_states);
arma::mat h_states = hv_states + hc_states;
arma::mat h_probs = prob(h_states);
// Sample hidden
if (m_params.doRaoBlackwell)
for (int gibbs=0; gibbs < m_params.numGibbs; gibbs++)
{
h_states = h_probs;
}
else
{
h_states = sample(h_probs);
}
// Update weights (positive phase)
dwhv = v_states.t() * h_states;
dbv = sum(v_states, 0);
dbh = sum(h_states, 0);
// Create visible reconstruction (a fantasy...) given hid
if (m_params.gibbsDoSampleHidden)
{
c_probs = prob(h_to_c(sample(hc_probs)));
}
else
{
c_probs = prob(h_to_c(hc_probs));
}
gibbs(h_probs, v_probs);
// Update weights (negative phase)
dwhv -= v_probs.t() * h_probs;
dbv -= sum(v_probs, 0);
dbh -= sum(h_probs, 0);
// Create hidden representation given v
if (m_params.gibbsDoSampleVisible)
{
hc_probs = prob(c_to_h(sample(c_probs)));
}
else
{
hc_probs = prob(c_to_h(c_probs));
}
}
}
void Rbm::weightUpdate(arma::mat const &v_states, arma::mat &dw, arma::mat &dbh, arma::mat &dbv)
void Rbm::weightUpdate_hv(arma::mat const &v_states, arma::mat &dw, arma::mat &dbhv, arma::mat &dbv)
{
arma::mat v_probs(v_states);
arma::mat h_states = v_to_h(v_states);
@@ -152,14 +145,43 @@ void Rbm::weightUpdate(arma::mat const &v_states, arma::mat &dw, arma::mat &dbh,
// Update weights (positive phase)
dw = v_states.t() * h_states;
dbv = sum(v_states, 0);
dbh = sum(h_states, 0);
dbhv = sum(h_states, 0);
gibbs(h_probs, v_probs);
gibbs_hv(h_probs, v_probs);
// Update weights (negative phase)
dw -= v_probs.t() * h_probs;
dbv -= sum(v_probs, 0);
dbh -= sum(h_probs, 0);
dbhv -= sum(h_probs, 0);
}
void Rbm::weightUpdate_hc(arma::mat const &c_states, arma::mat &dw, arma::mat &dbhc, arma::mat &dbc)
{
arma::mat c_probs(c_states);
arma::mat h_states = c_to_h(c_states);
arma::mat h_probs = prob(h_states);
// Sample hidden
if (m_params.doRaoBlackwell)
{
h_states = h_probs;
}
else
{
h_states = sample(h_probs);
}
// Update weights (positive phase)
dw = c_states.t() * h_states;
dbc = sum(c_states, 0);
dbhc = sum(h_states, 0);
gibbs_hc(h_probs, c_probs);
// Update weights (negative phase)
dw -= c_probs.t() * h_probs;
dbc -= sum(c_probs, 0);
dbhc -= sum(h_probs, 0);
}
void Rbm::train(const arma::mat& batch, IListener* pListener)
@@ -171,13 +193,17 @@ void Rbm::train(const arma::mat& batch, IListener* pListener)
int batchRowIndex = 0;
arma::mat grad_bias_v(arma::zeros(1, m_bv.n_cols));
arma::mat grad_bias_h(arma::zeros(1, m_bhv.n_cols));
arma::mat grad_bias_c(arma::zeros(1, m_bc.n_cols));
arma::mat grad_bias_hv(arma::zeros(1, m_bhv.n_cols));
arma::mat grad_bias_hc(arma::zeros(1, m_bhc.n_cols));
arma::mat grad_weight_hv(arma::zeros(m_whv.n_rows, m_whv.n_cols));
arma::mat grad_weight_hc(arma::zeros(m_whc.n_rows, m_whc.n_cols));
arma::mat momentum_whv = arma::zeros(m_whv.n_rows, m_whv.n_cols);
arma::mat momentum_whc = arma::zeros(m_whc.n_rows, m_whc.n_cols);
arma::mat momentum_bias_v(arma::zeros(1, m_bv.n_cols));
arma::mat momentum_bias_h(arma::zeros(1, m_bhv.n_cols));
arma::mat momentum_bias_c(arma::zeros(1, m_bc.n_cols));
arma::mat momentum_bias_hv(arma::zeros(1, m_bhv.n_cols));
arma::mat momentum_bias_hc(arma::zeros(1, m_bhc.n_cols));
arma::mat penalty_weights = arma::zeros(m_whv.n_rows, m_whv.n_cols);
int trainingSizeRemain = batch.n_rows;
@@ -225,9 +251,9 @@ void Rbm::train(const arma::mat& batch, IListener* pListener)
// Update weights (positive phase)
grad_weight = v_states.t() * hid_states;
grad_bias_v = sum(v_states, 0);
grad_bias_h = sum(hid_states, 0);
grad_bias_hv = sum(hid_states, 0);
for (int gibbs=0; gibbs < m_params.numGibbs; gibbs++)
for (int gibbs_hv=0; gibbs_hv < m_params.numGibbs; gibbs_hv++)
{
// Create visible reconstruction (a fantasy...) given hid
if (m_params.gibbsDoSampleHidden)
@@ -255,15 +281,16 @@ void Rbm::train(const arma::mat& batch, IListener* pListener)
// Update weights (negative phase)
grad_weight -= v_probs.t() * h_probs;
grad_bias_v -= sum(v_probs, 0);
grad_bias_h -= sum(h_probs, 0);
grad_bias_hv -= sum(h_probs, 0);
#else
if (m_bc.n_cols == 0)
{
weightUpdate(v_states, grad_weight_hv, grad_bias_h, grad_bias_v);
weightUpdate_hv(v_states, grad_weight_hv, grad_bias_hv, grad_bias_v);
}
else
{
weightUpdate(v_states, c_states, grad_weight_hv, grad_weight_hc, grad_bias_h, grad_bias_v, grad_bias_c);
weightUpdate_hv(v_states, grad_weight_hv, grad_bias_hv, grad_bias_v);
weightUpdate_hc(c_states, grad_weight_hc, grad_bias_hc, grad_bias_c);
}
#endif
@@ -272,12 +299,18 @@ void Rbm::train(const arma::mat& batch, IListener* pListener)
status.L1 = accu(abs(m_whv));
status.L2 = accu(m_whv % m_whv);
momentum_bias_v = m_params.momentum*momentum_bias_v + grad_bias_v;
momentum_bias_h = m_params.momentum*momentum_bias_h + grad_bias_h;
momentum_bias_hv = m_params.momentum*momentum_bias_hv + grad_bias_hv;
momentum_bias_c = m_params.momentum*momentum_bias_c + grad_bias_c;
momentum_bias_hc = m_params.momentum*momentum_bias_hc + grad_bias_hc;
momentum_whv = m_params.momentum*momentum_whv + grad_weight_hv - status.L2*penalty_weights;
momentum_whc = m_params.momentum*momentum_whc + grad_weight_hc;
m_bv += learning_rate*momentum_bias_v;
m_bhv += learning_rate*momentum_bias_h;
m_bc += learning_rate*momentum_bias_c;
m_bhv += learning_rate*momentum_bias_hv;
m_bhc += learning_rate*momentum_bias_hc;
m_whv += learning_rate*momentum_whv;
m_whc += learning_rate*momentum_whc;
progress += dProgress*miniBatchSizeActual;
status.progress = (int)(progress + 0.5);
source/Rbm.hpp
+4 -3
View File
@@ -142,10 +142,11 @@ public:
private:
Params m_params;
arma::mat sample(arma::mat const &src);
void weightUpdate(arma::mat const &v_states, arma::mat &dw, arma::mat &dbh, arma::mat &dbv);
void weightUpdate(arma::mat const &v_states, arma::mat const &c_states, arma::mat &dwhv, arma::mat &dwhc, arma::mat &dbh, arma::mat &dbv, arma::mat &dbc);
void weightUpdate_hv(arma::mat const &v_states, arma::mat &dwhv, arma::mat &dbhv, arma::mat &dbv);
void weightUpdate_hc(arma::mat const &c_states, arma::mat &dwhc, arma::mat &dbhc, arma::mat &dbc);
void uniform(arma::mat &srcDst, double stdDev=1.0, double mu=0.5);
void gibbs(arma::mat &h_states, arma::mat &v_states);
void gibbs_hv(arma::mat &hv_states, arma::mat &v_states);
void gibbs_hc(arma::mat &hc_states, arma::mat &c_states);
protected:
arma::mat m_whv;