jens/Rbm
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

- prepared for context processing

Browse Source 
git-svn-id: http://moon:8086/svn/software/trunk/projects/Rbm@757 b431acfa-c32f-4a4a-93f1-934dc6c82436
This commit is contained in:
jens
2022-01-08 11:52:01 +00:00
parent 1698ce0a1b
commit b46e53f207
4 changed files with 36 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
source/Layer.cpp
+5 -5
View File
@@ -15,7 +15,7 @@
using namespace std;
Layer::Layer(const string &name, size_t id, size_t numVisibleX, size_t numVisibleY, size_t numHidden)
: Rbm(numVisibleX*numVisibleY, numHidden)
: Rbm(numVisibleX*numVisibleY, numHidden, numHidden)
, next(nullptr)
, prev(nullptr)
, m_name(name)
@@ -86,7 +86,7 @@ bool Layer::loadWeights(const string &prjname)
result = fscanf(pFile, "%f", &v);
if (result > 0)
{
m_bh(i) = v;
m_bhv(i) = v;
}
}
for (i=0; i < numVisible; i++)
@@ -108,7 +108,7 @@ bool Layer::loadWeights(const string &prjname)
bool Layer::saveWeights(const string &prjname)
{
int numHidden = m_bh.n_elem;
int numHidden = m_bhv.n_elem;
int numVisible = m_bv.n_elem;
string filename = m_weightsFile;
if (prjname.size() > 0)
@@ -134,7 +134,7 @@ bool Layer::saveWeights(const string &prjname)
}
for (i=0; i < numHidden; i++)
{
fprintf(pFile, "%3.6f\n", m_bh(i));
fprintf(pFile, "%3.6f\n", m_bhv(i));
}
for (i=0; i < numVisible; i++)
{
@@ -158,7 +158,7 @@ Json::Value Layer::toJson() const
layer["weights_file"] = m_weightsFile;
layer["numVisibleX"] = (int)m_numVisibleX;
layer["numVisibleY"] = (int)m_numVisibleY;
layer["numHidden"] = (int)m_bh.n_elem;
layer["numHidden"] = (int)m_bhv.n_elem;
layer["rbm"] = Rbm::toJson();
return layer;
source/Layer.hpp
+1 -1
View File
@@ -64,7 +64,7 @@ public:
int numHidden()
{
return m_bh.n_elem;
return m_bhv.n_elem;
}
arma::mat toHiddenProbs(const arma::mat &visible) const
source/Rbm.cpp
+28 -18
View File
@@ -20,7 +20,8 @@ Rbm::Rbm(size_t numVisible, size_t numHidden, size_t numContext)
: m_params()
, m_whv(numVisible, numHidden)
, m_whc(numContext, numHidden)
, m_bh(1, numHidden)
, m_bhv(1, numHidden)
, m_bhc(1, numHidden)
, m_bv(1, numVisible)
, m_bc(1, numContext)
{
@@ -37,7 +38,8 @@ Rbm::Rbm(const Rbm& orig)
: m_params(orig.m_params)
, m_whv(orig.m_whv)
, m_whc(orig.m_whc)
, m_bh(orig.m_bh)
, m_bhv(orig.m_bhv)
, m_bhc(orig.m_bhc)
, m_bv(orig.m_bv)
, m_bc(orig.m_bc)
{
@@ -51,8 +53,9 @@ Rbm::~Rbm()
void Rbm::weightsInit(double stddev, double mu)
{
uniform(m_whv, stddev, mu);
uniform(m_whc, stddev, mu);
uniform(m_bh, stddev, mu);
uniform(m_whc, 0*stddev, mu);
uniform(m_bhv, stddev, mu);
uniform(m_bhc, 0*stddev, mu);
uniform(m_bv, stddev, mu);
uniform(m_bc, stddev, mu);
}
@@ -168,16 +171,15 @@ void Rbm::train(const arma::mat& batch, IListener* pListener)
int batchRowIndex = 0;
arma::mat grad_bias_v(arma::zeros(1, m_bv.n_cols));
arma::mat grad_bias_h(arma::zeros(1, m_bh.n_cols));
arma::mat grad_weight(arma::zeros(m_whv.n_rows, m_whv.n_cols));
arma::mat grad_bias_h(arma::zeros(1, m_bhv.n_cols));
arma::mat grad_bias_c(arma::zeros(1, m_bc.n_cols));
arma::mat grad_weight_hv(arma::zeros(m_whv.n_rows, m_whv.n_cols));
arma::mat grad_weight_hc(arma::zeros(m_whc.n_rows, m_whc.n_cols));
arma::mat momentum_whv = arma::zeros(m_whv.n_rows, m_whv.n_cols);
arma::mat momentum_bias_v(arma::zeros(1, m_bv.n_cols));
arma::mat momentum_bias_h(arma::zeros(1, m_bh.n_cols));
arma::mat momentum_bias_h(arma::zeros(1, m_bhv.n_cols));
arma::mat penalty_weights = arma::zeros(m_whv.n_rows, m_whv.n_cols);
arma::mat ctx_state(1, m_bc.n_cols);
arma::mat ctx_probs(1, m_bc.n_cols);
int trainingSizeRemain = batch.n_rows;
bool shouldAbort = false;
@@ -192,11 +194,12 @@ void Rbm::train(const arma::mat& batch, IListener* pListener)
double weight_decay = m_params.weightDecay/scaler;
#if RBM_TRAIN_FLAT
arma::mat h_probs(miniBatchSizeActual, m_bh.n_cols);
arma::mat h_probs(miniBatchSizeActual, m_bhv.n_cols);
arma::mat v_probs(miniBatchSizeActual, m_bv.n_cols);
arma::mat hid_states(miniBatchSizeActual, m_bh.n_cols);
arma::mat hid_states(miniBatchSizeActual, m_bhv.n_cols);
#endif
arma::mat v_states(miniBatch);
arma::mat c_states(arma::zeros(miniBatchSizeActual, m_bc.n_cols));
// Create hidden layer base on training data
if (m_params.doSampleBatch)
@@ -254,7 +257,14 @@ void Rbm::train(const arma::mat& batch, IListener* pListener)
grad_bias_v -= sum(v_probs, 0);
grad_bias_h -= sum(h_probs, 0);
#else
weightUpdate(v_states, grad_weight, grad_bias_h, grad_bias_v);
if (m_bc.n_cols == 0)
{
weightUpdate(v_states, grad_weight_hv, grad_bias_h, grad_bias_v);
}
else
{
weightUpdate(v_states, c_states, grad_weight_hv, grad_weight_hc, grad_bias_h, grad_bias_v, grad_bias_c);
}
#endif
penalty_weights = weight_decay*arma::sign(m_whv);
@@ -263,10 +273,10 @@ void Rbm::train(const arma::mat& batch, IListener* pListener)
status.L2 = accu(m_whv % m_whv);
momentum_bias_v = m_params.momentum*momentum_bias_v + grad_bias_v;
momentum_bias_h = m_params.momentum*momentum_bias_h + grad_bias_h;
momentum_whv = m_params.momentum*momentum_whv + grad_weight - status.L2*penalty_weights;
momentum_whv = m_params.momentum*momentum_whv + grad_weight_hv - status.L2*penalty_weights;
m_bv += learning_rate*momentum_bias_v;
m_bh += learning_rate*momentum_bias_h;
m_bhv += learning_rate*momentum_bias_h;
m_whv += learning_rate*momentum_whv;
progress += dProgress*miniBatchSizeActual;
@@ -326,7 +336,7 @@ arma::mat Rbm::sample(const arma::mat &src)
arma::mat Rbm::v_to_h(const arma::mat &visible) const
{
return visible * m_whv + arma::repmat(m_bh, visible.n_rows, 1);
return visible * m_whv + arma::repmat(m_bhv, visible.n_rows, 1);
}
arma::mat Rbm::h_to_v(const arma::mat &hidden) const
@@ -336,7 +346,7 @@ arma::mat Rbm::h_to_v(const arma::mat &hidden) const
arma::mat Rbm::c_to_h(const arma::mat &context) const
{
return context * m_whc + arma::repmat(m_bh, context.n_rows, 1);
return context * m_whc + arma::repmat(m_bhc, context.n_rows, 1);
}
arma::mat Rbm::h_to_c(const arma::mat &hidden) const
@@ -383,7 +393,7 @@ const arma::mat& Rbm::bv() const
const arma::mat& Rbm::bh() const
{
return m_bh;
return m_bhv;
}
source/Rbm.hpp
+2 -1
View File
@@ -150,7 +150,8 @@ private:
protected:
arma::mat m_whv;
arma::mat m_whc;
arma::mat m_bh;
arma::mat m_bhv;
arma::mat m_bhc;
arma::mat m_bv;
arma::mat m_bc;