jens/Rbm-legacy
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

- train(): added reconstruction error metric

Browse Source 
- train2(): 
  - added reconstruction error metric. 
  - added Gaussian units
  - added sparsity

- GUI: choose train() or train2()


git-svn-id: http://moon:8086/svn/software/trunk/projects/RBM@44 b431acfa-c32f-4a4a-93f1-934dc6c82436
This commit is contained in:
jens
2014-11-01 18:01:04 +00:00
parent 93fa8c64ce
commit 6e7b8bc151
3 changed files with 145 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Source/Rbm.hpp
+119 -72
View File
@@ -81,6 +81,8 @@ public:
MatrixXd sumWeights(m_w.getNumVisible(), m_w.getNumHidden());
MatrixXd deltaWeights(m_w.getNumVisible(), m_w.getNumHidden());
MatrixXd diffErr(1, m_w.getNumVisible());
const LayerArray<VisibleLayer> &vt = batch;
sigma = m_sigma;
@@ -96,6 +98,8 @@ public:
m_doCancel = false;
for (epoch=0; epoch < numEpochs; epoch++)
{
double err = 0;
if (m_doCancel)
{
m_doCancel = false;
@@ -125,6 +129,8 @@ public:
sumBiasV += vt[t].states();
sumBiasH += h.states();
diffErr = vt[t].states();
for (gibbs=0; gibbs < m_numGibbs; gibbs++)
{
h.statesUpdateStochastic();
@@ -170,6 +176,9 @@ public:
sumWeights -= v.states() * h.states().transpose();
sumBiasV -= v.states();
sumBiasH -= h.states();
diffErr -= v.states();
diffErr.array() *= diffErr.array();
err += diffErr.sum();
} // TrainingSize
@@ -215,20 +224,40 @@ public:
m_pListener->onEpochTrained(*this);
}
cout << "err =" << endl;
cout << err << endl;
} // Number of epochs
}
MatrixXd sample(const MatrixXd &src)
void sample(MatrixXd &src)
{
uint32_t i;
MatrixXd res(src);
for (i=0; i < src.array().size(); i++)
{
res.array()(i) = (double) src.array()(i) > Noise_Uniform(&m_noise);
src.array()(i) = (double)(src.array()(i) > Noise_Uniform(&m_noise));
}
}
return res;
void probsLogistic(MatrixXd &src, double lambda, double sigma)
{
double var = sigma*sigma;
src.array() *= -lambda/var;
src.array() = src.array().exp();
src.array() += 1;
src.array() = 1.0/src.array();
}
void sampleGaussian(MatrixXd &src, double lambda, double sigma)
{
uint32_t i;
for (i=0; i < src.array().size(); i++)
{
src.array()(i) = sigma*Noise_Gaussian(&m_noise) + lambda*src.array()(i);
}
}
void train2(const LayerArray<VisibleLayer> &vt, uint32_t numEpochs, uint32_t batchSize, double sigmaMin = 0.05)
@@ -240,113 +269,125 @@ public:
double dProgress = 1.0/numEpochs;
double kTrain = 1.0/vt.getSize();
if (batchSize > vt.getSize())
batchSize = vt.getSize();
// if (batchSize > vt.getSize())
batchSize = vt.getSize();
MatrixXd vp(m_w.getNumVisible(), batchSize);
MatrixXd vs(m_w.getNumVisible(), batchSize);
MatrixXd hp(m_w.getNumHidden(), batchSize);
MatrixXd hs(m_w.getNumHidden(), batchSize);
MatrixXd batch(m_w.getNumVisible(), batchSize);
MatrixXd v(batchSize, m_w.getNumVisible());
MatrixXd h(batchSize, m_w.getNumHidden());
MatrixXd batch(batchSize, m_w.getNumVisible());
VectorXd sumBiasV(m_w.getNumVisible());
VectorXd sumBiasH(m_w.getNumHidden());
MatrixXd sumBiasV(1, m_w.getNumVisible());
MatrixXd sumBiasH(1, m_w.getNumHidden());
MatrixXd sumWeights(m_w.getNumVisible(), m_w.getNumHidden());
VectorXd deltaBiasV(m_w.getNumVisible());
VectorXd deltaBiasH(m_w.getNumHidden());
MatrixXd deltaWeights(m_w.getNumVisible(), m_w.getNumHidden());
MatrixXd deltaBiasV(MatrixXd::Zero(1, m_w.getNumVisible()));
MatrixXd deltaBiasH(MatrixXd::Zero(1, m_w.getNumHidden()));
MatrixXd deltaWeights(MatrixXd::Zero(m_w.getNumVisible(), m_w.getNumHidden()));
MatrixXd diffErr(batchSize, m_w.getNumVisible());
m_progress = 0;
m_doCancel = false;
for (i=0; i < batchSize; i++)
{
t = (uint32_t)(0.5 + (vt.getSize()-1)*Noise_Uniform(&m_noise));
batch.col(i) = vt[t].states();
// t = (uint32_t)(0.5 + (vt.getSize()-1)*Noise_Uniform(&m_noise));
batch.row(i) = vt[i].states();
}
for (epoch=0; epoch < numEpochs; epoch++)
{
double err;
v = batch;
if (m_doCancel)
{
m_doCancel = false;
break;
}
sumWeights.fill(0);
sumBiasV.fill(0);
sumBiasH.fill(0);
for (i=0; i < batchSize; i++)
// Create hidden layer base on training data
h = v * m_w.weights();
h += m_w.hiddenBias().transpose().replicate(batchSize, 1);
probsLogistic(h, m_lambda, sigma);
if (!m_doRaoBlackwell)
{
vs = batch.col(i);
sample(h);
}
// Update weights (positive phase)
sumBiasV = v.colwise().sum();
if (!m_doSparse)
{
sumBiasH = h.colwise().sum();
}
sumWeights = v.transpose() * h;
diffErr = v;
// h.probsUpdateLogistic(vt[t], m_w, m_lambda, sigma);
hp = -vs.transpose() * m_w.weights();
hp.array() = hp.array().exp();
hp.array() += 1;
hp.array() = 1.0/hp.array();
for (gibbs=0; gibbs < m_numGibbs; gibbs++)
{
sample(h);
// Create hidden layer base on training data
hs = sample(hp);
// Create visible reconstruction (a fantasy...)
v = h * m_w.weights().transpose();
v += m_w.visibleBias().transpose().replicate(batchSize, 1);
// Update weights (positive phase)
sumBiasV += vp.colwise().sum();
if (m_doRaoBlackwell)
if (m_useVisibleGaussian)
{
sumWeights += vs * hp.transpose();
sumBiasH += hp.colwise().sum();
sampleGaussian(v, m_lambda, sigma);
}
else
{
sumWeights += vs * hs.transpose();
sumBiasH += hs.colwise().sum();
}
for (gibbs=0; gibbs < m_numGibbs; gibbs++)
{
// Create visible reconstruction (a fantasy...)
if (m_useProbsForHiddenReconstruction)
probsLogistic(v, m_lambda, sigma);
if (!m_useProbsForHiddenReconstruction)
{
vp = -hs * m_w.weights().transpose();
vp.array() = vp.array().exp();
vp.array() += 1;
vp.array() = 1.0/vp.array();
sample(v);
}
else
{
vs = sample(vp);
}
// Create hidden reconstruction
hp = -vs.transpose() * m_w.weights();
hp.array() = hp.array().exp();
hp.array() += 1;
hp.array() = 1.0/hp.array();
}
// Update weights (negative phase)
sumBiasV -= vp.colwise().sum();
if (m_doRaoBlackwell)
{
sumWeights -= vs * hp.transpose();
sumBiasH -= hp.colwise().sum();
}
else
{
sumWeights -= vs * hs.transpose();
sumBiasH -= hs.colwise().sum();
}
// Create hidden reconstruction
h = v * m_w.weights();
h += m_w.hiddenBias().transpose().replicate(batchSize, 1);
probsLogistic(h, m_lambda, sigma);
}
} // TrainingSize
if (!m_doRaoBlackwell)
{
sample(h);
}
// Update weights (negative phase)
sumBiasV -= v.colwise().sum();
if (!m_doSparse)
{
sumBiasH -= h.colwise().sum();
}
sumWeights -= v.transpose() * h;
diffErr -= v;
deltaWeights = m_momentum*deltaWeights + m_muWeights*kTrain*sumWeights - m_weightDecay*m_w.weights();
deltaWeights = m_momentum*deltaWeights + m_muWeights*(kTrain*sumWeights - m_weightDecay*m_w.weights());
m_w.weights() += deltaWeights;
deltaBiasV = m_momentum*deltaBiasV + m_muWeights*kTrain*sumBiasV;
m_w.visibleBias() += deltaBiasV;
deltaBiasH = m_momentum*deltaBiasH + m_muWeights*kTrain*sumBiasH;
if (m_doSparse)
{
h = v * m_w.weights();
h += m_w.hiddenBias().transpose().replicate(batchSize, 1);
probsLogistic(h, m_lambda, sigma);
sumBiasH.fill(m_sparsity);
sumBiasH -= h.colwise().mean();
deltaBiasH = m_momentum*deltaBiasH + m_muSparsity*sumBiasH;
// cout << "Mean(" << m_sparsity << ") = " << (double)sumBiasH.array().mean() << endl;
// cout << sumBiasH << endl;
}
else
{
deltaBiasH = m_momentum*deltaBiasH + m_muWeights*kTrain*sumBiasH;
}
m_w.hiddenBias() += deltaBiasH;
if (sigma > sigmaMin)
@@ -360,6 +401,12 @@ public:
m_pListener->onEpochTrained(*this);
}
diffErr.array() *= diffErr.array();
err = diffErr.colwise().sum().sum();
cout << "err =" << endl;
cout << err << endl;
} // Number of epochs
}