jens/Rbm-legacy
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[RBM]

Browse Source 
- DrawComponent: use fixed value scaling
- Rbm: fixed weight decay
- Rbm: fixed sparsity


git-svn-id: http://moon:8086/svn/software/trunk/projects/RBM@298 b431acfa-c32f-4a4a-93f1-934dc6c82436
This commit is contained in:
jens
2016-06-22 19:11:09 +00:00
parent 9d00654932
commit 866a0349db
4 changed files with 46 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Source/Rbm.hpp
+14 -37
View File
@@ -27,7 +27,7 @@ public:
Params()
: m_constantSigma(1.0)
, m_sigmaDecay(1.0)
, m_weightDecay(0.01)
, m_weightDecay(0.00001)
, m_lambda(1.0)
, m_sparsity(0.05)
, m_muWeights(0.1)
@@ -69,17 +69,6 @@ public:
, m_progress(0)
{
Noise_Init(&m_noise, 0x32727155);
#if 1
VectorXd a(4);
a << 1, 2, 3, 4;
VectorXd b(4);
b.array() = -a.array().exp();
cout << b << endl;
#endif
m_variableSigma.fill(m_params.m_constantSigma);
updateHiddenBatch();
}
@@ -260,7 +249,7 @@ public:
m_v.resize(batchSize, m_w.getNumVisible());
MatrixXd h(batchSize, m_w.getNumHidden());
MatrixXd dBiasV_curr(MatrixXd::Zero(1, m_w.getNumVisible()));
MatrixXd dBiasH_curr(MatrixXd::Zero(1, m_w.getNumHidden()));
MatrixXd dW_curr(MatrixXd::Zero(m_w.getNumVisible(), m_w.getNumHidden()));
@@ -289,9 +278,9 @@ public:
{
onProgressChanged();
// When the hidden units are being driven by data, always use stochastic binary states
if (m_params.m_doSampleBatch)
{
// When the hidden units are being driven by data, always use stochastic binary states
sample(batch_sampled, batch);
// Create hidden layer base on sampled training data
@@ -310,10 +299,7 @@ public:
// Update weights (positive phase)
dBiasV_curr = batch.colwise().sum();
if (!m_params.m_doSparse)
{
dBiasH_curr = h.colwise().sum();
}
dBiasH_curr = h.colwise().sum();
dW_curr = batch.transpose() * h;
for (gibbs=0; gibbs < m_params.m_numGibbs; gibbs++)
@@ -348,37 +334,28 @@ public:
// Update weights (negative phase)
dBiasV_curr -= m_v.colwise().sum();
if (!m_params.m_doSparse)
{
dBiasH_curr -= h.colwise().sum();
}
dBiasH_curr -= h.colwise().sum();
dW_curr -= m_v.transpose() * h;
m_w.visibleBias() += mu_biasV*(m_params.m_momentum*dBiasV + (1-m_params.m_momentum)*dBiasV_curr);
dBiasV = dBiasV_curr;
m_w.weights() += mu_w*(m_params.m_momentum*dW + (1-m_params.m_momentum)*dW_curr - m_params.m_weightDecay*m_w.weights());
dW = dW_curr;
if (m_params.m_doSparse)
{
// Create hidden representation given v
toHiddenBatch(h, batch);
dBiasH_curr = m_params.m_sparsity * MatrixXd::Ones(dBiasH.rows(), dBiasH.cols()) - h.colwise().mean();
m_w.hiddenBias() += m_params.m_muSparsity*(m_params.m_momentum*dBiasH + (1-m_params.m_momentum)*dBiasH_curr);
dBiasH = dBiasH_curr;
// cout << "Mean(" << m_sparsity << ") = " << (double)sumBiasH.array().mean() << endl;
// cout << sumBiasH << endl;
MatrixXd h1 = h-MatrixXd::Ones(h.rows(), h.cols())*m_params.m_sparsity;
RowVectorXd hm = h1.colwise().mean();
m_w.hiddenBias() -= m_params.m_muSparsity * hm;
}
else
{
m_w.hiddenBias() += mu_biasH*(m_params.m_momentum*dBiasH + (1-m_params.m_momentum)*dBiasH_curr);
dBiasH = dBiasH_curr;
}
dBiasH = dBiasH_curr;
m_w.weights() -= m_params.m_weightDecay*m_w.weights();
m_w.weights() += mu_w*(m_params.m_momentum*dW + (1-m_params.m_momentum)*dW_curr);
dW = dW_curr;
if (m_variableSigma[0] > sigmaMin)
{
m_variableSigma.array() *= m_params.m_sigmaDecay;