jens/Rbm-legacy
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

- added param sparsity

Browse Source 
- added gaussian visible unit
- added param sigma decay
- RBM modi and params are set using members
- use sigma instead of variance


git-svn-id: http://moon:8086/svn/software/trunk/projects/RBM@25 b431acfa-c32f-4a4a-93f1-934dc6c82436
This commit is contained in:
jens
2014-10-14 21:23:01 +00:00
parent 0166b986cb
commit 7f4438d698
5 changed files with 541 additions and 182 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Source/Rbm.hpp
+211 -58
View File
@@ -37,6 +37,16 @@ public:
: m_w(weights)
, m_pListener(pListener)
, m_progress(0)
, m_sigma(1.0)
, m_sigmaDecay(1.0)
, m_lambda(1.0)
, m_sparsity(0)
, m_useVisibleGaussian(false)
, m_useExpectations(false)
, m_doRaoBlackwell(false)
, m_useProbsForHiddenReconstruction(false)
, m_doRobbinsMonro(false)
, m_doSparse(false)
{
Noise_Init(&m_noise, 0x32727155);
}
@@ -63,43 +73,35 @@ public:
{
m_w.hiddenBias().array() += mu*h.states().array();
}
//#define RBM_SPARSE
void train(LayerArray<VisibleLayer> &vt, uint32_t numEpochs, double mu, uint32_t numGibbs = 1, bool useExpectations = false, bool doRaoBlackwell = false, bool useProbsForHiddenReconstruction = false, bool doRobbinsMonro = false)
void train(LayerArray<VisibleLayer> &vt, uint32_t numEpochs, double mu, uint32_t numGibbs = 1, double sigmaMin = 0.05)
{
uint32_t t, i;
uint32_t epoch;
uint32_t gibbs;
double sigma;
VisibleLayer v(m_w.getNumVisible());
HiddenLayer h(m_w.getNumHidden());
HiddenLayer *pH;
LayerArray<HiddenLayer> ht(vt.getSize(), m_w.getNumHidden());
sigma = m_sigma;
Weights w = m_w;
double dProgress = 1.0/numEpochs;
m_progress = 0;
#ifdef RBM_SPARSE
const double lambda = 0.05;
const double variance = 0.4;
const double penalty = 0.05;
#else
const double lambda = 1.0;
const double variance = 1.0;
const double penalty = 0.0;
#endif
if (useExpectations)
if (m_useExpectations)
{
mu /= vt.getSize();
}
if (doRobbinsMonro)
if (m_doRobbinsMonro)
{
for (i=0; i < vt.getSize(); i++)
{
// Create hidden layer base on training data
ht[i].probsUpdateLogistic(vt[i], w, lambda, variance);
ht[i].probsUpdateLogistic(vt[i], w, m_lambda, sigma);
}
}
@@ -107,11 +109,12 @@ public:
{
for (i=0; i < vt.getSize(); i++)
{
t = (uint32_t)(0.5 + (vt.getSize()-1)*Noise_Uniform(&m_noise, 0.5));
h.probsUpdateLogistic(vt[t], w, lambda, variance);
//t = (uint32_t)(0.5 + (vt.getSize()-1)*Noise_Uniform(&m_noise, 0.5));
t = i;
h.probsUpdateLogistic(vt[t], w, m_lambda, sigma);
// Create hidden layer base on training data
if (doRobbinsMonro)
if (m_doRobbinsMonro)
{
pH = &ht[t];
}
@@ -121,7 +124,7 @@ public:
}
// Update weights (positive phase)
if (doRaoBlackwell)
if (m_doRaoBlackwell)
{
pH->states() = pH->probs();
}
@@ -131,33 +134,46 @@ public:
}
weightsUpdate(vt[t], h, +mu);
visibleBiasUpdate(vt[t], +mu);
hiddenBiasUpdate(h, +mu);
if (!m_doSparse)
{
hiddenBiasUpdate(h, +mu);
}
for (gibbs=0; gibbs < numGibbs; gibbs++)
{
pH->statesUpdateStochastic();
// Create visible reconstruction (a fantasy...)
#ifdef RBM_SPARSE
v.probsUpdateGaussian(*pH, w, lambda, variance);
#else
v.probsUpdateLogistic(*pH, w);
#endif
if (useProbsForHiddenReconstruction)
if (m_useProbsForHiddenReconstruction)
{
if (m_useVisibleGaussian)
{
v.probsUpdateGaussian(*pH, w, m_lambda, sigma);
}
else
{
v.probsUpdateLogistic(*pH, w, m_lambda, sigma);
}
v.states() = v.probs();
}
else
{
v.statesUpdateStochastic();
if (m_useVisibleGaussian)
{
v.sampleGaussian(*pH, w, m_lambda, sigma);
}
else
{
v.probsUpdateLogistic(*pH, w, m_lambda, sigma);
v.statesUpdateStochastic();
}
}
// Create hidden reconstruction
pH->probsUpdateLogistic(v, w, lambda, variance);
pH->probsUpdateLogistic(v, w, m_lambda, sigma);
}
// Update weights (negative phase)
if (doRaoBlackwell)
if (m_doRaoBlackwell)
{
pH->states() = pH->probs();
}
@@ -167,33 +183,40 @@ public:
}
weightsUpdate(v, *pH, -mu);
visibleBiasUpdate(v, -mu);
hiddenBiasUpdate(*pH, -mu);
if (!useExpectations)
if (!m_doSparse)
{
hiddenBiasUpdate(*pH, -mu);
}
if (!m_useExpectations)
{
w = m_w;
}
} // TrainingSize
if (m_useExpectations)
{
w = m_w;
}
#ifdef RBM_SPARSE
if (m_doSparse)
{
HiddenLayer th(m_w.getNumHidden());
VectorXd m(th.states());
VectorXd m(m_w.getNumHidden());
m.fill(0);
for (i=0; i < vt.getSize(); i++)
{
m += expectHidden(vt[i].states(), lambda, variance, 10);
th.probsUpdateLogistic(vt[i], w, m_lambda, sigma);
m += th.probs();
}
m.array() = penalty - m.array();
m *= 1.0/vt.getSize();
m /= i;
m.array() = m_sparsity - m.array();
th.states() = m;
hiddenBiasUpdate(th, -mu);
}
#endif
if (useExpectations)
if (sigma > sigmaMin)
{
w = m_w;
sigma *= m_sigmaDecay;
}
m_progress += dProgress;
@@ -201,7 +224,8 @@ public:
{
m_pListener->onEpochTrained(*this);
}
}
} // Number of epochs
}
double getProgress() const
@@ -241,7 +265,7 @@ public:
for (j=0; j < vts.getSize(); j++)
{
h[j].setNumUnits(m_w.getNumHidden());
h[j].probsUpdateLogistic(vts.getAt(j), m_w);
h[j].probsUpdateLogistic(vts.getAt(j), m_w, m_lambda, m_sigma);
// h[j].statesAssignfromProbs();
h[j].statesUpdateStochastic();
}
@@ -280,7 +304,7 @@ public:
// Reconstruct
for (i=0; i < vts.getSize(); i++)
{
vts.getAt(i).probsUpdateLogistic(h[i], m_w);
vts.getAt(i).probsUpdateLogistic(h[i], m_w, m_lambda, m_sigma);
}
printf("A fantasy... (v^, t>)\n");
@@ -292,27 +316,32 @@ public:
delete [] h;
}
VectorXd toHidden(const VectorXd& visible, double lambda = 1.0, double variance = 1.0)
VectorXd toHidden(const VectorXd& visible)
{
HiddenLayer th(m_w.getNumHidden());
VisibleLayer tv(m_w.getNumVisible(), (const VectorXd*)&visible);
th.probsUpdateLogistic(tv, m_w, lambda, variance);
th.probsUpdateLogistic(tv, m_w, m_lambda, m_sigma);
return th.probs();
}
VectorXd toVisible(const VectorXd& hidden, double lambda = 1.0, double variance = 1.0)
VectorXd toVisible(const VectorXd& hidden)
{
HiddenLayer th(m_w.getNumHidden(), (const VectorXd*)&hidden);
VisibleLayer tv(m_w.getNumVisible());
tv.probsUpdateLogistic(th, m_w, lambda, variance);
if (m_useVisibleGaussian)
{
tv.probsUpdateGaussian(th, m_w, m_lambda, m_sigma);
}
else
{
tv.probsUpdateLogistic(th, m_w, m_lambda, m_sigma);
}
return tv.probs();
}
VectorXd expectHidden(VectorXd visible, uint32_t numIter, double lambda = 1.0, double variance = 1.0)
VectorXd expectHidden(VectorXd visible, uint32_t numIter)
{
uint32_t i;
VisibleLayer v(m_w.getNumVisible(), (const VectorXd*)&visible);
@@ -320,14 +349,21 @@ public:
for (i=0; i < numIter; i++)
{
h.probsUpdateLogistic(v, (Weights&)m_w, lambda, variance);
v.probsUpdateGaussian(h, (Weights&)m_w, lambda, variance);
h.probsUpdateLogistic(v, (Weights&)m_w, m_lambda, m_sigma);
if (m_useVisibleGaussian)
{
v.probsUpdateGaussian(h, (Weights&)m_w, m_lambda, m_sigma);
}
else
{
v.probsUpdateLogistic(h, (Weights&)m_w, m_lambda, m_sigma);
}
}
return h.probs();
}
VectorXd expectVisible(VectorXd visible, uint32_t numIter, double lambda = 1.0, double variance = 1.0)
VectorXd expectVisible(VectorXd visible, uint32_t numIter)
{
uint32_t i;
VisibleLayer v(m_w.getNumVisible(), (const VectorXd*)&visible);
@@ -335,18 +371,135 @@ public:
for (i=0; i < numIter; i++)
{
h.probsUpdateLogistic(v, (Weights&)m_w, lambda, variance);
v.probsUpdateLogistic(h, (Weights&)m_w, lambda, variance);
h.probsUpdateLogistic(v, (Weights&)m_w, m_lambda, m_sigma);
if (m_useVisibleGaussian)
{
v.probsUpdateGaussian(h, (Weights&)m_w, m_lambda, m_sigma);
}
else
{
v.probsUpdateLogistic(h, (Weights&)m_w, m_lambda, m_sigma);
}
}
return v.probs();
}
void setSigma(double value)
{
m_sigma = value;
}
void setSigmaDecay(double value)
{
m_sigmaDecay = value;
}
void setLambda(double value)
{
m_lambda = value;
}
void setSparsity(double value)
{
m_sparsity = value;
}
void setUseVisibleGaussian(bool flag)
{
m_useVisibleGaussian = flag;
}
void setUseExpectations(bool flag)
{
m_useExpectations = flag;
}
void setDoRaoBlackwell(bool flag)
{
m_doRaoBlackwell = flag;
}
void setUseProbsForHiddenReconstruction(bool flag)
{
m_useProbsForHiddenReconstruction = flag;
}
void setDoRobbinsMonro(bool flag)
{
m_doRobbinsMonro = flag;
}
void setDoSparse(bool flag)
{
m_doSparse = flag;
}
double getSparsity()
{
return m_sparsity;
}
double getSigma()
{
return m_sigma;
}
double getSigmaDecay()
{
return m_sigmaDecay;
}
double getLambda()
{
return m_lambda;
}
bool getUseVisibleGaussian()
{
return m_useVisibleGaussian;
}
bool getUseExpectations()
{
return m_useExpectations;
}
bool getDoRaoBlackwell()
{
return m_doRaoBlackwell;
}
bool getUseProbsForHiddenReconstruction()
{
return m_useProbsForHiddenReconstruction;
}
bool getRobbinsMonro()
{
return m_doRobbinsMonro;
}
bool getDoSparse()
{
return m_doSparse;
}
private:
Weights &m_w;
RbmListener *m_pListener;
noise_gen_t m_noise;
double m_progress;
double m_sigma;
double m_sigmaDecay;
double m_lambda;
double m_sparsity;
bool m_useVisibleGaussian;
bool m_useExpectations;
bool m_doRaoBlackwell;
bool m_useProbsForHiddenReconstruction;
bool m_doRobbinsMonro;
bool m_doSparse;
};