jens/Rbm-legacy
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[RBM]

Browse Source 
- GUI: added gaussian hidden, added mini batch size
- Rbm: added mini batch training revised sample functions, reverted to old weight decay


git-svn-id: http://moon:8086/svn/software/trunk/projects/RBM@305 b431acfa-c32f-4a4a-93f1-934dc6c82436
This commit is contained in:
jens
2016-07-07 19:27:33 +00:00
parent 4979ab41b4
commit 57cdfd7065
4 changed files with 161 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Source/Rbm.cpp
+102 -78
View File
@@ -17,6 +17,7 @@ Rbm::Rbm(Weights &weights, const MatrixXd &batch)
, m_variableSigma(weights.getNumVisible())
, m_progress(0)
{
setMiniBatchSize(batch.rows());
Noise_Init(&m_noise, 0x32727155);
m_variableSigma.fill(m_params.m_constantSigma);
updateHiddenBatch();
@@ -27,6 +28,49 @@ Rbm::~Rbm()
Noise_Free(&m_noise);
}
void Rbm::noiseGaussian(MatrixXd &dst)
{
for (size_t i=0; i < dst.rows(); i++)
{
for (size_t j=0; j < dst.cols(); j++)
{
dst(i, j) = Noise_Gaussian(&m_noise);
}
}
}
void Rbm::noiseUniform(MatrixXd &dst)
{
for (size_t i=0; i < dst.rows(); i++)
{
for (size_t j=0; j < dst.cols(); j++)
{
dst(i, j) = Noise_Uniform(&m_noise);
}
}
}
void Rbm::sampleGaussian(MatrixXd &dst, MatrixXd const &src, const MatrixXd &sigma)
{
MatrixXd n(src.rows(), src.cols());
noiseGaussian(n);
dst = sigma.array()*n.array() + src.array();
}
void Rbm::sampleGaussian(MatrixXd &srcDst, const MatrixXd &sigma)
{
MatrixXd n(srcDst.rows(), srcDst.cols());
noiseGaussian(n);
srcDst.array() += sigma.array()*n.array();
}
void Rbm::sampleGaussian(MatrixXd &srcDst, const double &sigma)
{
sampleGaussian(srcDst, sigma*MatrixXd::Ones(srcDst.rows(), srcDst.cols()));
}
void Rbm::sample(MatrixXd &srcDst)
{
sample(srcDst, srcDst);
@@ -34,42 +78,34 @@ void Rbm::sample(MatrixXd &srcDst)
void Rbm::sample(MatrixXd &dst, MatrixXd const &src)
{
uint32_t i;
MatrixXd n(src.rows(), src.cols());
for (i=0; i < src.array().size(); i++)
{
dst.array()(i) = (double)(src.array()(i) > Noise_Uniform(&m_noise));
}
noiseUniform(n);
dst = (src.array() > n.array()).cast<double>();
}
void Rbm::probsLogistic(MatrixXd &src)
{
src.array() = (-src.array()).exp();
src.array() += 1;
src.array() = 1.0/src.array();
src = (1 + (-src.array()).exp()).array().cwiseInverse();
}
void Rbm::probsLogistic(RowVectorXd &src)
{
src.array() = (-src.array()).exp();
src.array() += 1;
src.array() = 1.0/src.array();
src = (1 + (-src.array()).exp()).array().cwiseInverse();
}
void Rbm::probsLogistic(MatrixXd &src, const MatrixXd &sigma)
{
src.array() /= (sigma.array() + EPSILON_SIGMA);
src.array() = (-src.array()).exp();
src.array() += 1;
src.array() = 1.0/src.array();
probsLogistic(src);
}
void Rbm::probsLogistic(RowVectorXd &src, const RowVectorXd &sigma)
{
src.array() /= (sigma.array() + EPSILON_SIGMA);
src.array() = (-src.array()).exp();
src.array() += 1;
src.array() = 1.0/src.array();
probsLogistic(src);
}
void Rbm::probsGaussian(MatrixXd &src, const MatrixXd &sigma)
@@ -116,26 +152,6 @@ void Rbm::probsGaussian(RowVectorXd &src, const RowVectorXd &sigma)
src.array() *= k.array();
}
void Rbm::sampleGaussian(MatrixXd &dst, MatrixXd const &src, const MatrixXd &sigma)
{
uint32_t i;
for (i=0; i < src.array().size(); i++)
{
dst.array()(i) = sigma(i)*Noise_Gaussian(&m_noise) + src.array()(i);
}
}
void Rbm::sampleGaussian(MatrixXd &src, const MatrixXd &sigma)
{
uint32_t i;
for (i=0; i < src.array().size(); i++)
{
src.array()(i) = sigma(i)*Noise_Gaussian(&m_noise) + src.array()(i);
}
}
RowVectorXd Rbm::normalizeData(RowVectorXd const &src, RowVectorXd const &mu, RowVectorXd const &var)
{
// Remove mean
@@ -191,9 +207,8 @@ void Rbm::train(uint32_t numEpochs, double sigmaMin)
size_t trainingSize = m_batch.rows();
size_t trainingSizeRemain = trainingSize;
size_t batchRowIndex = 0;
const size_t miniBatchSize = 100;
double dProgress = 1.0/(numEpochs*(double)trainingSize/miniBatchSize);
double dProgress = 1.0/(numEpochs*(double)trainingSize/std::min(m_params.m_miniBatchSize, trainingSize));
MatrixXd dBiasV_curr(MatrixXd::Zero(1, m_w.getNumVisible()));
MatrixXd dBiasH_curr(MatrixXd::Zero(1, m_w.getNumHidden()));
@@ -206,14 +221,14 @@ void Rbm::train(uint32_t numEpochs, double sigmaMin)
while (trainingSizeRemain)
{
cout << "trainingSizeRemain: " << trainingSizeRemain << endl;
size_t toSlice = std::min(miniBatchSize, trainingSizeRemain);
size_t toSlice = std::min(m_params.m_miniBatchSize, trainingSizeRemain);
MatrixXd batch = m_batch.block(batchRowIndex, 0, toSlice, m_w.getNumVisible());
trainingSizeRemain -= toSlice;
batchRowIndex += toSlice;
size_t batchSize = batch.rows();
double mu_w = m_params.m_muWeights/batchSize;
double mu_biasV = m_params.m_muWeights/batchSize;
double mu_biasH = m_params.m_muWeights/batchSize;
MatrixXd diffErr(batchSize, m_w.getNumVisible());
double mu_w = m_params.m_muWeights/std::min(m_params.m_miniBatchSize, trainingSize);
double mu_biasV = m_params.m_muWeights/std::min(m_params.m_miniBatchSize, trainingSize);
double mu_biasH = m_params.m_muWeights/std::min(m_params.m_miniBatchSize, trainingSize);
MatrixXd batch_sampled(batchSize, m_w.getNumVisible());
MatrixXd v_sampled(batchSize, m_w.getNumVisible());
@@ -264,14 +279,19 @@ void Rbm::train(uint32_t numEpochs, double sigmaMin)
for (gibbs=0; gibbs < m_params.m_numGibbs; gibbs++)
{
sample(hid);
// Create visible reconstruction (a fantasy...) given hid
vis = hid * m_w.weights().transpose();
vis += m_w.visibleBias().replicate(batchSize, 1);
if (m_params.m_useHiddenGaussian)
{
sampleGaussian(hid, m_params.m_constantSigma);
}
else
{
sample(hid);
}
if (m_params.m_useVisibleGaussian)
{
// Create visible reconstruction (a fantasy...) given hid
vis = hid * m_w.weights().transpose();
vis += m_w.visibleBias().replicate(batchSize, 1);
sampleGaussian(v_sampled, vis, m_variableSigma.replicate(batchSize, 1));
hid = v_sampled * m_w.weights();
hid += m_w.hiddenBias().replicate(batchSize, 1);
@@ -280,6 +300,9 @@ void Rbm::train(uint32_t numEpochs, double sigmaMin)
}
else
{
// Create visible reconstruction (a fantasy...) given hid
vis = hid * m_w.weights().transpose();
vis += m_w.visibleBias().replicate(batchSize, 1);
probsLogistic(vis, m_variableSigma.replicate(batchSize, 1));
if (m_params.m_doSampleVisible)
{
@@ -319,26 +342,7 @@ void Rbm::train(uint32_t numEpochs, double sigmaMin)
}
dBiasH = dBiasH_curr;
MatrixXd p = m_w.weights();
if (m_params.m_weightDecay > 0)
{
for (size_t row=0; row < m_w.weights().rows(); row++)
{
for (size_t col=0; col < m_w.weights().cols(); col++)
{
if (p(row, col) >= 0)
{
p(row, col) = m_params.m_weightDecay;
}
else
{
p(row, col) -= m_params.m_weightDecay;
}
}
}
m_w.weights() -= mu_w*p;
}
m_w.weights() += mu_w*(m_params.m_momentum*dW + (1-m_params.m_momentum)*dW_curr);
m_w.weights() += mu_w*((m_params.m_momentum*dW + (1-m_params.m_momentum)*dW_curr) - m_params.m_weightDecay*m_w.weights());
dW = dW_curr;
if (m_params.m_sigmaDecay > 0)
@@ -353,15 +357,24 @@ void Rbm::train(uint32_t numEpochs, double sigmaMin)
} // Number of epochs
diffErr = batch - vis;
MatrixXd diffErr = batch - vis;
diffErr.array() *= diffErr.array();
double err = diffErr.colwise().sum().sum();
cout << "err =" << endl;
cout << err << endl;
cout << "error (per mini batch) = " << err << endl;
} // number of mini batches
updateHiddenBatch();
MatrixXd vis = m_h * m_w.weights().transpose();
vis += m_w.visibleBias().replicate(m_batch.rows(), 1);
probsLogistic(vis, m_variableSigma.replicate(m_batch.rows(), 1));
MatrixXd diffErr = m_batch - vis;
diffErr.array() *= diffErr.array();
double err = diffErr.colwise().sum().sum();
cout << "error (total) = " << err << endl;
onProgressChanged();
}
double Rbm::getProgress() const
@@ -445,6 +458,12 @@ void Rbm::setUseVisibleGaussian(bool flag)
onParamsChanged();
}
void Rbm::setUseHiddenGaussian(bool flag)
{
m_params.m_useHiddenGaussian = flag;
onParamsChanged();
}
void Rbm::setDoRaoBlackwell(bool flag)
{
m_params.m_doRaoBlackwell = flag;
@@ -495,6 +514,11 @@ void Rbm::setNumGibbs(size_t value)
onParamsChanged();
}
void Rbm::setMiniBatchSize(size_t size)
{
m_params.m_miniBatchSize = size;
}
void Rbm::setMuWeights(double value)
{
m_params.m_muWeights = value;
@@ -518,11 +542,6 @@ MatrixXd const& Rbm::getHiddenBatch()
return m_h;
}
MatrixXd const& Rbm::getVisibleBatch()
{
return m_v;
}
MatrixXd const& Rbm::getBatch()
{
return m_batch;
@@ -530,6 +549,10 @@ MatrixXd const& Rbm::getBatch()
void Rbm::updateHiddenBatch()
{
if (m_batch.rows() == 0)
{
return;
}
m_h.resize(m_batch.rows(), m_w.getNumHidden());
m_h = m_batch * m_w.weights();
m_h += m_w.hiddenBias().replicate(m_batch.rows(), 1);
@@ -540,3 +563,4 @@ Rbm::Params const& Rbm::params()
{
return m_params;
}