jens/Rbm-legacy
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[RBM]

Browse Source 
- committed last changes

git-svn-id: http://moon:8086/svn/software/trunk/projects/RBM@359 b431acfa-c32f-4a4a-93f1-934dc6c82436
This commit is contained in:
jens
2018-06-12 17:01:49 +00:00
parent 9f8d84008c
commit 80a67ba314
15 changed files with 155 additions and 264 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Source/Rbm.cpp
+47 -160
View File
@@ -14,11 +14,9 @@ void mylog(const char* format, ...);
Rbm::Rbm(Weights &weights, const MatrixXd &batch)
: m_w(weights)
, m_batch(batch)
, m_variableSigma(weights.getNumVisible())
, m_progress(0)
{
Noise_Init(&m_noise, 0x32727155);
m_variableSigma.fill(m_params.m_constantSigma);
updateHiddenBatch();
}
@@ -49,25 +47,17 @@ void Rbm::noiseUniform(MatrixXd &dst)
}
}
void Rbm::sampleGaussian(MatrixXd &dst, MatrixXd const &src, const MatrixXd &sigma)
void Rbm::sampleGaussian(MatrixXd &dst, MatrixXd const &src)
{
MatrixXd n(src.rows(), src.cols());
noiseGaussian(n);
dst = sigma.array()*n.array() + src.array();
dst = n.array() + src.array();
}
void Rbm::sampleGaussian(MatrixXd &srcDst, const MatrixXd &sigma)
void Rbm::sampleGaussian(MatrixXd &srcDst)
{
MatrixXd n(srcDst.rows(), srcDst.cols());
noiseGaussian(n);
srcDst.array() += sigma.array()*n.array();
}
void Rbm::sampleGaussian(MatrixXd &srcDst, const double &sigma)
{
sampleGaussian(srcDst, sigma*MatrixXd::Ones(srcDst.rows(), srcDst.cols()));
sampleGaussian(srcDst, srcDst);
}
void Rbm::sample(MatrixXd &srcDst)
@@ -85,116 +75,32 @@ void Rbm::sample(MatrixXd &dst, MatrixXd const &src)
}
void Rbm::probsLogistic(MatrixXd &src)
void Rbm::probsLogistic(MatrixXd &srcDst)
{
src = (1 + (-src.array()).exp()).array().cwiseInverse();
srcDst = (1 + (-srcDst.array()).exp()).array().cwiseInverse();
}
void Rbm::probsLogistic(RowVectorXd &src)
void Rbm::probsLogistic(RowVectorXd &srcDst)
{
src = (1 + (-src.array()).exp()).array().cwiseInverse();
srcDst = (1 + (-srcDst.array()).exp()).array().cwiseInverse();
}
void Rbm::probsLogistic(MatrixXd &src, const MatrixXd &sigma)
void Rbm::normalizeData(MatrixXd &dst, MatrixXd const &src)
{
src.array() /= (sigma.array() + EPSILON_SIGMA);
probsLogistic(src);
}
MatrixXd mean = src.rowwise().mean();
// cout << "mean" << ": " << endl << mean << endl;
void Rbm::probsLogistic(RowVectorXd &src, const RowVectorXd &sigma)
{
src.array() /= (sigma.array() + EPSILON_SIGMA);
probsLogistic(src);
}
dst = src - mean.replicate(1, src.cols());
// cout << "dst - mean" << ": " << endl << dst << endl;
void Rbm::probsGaussian(MatrixXd &src, const MatrixXd &sigma)
{
src.array() = 1 - src.array();
src.array() *= src.array();
src.array() *= -0.5;
MatrixXd x = dst.array().square();
MatrixXd var = sigma;
var.array() += EPSILON_SIGMA;
var.array() *= var.array();
MatrixXd var = x.rowwise().mean();
// cout << "var" << ": " << endl << var << endl;
src.array() /= var.array();
src.array() = src.array().exp();
MatrixXd k = var;
k.array() *= 2*3.14159265359;
k.array() = k.array().sqrt();
k.array() = 1.0/k.array();
src.array() *= k.array();
}
void Rbm::probsGaussian(RowVectorXd &src, const RowVectorXd &sigma)
{
src.array() = 1 - src.array();
src.array() *= src.array();
src.array() *= -0.5;
RowVectorXd var = sigma;
var.array() += EPSILON_SIGMA;
var.array() *= var.array();
src.array() /= var.array();
src.array() = src.array().exp();
RowVectorXd k = var;
k.array() *= 2*3.14159265359;
k.array() = k.array().sqrt();
k.array() = 1.0/k.array();
src.array() *= k.array();
}
RowVectorXd Rbm::normalizeData(RowVectorXd const &src, RowVectorXd const &mu, RowVectorXd const &var)
{
// Remove mean
RowVectorXd res = src - mu;
// res.array() /= var.array() + EPSILON_SIGMA;
// cout << __PRETTY_FUNCTION__ << ": " << res << endl;
return res;
}
RowVectorXd Rbm::calcMean(MatrixXd const &batch)
{
// Remove mean
RowVectorXd res = batch.colwise().mean();
// cout << __PRETTY_FUNCTION__ << ": " << res << endl;
return res;
}
RowVectorXd Rbm::calcSigma(MatrixXd const &batch)
{
MatrixXd x = batch.rowwise() - batch.colwise().mean();
x.array() *= x.array();
RowVectorXd res = x.colwise().mean().array().sqrt();
// cout << __PRETTY_FUNCTION__ << ": " << res << endl;
return res;
}
MatrixXd Rbm::calcZ(MatrixXd &v, MatrixXd &h)
{
MatrixXd t1(v.rows(), m_w.getNumVisible());
t1 = v - m_w.visibleBias().transpose().replicate(v.rows(), 1);
t1.array() *= t1.array();
t1.array() *= 0.5;
t1 -= (h * m_w.weights().transpose());
return t1;
MatrixXd stddev_norm = var.array().sqrt().cwiseInverse();
dst.array() *= stddev_norm.replicate(1, src.cols()).array();
// cout << "dst" << ": " << endl << dst << endl;
}
void Rbm::train(size_t numEpochs, size_t miniBatchSize, double sigmaMin)
@@ -216,12 +122,19 @@ void Rbm::train(size_t numEpochs, size_t miniBatchSize, double sigmaMin)
MatrixXd dBiasH(MatrixXd::Zero(1, m_w.getNumHidden()));
MatrixXd dW(MatrixXd::Zero(m_w.getNumVisible(), m_w.getNumHidden()));
MatrixXd __batch = m_batch_normalized;
if (m_params.m_doNormalizeData && !m_params.m_useVisibleGaussian)
{
probsLogistic(__batch);
}
m_progress = 0;
while (trainingSizeRemain)
{
cout << "trainingSizeRemain: " << trainingSizeRemain << endl;
size_t toSlice = std::min(miniBatchSize, trainingSizeRemain);
MatrixXd batch = m_batch.block(batchRowIndex, 0, toSlice, m_w.getNumVisible());
MatrixXd batch = __batch.block(batchRowIndex, 0, toSlice, m_w.getNumVisible());
trainingSizeRemain -= toSlice;
batchRowIndex += toSlice;
size_t batchSize = batch.rows();
@@ -234,16 +147,6 @@ void Rbm::train(size_t numEpochs, size_t miniBatchSize, double sigmaMin)
MatrixXd vis(batchSize, m_w.getNumVisible());
MatrixXd hid(batchSize, m_w.getNumHidden());
if (m_params.m_doNormalizeData)
{
RowVectorXd mean = calcMean(batch);
for (i=0; i < batchSize; i++)
{
RowVectorXd x = batch.row(i);
batch.row(i) = normalizeData(x, mean, m_variableSigma);
}
}
for (epoch=0; epoch < numEpochs; epoch++)
{
onProgressChanged();
@@ -280,7 +183,7 @@ void Rbm::train(size_t numEpochs, size_t miniBatchSize, double sigmaMin)
{
if (m_params.m_useHiddenGaussian)
{
sampleGaussian(hid, m_params.m_constantSigma);
sampleGaussian(hid);
}
else
{
@@ -291,7 +194,7 @@ void Rbm::train(size_t numEpochs, size_t miniBatchSize, double sigmaMin)
// Create visible reconstruction (a fantasy...) given hid
vis = hid * m_w.weights().transpose();
vis += m_w.visibleBias().replicate(batchSize, 1);
sampleGaussian(v_sampled, vis, m_variableSigma.replicate(batchSize, 1));
sampleGaussian(v_sampled, vis);
hid = v_sampled * m_w.weights();
hid += m_w.hiddenBias().replicate(batchSize, 1);
probsLogistic(hid);
@@ -302,7 +205,7 @@ void Rbm::train(size_t numEpochs, size_t miniBatchSize, double sigmaMin)
// Create visible reconstruction (a fantasy...) given hid
vis = hid * m_w.weights().transpose();
vis += m_w.visibleBias().replicate(batchSize, 1);
probsLogistic(vis, m_variableSigma.replicate(batchSize, 1));
probsLogistic(vis);
if (m_params.m_doSampleVisible)
{
sample(v_sampled, vis);
@@ -348,7 +251,6 @@ void Rbm::train(size_t numEpochs, size_t miniBatchSize, double sigmaMin)
{
if (m_variableSigma[0] > sigmaMin)
{
m_variableSigma.array() *= (1-m_params.m_sigmaDecay);
}
}
@@ -365,9 +267,9 @@ void Rbm::train(size_t numEpochs, size_t miniBatchSize, double sigmaMin)
updateHiddenBatch();
MatrixXd vis = m_h * m_w.weights().transpose();
vis += m_w.visibleBias().replicate(m_batch.rows(), 1);
probsLogistic(vis, m_variableSigma.replicate(m_batch.rows(), 1));
MatrixXd diffErr = m_batch - vis;
vis += m_w.visibleBias().replicate(__batch.rows(), 1);
probsLogistic(vis);
MatrixXd diffErr = __batch - vis;
diffErr.array() *= diffErr.array();
double err = diffErr.colwise().sum().sum();
cout << "error (total) = " << err << endl;
@@ -381,18 +283,6 @@ double Rbm::getProgress() const
return m_progress;
}
double Rbm::getEnergy(const VectorXd& visible, const VectorXd& hidden)
{
double energy;
double sigma = m_variableSigma.array().mean();
energy = m_w.visibleBias() * visible;
energy += m_w.hiddenBias() * hidden;
energy += visible.transpose() * m_w.weights() * hidden;
return -energy/(sigma*sigma);
}
void Rbm::toHidden(RowVectorXd &h, RowVectorXd const &v)
{
h = v * m_w.weights();
@@ -405,20 +295,15 @@ void Rbm::toVisible(RowVectorXd &v, RowVectorXd const &h)
v = h * m_w.weights().transpose();
v += m_w.visibleBias();
if (m_params.m_useVisibleGaussian)
if (!m_params.m_useVisibleGaussian)
{
// probsGaussian(v, m_variableSigma);
}
else
{
probsLogistic(v, m_variableSigma);
probsLogistic(v);
}
}
void Rbm::setConstantSigma(double value)
{
m_params.m_constantSigma = value;
m_variableSigma.fill(m_params.m_constantSigma);
onParamsChanged();
}
@@ -496,14 +381,6 @@ void Rbm::setNormalizeData(bool flag)
void Rbm::setDoLearnVariance(bool flag)
{
m_params.m_doLearnVariance = flag;
if (m_params.m_doLearnVariance && m_batch.rows())
{
m_variableSigma = calcSigma(m_batch);
}
else
{
m_variableSigma.fill(m_params.m_constantSigma);
}
onParamsChanged();
}
@@ -538,7 +415,7 @@ MatrixXd const& Rbm::getHiddenBatch()
MatrixXd const& Rbm::getBatch()
{
return m_batch;
return m_batch_normalized;
}
void Rbm::updateHiddenBatch()
@@ -547,8 +424,18 @@ void Rbm::updateHiddenBatch()
{
return;
}
m_batch_normalized.resize(m_batch.rows(), m_w.getNumHidden());
if (m_params.m_doNormalizeData)
{
normalizeData(m_batch_normalized, m_batch);
}
else
{
m_batch_normalized = m_batch;
}
m_h.resize(m_batch.rows(), m_w.getNumHidden());
m_h = m_batch * m_w.weights();
m_h = m_batch_normalized * m_w.weights();
m_h += m_w.hiddenBias().replicate(m_batch.rows(), 1);
probsLogistic(m_h);
}