jens
866a0349db
- DrawComponent: use fixed value scaling - Rbm: fixed weight decay - Rbm: fixed sparsity git-svn-id: http://moon:8086/svn/software/trunk/projects/RBM@298 b431acfa-c32f-4a4a-93f1-934dc6c82436
594 lines
12 KiB
C++
594 lines
12 KiB
C++
/*
|
|
* Rbm.hpp
|
|
*
|
|
* Created on: 21.09.2014
|
|
* Author: jens
|
|
*/
|
|
|
|
#ifndef RBM_HPP_
|
|
#define RBM_HPP_
|
|
|
|
#include "Weights.hpp"
|
|
#include <cmath>
|
|
#include <Eigen/Dense>
|
|
|
|
using namespace Eigen;
|
|
|
|
void mylog(const char* format, ...);
|
|
#define printf mylog
|
|
|
|
#define EPSILON_SIGMA 0.001
|
|
|
|
class Rbm
|
|
{
|
|
public:
|
|
struct Params
|
|
{
|
|
Params()
|
|
: m_constantSigma(1.0)
|
|
, m_sigmaDecay(1.0)
|
|
, m_weightDecay(0.00001)
|
|
, m_lambda(1.0)
|
|
, m_sparsity(0.05)
|
|
, m_muWeights(0.1)
|
|
, m_muSparsity(0.01)
|
|
, m_momentum(0.5)
|
|
, m_useVisibleGaussian(false)
|
|
, m_doRaoBlackwell(true)
|
|
, m_doSampleVisible(false)
|
|
, m_doSampleBatch(false)
|
|
, m_doSparse(false)
|
|
, m_doNormalizeData(false)
|
|
, m_doLearnVariance(false)
|
|
, m_numGibbs(1)
|
|
{
|
|
}
|
|
|
|
double m_constantSigma;
|
|
double m_sigmaDecay;
|
|
double m_weightDecay;
|
|
double m_lambda;
|
|
double m_sparsity;
|
|
double m_muWeights;
|
|
double m_muSparsity;
|
|
double m_momentum;
|
|
bool m_useVisibleGaussian;
|
|
bool m_doRaoBlackwell;
|
|
bool m_doSampleVisible;
|
|
bool m_doSampleBatch;
|
|
bool m_doSparse;
|
|
bool m_doNormalizeData;
|
|
bool m_doLearnVariance;
|
|
size_t m_numGibbs;
|
|
};
|
|
|
|
Rbm(Weights &weights, const MatrixXd &batch)
|
|
: m_w(weights)
|
|
, m_batch(batch)
|
|
, m_variableSigma(weights.getNumVisible())
|
|
, m_progress(0)
|
|
{
|
|
Noise_Init(&m_noise, 0x32727155);
|
|
m_variableSigma.fill(m_params.m_constantSigma);
|
|
updateHiddenBatch();
|
|
}
|
|
|
|
~Rbm()
|
|
{
|
|
Noise_Free(&m_noise);
|
|
}
|
|
|
|
void sample(MatrixXd &srcDst)
|
|
{
|
|
sample(srcDst, srcDst);
|
|
}
|
|
|
|
void sample(MatrixXd &dst, MatrixXd const &src)
|
|
{
|
|
uint32_t i;
|
|
|
|
for (i=0; i < src.array().size(); i++)
|
|
{
|
|
dst.array()(i) = (double)(src.array()(i) > Noise_Uniform(&m_noise));
|
|
}
|
|
}
|
|
|
|
void probsLogistic(MatrixXd &src)
|
|
{
|
|
src.array() = (-src.array()).exp();
|
|
src.array() += 1;
|
|
src.array() = 1.0/src.array();
|
|
}
|
|
|
|
void probsLogistic(RowVectorXd &src)
|
|
{
|
|
src.array() = (-src.array()).exp();
|
|
src.array() += 1;
|
|
src.array() = 1.0/src.array();
|
|
}
|
|
|
|
void probsLogistic(MatrixXd &src, const MatrixXd &sigma)
|
|
{
|
|
src.array() /= (sigma.array() + EPSILON_SIGMA);
|
|
src.array() = (-src.array()).exp();
|
|
src.array() += 1;
|
|
src.array() = 1.0/src.array();
|
|
}
|
|
|
|
void probsLogistic(RowVectorXd &src, const RowVectorXd &sigma)
|
|
{
|
|
src.array() /= (sigma.array() + EPSILON_SIGMA);
|
|
src.array() = (-src.array()).exp();
|
|
src.array() += 1;
|
|
src.array() = 1.0/src.array();
|
|
}
|
|
|
|
void probsGaussian(MatrixXd &src, const MatrixXd &sigma)
|
|
{
|
|
src.array() = 1 - src.array();
|
|
src.array() *= src.array();
|
|
src.array() *= -0.5;
|
|
|
|
MatrixXd var = sigma;
|
|
var.array() += EPSILON_SIGMA;
|
|
var.array() *= var.array();
|
|
|
|
src.array() /= var.array();
|
|
src.array() = src.array().exp();
|
|
|
|
MatrixXd k = var;
|
|
|
|
k.array() *= 2*3.14159265359;
|
|
k.array() = k.array().sqrt();
|
|
k.array() = 1.0/k.array();
|
|
|
|
src.array() *= k.array();
|
|
}
|
|
|
|
void probsGaussian(RowVectorXd &src, const RowVectorXd &sigma)
|
|
{
|
|
src.array() = 1 - src.array();
|
|
src.array() *= src.array();
|
|
src.array() *= -0.5;
|
|
|
|
RowVectorXd var = sigma;
|
|
var.array() += EPSILON_SIGMA;
|
|
var.array() *= var.array();
|
|
|
|
src.array() /= var.array();
|
|
src.array() = src.array().exp();
|
|
|
|
RowVectorXd k = var;
|
|
|
|
k.array() *= 2*3.14159265359;
|
|
k.array() = k.array().sqrt();
|
|
k.array() = 1.0/k.array();
|
|
|
|
src.array() *= k.array();
|
|
}
|
|
|
|
void sampleGaussian(MatrixXd &dst, MatrixXd const &src, const MatrixXd &sigma)
|
|
{
|
|
uint32_t i;
|
|
|
|
for (i=0; i < src.array().size(); i++)
|
|
{
|
|
dst.array()(i) = sigma(i)*Noise_Gaussian(&m_noise) + src.array()(i);
|
|
}
|
|
}
|
|
|
|
void sampleGaussian(MatrixXd &src, const MatrixXd &sigma)
|
|
{
|
|
uint32_t i;
|
|
|
|
for (i=0; i < src.array().size(); i++)
|
|
{
|
|
src.array()(i) = sigma(i)*Noise_Gaussian(&m_noise) + src.array()(i);
|
|
}
|
|
}
|
|
|
|
RowVectorXd normalizeData(RowVectorXd const &src, RowVectorXd const &mu, RowVectorXd const &var)
|
|
{
|
|
// Remove mean
|
|
RowVectorXd res = src - mu;
|
|
// res.array() /= var.array() + EPSILON_SIGMA;
|
|
|
|
// cout << __PRETTY_FUNCTION__ << ": " << res << endl;
|
|
return res;
|
|
}
|
|
|
|
RowVectorXd calcMean(MatrixXd const &batch)
|
|
{
|
|
// Remove mean
|
|
RowVectorXd res = batch.colwise().mean();
|
|
|
|
// cout << __PRETTY_FUNCTION__ << ": " << res << endl;
|
|
return res;
|
|
|
|
}
|
|
|
|
RowVectorXd calcSigma(MatrixXd const &batch)
|
|
{
|
|
MatrixXd x = batch.rowwise() - batch.colwise().mean();
|
|
|
|
x.array() *= x.array();
|
|
|
|
RowVectorXd res = x.colwise().mean().array().sqrt();
|
|
|
|
// cout << __PRETTY_FUNCTION__ << ": " << res << endl;
|
|
return res;
|
|
}
|
|
|
|
MatrixXd calcZ(MatrixXd &v, MatrixXd &h)
|
|
{
|
|
|
|
MatrixXd t1(v.rows(), m_w.getNumVisible());
|
|
|
|
t1 = v - m_w.visibleBias().transpose().replicate(v.rows(), 1);
|
|
t1.array() *= t1.array();
|
|
t1.array() *= 0.5;
|
|
|
|
t1 -= (h * m_w.weights().transpose());
|
|
|
|
return t1;
|
|
}
|
|
|
|
void train(uint32_t numEpochs, double sigmaMin = 0.05)
|
|
{
|
|
uint32_t t, i;
|
|
uint32_t epoch;
|
|
uint32_t gibbs;
|
|
|
|
size_t batchSize = m_batch.rows();
|
|
|
|
double dProgress = 1.0/numEpochs;
|
|
double mu_w = m_params.m_muWeights/batchSize;
|
|
double mu_biasV = m_params.m_muWeights/batchSize;
|
|
double mu_biasH = m_params.m_muWeights/batchSize;
|
|
|
|
|
|
m_v.resize(batchSize, m_w.getNumVisible());
|
|
MatrixXd h(batchSize, m_w.getNumHidden());
|
|
|
|
MatrixXd dBiasV_curr(MatrixXd::Zero(1, m_w.getNumVisible()));
|
|
MatrixXd dBiasH_curr(MatrixXd::Zero(1, m_w.getNumHidden()));
|
|
MatrixXd dW_curr(MatrixXd::Zero(m_w.getNumVisible(), m_w.getNumHidden()));
|
|
MatrixXd dBiasV(MatrixXd::Zero(1, m_w.getNumVisible()));
|
|
MatrixXd dBiasH(MatrixXd::Zero(1, m_w.getNumHidden()));
|
|
MatrixXd dW(MatrixXd::Zero(m_w.getNumVisible(), m_w.getNumHidden()));
|
|
|
|
MatrixXd diffErr(batchSize, m_w.getNumVisible());
|
|
|
|
MatrixXd batch = m_batch;
|
|
MatrixXd batch_sampled(batchSize, m_w.getNumVisible());
|
|
MatrixXd v_sampled(batchSize, m_w.getNumVisible());
|
|
|
|
if (m_params.m_doNormalizeData)
|
|
{
|
|
RowVectorXd mean = calcMean(batch);
|
|
for (i=0; i < batchSize; i++)
|
|
{
|
|
RowVectorXd x = batch.row(i);
|
|
batch.row(i) = normalizeData(x, mean, m_variableSigma);
|
|
}
|
|
}
|
|
|
|
m_progress = 0;
|
|
for (epoch=0; epoch < numEpochs; epoch++)
|
|
{
|
|
onProgressChanged();
|
|
|
|
if (m_params.m_doSampleBatch)
|
|
{
|
|
// When the hidden units are being driven by data, always use stochastic binary states
|
|
sample(batch_sampled, batch);
|
|
|
|
// Create hidden layer base on sampled training data
|
|
toHiddenBatch(h, batch_sampled);
|
|
}
|
|
else
|
|
{
|
|
// Create hidden layer base on training data
|
|
toHiddenBatch(h, batch);
|
|
}
|
|
// Sample hidden
|
|
if (!m_params.m_doRaoBlackwell)
|
|
{
|
|
sample(h);
|
|
}
|
|
|
|
// Update weights (positive phase)
|
|
dBiasV_curr = batch.colwise().sum();
|
|
dBiasH_curr = h.colwise().sum();
|
|
dW_curr = batch.transpose() * h;
|
|
|
|
for (gibbs=0; gibbs < m_params.m_numGibbs; gibbs++)
|
|
{
|
|
// Create visible reconstruction (a fantasy...) given h
|
|
toVisibleBatch(m_v, h);
|
|
if (m_params.m_useVisibleGaussian)
|
|
{
|
|
sampleGaussian(v_sampled, m_v, m_variableSigma.replicate(batchSize, 1));
|
|
toHiddenBatch(h, v_sampled);
|
|
}
|
|
else
|
|
{
|
|
probsLogistic(m_v, m_variableSigma.replicate(batchSize, 1));
|
|
if (m_params.m_doSampleVisible)
|
|
{
|
|
sample(v_sampled, m_v);
|
|
// Create hidden representation given sampled v
|
|
toHiddenBatch(h, v_sampled);
|
|
}
|
|
else
|
|
{
|
|
// Create hidden representation given v
|
|
toHiddenBatch(h, m_v);
|
|
}
|
|
}
|
|
if (!m_params.m_doRaoBlackwell)
|
|
{
|
|
sample(h);
|
|
}
|
|
}
|
|
|
|
// Update weights (negative phase)
|
|
dBiasV_curr -= m_v.colwise().sum();
|
|
dBiasH_curr -= h.colwise().sum();
|
|
dW_curr -= m_v.transpose() * h;
|
|
|
|
m_w.visibleBias() += mu_biasV*(m_params.m_momentum*dBiasV + (1-m_params.m_momentum)*dBiasV_curr);
|
|
dBiasV = dBiasV_curr;
|
|
|
|
if (m_params.m_doSparse)
|
|
{
|
|
MatrixXd h1 = h-MatrixXd::Ones(h.rows(), h.cols())*m_params.m_sparsity;
|
|
RowVectorXd hm = h1.colwise().mean();
|
|
m_w.hiddenBias() -= m_params.m_muSparsity * hm;
|
|
}
|
|
else
|
|
{
|
|
m_w.hiddenBias() += mu_biasH*(m_params.m_momentum*dBiasH + (1-m_params.m_momentum)*dBiasH_curr);
|
|
}
|
|
dBiasH = dBiasH_curr;
|
|
|
|
m_w.weights() -= m_params.m_weightDecay*m_w.weights();
|
|
m_w.weights() += mu_w*(m_params.m_momentum*dW + (1-m_params.m_momentum)*dW_curr);
|
|
dW = dW_curr;
|
|
|
|
if (m_variableSigma[0] > sigmaMin)
|
|
{
|
|
m_variableSigma.array() *= m_params.m_sigmaDecay;
|
|
}
|
|
|
|
m_progress += dProgress;
|
|
|
|
diffErr = m_batch - m_v;
|
|
diffErr.array() *= diffErr.array();
|
|
double err = diffErr.colwise().sum().sum();
|
|
|
|
cout << "err =" << endl;
|
|
cout << err << endl;
|
|
|
|
} // Number of epochs
|
|
|
|
updateHiddenBatch();
|
|
onProgressChanged();
|
|
}
|
|
|
|
double getProgress() const
|
|
{
|
|
return m_progress;
|
|
}
|
|
|
|
double getEnergy(const VectorXd& visible, const VectorXd& hidden)
|
|
{
|
|
double energy;
|
|
double sigma = m_variableSigma.array().mean();
|
|
|
|
energy = m_w.visibleBias() * visible;
|
|
energy += m_w.hiddenBias() * hidden;
|
|
energy += visible.transpose() * m_w.weights() * hidden;
|
|
|
|
return -energy/(sigma*sigma);
|
|
}
|
|
|
|
void toHidden(RowVectorXd &h, RowVectorXd const &v)
|
|
{
|
|
h = v * m_w.weights();
|
|
h += m_w.hiddenBias();
|
|
probsLogistic(h);
|
|
}
|
|
|
|
void toVisible(RowVectorXd &v, RowVectorXd const &h)
|
|
{
|
|
v = h * m_w.weights().transpose();
|
|
v += m_w.visibleBias();
|
|
|
|
if (m_params.m_useVisibleGaussian)
|
|
{
|
|
probsGaussian(v, m_variableSigma);
|
|
}
|
|
else
|
|
{
|
|
probsLogistic(v, m_variableSigma);
|
|
}
|
|
}
|
|
|
|
void setConstantSigma(double value)
|
|
{
|
|
m_params.m_constantSigma = value;
|
|
m_variableSigma.fill(m_params.m_constantSigma);
|
|
onParamsChanged();
|
|
}
|
|
|
|
RowVectorXd& getVariableSigma()
|
|
{
|
|
return m_variableSigma;
|
|
}
|
|
|
|
void setSigmaDecay(double value)
|
|
{
|
|
m_params.m_sigmaDecay = value;
|
|
onParamsChanged();
|
|
}
|
|
|
|
void setWeightDecay(double value)
|
|
{
|
|
m_params.m_weightDecay = value;
|
|
onParamsChanged();
|
|
}
|
|
|
|
void setLambda(double value)
|
|
{
|
|
m_params.m_lambda = value;
|
|
onParamsChanged();
|
|
}
|
|
|
|
void setSparsity(double value)
|
|
{
|
|
m_params.m_sparsity = value;
|
|
onParamsChanged();
|
|
}
|
|
|
|
void setUseVisibleGaussian(bool flag)
|
|
{
|
|
m_params.m_useVisibleGaussian = flag;
|
|
onParamsChanged();
|
|
}
|
|
|
|
void setDoRaoBlackwell(bool flag)
|
|
{
|
|
m_params.m_doRaoBlackwell = flag;
|
|
onParamsChanged();
|
|
}
|
|
|
|
void setDoSampleVisible(bool flag)
|
|
{
|
|
m_params.m_doSampleVisible = flag;
|
|
onParamsChanged();
|
|
}
|
|
|
|
void setDoSampleBatch(bool flag)
|
|
{
|
|
m_params.m_doSampleBatch = flag;
|
|
onParamsChanged();
|
|
}
|
|
|
|
void setDoSparse(bool flag)
|
|
{
|
|
m_params.m_doSparse = flag;
|
|
onParamsChanged();
|
|
}
|
|
|
|
void setNormalizeData(bool flag)
|
|
{
|
|
m_params.m_doNormalizeData = flag;
|
|
onParamsChanged();
|
|
}
|
|
|
|
void setDoLearnVariance(bool flag)
|
|
{
|
|
m_params.m_doLearnVariance = flag;
|
|
if (m_params.m_doLearnVariance && m_batch.rows())
|
|
{
|
|
m_variableSigma = calcSigma(m_batch);
|
|
}
|
|
else
|
|
{
|
|
m_variableSigma.fill(m_params.m_constantSigma);
|
|
}
|
|
onParamsChanged();
|
|
}
|
|
|
|
void setNumGibbs(size_t value)
|
|
{
|
|
m_params.m_numGibbs = value;
|
|
onParamsChanged();
|
|
}
|
|
|
|
void setMuWeights(double value)
|
|
{
|
|
m_params.m_muWeights = value;
|
|
onParamsChanged();
|
|
}
|
|
|
|
void setMuSparsity(double value)
|
|
{
|
|
m_params.m_muSparsity = value;
|
|
onParamsChanged();
|
|
}
|
|
|
|
void setMomentum(double value)
|
|
{
|
|
m_params.m_momentum = value;
|
|
onParamsChanged();
|
|
}
|
|
|
|
MatrixXd const& getHiddenBatch()
|
|
{
|
|
return m_h;
|
|
}
|
|
|
|
MatrixXd const& getVisibleBatch()
|
|
{
|
|
return m_v;
|
|
}
|
|
|
|
MatrixXd const& getBatch()
|
|
{
|
|
return m_batch;
|
|
}
|
|
|
|
void updateHiddenBatch()
|
|
{
|
|
m_h.resize(m_batch.rows(), m_w.getNumHidden());
|
|
toHiddenBatch(m_h, m_batch);
|
|
}
|
|
|
|
Params const& params()
|
|
{
|
|
return m_params;
|
|
}
|
|
|
|
private:
|
|
Weights &m_w;
|
|
MatrixXd const &m_batch;
|
|
MatrixXd m_v;
|
|
MatrixXd m_h;
|
|
RowVectorXd m_variableSigma;
|
|
noise_gen_t m_noise;
|
|
double m_progress;
|
|
Params m_params;
|
|
|
|
void toHiddenBatch(MatrixXd &h, MatrixXd const &v)
|
|
{
|
|
if (v.cols() == m_w.weights().rows())
|
|
{
|
|
h = v * m_w.weights();
|
|
h += m_w.hiddenBias().replicate(m_batch.rows(), 1);
|
|
probsLogistic(h);
|
|
}
|
|
}
|
|
|
|
void toVisibleBatch(MatrixXd &v, MatrixXd const &h)
|
|
{
|
|
v = h * m_w.weights().transpose();
|
|
v += m_w.visibleBias().replicate(m_batch.rows(), 1);
|
|
}
|
|
|
|
protected:
|
|
virtual void onProgressChanged()
|
|
{
|
|
}
|
|
|
|
virtual void onParamsChanged()
|
|
{
|
|
}
|
|
};
|
|
|
|
|
|
|
|
|
|
#endif /* RBM_HPP_ */
|