424 lines
10 KiB
C++
424 lines
10 KiB
C++
/*
|
|
* To change this license header, choose License Headers in Project Properties.
|
|
* To change this template file, choose Tools | Templates
|
|
* and open the template in the editor.
|
|
*/
|
|
|
|
/*
|
|
* File: Rbm.cpp
|
|
* Author: jens
|
|
*
|
|
* Created on 21. Oktober 2019, 21:28
|
|
*/
|
|
|
|
#include <cassert>
|
|
#include "Rbm.hpp"
|
|
#include "matutils.hpp"
|
|
|
|
using namespace Matutils;
|
|
|
|
Rbm::Rbm(size_t numVisible, size_t numHidden)
|
|
: m_params()
|
|
, m_whv(numVisible, numHidden)
|
|
, m_bh(1, numHidden)
|
|
, m_bv(1, numVisible)
|
|
{
|
|
assert(numVisible > 0);
|
|
assert(numHidden > 0);
|
|
}
|
|
|
|
Rbm::Rbm(const Rbm& orig)
|
|
: m_params(orig.m_params)
|
|
, m_whv(orig.m_whv)
|
|
, m_bh(orig.m_bh)
|
|
, m_bv(orig.m_bv)
|
|
{
|
|
}
|
|
|
|
Rbm::~Rbm()
|
|
{
|
|
}
|
|
|
|
size_t Rbm::numHidden() const
|
|
{
|
|
return m_bh.size();
|
|
}
|
|
|
|
size_t Rbm::numVisible() const
|
|
{
|
|
return m_bv.size();
|
|
}
|
|
|
|
Rbm::Params& Rbm::params()
|
|
{
|
|
return m_params;
|
|
}
|
|
|
|
arma::mat Rbm::rms_error(arma::mat diffErr)
|
|
{
|
|
arma::mat diffErr_squared = diffErr % diffErr;
|
|
return arma::sum(diffErr_squared, 1) * 1.0 / diffErr_squared.n_cols;
|
|
}
|
|
|
|
double Rbm::rms_error_accu(arma::mat diffErr)
|
|
{
|
|
arma::mat diffErr_squared = diffErr % diffErr;
|
|
return arma::accu(diffErr_squared) / diffErr_squared.n_elem;
|
|
}
|
|
|
|
arma::mat Rbm::toHiddenProbs(const arma::mat& visible) const
|
|
{
|
|
return Rbm::prob(v_to_h(visible));
|
|
}
|
|
|
|
arma::mat Rbm::toVisibleProbs(const arma::mat& hidden) const
|
|
{
|
|
return Rbm::prob(h_to_v(hidden));
|
|
}
|
|
|
|
void Rbm::weightsAssign(const arma::mat& w, const arma::mat& bh, const arma::mat& bv)
|
|
{
|
|
m_whv.submat(0, 0, w.n_rows - 1, w.n_cols - 1) = w;
|
|
m_bh.submat(0, 0, bh.n_rows - 1, bh.n_cols - 1) = bh;
|
|
m_bv.submat(0, 0, bv.n_rows - 1, bv.n_cols - 1) = bv;
|
|
}
|
|
|
|
void Rbm::weightsInit(double stddev, double mu)
|
|
{
|
|
uniform(m_whv, stddev, mu);
|
|
uniform(m_bh, 0, mu);
|
|
uniform(m_bv, 0, mu);
|
|
}
|
|
|
|
void Rbm::fromJson(Json::Value rbm)
|
|
{
|
|
std::cout << "Importing Rbm" << std::endl;
|
|
m_params.fromJson(rbm["params"]);
|
|
}
|
|
|
|
Json::Value Rbm::toJson() const
|
|
{
|
|
std::cout << "Exporting Rbm" << std::endl;
|
|
Json::Value rbm;
|
|
rbm["params"] = m_params.toJson();
|
|
return rbm;
|
|
}
|
|
|
|
void Rbm::gibbs_vh(arma::mat &v_probs, arma::mat &h_probs) const
|
|
{
|
|
for (int i=0; i < m_params.numGibbs; i++)
|
|
{
|
|
// Create hidden representation given v
|
|
h_probs = prob(v_to_h(v_probs));
|
|
|
|
// Create visible reconstruction (a fantasy...) given hid
|
|
v_probs = prob(h_to_v(h_probs));
|
|
}
|
|
}
|
|
|
|
void Rbm::gibbs_hv(arma::mat &h_probs, arma::mat &v_probs) const
|
|
{
|
|
for (int i=0; i < m_params.numGibbs; i++)
|
|
{
|
|
// Create visible reconstruction (a fantasy...) given hid
|
|
v_probs = prob(h_to_v(h_probs));
|
|
|
|
// Create hidden representation given v
|
|
h_probs = prob(v_to_h(v_probs));
|
|
}
|
|
}
|
|
|
|
#if 1
|
|
void Rbm::contrastiveDivergence(arma::mat const &v_data, arma::mat &dw, arma::mat &dbh, arma::mat &dbv)
|
|
{
|
|
arma::mat v_probs(v_data);
|
|
arma::mat h_states = v_to_h(v_data);
|
|
arma::mat h_probs;
|
|
|
|
// Start positive phase
|
|
arma::mat poshidprobs = prob(v_to_h(v_data));
|
|
arma::mat posprods = v_data.t() * poshidprobs;
|
|
arma::mat poshidact = arma::sum(poshidprobs);
|
|
arma::mat posvisact = arma::sum(v_data);
|
|
|
|
// End of positive phase
|
|
arma::mat poshidstates = sample(poshidprobs);
|
|
|
|
// Start negative phase
|
|
arma::mat negdata = prob(h_to_v(poshidstates));
|
|
arma::mat neghidprobs = prob(v_to_h(negdata));
|
|
arma::mat negprods = negdata.t() * neghidprobs;
|
|
arma::mat neghidact = arma::sum(neghidprobs);
|
|
arma::mat negvisact = arma::sum(negdata);
|
|
|
|
// Update weight deltas
|
|
dw = posprods - negprods;
|
|
dbv = posvisact - negvisact;
|
|
dbh = poshidact - neghidact;
|
|
}
|
|
|
|
void Rbm::train(arma::mat const &batch, IListener* pListener)
|
|
{
|
|
Status status;
|
|
double dProgress = 100.0/(batch.n_rows*m_params.numEpochs);
|
|
double progress = 0;
|
|
int lastProgress = -100;
|
|
int batchRowIndex = 0;
|
|
|
|
arma::mat dbv(arma::zeros(1, m_bv.n_cols));
|
|
arma::mat dbh(arma::zeros(1, m_bh.n_cols));
|
|
arma::mat dwhv(arma::zeros(m_whv.n_rows, m_whv.n_cols));
|
|
arma::mat inc_whv = arma::zeros(m_whv.n_rows, m_whv.n_cols);
|
|
arma::mat inc_bv(arma::zeros(1, m_bv.n_cols));
|
|
arma::mat inc_bh(arma::zeros(1, m_bh.n_cols));
|
|
|
|
int trainingSizeRemain = batch.n_rows;
|
|
|
|
bool shouldAbort = false;
|
|
while (trainingSizeRemain && !shouldAbort)
|
|
{
|
|
int miniBatchSizeActual = std::min(m_params.miniBatchSize, trainingSizeRemain);
|
|
arma::mat miniBatch = batch.rows(batchRowIndex, batchRowIndex+miniBatchSizeActual-1);
|
|
trainingSizeRemain -= miniBatchSizeActual;
|
|
batchRowIndex += miniBatchSizeActual;
|
|
int numcases = std::min(m_params.miniBatchSize, (int)batch.n_rows);
|
|
|
|
arma::mat v_states(miniBatch);
|
|
|
|
// Create hidden layer base on training data
|
|
if (m_params.doSampleBatch)
|
|
{
|
|
// When the hidden units are being driven by data, always use stochastic binary states
|
|
v_states = sample(miniBatch);
|
|
}
|
|
|
|
for (int epoch=0; epoch < m_params.numEpochs; epoch++)
|
|
{
|
|
// Contrastive divergence learning: calculate gradients
|
|
contrastiveDivergence(v_states, dwhv, dbh, dbv);
|
|
|
|
// Adjust weight and biases
|
|
inc_bv = m_params.momentum*inc_bv + m_params.learningRate/numcases*dbv;
|
|
inc_bh = m_params.momentum*inc_bh + m_params.learningRate/numcases*dbh;
|
|
inc_whv = m_params.momentum*inc_whv + m_params.learningRate*(dwhv/numcases - m_params.weightDecay*m_whv);
|
|
|
|
m_bv += inc_bv;
|
|
m_bh += inc_bh;
|
|
m_whv += inc_whv;
|
|
|
|
progress += dProgress*miniBatchSizeActual;
|
|
status.progress = (int)(progress + 0.5);
|
|
|
|
// Update status
|
|
if (status.progress != lastProgress)
|
|
{
|
|
lastProgress = status.progress;
|
|
|
|
// Calculate error
|
|
status.err = rms_error_accu(miniBatch - prob(h_to_v(prob(v_to_h(v_states)))));
|
|
if (pListener)
|
|
{
|
|
if(!pListener->onProgress(this, status))
|
|
{
|
|
shouldAbort = true;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
} // Number of epochs
|
|
|
|
} // number of mini batches
|
|
|
|
// Update final status
|
|
status.err_total = rms_error_accu(batch - prob(h_to_v(prob(v_to_h(batch)))));
|
|
|
|
if (pListener)
|
|
{
|
|
pListener->onProgress(this, status);
|
|
}
|
|
}
|
|
#else
|
|
void Rbm::contrastiveDivergence(arma::mat const &v_states, arma::mat &dw, arma::mat &dbh, arma::mat &dbv)
|
|
{
|
|
arma::mat v_probs(v_states);
|
|
arma::mat h_states = v_to_h(v_states);
|
|
arma::mat h_probs;
|
|
|
|
// Sample hidden
|
|
if (m_params.doGaussianHidden)
|
|
{
|
|
h_probs = h_states;
|
|
h_states = h_probs + arma::randn(h_probs.n_rows, h_probs.n_cols);
|
|
}
|
|
else if (m_params.doRaoBlackwell)
|
|
{
|
|
h_probs = prob(v_to_h(v_states));
|
|
h_states = h_probs;
|
|
}
|
|
else
|
|
{
|
|
h_probs = prob(v_to_h(v_states));
|
|
h_states = sample(h_probs);
|
|
}
|
|
|
|
// Update weights (positive phase)
|
|
dw = v_states.t() * h_states;
|
|
dbv = sum(v_states, 0);
|
|
dbh = sum(h_states, 0);
|
|
|
|
// Gibbs sampling with training params
|
|
for (int i=0; i < m_params.numGibbs; i++)
|
|
{
|
|
// Create visible reconstruction (a fantasy...) given hid
|
|
if (m_params.gibbsDoSampleHidden)
|
|
{
|
|
v_probs = prob(h_to_v(sample(h_probs)));
|
|
}
|
|
else
|
|
{
|
|
v_probs = prob(h_to_v(h_probs));
|
|
}
|
|
|
|
// Create hidden representation given v
|
|
if (m_params.doGaussianHidden)
|
|
{
|
|
h_probs = v_to_h(v_probs);
|
|
}
|
|
else if (m_params.gibbsDoSampleVisible)
|
|
{
|
|
h_probs = prob(v_to_h(sample(v_probs)));
|
|
}
|
|
else
|
|
{
|
|
h_probs = prob(v_to_h(v_probs));
|
|
}
|
|
}
|
|
|
|
// Update weights (negative phase)
|
|
dw -= v_probs.t() * h_probs;
|
|
dbv -= sum(v_probs, 0);
|
|
dbh -= sum(h_probs, 0);
|
|
}
|
|
void Rbm::train(arma::mat const &batch, IListener* pListener)
|
|
{
|
|
Status status;
|
|
double dProgress = 100.0/(batch.n_rows*m_params.numEpochs);
|
|
double progress = 0;
|
|
int lastProgress = -100;
|
|
int batchRowIndex = 0;
|
|
|
|
arma::mat dbv(arma::zeros(1, m_bv.n_cols));
|
|
arma::mat dbh(arma::zeros(1, m_bh.n_cols));
|
|
arma::mat dwhv(arma::zeros(m_whv.n_rows, m_whv.n_cols));
|
|
arma::mat momentum_whv = arma::zeros(m_whv.n_rows, m_whv.n_cols);
|
|
arma::mat momentum_bv(arma::zeros(1, m_bv.n_cols));
|
|
arma::mat momentum_bh(arma::zeros(1, m_bh.n_cols));
|
|
arma::mat penalty_whv = arma::zeros(m_whv.n_rows, m_whv.n_cols);
|
|
|
|
int trainingSizeRemain = batch.n_rows;
|
|
|
|
bool shouldAbort = false;
|
|
while (trainingSizeRemain && !shouldAbort)
|
|
{
|
|
int miniBatchSizeActual = std::min(m_params.miniBatchSize, trainingSizeRemain);
|
|
arma::mat miniBatch = batch.rows(batchRowIndex, batchRowIndex+miniBatchSizeActual-1);
|
|
trainingSizeRemain -= miniBatchSizeActual;
|
|
batchRowIndex += miniBatchSizeActual;
|
|
int scaler = std::min(m_params.miniBatchSize, (int)batch.n_rows);
|
|
double learning_rate = m_params.learningRate/scaler;
|
|
double weight_decay = m_params.weightDecay/scaler;
|
|
|
|
arma::mat v_states(miniBatch);
|
|
|
|
// Create hidden layer base on training data
|
|
if (m_params.doSampleBatch)
|
|
{
|
|
// When the hidden units are being driven by data, always use stochastic binary states
|
|
v_states = sample(miniBatch);
|
|
}
|
|
|
|
for (int epoch=0; epoch < m_params.numEpochs; epoch++)
|
|
{
|
|
// Contrastive divergence learning: calculate gradients
|
|
contrastiveDivergence(v_states, dwhv, dbh, dbv);
|
|
|
|
// Adjust weight and biases
|
|
penalty_whv = weight_decay*arma::sign(m_whv);
|
|
|
|
status.L1 = accu(abs(m_whv));
|
|
status.L2 = accu(m_whv % m_whv);
|
|
momentum_bv = m_params.momentum*momentum_bv + dbv;
|
|
momentum_bh = m_params.momentum*momentum_bh + dbh;
|
|
momentum_whv = m_params.momentum*momentum_whv + dwhv - status.L2*penalty_whv;
|
|
|
|
m_bv += learning_rate*momentum_bv;
|
|
m_bh += learning_rate*momentum_bh;
|
|
m_whv += learning_rate*momentum_whv;
|
|
|
|
progress += dProgress*miniBatchSizeActual;
|
|
status.progress = (int)(progress + 0.5);
|
|
|
|
// Update status
|
|
if (status.progress != lastProgress)
|
|
{
|
|
lastProgress = status.progress;
|
|
|
|
// Calculate error
|
|
status.err = rms_error_accu(miniBatch - prob(h_to_v(prob(v_to_h(v_states)))));
|
|
if (pListener)
|
|
{
|
|
if(!pListener->onProgress(this, status))
|
|
{
|
|
shouldAbort = true;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
} // Number of epochs
|
|
|
|
} // number of mini batches
|
|
|
|
// Update final status
|
|
status.err_total = rms_error_accu(batch - prob(h_to_v(prob(v_to_h(batch)))));
|
|
|
|
if (pListener)
|
|
{
|
|
pListener->onProgress(this, status);
|
|
}
|
|
}
|
|
#endif
|
|
|
|
arma::mat Rbm::prob(const arma::mat &src)
|
|
{
|
|
return 1 / (1 + (arma::exp(-src)));
|
|
}
|
|
|
|
arma::mat Rbm::v_to_h(const arma::mat &visible) const
|
|
{
|
|
return visible * m_whv + arma::repmat(m_bh, visible.n_rows, 1);
|
|
}
|
|
|
|
arma::mat Rbm::h_to_v(const arma::mat &hidden) const
|
|
{
|
|
return hidden * m_whv.t() + arma::repmat(m_bv, hidden.n_rows, 1);
|
|
}
|
|
|
|
const arma::mat& Rbm::whv() const
|
|
{
|
|
return m_whv;
|
|
}
|
|
|
|
const arma::mat& Rbm::bv() const
|
|
{
|
|
return m_bv;
|
|
}
|
|
|
|
const arma::mat& Rbm::bh() const
|
|
{
|
|
return m_bh;
|
|
}
|
|
|
|
|