jens
66fe4a018d
git-svn-id: http://moon:8086/svn/software/trunk/projects/Rbm@746 b431acfa-c32f-4a4a-93f1-934dc6c82436
357 lines
8.2 KiB
C++
357 lines
8.2 KiB
C++
/*
|
|
* To change this license header, choose License Headers in Project Properties.
|
|
* To change this template file, choose Tools | Templates
|
|
* and open the template in the editor.
|
|
*/
|
|
|
|
/*
|
|
* File: Rbm.cpp
|
|
* Author: jens
|
|
*
|
|
* Created on 21. Oktober 2019, 21:28
|
|
*/
|
|
|
|
#include <cassert>
|
|
#include "Rbm.hpp"
|
|
|
|
#define RBM_TRAIN_FLAT 1
|
|
|
|
Rbm::Rbm(size_t numVisible, size_t numHidden, size_t numContext)
|
|
: m_params()
|
|
, m_whv(numVisible, numHidden)
|
|
, m_whc(numContext, numHidden)
|
|
, m_bh(1, numHidden)
|
|
, m_bv(1, numVisible)
|
|
, m_bc(1, numContext)
|
|
{
|
|
assert(numVisible > 0);
|
|
assert(numHidden > 0);
|
|
if (numContext)
|
|
{
|
|
assert(numContext == numHidden);
|
|
}
|
|
Noise_Init(&m_noise, 0x32727155);
|
|
}
|
|
|
|
Rbm::Rbm(const Rbm& orig)
|
|
: m_params(orig.m_params)
|
|
, m_whv(orig.m_whv)
|
|
, m_whc(orig.m_whc)
|
|
, m_bh(orig.m_bh)
|
|
, m_bv(orig.m_bv)
|
|
, m_bc(orig.m_bc)
|
|
{
|
|
}
|
|
|
|
Rbm::~Rbm()
|
|
{
|
|
Noise_Free(&m_noise);
|
|
}
|
|
|
|
void Rbm::weightsInit(double stddev, double mu)
|
|
{
|
|
uniform(m_whv, stddev, mu);
|
|
uniform(m_whc, stddev, mu);
|
|
uniform(m_bh, stddev, mu);
|
|
uniform(m_bv, stddev, mu);
|
|
uniform(m_bc, stddev, mu);
|
|
}
|
|
|
|
void Rbm::fromJson(Json::Value rbm)
|
|
{
|
|
std::cout << "Importing Rbm" << std::endl;
|
|
m_params.fromJson(rbm["params"]);
|
|
}
|
|
|
|
Json::Value Rbm::toJson() const
|
|
{
|
|
std::cout << "Exporting Rbm" << std::endl;
|
|
Json::Value rbm;
|
|
rbm["params"] = m_params.toJson();
|
|
return rbm;
|
|
}
|
|
|
|
void Rbm::weightUpdate(arma::mat &v_state, arma::mat &dw, arma::mat &dbh, arma::mat &dbv)
|
|
{
|
|
arma::mat hid_probs = probsLogistic(toHiddenState(v_state));
|
|
arma::mat vis_probs(dbv.n_rows, dbv.n_cols);
|
|
|
|
arma::mat h_state = hid_probs;
|
|
|
|
// Sample hidden
|
|
if (!m_params.doRaoBlackwell)
|
|
{
|
|
h_state = sample(hid_probs);
|
|
}
|
|
|
|
// Update weights (positive phase)
|
|
dw = v_state.t() * h_state;
|
|
dbv = sum(v_state, 0);
|
|
dbh = sum(h_state, 0);
|
|
|
|
for (int gibbs=0; gibbs < m_params.numGibbs; gibbs++)
|
|
{
|
|
// Create visible reconstruction (a fantasy...) given hid
|
|
if (m_params.gibbsDoSampleHidden)
|
|
{
|
|
vis_probs = toVisibleProbs(sample(hid_probs));
|
|
}
|
|
else
|
|
{
|
|
vis_probs = toVisibleProbs(hid_probs);
|
|
}
|
|
|
|
// Create hidden representation given v
|
|
if (m_params.gibbsDoSampleVisible)
|
|
{
|
|
h_state = toHiddenState(sample(vis_probs));
|
|
}
|
|
else
|
|
{
|
|
h_state = toHiddenState(vis_probs);
|
|
}
|
|
|
|
hid_probs = probsLogistic(h_state);
|
|
|
|
}
|
|
|
|
// Update weights (negative phase)
|
|
dw -= vis_probs.t() * hid_probs;
|
|
dbv -= sum(vis_probs, 0);
|
|
dbh -= sum(hid_probs, 0);
|
|
}
|
|
|
|
void Rbm::train(const arma::mat& batch, IListener* pListener)
|
|
{
|
|
Status status;
|
|
double dProgress = 100.0/(batch.n_rows*m_params.numEpochs);
|
|
double progress = 0;
|
|
int lastProgress = -100;
|
|
int batchRowIndex = 0;
|
|
|
|
arma::mat grad_bias_v(arma::zeros(1, m_bv.n_cols));
|
|
arma::mat grad_bias_h(arma::zeros(1, m_bh.n_cols));
|
|
arma::mat grad_weight(arma::zeros(m_whv.n_rows, m_whv.n_cols));
|
|
arma::mat momentum_whv = arma::zeros(m_whv.n_rows, m_whv.n_cols);
|
|
arma::mat momentum_bias_v(arma::zeros(1, m_bv.n_cols));
|
|
arma::mat momentum_bias_h(arma::zeros(1, m_bh.n_cols));
|
|
arma::mat penalty_weights = arma::zeros(m_whv.n_rows, m_whv.n_cols);
|
|
|
|
arma::mat ctx_state(1, m_bc.n_cols);
|
|
arma::mat ctx_probs(1, m_bc.n_cols);
|
|
|
|
int trainingSizeRemain = batch.n_rows;
|
|
|
|
bool shouldAbort = false;
|
|
while (trainingSizeRemain && !shouldAbort)
|
|
{
|
|
int miniBatchSizeActual = std::min(m_params.miniBatchSize, trainingSizeRemain);
|
|
arma::mat miniBatch = batch.rows(batchRowIndex, batchRowIndex+miniBatchSizeActual-1);
|
|
trainingSizeRemain -= miniBatchSizeActual;
|
|
batchRowIndex += miniBatchSizeActual;
|
|
int scaler = std::min(m_params.miniBatchSize, (int)batch.n_rows);
|
|
double learning_rate = m_params.learningRate/scaler;
|
|
double weight_decay = m_params.weightDecay/scaler;
|
|
|
|
#if RBM_TRAIN_FLAT
|
|
arma::mat hid_probs(miniBatchSizeActual, m_bh.n_cols);
|
|
arma::mat vis_probs(miniBatchSizeActual, m_bv.n_cols);
|
|
#endif
|
|
|
|
arma::mat hid_state(miniBatchSizeActual, m_bh.n_cols);
|
|
arma::mat vis_state(miniBatchSizeActual, m_bv.n_cols);
|
|
|
|
// Create hidden layer base on training data
|
|
if (m_params.doSampleBatch)
|
|
{
|
|
// When the hidden units are being driven by data, always use stochastic binary states
|
|
vis_state = sample(miniBatch);
|
|
}
|
|
else
|
|
{
|
|
vis_state = miniBatch;
|
|
}
|
|
|
|
for (int epoch=0; epoch < m_params.numEpochs; epoch++)
|
|
{
|
|
#if RBM_TRAIN_FLAT
|
|
// Sample hidden
|
|
hid_probs = probsLogistic(toHiddenState(vis_state));
|
|
if (m_params.doRaoBlackwell)
|
|
{
|
|
hid_state = hid_probs;
|
|
}
|
|
else
|
|
{
|
|
hid_state = sample(hid_probs);
|
|
}
|
|
|
|
// Update weights (positive phase)
|
|
grad_weight = vis_state.t() * hid_state;
|
|
grad_bias_v = sum(vis_state, 0);
|
|
grad_bias_h = sum(hid_state, 0);
|
|
|
|
for (int gibbs=0; gibbs < m_params.numGibbs; gibbs++)
|
|
{
|
|
// Create visible reconstruction (a fantasy...) given hid
|
|
if (m_params.gibbsDoSampleHidden)
|
|
{
|
|
vis_probs = toVisibleProbs(sample(hid_probs));
|
|
}
|
|
else
|
|
{
|
|
vis_probs = toVisibleProbs(hid_probs);
|
|
}
|
|
|
|
// Create hidden representation given v
|
|
if (m_params.gibbsDoSampleVisible)
|
|
{
|
|
hid_state = toHiddenState(sample(vis_probs));
|
|
}
|
|
else
|
|
{
|
|
hid_state = toHiddenState(vis_probs);
|
|
}
|
|
hid_probs = probsLogistic(hid_state);
|
|
|
|
}
|
|
|
|
// Update weights (negative phase)
|
|
grad_weight -= vis_probs.t() * hid_probs;
|
|
grad_bias_v -= sum(vis_probs, 0);
|
|
grad_bias_h -= sum(hid_probs, 0);
|
|
#else
|
|
weightUpdate(vis_state, grad_weight, grad_bias_h, grad_bias_v);
|
|
#endif
|
|
|
|
penalty_weights = weight_decay*arma::sign(m_whv);
|
|
|
|
status.L1 = accu(abs(m_whv));
|
|
status.L2 = accu(m_whv % m_whv);
|
|
momentum_bias_v = m_params.momentum*momentum_bias_v + grad_bias_v;
|
|
momentum_bias_h = m_params.momentum*momentum_bias_h + grad_bias_h;
|
|
momentum_whv = m_params.momentum*momentum_whv + grad_weight - status.L2*penalty_weights;
|
|
|
|
m_bv += learning_rate*momentum_bias_v;
|
|
m_bh += learning_rate*momentum_bias_h;
|
|
m_whv += learning_rate*momentum_whv;
|
|
|
|
progress += dProgress*miniBatchSizeActual;
|
|
status.progress = (int)(progress + 0.5);
|
|
|
|
if (status.progress != lastProgress)
|
|
{
|
|
lastProgress = status.progress;
|
|
|
|
// Calculate error
|
|
arma::mat diffErr = miniBatch - toVisibleProbs(toHiddenProbs(vis_state));
|
|
arma::mat diffErr_squared = diffErr % diffErr;
|
|
status.err = accu(diffErr_squared)/diffErr_squared.n_elem;
|
|
if (pListener)
|
|
{
|
|
if(!pListener->onProgress(this, status))
|
|
{
|
|
shouldAbort = true;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
} // Number of epochs
|
|
|
|
} // number of mini batches
|
|
|
|
arma::mat diffErr = batch - toVisibleProbs(toHiddenProbs(batch));
|
|
arma::mat diffErr_squared = diffErr % diffErr;
|
|
status.err_total = accu(diffErr_squared)/diffErr_squared.n_elem;
|
|
|
|
if (pListener)
|
|
{
|
|
pListener->onProgress(this, status);
|
|
}
|
|
}
|
|
|
|
arma::mat Rbm::probsLogistic(const arma::mat &src)
|
|
{
|
|
return 1 / (1 + (arma::exp(-src)));
|
|
}
|
|
|
|
arma::mat Rbm::sample(const arma::mat &src)
|
|
{
|
|
arma::mat dst = src;
|
|
uniform(dst);
|
|
|
|
for (size_t i=0; i < src.n_rows; i++)
|
|
{
|
|
for (size_t j=0; j < src.n_cols; j++)
|
|
{
|
|
dst(i, j) = src(i, j) >= dst(i, j);
|
|
}
|
|
}
|
|
return dst;
|
|
}
|
|
|
|
arma::mat Rbm::toHiddenState(const arma::mat &visible) const
|
|
{
|
|
return visible * m_whv + arma::repmat(m_bh, visible.n_rows, 1);
|
|
}
|
|
|
|
arma::mat Rbm::toVisibleState(const arma::mat &hidden) const
|
|
{
|
|
return hidden * m_whv.t() + arma::repmat(m_bv, hidden.n_rows, 1);
|
|
}
|
|
|
|
arma::mat Rbm::toHiddenProbs(const arma::mat &visible) const
|
|
{
|
|
return probsLogistic(toHiddenState(visible));
|
|
}
|
|
|
|
arma::mat Rbm::toVisibleProbs(const arma::mat &hidden) const
|
|
{
|
|
return probsLogistic(toVisibleState(hidden));
|
|
}
|
|
|
|
arma::mat Rbm::normalize(const arma::mat& src)
|
|
{
|
|
double mean = arma::accu(src)/src.n_elem;
|
|
arma::mat x = src - mean;
|
|
arma::mat x2 = x % x;
|
|
double stddev = sqrt(arma::accu(x2)/x2.n_elem);
|
|
|
|
std::cout << "mean" << " : " << std::endl << mean << std::endl;
|
|
std::cout << "stddev" << ": " << std::endl << stddev << std::endl;
|
|
return x/stddev;
|
|
}
|
|
|
|
void Rbm::uniform(arma::mat& srcDst, double stdDev, double mu)
|
|
{
|
|
#if 1
|
|
for (size_t i=0; i < srcDst.n_rows; i++)
|
|
{
|
|
for (size_t j=0; j < srcDst.n_cols; j++)
|
|
{
|
|
srcDst(i, j) = stdDev*(Noise_Uniform(&m_noise) + mu - 0.5);
|
|
}
|
|
}
|
|
#else
|
|
srcDst = stdDev*(arma::randu(srcDst.n_rows, srcDst.n_cols) + mu - 0.5);
|
|
#endif
|
|
}
|
|
|
|
const arma::mat& Rbm::w() const
|
|
{
|
|
return m_whv;
|
|
}
|
|
|
|
const arma::mat& Rbm::bv() const
|
|
{
|
|
return m_bv;
|
|
}
|
|
|
|
const arma::mat& Rbm::bh() const
|
|
{
|
|
return m_bh;
|
|
}
|
|
|
|
|