jens/Rbm
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

- improved RBM

Browse Source 
git-svn-id: http://moon:8086/svn/software/trunk/projects/Rbm@567 b431acfa-c32f-4a4a-93f1-934dc6c82436
This commit is contained in:
jens
2019-10-24 18:22:30 +00:00
parent 0c71b2e6b8
commit 5b4aed047f
3 changed files with 60 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
source/Rbm.cpp
+43 -22
View File
@@ -21,9 +21,9 @@ Rbm::Rbm(const Params& params, arma::mat &w, arma::mat &bv, arma::mat &bh)
, m_bh(bh)
{
Noise_Init(&m_noise, 0x32727155);
uniform(m_w, 0.0, 0.1);
uniform(m_bh, 0.0, 0.1);
uniform(m_bv, 0.0, 0.1);
uniform(m_w, 0.0, m_params.m_weightInit);
uniform(m_bh, 0.0, m_params.m_weightInit);
uniform(m_bv, 0.0, m_params.m_weightInit);
}
Rbm::Rbm(const Rbm& orig)
@@ -90,7 +90,7 @@ void Rbm::train(const arma::mat& batch, size_t numEpochs, size_t miniBatchSize,
vis_state = miniBatch;
}
hid_state = vis_state * m_w + arma::repmat(m_bh, miniBatchSizeActual, 1);
hid_state = toHiddenState(vis_state);
hid_probs = probsLogistic(hid_state);
// Sample hidden
@@ -110,32 +110,33 @@ void Rbm::train(const arma::mat& batch, size_t numEpochs, size_t miniBatchSize,
for (gibbs=0; gibbs < m_params.m_numGibbs; gibbs++)
{
// Create hidden representation given v
hid_state = sample(hid_probs);
// Create visible reconstruction (a fantasy...) given hid
vis_state = hid_state * m_w.t() + arma::repmat(m_bv, miniBatchSizeActual, 1);
vis_probs = probsLogistic(vis_state);
if (m_params.m_doSampleVisible)
if (m_params.m_gibbsDoSampleHidden)
{
vis_state = sample(vis_probs);
vis_probs = toVisibleProbs(sample(hid_probs));
}
else
{
vis_state = vis_probs;
vis_probs = toVisibleProbs(hid_probs);
}
// Create hidden representation given v
hid_state = vis_state * m_w + repmat(m_bh, miniBatchSizeActual, 1);
if (m_params.m_gibbsDoSampleVisible)
{
hid_state = toHiddenState(sample(vis_probs));
}
else
{
hid_state = toHiddenState(vis_probs);
}
hid_probs = probsLogistic(hid_state);
}
// Update weights (negative phase)
grad_weight -= vis_probs.t() * hid_probs;
grad_bias_v -= sum(vis_probs, 0);
grad_bias_h -= sum(hid_probs, 0);
grad_weight -= vis_probs.t() * hid_probs;
penalty_weights = weight_decay*arma::sign(m_w);
@@ -157,7 +158,8 @@ void Rbm::train(const arma::mat& batch, size_t numEpochs, size_t miniBatchSize,
arma::mat diffErr = miniBatch - vis_probs;
arma::mat diffErr_squared = diffErr % diffErr;
status.err = accu(diffErr_squared)/diffErr_squared.n_elem;
status.err_total = 0;
if (pListener)
{
if(!pListener->onProgress(status))
@@ -167,6 +169,17 @@ void Rbm::train(const arma::mat& batch, size_t numEpochs, size_t miniBatchSize,
}
} // number of mini batches
arma::mat hid_probs = toHiddenProbs(batch);
arma::mat vis_probs = toVisibleProbs(hid_probs);
arma::mat diffErr = batch - vis_probs;
arma::mat diffErr_squared = diffErr % diffErr;
status.err_total = accu(diffErr_squared)/diffErr_squared.n_elem;
if (pListener)
{
pListener->onProgress(status);
}
}
arma::mat Rbm::probsLogistic(const arma::mat &src)
@@ -187,16 +200,24 @@ arma::mat Rbm::sample(const arma::mat &src)
return dst;
}
arma::mat Rbm::toHidden(const arma::mat &visible)
arma::mat Rbm::toHiddenState(const arma::mat &visible)
{
arma::mat h = visible * m_w + arma::repmat(m_bh, visible.n_rows, 1);
return probsLogistic(h);
return visible * m_w + arma::repmat(m_bh, visible.n_rows, 1);
}
arma::mat Rbm::toVisible(const arma::mat &hidden)
arma::mat Rbm::toVisibleState(const arma::mat &hidden)
{
arma::mat v = hidden * m_w.t() + arma::repmat(m_bv, hidden.n_rows, 1);
return probsLogistic(v);
return hidden * m_w.t() + arma::repmat(m_bv, hidden.n_rows, 1);
}
arma::mat Rbm::toHiddenProbs(const arma::mat &visible)
{
return probsLogistic(toHiddenState(visible));
}
arma::mat Rbm::toVisibleProbs(const arma::mat &hidden)
{
return probsLogistic(toVisibleState(hidden));
}
void Rbm::uniform(arma::mat& srcDst, double mu, double stdDev)
source/Rbm.hpp
+14 -6
View File
@@ -24,21 +24,25 @@ public:
struct Params
{
Params()
: m_weightDecay(0.0)
: m_weightInit(0.01)
, m_weightDecay(0.0)
, m_learningRate(0.1)
, m_momentum(0.5)
, m_doRaoBlackwell(true)
, m_doSampleVisible(false)
, m_gibbsDoSampleVisible(false)
, m_gibbsDoSampleHidden(false)
, m_doSampleBatch(false)
, m_numGibbs(1)
, m_numGibbs(5)
{
}
double m_weightInit;
double m_weightDecay;
double m_learningRate;
double m_momentum;
bool m_doRaoBlackwell;
bool m_doSampleVisible;
bool m_gibbsDoSampleVisible;
bool m_gibbsDoSampleHidden;
bool m_doSampleBatch;
size_t m_numGibbs;
};
@@ -49,6 +53,7 @@ public:
size_t trainingSizeRemain;
double progress;
double err;
double err_total;
double L1;
double L2;
};
@@ -70,8 +75,11 @@ public:
virtual ~Rbm();
void train(arma::mat const &batch, size_t numEpochs, size_t sizeMiniBatch, IListener *pListener);
arma::mat toHidden(const arma::mat &v);
arma::mat toVisible(const arma::mat &h);
arma::mat toHiddenState(const arma::mat &visible);
arma::mat toVisibleState(const arma::mat &hidden);
arma::mat toHiddenProbs(const arma::mat &visible);
arma::mat toVisibleProbs(const arma::mat &hidden);
arma::mat& weights();
arma::mat& bias_visible();
arma::mat& bias_hidden();
source/main.cpp
+3 -2
View File
@@ -16,6 +16,7 @@ class RbmListener : public Rbm::IListener
std::cout << "epoch : " << status.epoch << std::endl;
std::cout << "trainingSizeRemain: " << status.trainingSizeRemain << std::endl;
std::cout << "error (per mini batch) = " << status.err << std::endl;
std::cout << "error (total) = " << status.err_total << std::endl;
std::cout << "L1 = " << status.L1 << std::endl;
std::cout << "L2 = " << status.L2 << std::endl;
@@ -122,7 +123,7 @@ int main()
rbm.train(batch, 100, 100, &statusDisplay);
saveWeight("mnist_2.weights.dat", 28, 28, w, bv, bh);
arma::mat v = arma::randu(numTraining, numVisible);
arma::mat h = rbm.toHidden(v);
arma::mat r = rbm.toVisible(h);
arma::mat h = rbm.toHiddenProbs(v);
arma::mat r = rbm.toVisibleProbs(h);
return 0;
}