diff --git a/source/Rbm.cpp b/source/Rbm.cpp index 0bc1e4a..014fce5 100644 --- a/source/Rbm.cpp +++ b/source/Rbm.cpp @@ -21,9 +21,9 @@ Rbm::Rbm(const Params& params, arma::mat &w, arma::mat &bv, arma::mat &bh) , m_bh(bh) { Noise_Init(&m_noise, 0x32727155); - uniform(m_w, 0.0, 0.1); - uniform(m_bh, 0.0, 0.1); - uniform(m_bv, 0.0, 0.1); + uniform(m_w, 0.0, m_params.m_weightInit); + uniform(m_bh, 0.0, m_params.m_weightInit); + uniform(m_bv, 0.0, m_params.m_weightInit); } Rbm::Rbm(const Rbm& orig) @@ -90,7 +90,7 @@ void Rbm::train(const arma::mat& batch, size_t numEpochs, size_t miniBatchSize, vis_state = miniBatch; } - hid_state = vis_state * m_w + arma::repmat(m_bh, miniBatchSizeActual, 1); + hid_state = toHiddenState(vis_state); hid_probs = probsLogistic(hid_state); // Sample hidden @@ -110,32 +110,33 @@ void Rbm::train(const arma::mat& batch, size_t numEpochs, size_t miniBatchSize, for (gibbs=0; gibbs < m_params.m_numGibbs; gibbs++) { - - // Create hidden representation given v - hid_state = sample(hid_probs); - // Create visible reconstruction (a fantasy...) given hid - vis_state = hid_state * m_w.t() + arma::repmat(m_bv, miniBatchSizeActual, 1); - vis_probs = probsLogistic(vis_state); - if (m_params.m_doSampleVisible) + if (m_params.m_gibbsDoSampleHidden) { - vis_state = sample(vis_probs); + vis_probs = toVisibleProbs(sample(hid_probs)); } else { - vis_state = vis_probs; + vis_probs = toVisibleProbs(hid_probs); } // Create hidden representation given v - hid_state = vis_state * m_w + repmat(m_bh, miniBatchSizeActual, 1); + if (m_params.m_gibbsDoSampleVisible) + { + hid_state = toHiddenState(sample(vis_probs)); + } + else + { + hid_state = toHiddenState(vis_probs); + } hid_probs = probsLogistic(hid_state); } // Update weights (negative phase) + grad_weight -= vis_probs.t() * hid_probs; grad_bias_v -= sum(vis_probs, 0); grad_bias_h -= sum(hid_probs, 0); - grad_weight -= vis_probs.t() * hid_probs; penalty_weights = weight_decay*arma::sign(m_w); @@ -157,7 +158,8 @@ void Rbm::train(const arma::mat& batch, size_t numEpochs, size_t miniBatchSize, arma::mat diffErr = miniBatch - vis_probs; arma::mat diffErr_squared = diffErr % diffErr; status.err = accu(diffErr_squared)/diffErr_squared.n_elem; - + status.err_total = 0; + if (pListener) { if(!pListener->onProgress(status)) @@ -167,6 +169,17 @@ void Rbm::train(const arma::mat& batch, size_t numEpochs, size_t miniBatchSize, } } // number of mini batches + + arma::mat hid_probs = toHiddenProbs(batch); + arma::mat vis_probs = toVisibleProbs(hid_probs); + arma::mat diffErr = batch - vis_probs; + arma::mat diffErr_squared = diffErr % diffErr; + status.err_total = accu(diffErr_squared)/diffErr_squared.n_elem; + + if (pListener) + { + pListener->onProgress(status); + } } arma::mat Rbm::probsLogistic(const arma::mat &src) @@ -187,16 +200,24 @@ arma::mat Rbm::sample(const arma::mat &src) return dst; } -arma::mat Rbm::toHidden(const arma::mat &visible) +arma::mat Rbm::toHiddenState(const arma::mat &visible) { - arma::mat h = visible * m_w + arma::repmat(m_bh, visible.n_rows, 1); - return probsLogistic(h); + return visible * m_w + arma::repmat(m_bh, visible.n_rows, 1); } -arma::mat Rbm::toVisible(const arma::mat &hidden) +arma::mat Rbm::toVisibleState(const arma::mat &hidden) { - arma::mat v = hidden * m_w.t() + arma::repmat(m_bv, hidden.n_rows, 1); - return probsLogistic(v); + return hidden * m_w.t() + arma::repmat(m_bv, hidden.n_rows, 1); +} + +arma::mat Rbm::toHiddenProbs(const arma::mat &visible) +{ + return probsLogistic(toHiddenState(visible)); +} + +arma::mat Rbm::toVisibleProbs(const arma::mat &hidden) +{ + return probsLogistic(toVisibleState(hidden)); } void Rbm::uniform(arma::mat& srcDst, double mu, double stdDev) diff --git a/source/Rbm.hpp b/source/Rbm.hpp index cccd264..5b98b5a 100644 --- a/source/Rbm.hpp +++ b/source/Rbm.hpp @@ -24,21 +24,25 @@ public: struct Params { Params() - : m_weightDecay(0.0) + : m_weightInit(0.01) + , m_weightDecay(0.0) , m_learningRate(0.1) , m_momentum(0.5) , m_doRaoBlackwell(true) - , m_doSampleVisible(false) + , m_gibbsDoSampleVisible(false) + , m_gibbsDoSampleHidden(false) , m_doSampleBatch(false) - , m_numGibbs(1) + , m_numGibbs(5) { } + double m_weightInit; double m_weightDecay; double m_learningRate; double m_momentum; bool m_doRaoBlackwell; - bool m_doSampleVisible; + bool m_gibbsDoSampleVisible; + bool m_gibbsDoSampleHidden; bool m_doSampleBatch; size_t m_numGibbs; }; @@ -49,6 +53,7 @@ public: size_t trainingSizeRemain; double progress; double err; + double err_total; double L1; double L2; }; @@ -70,8 +75,11 @@ public: virtual ~Rbm(); void train(arma::mat const &batch, size_t numEpochs, size_t sizeMiniBatch, IListener *pListener); - arma::mat toHidden(const arma::mat &v); - arma::mat toVisible(const arma::mat &h); + + arma::mat toHiddenState(const arma::mat &visible); + arma::mat toVisibleState(const arma::mat &hidden); + arma::mat toHiddenProbs(const arma::mat &visible); + arma::mat toVisibleProbs(const arma::mat &hidden); arma::mat& weights(); arma::mat& bias_visible(); arma::mat& bias_hidden(); diff --git a/source/main.cpp b/source/main.cpp index 86cf75c..d1ba959 100644 --- a/source/main.cpp +++ b/source/main.cpp @@ -16,6 +16,7 @@ class RbmListener : public Rbm::IListener std::cout << "epoch : " << status.epoch << std::endl; std::cout << "trainingSizeRemain: " << status.trainingSizeRemain << std::endl; std::cout << "error (per mini batch) = " << status.err << std::endl; + std::cout << "error (total) = " << status.err_total << std::endl; std::cout << "L1 = " << status.L1 << std::endl; std::cout << "L2 = " << status.L2 << std::endl; @@ -122,7 +123,7 @@ int main() rbm.train(batch, 100, 100, &statusDisplay); saveWeight("mnist_2.weights.dat", 28, 28, w, bv, bh); arma::mat v = arma::randu(numTraining, numVisible); - arma::mat h = rbm.toHidden(v); - arma::mat r = rbm.toVisible(h); + arma::mat h = rbm.toHiddenProbs(v); + arma::mat r = rbm.toVisibleProbs(h); return 0; } \ No newline at end of file