diff --git a/source/Rbm.cpp b/source/Rbm.cpp index e284ba3..c1642a7 100644 --- a/source/Rbm.cpp +++ b/source/Rbm.cpp @@ -88,6 +88,16 @@ arma::mat Rbm::toVisibleProbs(const arma::mat& hidden) const return prob(state); } +arma::mat Rbm::sampleVisible(const arma::mat& visible) const +{ + return m_params.doGaussianVisible ? sample_gaussian(visible) : sample(visible); +} + +arma::mat Rbm::sampleHidden(const arma::mat& hidden) const +{ + return m_params.doGaussianHidden ? sample_gaussian(hidden) : sample(hidden); +} + void Rbm::weightsAssign(const arma::mat& w, const arma::mat& bh, const arma::mat& bv) { m_whv.submat(0, 0, w.n_rows - 1, w.n_cols - 1) = w; @@ -210,7 +220,7 @@ void Rbm::cd_hinton(arma::mat const &v_data, arma::mat &dw, arma::mat &dbh, arma } else { - posprods = v_data.t() * sample(poshidprobs); + posprods = v_data.t() * sampleHidden(poshidprobs); } arma::mat negprods; @@ -218,11 +228,11 @@ void Rbm::cd_hinton(arma::mat const &v_data, arma::mat &dw, arma::mat &dbh, arma arma::mat negvisact; for (int i=0; i < m_params.numGibbs; i++) { - // Start negative phase + // Start negative phase arma::mat negdata; if (m_params.gibbsDoSampleHidden) { - negdata = toVisibleProbs(sample(poshidstates)); + negdata = toVisibleProbs(sampleHidden(poshidstates)); } else { @@ -231,11 +241,11 @@ void Rbm::cd_hinton(arma::mat const &v_data, arma::mat &dw, arma::mat &dbh, arma arma::mat neghidprobs; if (m_params.gibbsDoSampleVisible) { - neghidprobs = toHiddenProbs(sample(negdata)); + neghidprobs = toHiddenProbs(sampleVisible(negdata)); } else { - neghidprobs = toHiddenProbs(negdata); + neghidprobs = toHiddenProbs(negdata); } negprods = negdata.t() * neghidprobs; neghidact = arma::sum(neghidprobs); @@ -280,7 +290,7 @@ void Rbm::cd_jens(arma::mat const &v_states, arma::mat &dw, arma::mat &dbh, arma // Create visible reconstruction (a fantasy...) given hid if (m_params.gibbsDoSampleHidden) { - v_probs = toVisibleProbs(sample(h_probs)); + v_probs = toVisibleProbs(sampleHidden(h_probs)); } else { @@ -290,7 +300,7 @@ void Rbm::cd_jens(arma::mat const &v_states, arma::mat &dw, arma::mat &dbh, arma // Create hidden representation given v if (m_params.gibbsDoSampleVisible) { - h_probs = toHiddenProbs(sample(v_probs)); + h_probs = toHiddenProbs(sampleVisible(v_probs)); } else { @@ -342,8 +352,8 @@ void Rbm::train(arma::mat const &batch, IListener* pListener) // Create hidden layer base on training data if (m_params.doSampleBatch) { - // When the hidden units are being driven by data, always use stochastic binary states - v_states = sample(miniBatch); + // Sample the visible batch (respects doGaussianVisible) + v_states = sampleVisible(miniBatch); } // Contrastive divergence learning: calculate gradients diff --git a/source/Rbm.hpp b/source/Rbm.hpp index b9c3ebf..8f93276 100644 --- a/source/Rbm.hpp +++ b/source/Rbm.hpp @@ -164,7 +164,14 @@ private: void cd(arma::mat const &v_states, arma::mat &dwhv, arma::mat &dbhv, arma::mat &dbv); arma::mat v_to_h(const arma::mat &visible) const; arma::mat h_to_v(const arma::mat &hidden) const; - + + // Sample a visible/hidden unit's state from its probability/mean matrix, + // dispatching to the Bernoulli or Gaussian sampler per doGaussianVisible/ + // doGaussianHidden -- CD's Gibbs steps must not binary-threshold a + // Gaussian unit's continuous activation. + arma::mat sampleVisible(const arma::mat &visible) const; + arma::mat sampleHidden(const arma::mat &hidden) const; + }; #endif /* RBM_HPP */ diff --git a/source/matutils.hpp b/source/matutils.hpp index 9cca18d..cd4d02e 100644 --- a/source/matutils.hpp +++ b/source/matutils.hpp @@ -31,7 +31,15 @@ namespace Matutils arma::umat res = (src > uniform(src)); return arma::conv_to::from(res); } - + + // src holds a Gaussian unit's mean; returns a stochastic sample around it. + // Binary-thresholding a continuous Gaussian activation (via sample() above) + // is meaningless, so Gaussian-visible/hidden units need this instead. + inline arma::mat sample_gaussian(const arma::mat &src) + { + return src + arma::randn(arma::size(src)); + } + inline arma::mat prob(const arma::mat &src) { return 1 / (1 + (arma::exp(-src)));