From eb29e33b819e98d3a05a85020643f3dd5d3cfbf1 Mon Sep 17 00:00:00 2001 From: Jens Ahrensfeld Date: Mon, 27 Jul 2026 13:45:03 +0200 Subject: [PATCH] Fix Bernoulli-only sample() applied to Gaussian visible/hidden units Matutils::sample() always binary-thresholds (src > uniform(src)), but it was the only sampler in the codebase and was called unconditionally on h_probs/v_probs/miniBatch in several CD Gibbs-loop branches regardless of doGaussianVisible/doGaussianHidden. Binary-thresholding a Gaussian unit's continuous activation is meaningless -- it would corrupt any Gaussian-visible/hidden RBM (image-domain experiments via the GUI or TEST target); doesn't affect poet's plain BB-RBM path since both flags are false there. Add sample_gaussian() (mean + N(0,1) noise) alongside the existing Bernoulli sample() in matutils.hpp, plus Rbm::sampleVisible/sampleHidden helpers that dispatch to the right one per the RBM's configured type. Replace every visible/hidden Gibbs-step sample() call in cd_jens (the active path) and cd_hinton (compiled but currently unused, behind USE_CD_HINTON) with the appropriate dispatch helper, and fix the same issue in Rbm::train's doSampleBatch path. Behavior is unchanged for any RBM with doGaussianVisible/doGaussianHidden both false (confirmed: poet.elf f output identical before/after). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_016K8Gu7Qejd11JbdiHZqYAs --- source/Rbm.cpp | 28 +++++++++++++++++++--------- source/Rbm.hpp | 9 ++++++++- source/matutils.hpp | 10 +++++++++- 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/source/Rbm.cpp b/source/Rbm.cpp index e284ba3..c1642a7 100644 --- a/source/Rbm.cpp +++ b/source/Rbm.cpp @@ -88,6 +88,16 @@ arma::mat Rbm::toVisibleProbs(const arma::mat& hidden) const return prob(state); } +arma::mat Rbm::sampleVisible(const arma::mat& visible) const +{ + return m_params.doGaussianVisible ? sample_gaussian(visible) : sample(visible); +} + +arma::mat Rbm::sampleHidden(const arma::mat& hidden) const +{ + return m_params.doGaussianHidden ? sample_gaussian(hidden) : sample(hidden); +} + void Rbm::weightsAssign(const arma::mat& w, const arma::mat& bh, const arma::mat& bv) { m_whv.submat(0, 0, w.n_rows - 1, w.n_cols - 1) = w; @@ -210,7 +220,7 @@ void Rbm::cd_hinton(arma::mat const &v_data, arma::mat &dw, arma::mat &dbh, arma } else { - posprods = v_data.t() * sample(poshidprobs); + posprods = v_data.t() * sampleHidden(poshidprobs); } arma::mat negprods; @@ -218,11 +228,11 @@ void Rbm::cd_hinton(arma::mat const &v_data, arma::mat &dw, arma::mat &dbh, arma arma::mat negvisact; for (int i=0; i < m_params.numGibbs; i++) { - // Start negative phase + // Start negative phase arma::mat negdata; if (m_params.gibbsDoSampleHidden) { - negdata = toVisibleProbs(sample(poshidstates)); + negdata = toVisibleProbs(sampleHidden(poshidstates)); } else { @@ -231,11 +241,11 @@ void Rbm::cd_hinton(arma::mat const &v_data, arma::mat &dw, arma::mat &dbh, arma arma::mat neghidprobs; if (m_params.gibbsDoSampleVisible) { - neghidprobs = toHiddenProbs(sample(negdata)); + neghidprobs = toHiddenProbs(sampleVisible(negdata)); } else { - neghidprobs = toHiddenProbs(negdata); + neghidprobs = toHiddenProbs(negdata); } negprods = negdata.t() * neghidprobs; neghidact = arma::sum(neghidprobs); @@ -280,7 +290,7 @@ void Rbm::cd_jens(arma::mat const &v_states, arma::mat &dw, arma::mat &dbh, arma // Create visible reconstruction (a fantasy...) given hid if (m_params.gibbsDoSampleHidden) { - v_probs = toVisibleProbs(sample(h_probs)); + v_probs = toVisibleProbs(sampleHidden(h_probs)); } else { @@ -290,7 +300,7 @@ void Rbm::cd_jens(arma::mat const &v_states, arma::mat &dw, arma::mat &dbh, arma // Create hidden representation given v if (m_params.gibbsDoSampleVisible) { - h_probs = toHiddenProbs(sample(v_probs)); + h_probs = toHiddenProbs(sampleVisible(v_probs)); } else { @@ -342,8 +352,8 @@ void Rbm::train(arma::mat const &batch, IListener* pListener) // Create hidden layer base on training data if (m_params.doSampleBatch) { - // When the hidden units are being driven by data, always use stochastic binary states - v_states = sample(miniBatch); + // Sample the visible batch (respects doGaussianVisible) + v_states = sampleVisible(miniBatch); } // Contrastive divergence learning: calculate gradients diff --git a/source/Rbm.hpp b/source/Rbm.hpp index b9c3ebf..8f93276 100644 --- a/source/Rbm.hpp +++ b/source/Rbm.hpp @@ -164,7 +164,14 @@ private: void cd(arma::mat const &v_states, arma::mat &dwhv, arma::mat &dbhv, arma::mat &dbv); arma::mat v_to_h(const arma::mat &visible) const; arma::mat h_to_v(const arma::mat &hidden) const; - + + // Sample a visible/hidden unit's state from its probability/mean matrix, + // dispatching to the Bernoulli or Gaussian sampler per doGaussianVisible/ + // doGaussianHidden -- CD's Gibbs steps must not binary-threshold a + // Gaussian unit's continuous activation. + arma::mat sampleVisible(const arma::mat &visible) const; + arma::mat sampleHidden(const arma::mat &hidden) const; + }; #endif /* RBM_HPP */ diff --git a/source/matutils.hpp b/source/matutils.hpp index 9cca18d..cd4d02e 100644 --- a/source/matutils.hpp +++ b/source/matutils.hpp @@ -31,7 +31,15 @@ namespace Matutils arma::umat res = (src > uniform(src)); return arma::conv_to::from(res); } - + + // src holds a Gaussian unit's mean; returns a stochastic sample around it. + // Binary-thresholding a continuous Gaussian activation (via sample() above) + // is meaningless, so Gaussian-visible/hidden units need this instead. + inline arma::mat sample_gaussian(const arma::mat &src) + { + return src + arma::randn(arma::size(src)); + } + inline arma::mat prob(const arma::mat &src) { return 1 / (1 + (arma::exp(-src)));