From be64a88e04def4916e9995d5cc5f13b4eb8f14b9 Mon Sep 17 00:00:00 2001
From: Jens Ahrensfeld <jens@jayfield.org>
Date: Thu, 13 Jan 2022 07:19:47 +0000
Subject: [PATCH] - Layer: do gibbs sampling in calcContextBatch() - RBM: added
 Gibbs sampler - Stack adjust training column vector according to needs

git-svn-id: http://moon:8086/svn/software/trunk/projects/Rbm@797 b431acfa-c32f-4a4a-93f1-934dc6c82436
---
 source/Layer.hpp |  4 +++-
 source/Rbm.cpp   | 38 ++++++++++++++++++++++++++++++++------
 source/Rbm.hpp   |  3 ++-
 source/Stack.cpp | 12 +++++++++---
 source/poet.cpp  |  4 ++--
 5 files changed, 48 insertions(+), 13 deletions(-)

diff --git a/source/Layer.hpp b/source/Layer.hpp
index 9fd043f..76cfbee 100644
--- a/source/Layer.hpp
+++ b/source/Layer.hpp
@@ -40,7 +40,9 @@ public:
 		{
 			for (int i=1; i < numTraining; i++)
 			{
-				arma::mat h = toHiddenProbs(batch.row(i-1));
+				arma::mat v = batch.row(i-1);
+				arma::mat h = arma::zeros(1, numHidden());
+				gibbs_vh(v, h);
 				arma::mat training_with_ctx = arma::join_rows(batch.row(i).cols(0, numVisible()-m_numContext-1), h);
 				batch.row(i) = training_with_ctx;
 			}
diff --git a/source/Rbm.cpp b/source/Rbm.cpp
index 0ff10a0..b620b8a 100644
--- a/source/Rbm.cpp
+++ b/source/Rbm.cpp
@@ -61,28 +61,54 @@ Json::Value Rbm::toJson() const
 	return rbm;
 }
 
-void Rbm::gibbs(arma::mat &hv_probs, arma::mat &v_probs)
+void Rbm::gibbs_vh(arma::mat &v_probs, arma::mat &h_probs)
+{
+	for (int i=0; i < m_params.numGibbs; i++)
+	{
+		// Create hidden representation given v
+		if (m_params.gibbsDoSampleVisible)
+		{
+			h_probs = prob(v_to_h(sample(v_probs)));
+		}
+		else
+		{
+			h_probs = prob(v_to_h(v_probs));
+		}
+
+		// Create visible reconstruction (a fantasy...) given hid
+		if (m_params.gibbsDoSampleHidden)
+		{
+			v_probs = prob(h_to_v(sample(h_probs)));
+		}
+		else
+		{
+			v_probs = prob(h_to_v(h_probs));
+		}
+	}
+}
+
+void Rbm::gibbs_hv(arma::mat &h_probs, arma::mat &v_probs)
 {
 	for (int i=0; i < m_params.numGibbs; i++)
 	{
 		// Create visible reconstruction (a fantasy...) given hid
 		if (m_params.gibbsDoSampleHidden)
 		{
-			v_probs = prob(h_to_v(sample(hv_probs)));
+			v_probs = prob(h_to_v(sample(h_probs)));
 		}
 		else
 		{
-			v_probs = prob(h_to_v(hv_probs));
+			v_probs = prob(h_to_v(h_probs));
 		}
 
 		// Create hidden representation given v
 		if (m_params.gibbsDoSampleVisible)
 		{
-			hv_probs = prob(v_to_h(sample(v_probs)));
+			h_probs = prob(v_to_h(sample(v_probs)));
 		}
 		else
 		{
-			hv_probs = prob(v_to_h(v_probs));
+			h_probs = prob(v_to_h(v_probs));
 		}
 	}
 }
@@ -108,7 +134,7 @@ void Rbm::contrastiveDivergence(arma::mat const &v_states, arma::mat &dw, arma::
 	dbv = sum(v_states, 0);
 	dbhv = sum(h_states, 0);
 
-	gibbs(h_probs, v_probs);
+	gibbs_hv(h_probs, v_probs);
 
 	// Update weights (negative phase)
 	dw -= v_probs.t() * h_probs;
diff --git a/source/Rbm.hpp b/source/Rbm.hpp
index b3d3607..7024141 100644
--- a/source/Rbm.hpp
+++ b/source/Rbm.hpp
@@ -176,13 +176,14 @@ public:
 		arma::mat diffErr_squared = diffErr % diffErr;
 		return arma::sum(diffErr_squared, 1) * 1.0/diffErr_squared.n_cols;
 	}
+	void gibbs_hv(arma::mat &h_states, arma::mat &v_states);
+	void gibbs_vh(arma::mat &v_probs, arma::mat &h_probs);
 	
 private:
 	Params m_params;
 	arma::mat sample(arma::mat const &src);
 	void contrastiveDivergence(arma::mat const &v_states, arma::mat &dwhv, arma::mat &dbhv, arma::mat &dbv);
 	void uniform(arma::mat &srcDst, double stdDev=1.0, double mu=0.5);
-	void gibbs(arma::mat &hv_states, arma::mat &v_states);
 	
 protected:
 	arma::mat m_bhv;
diff --git a/source/Stack.cpp b/source/Stack.cpp
index 82b4d61..c6b47bc 100644
--- a/source/Stack.cpp
+++ b/source/Stack.cpp
@@ -247,13 +247,19 @@ size_t Stack::loadTrainingBatch(bool doNormalize)
 	}
 	
 	// Migrate context part to training data
-	size_t numContext = getLayer(0)->context().n_cols;
 	size_t numTraining = m_trainingBatch.n_rows;
-	if (numContext > 0 and m_trainingBatch.n_cols != getLayer(0)->numVisible())
+	size_t numVisible = getLayer(0)->numVisible();
+	
+	if (m_trainingBatch.n_cols < numVisible)
 	{
-		arma::mat training_with_ctx = arma::join_rows(m_trainingBatch, arma::zeros(numTraining, numContext));
+		size_t diff = numVisible - m_trainingBatch.n_cols;
+		arma::mat training_with_ctx = arma::join_rows(m_trainingBatch, arma::zeros(numTraining, diff));
 		m_trainingBatch = training_with_ctx;
 	}
+	else if (m_trainingBatch.n_cols > numVisible)
+	{
+		m_trainingBatch = m_trainingBatch.submat(0, 0, numTraining-1, numVisible-1);
+	}
 
 	return m_trainingBatch.n_rows;
 }
diff --git a/source/poet.cpp b/source/poet.cpp
index 881590e..bbb69c5 100644
--- a/source/poet.cpp
+++ b/source/poet.cpp
@@ -81,8 +81,8 @@ int main()
 	
 	Stack stack(".", project);
 
-	arma::mat batch = createTraining("moby_ch1_small.txt");
-	batch.save("poet_small.training.dat", arma::arma_ascii);
+	arma::mat batch = createTraining("moby_ch1.txt");
+	batch.save("poet.training.dat", arma::arma_ascii);
 	
 	return 0;
 }