- gibbsDoSampleHidden and gibbsDoSampleVisible onl used only training

- very good results

git-svn-id: http://moon:8086/svn/software/trunk/projects/Rbm@852 b431acfa-c32f-4a4a-93f1-934dc6c82436

poet_2v_5s.prj

@@ -6,31 +6,7 @@
 			{
 				"id" : 0,
 				"name" : "Layer",
-				"numContext" : 8,
+				"numContext" : 0,
-				"numHidden" : 64,
-				"numVisibleX" : 37,
-				"numVisibleY" : 2,
-				"rbm" : 
-				{
-					"params" : 
-					{
-						"doRaoBlackwell" : 1,
-						"doSampleBatch" : 0,
-						"gibbsDoSampleHidden" : 0,
-						"gibbsDoSampleVisible" : 0,
-						"learningRate" : 0.10000000000000001,
-						"miniBatchSize" : 1000,
-						"momentum" : 0.5,
-						"numEpochs" : 10000,
-						"numGibbs" : 1,
-						"weightDecay" : 0.0
-					}
-				}
-			},
-			{
-				"id" : 1,
-				"name" : "Layer",
-				"numContext" : 64,
 				"numHidden" : 128,
 				"numVisibleX" : 37,
 				"numVisibleY" : 2,
@@ -40,12 +16,36 @@
 					{
 						"doRaoBlackwell" : 1,
 						"doSampleBatch" : 0,
-						"gibbsDoSampleHidden" : 0,
+						"gibbsDoSampleHidden" : 1,
 						"gibbsDoSampleVisible" : 0,
-						"learningRate" : 0.10000000000000001,
+						"learningRate" : 0.250000000000,
 						"miniBatchSize" : 1000,
 						"momentum" : 0.5,
-						"numEpochs" : 5000,
+						"numEpochs" : 1000,
+						"numGibbs" : 1,
+						"weightDecay" : 0.0
+					}
+				}
+			},
+			{
+				"id" : 1,
+				"name" : "Layer",
+				"numContext" : 128,
+				"numHidden" : 128,
+				"numVisibleX" : 37,
+				"numVisibleY" : 2,
+				"rbm" : 
+				{
+					"params" : 
+					{
+						"doRaoBlackwell" : 1,
+						"doSampleBatch" : 0,
+						"gibbsDoSampleHidden" : 1,
+						"gibbsDoSampleVisible" : 0,
+						"learningRate" : 0.250000000000,
+						"miniBatchSize" : 1000,
+						"momentum" : 0.5,
+						"numEpochs" : 1000,
 						"numGibbs" : 1,
 						"weightDecay" : 0.0
 					}
@@ -55,7 +55,7 @@
 				"id" : 2,
 				"name" : "Layer",
 				"numContext" : 128,
-				"numHidden" : 256,
+				"numHidden" : 128,
 				"numVisibleX" : 37,
 				"numVisibleY" : 2,
 				"rbm" : 
@@ -64,12 +64,12 @@
 					{
 						"doRaoBlackwell" : 1,
 						"doSampleBatch" : 0,
-						"gibbsDoSampleHidden" : 0,
+						"gibbsDoSampleHidden" : 1,
 						"gibbsDoSampleVisible" : 0,
-						"learningRate" : 0.10000000000000001,
+						"learningRate" : 0.250000000000,
 						"miniBatchSize" : 1000,
 						"momentum" : 0.5,
-						"numEpochs" : 2500,
+						"numEpochs" : 1000,
 						"numGibbs" : 1,
 						"weightDecay" : 0.0
 					}
@@ -78,8 +78,8 @@
 			{
 				"id" : 3,
 				"name" : "Layer",
-				"numContext" : 256,
+				"numContext" : 128,
-				"numHidden" : 512,
+				"numHidden" : 128,
 				"numVisibleX" : 37,
 				"numVisibleY" : 2,
 				"rbm" : 
@@ -88,12 +88,12 @@
 					{
 						"doRaoBlackwell" : 1,
 						"doSampleBatch" : 0,
-						"gibbsDoSampleHidden" : 0,
+						"gibbsDoSampleHidden" : 1,
 						"gibbsDoSampleVisible" : 0,
-						"learningRate" : 0.10000000000000001,
+						"learningRate" : 0.250000000000,
 						"miniBatchSize" : 1000,
 						"momentum" : 0.5,
-						"numEpochs" : 2500,
+						"numEpochs" : 1000,
 						"numGibbs" : 1,
 						"weightDecay" : 0.0
 					}
@@ -102,8 +102,8 @@
 			{
 				"id" : 4,
 				"name" : "Layer",
-				"numContext" : 512,
+				"numContext" : 128,
-				"numHidden" : 1024,
+				"numHidden" : 128,
 				"numVisibleX" : 37,
 				"numVisibleY" : 2,
 				"rbm" : 
@@ -112,9 +112,9 @@
 					{
 						"doRaoBlackwell" : 1,
 						"doSampleBatch" : 0,
-						"gibbsDoSampleHidden" : 0,
+						"gibbsDoSampleHidden" : 1,
 						"gibbsDoSampleVisible" : 0,
-						"learningRate" : 0.10000000000000001,
+						"learningRate" : 0.250000000000,
 						"miniBatchSize" : 1000,
 						"momentum" : 0.5,
 						"numEpochs" : 1000,

source/Rbm.cpp

@@ -108,24 +108,10 @@ void Rbm::gibbs_vh(arma::mat &v_probs, arma::mat &h_probs)
 	for (int i=0; i < m_params.numGibbs; i++)
 	{
 		// Create hidden representation given v
-		if (m_params.gibbsDoSampleVisible)
+		h_probs = prob(v_to_h(v_probs));
-		{
-			h_probs = prob(v_to_h(sample(v_probs)));
-		}
-		else
-		{
-			h_probs = prob(v_to_h(v_probs));
-		}
 
 		// Create visible reconstruction (a fantasy...) given hid
-		if (m_params.gibbsDoSampleHidden)
+		v_probs = prob(h_to_v(h_probs));
-		{
-			v_probs = prob(h_to_v(sample(h_probs)));
-		}
-		else
-		{
-			v_probs = prob(h_to_v(h_probs));
-		}
 	}
 }
 
@@ -134,24 +120,10 @@ void Rbm::gibbs_hv(arma::mat &h_probs, arma::mat &v_probs)
 	for (int i=0; i < m_params.numGibbs; i++)
 	{
 		// Create visible reconstruction (a fantasy...) given hid
-		if (m_params.gibbsDoSampleHidden)
+		v_probs = prob(h_to_v(h_probs));
-		{
-			v_probs = prob(h_to_v(sample(h_probs)));
-		}
-		else
-		{
-			v_probs = prob(h_to_v(h_probs));
-		}
 
 		// Create hidden representation given v
-		if (m_params.gibbsDoSampleVisible)
+		h_probs = prob(v_to_h(v_probs));
-		{
-			h_probs = prob(v_to_h(sample(v_probs)));
-		}
-		else
-		{
-			h_probs = prob(v_to_h(v_probs));
-		}
 	}
 }
 
@@ -176,7 +148,29 @@ void Rbm::contrastiveDivergence(arma::mat const &v_states, arma::mat &dw, arma::
 	dbv = sum(v_states, 0);
 	dbhv = sum(h_states, 0);
 
-	gibbs_hv(h_probs, v_probs);
+	// Gibbs sampling with training params
+	for (int i=0; i < m_params.numGibbs; i++)
+	{
+		// Create visible reconstruction (a fantasy...) given hid
+		if (m_params.gibbsDoSampleHidden)
+		{
+			v_probs = prob(h_to_v(sample(h_probs)));
+		}
+		else
+		{
+			v_probs = prob(h_to_v(h_probs));
+		}
+
+		// Create hidden representation given v
+		if (m_params.gibbsDoSampleVisible)
+		{
+			h_probs = prob(v_to_h(sample(v_probs)));
+		}
+		else
+		{
+			h_probs = prob(v_to_h(v_probs));
+		}
+	}
 
 	// Update weights (negative phase)
 	dw -= v_probs.t() * h_probs;

source/poet.cpp

@@ -19,19 +19,69 @@ using namespace arma;
 class RbmListener : public Rbm::IListener
 {
 	public:
-	RbmListener() {}
+	RbmListener(AStack &stack)
+	: m_stack(stack)
+	, m_last_progress(0)
+	{
+	}
 	virtual ~RbmListener() {}
 	
 	bool onProgress(Rbm *pRbm, const Rbm::Status &status)
 	{
-		std::cout << "Progress :          " << status.progress << " %" << std::endl;
+		if (status.progress == 0)
-		std::cout << "error (per mini batch) = " << status.err << std::endl;
+		{
-		std::cout << "error (total)          = " << status.err_total << std::endl;
+			m_last_progress = -10;
-		std::cout << "L1                     = " << status.L1 << std::endl;
+		}
-		std::cout << "L2                     = " << status.L2 << std::endl;
+		if ((status.progress - m_last_progress) == 10)
+		{
+			std::cout << "Progress :          " << status.progress << " %" << std::endl;
+			std::cout << "error (per mini batch) = " << status.err << std::endl;
+			std::cout << "error (total)          = " << status.err_total << std::endl;
+			std::cout << "L1                     = " << status.L1 << std::endl;
+			std::cout << "L2                     = " << status.L2 << std::endl;
+			m_last_progress = status.progress;
+			forward();
+		}
 		
 		return true;
 	}
+	
+	void forward()
+	{
+		RnnStack *stack = reinterpret_cast<RnnStack*>(&m_stack);
+		arma::mat state;
+		arma::mat curr;
+		arma::mat next;
+		std::string curr_str;
+		std::string next_str;
+		arma::mat t_vc = stack->trainingBatch();
+		
+		for (int i=0; i < stack->getSeqLen(); i++)
+		{
+			curr = stack->to_curr(t_vc.row(i));
+			curr_str.append(1,RnnTextHelper::idx2ch(curr.index_max()));
+			arma::mat r = stack->step_forward(state, curr);
+			next = stack->to_next(r, true);
+			next_str.append(1,RnnTextHelper::idx2ch(next.index_max()));
+		}
+		cout << "Curr: " << curr_str << std::endl;
+		cout << "Next: " << next_str << std::endl;
+
+		for (int i=stack->getSeqLen(); i < t_vc.n_rows; i++)
+		{
+			curr = next;
+			curr_str.append(1,RnnTextHelper::idx2ch(curr.index_max()));
+			arma::mat r = stack->step_forward(state, curr);
+			next = stack->to_next(r, true);
+			curr = stack->to_curr(r);
+			next_str.append(1,RnnTextHelper::idx2ch(next.index_max()));
+		}
+		cout << "Curr: " << curr_str << std::endl;
+		cout << "Next: " << next_str << std::endl;
+
+	}
+	AStack &m_stack;
+	int m_last_progress;
 };
 
 int main(int argc, char *argv[])
@@ -88,11 +138,7 @@ int main(int argc, char *argv[])
 	
 	if (command == Command::Train)
 	{
-		for (int i=0; i < stack->numLayers(); i++)
+		RbmListener listener(*stack);
-		{
-			stack->getLayer(i)->params().gibbsDoSampleHidden = true;
-		}
-		RbmListener listener;
 
 		// Phase 10
 		stack->train(t_vc, &listener);