From 6e7b8bc1517345a56abf83d3c51a7c8d0e9c1a99 Mon Sep 17 00:00:00 2001
From: Jens Ahrensfeld <jens@jayfield.org>
Date: Sat, 1 Nov 2014 18:01:04 +0000
Subject: [PATCH] - train(): added reconstruction error metric - train2():   -
 added reconstruction error metric.   - added Gaussian units   - added
 sparsity

- GUI: choose train() or train2()


git-svn-id: http://moon:8086/svn/software/trunk/projects/RBM@44 b431acfa-c32f-4a4a-93f1-934dc6c82436
---
 Source/MainComponent.cpp |  29 +++++-
 Source/MainComponent.h   |   1 +
 Source/Rbm.hpp           | 191 ++++++++++++++++++++++++---------------
 3 files changed, 145 insertions(+), 76 deletions(-)

diff --git a/Source/MainComponent.cpp b/Source/MainComponent.cpp
index 362465e..e504d76 100644
--- a/Source/MainComponent.cpp
+++ b/Source/MainComponent.cpp
@@ -280,6 +280,10 @@ MainComponent::MainComponent ()
     weightInitLabel->setColour (TextEditor::backgroundColourId, Colour (0x00000000));
     weightInitLabel->addListener (this);
 
+    addAndMakeVisible (rbmTrainV2ToggleButton = new ToggleButton ("rbmTrainV2ToggleButton toggle button"));
+    rbmTrainV2ToggleButton->setButtonText (TRANS("Train Ver. 2"));
+    rbmTrainV2ToggleButton->addListener (this);
+
 
     //[UserPreSize]
 	m_vNumX = 16;
@@ -340,6 +344,7 @@ MainComponent::~MainComponent()
     momentumLabel = nullptr;
     sparsityLearningRateLabel = nullptr;
     weightInitLabel = nullptr;
+    rbmTrainV2ToggleButton = nullptr;
 
 
     //[Destructor]. You can add your own custom destruction code here..
@@ -461,6 +466,7 @@ void MainComponent::resized()
     momentumLabel->setBounds (208, 368, 72, 24);
     sparsityLearningRateLabel->setBounds (416, 320, 72, 24);
     weightInitLabel->setBounds (416, 368, 72, 24);
+    rbmTrainV2ToggleButton->setBounds (336, 200, 128, 24);
     //[UserResized] Add your own custom resize handling here..
     DrawTraining->setBounds (16, 16, 100, 100);
     DrawReconstruction->setBounds (110+16, 16, 100, 100);
@@ -618,6 +624,11 @@ void MainComponent::buttonClicked (Button* buttonThatWasClicked)
     	m_pRbm->setDoSparse(buttonThatWasClicked->getToggleState());
         //[/UserButtonCode_rbmDoSparseToggleButton]
     }
+    else if (buttonThatWasClicked == rbmTrainV2ToggleButton)
+    {
+        //[UserButtonCode_rbmTrainV2ToggleButton] -- add your button handler code here..
+        //[/UserButtonCode_rbmTrainV2ToggleButton]
+    }
 
     //[UserbuttonClicked_Post]
     //[/UserbuttonClicked_Post]
@@ -848,8 +859,8 @@ void MainComponent::redrawReconstruction()
 {
 	DrawHidden->setData(m_pRbm->toHidden(DrawTraining->getData()));
 	DrawReconstruction->setData(m_pRbm->toVisible(DrawHidden->getData()));
-	double energy = m_pRbm->getEnergy(DrawTraining->getData(), DrawHidden->getData());
-	cout << "Energy(" << 0 <<") = " << energy << endl;
+//	double energy = m_pRbm->getEnergy(DrawTraining->getData(), DrawHidden->getData());
+//	cout << "Energy(" << 0 <<") = " << energy << endl;
 
 }
 
@@ -862,8 +873,14 @@ void MainComponent::redrawWeights(int index)
 void MainComponent::run()
 {
 	trainButton->setEnabled(false);
-	m_pRbm->train(m_layers, numEpochslabel->getText().getIntValue());
-//	m_pRbm->train2(m_layers, numEpochslabel->getText().getIntValue(), 100);
+	if (rbmTrainV2ToggleButton->getToggleState())
+	{
+		m_pRbm->train2(m_layers, numEpochslabel->getText().getIntValue(), 100);
+	}
+	else
+	{
+		m_pRbm->train(m_layers, numEpochslabel->getText().getIntValue());
+	}
 	trainButton->setEnabled(true);
 }
 
@@ -1046,6 +1063,10 @@ BEGIN_JUCER_METADATA
          edBkgCol="0" labelText="0.001" editableSingleClick="1" editableDoubleClick="1"
          focusDiscardsChanges="0" fontname="Default font" fontsize="15"
          bold="0" italic="0" justification="36"/>
+  <TOGGLEBUTTON name="rbmTrainV2ToggleButton toggle button" id="92c647c1f8b110a2"
+                memberName="rbmTrainV2ToggleButton" virtualName="" explicitFocusOrder="0"
+                pos="336 200 128 24" buttonText="Train Ver. 2" connectedEdges="0"
+                needsCallback="1" radioGroupId="0" state="0"/>
 </JUCER_COMPONENT>
 
 END_JUCER_METADATA
diff --git a/Source/MainComponent.h b/Source/MainComponent.h
index 9e7e5b7..cb2dc0e 100644
--- a/Source/MainComponent.h
+++ b/Source/MainComponent.h
@@ -129,6 +129,7 @@ private:
     ScopedPointer<Label> momentumLabel;
     ScopedPointer<Label> sparsityLearningRateLabel;
     ScopedPointer<Label> weightInitLabel;
+    ScopedPointer<ToggleButton> rbmTrainV2ToggleButton;
 
 
     //==============================================================================
diff --git a/Source/Rbm.hpp b/Source/Rbm.hpp
index c250b6d..836b10a 100644
--- a/Source/Rbm.hpp
+++ b/Source/Rbm.hpp
@@ -81,6 +81,8 @@ public:
 		MatrixXd sumWeights(m_w.getNumVisible(), m_w.getNumHidden());
 		MatrixXd deltaWeights(m_w.getNumVisible(), m_w.getNumHidden());
 
+		MatrixXd diffErr(1, m_w.getNumVisible());
+
 		const LayerArray<VisibleLayer> &vt = batch;
 
 		sigma = m_sigma;
@@ -96,6 +98,8 @@ public:
 		m_doCancel = false;
 		for (epoch=0; epoch < numEpochs; epoch++)
 		{
+			double err = 0;
+
 			if (m_doCancel)
 			{
 				m_doCancel = false;
@@ -125,6 +129,8 @@ public:
 				sumBiasV += vt[t].states();
 				sumBiasH += h.states();
 
+				diffErr = vt[t].states();
+
 				for (gibbs=0; gibbs < m_numGibbs; gibbs++)
 				{
 					h.statesUpdateStochastic();
@@ -170,6 +176,9 @@ public:
 				sumWeights -= v.states() * h.states().transpose();
 				sumBiasV -= v.states();
 				sumBiasH -= h.states();
+				diffErr -= v.states();
+				diffErr.array() *= diffErr.array();
+				err += diffErr.sum();
 
 			} // TrainingSize
 
@@ -215,20 +224,40 @@ public:
 				m_pListener->onEpochTrained(*this);
 			}
 
+			cout << "err =" << endl;
+			cout << err << endl;
+
 		} // Number of epochs
 	}
 
-	MatrixXd sample(const MatrixXd &src)
+	void sample(MatrixXd &src)
 	{
 		uint32_t i;
-		MatrixXd res(src);
 
 		for (i=0; i < src.array().size(); i++)
 		{
-			res.array()(i) = (double) src.array()(i) > Noise_Uniform(&m_noise);
+			src.array()(i) = (double)(src.array()(i) > Noise_Uniform(&m_noise));
 		}
+	}
 
-		return res;
+	void probsLogistic(MatrixXd &src, double lambda, double sigma)
+	{
+		double var = sigma*sigma;
+
+		src.array() *= -lambda/var;
+		src.array() = src.array().exp();
+		src.array() += 1;
+		src.array() = 1.0/src.array();
+	}
+
+	void sampleGaussian(MatrixXd &src, double lambda, double sigma)
+	{
+		uint32_t i;
+
+		for (i=0; i < src.array().size(); i++)
+		{
+			src.array()(i) = sigma*Noise_Gaussian(&m_noise) + lambda*src.array()(i);
+		}
 	}
 
 	void train2(const LayerArray<VisibleLayer> &vt, uint32_t numEpochs, uint32_t batchSize, double sigmaMin = 0.05)
@@ -240,113 +269,125 @@ public:
 		double dProgress = 1.0/numEpochs;
 		double kTrain = 1.0/vt.getSize();
 
-		if (batchSize > vt.getSize())
-			batchSize = vt.getSize();
+//		if (batchSize > vt.getSize())
+		batchSize = vt.getSize();
 
-		MatrixXd vp(m_w.getNumVisible(), batchSize);
-		MatrixXd vs(m_w.getNumVisible(), batchSize);
-		MatrixXd hp(m_w.getNumHidden(), batchSize);
-		MatrixXd hs(m_w.getNumHidden(), batchSize);
-		MatrixXd batch(m_w.getNumVisible(), batchSize);
+		MatrixXd v(batchSize, m_w.getNumVisible());
+		MatrixXd h(batchSize, m_w.getNumHidden());
+		MatrixXd batch(batchSize, m_w.getNumVisible());
 
-		VectorXd sumBiasV(m_w.getNumVisible());
-		VectorXd sumBiasH(m_w.getNumHidden());
+		MatrixXd sumBiasV(1, m_w.getNumVisible());
+		MatrixXd sumBiasH(1, m_w.getNumHidden());
 		MatrixXd sumWeights(m_w.getNumVisible(), m_w.getNumHidden());
 
-		VectorXd deltaBiasV(m_w.getNumVisible());
-		VectorXd deltaBiasH(m_w.getNumHidden());
-		MatrixXd deltaWeights(m_w.getNumVisible(), m_w.getNumHidden());
+		MatrixXd deltaBiasV(MatrixXd::Zero(1, m_w.getNumVisible()));
+		MatrixXd deltaBiasH(MatrixXd::Zero(1, m_w.getNumHidden()));
+		MatrixXd deltaWeights(MatrixXd::Zero(m_w.getNumVisible(), m_w.getNumHidden()));
+
+		MatrixXd diffErr(batchSize, m_w.getNumVisible());
 
 		m_progress = 0;
 		m_doCancel = false;
 
 		for (i=0; i < batchSize; i++)
 		{
-			t = (uint32_t)(0.5 + (vt.getSize()-1)*Noise_Uniform(&m_noise));
-			batch.col(i) = vt[t].states();
+//			t = (uint32_t)(0.5 + (vt.getSize()-1)*Noise_Uniform(&m_noise));
+			batch.row(i) = vt[i].states();
 		}
 
 		for (epoch=0; epoch < numEpochs; epoch++)
 		{
+			double err;
+
+			v = batch;
 			if (m_doCancel)
 			{
 				m_doCancel = false;
 				break;
 			}
 
-			sumWeights.fill(0);
-			sumBiasV.fill(0);
-			sumBiasH.fill(0);
-			for (i=0; i < batchSize; i++)
+			// Create hidden layer base on training data
+			h = v * m_w.weights();
+			h += m_w.hiddenBias().transpose().replicate(batchSize, 1);
+			probsLogistic(h, m_lambda, sigma);
+
+			if (!m_doRaoBlackwell)
 			{
-				vs = batch.col(i);
+				sample(h);
+			}
+			// Update weights (positive phase)
+			sumBiasV = v.colwise().sum();
+			if (!m_doSparse)
+			{
+				sumBiasH = h.colwise().sum();
+			}
+			sumWeights = v.transpose() * h;
+			diffErr = v;
 
-				// h.probsUpdateLogistic(vt[t], m_w, m_lambda, sigma);
-				hp = -vs.transpose() * m_w.weights();
-				hp.array() = hp.array().exp();
-				hp.array() += 1;
-				hp.array() = 1.0/hp.array();
+			for (gibbs=0; gibbs < m_numGibbs; gibbs++)
+			{
+				sample(h);
 
-				// Create hidden layer base on training data
-				hs = sample(hp);
+				// Create visible reconstruction (a fantasy...)
+				v = h * m_w.weights().transpose();
+				v += m_w.visibleBias().transpose().replicate(batchSize, 1);
 
-				// Update weights (positive phase)
-				sumBiasV += vp.colwise().sum();
-				if (m_doRaoBlackwell)
+				if (m_useVisibleGaussian)
 				{
-					sumWeights += vs * hp.transpose();
-					sumBiasH += hp.colwise().sum();
+					sampleGaussian(v, m_lambda, sigma);
 				}
 				else
 				{
-					sumWeights += vs * hs.transpose();
-					sumBiasH += hs.colwise().sum();
-				}
-
-				for (gibbs=0; gibbs < m_numGibbs; gibbs++)
-				{
-					// Create visible reconstruction (a fantasy...)
-					if (m_useProbsForHiddenReconstruction)
+					probsLogistic(v, m_lambda, sigma);
+					if (!m_useProbsForHiddenReconstruction)
 					{
-						vp = -hs * m_w.weights().transpose();
-						vp.array() = vp.array().exp();
-						vp.array() += 1;
-						vp.array() = 1.0/vp.array();
+						sample(v);
 					}
-					else
-					{
-						vs = sample(vp);
-					}
-
-					// Create hidden reconstruction
-					hp = -vs.transpose() * m_w.weights();
-					hp.array() = hp.array().exp();
-					hp.array() += 1;
-					hp.array() = 1.0/hp.array();
 				}
 
-				// Update weights (negative phase)
-				sumBiasV -= vp.colwise().sum();
-				if (m_doRaoBlackwell)
-				{
-					sumWeights -= vs * hp.transpose();
-					sumBiasH -= hp.colwise().sum();
-				}
-				else
-				{
-					sumWeights -= vs * hs.transpose();
-					sumBiasH -= hs.colwise().sum();
-				}
+				// Create hidden reconstruction
+				h = v * m_w.weights();
+				h += m_w.hiddenBias().transpose().replicate(batchSize, 1);
+				probsLogistic(h, m_lambda, sigma);
+			}
 
-			} // TrainingSize
+			if (!m_doRaoBlackwell)
+			{
+				sample(h);
+			}
+			// Update weights (negative phase)
+			sumBiasV -= v.colwise().sum();
+			if (!m_doSparse)
+			{
+				sumBiasH -= h.colwise().sum();
+			}
+			sumWeights -= v.transpose() * h;
+			diffErr -= v;
 
-			deltaWeights = m_momentum*deltaWeights + m_muWeights*kTrain*sumWeights - m_weightDecay*m_w.weights();
+			deltaWeights = m_momentum*deltaWeights + m_muWeights*(kTrain*sumWeights - m_weightDecay*m_w.weights());
 			m_w.weights() += deltaWeights;
 
 			deltaBiasV = m_momentum*deltaBiasV + m_muWeights*kTrain*sumBiasV;
 			m_w.visibleBias() += deltaBiasV;
 
-			deltaBiasH = m_momentum*deltaBiasH + m_muWeights*kTrain*sumBiasH;
+			if (m_doSparse)
+			{
+				h = v * m_w.weights();
+				h += m_w.hiddenBias().transpose().replicate(batchSize, 1);
+				probsLogistic(h, m_lambda, sigma);
+
+				sumBiasH.fill(m_sparsity);
+				sumBiasH -= h.colwise().mean();
+
+				deltaBiasH = m_momentum*deltaBiasH + m_muSparsity*sumBiasH;
+
+//				cout << "Mean(" << m_sparsity << ") = " << (double)sumBiasH.array().mean() << endl;
+//				cout << sumBiasH << endl;
+			}
+			else
+			{
+				deltaBiasH = m_momentum*deltaBiasH + m_muWeights*kTrain*sumBiasH;
+			}
 			m_w.hiddenBias() += deltaBiasH;
 
 			if (sigma > sigmaMin)
@@ -360,6 +401,12 @@ public:
 				m_pListener->onEpochTrained(*this);
 			}
 
+			diffErr.array() *= diffErr.array();
+			err = diffErr.colwise().sum().sum();
+
+			cout << "err =" << endl;
+			cout << err << endl;
+
 		} // Number of epochs
 	}