[RBM]

- moved sigma and mean from WEIGHTS to RBM
- use Gibbs slider also for reconstruction draw


git-svn-id: http://moon:8086/svn/software/trunk/projects/RBM@287 b431acfa-c32f-4a4a-93f1-934dc6c82436

Source/MainComponent.cpp

@@ -74,7 +74,7 @@ MainComponent::MainComponent ()
     WeightsSlider->addListener (this);
 
     addAndMakeVisible (numEpochslabel = new Label ("Num Epochs label",
-                                                   TRANS("100")));
+                                                   TRANS("1000")));
     numEpochslabel->setFont (Font (15.00f, Font::plain));
     numEpochslabel->setJustificationType (Justification::centred);
     numEpochslabel->setEditable (true, true, false);
@@ -118,7 +118,7 @@ MainComponent::MainComponent ()
     createButton->addListener (this);
 
     addAndMakeVisible (projectNameLabel = new Label ("Project Name label",
-                                                     TRANS("TestPrj")));
+                                                     TRANS("test")));
     projectNameLabel->setFont (Font (15.00f, Font::plain));
     projectNameLabel->setJustificationType (Justification::centred);
     projectNameLabel->setEditable (true, true, false);
@@ -482,14 +482,14 @@ void MainComponent::buttonClicked (Button* buttonThatWasClicked)
     else if (buttonThatWasClicked == reconstructButton)
     {
         //[UserButtonCode_reconstructButton] -- add your button handler code here..
-    	redrawReconstruction();
+    	redrawReconstruction(numGibbsSlider->getValue());
         //[/UserButtonCode_reconstructButton]
     }
     else if (buttonThatWasClicked == ShakeButton)
     {
         //[UserButtonCode_ShakeButton] -- add your button handler code here..
     	m_weights->shuffle(weightInitLabel->getText().getFloatValue());
-    	redrawReconstruction();
+    	redrawReconstruction(numGibbsSlider->getValue());
     	redrawWeights((int)WeightsSlider->getValue());
         //[/UserButtonCode_ShakeButton]
     }
@@ -497,7 +497,7 @@ void MainComponent::buttonClicked (Button* buttonThatWasClicked)
     {
         //[UserButtonCode_testButton] -- add your button handler code here..
 		DrawTraining->setData(DrawReconstruction->getData());
-		redrawReconstruction();
+		redrawReconstruction(numGibbsSlider->getValue());
         //[/UserButtonCode_testButton]
     }
     else if (buttonThatWasClicked == createButton)
@@ -546,17 +546,7 @@ void MainComponent::buttonClicked (Button* buttonThatWasClicked)
     else if (buttonThatWasClicked == reconstructEquButton)
     {
         //[UserButtonCode_reconstructEquButton] -- add your button handler code here..
-    	uint32_t i;
+		redrawReconstruction(100);
-    	VectorXd V, H;
-
-    	V = DrawTraining->getData();
-    	for (i=0; i < 100; i++)
-    	{
-        	H = m_pRbm->toHidden(V);
-        	DrawHidden->setData(H);
-        	V = m_pRbm->toVisible(H);
-    		DrawReconstruction->setData(V);
-    	}
         //[/UserButtonCode_reconstructEquButton]
     }
     else if (buttonThatWasClicked == rbmDoRaoBlackwellToggleButton)
@@ -575,7 +565,7 @@ void MainComponent::buttonClicked (Button* buttonThatWasClicked)
     {
         //[UserButtonCode_rbmUseVisibleGaussianToggleButton] -- add your button handler code here..
     	m_pRbm->setUseVisibleGaussian(buttonThatWasClicked->getToggleState());
-    	redrawReconstruction();
+    	redrawReconstruction(numGibbsSlider->getValue());
         //[/UserButtonCode_rbmUseVisibleGaussianToggleButton]
     }
     else if (buttonThatWasClicked == rbmDoSparseToggleButton)
@@ -588,6 +578,10 @@ void MainComponent::buttonClicked (Button* buttonThatWasClicked)
     {
         //[UserButtonCode_rbmLearnVarianceButton] -- add your button handler code here..
     	m_pRbm->setDoLearnVariance(buttonThatWasClicked->getToggleState());
+		if (!buttonThatWasClicked->getToggleState())
+		{
+			m_pRbm->setSigma(sigmaLabel->getText().getFloatValue());
+		}
         //[/UserButtonCode_rbmLearnVarianceButton]
     }
     else if (buttonThatWasClicked == rbmNormalizeDataToggleButton)
@@ -614,7 +608,7 @@ void MainComponent::sliderValueChanged (Slider* sliderThatWasMoved)
 			RowVectorXd t = m_layers.getAt((int)sliderThatWasMoved->getValue());
 
 			DrawTraining->setData(t);
-			redrawReconstruction();
+			redrawReconstruction(numGibbsSlider->getValue());
 		}
         //[/UserSliderCode_patterSlider]
     }
@@ -628,6 +622,7 @@ void MainComponent::sliderValueChanged (Slider* sliderThatWasMoved)
     {
         //[UserSliderCode_numGibbsSlider] -- add your slider handling code here..
     	m_pRbm->setNumGibbs((uint32_t)sliderThatWasMoved->getValue());
+		redrawReconstruction(sliderThatWasMoved->getValue());
         //[/UserSliderCode_numGibbsSlider]
     }
     else if (sliderThatWasMoved == m_progressBarSlider)
@@ -680,14 +675,16 @@ void MainComponent::labelTextChanged (Label* labelThatHasChanged)
     {
         //[UserLabelCode_lambdaLabel] -- add your label text handling code here..
     	m_pRbm->setLambda(labelThatHasChanged->getText().getFloatValue());
-    	redrawReconstruction();
+    	redrawReconstruction(numGibbsSlider->getValue());
         //[/UserLabelCode_lambdaLabel]
     }
     else if (labelThatHasChanged == sigmaLabel)
     {
         //[UserLabelCode_sigmaLabel] -- add your label text handling code here..
     	m_pRbm->setSigma(labelThatHasChanged->getText().getFloatValue());
-    	redrawReconstruction();
+		DrawVars->setData(m_pRbm->getSigma());
+    	redrawReconstruction(numGibbsSlider->getValue());
+
         //[/UserLabelCode_sigmaLabel]
     }
     else if (labelThatHasChanged == sparsityLabel)
@@ -844,7 +841,7 @@ void MainComponent::create(const char *pFilename)
 
 	resized();
 
-	redrawReconstruction();
+	redrawReconstruction(numGibbsSlider->getValue());
    	redrawWeights((int)WeightsSlider->getValue());
 }
 
@@ -868,7 +865,7 @@ void MainComponent::onChanged(const LayerArray &obj)
 void MainComponent::onEpochTrained(const Rbm &obj)
 {
 //	mylog("Training %f %%\n", obj.getProgress()*100);
-	redrawReconstruction();
+	redrawReconstruction(numGibbsSlider->getValue());
 	redrawWeights((int)WeightsSlider->getValue());
 	m_progressBarSlider->setValue((int)(100*obj.getProgress()), dontSendNotification);
 }
@@ -881,25 +878,23 @@ void MainComponent::onDraw(DrawComponent &obj)
 	}
 	if (&obj == DrawTraining)
 	{
-    	redrawReconstruction();
+    	redrawReconstruction(numGibbsSlider->getValue());
 	}
 }
 
-void MainComponent::redrawReconstruction()
+void MainComponent::redrawReconstruction(uint32_t numIter)
 {
-	RowVectorXd h = m_pRbm->toHidden(DrawTraining->getData());
+	uint32_t i;
-	DrawHidden->setData(h);
+	VectorXd V, H;
-	
-	RowVectorXd v = m_pRbm->toVisible(DrawHidden->getData());
-	DrawReconstruction->setData(v);
-	
-//	RowVectorXd dataV(m_weights->getNumVisible());
-//	DrawHidden->setData(m_pRbm->toHidden(dataV));
-
-//	RowVectorXd dataH(m_weights->getNumHidden());
-//	VectorXd v = m_pRbm->toVisible(dataH);
-//	DrawReconstruction->setData(v);
 
+	V = DrawTraining->getData();
+	for (i=0; i < numIter; i++)
+	{
+		H = m_pRbm->toHidden(V);
+		DrawHidden->setData(H);
+		V = m_pRbm->toVisible(H);
+		DrawReconstruction->setData(V);
+	}
 
 	if ((m_vNumX == 27) && (m_vNumY == 4))
 	{
@@ -933,14 +928,14 @@ void MainComponent::redrawWeights(int index)
 {
 	VectorXd w = m_weights->weights().col(index);
 	DrawWeights->setData(w);
-	DrawVars->setData(m_weights->sigma());
+	DrawVars->setData(m_pRbm->getSigma());
 
 }
 
 void MainComponent::run()
 {
 //	trainButton->setEnabled(false);
-	m_pRbm->train(m_layers, numEpochslabel->getText().getIntValue(), 100);
+	m_pRbm->train(m_layers, numEpochslabel->getText().getIntValue());
 //	trainButton->setEnabled(true);
 }
 

Source/MainComponent.h

@@ -93,7 +93,7 @@ private:
 	void onChanged(const LayerArray &obj);
 	void onEpochTrained(const Rbm &obj);
 	void onDraw(DrawComponent &obj);
-	void redrawReconstruction();
+	void redrawReconstruction(uint32_t numIter);
 	void redrawWeights(int index);
 	void run();
 	String m_baseDir;

Source/Rbm.hpp

@@ -41,9 +41,9 @@ public:
 	: m_w(weights)
 	, m_v(weights.getNumVisible())
 	, m_h(weights.getNumHidden())
+	, m_sigmas(weights.getNumVisible())
 	, m_pListener(pListener)
 	, m_progress(0)
-	, m_sigma(1.0)
 	, m_sigmaDecay(1.0)
 	, m_weightDecay(0.0)
 	, m_lambda(1.0)
@@ -222,7 +222,7 @@ public:
 		return t1;
 	}
 
-	void train(const LayerArray &vt, uint32_t numEpochs, uint32_t batchSize, double sigmaMin = 0.05)
+	void train(const LayerArray &vt, uint32_t numEpochs, double sigmaMin = 0.05)
 	{
 		uint32_t t, i;
 		uint32_t epoch;
@@ -230,7 +230,7 @@ public:
 		double dProgress = 1.0/numEpochs;
 		double kTrain = 1.0/vt.getSize();
 
-		batchSize = vt.getSize();
+		size_t batchSize = vt.getSize();
 
 		MatrixXd v(batchSize, m_w.getNumVisible());
 		MatrixXd h(batchSize, m_w.getNumHidden());
@@ -252,19 +252,18 @@ public:
 
 		batch = vt.data();
 
-		m_w.mean() = calcMean(batch);
-
 		if (m_doLearnVariance)
 		{
-			m_w.sigma() = calcSigma(batch);
+			m_sigmas = calcSigma(batch);
 		}
 
 		if (m_doNormalizeData)
 		{
+			RowVectorXd mean = calcMean(batch);
 			for (i=0; i < batchSize; i++)
 			{
 				RowVectorXd x = batch.row(i);
-				batch.row(i) = normalizeData(x, m_w.mean(), m_w.sigma());
+				batch.row(i) = normalizeData(x, mean, m_sigmas);
 			}
 		}
 
@@ -310,12 +309,12 @@ public:
 				{
 					if (!m_useProbsForHiddenReconstruction)
 					{
-						sampleGaussian(v, m_w.sigma().replicate(batchSize, 1));
+						sampleGaussian(v, m_sigmas.replicate(batchSize, 1));
 					}
 				}
 				else
 				{
-					probsLogistic(v, m_w.sigma().replicate(batchSize, 1));
+					probsLogistic(v, m_sigmas.replicate(batchSize, 1));
 					if (!m_useProbsForHiddenReconstruction)
 					{
 						sample(v);
@@ -367,9 +366,9 @@ public:
 			}
 			m_w.hiddenBias() += deltaBiasH;
 
-			if (m_w.sigma()[0] > sigmaMin)
+			if (m_sigmas[0] > sigmaMin)
 			{
-				m_w.sigma().array() *= m_sigmaDecay;
+				m_sigmas.array() *= m_sigmaDecay;
 			}
 
 			m_progress += dProgress;
@@ -395,12 +394,13 @@ public:
 	double getEnergy(const VectorXd& visible, const VectorXd& hidden)
 	{
 		double energy;
+		double sigma = m_sigmas.array().mean();
 
 		energy = m_w.visibleBias() * visible;
 		energy += m_w.hiddenBias() * hidden;
 		energy += visible.transpose() * m_w.weights() * hidden;
-
+		
-		return -energy/(m_sigma*m_sigma);
+		return -energy/(sigma*sigma);
 	}
 
 	RowVectorXd const & toHidden(const RowVectorXd& v)
@@ -419,18 +419,23 @@ public:
 
 		if (m_useVisibleGaussian)
 		{
-//			probsGaussian(v, m_w.sigma());
+//			probsGaussian(v, m_sigmas);
 		}
 		else
 		{
-			probsLogistic(m_v, m_w.sigma());
+			probsLogistic(m_v, m_sigmas);
 		}
 		return m_v;
 	}
 
    void setSigma(double value)
    {
-		m_sigma = value;
+		m_sigmas.fill(value);
+   }
+
+   RowVectorXd& getSigma()
+   {
+		return m_sigmas;
    }
 
    void setSigmaDecay(double value)
@@ -481,10 +486,6 @@ public:
    void setDoLearnVariance(bool flag)
    {
 	   m_doLearnVariance = flag;
-	   if (!flag)
-	   {
-			m_w.sigma().fill(m_sigma);
-	   }
    }
 
    void setNumGibbs(uint32_t value)
@@ -517,10 +518,11 @@ private:
 	Weights &m_w;
 	RowVectorXd m_h;
 	RowVectorXd m_v;
+	RowVectorXd m_sigmas;
+
 	RbmListener *m_pListener;
 	noise_gen_t m_noise;
 	double m_progress;
-	double m_sigma;
 	double m_sigmaDecay;
 	double m_weightDecay;
 	double m_lambda;

Source/Weights.hpp

@@ -80,8 +80,6 @@ public:
 		m_numHidden = numHidden;
 
 		m_w.resize(m_numVisible, m_numHidden);
-		m_sigma.resize(m_numVisible);
-		m_mean.resize(m_numVisible);
 		m_bv.resize(m_numVisible);
 		m_bh.resize(m_numHidden);
 		shuffle(1.0);
@@ -92,25 +90,8 @@ public:
 		uint32_t i, j;
 		double kdev = stdDev*sqrt(12.0);
 
-		for (i=0; i < m_numVisible; i++)
+		m_bv.array().fill(Noise_Uniform(&m_noise));
-		{
+		m_bh.array().fill(Noise_Uniform(&m_noise));
-			m_sigma(i) = 1; //kdev*Noise_Uniform(&m_noise);
-		}
-
-		for (i=0; i < m_numVisible; i++)
-		{
-			m_mean(i) = 0; //kdev*Noise_Uniform(&m_noise);
-		}
-
-		for (i=0; i < m_numVisible; i++)
-		{
-			m_bv(i) = 0; //kdev*Noise_Uniform(&m_noise);
-		}
-
-		for (j=0; j < m_numHidden; j++)
-		{
-			m_bh(j) = 0; //kdev*Noise_Uniform(&m_noise);
-		}
 
 		for (i=0; i < m_numVisible; i++)
 		{
@@ -126,8 +107,6 @@ public:
 		m_bv = rhs.m_bv;
 		m_bh = rhs.m_bh;
 		m_w = rhs.m_w;
-		m_sigma = rhs.m_sigma;
-		m_mean = rhs.m_mean;
 
 		return *this;
 	}
@@ -142,16 +121,6 @@ public:
 		return m_bv;
 	}
 
-	RowVectorXd& sigma()
-	{
-		return m_sigma;
-	}
-
-	RowVectorXd& mean()
-	{
-		return m_mean;
-	}
-
 	RowVectorXd& hiddenBias()
 	{
 		return m_bh;
@@ -294,8 +263,6 @@ private:
 	MatrixXd m_w;
 	RowVectorXd m_bv;
 	RowVectorXd m_bh;
-	RowVectorXd m_sigma;
-	RowVectorXd m_mean;
 
 	void free()
 	{