[RBM]

- committed last changes

git-svn-id: http://moon:8086/svn/software/trunk/projects/RBM@359 b431acfa-c32f-4a4a-93f1-934dc6c82436

Source/LayerArray.hpp

@@ -142,6 +142,7 @@ public:
 				fscanf(pFile, "%f", &v);
 				data(j) = v;
 			}
+//			fscanf(pFile, "%d", &numUnits);
 			m_data.row(i) = data;
 			if (!(i%100) || (i == (size-1)))
 				printf("Loaded training sample %u of %u\n", i, size);

Source/MainComponent.cpp

@@ -585,6 +585,9 @@ void MainComponent::buttonClicked (Button* buttonThatWasClicked)
     {
         //[UserButtonCode_rbmNormalizeDataToggleButton] -- add your button handler code here..
     	m_pRbmComponentCurr->setNormalizeData(buttonThatWasClicked->getToggleState());
+		m_pRbmComponentCurr->batchchanged();
+    	m_pRbmComponentCurr->redrawWeights();
+    	m_pRbmComponentCurr->redrawReconstruction();
         //[/UserButtonCode_rbmNormalizeDataToggleButton]
     }
     else if (buttonThatWasClicked == rbmDoSampleBatch)
@@ -852,7 +855,6 @@ void MainComponent::create(juce::String const &projectName)
 	m_pRbmComponentCurr->batchchanged();
 	m_pRbmComponentCurr->redrawReconstruction();
 	m_pRbmComponentCurr->redrawWeights();
-	m_pRbmComponentCurr->redrawVariances();
 
 	if (((id+1) < DBN_SIZE) and shouldAddItem)
 		m_rbmSelect->addItem(String(id+1), id+2);
@@ -898,7 +900,8 @@ void MainComponent::updateControls()
 	rbmDoSparseToggleButton->setToggleState(m_pRbmComponentCurr->params().m_doSparse, dontSendNotification);
    	rbmLearnVarianceButton->setToggleState(m_pRbmComponentCurr->params().m_doLearnVariance, dontSendNotification);
 	rbmDoSampleBatch->setToggleState(m_pRbmComponentCurr->params().m_doSampleBatch, dontSendNotification);
-
+	rbmNormalizeDataToggleButton->setToggleState(m_pRbmComponentCurr->params().m_doNormalizeData, dontSendNotification);
+	
 	lambdaLabel->setText(String(m_pRbmComponentCurr->params().m_lambda), dontSendNotification);
 	sigmaDecayLabel->setText(String(m_pRbmComponentCurr->params().m_sigmaDecay), dontSendNotification);
 	weightDecayLabel->setText(String(m_pRbmComponentCurr->params().m_weightDecay), dontSendNotification);

Source/Rbm.cpp

@@ -14,11 +14,9 @@ void mylog(const char* format, ...);
 Rbm::Rbm(Weights &weights, const MatrixXd &batch)
 : m_w(weights)
 , m_batch(batch)
-, m_variableSigma(weights.getNumVisible())
 , m_progress(0)
 {
 	Noise_Init(&m_noise, 0x32727155);
-	m_variableSigma.fill(m_params.m_constantSigma);
 	updateHiddenBatch();
 }
 
@@ -49,25 +47,17 @@ void Rbm::noiseUniform(MatrixXd &dst)
 	}
 }
 
-void Rbm::sampleGaussian(MatrixXd &dst, MatrixXd const &src, const MatrixXd &sigma)
+void Rbm::sampleGaussian(MatrixXd &dst, MatrixXd const &src)
 {
 	MatrixXd n(src.rows(), src.cols());
 
 	noiseGaussian(n);
-	dst = sigma.array()*n.array() + src.array();
+	dst = n.array() + src.array();
 }
 
-void Rbm::sampleGaussian(MatrixXd &srcDst, const MatrixXd &sigma)
+void Rbm::sampleGaussian(MatrixXd &srcDst)
 {
-	MatrixXd n(srcDst.rows(), srcDst.cols());
-
-	noiseGaussian(n);
-	srcDst.array() += sigma.array()*n.array();
-}
-
-void Rbm::sampleGaussian(MatrixXd &srcDst, const double &sigma)
-{
-	sampleGaussian(srcDst, sigma*MatrixXd::Ones(srcDst.rows(), srcDst.cols()));
+	sampleGaussian(srcDst, srcDst);
 }
 
 void Rbm::sample(MatrixXd &srcDst)
@@ -85,116 +75,32 @@ void Rbm::sample(MatrixXd &dst, MatrixXd const &src)
 	
 }
 
-void Rbm::probsLogistic(MatrixXd &src)
+void Rbm::probsLogistic(MatrixXd &srcDst)
 {
-	src = (1 + (-src.array()).exp()).array().cwiseInverse();
+	srcDst = (1 + (-srcDst.array()).exp()).array().cwiseInverse();
 }
 
-void Rbm::probsLogistic(RowVectorXd &src)
+void Rbm::probsLogistic(RowVectorXd &srcDst)
 {
-	src = (1 + (-src.array()).exp()).array().cwiseInverse();
+	srcDst = (1 + (-srcDst.array()).exp()).array().cwiseInverse();
 }
 
-void Rbm::probsLogistic(MatrixXd &src, const MatrixXd &sigma)
+void Rbm::normalizeData(MatrixXd &dst, MatrixXd const &src)
 {
-	src.array() /= (sigma.array() + EPSILON_SIGMA);
-	probsLogistic(src);
-}
+	MatrixXd mean = src.rowwise().mean();
+//	cout << "mean" << ": " << endl << mean << endl;
 
-void Rbm::probsLogistic(RowVectorXd &src, const RowVectorXd &sigma)
-{
-	src.array() /= (sigma.array() + EPSILON_SIGMA);
-	probsLogistic(src);
-}
+	dst = src - mean.replicate(1, src.cols());
+//	cout << "dst - mean" << ": " << endl << dst << endl;
 
-void Rbm::probsGaussian(MatrixXd &src, const MatrixXd &sigma)
-{
-	src.array() = 1 - src.array();
-	src.array() *= src.array();
-	src.array() *= -0.5;
+	MatrixXd x = dst.array().square();
 
-	MatrixXd var = sigma;
-	var.array() += EPSILON_SIGMA;
-	var.array() *= var.array();
+	MatrixXd var = x.rowwise().mean();
+//	cout << "var" << ": " << endl << var << endl;
 
-	src.array() /= var.array();
-	src.array() = src.array().exp();
-
-	MatrixXd k = var;
-
-	k.array() *= 2*3.14159265359;
-	k.array() = k.array().sqrt();
-	k.array() = 1.0/k.array();
-
-	src.array() *= k.array();
-}
-
-void Rbm::probsGaussian(RowVectorXd &src, const RowVectorXd &sigma)
-{
-	src.array() = 1 - src.array();
-	src.array() *= src.array();
-	src.array() *= -0.5;
-
-	RowVectorXd var = sigma;
-	var.array() += EPSILON_SIGMA;
-	var.array() *= var.array();
-
-	src.array() /= var.array();
-	src.array() = src.array().exp();
-
-	RowVectorXd k = var;
-
-	k.array() *= 2*3.14159265359;
-	k.array() = k.array().sqrt();
-	k.array() = 1.0/k.array();
-
-	src.array() *= k.array();
-}
-
-RowVectorXd Rbm::normalizeData(RowVectorXd const &src, RowVectorXd const &mu, RowVectorXd const &var)
-{
-	// Remove mean
-	RowVectorXd res = src - mu;
-//		res.array() /= var.array() + EPSILON_SIGMA;
-
-//		cout << __PRETTY_FUNCTION__ << ": " << res << endl;
-	return res;
-}
-
-RowVectorXd Rbm::calcMean(MatrixXd const &batch)
-{
-	// Remove mean
-	RowVectorXd res = batch.colwise().mean();
-
-//		cout << __PRETTY_FUNCTION__ << ": " << res << endl;
-	return res;
-
-}
-
-RowVectorXd Rbm::calcSigma(MatrixXd const &batch)
-{
-	MatrixXd x = batch.rowwise() - batch.colwise().mean();
-
-	x.array() *= x.array();
-
-	RowVectorXd res = x.colwise().mean().array().sqrt();
-
-//		cout << __PRETTY_FUNCTION__ << ": " << res << endl;
-	return res;
-}
-
-MatrixXd Rbm::calcZ(MatrixXd &v, MatrixXd &h)
-{
-
-	MatrixXd t1(v.rows(), m_w.getNumVisible());
-
-	t1 = v - m_w.visibleBias().transpose().replicate(v.rows(), 1);
-	t1.array() *= t1.array();
-	t1.array() *= 0.5;
-
-	t1 -= (h * m_w.weights().transpose());
-
-	return t1;
+	MatrixXd stddev_norm = var.array().sqrt().cwiseInverse();
+	dst.array() *= stddev_norm.replicate(1, src.cols()).array();
+//	cout << "dst" << ": " << endl << dst << endl;
 }
 
 void Rbm::train(size_t numEpochs, size_t miniBatchSize, double sigmaMin)
@@ -216,12 +122,19 @@ void Rbm::train(size_t numEpochs, size_t miniBatchSize, double sigmaMin)
 	MatrixXd dBiasH(MatrixXd::Zero(1, m_w.getNumHidden()));
 	MatrixXd dW(MatrixXd::Zero(m_w.getNumVisible(), m_w.getNumHidden()));
 
+	MatrixXd __batch = m_batch_normalized;
+
+	if (m_params.m_doNormalizeData && !m_params.m_useVisibleGaussian)
+	{
+		probsLogistic(__batch);
+	}
+	
 	m_progress = 0;
 	while (trainingSizeRemain)
 	{
 		cout << "trainingSizeRemain: " << trainingSizeRemain << endl;
 		size_t toSlice = std::min(miniBatchSize, trainingSizeRemain);
-		MatrixXd batch = m_batch.block(batchRowIndex, 0, toSlice, m_w.getNumVisible());
+		MatrixXd batch = __batch.block(batchRowIndex, 0, toSlice, m_w.getNumVisible());
 		trainingSizeRemain -= toSlice;
 		batchRowIndex += toSlice;
 		size_t batchSize = batch.rows();
@@ -234,16 +147,6 @@ void Rbm::train(size_t numEpochs, size_t miniBatchSize, double sigmaMin)
 		MatrixXd vis(batchSize, m_w.getNumVisible());
 		MatrixXd hid(batchSize, m_w.getNumHidden());
 		
-		if (m_params.m_doNormalizeData)
-		{
-			RowVectorXd mean = calcMean(batch);
-			for (i=0; i < batchSize; i++)
-			{
-				RowVectorXd x = batch.row(i);
-				batch.row(i) = normalizeData(x, mean, m_variableSigma);
-			}
-		}
-		
 		for (epoch=0; epoch < numEpochs; epoch++)
 		{
 			onProgressChanged();
@@ -280,7 +183,7 @@ void Rbm::train(size_t numEpochs, size_t miniBatchSize, double sigmaMin)
 			{
 				if (m_params.m_useHiddenGaussian)
 				{
-					sampleGaussian(hid, m_params.m_constantSigma);
+					sampleGaussian(hid);
 				}
 				else
 				{
@@ -291,7 +194,7 @@ void Rbm::train(size_t numEpochs, size_t miniBatchSize, double sigmaMin)
 					// Create visible reconstruction (a fantasy...) given hid
 					vis = hid * m_w.weights().transpose();
 					vis += m_w.visibleBias().replicate(batchSize, 1);
-					sampleGaussian(v_sampled, vis, m_variableSigma.replicate(batchSize, 1));
+					sampleGaussian(v_sampled, vis);
 					hid = v_sampled * m_w.weights();
 					hid += m_w.hiddenBias().replicate(batchSize, 1);
 					probsLogistic(hid);
@@ -302,7 +205,7 @@ void Rbm::train(size_t numEpochs, size_t miniBatchSize, double sigmaMin)
 					// Create visible reconstruction (a fantasy...) given hid
 					vis = hid * m_w.weights().transpose();
 					vis += m_w.visibleBias().replicate(batchSize, 1);
-					probsLogistic(vis, m_variableSigma.replicate(batchSize, 1));
+					probsLogistic(vis);
 					if (m_params.m_doSampleVisible)
 					{
 						sample(v_sampled, vis);
@@ -348,7 +251,6 @@ void Rbm::train(size_t numEpochs, size_t miniBatchSize, double sigmaMin)
 			{
 				if (m_variableSigma[0] > sigmaMin)
 				{
-					m_variableSigma.array() *= (1-m_params.m_sigmaDecay);
 				}
 			}
 
@@ -365,9 +267,9 @@ void Rbm::train(size_t numEpochs, size_t miniBatchSize, double sigmaMin)
 	updateHiddenBatch();
 	
 	MatrixXd vis = m_h * m_w.weights().transpose();
-	vis += m_w.visibleBias().replicate(m_batch.rows(), 1);
-	probsLogistic(vis, m_variableSigma.replicate(m_batch.rows(), 1));
-	MatrixXd diffErr = m_batch - vis;
+	vis += m_w.visibleBias().replicate(__batch.rows(), 1);
+	probsLogistic(vis);
+	MatrixXd diffErr = __batch - vis;
 	diffErr.array() *= diffErr.array();
 	double err = diffErr.colwise().sum().sum();
 	cout << "error (total) = " << err << endl;
@@ -381,18 +283,6 @@ double Rbm::getProgress() const
 	return m_progress;
 }
 
-double Rbm::getEnergy(const VectorXd& visible, const VectorXd& hidden)
-{
-	double energy;
-	double sigma = m_variableSigma.array().mean();
-
-	energy = m_w.visibleBias() * visible;
-	energy += m_w.hiddenBias() * hidden;
-	energy += visible.transpose() * m_w.weights() * hidden;
-
-	return -energy/(sigma*sigma);
-}
-
 void Rbm::toHidden(RowVectorXd &h, RowVectorXd const &v)
 {
 	h = v * m_w.weights();
@@ -405,20 +295,15 @@ void Rbm::toVisible(RowVectorXd &v, RowVectorXd const &h)
 	v = h * m_w.weights().transpose();
 	v += m_w.visibleBias();
 
-	if (m_params.m_useVisibleGaussian)
+	if (!m_params.m_useVisibleGaussian)
 	{
-//		probsGaussian(v, m_variableSigma);
-	}
-	else
-	{
-		probsLogistic(v, m_variableSigma);
+		probsLogistic(v);
 	}
 }
 
 void Rbm::setConstantSigma(double value)
 {	
 	 m_params.m_constantSigma = value;
-	 m_variableSigma.fill(m_params.m_constantSigma);
 	 onParamsChanged();
 }
 
@@ -496,14 +381,6 @@ void Rbm::setNormalizeData(bool flag)
 void Rbm::setDoLearnVariance(bool flag)
 {
 	m_params.m_doLearnVariance = flag;
-	if (m_params.m_doLearnVariance && m_batch.rows())
-	{
-		m_variableSigma = calcSigma(m_batch);
-	}
-	else
-	{
-		 m_variableSigma.fill(m_params.m_constantSigma);
-	}
 	onParamsChanged();
 }
 
@@ -538,7 +415,7 @@ MatrixXd const& Rbm::getHiddenBatch()
 
 MatrixXd const& Rbm::getBatch()
 {
-	return m_batch;
+	return m_batch_normalized;
 }
 
 void Rbm::updateHiddenBatch()
@@ -547,8 +424,18 @@ void Rbm::updateHiddenBatch()
 	{
 		return;
 	}
+	m_batch_normalized.resize(m_batch.rows(), m_w.getNumHidden());
+	
+	if (m_params.m_doNormalizeData)
+	{
+		normalizeData(m_batch_normalized, m_batch);
+	}
+	else
+	{
+		m_batch_normalized = m_batch;
+	}
 	m_h.resize(m_batch.rows(), m_w.getNumHidden());
-	m_h = m_batch * m_w.weights();
+	m_h = m_batch_normalized * m_w.weights();
 	m_h += m_w.hiddenBias().replicate(m_batch.rows(), 1);
 	probsLogistic(m_h);
 }

Source/Rbm.hpp

@@ -63,22 +63,13 @@ public:
 	~Rbm();
 	void sample(MatrixXd &srcDst);
 	void sample(MatrixXd &dst, MatrixXd const &src);
-	static void probsLogistic(MatrixXd &src);
-	static void probsLogistic(RowVectorXd &src);
-	static void probsLogistic(MatrixXd &src, const MatrixXd &sigma);
-	static void probsLogistic(RowVectorXd &src, const RowVectorXd &sigma);
-	static void probsGaussian(MatrixXd &src, const MatrixXd &sigma);
-	static void probsGaussian(RowVectorXd &src, const RowVectorXd &sigma);
-	void sampleGaussian(MatrixXd &dst, MatrixXd const &src, const MatrixXd &sigma);
-	void sampleGaussian(MatrixXd &srcDst, const MatrixXd &sigma);
-	void sampleGaussian(MatrixXd &srcDst, const double &sigma);
-	RowVectorXd normalizeData(RowVectorXd const &src, RowVectorXd const &mu, RowVectorXd const &var);
-	RowVectorXd calcMean(MatrixXd const &batch);
-	RowVectorXd calcSigma(MatrixXd const &batch);
-	MatrixXd calcZ(MatrixXd &v, MatrixXd &h);
+	static void probsLogistic(MatrixXd &srcDst);
+	static void probsLogistic(RowVectorXd &srcDst);
+	void sampleGaussian(MatrixXd &dst, MatrixXd const &src);
+	void sampleGaussian(MatrixXd &srcDst);
+	void normalizeData(MatrixXd &dst, MatrixXd const &src);
 	void train(size_t numEpochs, size_t miniBatchSize, double sigmaMin = 0.05);
 	double getProgress() const;
-	double getEnergy(const VectorXd& visible, const VectorXd& hidden);
 	void toHidden(RowVectorXd &h, RowVectorXd const &v);
 	void toVisible(RowVectorXd &v, RowVectorXd const &h);
 	void setConstantSigma(double value);
@@ -107,6 +98,7 @@ public:
 private:
 	Weights &m_w;
 	MatrixXd const &m_batch;
+	MatrixXd m_batch_normalized;
 	MatrixXd m_h;
 	RowVectorXd m_variableSigma;
 	noise_gen_t m_noise;

Source/RbmComponent.cpp

@@ -46,12 +46,10 @@ RbmComponent::RbmComponent (Weights &_weights, RbmComponent *pRbmUpper, RbmCompo
 
 	
     addAndMakeVisible (DrawWeights = new DrawComponent (getTopWeights().getNumVisibleX(), getTopWeights().getNumVisibleY(), 0.5, 0.5));
-    addAndMakeVisible (DrawVars = new DrawComponent (vNumX, vNumY));
     addAndMakeVisible (DrawHidden = new DrawComponent (hNum, 1));
 	DrawHidden->setListener(this);
 	redrawWeights();
 	redrawReconstruction();
-	redrawVariances();
 
     setSize (430, 130);
 	resized();
@@ -80,12 +78,10 @@ RbmComponent::RbmComponent (Weights &_weights, MatrixXd const &batch, RbmCompone
 
 	
     addAndMakeVisible (DrawWeights = new DrawComponent (getTopWeights().getNumVisibleX(), getTopWeights().getNumVisibleY(), 0.5, 0.5));
-    addAndMakeVisible (DrawVars = new DrawComponent (vNumX, vNumY));
     addAndMakeVisible (DrawHidden = new DrawComponent (hNum, 1));
 	DrawHidden->setListener(this);
 	redrawWeights();
 	redrawReconstruction();
-	redrawVariances();
 
     setSize (430, 130);
 	resized();
@@ -116,7 +112,6 @@ RbmComponent::~RbmComponent()
     DrawTraining = nullptr;
     DrawReconstruction = nullptr;
     DrawWeights = nullptr;
-    DrawVars = nullptr;
     DrawHidden = nullptr;
     //[/Destructor]
 }
@@ -198,7 +193,6 @@ void RbmComponent::resized()
 	DrawTraining->setBoundsRelative(0, 0, 100./430, 100./130);
 	DrawReconstruction->setBoundsRelative(110./430, 0, 100./430, 100./130);
     DrawWeights->setBoundsRelative (220./430, 0, 100./430, 100./130);
-    DrawVars->setBoundsRelative (330./430, 0, 100./430, 100./130);
     DrawHidden->setBoundsRelative (0, 110./130, 430./430, 20./130);
 }
 
@@ -305,16 +299,9 @@ void RbmComponent::redrawWeights()
 	DrawWeights->DrawData();
 }
 
-void RbmComponent::redrawVariances()
-{
-	DrawVars->getData() = getVariableSigma();
- 	DrawVars->DrawData();
-}
-
 void RbmComponent::onParamsChanged()
 {
 	redrawWeights();
-	redrawVariances();
    	redrawReconstruction();
 }
 
@@ -325,7 +312,6 @@ void RbmComponent::batchchanged()
 	{
 		lower->batchchanged();
 	}
-	redrawVariances();
 }
 
 void RbmComponent::setWeightsIndex(size_t index)

Source/RbmComponent.h

@@ -108,7 +108,6 @@ public:
 
 	void redrawWeights();
 	void redrawReconstruction();
-	void redrawVariances();
 	RowVectorXd const& getTrainingData();
 	
 	void downPass(RowVectorXd &dst, RowVectorXd const &src) override
@@ -143,7 +142,6 @@ private:
     ScopedPointer<DrawComponent> DrawTraining;
     ScopedPointer<DrawComponent> DrawReconstruction;
     ScopedPointer<DrawComponent> DrawWeights;
-    ScopedPointer<DrawComponent> DrawVars;
     ScopedPointer<DrawComponent> DrawHidden;
 	size_t m_currWeightIndexToDraw;
 	size_t m_currTrainingIndexToDraw;

Source/Weights.hpp

@@ -13,7 +13,7 @@
 #include <stdint.h>
 #include <iostream>
 #include <Eigen/Dense>
-
+#include <stdio.h>
 #include "noise.h"
 
 using namespace std;