[RBM]

- DrawComponent: use fixed value scaling
- Rbm: fixed weight decay
- Rbm: fixed sparsity


git-svn-id: http://moon:8086/svn/software/trunk/projects/RBM@298 b431acfa-c32f-4a4a-93f1-934dc6c82436

Source/DrawComponent.cpp

@@ -29,13 +29,15 @@ void mylog(const char* format, ...);
 //[/MiscUserDefs]
 
 //==============================================================================
-DrawComponent::DrawComponent (int width, int height)
+DrawComponent::DrawComponent (int width, int height, float offset, float scale)
     : m_pListener(nullptr)
 {
 
     //[UserPreSize]
     m_width = width;
     m_height = height;
+	m_offset = offset;
+	m_scale = scale;
     m_scaleX = 1.0;
     m_scaleY = 1.0;
 	m_pG = nullptr;
@@ -231,36 +233,19 @@ RowVectorXd& DrawComponent::getData ()
 void DrawComponent::DrawData ()
 {
 
-	double a;
 	RowVectorXd temp = m_data;
-	double min = +1E12;
-	double max = -1E12;
 
 	for (int i=0; i < m_width*m_height; i++)
 	{
-		min = std::min<double>(min, (double)temp[i]);
+		temp[i] = std::min<double>(1.0, (double)temp[i]);
-		max = std::max<double>(max, (double)temp[i]);
+		temp[i] = std::max<double>(-1.0, (double)temp[i]);
-	}
-
-	if (min < 0)
-	{
-		for (int i=0; i < m_width*m_height; i++)
-		{
-			temp[i] -= min;
-		}
-		max -= min;
-	}
-	for (int i=0; i < m_width*m_height; i++)
-	{
-		temp[i] /= max;
 	}
 
 	for (int i=0; i < m_height; i++)
 	{
 		for (int j=0; j < m_width; j++)
 		{
-        	a = std::min<double>(std::max<double>((double)temp[i*m_width + j], 0), 1);
+			m_pG->setColour(Colour(Colours::white).greyLevel(m_offset + m_scale*temp[i*m_width + j]));
-        	m_pG->setColour(Colour(Colours::white).greyLevel(a));
 			m_pG->fillRect(m_scaleX*j, m_scaleY*i, m_scaleX, m_scaleY);
 		}
 	}

Source/DrawComponent.h

@@ -49,7 +49,7 @@ class DrawComponent  : public Component
 {
 public:
     //==============================================================================
-    DrawComponent (int width, int height);
+    DrawComponent (int width, int height, float offset=0.0, float scale=1.0);
     ~DrawComponent();
 
     //==============================================================================
@@ -79,6 +79,8 @@ private:
     DrawListener *m_pListener;
     int m_width;
     int m_height;
+	float m_offset;
+	float m_scale;
     float m_scaleX;
     float m_scaleY;
     ScopedPointer<Graphics>m_pG;

Source/Rbm.hpp

@@ -27,7 +27,7 @@ public:
 		Params()
 		: m_constantSigma(1.0)
 		, m_sigmaDecay(1.0)
-		, m_weightDecay(0.01)
+		, m_weightDecay(0.00001)
 		, m_lambda(1.0)
 		, m_sparsity(0.05)
 		, m_muWeights(0.1)
@@ -69,17 +69,6 @@ public:
 	, m_progress(0)
 	{
 		Noise_Init(&m_noise, 0x32727155);
-
-#if 1
-		VectorXd a(4);
-		a << 1, 2, 3, 4;
-		VectorXd b(4);
-
-		b.array() = -a.array().exp();
-
-		cout << b << endl;
-		
-#endif
 		m_variableSigma.fill(m_params.m_constantSigma);
 		updateHiddenBatch();
 	}
@@ -289,9 +278,9 @@ public:
 		{
 			onProgressChanged();
 
-			// When the hidden units are being driven by data, always use stochastic binary states
 			if (m_params.m_doSampleBatch)
 			{
+				// When the hidden units are being driven by data, always use stochastic binary states
 				sample(batch_sampled, batch);
 
 				// Create hidden layer base on sampled training data
@@ -310,10 +299,7 @@ public:
 
 			// Update weights (positive phase)
 			dBiasV_curr = batch.colwise().sum();
-			if (!m_params.m_doSparse)
-			{
 			dBiasH_curr = h.colwise().sum();
-			}
 			dW_curr = batch.transpose() * h;
 
 			for (gibbs=0; gibbs < m_params.m_numGibbs; gibbs++)
@@ -348,36 +334,27 @@ public:
 
 			// Update weights (negative phase)
 			dBiasV_curr -= m_v.colwise().sum();
-			if (!m_params.m_doSparse)
-			{
 			dBiasH_curr -= h.colwise().sum();
-			}
 			dW_curr -= m_v.transpose() * h;
 
 			m_w.visibleBias() += mu_biasV*(m_params.m_momentum*dBiasV + (1-m_params.m_momentum)*dBiasV_curr);
 			dBiasV = dBiasV_curr;
 			
-			m_w.weights() += mu_w*(m_params.m_momentum*dW + (1-m_params.m_momentum)*dW_curr - m_params.m_weightDecay*m_w.weights());
-			dW = dW_curr;
-			
 			if (m_params.m_doSparse)
 			{
-				// Create hidden representation given v
+				MatrixXd h1 = h-MatrixXd::Ones(h.rows(), h.cols())*m_params.m_sparsity;
-				toHiddenBatch(h, batch);
+				RowVectorXd hm = h1.colwise().mean();
-
+				m_w.hiddenBias() -=  m_params.m_muSparsity * hm;
-				dBiasH_curr = m_params.m_sparsity * MatrixXd::Ones(dBiasH.rows(), dBiasH.cols()) - h.colwise().mean();
-
-				m_w.hiddenBias() += m_params.m_muSparsity*(m_params.m_momentum*dBiasH + (1-m_params.m_momentum)*dBiasH_curr);
-				dBiasH = dBiasH_curr;
-				
-//				cout << "Mean(" << m_sparsity << ") = " << (double)sumBiasH.array().mean() << endl;
-//				cout << sumBiasH << endl;
 			}
 			else
 			{
 				m_w.hiddenBias() += mu_biasH*(m_params.m_momentum*dBiasH + (1-m_params.m_momentum)*dBiasH_curr);
-				dBiasH = dBiasH_curr;
 			}
+			dBiasH = dBiasH_curr;
+			
+			m_w.weights() -= m_params.m_weightDecay*m_w.weights();
+			m_w.weights() += mu_w*(m_params.m_momentum*dW + (1-m_params.m_momentum)*dW_curr);
+			dW = dW_curr;
 			
 			if (m_variableSigma[0] > sigmaMin)
 			{

Source/RbmComponent.cpp

@@ -44,7 +44,7 @@ RbmComponent::RbmComponent (Weights &weights, MatrixXd const &batch, RbmComponen
 	DrawTraining->setListener(this);
 
     addAndMakeVisible (DrawReconstruction = new DrawComponent (vNumX, vNumY));
-    addAndMakeVisible (DrawWeights = new DrawComponent (vNumX, vNumY));
+    addAndMakeVisible (DrawWeights = new DrawComponent (vNumX, vNumY, 0.5, 0.5));
     addAndMakeVisible (DrawVars = new DrawComponent (vNumX, vNumY));
     addAndMakeVisible (DrawHidden = new DrawComponent (hNum, 1));
 	DrawHidden->setListener(this);
@@ -56,6 +56,21 @@ RbmComponent::RbmComponent (Weights &weights, MatrixXd const &batch, RbmComponen
     setSize (430, 130);
 	resized();
 
+	MatrixXd m = MatrixXd::Random(4,3);
+	MatrixXd m2 = MatrixXd::Zero(4,3);
+	cout << "Rows = " << m.rows() << ", cols = " << m.cols() << endl;
+	cout << "m = " << endl << m << endl;
+	cout << "max = " << endl << m.rowwise().maxCoeff() << endl;
+	cout << "m.colwise().sum() = " << endl << m.colwise().sum() << endl;
+
+	for (MatrixXf::Index i=0; i < m.rows(); i++)
+	{
+		MatrixXf::Index j = 8;
+		cout << "max = " << endl << m.row(i).maxCoeff(&j);
+		cout << "at row = " << i << ", col = " << j << endl;
+		m2(i, j) = m(i, j);
+	}
+	cout << "m2 = " << endl << m2 << endl;
 }
 
 RbmComponent::~RbmComponent()