- use expectations

- improved gibbs sampling - LayerArray is template class git-svn-id: http://moon:8086/svn/software/trunk/projects/RBM@18 b431acfa-c32f-4a4a-93f1-934dc6c82436
2014-10-06 17:08:50 +00:00
parent f62f1283f8
commit fc758b853a
122 changed files with 1053 additions and 1149 deletions
@@ -16,13 +16,26 @@
 void mylog(const char* format, ...);
 #define printf mylog

+class Rbm;
+
+class RbmListener
+{
+public:
+	RbmListener() {}
+	virtual ~RbmListener() {}
+
+	virtual void onEpochTrained(const Rbm &obj) = 0;
+};
+
 class Rbm
 {
 public:
-	Rbm(Weights &weights)
+	Rbm(Weights &weights, RbmListener *pListener = nullptr)
 	: m_w(weights)
+	, m_pListener(pListener)
 	, m_tv(weights.getNumVisible())
 	, m_th(weights.getNumHidden())
+	, m_progress(0)
 	{
 		Noise_Init(&m_noise, 0x32727155);
 	}
@@ -32,95 +45,100 @@ public:
 		Noise_Free(&m_noise);
 	}

-	void weightsUpdate(VisibleLayer &v, VisibleLayer &vr, HiddenLayer &h, HiddenLayer &hr, double mu)
+	void weightsUpdate(VisibleLayer &v, HiddenLayer &h, double mu)
 	{
 		uint32_t i, j;
 		double dw;

+		double **ppW = m_w.getWeights();
+		const double *pH = h.getStates();
+		const double *pV = v.getStates();
+
 		// Update weights
 		for (i=0; i < m_w.getNumHidden(); i++)
 		{
-			dw = 0;
 			for (j=0; j < m_w.getNumVisible(); j++)
 			{
-				dw = v.getStates()[j] * h.getStates()[i];
-				m_w.getWeights()[i][j] += mu*dw;
+				dw = pV[j] * pH[i];
+				ppW[i][j] += mu*dw;
 			}
 		}
-
-		for (i=0; i < m_w.getNumHidden(); i++)
-		{
-			dw = 0;
-			for (j=0; j < m_w.getNumVisible(); j++)
-			{
-				dw = vr.getStates()[j] * hr.getStates()[i];
-				m_w.getWeights()[i][j] -= mu*dw;
-			}
-		}
-#if 1
+		double *pBias = m_w.getBiasVisible();
 		for (i=0; i < m_w.getNumVisible(); i++)
 		{
-			dw = v.getStates()[i] - vr.getStates()[i];
-			m_w.getBiasVisible()[i] += mu*dw;
+			dw = pV[i];
+			pBias[i] += mu*dw;
 		}

+		pBias = m_w.getBiasHidden();
 		for (i=0; i < m_w.getNumHidden(); i++)
 		{
-			dw = h.getStates()[i] - hr.getStates()[i];
-			m_w.getBiasHidden()[i] += mu*dw;
+			dw = pH[i];
+			pBias[i] += mu*dw;
 		}
-#endif
 	}

-	void train(VisibleLayerArray &vts, uint32_t numEpochs, double mu)
+	void train(LayerArray<VisibleLayer> &vts, uint32_t numEpochs, double mu, uint32_t numGibbs = 1, bool useExpectations=false)
 	{
+		uint32_t t;
 		uint32_t epoch;
-		uint32_t trainingPatternIndex;
-		VisibleLayer vr(m_w.getNumVisible());
+		uint32_t gibbs;
+		VisibleLayer v(m_w.getNumVisible());
 		HiddenLayer h(m_w.getNumHidden());
-		HiddenLayer hr(m_w.getNumHidden());
-		const uint32_t monitorInterval = 100; 		// epochs
-		uint32_t monitorCount = monitorInterval; 	// epochs
+		Weights w = m_w;
+
+		double dProgress = 1.0/numEpochs;
+		m_progress = 0;

 		for (epoch=0; epoch < numEpochs; epoch++)
 		{
-			trainingPatternIndex = (uint32_t)((vts.getSize())*Noise_Uniform(&m_noise, 0.5));
-
-			if (trainingPatternIndex == vts.getSize())
-				continue;
-
-			// Assign training data
-			VisibleLayer &vt = vts.getAt(trainingPatternIndex);
-
-			// Create hidden layer base on training data
-			h.probsUpdate(vt, m_w);
-//			h.statesAssignfromProbs();
-			h.statesUpdateStochastic();
-
-			// Create visible reconstruction (a fantasy...)
-			vr = vt;
-			vr.probsUpdate(h, m_w);
-//			vr.statesAssignfromProbs();
-			vr.statesUpdateStochastic();
-
-			// Create hidden reconstruction
-			hr.probsUpdate(vr, m_w);
-//			hr.statesAssignfromProbs();
-			hr.statesUpdateStochastic();
-
-			// Update weights
-			weightsUpdate(vt, vr, h, hr, mu);
-
-			if (!monitorCount)
+			for (t=0; t < vts.getSize(); t++)
 			{
-				monitorCount = monitorInterval;
-//				printf("Epoch #%d\n", epoch);
-//				prob();
+				// Create hidden layer base on training data
+				h.probsUpdate(vts[t], w);
+				h.statesUpdateStochastic();
+
+				// Update weights (positive phase)
+				weightsUpdate(vts[t], h, +mu/vts.getSize());
+
+				for (gibbs=0; gibbs < numGibbs; gibbs++)
+				{
+					// Create visible reconstruction (a fantasy...)
+					v.probsUpdate(h, w);
+					v.statesUpdateStochastic();
+
+					// Create hidden reconstruction
+					h.probsUpdate(v, w);
+					h.statesUpdateStochastic();
+				}
+				// Update weights (negative phase)
+				h.statesAssignfromProbs();
+				weightsUpdate(v, h, -mu/vts.getSize());
+
+				if (!useExpectations)
+				{
+					w = m_w;
+				}
+
+			}
+			if (useExpectations)
+			{
+				w = m_w;
+			}
+
+			m_progress += dProgress;
+			if (m_pListener)
+			{
+				m_pListener->onEpochTrained(*this);
 			}
-			monitorCount--;
 		}
 	}

+	double getProgress() const
+	{
+		return m_progress;
+	}
+
 	double getEnergy(VisibleLayer &v, HiddenLayer &h)
 	{
 		uint32_t i, j;
@@ -138,7 +156,7 @@ public:
 		return energy;
 	}

-	void prob(VisibleLayerArray &vts)
+	void prob(LayerArray<VisibleLayer> &vts)
 	{
 		uint32_t i, j;
 		double z;
@@ -264,9 +282,11 @@ public:

 private:
 	Weights &m_w;
+	RbmListener *m_pListener;
 	VisibleLayer m_tv;
 	HiddenLayer m_th;
 	noise_gen_t m_noise;
+	double m_progress;

 };