- use custom noise for bugfixing

- init weights with zero mean uniform noise

git-svn-id: http://moon:8086/svn/software/trunk/projects/Rbm@598 b431acfa-c32f-4a4a-93f1-934dc6c82436

source/Rbm.cpp

@@ -13,14 +13,13 @@
 
 #include "Rbm.hpp"
 
-Rbm::Rbm(size_t numVisible, size_t numHidden, double weightInit)
+Rbm::Rbm(size_t numVisible, size_t numHidden)
 : m_params()
-, m_weightInit(weightInit)
 , m_w(numVisible, numHidden)
 , m_bv(1, numVisible)
 , m_bh(1, numHidden)
 {
-	weightsInit(0.0, m_weightInit);
+	Noise_Init(&m_noise, 0x32727155);
 }
 
 Rbm::Rbm(const Rbm& orig)
@@ -33,13 +32,14 @@ Rbm::Rbm(const Rbm& orig)
 
 Rbm::~Rbm()
 {
+	Noise_Free(&m_noise);
 }
 
-void Rbm::weightsInit(double mu, double stddev)
+void Rbm::weightsInit(double stddev, double mu)
 {
-	uniform(m_w, mu, stddev);
+	uniform(m_w, stddev, mu);
-	uniform(m_bh, mu, stddev);
+	uniform(m_bv, stddev, mu);
-	uniform(m_bv, mu, stddev);
+	uniform(m_bh, stddev, mu);
 
 }
 
@@ -53,7 +53,6 @@ Json::Value Rbm::toJson() const
 	Json::Value rbm;
 	rbm["numVisible"] = m_bv.n_elem;
 	rbm["numHidden"] = m_bh.n_elem;
-	rbm["weightInit"] = m_weightInit;
 	rbm["params"] = m_params.toJson();
 	return rbm;
 }
@@ -239,9 +238,19 @@ arma::mat Rbm::toVisibleProbs(const arma::mat &hidden) const
 	return probsLogistic(toVisibleState(hidden));
 }
 
-void Rbm::uniform(arma::mat& srcDst, double mu, double stdDev)
+void Rbm::uniform(arma::mat& srcDst, double stdDev, double mu)
 {
+#if 1
+	for (size_t i=0; i < srcDst.n_rows; i++)
+	{
+		for (size_t j=0; j < srcDst.n_cols; j++)
+		{
+			srcDst(i, j) = stdDev*(Noise_Uniform(&m_noise) + mu - 0.5);
+		}
+	}
+#else
 	srcDst = stdDev*arma::randu(srcDst.n_rows, srcDst.n_cols) + mu;
+#endif
 }
 
 const arma::mat& Rbm::w() const

source/Rbm.hpp

@@ -18,6 +18,7 @@
 
 #include <armadillo>
 #include <jsoncpp/json/json.h>
+#include "noise.h"
 
 class Rbm
 {
@@ -108,11 +109,11 @@ public:
 		}
 	};
 
-	Rbm(size_t numVisible, size_t numHidden, double weightInit=0.01);
+	Rbm(size_t numVisible, size_t numHidden);
 	Rbm(const Rbm& orig);
 	virtual ~Rbm();
 
-	void weightsInit(double mu, double stddev);
+	void weightsInit(double stddev, double mu=0.0);
 	void train(arma::mat const &batch, size_t miniBatchSize, size_t numEpochs, IListener *pListener);
 	
 	arma::mat toHiddenState(const arma::mat &visible) const;
@@ -128,16 +129,17 @@ public:
 	
 private:
 	const Params m_params;
-	double m_weightInit;
 	arma::mat sample(arma::mat const &src);
 	static arma::mat probsLogistic(arma::mat const &src);
-	void uniform(arma::mat &srcDst, double mu=0.0, double stdDev=1.0);
+	void uniform(arma::mat &srcDst, double stdDev=1.0, double mu=0.5);
 
 protected:
 	arma::mat m_w;
 	arma::mat m_bh;
 	arma::mat m_bv;
 
+private:
+	noise_gen_t m_noise;
 };
 
 #endif /* RBM_HPP */

source/Stack.cpp

@@ -120,7 +120,7 @@ void Stack::weightsInit(double stddev)
 	Layer *pLayer = m_pLayers;
 	while(pLayer)
 	{
-		pLayer->weightsInit(0, stddev);
+		pLayer->weightsInit(stddev);
 		pLayer = pLayer->upper;
 	}
 }

source/noise.c

@@ -0,0 +1,80 @@
+// --------------------------------------------------------------
+// --------------------------------------------------------------
+
+#include <string.h>
+#include <stdlib.h>
+#include <time.h>
+#include <math.h>
+
+#include "noise.h"
+
+// --------------------------------------------------------------
+// internal funcs
+// --------------------------------------------------------------
+#define PM_IA 16807
+#define PM_IM 2147483647
+#define PM_AM (1.0/PM_IM)
+#define PM_IQ 127773
+#define PM_IR 2836
+#define PM_MASK 123459876
+
+//	'Minimal' random number generator of Park and Miller. Returns a uniform random deviate
+//	between 0.0 and 1.0. Set or reset idum to any integer value (except the unlikely value MASK)
+//	to initialize the sequence; idum must not be altered between calls for successive deviates in
+//	a sequence.
+double ran0(long *idum)
+{
+	long k;
+	double ans;
+
+	*idum ^= PM_MASK;				// XORing with MASK allows use of zero and other
+	k=(*idum)/PM_IQ;				// simple bit patterns for idum.
+	*idum=PM_IA*(*idum-k*PM_IQ)-PM_IR *k;		// Compute idum=(IA*idum) % IM without over-
+							// flows by Schrage’s method.
+	if (*idum < 0)
+		*idum += PM_IM;
+
+	ans=PM_AM*(*idum);				// Convert idumto a floating result.
+	*idum ^= PM_MASK;				// Unmask before return.
+
+	return ans;
+}
+
+// --------------------------------------------------------------
+// Exported functions
+// --------------------------------------------------------------
+void Noise_Init(noise_gen_t *pObj, long seed)
+{
+
+	pObj->state = seed;
+}
+
+void Noise_Free(noise_gen_t *pObj)
+{
+}
+
+// 0 .. 1
+double Noise_Uniform(noise_gen_t *pObj)
+{
+	return ran0(&pObj->state);
+}
+
+double Noise_Gaussian(noise_gen_t *pObj)
+{
+	double U1, U2, V1, V2, S, Y;
+
+	do
+	{
+		U1 = Noise_Uniform(pObj);		         // U1=[0,1]
+		U2 = Noise_Uniform(pObj);                // U2=[0,1]
+		V1 = 2 * U1 - 1;          // V1=[-1,1]
+		V2 = 2 * U2 - 1;          // V2=[-1,1]
+		S = V1 * V1 + V2 * V2;
+
+	} while (S >= 1);
+
+	// X = sqrt(-2 * log(S) / S) * V1;
+	Y = sqrt(-2 * log(S) / S) * V2;
+
+	return Y;
+}

source/noise.h

@@ -0,0 +1,42 @@
+/*
+  ==============================================================================
+
+    noise.h
+    Created: 21 Sep 2014 1:55:07pm
+    Author:  jens
+
+  ==============================================================================
+*/
+// --------------------------------------------------------------
+#ifndef _NOISE_H_
+#define	_NOISE_H_
+
+#define __func__ ""
+// --------------------------------------------------------------
+// Types
+// --------------------------------------------------------------
+typedef struct _snoise_gen_t
+{
+	long state;
+} noise_gen_t;
+
+// --------------------------------------------------------------
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+// --------------------------------------------------------------
+// Exported functions
+// --------------------------------------------------------------
+void Noise_Init(noise_gen_t *pObj, long seed);
+void Noise_Free(noise_gen_t *pObj);
+double Noise_Uniform(noise_gen_t *pObj);
+double Noise_Gaussian(noise_gen_t *pObj);
+
+#if defined(__cplusplus)
+}
+
+#endif
+// --------------------------------------------------------------
+
+#endif // _NOISE_H_