- load training, save weights

- provide params at construction time

git-svn-id: http://moon:8086/svn/software/trunk/projects/Rbm@566 b431acfa-c32f-4a4a-93f1-934dc6c82436

source/Rbm.cpp

@@ -14,19 +14,21 @@
 #include "Rbm.hpp"
 #include "noise.h"
 
-Rbm::Rbm(arma::mat &w, arma::mat &bv, arma::mat &bh)
+Rbm::Rbm(const Params& params, arma::mat &w, arma::mat &bv, arma::mat &bh)
-: m_w(w)
+: m_params(params)
+, m_w(w)
 , m_bv(bv)
 , m_bh(bh)
 {
 	Noise_Init(&m_noise, 0x32727155);
-	uniform(m_w);
+	uniform(m_w, 0.0, 0.1);
-	uniform(m_bh);
+	uniform(m_bh, 0.0, 0.1);
-	uniform(m_bv);
+	uniform(m_bv, 0.0, 0.1);
 }
 
 Rbm::Rbm(const Rbm& orig)
-: m_w(orig.m_w)
+: m_params(orig.m_params)
+, m_w(orig.m_w)
 , m_bv(orig.m_bv)
 , m_bh(orig.m_bh)
 {
@@ -36,7 +38,7 @@ Rbm::~Rbm()
 {
 }
 
-void Rbm::train(const arma::mat& batch, size_t numEpochs, size_t miniBatchSize, const Params& params, IListener* pListener)
+void Rbm::train(const arma::mat& batch, size_t numEpochs, size_t miniBatchSize, IListener* pListener)
 {
 	size_t epoch;
 	size_t gibbs;
@@ -66,8 +68,8 @@ void Rbm::train(const arma::mat& batch, size_t numEpochs, size_t miniBatchSize,
 		trainingSizeRemain -= toSlice;
 		batchRowIndex += toSlice;
 		size_t miniBatchSizeActual = miniBatch.n_rows;
-		double learning_rate = params.m_learningRate/std::min(miniBatchSizeActual, trainingSize);
+		double learning_rate = m_params.m_learningRate/std::min(miniBatchSizeActual, trainingSize);
-		double weight_decay = params.m_weightDecay/std::min(miniBatchSizeActual, trainingSize);
+		double weight_decay = m_params.m_weightDecay/std::min(miniBatchSizeActual, trainingSize);
 
 		arma::mat vis_state(miniBatchSizeActual, m_w.n_rows);
 		arma::mat vis_probs(miniBatchSizeActual, m_w.n_rows);
@@ -78,7 +80,7 @@ void Rbm::train(const arma::mat& batch, size_t numEpochs, size_t miniBatchSize,
 		{
 
 			// Create hidden layer base on training data
-			if (params.m_doSampleBatch)
+			if (m_params.m_doSampleBatch)
 			{
 				// When the hidden units are being driven by data, always use stochastic binary states
 				vis_state = sample(miniBatch);
@@ -92,7 +94,7 @@ void Rbm::train(const arma::mat& batch, size_t numEpochs, size_t miniBatchSize,
 			hid_probs = probsLogistic(hid_state);
 
 			// Sample hidden
-			if (params.m_doRaoBlackwell)
+			if (m_params.m_doRaoBlackwell)
 			{
 				hid_state = hid_probs;
 			}
@@ -106,7 +108,7 @@ void Rbm::train(const arma::mat& batch, size_t numEpochs, size_t miniBatchSize,
 			grad_bias_v = sum(vis_state, 0);
 			grad_bias_h = sum(hid_state, 0);
 
-			for (gibbs=0; gibbs < params.m_numGibbs; gibbs++)
+			for (gibbs=0; gibbs < m_params.m_numGibbs; gibbs++)
 			{
 
 				// Create hidden representation given v
@@ -115,7 +117,7 @@ void Rbm::train(const arma::mat& batch, size_t numEpochs, size_t miniBatchSize,
 				// Create visible reconstruction (a fantasy...) given hid
 				vis_state = hid_state * m_w.t() + arma::repmat(m_bv, miniBatchSizeActual, 1);
 				vis_probs = probsLogistic(vis_state);
-				if (params.m_doSampleVisible)
+				if (m_params.m_doSampleVisible)
 				{
 					vis_state = sample(vis_probs);
 				}
@@ -139,9 +141,9 @@ void Rbm::train(const arma::mat& batch, size_t numEpochs, size_t miniBatchSize,
 			
 			status.L1 = accu(abs(m_w));
 			status.L2 = accu(m_w % m_w);
-			momentum_bias_v = params.m_momentum*momentum_bias_v + grad_bias_v;
+			momentum_bias_v = m_params.m_momentum*momentum_bias_v + grad_bias_v;
-			momentum_bias_h = params.m_momentum*momentum_bias_h + grad_bias_h;
+			momentum_bias_h = m_params.m_momentum*momentum_bias_h + grad_bias_h;
-			momentum_weights = params.m_momentum*momentum_weights + grad_weight - status.L2*penalty_weights;
+			momentum_weights = m_params.m_momentum*momentum_weights + grad_weight - status.L2*penalty_weights;
 			
 			m_bv += learning_rate*momentum_bias_v;
 			m_bh += learning_rate*momentum_bias_h;
@@ -154,7 +156,7 @@ void Rbm::train(const arma::mat& batch, size_t numEpochs, size_t miniBatchSize,
 		status.epoch = epoch;
 		arma::mat diffErr = miniBatch - vis_probs;
 		arma::mat diffErr_squared = diffErr % diffErr;
-		status.err = accu(diffErr_squared);
+		status.err = accu(diffErr_squared)/diffErr_squared.n_elem;
 
 		if (pListener)
 		{

source/Rbm.hpp

@@ -24,7 +24,7 @@ public:
 	struct Params
 	{
 		Params()
-		: m_weightDecay(0.01)
+		: m_weightDecay(0.0)
 		, m_learningRate(0.1)
 		, m_momentum(0.5)
 		, m_doRaoBlackwell(true)
@@ -65,11 +65,11 @@ public:
 		}
 	};
 
-	Rbm(arma::mat &w, arma::mat &bv, arma::mat &bh);
+	Rbm(const Params& params, arma::mat &w, arma::mat &bv, arma::mat &bh);
 	Rbm(const Rbm& orig);
 	virtual ~Rbm();
 
-	void train(arma::mat const &batch, size_t numEpochs, size_t sizeMiniBatch, Params const &params, IListener *pListener);
+	void train(arma::mat const &batch, size_t numEpochs, size_t sizeMiniBatch, IListener *pListener);
 	arma::mat toHidden(const arma::mat &v);
 	arma::mat toVisible(const arma::mat &h);
 	arma::mat& weights();
@@ -78,6 +78,7 @@ public:
 	
 private:
 	noise_gen_t m_noise;
+	const Params &m_params;
 	arma::mat &m_w;
 	arma::mat &m_bh;
 	arma::mat &m_bv;

source/main.cpp

@@ -23,45 +23,105 @@ class RbmListener : public Rbm::IListener
 	}
 };
 
+arma::mat loadTraining(const char *pFilename)
+{
+	uint32_t numTraining = 0;
+	uint32_t numVisible = 0;
+	FILE *pFile;
+
+	pFile = fopen(pFilename,"r");
+
+	if (!pFile)
+	{
+		return 0;
+	}
+
+	int result = fscanf(pFile, "%d\n", &numTraining);
+	if (result < 0)
+	{
+		return 0;
+	}
+	result = fscanf(pFile, "%d\n", &numVisible);
+	if (result < 0)
+	{
+		return 0;
+	}
+	arma::mat data = arma::zeros(numTraining, numVisible);
+
+	uint32_t i, j;
+	for (i=0; i < numTraining; i++)
+	{
+		for (j=0; j < numVisible; j++)
+		{
+			float v;
+			int result = fscanf(pFile, "%f", &v);
+			if (result > 0)
+			{
+				data(i, j) = v;
+			}
+		}
+	}
+	fclose(pFile);
+	return data;
+	
+}
+
+void saveWeight(const char *pFilename, size_t numVisibleX, size_t numVisibleY, const arma::mat &w, const arma::mat &bv, const arma::mat &bh)
+{
+	FILE *pFile;
+
+	pFile = fopen(pFilename,"w");
+
+	if (!pFile)
+		return;
+
+	size_t numHidden = bh.n_elem;
+	size_t numVisible = bv.n_elem;
+	fprintf(pFile, "%d %d %d\n", (int)numVisibleX, (int)numVisibleY, (int)numHidden);
+
+	uint32_t i, j;
+
+	for (i=0; i < numVisible; i++)
+	{
+		fprintf(pFile, "%3.6f\n", bv(i));
+	}
+	for (i=0; i < numHidden; i++)
+	{
+		fprintf(pFile, "%3.6f\n", bh(i));
+	}
+	for (i=0; i < numVisible; i++)
+	{
+		for (j=0; j < numHidden; j++)
+		{
+			fprintf(pFile, "%3.6f ", w(i,j));
+		}
+		fprintf(pFile, "\n");
+	}
+
+	fclose(pFile);
+}
+
 int main()
 {
 	printf("Hallo, Welt!\n");
 
-	// Position of a particle                     //    |
-	arma::vec Pos = {{0},                         //    | (0,1)
-				   {1}};                        //    +---x-->
-
-	// Rotation matrix 
-	double phi = -3.1416/2; 
-	arma::mat RotM = {{+cos(phi), -sin(phi)},
-					{+sin(phi), +cos(phi)}};
-
-	Pos.print("Current position of the particle:");
-	std::cout << "Rotating the point " << phi*180/3.1416 << " deg" << std::endl;
-
-	Pos = RotM*Pos;
-
-	Pos.print("New position of the particle:");   //    ^
-												//    x (1,0)
-												//    | 
-	arma::mat Z = arma::eye<arma::mat>(4000,4000);
-//	Z.print("Z:");
-	printf("Z.n_rows = %d\n", (int)Z.n_rows);
-	printf("Z.n_cols = %d\n", (int)Z.n_cols);
-	printf("Z.n_elem = %d\n", (int)Z.n_elem);
-	//    +------>
-
-	
 	RbmListener statusDisplay;
 	Rbm::Params params;
-	arma::mat w = arma::zeros(28*28, 64);
+	arma::mat batch = loadTraining("mnist_2.training.dat");
-	arma::mat bv = arma::zeros(1, 28*28);
-	arma::mat bh = arma::zeros(1, 64);
-	Rbm rbm(w, bv, bh);
-	arma::mat batch = arma::randu(100, 28*28);
-	rbm.train(batch, 10, 10, params, &statusDisplay);
 
-	arma::mat v = arma::randu(100, 28*28);
+	size_t numTraining = batch.n_rows;
+	size_t numVisible = batch.n_cols;
+	size_t numHidden = 64;
+
+	printf("Loaded %d training samples\n", (int)numTraining);
+
+	arma::mat w = arma::zeros(numVisible, numHidden);
+	arma::mat bv = arma::zeros(1, numVisible);
+	arma::mat bh = arma::zeros(1, numHidden);
+	Rbm rbm(params, w, bv, bh);
+	rbm.train(batch, 100, 100, &statusDisplay);
+	saveWeight("mnist_2.weights.dat", 28, 28, w, bv, bh);
+	arma::mat v = arma::randu(numTraining, numVisible);
 	arma::mat h = rbm.toHidden(v);
 	arma::mat r = rbm.toVisible(h);
 	return 0;