- improved RBM

git-svn-id: http://moon:8086/svn/software/trunk/projects/Rbm@567 b431acfa-c32f-4a4a-93f1-934dc6c82436

source/Rbm.cpp

@@ -21,9 +21,9 @@ Rbm::Rbm(const Params& params, arma::mat &w, arma::mat &bv, arma::mat &bh)
 , m_bh(bh)
 {
 	Noise_Init(&m_noise, 0x32727155);
-	uniform(m_w, 0.0, 0.1);
+	uniform(m_w, 0.0, m_params.m_weightInit);
-	uniform(m_bh, 0.0, 0.1);
+	uniform(m_bh, 0.0, m_params.m_weightInit);
-	uniform(m_bv, 0.0, 0.1);
+	uniform(m_bv, 0.0, m_params.m_weightInit);
 }
 
 Rbm::Rbm(const Rbm& orig)
@@ -90,7 +90,7 @@ void Rbm::train(const arma::mat& batch, size_t numEpochs, size_t miniBatchSize,
 				vis_state = miniBatch;
 			}
 			
-			hid_state = vis_state * m_w + arma::repmat(m_bh, miniBatchSizeActual, 1);
+			hid_state = toHiddenState(vis_state);
 			hid_probs = probsLogistic(hid_state);
 
 			// Sample hidden
@@ -110,32 +110,33 @@ void Rbm::train(const arma::mat& batch, size_t numEpochs, size_t miniBatchSize,
 
 			for (gibbs=0; gibbs < m_params.m_numGibbs; gibbs++)
 			{
-
-				// Create hidden representation given v
-				hid_state = sample(hid_probs);
-
 				// Create visible reconstruction (a fantasy...) given hid
-				vis_state = hid_state * m_w.t() + arma::repmat(m_bv, miniBatchSizeActual, 1);
+				if (m_params.m_gibbsDoSampleHidden)
-				vis_probs = probsLogistic(vis_state);
-				if (m_params.m_doSampleVisible)
 				{
-					vis_state = sample(vis_probs);
+					vis_probs = toVisibleProbs(sample(hid_probs));
 				}
 				else
 				{
-					vis_state = vis_probs;
+					vis_probs = toVisibleProbs(hid_probs);
 				}
 
 				// Create hidden representation given v
-				hid_state = vis_state * m_w + repmat(m_bh, miniBatchSizeActual, 1);
+				if (m_params.m_gibbsDoSampleVisible)
+				{
+					hid_state = toHiddenState(sample(vis_probs));
+				}
+				else
+				{
+					hid_state = toHiddenState(vis_probs);
+				}
 				hid_probs = probsLogistic(hid_state);
 			
 			}
 
 			// Update weights (negative phase)
+			grad_weight -= vis_probs.t() * hid_probs;
 			grad_bias_v -= sum(vis_probs, 0);
 			grad_bias_h -= sum(hid_probs, 0);
-			grad_weight -= vis_probs.t() * hid_probs;
 
 			penalty_weights = weight_decay*arma::sign(m_w);
 			
@@ -157,7 +158,8 @@ void Rbm::train(const arma::mat& batch, size_t numEpochs, size_t miniBatchSize,
 		arma::mat diffErr = miniBatch - vis_probs;
 		arma::mat diffErr_squared = diffErr % diffErr;
 		status.err = accu(diffErr_squared)/diffErr_squared.n_elem;
-
+		status.err_total = 0;
+		
 		if (pListener)
 		{
 			if(!pListener->onProgress(status))
@@ -167,6 +169,17 @@ void Rbm::train(const arma::mat& batch, size_t numEpochs, size_t miniBatchSize,
 		}
 		
 	} // number of mini batches
+
+	arma::mat hid_probs = toHiddenProbs(batch);
+	arma::mat vis_probs = toVisibleProbs(hid_probs);
+	arma::mat diffErr = batch - vis_probs;
+	arma::mat diffErr_squared = diffErr % diffErr;
+	status.err_total = accu(diffErr_squared)/diffErr_squared.n_elem;
+
+	if (pListener)
+	{
+		pListener->onProgress(status);
+	}
 }
 
 arma::mat Rbm::probsLogistic(const arma::mat &src)
@@ -187,16 +200,24 @@ arma::mat Rbm::sample(const arma::mat &src)
 	return dst;
 }
 
-arma::mat Rbm::toHidden(const arma::mat &visible)
+arma::mat Rbm::toHiddenState(const arma::mat &visible)
 {
-	arma::mat h = visible * m_w  + arma::repmat(m_bh, visible.n_rows, 1);
+	return visible * m_w  + arma::repmat(m_bh, visible.n_rows, 1);
-	return probsLogistic(h);
 }
 
-arma::mat Rbm::toVisible(const arma::mat &hidden)
+arma::mat Rbm::toVisibleState(const arma::mat &hidden)
 {
-	arma::mat v = hidden * m_w.t() + arma::repmat(m_bv, hidden.n_rows, 1);
+	return hidden * m_w.t() + arma::repmat(m_bv, hidden.n_rows, 1);
-	return probsLogistic(v);
+}
+
+arma::mat Rbm::toHiddenProbs(const arma::mat &visible)
+{
+	return probsLogistic(toHiddenState(visible));
+}
+
+arma::mat Rbm::toVisibleProbs(const arma::mat &hidden)
+{
+	return probsLogistic(toVisibleState(hidden));
 }
 
 void Rbm::uniform(arma::mat& srcDst, double mu, double stdDev)

source/Rbm.hpp

@@ -24,21 +24,25 @@ public:
 	struct Params
 	{
 		Params()
-		: m_weightDecay(0.0)
+		: m_weightInit(0.01)
+		, m_weightDecay(0.0)
 		, m_learningRate(0.1)
 		, m_momentum(0.5)
 		, m_doRaoBlackwell(true)
-		, m_doSampleVisible(false)
+		, m_gibbsDoSampleVisible(false)
+		, m_gibbsDoSampleHidden(false)
 		, m_doSampleBatch(false)
-		, m_numGibbs(1)
+		, m_numGibbs(5)
 		{
 		}
 
+		double m_weightInit;
 		double m_weightDecay;
 		double m_learningRate;
 		double m_momentum;
 		bool m_doRaoBlackwell;
-		bool m_doSampleVisible;
+		bool m_gibbsDoSampleVisible;
+		bool m_gibbsDoSampleHidden;
 		bool m_doSampleBatch;
 		size_t m_numGibbs;
 	};
@@ -49,6 +53,7 @@ public:
 		size_t trainingSizeRemain;
 		double progress;
 		double err;
+		double err_total;
 		double L1;
 		double L2;
 	};
@@ -70,8 +75,11 @@ public:
 	virtual ~Rbm();
 
 	void train(arma::mat const &batch, size_t numEpochs, size_t sizeMiniBatch, IListener *pListener);
-	arma::mat toHidden(const arma::mat &v);
+	
-	arma::mat toVisible(const arma::mat &h);
+	arma::mat toHiddenState(const arma::mat &visible);
+	arma::mat toVisibleState(const arma::mat &hidden);
+	arma::mat toHiddenProbs(const arma::mat &visible);
+	arma::mat toVisibleProbs(const arma::mat &hidden);
 	arma::mat& weights();
 	arma::mat& bias_visible();
 	arma::mat& bias_hidden();

source/main.cpp

@@ -16,6 +16,7 @@ class RbmListener : public Rbm::IListener
 		std::cout << "epoch :             " << status.epoch << std::endl;
 		std::cout << "trainingSizeRemain: " << status.trainingSizeRemain << std::endl;
 		std::cout << "error (per mini batch) = " << status.err << std::endl;
+		std::cout << "error (total)          = " << status.err_total << std::endl;
 		std::cout << "L1                     = " << status.L1 << std::endl;
 		std::cout << "L2                     = " << status.L2 << std::endl;
 		
@@ -122,7 +123,7 @@ int main()
 	rbm.train(batch, 100, 100, &statusDisplay);
 	saveWeight("mnist_2.weights.dat", 28, 28, w, bv, bh);
 	arma::mat v = arma::randu(numTraining, numVisible);
-	arma::mat h = rbm.toHidden(v);
+	arma::mat h = rbm.toHiddenProbs(v);
-	arma::mat r = rbm.toVisible(h);
+	arma::mat r = rbm.toVisibleProbs(h);
 	return 0;
 }