- train whv and whc separately

git-svn-id: http://moon:8086/svn/software/trunk/projects/Rbm@758 b431acfa-c32f-4a4a-93f1-934dc6c82436

source/Rbm.cpp

@@ -53,9 +53,9 @@ Rbm::~Rbm()
 void Rbm::weightsInit(double stddev, double mu)
 {
 	uniform(m_whv, stddev, mu);
-	uniform(m_whc, 0*stddev, mu);
+	uniform(m_whc, 0.01*stddev, mu);
 	uniform(m_bhv, stddev, mu);
-	uniform(m_bhc, 0*stddev, mu);
+	uniform(m_bhc, 0.01*stddev, mu);
 	uniform(m_bv, stddev, mu);
 	uniform(m_bc, stddev, mu);
 }
@@ -74,66 +74,59 @@ Json::Value Rbm::toJson() const
 	return rbm;
 }
 
-void Rbm::gibbs(arma::mat &h_probs, arma::mat &v_probs)
+void Rbm::gibbs_hv(arma::mat &hv_probs, arma::mat &v_probs)
 {
 	for (int gibbs=0; gibbs < m_params.numGibbs; gibbs++)
 	{
 		// Create visible reconstruction (a fantasy...) given hid
 		if (m_params.gibbsDoSampleHidden)
 		{
-			v_probs = prob(h_to_v(sample(h_probs)));
+			v_probs = prob(h_to_v(sample(hv_probs)));
 		}
 		else
 		{
-			v_probs = prob(h_to_v(h_probs));
+			v_probs = prob(h_to_v(hv_probs));
 		}
 
 		// Create hidden representation given v
 		if (m_params.gibbsDoSampleVisible)
 		{
-			h_probs = prob(v_to_h(sample(v_probs)));
+			hv_probs = prob(v_to_h(sample(v_probs)));
 		}
 		else
 		{
-			h_probs = prob(v_to_h(v_probs));
+			hv_probs = prob(v_to_h(v_probs));
 		}
 	}
 }
 
-void Rbm::weightUpdate(arma::mat const &v_states, arma::mat const &c_states, arma::mat &dwhv, arma::mat &dwhc, arma::mat &dbh, arma::mat &dbv, arma::mat &dbc)
+void Rbm::gibbs_hc(arma::mat &hc_probs, arma::mat &c_probs)
 {
-	arma::mat v_probs(v_states);
+	for (int gibbs=0; gibbs < m_params.numGibbs; gibbs++)
-	arma::mat hv_states = v_to_h(v_states);
-	arma::mat hc_states = c_to_h(c_states);
-	
-	arma::mat h_states = hv_states + hc_states;
-	arma::mat h_probs = prob(h_states);
-	
-	// Sample hidden
-	if (m_params.doRaoBlackwell)
 	{
-		h_states = h_probs;
+		// Create visible reconstruction (a fantasy...) given hid
+		if (m_params.gibbsDoSampleHidden)
+		{
+			c_probs = prob(h_to_c(sample(hc_probs)));
 		}
 		else
 		{
-		h_states = sample(h_probs);
+			c_probs = prob(h_to_c(hc_probs));
 		}
 
-	// Update weights (positive phase)
+		// Create hidden representation given v
-	dwhv = v_states.t() * h_states;
+		if (m_params.gibbsDoSampleVisible)
-	dbv = sum(v_states, 0);
+		{
-	dbh = sum(h_states, 0);
+			hc_probs = prob(c_to_h(sample(c_probs)));
-
+		}
-	gibbs(h_probs, v_probs);
+		else
-
+		{
-	// Update weights (negative phase)
+			hc_probs = prob(c_to_h(c_probs));
-	dwhv -= v_probs.t() * h_probs;
+		}
-	dbv -= sum(v_probs, 0);
+	}
-	dbh -= sum(h_probs, 0);
-	
 }
 
-void Rbm::weightUpdate(arma::mat const &v_states, arma::mat &dw, arma::mat &dbh, arma::mat &dbv)
+void Rbm::weightUpdate_hv(arma::mat const &v_states, arma::mat &dw, arma::mat &dbhv, arma::mat &dbv)
 {
 	arma::mat v_probs(v_states);
 	arma::mat h_states = v_to_h(v_states);
@@ -152,14 +145,43 @@ void Rbm::weightUpdate(arma::mat const &v_states, arma::mat &dw, arma::mat &dbh,
 	// Update weights (positive phase)
 	dw = v_states.t() * h_states;
 	dbv = sum(v_states, 0);
-	dbh = sum(h_states, 0);
+	dbhv = sum(h_states, 0);
 
-	gibbs(h_probs, v_probs);
+	gibbs_hv(h_probs, v_probs);
 
 	// Update weights (negative phase)
 	dw -= v_probs.t() * h_probs;
 	dbv -= sum(v_probs, 0);
-	dbh -= sum(h_probs, 0);
+	dbhv -= sum(h_probs, 0);
+}
+
+void Rbm::weightUpdate_hc(arma::mat const &c_states, arma::mat &dw, arma::mat &dbhc, arma::mat &dbc)
+{
+	arma::mat c_probs(c_states);
+	arma::mat h_states = c_to_h(c_states);
+	arma::mat h_probs = prob(h_states);
+	
+	// Sample hidden
+	if (m_params.doRaoBlackwell)
+	{
+		h_states = h_probs;
+	}
+	else
+	{
+		h_states = sample(h_probs);
+	}
+	
+	// Update weights (positive phase)
+	dw = c_states.t() * h_states;
+	dbc = sum(c_states, 0);
+	dbhc = sum(h_states, 0);
+
+	gibbs_hc(h_probs, c_probs);
+
+	// Update weights (negative phase)
+	dw -= c_probs.t() * h_probs;
+	dbc -= sum(c_probs, 0);
+	dbhc -= sum(h_probs, 0);
 }
 
 void Rbm::train(const arma::mat& batch, IListener* pListener)
@@ -171,13 +193,17 @@ void Rbm::train(const arma::mat& batch, IListener* pListener)
 	int batchRowIndex = 0;
 	
 	arma::mat grad_bias_v(arma::zeros(1, m_bv.n_cols));
-	arma::mat grad_bias_h(arma::zeros(1, m_bhv.n_cols));
 	arma::mat grad_bias_c(arma::zeros(1, m_bc.n_cols));
+	arma::mat grad_bias_hv(arma::zeros(1, m_bhv.n_cols));
+	arma::mat grad_bias_hc(arma::zeros(1, m_bhc.n_cols));
 	arma::mat grad_weight_hv(arma::zeros(m_whv.n_rows, m_whv.n_cols));
 	arma::mat grad_weight_hc(arma::zeros(m_whc.n_rows, m_whc.n_cols));
 	arma::mat momentum_whv = arma::zeros(m_whv.n_rows, m_whv.n_cols);
+	arma::mat momentum_whc = arma::zeros(m_whc.n_rows, m_whc.n_cols);
 	arma::mat momentum_bias_v(arma::zeros(1, m_bv.n_cols));
-	arma::mat momentum_bias_h(arma::zeros(1, m_bhv.n_cols));
+	arma::mat momentum_bias_c(arma::zeros(1, m_bc.n_cols));
+	arma::mat momentum_bias_hv(arma::zeros(1, m_bhv.n_cols));
+	arma::mat momentum_bias_hc(arma::zeros(1, m_bhc.n_cols));
 	arma::mat penalty_weights = arma::zeros(m_whv.n_rows, m_whv.n_cols);
 	
 	int trainingSizeRemain = batch.n_rows;
@@ -225,9 +251,9 @@ void Rbm::train(const arma::mat& batch, IListener* pListener)
 			// Update weights (positive phase)
 			grad_weight = v_states.t() * hid_states;
 			grad_bias_v = sum(v_states, 0);
-			grad_bias_h = sum(hid_states, 0);
+			grad_bias_hv = sum(hid_states, 0);
 
-			for (int gibbs=0; gibbs < m_params.numGibbs; gibbs++)
+			for (int gibbs_hv=0; gibbs_hv < m_params.numGibbs; gibbs_hv++)
 			{
 				// Create visible reconstruction (a fantasy...) given hid
 				if (m_params.gibbsDoSampleHidden)
@@ -255,15 +281,16 @@ void Rbm::train(const arma::mat& batch, IListener* pListener)
 			// Update weights (negative phase)
 			grad_weight -= v_probs.t() * h_probs;
 			grad_bias_v -= sum(v_probs, 0);
-			grad_bias_h -= sum(h_probs, 0);
+			grad_bias_hv -= sum(h_probs, 0);
 #else
 			if (m_bc.n_cols == 0)
 			{
-				weightUpdate(v_states, grad_weight_hv, grad_bias_h, grad_bias_v);
+				weightUpdate_hv(v_states, grad_weight_hv, grad_bias_hv, grad_bias_v);
 			}
 			else
 			{
-				weightUpdate(v_states, c_states, grad_weight_hv, grad_weight_hc, grad_bias_h, grad_bias_v, grad_bias_c);
+				weightUpdate_hv(v_states, grad_weight_hv, grad_bias_hv, grad_bias_v);
+				weightUpdate_hc(c_states, grad_weight_hc, grad_bias_hc, grad_bias_c);
 			}
 #endif
 			
@@ -272,12 +299,18 @@ void Rbm::train(const arma::mat& batch, IListener* pListener)
 			status.L1 = accu(abs(m_whv));
 			status.L2 = accu(m_whv % m_whv);
 			momentum_bias_v = m_params.momentum*momentum_bias_v + grad_bias_v;
-			momentum_bias_h = m_params.momentum*momentum_bias_h + grad_bias_h;
+			momentum_bias_hv = m_params.momentum*momentum_bias_hv + grad_bias_hv;
+			momentum_bias_c = m_params.momentum*momentum_bias_c + grad_bias_c;
+			momentum_bias_hc = m_params.momentum*momentum_bias_hc + grad_bias_hc;
 			momentum_whv = m_params.momentum*momentum_whv + grad_weight_hv - status.L2*penalty_weights;
+			momentum_whc = m_params.momentum*momentum_whc + grad_weight_hc;
 			
 			m_bv += learning_rate*momentum_bias_v;
-			m_bhv += learning_rate*momentum_bias_h;
+			m_bc += learning_rate*momentum_bias_c;
+			m_bhv += learning_rate*momentum_bias_hv;
+			m_bhc += learning_rate*momentum_bias_hc;
 			m_whv += learning_rate*momentum_whv;
+			m_whc += learning_rate*momentum_whc;
 			
 			progress += dProgress*miniBatchSizeActual;
 			status.progress = (int)(progress + 0.5);

source/Rbm.hpp

@@ -142,10 +142,11 @@ public:
 private:
 	Params m_params;
 	arma::mat sample(arma::mat const &src);
-	void weightUpdate(arma::mat const &v_states, arma::mat &dw, arma::mat &dbh, arma::mat &dbv);
+	void weightUpdate_hv(arma::mat const &v_states, arma::mat &dwhv, arma::mat &dbhv, arma::mat &dbv);
-	void weightUpdate(arma::mat const &v_states, arma::mat const &c_states, arma::mat &dwhv, arma::mat &dwhc, arma::mat &dbh, arma::mat &dbv, arma::mat &dbc);
+	void weightUpdate_hc(arma::mat const &c_states, arma::mat &dwhc, arma::mat &dbhc, arma::mat &dbc);
 	void uniform(arma::mat &srcDst, double stdDev=1.0, double mu=0.5);
-	void gibbs(arma::mat &h_states, arma::mat &v_states);
+	void gibbs_hv(arma::mat &hv_states, arma::mat &v_states);
+	void gibbs_hc(arma::mat &hc_states, arma::mat &c_states);
 	
 protected:
 	arma::mat m_whv;