- prepared for context units

- simplified status report

git-svn-id: http://moon:8086/svn/software/trunk/projects/Rbm@745 b431acfa-c32f-4a4a-93f1-934dc6c82436

source/Layer.cpp

@@ -97,7 +97,7 @@ bool Layer::loadWeights(const string &prjname)
 			result = fscanf(pFile, "%f", &v);
 			if (result > 0)
 			{
-				m_w(i, j) = v;
+				m_whv(i, j) = v;
 			}
 		}
 	}
@@ -140,7 +140,7 @@ bool Layer::saveWeights(const string &prjname)
 	{
 		for (j=0; j < numHidden; j++)
 		{
-			fprintf(pFile, "%3.6f ", m_w(i,j));
+			fprintf(pFile, "%3.6f ", m_whv(i,j));
 		}
 		fprintf(pFile, "\n");
 	}

source/MainComponent.cpp

@@ -903,6 +903,8 @@ void MainComponent::run()
 
 bool MainComponent::onProgress(Rbm *pRbm, const Rbm::Status &status)
 {
+	printf("[%3u] : Error: %f\n", status.progress, status.err);
+
 	RbmComponent *pComp = static_cast<RbmComponent*>(pRbm);
 	pComp->upPass(pComp->DrawTraining->getData());
 	pComp->redrawReconstruction();

source/Rbm.cpp

@@ -14,22 +14,30 @@
 #include <cassert>
 #include "Rbm.hpp"
 
-Rbm::Rbm(size_t numVisible, size_t numHidden)
+Rbm::Rbm(size_t numVisible, size_t numHidden, size_t numContext)
 : m_params()
-, m_w(numVisible, numHidden)
+, m_whv(numVisible, numHidden)
+, m_whc(numContext, numHidden)
 , m_bh(1, numHidden)
 , m_bv(1, numVisible)
+, m_bc(1, numContext)
 {
 	assert(numVisible > 0);
 	assert(numHidden > 0);
+	if (numContext)
+	{
+		assert(numContext == numHidden);
+	}
 	Noise_Init(&m_noise, 0x32727155);
 }
 
 Rbm::Rbm(const Rbm& orig)
 : m_params(orig.m_params)
-, m_w(orig.m_w)
+, m_whv(orig.m_whv)
+, m_whc(orig.m_whc)
 , m_bh(orig.m_bh)
 , m_bv(orig.m_bv)
+, m_bc(orig.m_bc)
 {
 }
 
@@ -40,9 +48,11 @@ Rbm::~Rbm()
 
 void Rbm::weightsInit(double stddev, double mu)
 {
-	uniform(m_w, stddev, mu);
+	uniform(m_whv, stddev, mu);
+	uniform(m_whc, stddev, mu);
 	uniform(m_bh, stddev, mu);
 	uniform(m_bv, stddev, mu);
+	uniform(m_bc, stddev, mu);
 }
 
 void Rbm::fromJson(Json::Value rbm)
@@ -69,11 +79,11 @@ void Rbm::train(const arma::mat& batch, IListener* pListener)
 	
 	arma::mat grad_bias_v(arma::zeros(1, m_bv.n_cols));
 	arma::mat grad_bias_h(arma::zeros(1, m_bh.n_cols));
-	arma::mat grad_weight(arma::zeros(m_w.n_rows, m_w.n_cols));
+	arma::mat grad_weight(arma::zeros(m_whv.n_rows, m_whv.n_cols));
-	arma::mat momentum_weights = arma::zeros(m_w.n_rows, m_w.n_cols);
+	arma::mat momentum_weights = arma::zeros(m_whv.n_rows, m_whv.n_cols);
 	arma::mat momentum_bias_v(arma::zeros(1, m_bv.n_cols));
 	arma::mat momentum_bias_h(arma::zeros(1, m_bh.n_cols));
-	arma::mat penalty_weights = arma::zeros(m_w.n_rows, m_w.n_cols);
+	arma::mat penalty_weights = arma::zeros(m_whv.n_rows, m_whv.n_cols);
 	
 	int trainingSizeRemain = batch.n_rows;
 
@@ -107,19 +117,6 @@ void Rbm::train(const arma::mat& batch, IListener* pListener)
 		for (int epoch=0; epoch < m_params.numEpochs; epoch++)
 		{
 
-			if (status.progress != lastProgress)
-			{
-				lastProgress = status.progress;
-				if (pListener)
-				{
-					if(!pListener->onProgress(this, status))
-					{
-						shouldAbort = true;
-						break;
-					}
-				}
-			}
-
 			hid_probs = probsLogistic(toHiddenState(vis_state));
 
 			// Sample hidden
@@ -167,26 +164,40 @@ void Rbm::train(const arma::mat& batch, IListener* pListener)
 			grad_bias_v -= sum(vis_probs, 0);
 			grad_bias_h -= sum(hid_probs, 0);
 
-			penalty_weights = weight_decay*arma::sign(m_w);
+			penalty_weights = weight_decay*arma::sign(m_whv);
 			
-			status.L1 = accu(abs(m_w));
+			status.L1 = accu(abs(m_whv));
-			status.L2 = accu(m_w % m_w);
+			status.L2 = accu(m_whv % m_whv);
 			momentum_bias_v = m_params.momentum*momentum_bias_v + grad_bias_v;
 			momentum_bias_h = m_params.momentum*momentum_bias_h + grad_bias_h;
 			momentum_weights = m_params.momentum*momentum_weights + grad_weight - status.L2*penalty_weights;
 			
 			m_bv += learning_rate*momentum_bias_v;
 			m_bh += learning_rate*momentum_bias_h;
-			m_w += learning_rate*momentum_weights;
+			m_whv += learning_rate*momentum_weights;
 			
 			progress += dProgress*miniBatchSizeActual;
 			status.progress = (int)(progress + 0.5);
 
-		} // Number of epochs
+			if (status.progress != lastProgress)
+			{
+				lastProgress = status.progress;
 				
+				// Calculate error
 				arma::mat diffErr = miniBatch - vis_probs;
 				arma::mat diffErr_squared = diffErr % diffErr;
 				status.err = accu(diffErr_squared)/diffErr_squared.n_elem;
+				if (pListener)
+				{
+					if(!pListener->onProgress(this, status))
+					{
+						shouldAbort = true;
+						break;
+					}
+				}
+			}
+			
+		} // Number of epochs
 			
 	} // number of mini batches
 
@@ -224,12 +235,12 @@ arma::mat Rbm::sample(const arma::mat &src)
 
 arma::mat Rbm::toHiddenState(const arma::mat &visible) const
 {
-	return visible * m_w  + arma::repmat(m_bh, visible.n_rows, 1);
+	return visible * m_whv  + arma::repmat(m_bh, visible.n_rows, 1);
 }
 
 arma::mat Rbm::toVisibleState(const arma::mat &hidden) const
 {
-	return hidden * m_w.t() + arma::repmat(m_bv, hidden.n_rows, 1);
+	return hidden * m_whv.t() + arma::repmat(m_bv, hidden.n_rows, 1);
 }
 
 arma::mat Rbm::toHiddenProbs(const arma::mat &visible) const
@@ -271,7 +282,7 @@ void Rbm::uniform(arma::mat& srcDst, double stdDev, double mu)
 
 const arma::mat& Rbm::w() const
 {
-	return m_w;
+	return m_whv;
 }
 
 const arma::mat& Rbm::bv() const

source/Rbm.hpp

@@ -114,7 +114,7 @@ public:
 		}
 	};
 
-	Rbm(size_t numVisible, size_t numHidden);
+	Rbm(size_t numVisible, size_t numHidden, size_t numContext=0);
 	Rbm(const Rbm& orig);
 	virtual ~Rbm();
 
@@ -144,9 +144,11 @@ private:
 	void uniform(arma::mat &srcDst, double stdDev=1.0, double mu=0.5);
 
 protected:
-	arma::mat m_w;
+	arma::mat m_whv;
+	arma::mat m_whc;
 	arma::mat m_bh;
 	arma::mat m_bv;
+	arma::mat m_bc;
 
 private:
 	noise_gen_t m_noise;