- bugfix: weightDecay needs to be multiplied by mu_weights

- Biases are initialized with zero
- developing full matrix calculation in train2()


git-svn-id: http://moon:8086/svn/software/trunk/projects/RBM@43 b431acfa-c32f-4a4a-93f1-934dc6c82436

Source/MainComponent.cpp

@@ -863,6 +863,7 @@ void MainComponent::run()
 {
 	trainButton->setEnabled(false);
 	m_pRbm->train(m_layers, numEpochslabel->getText().getIntValue());
+//	m_pRbm->train2(m_layers, numEpochslabel->getText().getIntValue(), 100);
 	trainButton->setEnabled(true);
 }
 

Source/Rbm.hpp

@@ -173,7 +173,7 @@ public:
 
 			} // TrainingSize
 
-			deltaWeights = m_momentum*deltaWeights + m_muWeights*kTrain*sumWeights - m_weightDecay*m_w.weights();
+			deltaWeights = m_momentum*deltaWeights + m_muWeights*(kTrain*sumWeights - m_weightDecay*m_w.weights());
 			m_w.weights() += deltaWeights;
 
 			deltaBiasV = m_momentum*deltaBiasV + m_muWeights*kTrain*sumBiasV;
@@ -218,6 +218,151 @@ public:
 		} // Number of epochs
 	}
 
+	MatrixXd sample(const MatrixXd &src)
+	{
+		uint32_t i;
+		MatrixXd res(src);
+
+		for (i=0; i < src.array().size(); i++)
+		{
+			res.array()(i) = (double) src.array()(i) > Noise_Uniform(&m_noise);
+		}
+
+		return res;
+	}
+
+	void train2(const LayerArray<VisibleLayer> &vt, uint32_t numEpochs, uint32_t batchSize, double sigmaMin = 0.05)
+	{
+		uint32_t t, i;
+		uint32_t epoch;
+		uint32_t gibbs;
+		double sigma = m_sigma;
+		double dProgress = 1.0/numEpochs;
+		double kTrain = 1.0/vt.getSize();
+
+		if (batchSize > vt.getSize())
+			batchSize = vt.getSize();
+
+		MatrixXd vp(m_w.getNumVisible(), batchSize);
+		MatrixXd vs(m_w.getNumVisible(), batchSize);
+		MatrixXd hp(m_w.getNumHidden(), batchSize);
+		MatrixXd hs(m_w.getNumHidden(), batchSize);
+		MatrixXd batch(m_w.getNumVisible(), batchSize);
+
+		VectorXd sumBiasV(m_w.getNumVisible());
+		VectorXd sumBiasH(m_w.getNumHidden());
+		MatrixXd sumWeights(m_w.getNumVisible(), m_w.getNumHidden());
+
+		VectorXd deltaBiasV(m_w.getNumVisible());
+		VectorXd deltaBiasH(m_w.getNumHidden());
+		MatrixXd deltaWeights(m_w.getNumVisible(), m_w.getNumHidden());
+
+		m_progress = 0;
+		m_doCancel = false;
+
+		for (i=0; i < batchSize; i++)
+		{
+			t = (uint32_t)(0.5 + (vt.getSize()-1)*Noise_Uniform(&m_noise));
+			batch.col(i) = vt[t].states();
+		}
+
+		for (epoch=0; epoch < numEpochs; epoch++)
+		{
+			if (m_doCancel)
+			{
+				m_doCancel = false;
+				break;
+			}
+
+			sumWeights.fill(0);
+			sumBiasV.fill(0);
+			sumBiasH.fill(0);
+			for (i=0; i < batchSize; i++)
+			{
+				vs = batch.col(i);
+
+				// h.probsUpdateLogistic(vt[t], m_w, m_lambda, sigma);
+				hp = -vs.transpose() * m_w.weights();
+				hp.array() = hp.array().exp();
+				hp.array() += 1;
+				hp.array() = 1.0/hp.array();
+
+				// Create hidden layer base on training data
+				hs = sample(hp);
+
+				// Update weights (positive phase)
+				sumBiasV += vp.colwise().sum();
+				if (m_doRaoBlackwell)
+				{
+					sumWeights += vs * hp.transpose();
+					sumBiasH += hp.colwise().sum();
+				}
+				else
+				{
+					sumWeights += vs * hs.transpose();
+					sumBiasH += hs.colwise().sum();
+				}
+
+				for (gibbs=0; gibbs < m_numGibbs; gibbs++)
+				{
+					// Create visible reconstruction (a fantasy...)
+					if (m_useProbsForHiddenReconstruction)
+					{
+						vp = -hs * m_w.weights().transpose();
+						vp.array() = vp.array().exp();
+						vp.array() += 1;
+						vp.array() = 1.0/vp.array();
+					}
+					else
+					{
+						vs = sample(vp);
+					}
+
+					// Create hidden reconstruction
+					hp = -vs.transpose() * m_w.weights();
+					hp.array() = hp.array().exp();
+					hp.array() += 1;
+					hp.array() = 1.0/hp.array();
+				}
+
+				// Update weights (negative phase)
+				sumBiasV -= vp.colwise().sum();
+				if (m_doRaoBlackwell)
+				{
+					sumWeights -= vs * hp.transpose();
+					sumBiasH -= hp.colwise().sum();
+				}
+				else
+				{
+					sumWeights -= vs * hs.transpose();
+					sumBiasH -= hs.colwise().sum();
+				}
+
+			} // TrainingSize
+
+			deltaWeights = m_momentum*deltaWeights + m_muWeights*kTrain*sumWeights - m_weightDecay*m_w.weights();
+			m_w.weights() += deltaWeights;
+
+			deltaBiasV = m_momentum*deltaBiasV + m_muWeights*kTrain*sumBiasV;
+			m_w.visibleBias() += deltaBiasV;
+
+			deltaBiasH = m_momentum*deltaBiasH + m_muWeights*kTrain*sumBiasH;
+			m_w.hiddenBias() += deltaBiasH;
+
+			if (sigma > sigmaMin)
+			{
+				sigma *= m_sigmaDecay;
+			}
+
+			m_progress += dProgress;
+			if (m_pListener)
+			{
+				m_pListener->onEpochTrained(*this);
+			}
+
+		} // Number of epochs
+	}
+
 	double getProgress() const
 	{
 		return m_progress;

Source/Weights.hpp

@@ -72,12 +72,12 @@ public:
 
 		for (i=0; i < m_numVisible; i++)
 		{
-			m_bv(i) = kdev*Noise_Uniform(&m_noise);
+			m_bv(i) = 0; //kdev*Noise_Uniform(&m_noise);
 		}
 
 		for (j=0; j < m_numHidden; j++)
 		{
-			m_bh(j) = kdev*Noise_Uniform(&m_noise);
+			m_bh(j) = 0; //kdev*Noise_Uniform(&m_noise);
 		}
 
 		for (i=0; i < m_numVisible; i++)