- fully refactored traning

- removed non working stuff

git-svn-id: http://moon:8086/svn/software/trunk/projects/RBM@553 b431acfa-c32f-4a4a-93f1-934dc6c82436

Source/MainComponent.cpp

@@ -485,6 +485,14 @@ void MainComponent::buttonClicked (Button* buttonThatWasClicked)
     if (buttonThatWasClicked == trainButton)
     {
         //[UserButtonCode_trainButton] -- add your button handler code here..
+		if (rbmNormalizeDataToggleButton->getToggleState() == true)
+		{
+			m_trainingData = Rbm::normalizeData(m_layers.data());
+		}
+		else
+		{
+			m_trainingData = m_layers.data();
+		}
     	startThread();
         //[/UserButtonCode_trainButton]
     }
@@ -565,33 +573,21 @@ void MainComponent::buttonClicked (Button* buttonThatWasClicked)
     else if (buttonThatWasClicked == rbmUseVisibleGaussianToggleButton)
     {
         //[UserButtonCode_rbmUseVisibleGaussianToggleButton] -- add your button handler code here..
-    	m_pRbmComponentCurr->setUseVisibleGaussian(buttonThatWasClicked->getToggleState());
-    	m_pRbmComponentCurr->redrawReconstruction();
         //[/UserButtonCode_rbmUseVisibleGaussianToggleButton]
     }
     else if (buttonThatWasClicked == rbmDoSparseToggleButton)
     {
         //[UserButtonCode_rbmDoSparseToggleButton] -- add your button handler code here..
-    	m_pRbmComponentCurr->setDoSparse(buttonThatWasClicked->getToggleState());
         //[/UserButtonCode_rbmDoSparseToggleButton]
     }
     else if (buttonThatWasClicked == rbmLearnVarianceButton)
     {
         //[UserButtonCode_rbmLearnVarianceButton] -- add your button handler code here..
-    	m_pRbmComponentCurr->setDoLearnVariance(buttonThatWasClicked->getToggleState());
-		if (!buttonThatWasClicked->getToggleState())
-		{
-			m_pRbmComponentCurr->setConstantSigma(sigmaLabel->getText().getFloatValue());
-		}
         //[/UserButtonCode_rbmLearnVarianceButton]
     }
     else if (buttonThatWasClicked == rbmNormalizeDataToggleButton)
     {
         //[UserButtonCode_rbmNormalizeDataToggleButton] -- add your button handler code here..
-    	m_pRbmComponentCurr->setNormalizeData(buttonThatWasClicked->getToggleState());
-		m_pRbmComponentCurr->batchchanged();
-    	m_pRbmComponentCurr->redrawWeights();
-    	m_pRbmComponentCurr->redrawReconstruction();
         //[/UserButtonCode_rbmNormalizeDataToggleButton]
     }
     else if (buttonThatWasClicked == rbmDoSampleBatch)
@@ -603,8 +599,7 @@ void MainComponent::buttonClicked (Button* buttonThatWasClicked)
     else if (buttonThatWasClicked == rbmUseHiddenGaussianToggleButton)
     {
         //[UserButtonCode_rbmUseHiddenGaussianToggleButton] -- add your button handler code here..
-		m_pRbmComponentCurr->setUseHiddenGaussian(buttonThatWasClicked->getToggleState());
-        //[/UserButtonCode_rbmUseHiddenGaussianToggleButton]
+       //[/UserButtonCode_rbmUseHiddenGaussianToggleButton]
     }
 
     //[UserbuttonClicked_Post]
@@ -683,27 +678,22 @@ void MainComponent::labelTextChanged (Label* labelThatHasChanged)
     else if (labelThatHasChanged == lambdaLabel)
     {
         //[UserLabelCode_lambdaLabel] -- add your label text handling code here..
-    	m_pRbmComponentCurr->setLambda(labelThatHasChanged->getText().getFloatValue());
-    	m_pRbmComponentCurr->redrawReconstruction();
         //[/UserLabelCode_lambdaLabel]
     }
     else if (labelThatHasChanged == sigmaLabel)
     {
         //[UserLabelCode_sigmaLabel] -- add your label text handling code here..
-    	m_pRbmComponentCurr->setConstantSigma(labelThatHasChanged->getText().getFloatValue());
         //[/UserLabelCode_sigmaLabel]
     }
     else if (labelThatHasChanged == sparsityLabel)
     {
         //[UserLabelCode_sparsityLabel] -- add your label text handling code here..
-    	m_pRbmComponentCurr->setSparsity(labelThatHasChanged->getText().getFloatValue());
         //[/UserLabelCode_sparsityLabel]
     }
     else if (labelThatHasChanged == sigmaDecayLabel)
     {
         //[UserLabelCode_sigmaDecayLabel] -- add your label text handling code here..
-    	m_pRbmComponentCurr->setSigmaDecay(labelThatHasChanged->getText().getFloatValue());
-        //[/UserLabelCode_sigmaDecayLabel]
+         //[/UserLabelCode_sigmaDecayLabel]
     }
     else if (labelThatHasChanged == weightDecayLabel)
     {
@@ -720,7 +710,6 @@ void MainComponent::labelTextChanged (Label* labelThatHasChanged)
     else if (labelThatHasChanged == sparsityLearningRateLabel)
     {
         //[UserLabelCode_sparsityLearningRateLabel] -- add your label text handling code here..
-    	m_pRbmComponentCurr->setMuSparsity(labelThatHasChanged->getText().getFloatValue());
         //[/UserLabelCode_sparsityLearningRateLabel]
     }
     else if (labelThatHasChanged == weightInitLabel)
@@ -843,7 +832,7 @@ void MainComponent::create(juce::String const &projectName)
 		{
 			m_weights[id] = new Weights(numVisibleLabel->getText().getIntValue(), numVisibleYLabel->getText().getIntValue(), numHiddenLabel->getText().getIntValue());
 		}
-		addAndMakeVisible(m_pRbmComponent[id] = new RbmComponent(*m_weights[id], m_layers.data(), *this));
+		addAndMakeVisible(m_pRbmComponent[id] = new RbmComponent(*m_weights[id], m_trainingData, *this));
 	}
 	else
 	{
@@ -904,20 +893,10 @@ void MainComponent::updateControls()
 	rbmDoRaoBlackwellToggleButton->setToggleState(m_pRbmComponentCurr->params().m_doRaoBlackwell, dontSendNotification);
 	rbmDoSampleVisibleToggleButton->setToggleState(m_pRbmComponentCurr->params().m_doSampleVisible, dontSendNotification);
 	rbmDoSampleBatch->setToggleState(m_pRbmComponentCurr->params().m_doSampleBatch, dontSendNotification);
-	rbmUseVisibleGaussianToggleButton->setToggleState(m_pRbmComponentCurr->params().m_useVisibleGaussian, dontSendNotification);
-	rbmUseHiddenGaussianToggleButton->setToggleState(m_pRbmComponentCurr->params().m_useHiddenGaussian, dontSendNotification);
-	rbmDoSparseToggleButton->setToggleState(m_pRbmComponentCurr->params().m_doSparse, dontSendNotification);
-   	rbmLearnVarianceButton->setToggleState(m_pRbmComponentCurr->params().m_doLearnVariance, dontSendNotification);
-	rbmDoSampleBatch->setToggleState(m_pRbmComponentCurr->params().m_doSampleBatch, dontSendNotification);
-	rbmNormalizeDataToggleButton->setToggleState(m_pRbmComponentCurr->params().m_doNormalizeData, dontSendNotification);
+ 	rbmDoSampleBatch->setToggleState(m_pRbmComponentCurr->params().m_doSampleBatch, dontSendNotification);
 	
-	lambdaLabel->setText(String(m_pRbmComponentCurr->params().m_lambda), dontSendNotification);
-	sigmaDecayLabel->setText(String(m_pRbmComponentCurr->params().m_sigmaDecay), dontSendNotification);
 	weightDecayLabel->setText(String(m_pRbmComponentCurr->params().m_weightDecay), dontSendNotification);
-	sparsityLabel->setText(String(m_pRbmComponentCurr->params().m_sparsity), dontSendNotification);
-	sigmaLabel->setText(String(m_pRbmComponentCurr->params().m_constantSigma), dontSendNotification);
-	sparsityLearningRateLabel->setText(String(m_pRbmComponentCurr->params().m_muSparsity), dontSendNotification);
-	learningRateLabel->setText(String(m_pRbmComponentCurr->params().m_muWeights), dontSendNotification);
+	learningRateLabel->setText(String(m_pRbmComponentCurr->params().m_learningRate), dontSendNotification);
 	momentumLabel->setText(String(m_pRbmComponentCurr->params().m_momentum), dontSendNotification);
 	numVisibleLabel->setText(String(m_weightsCurr->getNumVisibleX()), dontSendNotification );
 	numVisibleYLabel->setText(String(m_weightsCurr->getNumVisibleY()), dontSendNotification );

Source/MainComponent.h

@@ -82,6 +82,7 @@ private:
 	Weights *m_weightsCurr;
 	RbmComponent *m_pRbmComponentCurr;
 	LayerArray m_layers;
+	MatrixXd m_trainingData;
 	void save();
 	void create(juce::String const &projectName="");
 	void destroy();

Source/Rbm.cpp

@@ -60,47 +60,42 @@ void Rbm::sampleGaussian(MatrixXd &srcDst)
 	sampleGaussian(srcDst, srcDst);
 }
 
-void Rbm::sample(MatrixXd &srcDst)
-{
-	sample(srcDst, srcDst);
-}
-
-void Rbm::sample(MatrixXd &dst, MatrixXd const &src)
+MatrixXd Rbm::sample(MatrixXd const &src)
 {
 	MatrixXd n(src.rows(), src.cols());
 
 	noiseUniform(n);
 
-	dst = (src.array() > n.array()).cast<double>();
+	return (src.array() > n.array()).cast<double>();
+}
+
+void Rbm::sample(MatrixXd &dst, MatrixXd const &src)
+{
+	dst = sample(src);
+}
+
+MatrixXd Rbm::probsLogistic(MatrixXd const &src)
+{
+	return (1 + (-src.array()).exp()).array().cwiseInverse();
+}
+
+RowVectorXd Rbm::probsLogistic(RowVectorXd const &src)
+{
+	return (1 + (-src.array()).exp()).array().cwiseInverse();
+}
+
+MatrixXd Rbm::normalizeData(MatrixXd const &src)
+{
+	double mean = src.array().mean();
+	cout << "mean" << ": " << endl << mean << endl;
+
+	MatrixXd x = src - mean*MatrixXd::Ones(src.rows(), src.cols());
+	MatrixXd x2 = x.cwiseProduct(x);
+	double stddev = sqrt(x2.array().mean());
+	cout << "stddev" << ": " << endl << stddev << endl;
+	
+	return x;
 	
-}
-
-void Rbm::probsLogistic(MatrixXd &srcDst)
-{
-	srcDst = (1 + (-srcDst.array()).exp()).array().cwiseInverse();
-}
-
-void Rbm::probsLogistic(RowVectorXd &srcDst)
-{
-	srcDst = (1 + (-srcDst.array()).exp()).array().cwiseInverse();
-}
-
-void Rbm::normalizeData(MatrixXd &dst, MatrixXd const &src)
-{
-	MatrixXd mean = src.rowwise().mean();
-//	cout << "mean" << ": " << endl << mean << endl;
-
-	dst = src - mean.replicate(1, src.cols());
-//	cout << "dst - mean" << ": " << endl << dst << endl;
-
-	MatrixXd x = dst.array().square();
-
-	MatrixXd var = x.rowwise().mean();
-//	cout << "var" << ": " << endl << var << endl;
-
-	MatrixXd stddev_norm = var.array().sqrt().cwiseInverse();
-	dst.array() *= stddev_norm.replicate(1, src.cols()).array();
-//	cout << "dst" << ": " << endl << dst << endl;
 }
 
 void Rbm::train(size_t numEpochs, size_t miniBatchSize, double sigmaMin)
@@ -118,7 +113,7 @@ void Rbm::train(size_t numEpochs, size_t miniBatchSize, double sigmaMin)
 	MatrixXd grad_bias_v(MatrixXd::Zero(1, m_w.getNumVisible()));
 	MatrixXd grad_bias_h(MatrixXd::Zero(1, m_w.getNumHidden()));
 	MatrixXd grad_weight(MatrixXd::Zero(m_w.getNumVisible(), m_w.getNumHidden()));
-	MatrixXd __batch = m_batch_normalized;
+	MatrixXd __batch = m_batch;
 	MatrixXd momentum_weights = MatrixXd::Zero(m_w.getNumVisible(), m_w.getNumHidden());
 	MatrixXd momentum_bias_v(MatrixXd::Zero(1, m_w.getNumVisible()));
 	MatrixXd momentum_bias_h(MatrixXd::Zero(1, m_w.getNumHidden()));
@@ -126,11 +121,6 @@ void Rbm::train(size_t numEpochs, size_t miniBatchSize, double sigmaMin)
 	double L1 = 0;
 	double L2 = 0;
 	
-	if (m_params.m_doNormalizeData && !m_params.m_useVisibleGaussian)
-	{
-		probsLogistic(__batch);
-	}
-	
 	m_progress = 0;
 	while (trainingSizeRemain)
 	{
@@ -140,94 +130,75 @@ void Rbm::train(size_t numEpochs, size_t miniBatchSize, double sigmaMin)
 		trainingSizeRemain -= toSlice;
 		batchRowIndex += toSlice;
 		size_t batchSize = batch.rows();
-		double learning_rate = m_params.m_muWeights/std::min(miniBatchSize, trainingSize);
+		double learning_rate = m_params.m_learningRate/std::min(miniBatchSize, trainingSize);
+		double weight_decay = m_params.m_weightDecay/std::min(miniBatchSize, trainingSize);
 
-		MatrixXd batch_sampled(batchSize, m_w.getNumVisible());
-		MatrixXd v_sampled(batchSize, m_w.getNumVisible()); 
-		MatrixXd vis(batchSize, m_w.getNumVisible());
-		MatrixXd hid(batchSize, m_w.getNumHidden());
+		MatrixXd vis_state(batchSize, m_w.getNumVisible());
+		MatrixXd vis_probs(batchSize, m_w.getNumVisible());
+		MatrixXd hid_state(batchSize, m_w.getNumHidden());
+		MatrixXd hid_probs(batchSize, m_w.getNumHidden());
 
 		for (epoch=0; epoch < numEpochs; epoch++)
 		{
 			onProgressChanged();
 
+			// Create hidden layer base on training data
 			if (m_params.m_doSampleBatch)
 			{
 				// When the hidden units are being driven by data, always use stochastic binary states
-				sample(batch_sampled, batch);
-
-				// Create hidden layer base on sampled training data
-				hid = batch_sampled * m_w.weights();
-				hid += m_w.hiddenBias().replicate(batchSize, 1);
-				probsLogistic(hid);
+				vis_state = sample(batch);
 			}
 			else
 			{
-				// Create hidden layer base on training data
-				hid = batch * m_w.weights();
-				hid += m_w.hiddenBias().replicate(batchSize, 1);
-				probsLogistic(hid);
+				vis_state = batch;
 			}
+			
+			hid_state = vis_state * m_w.weights() + m_w.hiddenBias().replicate(batchSize, 1);
+			hid_probs = probsLogistic(hid_state);
+
 			// Sample hidden
-			if (!m_params.m_doRaoBlackwell)
+			if (m_params.m_doRaoBlackwell)
 			{
-				sample(hid);
+				hid_state = hid_probs;
+			}
+			else
+			{
+				hid_state = sample(hid_probs);
 			}
 
 			// Update weights (positive phase)
-			grad_bias_v = batch.colwise().sum();
-			grad_bias_h = hid.colwise().sum();
-			grad_weight = batch.transpose() * hid;
+			grad_weight = vis_state.transpose() * hid_state;
+			grad_bias_v = vis_state.colwise().sum();
+			grad_bias_h = hid_state.colwise().sum();
 
 			for (gibbs=0; gibbs < m_params.m_numGibbs; gibbs++)
 			{
-				if (m_params.m_useHiddenGaussian)
+
+				// Create hidden representation given v
+				hid_state = sample(hid_probs);
+
+				// Create visible reconstruction (a fantasy...) given hid
+				vis_state = hid_state * m_w.weights().transpose() + m_w.visibleBias().replicate(batchSize, 1);
+				vis_probs = probsLogistic(vis_state);
+				if (m_params.m_doSampleVisible)
 				{
-					sampleGaussian(hid);
+					vis_state = sample(vis_probs);
 				}
 				else
 				{
-					sample(hid);
-				}
-				if (m_params.m_useVisibleGaussian)
-				{
-					// Create visible reconstruction (a fantasy...) given hid
-					vis = hid * m_w.weights().transpose();
-					vis += m_w.visibleBias().replicate(batchSize, 1);
-					sampleGaussian(v_sampled, vis);
-					hid = v_sampled * m_w.weights();
-					hid += m_w.hiddenBias().replicate(batchSize, 1);
-					probsLogistic(hid);
-					
-				}
-				else
-				{
-					// Create visible reconstruction (a fantasy...) given hid
-					vis = hid * m_w.weights().transpose();
-					vis += m_w.visibleBias().replicate(batchSize, 1);
-					probsLogistic(vis);
-					if (m_params.m_doSampleVisible)
-					{
-						sample(v_sampled, vis);
-						// Create hidden representation given sampled v
-						hid = v_sampled * m_w.weights();
-						hid += m_w.hiddenBias().replicate(batchSize, 1);
-						probsLogistic(hid);
-					}
-					else
-					{
-						// Create hidden representation given v
-						hid = vis * m_w.weights();
-						hid += m_w.hiddenBias().replicate(batchSize, 1);
-						probsLogistic(hid);
-					}
+					vis_state = vis_probs;
 				}
+
+				// Create hidden representation given v
+				hid_state = vis_state * m_w.weights() + m_w.hiddenBias().replicate(batchSize, 1);
+				hid_probs = probsLogistic(hid_state);
+			
 			}
 
 			// Update weights (negative phase)
-			grad_bias_v -= vis.colwise().sum();
-			grad_bias_h -= hid.colwise().sum();
-			grad_weight -= vis.transpose() * hid;
+			grad_bias_v -= vis_probs.colwise().sum();
+			grad_bias_h -= hid_probs.colwise().sum();
+			grad_weight -= vis_probs.transpose() * hid_probs;
 
 			for (int i=0; i < m_w.weights().rows(); i++)
 			{
@@ -235,11 +206,11 @@ void Rbm::train(size_t numEpochs, size_t miniBatchSize, double sigmaMin)
 				{
 					if (m_w.weights()(i,j) >= 0)
 					{
-						penalty_weights(i,j) = m_params.m_weightDecay;
+						penalty_weights(i,j) = weight_decay;
 					}
 					else
 					{
-						penalty_weights(i,j) = -m_params.m_weightDecay;
+						penalty_weights(i,j) = -weight_decay;
 					}
 				}
 			}
@@ -251,34 +222,16 @@ void Rbm::train(size_t numEpochs, size_t miniBatchSize, double sigmaMin)
 			momentum_weights = m_params.m_momentum*momentum_weights + grad_weight - L2*penalty_weights;
 			
 			m_w.visibleBias() += learning_rate*momentum_bias_v;
-
-			if (m_params.m_doSparse)
-			{
-				MatrixXd h1 = hid-MatrixXd::Ones(hid.rows(), hid.cols())*m_params.m_sparsity;
-				RowVectorXd hm = h1.colwise().mean();
-				m_w.hiddenBias() -=  m_params.m_muSparsity * hm;
-			}
-			else
-			{
-				m_w.hiddenBias() += learning_rate*momentum_bias_h;
-			}
-
+			m_w.hiddenBias() += learning_rate*momentum_bias_h;
 			m_w.weights() += learning_rate*momentum_weights;
 			
-			if (m_params.m_sigmaDecay > 0)
-			{
-				if (m_variableSigma[0] > sigmaMin)
-				{
-				}
-			}
-
 			m_progress += dProgress;
 
 		} // Number of epochs
 			
-		MatrixXd diffErr = batch - vis;
+		MatrixXd diffErr = batch - vis_probs;
 		diffErr.array() *= diffErr.array();
-		double err = diffErr.colwise().sum().sum();
+		double err = diffErr.colwise().sum().mean();
 		cout << "error (per mini batch) = " << err << endl;
 		cout << "L1                     = " << L1 << endl;
 		cout << "L2                     = " << L2 << endl;
@@ -288,10 +241,9 @@ void Rbm::train(size_t numEpochs, size_t miniBatchSize, double sigmaMin)
 	
 	MatrixXd vis = m_h * m_w.weights().transpose();
 	vis += m_w.visibleBias().replicate(__batch.rows(), 1);
-	probsLogistic(vis);
-	MatrixXd diffErr = __batch - vis;
+	MatrixXd diffErr = __batch - probsLogistic(vis);
 	diffErr.array() *= diffErr.array();
-	double err = diffErr.colwise().sum().sum();
+	double err = diffErr.colwise().sum().mean();
 	cout << "error (total) = " << err << endl;
 
 	onProgressChanged();
@@ -307,35 +259,14 @@ void Rbm::toHidden(RowVectorXd &h, RowVectorXd const &v)
 {
 	h = v * m_w.weights();
 	h += m_w.hiddenBias();
-	probsLogistic(h);
+	h = probsLogistic(h);
 }
 
 void Rbm::toVisible(RowVectorXd &v, RowVectorXd const &h)
 {
 	v = h * m_w.weights().transpose();
 	v += m_w.visibleBias();
-
-	if (!m_params.m_useVisibleGaussian)
-	{
-		probsLogistic(v);
-	}
-}
-
-void Rbm::setConstantSigma(double value)
-{	
-	 m_params.m_constantSigma = value;
-	 onParamsChanged();
-}
-
-RowVectorXd& Rbm::getVariableSigma()
-{
-	 return m_variableSigma;
-}
-
-void Rbm::setSigmaDecay(double value)
-{
-	m_params.m_sigmaDecay = value;
-	onParamsChanged();
+	v = probsLogistic(v);
 }
 
 void Rbm::setWeightDecay(double value)
@@ -344,30 +275,6 @@ void Rbm::setWeightDecay(double value)
 	onParamsChanged();
 }
 
-void Rbm::setLambda(double value)
-{
-	m_params.m_lambda = value;
-	onParamsChanged();
-}
-
-void Rbm::setSparsity(double value)
-{
-	m_params.m_sparsity = value;
-	onParamsChanged();
-}
-
-void Rbm::setUseVisibleGaussian(bool flag)
-{
-	m_params.m_useVisibleGaussian = flag;
-	onParamsChanged();
-}
-
-void Rbm::setUseHiddenGaussian(bool flag)
-{
-	m_params.m_useHiddenGaussian = flag;
-	onParamsChanged();
-}
-
 void Rbm::setDoRaoBlackwell(bool flag)
 {
 	m_params.m_doRaoBlackwell = flag;
@@ -386,24 +293,6 @@ void Rbm::setDoSampleBatch(bool flag)
 	onParamsChanged();
 }
 
-void Rbm::setDoSparse(bool flag)
-{
-	m_params.m_doSparse = flag;
-	onParamsChanged();
-}
-
-void Rbm::setNormalizeData(bool flag)
-{
-	m_params.m_doNormalizeData = flag;
-	onParamsChanged();
-}
-
-void Rbm::setDoLearnVariance(bool flag)
-{
-	m_params.m_doLearnVariance = flag;
-	onParamsChanged();
-}
-
 void Rbm::setNumGibbs(size_t value)
 {
 	m_params.m_numGibbs = value;
@@ -412,13 +301,7 @@ void Rbm::setNumGibbs(size_t value)
 
 void Rbm::setMuWeights(double value)
 {
-	m_params.m_muWeights = value;
-	onParamsChanged();
-}
-
-void Rbm::setMuSparsity(double value)
-{
-	m_params.m_muSparsity = value;
+	m_params.m_learningRate = value;
 	onParamsChanged();
 }
 
@@ -435,7 +318,7 @@ MatrixXd const& Rbm::getHiddenBatch()
 
 MatrixXd const& Rbm::getBatch()
 {
-	return m_batch_normalized;
+	return m_batch;
 }
 
 void Rbm::updateHiddenBatch()
@@ -444,20 +327,11 @@ void Rbm::updateHiddenBatch()
 	{
 		return;
 	}
-	m_batch_normalized.resize(m_batch.rows(), m_w.getNumHidden());
 	
-	if (m_params.m_doNormalizeData)
-	{
-		normalizeData(m_batch_normalized, m_batch);
-	}
-	else
-	{
-		m_batch_normalized = m_batch;
-	}
 	m_h.resize(m_batch.rows(), m_w.getNumHidden());
-	m_h = m_batch_normalized * m_w.weights();
+	m_h = m_batch * m_w.weights();
 	m_h += m_w.hiddenBias().replicate(m_batch.rows(), 1);
-	probsLogistic(m_h);
+	m_h = probsLogistic(m_h);
 }
 
 Rbm::Params const& Rbm::params()

Source/Rbm.hpp

@@ -20,75 +20,44 @@ public:
 	struct Params
 	{
 		Params()
-		: m_constantSigma(1.0)
-		, m_sigmaDecay(0.0)
-		, m_weightDecay(0.0)
-		, m_lambda(1.0)
-		, m_sparsity(0.05)
-		, m_muWeights(0.1)
-		, m_muSparsity(0.01)
+		: m_weightDecay(0.01)
+		, m_learningRate(0.1)
 		, m_momentum(0.5)
-		, m_useVisibleGaussian(false)
-		, m_useHiddenGaussian(false)
 		, m_doRaoBlackwell(true)
 		, m_doSampleVisible(false)
 		, m_doSampleBatch(false)
-		, m_doSparse(false)
-		, m_doNormalizeData(false)
-		, m_doLearnVariance(false)
 		, m_numGibbs(1)
 		{
 		}
 		
-		double m_constantSigma;
-		double m_sigmaDecay;
 		double m_weightDecay;
-		double m_lambda;
-		double m_sparsity;
-		double m_muWeights;
-		double m_muSparsity;
+		double m_learningRate;
 		double m_momentum;
-		bool m_useVisibleGaussian;
-		bool m_useHiddenGaussian;
 		bool m_doRaoBlackwell;
 		bool m_doSampleVisible;
 		bool m_doSampleBatch;
-		bool m_doSparse;
-		bool m_doNormalizeData;
-		bool m_doLearnVariance;
 		size_t m_numGibbs;
 	};
 	
 	Rbm(Weights &weights, const MatrixXd &batch);
 	~Rbm();
-	void sample(MatrixXd &srcDst);
+	MatrixXd sample(MatrixXd const &src);
 	void sample(MatrixXd &dst, MatrixXd const &src);
-	static void probsLogistic(MatrixXd &srcDst);
-	static void probsLogistic(RowVectorXd &srcDst);
+	static MatrixXd probsLogistic(MatrixXd const &src);
+	static RowVectorXd probsLogistic(RowVectorXd const &src);
 	void sampleGaussian(MatrixXd &dst, MatrixXd const &src);
 	void sampleGaussian(MatrixXd &srcDst);
-	void normalizeData(MatrixXd &dst, MatrixXd const &src);
+	static MatrixXd normalizeData(MatrixXd const &src);
 	void train(size_t numEpochs, size_t miniBatchSize, double sigmaMin = 0.05);
 	double getProgress() const;
 	void toHidden(RowVectorXd &h, RowVectorXd const &v);
 	void toVisible(RowVectorXd &v, RowVectorXd const &h);
-	void setConstantSigma(double value);
-	RowVectorXd& getVariableSigma();
-	void setSigmaDecay(double value);
 	void setWeightDecay(double value);
-	void setLambda(double value);
-	void setSparsity(double value);
-	void setUseVisibleGaussian(bool flag);
-	void setUseHiddenGaussian(bool flag);
 	void setDoRaoBlackwell(bool flag);
 	void setDoSampleVisible(bool flag);
 	void setDoSampleBatch(bool flag);
-	void setDoSparse(bool flag);
-	void setNormalizeData(bool flag);
-	void setDoLearnVariance(bool flag);
 	void setNumGibbs(size_t value);
 	void setMuWeights(double value);
-	void setMuSparsity(double value);
 	void setMomentum(double value);
 	MatrixXd const& getHiddenBatch();
 	MatrixXd const& getBatch();
@@ -98,9 +67,7 @@ public:
 private:
 	Weights &m_w;
 	MatrixXd const &m_batch;
-	MatrixXd m_batch_normalized;
 	MatrixXd m_h;
-	RowVectorXd m_variableSigma;
 	noise_gen_t m_noise;
 	double m_progress;
 	Params m_params;