From 57cdfd706598ee14161b121881d9dad462c6a707 Mon Sep 17 00:00:00 2001
From: Jens Ahrensfeld <jens@jayfield.org>
Date: Thu, 7 Jul 2016 19:27:33 +0000
Subject: [PATCH] [RBM] - GUI: added gaussian hidden, added mini batch size -
 Rbm: added mini batch training revised sample functions, reverted to old
 weight decay

git-svn-id: http://moon:8086/svn/software/trunk/projects/RBM@305 b431acfa-c32f-4a4a-93f1-934dc6c82436
---
 Source/MainComponent.cpp |  53 ++++++++++--
 Source/MainComponent.h   |   2 +
 Source/Rbm.cpp           | 180 ++++++++++++++++++++++-----------------
 Source/Rbm.hpp           |  13 ++-
 4 files changed, 161 insertions(+), 87 deletions(-)

diff --git a/Source/MainComponent.cpp b/Source/MainComponent.cpp
index 0ac58e3..c11825f 100644
--- a/Source/MainComponent.cpp
+++ b/Source/MainComponent.cpp
@@ -277,6 +277,20 @@ MainComponent::MainComponent ()
     rbmDoSampleBatch->setButtonText (TRANS("Sample training"));
     rbmDoSampleBatch->addListener (this);
 
+    addAndMakeVisible (sizeMiniBatch = new Label ("sizeMiniBatch",
+                                                  TRANS("0")));
+    sizeMiniBatch->setTooltip (TRANS("Maximum number of training samples per train"));
+    sizeMiniBatch->setFont (Font (15.00f, Font::plain));
+    sizeMiniBatch->setJustificationType (Justification::centred);
+    sizeMiniBatch->setEditable (true, true, false);
+    sizeMiniBatch->setColour (TextEditor::textColourId, Colours::black);
+    sizeMiniBatch->setColour (TextEditor::backgroundColourId, Colour (0x00000000));
+    sizeMiniBatch->addListener (this);
+
+    addAndMakeVisible (rbmUseHiddenGaussianToggleButton = new ToggleButton ("rbmUseHiddenGaussian toggle button"));
+    rbmUseHiddenGaussianToggleButton->setButtonText (TRANS("Use gaussian hidden"));
+    rbmUseHiddenGaussianToggleButton->addListener (this);
+
 
     //[UserPreSize]
 	memset(m_pRbmComponent, 0, sizeof(m_pRbmComponent));
@@ -334,6 +348,8 @@ MainComponent::~MainComponent()
     rbmNormalizeDataToggleButton = nullptr;
     m_rbmSelect = nullptr;
     rbmDoSampleBatch = nullptr;
+    sizeMiniBatch = nullptr;
+    rbmUseHiddenGaussianToggleButton = nullptr;
 
 
     //[Destructor]. You can add your own custom destruction code here..
@@ -443,14 +459,16 @@ void MainComponent::resized()
     sparsityLabel->setBounds (740, 152, 72, 24);
     sigmaDecayLabel->setBounds (836, 152, 72, 24);
     weightDecayLabel->setBounds (836, 200, 72, 24);
-    m_progressBarSlider->setBounds (844, 20, 220, 24);
+    m_progressBarSlider->setBounds (828, 20, 220, 24);
     momentumLabel->setBounds (740, 200, 72, 24);
     sparsityLearningRateLabel->setBounds (948, 152, 72, 24);
     weightInitLabel->setBounds (948, 200, 72, 24);
-    rbmLearnVarianceButton->setBounds (976, 92, 128, 24);
-    rbmNormalizeDataToggleButton->setBounds (844, 92, 124, 24);
+    rbmLearnVarianceButton->setBounds (1004, 92, 128, 24);
+    rbmNormalizeDataToggleButton->setBounds (1004, 60, 124, 24);
     m_rbmSelect->setBounds (812, 280, 62, 24);
     rbmDoSampleBatch->setBounds (844, 60, 128, 24);
+    sizeMiniBatch->setBounds (1064, 20, 48, 24);
+    rbmUseHiddenGaussianToggleButton->setBounds (840, 92, 156, 24);
     //[UserResized] Add your own custom resize handling here..
     //[/UserResized]
 }
@@ -575,6 +593,12 @@ void MainComponent::buttonClicked (Button* buttonThatWasClicked)
 		m_pRbmComponentCurr->setDoSampleBatch(buttonThatWasClicked->getToggleState());
         //[/UserButtonCode_rbmDoSampleBatch]
     }
+    else if (buttonThatWasClicked == rbmUseHiddenGaussianToggleButton)
+    {
+        //[UserButtonCode_rbmUseHiddenGaussianToggleButton] -- add your button handler code here..
+		m_pRbmComponentCurr->setUseHiddenGaussian(buttonThatWasClicked->getToggleState());
+        //[/UserButtonCode_rbmUseHiddenGaussianToggleButton]
+    }
 
     //[UserbuttonClicked_Post]
     //[/UserbuttonClicked_Post]
@@ -697,6 +721,12 @@ void MainComponent::labelTextChanged (Label* labelThatHasChanged)
         //[UserLabelCode_weightInitLabel] -- add your label text handling code here..
         //[/UserLabelCode_weightInitLabel]
     }
+    else if (labelThatHasChanged == sizeMiniBatch)
+    {
+        //[UserLabelCode_sizeMiniBatch] -- add your label text handling code here..
+		m_pRbmComponentCurr->setMiniBatchSize((size_t)labelThatHasChanged->getText().getFloatValue());
+        //[/UserLabelCode_sizeMiniBatch]
+    }
 
     //[UserlabelTextChanged_Post]
     //[/UserlabelTextChanged_Post]
@@ -867,6 +897,7 @@ void MainComponent::updateControls()
 	rbmDoSampleVisibleToggleButton->setToggleState(m_pRbmComponentCurr->params().m_doSampleVisible, dontSendNotification);
 	rbmDoSampleBatch->setToggleState(m_pRbmComponentCurr->params().m_doSampleBatch, dontSendNotification);
 	rbmUseVisibleGaussianToggleButton->setToggleState(m_pRbmComponentCurr->params().m_useVisibleGaussian, dontSendNotification);
+	rbmUseHiddenGaussianToggleButton->setToggleState(m_pRbmComponentCurr->params().m_useHiddenGaussian, dontSendNotification);
 	rbmDoSparseToggleButton->setToggleState(m_pRbmComponentCurr->params().m_doSparse, dontSendNotification);
    	rbmLearnVarianceButton->setToggleState(m_pRbmComponentCurr->params().m_doLearnVariance, dontSendNotification);
 	rbmDoSampleBatch->setToggleState(m_pRbmComponentCurr->params().m_doSampleBatch, dontSendNotification);
@@ -882,6 +913,7 @@ void MainComponent::updateControls()
 	numVisibleLabel->setText(String(m_weightsCurr->getNumVisibleX()), dontSendNotification );
 	numVisibleYLabel->setText(String(m_weightsCurr->getNumVisibleY()), dontSendNotification );
 	numHiddenLabel->setText(String(m_weightsCurr->getNumHidden()), dontSendNotification );
+	sizeMiniBatch->setText(String(m_pRbmComponentCurr->params().m_miniBatchSize), dontSendNotification);
 
 	WeightsSlider->setRange(0, m_weightsCurr->getNumHidden()-1, 1);
 	numGibbsSlider->setValue(m_pRbmComponentCurr->params().m_numGibbs);
@@ -1054,7 +1086,7 @@ BEGIN_JUCER_METADATA
          focusDiscardsChanges="0" fontname="Default font" fontsize="15"
          bold="0" italic="0" justification="36"/>
   <SLIDER name="progressBar slider" id="8c8748f39f6d0ec6" memberName="m_progressBarSlider"
-          virtualName="" explicitFocusOrder="0" pos="844 20 220 24" tooltip="Progress"
+          virtualName="" explicitFocusOrder="0" pos="828 20 220 24" tooltip="Progress"
           min="0" max="100" int="1" style="LinearBar" textBoxPos="TextBoxLeft"
           textBoxEditable="0" textBoxWidth="80" textBoxHeight="20" skewFactor="1"/>
   <LABEL name="Momentum label" id="570ca1355ccd4cdd" memberName="momentumLabel"
@@ -1073,10 +1105,10 @@ BEGIN_JUCER_METADATA
          focusDiscardsChanges="0" fontname="Default font" fontsize="15"
          bold="0" italic="0" justification="36"/>
   <TOGGLEBUTTON name="rbmLearnVariance button" id="92c647c1f8b110a2" memberName="rbmLearnVarianceButton"
-                virtualName="" explicitFocusOrder="0" pos="976 92 128 24" buttonText="Learn Variance"
+                virtualName="" explicitFocusOrder="0" pos="1004 92 128 24" buttonText="Learn Variance"
                 connectedEdges="0" needsCallback="1" radioGroupId="0" state="0"/>
   <TOGGLEBUTTON name="rbmNormalizeData toggle button" id="739772af1b096120" memberName="rbmNormalizeDataToggleButton"
-                virtualName="" explicitFocusOrder="0" pos="844 92 124 24" buttonText="Normalize data"
+                virtualName="" explicitFocusOrder="0" pos="1004 60 124 24" buttonText="Normalize data"
                 connectedEdges="0" needsCallback="1" radioGroupId="0" state="0"/>
   <COMBOBOX name="RBM Selector" id="71115a4f965bfd38" memberName="m_rbmSelect"
             virtualName="" explicitFocusOrder="0" pos="812 280 62 24" editable="0"
@@ -1085,6 +1117,15 @@ BEGIN_JUCER_METADATA
                 virtualName="" explicitFocusOrder="0" pos="844 60 128 24" tooltip="Sample training data during learning&#10;"
                 buttonText="Sample training" connectedEdges="0" needsCallback="1"
                 radioGroupId="0" state="0"/>
+  <LABEL name="sizeMiniBatch" id="dd9e3e6f2b7b22f8" memberName="sizeMiniBatch"
+         virtualName="" explicitFocusOrder="0" pos="1064 20 48 24" tooltip="Maximum number of training samples per train"
+         edTextCol="ff000000" edBkgCol="0" labelText="0" editableSingleClick="1"
+         editableDoubleClick="1" focusDiscardsChanges="0" fontname="Default font"
+         fontsize="15" bold="0" italic="0" justification="36"/>
+  <TOGGLEBUTTON name="rbmUseHiddenGaussian toggle button" id="92e05c920283616b"
+                memberName="rbmUseHiddenGaussianToggleButton" virtualName=""
+                explicitFocusOrder="0" pos="840 92 156 24" buttonText="Use gaussian hidden"
+                connectedEdges="0" needsCallback="1" radioGroupId="0" state="0"/>
 </JUCER_COMPONENT>
 
 END_JUCER_METADATA
diff --git a/Source/MainComponent.h b/Source/MainComponent.h
index dd504f0..0273415 100644
--- a/Source/MainComponent.h
+++ b/Source/MainComponent.h
@@ -150,6 +150,8 @@ private:
     ScopedPointer<ToggleButton> rbmNormalizeDataToggleButton;
     ScopedPointer<ComboBox> m_rbmSelect;
     ScopedPointer<ToggleButton> rbmDoSampleBatch;
+    ScopedPointer<Label> sizeMiniBatch;
+    ScopedPointer<ToggleButton> rbmUseHiddenGaussianToggleButton;
 
 
     //==============================================================================
diff --git a/Source/Rbm.cpp b/Source/Rbm.cpp
index c20dcc5..3bf1d09 100644
--- a/Source/Rbm.cpp
+++ b/Source/Rbm.cpp
@@ -17,6 +17,7 @@ Rbm::Rbm(Weights &weights, const MatrixXd &batch)
 , m_variableSigma(weights.getNumVisible())
 , m_progress(0)
 {
+	setMiniBatchSize(batch.rows());
 	Noise_Init(&m_noise, 0x32727155);
 	m_variableSigma.fill(m_params.m_constantSigma);
 	updateHiddenBatch();
@@ -27,6 +28,49 @@ Rbm::~Rbm()
 	Noise_Free(&m_noise);
 }
 
+void Rbm::noiseGaussian(MatrixXd &dst)
+{
+	for (size_t i=0; i < dst.rows(); i++)
+	{
+		for (size_t j=0; j < dst.cols(); j++)
+		{
+			dst(i, j) = Noise_Gaussian(&m_noise);
+		}
+	}
+}
+
+void Rbm::noiseUniform(MatrixXd &dst)
+{
+	for (size_t i=0; i < dst.rows(); i++)
+	{
+		for (size_t j=0; j < dst.cols(); j++)
+		{
+			dst(i, j) = Noise_Uniform(&m_noise);
+		}
+	}
+}
+
+void Rbm::sampleGaussian(MatrixXd &dst, MatrixXd const &src, const MatrixXd &sigma)
+{
+	MatrixXd n(src.rows(), src.cols());
+
+	noiseGaussian(n);
+	dst = sigma.array()*n.array() + src.array();
+}
+
+void Rbm::sampleGaussian(MatrixXd &srcDst, const MatrixXd &sigma)
+{
+	MatrixXd n(srcDst.rows(), srcDst.cols());
+
+	noiseGaussian(n);
+	srcDst.array() += sigma.array()*n.array();
+}
+
+void Rbm::sampleGaussian(MatrixXd &srcDst, const double &sigma)
+{
+	sampleGaussian(srcDst, sigma*MatrixXd::Ones(srcDst.rows(), srcDst.cols()));
+}
+
 void Rbm::sample(MatrixXd &srcDst)
 {
 	sample(srcDst, srcDst);
@@ -34,42 +78,34 @@ void Rbm::sample(MatrixXd &srcDst)
 
 void Rbm::sample(MatrixXd &dst, MatrixXd const &src)
 {
-	uint32_t i;
+	MatrixXd n(src.rows(), src.cols());
 
-	for (i=0; i < src.array().size(); i++)
-	{
-		dst.array()(i) = (double)(src.array()(i) > Noise_Uniform(&m_noise));
-	}
+	noiseUniform(n);
+
+	dst = (src.array() > n.array()).cast<double>();
+	
 }
 
 void Rbm::probsLogistic(MatrixXd &src)
 {
-	src.array() = (-src.array()).exp();
-	src.array() += 1;
-	src.array() = 1.0/src.array();
+	src = (1 + (-src.array()).exp()).array().cwiseInverse();
 }
 
 void Rbm::probsLogistic(RowVectorXd &src)
 {
-	src.array() = (-src.array()).exp();
-	src.array() += 1;
-	src.array() = 1.0/src.array();
+	src = (1 + (-src.array()).exp()).array().cwiseInverse();
 }
 
 void Rbm::probsLogistic(MatrixXd &src, const MatrixXd &sigma)
 {
 	src.array() /= (sigma.array() + EPSILON_SIGMA);
-	src.array() = (-src.array()).exp();
-	src.array() += 1;
-	src.array() = 1.0/src.array();
+	probsLogistic(src);
 }
 
 void Rbm::probsLogistic(RowVectorXd &src, const RowVectorXd &sigma)
 {
 	src.array() /= (sigma.array() + EPSILON_SIGMA);
-	src.array() = (-src.array()).exp();
-	src.array() += 1;
-	src.array() = 1.0/src.array();
+	probsLogistic(src);
 }
 
 void Rbm::probsGaussian(MatrixXd &src, const MatrixXd &sigma)
@@ -116,26 +152,6 @@ void Rbm::probsGaussian(RowVectorXd &src, const RowVectorXd &sigma)
 	src.array() *= k.array();
 }
 
-void Rbm::sampleGaussian(MatrixXd &dst, MatrixXd const &src, const MatrixXd &sigma)
-{
-	uint32_t i;
-
-	for (i=0; i < src.array().size(); i++)
-	{
-		dst.array()(i) = sigma(i)*Noise_Gaussian(&m_noise) + src.array()(i);
-	}
-}
-
-void Rbm::sampleGaussian(MatrixXd &src, const MatrixXd &sigma)
-{
-	uint32_t i;
-
-	for (i=0; i < src.array().size(); i++)
-	{
-		src.array()(i) = sigma(i)*Noise_Gaussian(&m_noise) + src.array()(i);
-	}
-}
-
 RowVectorXd Rbm::normalizeData(RowVectorXd const &src, RowVectorXd const &mu, RowVectorXd const &var)
 {
 	// Remove mean
@@ -191,9 +207,8 @@ void Rbm::train(uint32_t numEpochs, double sigmaMin)
 	size_t trainingSize = m_batch.rows();
 	size_t trainingSizeRemain = trainingSize;
 	size_t batchRowIndex = 0;
-	const size_t miniBatchSize = 100;
 	
-	double dProgress = 1.0/(numEpochs*(double)trainingSize/miniBatchSize);
+	double dProgress = 1.0/(numEpochs*(double)trainingSize/std::min(m_params.m_miniBatchSize, trainingSize));
 
 	MatrixXd dBiasV_curr(MatrixXd::Zero(1, m_w.getNumVisible()));
 	MatrixXd dBiasH_curr(MatrixXd::Zero(1, m_w.getNumHidden()));
@@ -206,14 +221,14 @@ void Rbm::train(uint32_t numEpochs, double sigmaMin)
 	while (trainingSizeRemain)
 	{
 		cout << "trainingSizeRemain: " << trainingSizeRemain << endl;
-		size_t toSlice = std::min(miniBatchSize, trainingSizeRemain);
+		size_t toSlice = std::min(m_params.m_miniBatchSize, trainingSizeRemain);
 		MatrixXd batch = m_batch.block(batchRowIndex, 0, toSlice, m_w.getNumVisible());
 		trainingSizeRemain -= toSlice;
+		batchRowIndex += toSlice;
 		size_t batchSize = batch.rows();
-		double mu_w = m_params.m_muWeights/batchSize;
-		double mu_biasV = m_params.m_muWeights/batchSize;
-		double mu_biasH = m_params.m_muWeights/batchSize;
-		MatrixXd diffErr(batchSize, m_w.getNumVisible());
+		double mu_w = m_params.m_muWeights/std::min(m_params.m_miniBatchSize, trainingSize);
+		double mu_biasV = m_params.m_muWeights/std::min(m_params.m_miniBatchSize, trainingSize);
+		double mu_biasH = m_params.m_muWeights/std::min(m_params.m_miniBatchSize, trainingSize);
 
 		MatrixXd batch_sampled(batchSize, m_w.getNumVisible());
 		MatrixXd v_sampled(batchSize, m_w.getNumVisible()); 
@@ -264,14 +279,19 @@ void Rbm::train(uint32_t numEpochs, double sigmaMin)
 
 			for (gibbs=0; gibbs < m_params.m_numGibbs; gibbs++)
 			{
-				sample(hid);
-
-				// Create visible reconstruction (a fantasy...) given hid
-				vis = hid * m_w.weights().transpose();
-				vis += m_w.visibleBias().replicate(batchSize, 1);
-				
+				if (m_params.m_useHiddenGaussian)
+				{
+					sampleGaussian(hid, m_params.m_constantSigma);
+				}
+				else
+				{
+					sample(hid);
+				}
 				if (m_params.m_useVisibleGaussian)
 				{
+					// Create visible reconstruction (a fantasy...) given hid
+					vis = hid * m_w.weights().transpose();
+					vis += m_w.visibleBias().replicate(batchSize, 1);
 					sampleGaussian(v_sampled, vis, m_variableSigma.replicate(batchSize, 1));
 					hid = v_sampled * m_w.weights();
 					hid += m_w.hiddenBias().replicate(batchSize, 1);
@@ -280,6 +300,9 @@ void Rbm::train(uint32_t numEpochs, double sigmaMin)
 				}
 				else
 				{
+					// Create visible reconstruction (a fantasy...) given hid
+					vis = hid * m_w.weights().transpose();
+					vis += m_w.visibleBias().replicate(batchSize, 1);
 					probsLogistic(vis, m_variableSigma.replicate(batchSize, 1));
 					if (m_params.m_doSampleVisible)
 					{
@@ -319,26 +342,7 @@ void Rbm::train(uint32_t numEpochs, double sigmaMin)
 			}
 			dBiasH = dBiasH_curr;
 
-			MatrixXd p = m_w.weights();
-			if (m_params.m_weightDecay > 0)
-			{
-				for (size_t row=0; row < m_w.weights().rows(); row++)
-				{
-					for (size_t col=0; col < m_w.weights().cols(); col++)
-					{
-						if (p(row, col) >= 0)
-						{
-							p(row, col) = m_params.m_weightDecay;
-						}
-						else
-						{
-							p(row, col) -= m_params.m_weightDecay;
-						}
-					}
-				}
-				m_w.weights() -= mu_w*p;
-			}
-			m_w.weights() += mu_w*(m_params.m_momentum*dW + (1-m_params.m_momentum)*dW_curr);
+			m_w.weights() += mu_w*((m_params.m_momentum*dW + (1-m_params.m_momentum)*dW_curr) - m_params.m_weightDecay*m_w.weights());
 			dW = dW_curr;
 
 			if (m_params.m_sigmaDecay > 0)
@@ -353,15 +357,24 @@ void Rbm::train(uint32_t numEpochs, double sigmaMin)
 
 		} // Number of epochs
 			
-		diffErr = batch - vis;
+		MatrixXd diffErr = batch - vis;
 		diffErr.array() *= diffErr.array();
 		double err = diffErr.colwise().sum().sum();
-		cout << "err =" << endl;
-		cout << err << endl;
+		cout << "error (per mini batch) = " << err << endl;
 	} // number of mini batches
 	
 	updateHiddenBatch();
+	
+	MatrixXd vis = m_h * m_w.weights().transpose();
+	vis += m_w.visibleBias().replicate(m_batch.rows(), 1);
+	probsLogistic(vis, m_variableSigma.replicate(m_batch.rows(), 1));
+	MatrixXd diffErr = m_batch - vis;
+	diffErr.array() *= diffErr.array();
+	double err = diffErr.colwise().sum().sum();
+	cout << "error (total) = " << err << endl;
+
 	onProgressChanged();
+	
 }
 
 double Rbm::getProgress() const
@@ -445,6 +458,12 @@ void Rbm::setUseVisibleGaussian(bool flag)
 	onParamsChanged();
 }
 
+void Rbm::setUseHiddenGaussian(bool flag)
+{
+	m_params.m_useHiddenGaussian = flag;
+	onParamsChanged();
+}
+
 void Rbm::setDoRaoBlackwell(bool flag)
 {
 	m_params.m_doRaoBlackwell = flag;
@@ -495,6 +514,11 @@ void Rbm::setNumGibbs(size_t value)
 	onParamsChanged();
 }
 
+void Rbm::setMiniBatchSize(size_t size)
+{
+	m_params.m_miniBatchSize = size;
+}
+
 void Rbm::setMuWeights(double value)
 {
 	m_params.m_muWeights = value;
@@ -518,11 +542,6 @@ MatrixXd const& Rbm::getHiddenBatch()
 	return m_h;
 }
 
-MatrixXd const& Rbm::getVisibleBatch()
-{
-	return m_v;
-}
-
 MatrixXd const& Rbm::getBatch()
 {
 	return m_batch;
@@ -530,6 +549,10 @@ MatrixXd const& Rbm::getBatch()
 
 void Rbm::updateHiddenBatch()
 {
+	if (m_batch.rows() == 0)
+	{
+		return;
+	}
 	m_h.resize(m_batch.rows(), m_w.getNumHidden());
 	m_h = m_batch * m_w.weights();
 	m_h += m_w.hiddenBias().replicate(m_batch.rows(), 1);
@@ -540,3 +563,4 @@ Rbm::Params const& Rbm::params()
 {
 	return m_params;
 }
+
diff --git a/Source/Rbm.hpp b/Source/Rbm.hpp
index fa6f296..639996e 100644
--- a/Source/Rbm.hpp
+++ b/Source/Rbm.hpp
@@ -29,6 +29,7 @@ public:
 		, m_muSparsity(0.01)
 		, m_momentum(0.5)
 		, m_useVisibleGaussian(false)
+		, m_useHiddenGaussian(false)
 		, m_doRaoBlackwell(true)
 		, m_doSampleVisible(false)
 		, m_doSampleBatch(false)
@@ -36,6 +37,7 @@ public:
 		, m_doNormalizeData(false)
 		, m_doLearnVariance(false)
 		, m_numGibbs(1)
+		, m_miniBatchSize(100)
 		{
 		}
 		
@@ -48,6 +50,7 @@ public:
 		double m_muSparsity;
 		double m_momentum;
 		bool m_useVisibleGaussian;
+		bool m_useHiddenGaussian;
 		bool m_doRaoBlackwell;
 		bool m_doSampleVisible;
 		bool m_doSampleBatch;
@@ -55,6 +58,7 @@ public:
 		bool m_doNormalizeData;
 		bool m_doLearnVariance;
 		size_t m_numGibbs;
+		size_t m_miniBatchSize;
 	};
 	
 	Rbm(Weights &weights, const MatrixXd &batch);
@@ -68,7 +72,8 @@ public:
 	static void probsGaussian(MatrixXd &src, const MatrixXd &sigma);
 	static void probsGaussian(RowVectorXd &src, const RowVectorXd &sigma);
 	void sampleGaussian(MatrixXd &dst, MatrixXd const &src, const MatrixXd &sigma);
-	void sampleGaussian(MatrixXd &src, const MatrixXd &sigma);
+	void sampleGaussian(MatrixXd &srcDst, const MatrixXd &sigma);
+	void sampleGaussian(MatrixXd &srcDst, const double &sigma);
 	RowVectorXd normalizeData(RowVectorXd const &src, RowVectorXd const &mu, RowVectorXd const &var);
 	RowVectorXd calcMean(MatrixXd const &batch);
 	RowVectorXd calcSigma(MatrixXd const &batch);
@@ -85,6 +90,7 @@ public:
 	void setLambda(double value);
 	void setSparsity(double value);
 	void setUseVisibleGaussian(bool flag);
+	void setUseHiddenGaussian(bool flag);
 	void setDoRaoBlackwell(bool flag);
 	void setDoSampleVisible(bool flag);
 	void setDoSampleBatch(bool flag);
@@ -92,11 +98,11 @@ public:
 	void setNormalizeData(bool flag);
 	void setDoLearnVariance(bool flag);
 	void setNumGibbs(size_t value);
+	void setMiniBatchSize(size_t size);
 	void setMuWeights(double value);
 	void setMuSparsity(double value);
 	void setMomentum(double value);
 	MatrixXd const& getHiddenBatch();
-	MatrixXd const& getVisibleBatch();
 	MatrixXd const& getBatch();
 	void updateHiddenBatch();
 	Params const& params();
@@ -104,12 +110,13 @@ public:
 private:
 	Weights &m_w;
 	MatrixXd const &m_batch;
-	MatrixXd m_v;
 	MatrixXd m_h;
 	RowVectorXd m_variableSigma;
 	noise_gen_t m_noise;
 	double m_progress;
 	Params m_params;
+	void noiseGaussian(MatrixXd &dst);
+	void noiseUniform(MatrixXd &dst);
 	
 protected:
 	virtual void onProgressChanged()