- added param sparsity

- added gaussian visible unit
- added param sigma decay
- RBM modi and params are set using members
- use sigma instead of variance


git-svn-id: http://moon:8086/svn/software/trunk/projects/RBM@25 b431acfa-c32f-4a4a-93f1-934dc6c82436

RBM.jucer

@@ -17,7 +17,7 @@
     </GROUP>
   </MAINGROUP>
   <EXPORTFORMATS>
-    <LINUX_MAKE targetFolder="Builds/Linux">
+    <LINUX_MAKE targetFolder="Builds/Linux" extraCompilerFlags="">
       <CONFIGURATIONS>
         <CONFIGURATION name="Debug" libraryPath="/usr/X11R6/lib/" isDebug="1" optimisation="1"
                        targetName="RBM" headerPath="/usr/local/include/eigen3"/>

Source/Layer.hpp

@@ -65,24 +65,25 @@ public:
 		m_states.fill(value);
 	}
 
-	void probsUpdateLogistic(Layer &layer, Weights &weights, double lambda = 1.0, double variance = 1.0)
+	void probsUpdateLogistic(Layer &layer, Weights &weights, double lambda, double sigma)
 	{
 		uint32_t i;
+
 		for (i=0; i < m_numUnits; i++)
 		{
-			m_probs(i) = lambda/variance*accum(layer, weights, i);
+			m_probs(i) = lambda/sigma*accum(layer, weights, i);
 		}
 		logSigmoid(m_probs);
 	}
 
-	void probsUpdateGaussian(Layer &layer, Weights &weights, double lambda, double variance)
+	void probsUpdateGaussian(Layer &layer, Weights &weights, double lambda, double sigma)
 	{
 		uint32_t i;
 		for (i=0; i < m_numUnits; i++)
 		{
 			m_probs(i) = lambda*accum(layer, weights, i);
 		}
-		gaussProb(m_probs, variance);
+		gaussProb(m_probs, sigma);
 	}
 
 	void statesUpdateStochastic()
@@ -97,6 +98,16 @@ public:
 		}
 	}
 
+	void sampleGaussian(Layer &layer, Weights &weights, double lambda, double sigma)
+	{
+		uint32_t i;
+
+		for (i=0; i < m_numUnits; i++)
+		{
+			m_states(i) = Noise_Gaussian(&m_noise, 0, sigma) + lambda*accum(layer, weights, i);
+		}
+	}
+
 	VectorXd& probs()
 	{
 		return m_probs;
@@ -133,17 +144,18 @@ protected:
 		}
 	}
 
-	void gaussProb(const VectorXd &x, double var)
+	void gaussProb(const VectorXd &x, double sigma)
 	{
-		double k = 1.0/sqrt(var*2*3.14159265359);
+		double var = sigma*sigma;
-		double mu = 0;
+		double k = 1.0/sqrt(2*3.14159265359*var);
+		double mu = 1;
 
 		uint32_t i;
 
 		for (i=0; i < m_numUnits; i++)
 		{
-			double x2 = ((double)x[i]-mu) * ((double)x[i]-mu);
+			double x2 = ((double)x[i]-mu);
-			m_probs[i] = k*exp(-x2/(2*var));
+			m_probs[i] = k*exp(-x2*x2/(2*var));
 		}
 	}
 };

Source/MainComponent.cpp

@@ -52,14 +52,12 @@ void mylog(const char* format, ...)
 
 //[/MiscUserDefs]
 
-using namespace Eigen;
-using namespace std;
 //==============================================================================
 MainComponent::MainComponent ()
     : m_layers(this),
       m_pRbm(nullptr),
-      Draw(nullptr),
+      DrawTraining(nullptr),
-      Draw2(nullptr),
+      DrawReconstruction(nullptr),
       DrawWeights(nullptr),
       DrawHidden(nullptr)
 {
@@ -92,7 +90,7 @@ MainComponent::MainComponent ()
     WeightsSlider->addListener (this);
 
     addAndMakeVisible (numEpochslabel = new Label ("Num Epochs label",
-                                                   TRANS("99999")));
+                                                   TRANS("100")));
     numEpochslabel->setFont (Font (15.00f, Font::plain));
     numEpochslabel->setJustificationType (Justification::centred);
     numEpochslabel->setEditable (true, true, false);
@@ -101,7 +99,7 @@ MainComponent::MainComponent ()
     numEpochslabel->addListener (this);
 
     addAndMakeVisible (learningRateLabel = new Label ("Learning Rate label",
-                                                      TRANS("0.001")));
+                                                      TRANS("0.03")));
     learningRateLabel->setFont (Font (15.00f, Font::plain));
     learningRateLabel->setJustificationType (Justification::centred);
     learningRateLabel->setEditable (true, true, false);
@@ -136,7 +134,7 @@ MainComponent::MainComponent ()
     createButton->addListener (this);
 
     addAndMakeVisible (projectNameLabel = new Label ("Project Name label",
-                                                     TRANS("ProjectName")));
+                                                     TRANS("TestPrj")));
     projectNameLabel->setFont (Font (15.00f, Font::plain));
     projectNameLabel->setJustificationType (Justification::centred);
     projectNameLabel->setEditable (true, true, false);
@@ -203,6 +201,50 @@ MainComponent::MainComponent ()
     rbmReduceVarianceToggleButton->setButtonText (TRANS("Reduce Variance"));
     rbmReduceVarianceToggleButton->addListener (this);
 
+    addAndMakeVisible (lambdaLabel = new Label ("Lambda label",
+                                                TRANS("1.0")));
+    lambdaLabel->setFont (Font (15.00f, Font::plain));
+    lambdaLabel->setJustificationType (Justification::centred);
+    lambdaLabel->setEditable (true, true, false);
+    lambdaLabel->setColour (TextEditor::textColourId, Colours::black);
+    lambdaLabel->setColour (TextEditor::backgroundColourId, Colour (0x00000000));
+    lambdaLabel->addListener (this);
+
+    addAndMakeVisible (sigmaLabel = new Label ("Sigma label",
+                                               TRANS("0.5")));
+    sigmaLabel->setFont (Font (15.00f, Font::plain));
+    sigmaLabel->setJustificationType (Justification::centred);
+    sigmaLabel->setEditable (true, true, false);
+    sigmaLabel->setColour (TextEditor::textColourId, Colours::black);
+    sigmaLabel->setColour (TextEditor::backgroundColourId, Colour (0x00000000));
+    sigmaLabel->addListener (this);
+
+    addAndMakeVisible (rbmUseVisibleGaussianToggleButton = new ToggleButton ("rbmUseVisibleGaussian toggle button"));
+    rbmUseVisibleGaussianToggleButton->setButtonText (TRANS("Use gaussian visible"));
+    rbmUseVisibleGaussianToggleButton->addListener (this);
+
+    addAndMakeVisible (rbmDoSparseToggleButton = new ToggleButton ("rbmDoSparse toggle button"));
+    rbmDoSparseToggleButton->setButtonText (TRANS("Do sparse"));
+    rbmDoSparseToggleButton->addListener (this);
+
+    addAndMakeVisible (sparsityLabel = new Label ("Sparsity label",
+                                                  TRANS("0.05")));
+    sparsityLabel->setFont (Font (15.00f, Font::plain));
+    sparsityLabel->setJustificationType (Justification::centred);
+    sparsityLabel->setEditable (true, true, false);
+    sparsityLabel->setColour (TextEditor::textColourId, Colours::black);
+    sparsityLabel->setColour (TextEditor::backgroundColourId, Colour (0x00000000));
+    sparsityLabel->addListener (this);
+
+    addAndMakeVisible (sigmaDecayLabel = new Label ("SigmaDecay label",
+                                                    TRANS("1.0")));
+    sigmaDecayLabel->setFont (Font (15.00f, Font::plain));
+    sigmaDecayLabel->setJustificationType (Justification::centred);
+    sigmaDecayLabel->setEditable (true, true, false);
+    sigmaDecayLabel->setColour (TextEditor::textColourId, Colours::black);
+    sigmaDecayLabel->setColour (TextEditor::backgroundColourId, Colour (0x00000000));
+    sigmaDecayLabel->addListener (this);
+
 
     //[UserPreSize]
 	m_vNumX = 16;
@@ -217,19 +259,12 @@ MainComponent::MainComponent ()
 
     //[/UserPreSize]
 
-    setSize (600, 400);
+    setSize (800, 600);
 
 
     //[Constructor] You can add your own custom stuff here..
 
-	projectNameLabel->setText(String("TestPrj"), dontSendNotification );
+    //[/Constructor]
-	numEpochslabel->setText(String(100), dontSendNotification );
-	learningRateLabel->setText(String(0.1), dontSendNotification );
-	rbmUseExpectationsToggleButton->setToggleState(false, sendNotification);
-	rbmDoRaoBlackwellToggleButton->setToggleState(false, sendNotification);
-	rbmDoRobbinsMonroToggleButton->setToggleState(false, sendNotification);
-	rbmReduceVarianceToggleButton->setToggleState(false, sendNotification);
-	//[/Constructor]
 }
 
 MainComponent::~MainComponent()
@@ -263,11 +298,17 @@ MainComponent::~MainComponent()
     rbmDoRaoBlackwellToggleButton = nullptr;
     rbmDoRobbinsMonroToggleButton = nullptr;
     rbmReduceVarianceToggleButton = nullptr;
+    lambdaLabel = nullptr;
+    sigmaLabel = nullptr;
+    rbmUseVisibleGaussianToggleButton = nullptr;
+    rbmDoSparseToggleButton = nullptr;
+    sparsityLabel = nullptr;
+    sigmaDecayLabel = nullptr;
 
 
     //[Destructor]. You can add your own custom destruction code here..
-    Draw = nullptr;
+    DrawTraining = nullptr;
-    Draw2 = nullptr;
+    DrawReconstruction = nullptr;
     DrawWeights = nullptr;
     DrawHidden = nullptr;
     m_pRbm = nullptr;
@@ -283,41 +324,83 @@ void MainComponent::paint (Graphics& g)
 
     g.fillAll (Colours::white);
 
+    g.setColour (Colours::black);
+    g.setFont (Font (15.00f, Font::plain));
+    g.drawText (TRANS("Sigma"),
+                32, 306, 80, 14,
+                Justification::centred, true);
+
+    g.setColour (Colours::black);
+    g.setFont (Font (15.00f, Font::plain));
+    g.drawText (TRANS("Lambda"),
+                120, 306, 80, 14,
+                Justification::centred, true);
+
+    g.setColour (Colours::black);
+    g.setFont (Font (15.00f, Font::plain));
+    g.drawText (TRANS("Num. epochs"),
+                32, 354, 80, 14,
+                Justification::centred, true);
+
+    g.setColour (Colours::black);
+    g.setFont (Font (15.00f, Font::plain));
+    g.drawText (TRANS("Alpha"),
+                120, 354, 80, 14,
+                Justification::centred, true);
+
+    g.setColour (Colours::black);
+    g.setFont (Font (15.00f, Font::plain));
+    g.drawText (TRANS("Sparsity"),
+                208, 306, 80, 14,
+                Justification::centred, true);
+
+    g.setColour (Colours::black);
+    g.setFont (Font (15.00f, Font::plain));
+    g.drawText (TRANS("Sigma decay"),
+                296, 306, 96, 14,
+                Justification::centred, true);
+
     //[UserPaint] Add your own custom painting code here..
     //[/UserPaint]
 }
 
 void MainComponent::resized()
 {
-    trainButton->setBounds (120, 312, 72, 24);
+    trainButton->setBounds (120, 408, 72, 24);
     addButton->setBounds (24, 160, 72, 24);
-    patterSlider->setBounds (224, 312, 184, 24);
+    patterSlider->setBounds (224, 408, 184, 24);
-    reconstructButton->setBounds (24, 312, 72, 24);
+    reconstructButton->setBounds (24, 408, 72, 24);
-    ShakeButton->setBounds (120, 352, 72, 24);
+    ShakeButton->setBounds (120, 448, 72, 24);
-    WeightsSlider->setBounds (224, 352, 184, 24);
+    WeightsSlider->setBounds (224, 448, 184, 24);
-    numEpochslabel->setBounds (24, 280, 72, 24);
+    numEpochslabel->setBounds (32, 368, 72, 24);
-    learningRateLabel->setBounds (120, 280, 72, 24);
+    learningRateLabel->setBounds (120, 368, 72, 24);
     testButton->setBounds (120, 160, 72, 24);
-    numVisibleLabel->setBounds (440, 256, 72, 24);
+    numVisibleLabel->setBounds (456, 328, 72, 24);
-    numHiddenLabel->setBounds (488, 288, 72, 24);
+    numHiddenLabel->setBounds (504, 360, 72, 24);
-    createButton->setBounds (488, 320, 72, 24);
+    createButton->setBounds (504, 392, 72, 24);
-    projectNameLabel->setBounds (472, 224, 96, 24);
+    projectNameLabel->setBounds (488, 296, 96, 24);
-    loadButton->setBounds (440, 192, 72, 24);
+    loadButton->setBounds (456, 264, 72, 24);
-    saveButton->setBounds (520, 192, 72, 24);
+    saveButton->setBounds (536, 264, 72, 24);
-    numVisibleYLabel->setBounds (512, 256, 72, 24);
+    numVisibleYLabel->setBounds (528, 328, 72, 24);
     loadTrainingButton->setBounds (440, 48, 72, 24);
     saveTrainingButton->setBounds (520, 48, 72, 24);
     clearTrainingButton->setBounds (520, 88, 72, 24);
     removeTrainingButton->setBounds (440, 88, 72, 24);
-    numGibbsSlider->setBounds (224, 272, 184, 24);
+    numGibbsSlider->setBounds (224, 368, 184, 24);
-    reconstructEquButton->setBounds (24, 352, 72, 24);
+    reconstructEquButton->setBounds (24, 448, 72, 24);
     rbmUseExpectationsToggleButton->setBounds (24, 232, 128, 24);
     rbmDoRaoBlackwellToggleButton->setBounds (24, 200, 112, 24);
     rbmDoRobbinsMonroToggleButton->setBounds (168, 232, 120, 24);
     rbmReduceVarianceToggleButton->setBounds (168, 200, 128, 24);
+    lambdaLabel->setBounds (120, 320, 72, 24);
+    sigmaLabel->setBounds (32, 320, 72, 24);
+    rbmUseVisibleGaussianToggleButton->setBounds (168, 264, 160, 24);
+    rbmDoSparseToggleButton->setBounds (24, 264, 128, 24);
+    sparsityLabel->setBounds (208, 320, 72, 24);
+    sigmaDecayLabel->setBounds (304, 320, 72, 24);
     //[UserResized] Add your own custom resize handling here..
-    Draw->setBounds (16, 16, 100, 100);
+    DrawTraining->setBounds (16, 16, 100, 100);
-    Draw2->setBounds (110+16, 16, 100, 100);
+    DrawReconstruction->setBounds (110+16, 16, 100, 100);
     DrawWeights->setBounds (220+16, 16, 100, 100);
     DrawHidden->setBounds (16, 16+110, 320, 20);
     //[/UserResized]
@@ -331,34 +414,38 @@ void MainComponent::buttonClicked (Button* buttonThatWasClicked)
     if (buttonThatWasClicked == trainButton)
     {
         //[UserButtonCode_trainButton] -- add your button handler code here..
-    	m_pRbm->train(m_layers, numEpochslabel->getText().getIntValue(), learningRateLabel->getText().getFloatValue(), m_numGibbs, m_rbmUseExpectations, m_rbmDoRaoBlackwell, m_rbmReduceEstimatorVariance, m_rbmDoRobbinsMonro);
+    	m_pRbm->train(m_layers, numEpochslabel->getText().getIntValue(), learningRateLabel->getText().getFloatValue(), m_numGibbs);
+    	redrawReconstruction();
+    	redrawWeights((int)WeightsSlider->getValue());
         //[/UserButtonCode_trainButton]
     }
     else if (buttonThatWasClicked == addButton)
     {
         //[UserButtonCode_addButton] -- add your button handler code here..
-		Draw2->setData(Draw->getData());
+		DrawReconstruction->setData(DrawTraining->getData());
-		m_layers.add(&Draw->getData(), m_vNumX*m_vNumY);
+		m_layers.add(&DrawTraining->getData(), m_vNumX*m_vNumY);
 		patterSlider->setRange (0, m_layers.getSize()-1, 1);
         //[/UserButtonCode_addButton]
     }
     else if (buttonThatWasClicked == reconstructButton)
     {
         //[UserButtonCode_reconstructButton] -- add your button handler code here..
-		Draw2->setData(m_pRbm->toVisible(m_pRbm->toHidden(Draw->getData())));
+    	redrawReconstruction();
-    	DrawHidden->setData(m_pRbm->toHidden(Draw->getData()));
         //[/UserButtonCode_reconstructButton]
     }
     else if (buttonThatWasClicked == ShakeButton)
     {
         //[UserButtonCode_ShakeButton] -- add your button handler code here..
     	m_weights.shuffle(0.01);
+    	redrawReconstruction();
+    	redrawWeights((int)WeightsSlider->getValue());
         //[/UserButtonCode_ShakeButton]
     }
     else if (buttonThatWasClicked == testButton)
     {
         //[UserButtonCode_testButton] -- add your button handler code here..
-		Draw->setData(Draw2->getData());
+		DrawTraining->setData(DrawReconstruction->getData());
+		redrawReconstruction();
         //[/UserButtonCode_testButton]
     }
     else if (buttonThatWasClicked == createButton)
@@ -367,9 +454,6 @@ void MainComponent::buttonClicked (Button* buttonThatWasClicked)
 		m_vNumX = m_vNumX_next;
 		m_vNumY = m_vNumY_next;
 		m_hNum = m_hNum_next;
-		numVisibleLabel->setText(String(m_vNumX), dontSendNotification );
-		numVisibleYLabel->setText(String(m_vNumY), dontSendNotification );
-		numHiddenLabel->setText(String(m_hNum), dontSendNotification );
 		m_weights.setUnits(m_vNumX*m_vNumY, m_hNum);
 		create();
         //[/UserButtonCode_createButton]
@@ -378,6 +462,8 @@ void MainComponent::buttonClicked (Button* buttonThatWasClicked)
     {
         //[UserButtonCode_loadButton] -- add your button handler code here..
 		load();
+		redrawReconstruction();
+    	redrawWeights((int)WeightsSlider->getValue());
         //[/UserButtonCode_loadButton]
     }
     else if (buttonThatWasClicked == saveButton)
@@ -391,7 +477,8 @@ void MainComponent::buttonClicked (Button* buttonThatWasClicked)
         //[UserButtonCode_loadTrainingButton] -- add your button handler code here..
 		m_layers.clear();
 		m_layers.load((String(getBaseDir() + String(".trainingStates.dat"))).toUTF8());
-        //[/UserButtonCode_loadTrainingButton]
+		redrawReconstruction();
+       //[/UserButtonCode_loadTrainingButton]
     }
     else if (buttonThatWasClicked == saveTrainingButton)
     {
@@ -414,45 +501,56 @@ void MainComponent::buttonClicked (Button* buttonThatWasClicked)
     else if (buttonThatWasClicked == reconstructEquButton)
     {
         //[UserButtonCode_reconstructEquButton] -- add your button handler code here..
-  //  	Draw2->setData(m_pRbm->expectVisible(Draw->getData(), 100));
-
     	uint32_t i;
     	VectorXd V, H;
 
-    	V = Draw->getData();
+    	V = DrawTraining->getData();
     	for (i=0; i < 100; i++)
     	{
         	H = m_pRbm->toHidden(V);
         	DrawHidden->setData(H);
         	V = m_pRbm->toVisible(H);
-    		Draw2->setData(V);
+    		DrawReconstruction->setData(V);
     	}
         //[/UserButtonCode_reconstructEquButton]
     }
     else if (buttonThatWasClicked == rbmUseExpectationsToggleButton)
     {
         //[UserButtonCode_rbmUseExpectationsToggleButton] -- add your button handler code here..
-    	m_rbmUseExpectations = buttonThatWasClicked->getToggleState();
+    	m_pRbm->setUseExpectations(buttonThatWasClicked->getToggleState());
         //[/UserButtonCode_rbmUseExpectationsToggleButton]
     }
     else if (buttonThatWasClicked == rbmDoRaoBlackwellToggleButton)
     {
         //[UserButtonCode_rbmDoRaoBlackwellToggleButton] -- add your button handler code here..
-    	m_rbmDoRaoBlackwell = buttonThatWasClicked->getToggleState();
+    	m_pRbm->setDoRaoBlackwell(buttonThatWasClicked->getToggleState());
         //[/UserButtonCode_rbmDoRaoBlackwellToggleButton]
     }
     else if (buttonThatWasClicked == rbmDoRobbinsMonroToggleButton)
     {
         //[UserButtonCode_rbmDoRobbinsMonroToggleButton] -- add your button handler code here..
-    	m_rbmDoRobbinsMonro = buttonThatWasClicked->getToggleState();
+    	m_pRbm->setDoRobbinsMonro(buttonThatWasClicked->getToggleState());
         //[/UserButtonCode_rbmDoRobbinsMonroToggleButton]
     }
     else if (buttonThatWasClicked == rbmReduceVarianceToggleButton)
     {
         //[UserButtonCode_rbmReduceVarianceToggleButton] -- add your button handler code here..
-    	m_rbmReduceEstimatorVariance = buttonThatWasClicked->getToggleState();
+    	m_pRbm->setUseProbsForHiddenReconstruction(buttonThatWasClicked->getToggleState());
         //[/UserButtonCode_rbmReduceVarianceToggleButton]
     }
+    else if (buttonThatWasClicked == rbmUseVisibleGaussianToggleButton)
+    {
+        //[UserButtonCode_rbmUseVisibleGaussianToggleButton] -- add your button handler code here..
+    	m_pRbm->setUseVisibleGaussian(buttonThatWasClicked->getToggleState());
+    	redrawReconstruction();
+        //[/UserButtonCode_rbmUseVisibleGaussianToggleButton]
+    }
+    else if (buttonThatWasClicked == rbmDoSparseToggleButton)
+    {
+        //[UserButtonCode_rbmDoSparseToggleButton] -- add your button handler code here..
+    	m_pRbm->setDoSparse(buttonThatWasClicked->getToggleState());
+        //[/UserButtonCode_rbmDoSparseToggleButton]
+    }
 
     //[UserbuttonClicked_Post]
     //[/UserbuttonClicked_Post]
@@ -469,33 +567,14 @@ void MainComponent::sliderValueChanged (Slider* sliderThatWasMoved)
 		if (m_layers.getSize() > 0)
 		{
 			VisibleLayer &p = (VisibleLayer&)m_layers.getAt((int)sliderThatWasMoved->getValue());
-			Draw2->setData(p.states());
+			DrawReconstruction->setData(p.states());
 		}
         //[/UserSliderCode_patterSlider]
     }
     else if (sliderThatWasMoved == WeightsSlider)
     {
         //[UserSliderCode_WeightsSlider] -- add your slider handling code here..
-		VectorXd w = m_weights.weights().col((int)sliderThatWasMoved->getValue());
+    	redrawWeights((int)sliderThatWasMoved->getValue());
-		VectorXd temp = w;
-		double min = +1E12;
-		double max = -1E12;
-
-		for (int i=0; i < m_vNumX*m_vNumY; i++)
-		{
-    		min = std::min<double>(min, (double)temp[i]);
-    		max = std::max<double>(max, (double)temp[i]);
-		}
-		for (int i=0; i < m_vNumX*m_vNumY; i++)
-		{
-			temp[i] -= min;
-		}
-		for (int i=0; i < m_vNumX*m_vNumY; i++)
-		{
-			temp[i] /= (max-min);
-		}
-
-		DrawWeights->setData(temp);
         //[/UserSliderCode_WeightsSlider]
     }
     else if (sliderThatWasMoved == numGibbsSlider)
@@ -547,6 +626,32 @@ void MainComponent::labelTextChanged (Label* labelThatHasChanged)
         m_vNumY_next = labelThatHasChanged->getText().getIntValue();
         //[/UserLabelCode_numVisibleYLabel]
     }
+    else if (labelThatHasChanged == lambdaLabel)
+    {
+        //[UserLabelCode_lambdaLabel] -- add your label text handling code here..
+    	m_pRbm->setLambda(labelThatHasChanged->getText().getFloatValue());
+    	redrawReconstruction();
+        //[/UserLabelCode_lambdaLabel]
+    }
+    else if (labelThatHasChanged == sigmaLabel)
+    {
+        //[UserLabelCode_sigmaLabel] -- add your label text handling code here..
+    	m_pRbm->setSigma(labelThatHasChanged->getText().getFloatValue());
+    	redrawReconstruction();
+        //[/UserLabelCode_sigmaLabel]
+    }
+    else if (labelThatHasChanged == sparsityLabel)
+    {
+        //[UserLabelCode_sparsityLabel] -- add your label text handling code here..
+    	m_pRbm->setSparsity(labelThatHasChanged->getText().getFloatValue());
+        //[/UserLabelCode_sparsityLabel]
+    }
+    else if (labelThatHasChanged == sigmaDecayLabel)
+    {
+        //[UserLabelCode_sigmaDecayLabel] -- add your label text handling code here..
+    	m_pRbm->setSigmaDecay(labelThatHasChanged->getText().getFloatValue());
+        //[/UserLabelCode_sigmaDecayLabel]
+    }
 
     //[UserlabelTextChanged_Post]
     //[/UserlabelTextChanged_Post]
@@ -571,22 +676,37 @@ void MainComponent::save ()
 
 void MainComponent::create()
 {
-	Draw = nullptr;
+	DrawTraining = nullptr;
-	Draw2 = nullptr;
+	DrawReconstruction = nullptr;
 	DrawWeights = nullptr;
 	DrawHidden = nullptr;
 	m_pRbm = nullptr;
-	addAndMakeVisible (Draw = new DrawComponent (m_vNumX, m_vNumY));
+
-	Draw->setListener(this);
+	WeightsSlider->setRange(0, m_hNum-1, 1);
-    addAndMakeVisible (Draw2 = new DrawComponent (m_vNumX, m_vNumY));
+
+	addAndMakeVisible (DrawTraining = new DrawComponent (m_vNumX, m_vNumY));
+	DrawTraining->setListener(this);
+    addAndMakeVisible (DrawReconstruction = new DrawComponent (m_vNumX, m_vNumY));
     addAndMakeVisible (DrawWeights = new DrawComponent (m_vNumX, m_vNumY));
     addAndMakeVisible (DrawHidden = new DrawComponent (m_hNum, 1));
 	DrawHidden->setListener(this);
 	numVisibleLabel->setText(String(m_vNumX), dontSendNotification );
 	numVisibleYLabel->setText(String(m_vNumY), dontSendNotification );
 	numHiddenLabel->setText(String(m_hNum), dontSendNotification );
+
 	m_pRbm = new Rbm(m_weights, this);
-	WeightsSlider->setRange(0, m_hNum-1, 1);
+	m_pRbm->setUseExpectations(rbmUseExpectationsToggleButton->getToggleState());
+	m_pRbm->setDoRaoBlackwell(rbmDoRaoBlackwellToggleButton->getToggleState());
+	m_pRbm->setDoRobbinsMonro(rbmDoRobbinsMonroToggleButton->getToggleState());
+	m_pRbm->setUseProbsForHiddenReconstruction(rbmReduceVarianceToggleButton->getToggleState());
+	m_pRbm->setUseVisibleGaussian(rbmUseVisibleGaussianToggleButton->getToggleState());
+	m_pRbm->setDoSparse(rbmDoSparseToggleButton->getToggleState());
+
+	m_pRbm->setLambda(lambdaLabel->getText().getFloatValue());
+	m_pRbm->setSigma(sigmaLabel->getText().getFloatValue());
+	m_pRbm->setSigmaDecay(sigmaDecayLabel->getText().getFloatValue());
+	m_pRbm->setSparsity(sparsityLabel->getText().getFloatValue());
+
 	resized();
 }
 
@@ -617,15 +737,45 @@ void MainComponent::onDraw(DrawComponent &obj)
 {
 	if (&obj == DrawHidden)
 	{
-		Draw2->setData(m_pRbm->toVisible(obj.getData()));
+		DrawReconstruction->setData(m_pRbm->toVisible(obj.getData()));
 	}
-	if (&obj == Draw)
+	if (&obj == DrawTraining)
 	{
-		Draw2->setData(m_pRbm->toVisible(m_pRbm->toHidden(obj.getData())));
+		DrawReconstruction->setData(m_pRbm->toVisible(m_pRbm->toHidden(obj.getData())));
     	DrawHidden->setData(m_pRbm->toHidden(obj.getData()));
 	}
 }
 
+void MainComponent::redrawReconstruction()
+{
+	DrawHidden->setData(m_pRbm->toHidden(DrawTraining->getData()));
+	DrawReconstruction->setData(m_pRbm->toVisible(DrawHidden->getData()));
+}
+
+void MainComponent::redrawWeights(int index)
+{
+	VectorXd w = m_weights.weights().col(index);
+	VectorXd temp = w;
+	double min = +1E12;
+	double max = -1E12;
+
+	for (int i=0; i < m_vNumX*m_vNumY; i++)
+	{
+		min = std::min<double>(min, (double)temp[i]);
+		max = std::max<double>(max, (double)temp[i]);
+	}
+	for (int i=0; i < m_vNumX*m_vNumY; i++)
+	{
+		temp[i] -= min;
+	}
+	for (int i=0; i < m_vNumX*m_vNumY; i++)
+	{
+		temp[i] /= (max-min);
+	}
+
+	DrawWeights->setData(temp);
+}
+
 //[/MiscUserCode]
 
 
@@ -640,69 +790,82 @@ BEGIN_JUCER_METADATA
 
 <JUCER_COMPONENT documentType="Component" className="MainComponent" componentName=""
                  parentClasses="public Component, public LayerArrayListener&lt;VisibleLayer&gt;, public RbmListener, public DrawListener"
-                 constructorParams="" variableInitialisers="m_layers(this),&#10;m_pRbm(nullptr)&#10;Draw(nullptr),&#10;Draw2(nullptr),&#10;DrawWeights(nullptr),&#10;DrawHidden(nullptr)"
+                 constructorParams="" variableInitialisers="m_layers(this),&#10;m_pRbm(nullptr)&#10;DrawTraining(nullptr),&#10;DrawReconstruction(nullptr),&#10;DrawWeights(nullptr),&#10;DrawHidden(nullptr)"
                  snapPixels="8" snapActive="1" snapShown="1" overlayOpacity="0.330"
-                 fixedSize="1" initialWidth="600" initialHeight="400">
+                 fixedSize="1" initialWidth="800" initialHeight="600">
-  <BACKGROUND backgroundColour="ffffffff"/>
+  <BACKGROUND backgroundColour="ffffffff">
+    <TEXT pos="32 306 80 14" fill="solid: ff000000" hasStroke="0" text="Sigma"
+          fontname="Default font" fontsize="15" bold="0" italic="0" justification="36"/>
+    <TEXT pos="120 306 80 14" fill="solid: ff000000" hasStroke="0" text="Lambda"
+          fontname="Default font" fontsize="15" bold="0" italic="0" justification="36"/>
+    <TEXT pos="32 354 80 14" fill="solid: ff000000" hasStroke="0" text="Num. epochs"
+          fontname="Default font" fontsize="15" bold="0" italic="0" justification="36"/>
+    <TEXT pos="120 354 80 14" fill="solid: ff000000" hasStroke="0" text="Alpha"
+          fontname="Default font" fontsize="15" bold="0" italic="0" justification="36"/>
+    <TEXT pos="208 306 80 14" fill="solid: ff000000" hasStroke="0" text="Sparsity"
+          fontname="Default font" fontsize="15" bold="0" italic="0" justification="36"/>
+    <TEXT pos="296 306 96 14" fill="solid: ff000000" hasStroke="0" text="Sigma decay"
+          fontname="Default font" fontsize="15" bold="0" italic="0" justification="36"/>
+  </BACKGROUND>
   <TEXTBUTTON name="Train button" id="7909d74b5522987c" memberName="trainButton"
-              virtualName="" explicitFocusOrder="0" pos="120 312 72 24" buttonText="Train"
+              virtualName="" explicitFocusOrder="0" pos="120 408 72 24" buttonText="Train"
               connectedEdges="0" needsCallback="1" radioGroupId="0"/>
   <TEXTBUTTON name="Add button" id="4609f7b526ec1aef" memberName="addButton"
               virtualName="" explicitFocusOrder="0" pos="24 160 72 24" buttonText="Add"
               connectedEdges="0" needsCallback="1" radioGroupId="0"/>
   <SLIDER name="Pattern slider" id="c3e0a2c816db81d1" memberName="patterSlider"
-          virtualName="" explicitFocusOrder="0" pos="224 312 184 24" min="0"
+          virtualName="" explicitFocusOrder="0" pos="224 408 184 24" min="0"
           max="0" int="1" style="LinearHorizontal" textBoxPos="TextBoxLeft"
           textBoxEditable="1" textBoxWidth="80" textBoxHeight="20" skewFactor="1"/>
   <TEXTBUTTON name="Reconstruct button" id="c1901d121a5819d6" memberName="reconstructButton"
-              virtualName="" explicitFocusOrder="0" pos="24 312 72 24" buttonText="Reconstruct"
+              virtualName="" explicitFocusOrder="0" pos="24 408 72 24" buttonText="Reconstruct"
               connectedEdges="0" needsCallback="1" radioGroupId="0"/>
   <TEXTBUTTON name="Shake button" id="7fa4964a9a9ed199" memberName="ShakeButton"
-              virtualName="" explicitFocusOrder="0" pos="120 352 72 24" buttonText="Shake"
+              virtualName="" explicitFocusOrder="0" pos="120 448 72 24" buttonText="Shake"
               connectedEdges="0" needsCallback="1" radioGroupId="0"/>
   <SLIDER name="Weights slider" id="699075a1fc01458a" memberName="WeightsSlider"
-          virtualName="" explicitFocusOrder="0" pos="224 352 184 24" min="0"
+          virtualName="" explicitFocusOrder="0" pos="224 448 184 24" min="0"
           max="0" int="1" style="LinearHorizontal" textBoxPos="TextBoxLeft"
           textBoxEditable="1" textBoxWidth="80" textBoxHeight="20" skewFactor="1"/>
   <LABEL name="Num Epochs label" id="b23ae372ee931474" memberName="numEpochslabel"
-         virtualName="" explicitFocusOrder="0" pos="24 280 72 24" edTextCol="ff000000"
+         virtualName="" explicitFocusOrder="0" pos="32 368 72 24" edTextCol="ff000000"
-         edBkgCol="0" labelText="99999" editableSingleClick="1" editableDoubleClick="1"
+         edBkgCol="0" labelText="100" editableSingleClick="1" editableDoubleClick="1"
          focusDiscardsChanges="0" fontname="Default font" fontsize="15"
          bold="0" italic="0" justification="36"/>
   <LABEL name="Learning Rate label" id="49611a27914e910d" memberName="learningRateLabel"
-         virtualName="" explicitFocusOrder="0" pos="120 280 72 24" edTextCol="ff000000"
+         virtualName="" explicitFocusOrder="0" pos="120 368 72 24" edTextCol="ff000000"
-         edBkgCol="0" labelText="0.001" editableSingleClick="1" editableDoubleClick="1"
+         edBkgCol="0" labelText="0.03" editableSingleClick="1" editableDoubleClick="1"
          focusDiscardsChanges="0" fontname="Default font" fontsize="15"
          bold="0" italic="0" justification="36"/>
   <TEXTBUTTON name="Test button" id="a1e6fed732ae7ee1" memberName="testButton"
               virtualName="" explicitFocusOrder="0" pos="120 160 72 24" buttonText="Test"
               connectedEdges="0" needsCallback="1" radioGroupId="0"/>
   <LABEL name="Num Visible label" id="60acd702770b77dd" memberName="numVisibleLabel"
-         virtualName="" explicitFocusOrder="0" pos="440 256 72 24" edTextCol="ff000000"
+         virtualName="" explicitFocusOrder="0" pos="456 328 72 24" edTextCol="ff000000"
          edBkgCol="0" labelText="99999" editableSingleClick="1" editableDoubleClick="1"
          focusDiscardsChanges="0" fontname="Default font" fontsize="15"
          bold="0" italic="0" justification="36"/>
   <LABEL name="Num Hidden label" id="8a56c0419bd3ce86" memberName="numHiddenLabel"
-         virtualName="" explicitFocusOrder="0" pos="488 288 72 24" edTextCol="ff000000"
+         virtualName="" explicitFocusOrder="0" pos="504 360 72 24" edTextCol="ff000000"
          edBkgCol="0" labelText="99999" editableSingleClick="1" editableDoubleClick="1"
          focusDiscardsChanges="0" fontname="Default font" fontsize="15"
          bold="0" italic="0" justification="36"/>
   <TEXTBUTTON name="Create button" id="6c71593a581844eb" memberName="createButton"
-              virtualName="" explicitFocusOrder="0" pos="488 320 72 24" buttonText="Create"
+              virtualName="" explicitFocusOrder="0" pos="504 392 72 24" buttonText="Create"
               connectedEdges="0" needsCallback="1" radioGroupId="0"/>
   <LABEL name="Project Name label" id="b4e855167abdac6b" memberName="projectNameLabel"
-         virtualName="" explicitFocusOrder="0" pos="472 224 96 24" edTextCol="ff000000"
+         virtualName="" explicitFocusOrder="0" pos="488 296 96 24" edTextCol="ff000000"
-         edBkgCol="0" labelText="ProjectName" editableSingleClick="1"
+         edBkgCol="0" labelText="TestPrj" editableSingleClick="1" editableDoubleClick="1"
-         editableDoubleClick="1" focusDiscardsChanges="0" fontname="Default font"
+         focusDiscardsChanges="0" fontname="Default font" fontsize="15"
-         fontsize="15" bold="0" italic="0" justification="36"/>
+         bold="0" italic="0" justification="36"/>
   <TEXTBUTTON name="Load button" id="4e606c77b12e7d11" memberName="loadButton"
-              virtualName="" explicitFocusOrder="0" pos="440 192 72 24" buttonText="Load"
+              virtualName="" explicitFocusOrder="0" pos="456 264 72 24" buttonText="Load"
               connectedEdges="0" needsCallback="1" radioGroupId="0"/>
   <TEXTBUTTON name="Save button" id="c435e774a51e39bf" memberName="saveButton"
-              virtualName="" explicitFocusOrder="0" pos="520 192 72 24" buttonText="Save"
+              virtualName="" explicitFocusOrder="0" pos="536 264 72 24" buttonText="Save"
               connectedEdges="0" needsCallback="1" radioGroupId="0"/>
   <LABEL name="Num Visible Y label" id="101a4f77ca2445ba" memberName="numVisibleYLabel"
-         virtualName="" explicitFocusOrder="0" pos="512 256 72 24" edTextCol="ff000000"
+         virtualName="" explicitFocusOrder="0" pos="528 328 72 24" edTextCol="ff000000"
          edBkgCol="0" labelText="99999" editableSingleClick="1" editableDoubleClick="1"
          focusDiscardsChanges="0" fontname="Default font" fontsize="15"
          bold="0" italic="0" justification="36"/>
@@ -719,11 +882,11 @@ BEGIN_JUCER_METADATA
               virtualName="" explicitFocusOrder="0" pos="440 88 72 24" buttonText="Remove T"
               connectedEdges="0" needsCallback="1" radioGroupId="0"/>
   <SLIDER name="Num. Gibbs slider" id="fdd9d5ca4e05e18c" memberName="numGibbsSlider"
-          virtualName="" explicitFocusOrder="0" pos="224 272 184 24" min="1"
+          virtualName="" explicitFocusOrder="0" pos="224 368 184 24" min="1"
           max="10" int="1" style="LinearHorizontal" textBoxPos="TextBoxLeft"
           textBoxEditable="1" textBoxWidth="80" textBoxHeight="20" skewFactor="1"/>
   <TEXTBUTTON name="Reconstruct Equilibrium button" id="208a587f5556207" memberName="reconstructEquButton"
-              virtualName="" explicitFocusOrder="0" pos="24 352 72 24" buttonText="Reconst Equ."
+              virtualName="" explicitFocusOrder="0" pos="24 448 72 24" buttonText="Reconst Equ."
               connectedEdges="0" needsCallback="1" radioGroupId="0"/>
   <TOGGLEBUTTON name="rbmUseExpectations toggle button" id="62884e37cd027719"
                 memberName="rbmUseExpectationsToggleButton" virtualName="" explicitFocusOrder="0"
@@ -741,6 +904,33 @@ BEGIN_JUCER_METADATA
                 memberName="rbmReduceVarianceToggleButton" virtualName="" explicitFocusOrder="0"
                 pos="168 200 128 24" buttonText="Reduce Variance" connectedEdges="0"
                 needsCallback="1" radioGroupId="0" state="0"/>
+  <LABEL name="Lambda label" id="c51a5dfb587390b4" memberName="lambdaLabel"
+         virtualName="" explicitFocusOrder="0" pos="120 320 72 24" edTextCol="ff000000"
+         edBkgCol="0" labelText="1.0" editableSingleClick="1" editableDoubleClick="1"
+         focusDiscardsChanges="0" fontname="Default font" fontsize="15"
+         bold="0" italic="0" justification="36"/>
+  <LABEL name="Sigma label" id="68d7dad32ed357f8" memberName="sigmaLabel"
+         virtualName="" explicitFocusOrder="0" pos="32 320 72 24" edTextCol="ff000000"
+         edBkgCol="0" labelText="0.5" editableSingleClick="1" editableDoubleClick="1"
+         focusDiscardsChanges="0" fontname="Default font" fontsize="15"
+         bold="0" italic="0" justification="36"/>
+  <TOGGLEBUTTON name="rbmUseVisibleGaussian toggle button" id="c89b32a4deaa7f19"
+                memberName="rbmUseVisibleGaussianToggleButton" virtualName=""
+                explicitFocusOrder="0" pos="168 264 160 24" buttonText="Use gaussian visible"
+                connectedEdges="0" needsCallback="1" radioGroupId="0" state="0"/>
+  <TOGGLEBUTTON name="rbmDoSparse toggle button" id="ed3f2602ab35b5f2" memberName="rbmDoSparseToggleButton"
+                virtualName="" explicitFocusOrder="0" pos="24 264 128 24" buttonText="Do sparse"
+                connectedEdges="0" needsCallback="1" radioGroupId="0" state="0"/>
+  <LABEL name="Sparsity label" id="f4efe44dc0ca4894" memberName="sparsityLabel"
+         virtualName="" explicitFocusOrder="0" pos="208 320 72 24" edTextCol="ff000000"
+         edBkgCol="0" labelText="0.05" editableSingleClick="1" editableDoubleClick="1"
+         focusDiscardsChanges="0" fontname="Default font" fontsize="15"
+         bold="0" italic="0" justification="36"/>
+  <LABEL name="SigmaDecay label" id="d5f2f5fffdafd926" memberName="sigmaDecayLabel"
+         virtualName="" explicitFocusOrder="0" pos="304 320 72 24" edTextCol="ff000000"
+         edBkgCol="0" labelText="1.0" editableSingleClick="1" editableDoubleClick="1"
+         focusDiscardsChanges="0" fontname="Default font" fontsize="15"
+         bold="0" italic="0" justification="36"/>
 </JUCER_COMPONENT>
 
 END_JUCER_METADATA

Source/MainComponent.h

@@ -66,8 +66,8 @@ private:
     //[UserVariables]   -- You can add your own custom variables in this section.
     Weights m_weights;
 	ScopedPointer<Rbm> m_pRbm;
-    ScopedPointer<DrawComponent> Draw;
+    ScopedPointer<DrawComponent> DrawTraining;
-    ScopedPointer<DrawComponent> Draw2;
+    ScopedPointer<DrawComponent> DrawReconstruction;
     ScopedPointer<DrawComponent> DrawWeights;
     ScopedPointer<DrawComponent> DrawHidden;
 	uint32_t m_vNumX;
@@ -85,13 +85,11 @@ private:
 	void onChanged(const LayerArray<VisibleLayer> &obj);
 	void onEpochTrained(const Rbm &obj);
 	void onDraw(DrawComponent &obj);
+	void redrawReconstruction();
+	void redrawWeights(int index);
 	String m_baseDir;
 	double m_trainingProgress;
 	uint32_t m_numGibbs;
-	bool m_rbmUseExpectations;
-	bool m_rbmDoRaoBlackwell;
-	bool m_rbmDoRobbinsMonro;
-	bool m_rbmReduceEstimatorVariance;
     //[/UserVariables]
 
     //==============================================================================
@@ -121,6 +119,12 @@ private:
     ScopedPointer<ToggleButton> rbmDoRaoBlackwellToggleButton;
     ScopedPointer<ToggleButton> rbmDoRobbinsMonroToggleButton;
     ScopedPointer<ToggleButton> rbmReduceVarianceToggleButton;
+    ScopedPointer<Label> lambdaLabel;
+    ScopedPointer<Label> sigmaLabel;
+    ScopedPointer<ToggleButton> rbmUseVisibleGaussianToggleButton;
+    ScopedPointer<ToggleButton> rbmDoSparseToggleButton;
+    ScopedPointer<Label> sparsityLabel;
+    ScopedPointer<Label> sigmaDecayLabel;
 
 
     //==============================================================================

Source/Rbm.hpp

@@ -37,6 +37,16 @@ public:
 	: m_w(weights)
 	, m_pListener(pListener)
 	, m_progress(0)
+	, m_sigma(1.0)
+	, m_sigmaDecay(1.0)
+	, m_lambda(1.0)
+	, m_sparsity(0)
+	, m_useVisibleGaussian(false)
+	, m_useExpectations(false)
+	, m_doRaoBlackwell(false)
+	, m_useProbsForHiddenReconstruction(false)
+	, m_doRobbinsMonro(false)
+	, m_doSparse(false)
 	{
 		Noise_Init(&m_noise, 0x32727155);
 	}
@@ -63,43 +73,35 @@ public:
 	{
 		m_w.hiddenBias().array() += mu*h.states().array();
 	}
-//#define RBM_SPARSE
+
-	void train(LayerArray<VisibleLayer> &vt, uint32_t numEpochs, double mu, uint32_t numGibbs = 1, bool useExpectations = false, bool doRaoBlackwell = false, bool useProbsForHiddenReconstruction = false, bool doRobbinsMonro = false)
+	void train(LayerArray<VisibleLayer> &vt, uint32_t numEpochs, double mu, uint32_t numGibbs = 1, double sigmaMin = 0.05)
 	{
 		uint32_t t, i;
 		uint32_t epoch;
 		uint32_t gibbs;
+		double sigma;
 		VisibleLayer v(m_w.getNumVisible());
 		HiddenLayer h(m_w.getNumHidden());
 		HiddenLayer *pH;
 		LayerArray<HiddenLayer> ht(vt.getSize(), m_w.getNumHidden());
 
+		sigma = m_sigma;
 		Weights w = m_w;
 
 		double dProgress = 1.0/numEpochs;
 		m_progress = 0;
 
-#ifdef RBM_SPARSE
+		if (m_useExpectations)
-		const double lambda = 0.05;
-		const double variance = 0.4;
-		const double penalty = 0.05;
-#else
-		const double lambda = 1.0;
-		const double variance = 1.0;
-		const double penalty = 0.0;
-#endif
-
-		if (useExpectations)
 		{
 			mu /= vt.getSize();
 		}
 
-		if (doRobbinsMonro)
+		if (m_doRobbinsMonro)
 		{
 			for (i=0; i < vt.getSize(); i++)
 			{
 				// Create hidden layer base on training data
-				ht[i].probsUpdateLogistic(vt[i], w, lambda, variance);
+				ht[i].probsUpdateLogistic(vt[i], w, m_lambda, sigma);
 			}
 		}
 
@@ -107,11 +109,12 @@ public:
 		{
 			for (i=0; i < vt.getSize(); i++)
 			{
-				t = (uint32_t)(0.5 + (vt.getSize()-1)*Noise_Uniform(&m_noise, 0.5));
+				//t = (uint32_t)(0.5 + (vt.getSize()-1)*Noise_Uniform(&m_noise, 0.5));
-				h.probsUpdateLogistic(vt[t], w, lambda, variance);
+				t = i;
+				h.probsUpdateLogistic(vt[t], w, m_lambda, sigma);
 
 				// Create hidden layer base on training data
-				if (doRobbinsMonro)
+				if (m_doRobbinsMonro)
 				{
 					pH = &ht[t];
 				}
@@ -121,7 +124,7 @@ public:
 				}
 
 				// Update weights (positive phase)
-				if (doRaoBlackwell)
+				if (m_doRaoBlackwell)
 				{
 					pH->states() = pH->probs();
 				}
@@ -131,33 +134,46 @@ public:
 				}
 				weightsUpdate(vt[t], h, +mu);
 				visibleBiasUpdate(vt[t], +mu);
-				hiddenBiasUpdate(h, +mu);
+				if (!m_doSparse)
+				{
+					hiddenBiasUpdate(h, +mu);
+				}
 
 				for (gibbs=0; gibbs < numGibbs; gibbs++)
 				{
 					pH->statesUpdateStochastic();
 
 					// Create visible reconstruction (a fantasy...)
-#ifdef RBM_SPARSE
+					if (m_useProbsForHiddenReconstruction)
-					v.probsUpdateGaussian(*pH, w, lambda, variance);
-#else
-					v.probsUpdateLogistic(*pH, w);
-#endif
-
-					if (useProbsForHiddenReconstruction)
 					{
+						if (m_useVisibleGaussian)
+						{
+							v.probsUpdateGaussian(*pH, w, m_lambda, sigma);
+						}
+						else
+						{
+							v.probsUpdateLogistic(*pH, w, m_lambda, sigma);
+						}
 						v.states() = v.probs();
 					}
 					else
 					{
-						v.statesUpdateStochastic();
+						if (m_useVisibleGaussian)
+						{
+							v.sampleGaussian(*pH, w, m_lambda, sigma);
+						}
+						else
+						{
+							v.probsUpdateLogistic(*pH, w, m_lambda, sigma);
+							v.statesUpdateStochastic();
+						}
 					}
-
 					// Create hidden reconstruction
-					pH->probsUpdateLogistic(v, w, lambda, variance);
+					pH->probsUpdateLogistic(v, w, m_lambda, sigma);
 				}
+
 				// Update weights (negative phase)
-				if (doRaoBlackwell)
+				if (m_doRaoBlackwell)
 				{
 					pH->states() = pH->probs();
 				}
@@ -167,33 +183,40 @@ public:
 				}
 				weightsUpdate(v, *pH, -mu);
 				visibleBiasUpdate(v, -mu);
-				hiddenBiasUpdate(*pH, -mu);
+				if (!m_doSparse)
-
+				{
-				if (!useExpectations)
+					hiddenBiasUpdate(*pH, -mu);
+				}
+				if (!m_useExpectations)
 				{
 					w = m_w;
 				}
+			} // TrainingSize
 
+			if (m_useExpectations)
+			{
+				w = m_w;
 			}
-#ifdef RBM_SPARSE
+
+			if (m_doSparse)
 			{
 				HiddenLayer th(m_w.getNumHidden());
-				VectorXd m(th.states());
+				VectorXd m(m_w.getNumHidden());
 				m.fill(0);
+
 				for (i=0; i < vt.getSize(); i++)
 				{
-					m += expectHidden(vt[i].states(), lambda, variance, 10);
+					th.probsUpdateLogistic(vt[i], w, m_lambda, sigma);
+					m += th.probs();
 				}
-				m.array() = penalty - m.array();
+				m /= i;
-				m *= 1.0/vt.getSize();
+				m.array() = m_sparsity - m.array();
 				th.states() = m;
 				hiddenBiasUpdate(th, -mu);
 			}
-#endif
+			if (sigma > sigmaMin)
-
-			if (useExpectations)
 			{
-				w = m_w;
+				sigma *= m_sigmaDecay;
 			}
 
 			m_progress += dProgress;
@@ -201,7 +224,8 @@ public:
 			{
 				m_pListener->onEpochTrained(*this);
 			}
-		}
+
+		} // Number of epochs
 	}
 
 	double getProgress() const
@@ -241,7 +265,7 @@ public:
 		for (j=0; j < vts.getSize(); j++)
 		{
 			h[j].setNumUnits(m_w.getNumHidden());
-			h[j].probsUpdateLogistic(vts.getAt(j), m_w);
+			h[j].probsUpdateLogistic(vts.getAt(j), m_w, m_lambda, m_sigma);
 //			h[j].statesAssignfromProbs();
 			h[j].statesUpdateStochastic();
 		}
@@ -280,7 +304,7 @@ public:
 		// Reconstruct
 		for (i=0; i < vts.getSize(); i++)
 		{
-			vts.getAt(i).probsUpdateLogistic(h[i], m_w);
+			vts.getAt(i).probsUpdateLogistic(h[i], m_w, m_lambda, m_sigma);
 		}
 
 		printf("A fantasy... (v^, t>)\n");
@@ -292,27 +316,32 @@ public:
 		delete [] h;
 	}
 
-	VectorXd toHidden(const VectorXd& visible, double lambda = 1.0, double variance = 1.0)
+	VectorXd toHidden(const VectorXd& visible)
 	{
 		HiddenLayer th(m_w.getNumHidden());
 		VisibleLayer tv(m_w.getNumVisible(), (const VectorXd*)&visible);
 
-		th.probsUpdateLogistic(tv, m_w, lambda, variance);
+		th.probsUpdateLogistic(tv, m_w, m_lambda, m_sigma);
 
 		return th.probs();
 	}
 
-	VectorXd toVisible(const VectorXd& hidden, double lambda = 1.0, double variance = 1.0)
+	VectorXd toVisible(const VectorXd& hidden)
 	{
 		HiddenLayer th(m_w.getNumHidden(), (const VectorXd*)&hidden);
 		VisibleLayer tv(m_w.getNumVisible());
-
+		if (m_useVisibleGaussian)
-		tv.probsUpdateLogistic(th, m_w, lambda, variance);
+		{
-
+			tv.probsUpdateGaussian(th, m_w, m_lambda, m_sigma);
+		}
+		else
+		{
+			tv.probsUpdateLogistic(th, m_w, m_lambda, m_sigma);
+		}
 		return tv.probs();
 	}
 
-	VectorXd expectHidden(VectorXd visible, uint32_t numIter, double lambda = 1.0, double variance = 1.0)
+	VectorXd expectHidden(VectorXd visible, uint32_t numIter)
 	{
       	uint32_t i;
 		VisibleLayer v(m_w.getNumVisible(), (const VectorXd*)&visible);
@@ -320,14 +349,21 @@ public:
 
     	for (i=0; i < numIter; i++)
     	{
-    		h.probsUpdateLogistic(v, (Weights&)m_w, lambda, variance);
+    		h.probsUpdateLogistic(v, (Weights&)m_w, m_lambda, m_sigma);
-    		v.probsUpdateGaussian(h, (Weights&)m_w, lambda, variance);
+    		if (m_useVisibleGaussian)
+    		{
+    			v.probsUpdateGaussian(h, (Weights&)m_w, m_lambda, m_sigma);
+    		}
+    		else
+    		{
+    			v.probsUpdateLogistic(h, (Weights&)m_w, m_lambda, m_sigma);
+    		}
     	}
 
 		return h.probs();
 	}
 
-   VectorXd expectVisible(VectorXd visible, uint32_t numIter, double lambda = 1.0, double variance = 1.0)
+   VectorXd expectVisible(VectorXd visible, uint32_t numIter)
 	{
       	uint32_t i;
 		VisibleLayer v(m_w.getNumVisible(), (const VectorXd*)&visible);
@@ -335,18 +371,135 @@ public:
 
     	for (i=0; i < numIter; i++)
     	{
-    		h.probsUpdateLogistic(v, (Weights&)m_w, lambda, variance);
+    		h.probsUpdateLogistic(v, (Weights&)m_w, m_lambda, m_sigma);
-    		v.probsUpdateLogistic(h, (Weights&)m_w, lambda, variance);
+    		if (m_useVisibleGaussian)
+    		{
+    			v.probsUpdateGaussian(h, (Weights&)m_w, m_lambda, m_sigma);
+    		}
+    		else
+    		{
+    			v.probsUpdateLogistic(h, (Weights&)m_w, m_lambda, m_sigma);
+    		}
     	}
 
 		return v.probs();
 	}
 
+   void setSigma(double value)
+   {
+	   m_sigma = value;
+   }
+
+   void setSigmaDecay(double value)
+   {
+	   m_sigmaDecay = value;
+   }
+
+   void setLambda(double value)
+   {
+	   m_lambda = value;
+   }
+
+   void setSparsity(double value)
+   {
+	   m_sparsity = value;
+   }
+
+   void setUseVisibleGaussian(bool flag)
+   {
+	   m_useVisibleGaussian = flag;
+   }
+
+   void setUseExpectations(bool flag)
+   {
+	   m_useExpectations = flag;
+   }
+
+   void setDoRaoBlackwell(bool flag)
+   {
+	   m_doRaoBlackwell = flag;
+   }
+
+   void setUseProbsForHiddenReconstruction(bool flag)
+   {
+	   m_useProbsForHiddenReconstruction = flag;
+   }
+
+   void setDoRobbinsMonro(bool flag)
+   {
+	   m_doRobbinsMonro = flag;
+   }
+
+   void setDoSparse(bool flag)
+   {
+	   m_doSparse = flag;
+   }
+
+   double getSparsity()
+   {
+	   return m_sparsity;
+   }
+
+   double getSigma()
+   {
+	   return m_sigma;
+   }
+
+   double getSigmaDecay()
+   {
+	   return m_sigmaDecay;
+   }
+
+   double getLambda()
+   {
+	   return m_lambda;
+   }
+
+   bool getUseVisibleGaussian()
+   {
+	   return m_useVisibleGaussian;
+   }
+
+   bool getUseExpectations()
+   {
+	   return m_useExpectations;
+   }
+
+   bool getDoRaoBlackwell()
+   {
+	   return m_doRaoBlackwell;
+   }
+
+   bool getUseProbsForHiddenReconstruction()
+   {
+	   return m_useProbsForHiddenReconstruction;
+   }
+
+   bool getRobbinsMonro()
+   {
+	   return m_doRobbinsMonro;
+   }
+
+   bool getDoSparse()
+   {
+	   return m_doSparse;
+   }
+
 private:
 	Weights &m_w;
 	RbmListener *m_pListener;
 	noise_gen_t m_noise;
 	double m_progress;
+	double m_sigma;
+	double m_sigmaDecay;
+	double m_lambda;
+	double m_sparsity;
+	bool m_useVisibleGaussian;
+	bool m_useExpectations;
+	bool m_doRaoBlackwell;
+	bool m_useProbsForHiddenReconstruction;
+	bool m_doRobbinsMonro;
+	bool m_doSparse;
 
 };