- RBM: changed calcualation of pre-weight update data in RBM

- always use expectations
- removed "Use Expectations Button"
- removed Robbins-Monro
- added sparsity learning rate
- added momentum
- added weight decay
- added Slider as progress bar



git-svn-id: http://moon:8086/svn/software/trunk/projects/RBM@30 b431acfa-c32f-4a4a-93f1-934dc6c82436

Source/MainComponent.cpp

@@ -72,7 +72,7 @@ MainComponent::MainComponent ()
 
     addAndMakeVisible (patterSlider = new Slider ("Pattern slider"));
     patterSlider->setRange (0, 0, 1);
-    patterSlider->setSliderStyle (Slider::LinearHorizontal);
+    patterSlider->setSliderStyle (Slider::LinearBar);
     patterSlider->setTextBoxStyle (Slider::TextBoxLeft, false, 80, 20);
     patterSlider->addListener (this);
 
@@ -86,7 +86,7 @@ MainComponent::MainComponent ()
 
     addAndMakeVisible (WeightsSlider = new Slider ("Weights slider"));
     WeightsSlider->setRange (0, 0, 1);
-    WeightsSlider->setSliderStyle (Slider::LinearHorizontal);
+    WeightsSlider->setSliderStyle (Slider::LinearBar);
     WeightsSlider->setTextBoxStyle (Slider::TextBoxLeft, false, 80, 20);
     WeightsSlider->addListener (this);
 
@@ -100,7 +100,7 @@ MainComponent::MainComponent ()
     numEpochslabel->addListener (this);
 
     addAndMakeVisible (learningRateLabel = new Label ("Learning Rate label",
-                                                      TRANS("0.03")));
+                                                      TRANS("0.01")));
     learningRateLabel->setFont (Font (15.00f, Font::plain));
     learningRateLabel->setJustificationType (Justification::centred);
     learningRateLabel->setEditable (true, true, false);
@@ -178,7 +178,7 @@ MainComponent::MainComponent ()
 
     addAndMakeVisible (numGibbsSlider = new Slider ("Num. Gibbs slider"));
     numGibbsSlider->setRange (1, 10, 1);
-    numGibbsSlider->setSliderStyle (Slider::LinearHorizontal);
+    numGibbsSlider->setSliderStyle (Slider::LinearBar);
     numGibbsSlider->setTextBoxStyle (Slider::TextBoxLeft, false, 80, 20);
     numGibbsSlider->addListener (this);
 
@@ -186,18 +186,10 @@ MainComponent::MainComponent ()
     reconstructEquButton->setButtonText (TRANS("Reconst Equ."));
     reconstructEquButton->addListener (this);
 
-    addAndMakeVisible (rbmUseExpectationsToggleButton = new ToggleButton ("rbmUseExpectations toggle button"));
-    rbmUseExpectationsToggleButton->setButtonText (TRANS("Use Expectations"));
-    rbmUseExpectationsToggleButton->addListener (this);
-
     addAndMakeVisible (rbmDoRaoBlackwellToggleButton = new ToggleButton ("rbmDoRaoBlackwell toggle button"));
     rbmDoRaoBlackwellToggleButton->setButtonText (TRANS("Rao-Blackwell"));
     rbmDoRaoBlackwellToggleButton->addListener (this);
 
-    addAndMakeVisible (rbmDoRobbinsMonroToggleButton = new ToggleButton ("rbmDoRobbinsMonro toggle button"));
-    rbmDoRobbinsMonroToggleButton->setButtonText (TRANS("Robbins-Monro"));
-    rbmDoRobbinsMonroToggleButton->addListener (this);
-
     addAndMakeVisible (rbmReduceVarianceToggleButton = new ToggleButton ("rbmReduceVariance toggle button"));
     rbmReduceVarianceToggleButton->setButtonText (TRANS("Reduce Variance"));
     rbmReduceVarianceToggleButton->addListener (this);
@@ -212,7 +204,7 @@ MainComponent::MainComponent ()
     lambdaLabel->addListener (this);
 
     addAndMakeVisible (sigmaLabel = new Label ("Sigma label",
-                                               TRANS("0.5")));
+                                               TRANS("1.0")));
     sigmaLabel->setFont (Font (15.00f, Font::plain));
     sigmaLabel->setJustificationType (Justification::centred);
     sigmaLabel->setEditable (true, true, false);
@@ -246,17 +238,44 @@ MainComponent::MainComponent ()
     sigmaDecayLabel->setColour (TextEditor::backgroundColourId, Colour (0x00000000));
     sigmaDecayLabel->addListener (this);
 
-    addAndMakeVisible (m_progressBar = new ProgressBar (m_trainingProgress));
+    addAndMakeVisible (weightDecayLabel = new Label ("WeightDecay label",
-    m_progressBar->setName ("Progress Bar");
+                                                     TRANS("0.0002")));
+    weightDecayLabel->setFont (Font (15.00f, Font::plain));
+    weightDecayLabel->setJustificationType (Justification::centred);
+    weightDecayLabel->setEditable (true, true, false);
+    weightDecayLabel->setColour (TextEditor::textColourId, Colours::black);
+    weightDecayLabel->setColour (TextEditor::backgroundColourId, Colour (0x00000000));
+    weightDecayLabel->addListener (this);
+
+    addAndMakeVisible (m_progressBarSlider = new Slider ("progressBar slider"));
+    m_progressBarSlider->setRange (0, 100, 1);
+    m_progressBarSlider->setSliderStyle (Slider::LinearBar);
+    m_progressBarSlider->setTextBoxStyle (Slider::TextBoxLeft, true, 80, 20);
+    m_progressBarSlider->addListener (this);
+
+    addAndMakeVisible (momentumLabel = new Label ("Momentum label",
+                                                  TRANS("0.5")));
+    momentumLabel->setFont (Font (15.00f, Font::plain));
+    momentumLabel->setJustificationType (Justification::centred);
+    momentumLabel->setEditable (true, true, false);
+    momentumLabel->setColour (TextEditor::textColourId, Colours::black);
+    momentumLabel->setColour (TextEditor::backgroundColourId, Colour (0x00000000));
+    momentumLabel->addListener (this);
+
+    addAndMakeVisible (sparsityLearningRateLabel = new Label ("SparsityLearn label",
+                                                              TRANS("0.01")));
+    sparsityLearningRateLabel->setFont (Font (15.00f, Font::plain));
+    sparsityLearningRateLabel->setJustificationType (Justification::centred);
+    sparsityLearningRateLabel->setEditable (true, true, false);
+    sparsityLearningRateLabel->setColour (TextEditor::textColourId, Colours::black);
+    sparsityLearningRateLabel->setColour (TextEditor::backgroundColourId, Colour (0x00000000));
+    sparsityLearningRateLabel->addListener (this);
 
 
     //[UserPreSize]
 	m_vNumX = 16;
 	m_vNumY = 16;
 	m_hNum = 64;
-	m_vNumX_next = 16;
-	m_vNumY_next = 16;
-	m_hNum_next = 64;
 	m_numGibbs = 1;
 	m_weights.setUnits(m_vNumX*m_vNumY, m_hNum);
 	create();
@@ -267,8 +286,7 @@ MainComponent::MainComponent ()
 
 
     //[Constructor] You can add your own custom stuff here..
-    m_trainingProgress = 0;
+    m_progressBarSlider->setValue(100);
-    m_progressBar->setTextToDisplay("Finished");
     //[/Constructor]
 }
 
@@ -299,9 +317,7 @@ MainComponent::~MainComponent()
     removeTrainingButton = nullptr;
     numGibbsSlider = nullptr;
     reconstructEquButton = nullptr;
-    rbmUseExpectationsToggleButton = nullptr;
     rbmDoRaoBlackwellToggleButton = nullptr;
-    rbmDoRobbinsMonroToggleButton = nullptr;
     rbmReduceVarianceToggleButton = nullptr;
     lambdaLabel = nullptr;
     sigmaLabel = nullptr;
@@ -309,7 +325,10 @@ MainComponent::~MainComponent()
     rbmDoSparseToggleButton = nullptr;
     sparsityLabel = nullptr;
     sigmaDecayLabel = nullptr;
-    m_progressBar = nullptr;
+    weightDecayLabel = nullptr;
+    m_progressBarSlider = nullptr;
+    momentumLabel = nullptr;
+    sparsityLearningRateLabel = nullptr;
 
 
     //[Destructor]. You can add your own custom destruction code here..
@@ -366,6 +385,24 @@ void MainComponent::paint (Graphics& g)
                 296, 306, 96, 14,
                 Justification::centred, true);
 
+    g.setColour (Colours::black);
+    g.setFont (Font (15.00f, Font::plain));
+    g.drawText (TRANS("Weight decay"),
+                296, 354, 96, 14,
+                Justification::centred, true);
+
+    g.setColour (Colours::black);
+    g.setFont (Font (15.00f, Font::plain));
+    g.drawText (TRANS("Momentum"),
+                208, 354, 80, 14,
+                Justification::centred, true);
+
+    g.setColour (Colours::black);
+    g.setFont (Font (15.00f, Font::plain));
+    g.drawText (TRANS("Sparsity Alpha"),
+                408, 306, 96, 14,
+                Justification::centred, true);
+
     //[UserPaint] Add your own custom painting code here..
     //[/UserPaint]
 }
@@ -374,37 +411,38 @@ void MainComponent::resized()
 {
     trainButton->setBounds (120, 408, 72, 24);
     addButton->setBounds (24, 160, 72, 24);
-    patterSlider->setBounds (224, 408, 184, 24);
+    patterSlider->setBounds (224, 448, 184, 24);
     reconstructButton->setBounds (24, 408, 72, 24);
     ShakeButton->setBounds (120, 448, 72, 24);
-    WeightsSlider->setBounds (224, 448, 184, 24);
+    WeightsSlider->setBounds (224, 488, 184, 24);
     numEpochslabel->setBounds (32, 368, 72, 24);
     learningRateLabel->setBounds (120, 368, 72, 24);
     testButton->setBounds (120, 160, 72, 24);
-    numVisibleLabel->setBounds (456, 328, 72, 24);
+    numVisibleLabel->setBounds (504, 224, 72, 24);
-    numHiddenLabel->setBounds (504, 360, 72, 24);
+    numHiddenLabel->setBounds (552, 256, 72, 24);
-    createButton->setBounds (504, 392, 72, 24);
+    createButton->setBounds (552, 288, 72, 24);
-    projectNameLabel->setBounds (488, 296, 96, 24);
+    projectNameLabel->setBounds (536, 192, 96, 24);
-    loadButton->setBounds (456, 264, 72, 24);
+    loadButton->setBounds (504, 160, 72, 24);
-    saveButton->setBounds (536, 264, 72, 24);
+    saveButton->setBounds (584, 160, 72, 24);
-    numVisibleYLabel->setBounds (528, 328, 72, 24);
+    numVisibleYLabel->setBounds (576, 224, 72, 24);
-    loadTrainingButton->setBounds (440, 48, 72, 24);
+    loadTrainingButton->setBounds (504, 48, 72, 24);
-    saveTrainingButton->setBounds (520, 48, 72, 24);
+    saveTrainingButton->setBounds (584, 48, 72, 24);
-    clearTrainingButton->setBounds (520, 88, 72, 24);
+    clearTrainingButton->setBounds (584, 88, 72, 24);
-    removeTrainingButton->setBounds (440, 88, 72, 24);
+    removeTrainingButton->setBounds (504, 88, 72, 24);
-    numGibbsSlider->setBounds (224, 368, 184, 24);
+    numGibbsSlider->setBounds (224, 408, 184, 24);
     reconstructEquButton->setBounds (24, 448, 72, 24);
-    rbmUseExpectationsToggleButton->setBounds (24, 232, 128, 24);
     rbmDoRaoBlackwellToggleButton->setBounds (24, 200, 112, 24);
-    rbmDoRobbinsMonroToggleButton->setBounds (168, 232, 120, 24);
     rbmReduceVarianceToggleButton->setBounds (168, 200, 128, 24);
     lambdaLabel->setBounds (120, 320, 72, 24);
     sigmaLabel->setBounds (32, 320, 72, 24);
-    rbmUseVisibleGaussianToggleButton->setBounds (168, 264, 160, 24);
+    rbmUseVisibleGaussianToggleButton->setBounds (168, 232, 160, 24);
-    rbmDoSparseToggleButton->setBounds (24, 264, 128, 24);
+    rbmDoSparseToggleButton->setBounds (24, 232, 128, 24);
     sparsityLabel->setBounds (208, 320, 72, 24);
     sigmaDecayLabel->setBounds (304, 320, 72, 24);
-    m_progressBar->setBounds (224, 160, 168, 24);
+    weightDecayLabel->setBounds (304, 368, 72, 24);
+    m_progressBarSlider->setBounds (216, 160, 184, 24);
+    momentumLabel->setBounds (208, 368, 72, 24);
+    sparsityLearningRateLabel->setBounds (416, 320, 72, 24);
     //[UserResized] Add your own custom resize handling here..
     DrawTraining->setBounds (16, 16, 100, 100);
     DrawReconstruction->setBounds (110+16, 16, 100, 100);
@@ -459,9 +497,10 @@ void MainComponent::buttonClicked (Button* buttonThatWasClicked)
     else if (buttonThatWasClicked == createButton)
     {
         //[UserButtonCode_createButton] -- add your button handler code here..
-		m_vNumX = m_vNumX_next;
+		m_vNumX = numVisibleLabel->getText().getIntValue();
-		m_vNumY = m_vNumY_next;
+		m_vNumY = numVisibleYLabel->getText().getIntValue();
-		m_hNum = m_hNum_next;
+		m_hNum = numHiddenLabel->getText().getIntValue();
+
 		m_weights.setUnits(m_vNumX*m_vNumY, m_hNum);
 		create();
         //[/UserButtonCode_createButton]
@@ -522,24 +561,12 @@ void MainComponent::buttonClicked (Button* buttonThatWasClicked)
     	}
         //[/UserButtonCode_reconstructEquButton]
     }
-    else if (buttonThatWasClicked == rbmUseExpectationsToggleButton)
-    {
-        //[UserButtonCode_rbmUseExpectationsToggleButton] -- add your button handler code here..
-    	m_pRbm->setUseExpectations(buttonThatWasClicked->getToggleState());
-        //[/UserButtonCode_rbmUseExpectationsToggleButton]
-    }
     else if (buttonThatWasClicked == rbmDoRaoBlackwellToggleButton)
     {
         //[UserButtonCode_rbmDoRaoBlackwellToggleButton] -- add your button handler code here..
     	m_pRbm->setDoRaoBlackwell(buttonThatWasClicked->getToggleState());
         //[/UserButtonCode_rbmDoRaoBlackwellToggleButton]
     }
-    else if (buttonThatWasClicked == rbmDoRobbinsMonroToggleButton)
-    {
-        //[UserButtonCode_rbmDoRobbinsMonroToggleButton] -- add your button handler code here..
-    	m_pRbm->setDoRobbinsMonro(buttonThatWasClicked->getToggleState());
-        //[/UserButtonCode_rbmDoRobbinsMonroToggleButton]
-    }
     else if (buttonThatWasClicked == rbmReduceVarianceToggleButton)
     {
         //[UserButtonCode_rbmReduceVarianceToggleButton] -- add your button handler code here..
@@ -550,6 +577,23 @@ void MainComponent::buttonClicked (Button* buttonThatWasClicked)
     {
         //[UserButtonCode_rbmUseVisibleGaussianToggleButton] -- add your button handler code here..
     	m_pRbm->setUseVisibleGaussian(buttonThatWasClicked->getToggleState());
+   		rbmReduceVarianceToggleButton->setEnabled(!buttonThatWasClicked->getToggleState());
+
+   		if (buttonThatWasClicked->getToggleState())
+   		{
+   	    	rbmReduceVarianceToggleButton_binary = rbmReduceVarianceToggleButton->getToggleState();
+   			rbmReduceVarianceToggleButton->setToggleState(false, sendNotification);
+   			sigmaLabel->setText(String(sigma_gauss, 1), sendNotification);
+			sigmaDecayLabel->setText(String(sigmaDecay_gauss, 1), sendNotification);
+   		}
+   		else
+   		{
+   			sigma_gauss = sigmaLabel->getText().getFloatValue();
+   			sigmaDecay_gauss = sigmaDecayLabel->getText().getFloatValue();
+   			sigmaLabel->setText("1.0", sendNotification);
+			sigmaDecayLabel->setText("1.0", sendNotification);
+   			rbmReduceVarianceToggleButton->setToggleState(rbmReduceVarianceToggleButton_binary, sendNotification);
+   		}
     	redrawReconstruction();
         //[/UserButtonCode_rbmUseVisibleGaussianToggleButton]
     }
@@ -589,9 +633,14 @@ void MainComponent::sliderValueChanged (Slider* sliderThatWasMoved)
     else if (sliderThatWasMoved == numGibbsSlider)
     {
         //[UserSliderCode_numGibbsSlider] -- add your slider handling code here..
-    	m_numGibbs = (uint32_t)sliderThatWasMoved->getValue();
+    	m_pRbm->setNumGibbs((uint32_t)sliderThatWasMoved->getValue());
         //[/UserSliderCode_numGibbsSlider]
     }
+    else if (sliderThatWasMoved == m_progressBarSlider)
+    {
+        //[UserSliderCode_m_progressBarSlider] -- add your slider handling code here..
+        //[/UserSliderCode_m_progressBarSlider]
+    }
 
     //[UsersliderValueChanged_Post]
     //[/UsersliderValueChanged_Post]
@@ -610,18 +659,17 @@ void MainComponent::labelTextChanged (Label* labelThatHasChanged)
     else if (labelThatHasChanged == learningRateLabel)
     {
         //[UserLabelCode_learningRateLabel] -- add your label text handling code here..
+    	m_pRbm->setMuWeights(labelThatHasChanged->getText().getFloatValue());
         //[/UserLabelCode_learningRateLabel]
     }
     else if (labelThatHasChanged == numVisibleLabel)
     {
         //[UserLabelCode_numVisibleLabel] -- add your label text handling code here..
-        m_vNumX_next = labelThatHasChanged->getText().getIntValue();
         //[/UserLabelCode_numVisibleLabel]
     }
     else if (labelThatHasChanged == numHiddenLabel)
     {
         //[UserLabelCode_numHiddenLabel] -- add your label text handling code here..
-        m_hNum_next = labelThatHasChanged->getText().getIntValue();
         //[/UserLabelCode_numHiddenLabel]
     }
     else if (labelThatHasChanged == projectNameLabel)
@@ -632,7 +680,6 @@ void MainComponent::labelTextChanged (Label* labelThatHasChanged)
     else if (labelThatHasChanged == numVisibleYLabel)
     {
         //[UserLabelCode_numVisibleYLabel] -- add your label text handling code here..
-        m_vNumY_next = labelThatHasChanged->getText().getIntValue();
         //[/UserLabelCode_numVisibleYLabel]
     }
     else if (labelThatHasChanged == lambdaLabel)
@@ -661,6 +708,24 @@ void MainComponent::labelTextChanged (Label* labelThatHasChanged)
     	m_pRbm->setSigmaDecay(labelThatHasChanged->getText().getFloatValue());
         //[/UserLabelCode_sigmaDecayLabel]
     }
+    else if (labelThatHasChanged == weightDecayLabel)
+    {
+        //[UserLabelCode_weightDecayLabel] -- add your label text handling code here..
+    	m_pRbm->setWeightDecay(labelThatHasChanged->getText().getFloatValue());
+        //[/UserLabelCode_weightDecayLabel]
+    }
+    else if (labelThatHasChanged == momentumLabel)
+    {
+        //[UserLabelCode_momentumLabel] -- add your label text handling code here..
+    	m_pRbm->setMomentum(labelThatHasChanged->getText().getFloatValue());
+        //[/UserLabelCode_momentumLabel]
+    }
+    else if (labelThatHasChanged == sparsityLearningRateLabel)
+    {
+        //[UserLabelCode_sparsityLearningRateLabel] -- add your label text handling code here..
+    	m_pRbm->setMuSparsity(labelThatHasChanged->getText().getFloatValue());
+        //[/UserLabelCode_sparsityLearningRateLabel]
+    }
 
     //[UserlabelTextChanged_Post]
     //[/UserlabelTextChanged_Post]
@@ -704,9 +769,9 @@ void MainComponent::create()
 	numHiddenLabel->setText(String(m_hNum), dontSendNotification );
 
 	m_pRbm = new Rbm(m_weights, this);
-	m_pRbm->setUseExpectations(rbmUseExpectationsToggleButton->getToggleState());
+	sigma_gauss = 0.5;
+	sigmaDecay_gauss = 0.99;
 	m_pRbm->setDoRaoBlackwell(rbmDoRaoBlackwellToggleButton->getToggleState());
-	m_pRbm->setDoRobbinsMonro(rbmDoRobbinsMonroToggleButton->getToggleState());
 	m_pRbm->setUseProbsForHiddenReconstruction(rbmReduceVarianceToggleButton->getToggleState());
 	m_pRbm->setUseVisibleGaussian(rbmUseVisibleGaussianToggleButton->getToggleState());
 	m_pRbm->setDoSparse(rbmDoSparseToggleButton->getToggleState());
@@ -714,7 +779,10 @@ void MainComponent::create()
 	m_pRbm->setLambda(lambdaLabel->getText().getFloatValue());
 	m_pRbm->setSigma(sigmaLabel->getText().getFloatValue());
 	m_pRbm->setSigmaDecay(sigmaDecayLabel->getText().getFloatValue());
+	m_pRbm->setWeightDecay(weightDecayLabel->getText().getFloatValue());
 	m_pRbm->setSparsity(sparsityLabel->getText().getFloatValue());
+	m_pRbm->setNumGibbs((uint32_t)numGibbsSlider->getValue());
+
 
 	resized();
 }
@@ -738,10 +806,10 @@ void MainComponent::onChanged(const LayerArray<VisibleLayer> &obj)
 
 void MainComponent::onEpochTrained(const Rbm &obj)
 {
-	m_trainingProgress = obj.getProgress();
+//	mylog("Training %f %%\n", obj.getProgress()*100);
-//	mylog("Training %f %%\n", m_trainingProgress*100);
 	redrawReconstruction();
 	redrawWeights((int)WeightsSlider->getValue());
+	m_progressBarSlider->setValue((int)(100*obj.getProgress()), dontSendNotification);
 }
 
 void MainComponent::onDraw(DrawComponent &obj)
@@ -790,11 +858,8 @@ void MainComponent::redrawWeights(int index)
 void MainComponent::run()
 {
 	trainButton->setEnabled(false);
-	m_progressBar->setPercentageDisplay(true);
+	m_pRbm->train(m_layers, numEpochslabel->getText().getIntValue());
-	m_pRbm->train(m_layers, numEpochslabel->getText().getIntValue(), learningRateLabel->getText().getFloatValue(), m_numGibbs);
 	trainButton->setEnabled(true);
-	m_trainingProgress = 0.0;
-	m_progressBar->setTextToDisplay("Finished");
 }
 
 //[/MiscUserCode]
@@ -827,6 +892,12 @@ BEGIN_JUCER_METADATA
           fontname="Default font" fontsize="15" bold="0" italic="0" justification="36"/>
     <TEXT pos="296 306 96 14" fill="solid: ff000000" hasStroke="0" text="Sigma decay"
           fontname="Default font" fontsize="15" bold="0" italic="0" justification="36"/>
+    <TEXT pos="296 354 96 14" fill="solid: ff000000" hasStroke="0" text="Weight decay"
+          fontname="Default font" fontsize="15" bold="0" italic="0" justification="36"/>
+    <TEXT pos="208 354 80 14" fill="solid: ff000000" hasStroke="0" text="Momentum"
+          fontname="Default font" fontsize="15" bold="0" italic="0" justification="36"/>
+    <TEXT pos="408 306 96 14" fill="solid: ff000000" hasStroke="0" text="Sparsity Alpha"
+          fontname="Default font" fontsize="15" bold="0" italic="0" justification="36"/>
   </BACKGROUND>
   <TEXTBUTTON name="Train button" id="7909d74b5522987c" memberName="trainButton"
               virtualName="" explicitFocusOrder="0" pos="120 408 72 24" buttonText="Train"
@@ -835,9 +906,9 @@ BEGIN_JUCER_METADATA
               virtualName="" explicitFocusOrder="0" pos="24 160 72 24" buttonText="Add"
               connectedEdges="0" needsCallback="1" radioGroupId="0"/>
   <SLIDER name="Pattern slider" id="c3e0a2c816db81d1" memberName="patterSlider"
-          virtualName="" explicitFocusOrder="0" pos="224 408 184 24" min="0"
+          virtualName="" explicitFocusOrder="0" pos="224 448 184 24" min="0"
-          max="0" int="1" style="LinearHorizontal" textBoxPos="TextBoxLeft"
+          max="0" int="1" style="LinearBar" textBoxPos="TextBoxLeft" textBoxEditable="1"
-          textBoxEditable="1" textBoxWidth="80" textBoxHeight="20" skewFactor="1"/>
+          textBoxWidth="80" textBoxHeight="20" skewFactor="1"/>
   <TEXTBUTTON name="Reconstruct button" id="c1901d121a5819d6" memberName="reconstructButton"
               virtualName="" explicitFocusOrder="0" pos="24 408 72 24" buttonText="Reconstruct"
               connectedEdges="0" needsCallback="1" radioGroupId="0"/>
@@ -845,9 +916,9 @@ BEGIN_JUCER_METADATA
               virtualName="" explicitFocusOrder="0" pos="120 448 72 24" buttonText="Shake"
               connectedEdges="0" needsCallback="1" radioGroupId="0"/>
   <SLIDER name="Weights slider" id="699075a1fc01458a" memberName="WeightsSlider"
-          virtualName="" explicitFocusOrder="0" pos="224 448 184 24" min="0"
+          virtualName="" explicitFocusOrder="0" pos="224 488 184 24" min="0"
-          max="0" int="1" style="LinearHorizontal" textBoxPos="TextBoxLeft"
+          max="0" int="1" style="LinearBar" textBoxPos="TextBoxLeft" textBoxEditable="1"
-          textBoxEditable="1" textBoxWidth="80" textBoxHeight="20" skewFactor="1"/>
+          textBoxWidth="80" textBoxHeight="20" skewFactor="1"/>
   <LABEL name="Num Epochs label" id="b23ae372ee931474" memberName="numEpochslabel"
          virtualName="" explicitFocusOrder="0" pos="32 368 72 24" edTextCol="ff000000"
          edBkgCol="0" labelText="100" editableSingleClick="1" editableDoubleClick="1"
@@ -855,72 +926,64 @@ BEGIN_JUCER_METADATA
          bold="0" italic="0" justification="36"/>
   <LABEL name="Learning Rate label" id="49611a27914e910d" memberName="learningRateLabel"
          virtualName="" explicitFocusOrder="0" pos="120 368 72 24" edTextCol="ff000000"
-         edBkgCol="0" labelText="0.03" editableSingleClick="1" editableDoubleClick="1"
+         edBkgCol="0" labelText="0.01" editableSingleClick="1" editableDoubleClick="1"
          focusDiscardsChanges="0" fontname="Default font" fontsize="15"
          bold="0" italic="0" justification="36"/>
   <TEXTBUTTON name="Test button" id="a1e6fed732ae7ee1" memberName="testButton"
               virtualName="" explicitFocusOrder="0" pos="120 160 72 24" buttonText="Test"
               connectedEdges="0" needsCallback="1" radioGroupId="0"/>
   <LABEL name="Num Visible label" id="60acd702770b77dd" memberName="numVisibleLabel"
-         virtualName="" explicitFocusOrder="0" pos="456 328 72 24" edTextCol="ff000000"
+         virtualName="" explicitFocusOrder="0" pos="504 224 72 24" edTextCol="ff000000"
          edBkgCol="0" labelText="99999" editableSingleClick="1" editableDoubleClick="1"
          focusDiscardsChanges="0" fontname="Default font" fontsize="15"
          bold="0" italic="0" justification="36"/>
   <LABEL name="Num Hidden label" id="8a56c0419bd3ce86" memberName="numHiddenLabel"
-         virtualName="" explicitFocusOrder="0" pos="504 360 72 24" edTextCol="ff000000"
+         virtualName="" explicitFocusOrder="0" pos="552 256 72 24" edTextCol="ff000000"
          edBkgCol="0" labelText="99999" editableSingleClick="1" editableDoubleClick="1"
          focusDiscardsChanges="0" fontname="Default font" fontsize="15"
          bold="0" italic="0" justification="36"/>
   <TEXTBUTTON name="Create button" id="6c71593a581844eb" memberName="createButton"
-              virtualName="" explicitFocusOrder="0" pos="504 392 72 24" buttonText="Create"
+              virtualName="" explicitFocusOrder="0" pos="552 288 72 24" buttonText="Create"
               connectedEdges="0" needsCallback="1" radioGroupId="0"/>
   <LABEL name="Project Name label" id="b4e855167abdac6b" memberName="projectNameLabel"
-         virtualName="" explicitFocusOrder="0" pos="488 296 96 24" edTextCol="ff000000"
+         virtualName="" explicitFocusOrder="0" pos="536 192 96 24" edTextCol="ff000000"
          edBkgCol="0" labelText="TestPrj" editableSingleClick="1" editableDoubleClick="1"
          focusDiscardsChanges="0" fontname="Default font" fontsize="15"
          bold="0" italic="0" justification="36"/>
   <TEXTBUTTON name="Load button" id="4e606c77b12e7d11" memberName="loadButton"
-              virtualName="" explicitFocusOrder="0" pos="456 264 72 24" buttonText="Load"
+              virtualName="" explicitFocusOrder="0" pos="504 160 72 24" buttonText="Load"
               connectedEdges="0" needsCallback="1" radioGroupId="0"/>
   <TEXTBUTTON name="Save button" id="c435e774a51e39bf" memberName="saveButton"
-              virtualName="" explicitFocusOrder="0" pos="536 264 72 24" buttonText="Save"
+              virtualName="" explicitFocusOrder="0" pos="584 160 72 24" buttonText="Save"
               connectedEdges="0" needsCallback="1" radioGroupId="0"/>
   <LABEL name="Num Visible Y label" id="101a4f77ca2445ba" memberName="numVisibleYLabel"
-         virtualName="" explicitFocusOrder="0" pos="528 328 72 24" edTextCol="ff000000"
+         virtualName="" explicitFocusOrder="0" pos="576 224 72 24" edTextCol="ff000000"
          edBkgCol="0" labelText="99999" editableSingleClick="1" editableDoubleClick="1"
          focusDiscardsChanges="0" fontname="Default font" fontsize="15"
          bold="0" italic="0" justification="36"/>
   <TEXTBUTTON name="Load Training button" id="c603a32c128de05d" memberName="loadTrainingButton"
-              virtualName="" explicitFocusOrder="0" pos="440 48 72 24" buttonText="Load T"
+              virtualName="" explicitFocusOrder="0" pos="504 48 72 24" buttonText="Load T"
               connectedEdges="0" needsCallback="1" radioGroupId="0"/>
   <TEXTBUTTON name="Save Training button" id="8e665b97490357c4" memberName="saveTrainingButton"
-              virtualName="" explicitFocusOrder="0" pos="520 48 72 24" buttonText="Save T"
+              virtualName="" explicitFocusOrder="0" pos="584 48 72 24" buttonText="Save T"
               connectedEdges="0" needsCallback="1" radioGroupId="0"/>
   <TEXTBUTTON name="Clear Training button" id="b4c148828519c013" memberName="clearTrainingButton"
-              virtualName="" explicitFocusOrder="0" pos="520 88 72 24" buttonText="Clear T"
+              virtualName="" explicitFocusOrder="0" pos="584 88 72 24" buttonText="Clear T"
               connectedEdges="0" needsCallback="1" radioGroupId="0"/>
   <TEXTBUTTON name="Remove Training button" id="f0e1d068c39986f3" memberName="removeTrainingButton"
-              virtualName="" explicitFocusOrder="0" pos="440 88 72 24" buttonText="Remove T"
+              virtualName="" explicitFocusOrder="0" pos="504 88 72 24" buttonText="Remove T"
               connectedEdges="0" needsCallback="1" radioGroupId="0"/>
   <SLIDER name="Num. Gibbs slider" id="fdd9d5ca4e05e18c" memberName="numGibbsSlider"
-          virtualName="" explicitFocusOrder="0" pos="224 368 184 24" min="1"
+          virtualName="" explicitFocusOrder="0" pos="224 408 184 24" min="1"
-          max="10" int="1" style="LinearHorizontal" textBoxPos="TextBoxLeft"
+          max="10" int="1" style="LinearBar" textBoxPos="TextBoxLeft" textBoxEditable="1"
-          textBoxEditable="1" textBoxWidth="80" textBoxHeight="20" skewFactor="1"/>
+          textBoxWidth="80" textBoxHeight="20" skewFactor="1"/>
   <TEXTBUTTON name="Reconstruct Equilibrium button" id="208a587f5556207" memberName="reconstructEquButton"
               virtualName="" explicitFocusOrder="0" pos="24 448 72 24" buttonText="Reconst Equ."
               connectedEdges="0" needsCallback="1" radioGroupId="0"/>
-  <TOGGLEBUTTON name="rbmUseExpectations toggle button" id="62884e37cd027719"
-                memberName="rbmUseExpectationsToggleButton" virtualName="" explicitFocusOrder="0"
-                pos="24 232 128 24" buttonText="Use Expectations" connectedEdges="0"
-                needsCallback="1" radioGroupId="0" state="0"/>
   <TOGGLEBUTTON name="rbmDoRaoBlackwell toggle button" id="7f585de46e182418"
                 memberName="rbmDoRaoBlackwellToggleButton" virtualName="" explicitFocusOrder="0"
                 pos="24 200 112 24" buttonText="Rao-Blackwell" connectedEdges="0"
                 needsCallback="1" radioGroupId="0" state="0"/>
-  <TOGGLEBUTTON name="rbmDoRobbinsMonro toggle button" id="21974def5fe90b46"
-                memberName="rbmDoRobbinsMonroToggleButton" virtualName="" explicitFocusOrder="0"
-                pos="168 232 120 24" buttonText="Robbins-Monro" connectedEdges="0"
-                needsCallback="1" radioGroupId="0" state="0"/>
   <TOGGLEBUTTON name="rbmReduceVariance toggle button" id="1636f6389abe9225"
                 memberName="rbmReduceVarianceToggleButton" virtualName="" explicitFocusOrder="0"
                 pos="168 200 128 24" buttonText="Reduce Variance" connectedEdges="0"
@@ -932,15 +995,15 @@ BEGIN_JUCER_METADATA
          bold="0" italic="0" justification="36"/>
   <LABEL name="Sigma label" id="68d7dad32ed357f8" memberName="sigmaLabel"
          virtualName="" explicitFocusOrder="0" pos="32 320 72 24" edTextCol="ff000000"
-         edBkgCol="0" labelText="0.5" editableSingleClick="1" editableDoubleClick="1"
+         edBkgCol="0" labelText="1.0" editableSingleClick="1" editableDoubleClick="1"
          focusDiscardsChanges="0" fontname="Default font" fontsize="15"
          bold="0" italic="0" justification="36"/>
   <TOGGLEBUTTON name="rbmUseVisibleGaussian toggle button" id="c89b32a4deaa7f19"
                 memberName="rbmUseVisibleGaussianToggleButton" virtualName=""
-                explicitFocusOrder="0" pos="168 264 160 24" buttonText="Use gaussian visible"
+                explicitFocusOrder="0" pos="168 232 160 24" buttonText="Use gaussian visible"
                 connectedEdges="0" needsCallback="1" radioGroupId="0" state="0"/>
   <TOGGLEBUTTON name="rbmDoSparse toggle button" id="ed3f2602ab35b5f2" memberName="rbmDoSparseToggleButton"
-                virtualName="" explicitFocusOrder="0" pos="24 264 128 24" buttonText="Do sparse"
+                virtualName="" explicitFocusOrder="0" pos="24 232 128 24" buttonText="Do sparse"
                 connectedEdges="0" needsCallback="1" radioGroupId="0" state="0"/>
   <LABEL name="Sparsity label" id="f4efe44dc0ca4894" memberName="sparsityLabel"
          virtualName="" explicitFocusOrder="0" pos="208 320 72 24" edTextCol="ff000000"
@@ -952,9 +1015,25 @@ BEGIN_JUCER_METADATA
          edBkgCol="0" labelText="1.0" editableSingleClick="1" editableDoubleClick="1"
          focusDiscardsChanges="0" fontname="Default font" fontsize="15"
          bold="0" italic="0" justification="36"/>
-  <GENERICCOMPONENT name="Progress Bar" id="548820d4a30574c7" memberName="m_progressBar"
+  <LABEL name="WeightDecay label" id="67bf3a0bec436468" memberName="weightDecayLabel"
-                    virtualName="" explicitFocusOrder="0" pos="224 160 168 24" class="ProgressBar"
+         virtualName="" explicitFocusOrder="0" pos="304 368 72 24" edTextCol="ff000000"
-                    params="m_trainingProgress"/>
+         edBkgCol="0" labelText="0.0002" editableSingleClick="1" editableDoubleClick="1"
+         focusDiscardsChanges="0" fontname="Default font" fontsize="15"
+         bold="0" italic="0" justification="36"/>
+  <SLIDER name="progressBar slider" id="8c8748f39f6d0ec6" memberName="m_progressBarSlider"
+          virtualName="" explicitFocusOrder="0" pos="216 160 184 24" min="0"
+          max="100" int="1" style="LinearBar" textBoxPos="TextBoxLeft"
+          textBoxEditable="0" textBoxWidth="80" textBoxHeight="20" skewFactor="1"/>
+  <LABEL name="Momentum label" id="570ca1355ccd4cdd" memberName="momentumLabel"
+         virtualName="" explicitFocusOrder="0" pos="208 368 72 24" edTextCol="ff000000"
+         edBkgCol="0" labelText="0.5" editableSingleClick="1" editableDoubleClick="1"
+         focusDiscardsChanges="0" fontname="Default font" fontsize="15"
+         bold="0" italic="0" justification="36"/>
+  <LABEL name="SparsityLearn label" id="8e452129810c0ed3" memberName="sparsityLearningRateLabel"
+         virtualName="" explicitFocusOrder="0" pos="416 320 72 24" edTextCol="ff000000"
+         edBkgCol="0" labelText="0.01" editableSingleClick="1" editableDoubleClick="1"
+         focusDiscardsChanges="0" fontname="Default font" fontsize="15"
+         bold="0" italic="0" justification="36"/>
 </JUCER_COMPONENT>
 
 END_JUCER_METADATA

Source/MainComponent.h

@@ -74,9 +74,6 @@ private:
 	uint32_t m_vNumX;
 	uint32_t m_vNumY;
 	uint32_t m_hNum;
-	uint32_t m_vNumX_next;
-	uint32_t m_vNumY_next;
-	uint32_t m_hNum_next;
 	LayerArray<VisibleLayer> m_layers;
 	void load();
 	void save();
@@ -90,8 +87,10 @@ private:
 	void redrawWeights(int index);
 	void run();
 	String m_baseDir;
-	double m_trainingProgress;
 	uint32_t m_numGibbs;
+	bool rbmReduceVarianceToggleButton_binary;
+	double sigma_gauss;
+	double sigmaDecay_gauss;
     //[/UserVariables]
 
     //==============================================================================
@@ -117,9 +116,7 @@ private:
     ScopedPointer<TextButton> removeTrainingButton;
     ScopedPointer<Slider> numGibbsSlider;
     ScopedPointer<TextButton> reconstructEquButton;
-    ScopedPointer<ToggleButton> rbmUseExpectationsToggleButton;
     ScopedPointer<ToggleButton> rbmDoRaoBlackwellToggleButton;
-    ScopedPointer<ToggleButton> rbmDoRobbinsMonroToggleButton;
     ScopedPointer<ToggleButton> rbmReduceVarianceToggleButton;
     ScopedPointer<Label> lambdaLabel;
     ScopedPointer<Label> sigmaLabel;
@@ -127,7 +124,10 @@ private:
     ScopedPointer<ToggleButton> rbmDoSparseToggleButton;
     ScopedPointer<Label> sparsityLabel;
     ScopedPointer<Label> sigmaDecayLabel;
-    ScopedPointer<ProgressBar> m_progressBar;
+    ScopedPointer<Label> weightDecayLabel;
+    ScopedPointer<Slider> m_progressBarSlider;
+    ScopedPointer<Label> momentumLabel;
+    ScopedPointer<Label> sparsityLearningRateLabel;
 
 
     //==============================================================================

Source/Rbm.hpp

@@ -25,7 +25,9 @@ class RbmListener
 {
 public:
 	RbmListener() {}
-	virtual ~RbmListener() {}
+	virtual ~RbmListener()
+	{
+	}
 
 	virtual void onEpochTrained(const Rbm &obj) = 0;
 };
@@ -39,42 +41,29 @@ public:
 	, m_progress(0)
 	, m_sigma(1.0)
 	, m_sigmaDecay(1.0)
+	, m_weightDecay(0.0)
 	, m_lambda(1.0)
 	, m_sparsity(0)
+	, m_muWeights(0.01)
+	, m_muSparsity(0.01)
+	, m_momentum(0.5)
+	, m_doCancel(false)
 	, m_useVisibleGaussian(false)
-	, m_useExpectations(false)
 	, m_doRaoBlackwell(false)
 	, m_useProbsForHiddenReconstruction(false)
-	, m_doRobbinsMonro(false)
 	, m_doSparse(false)
+	, m_numGibbs(1)
 	{
 		Noise_Init(&m_noise, 0x32727155);
 	}
 
 	~Rbm()
 	{
+		cancel();
 		Noise_Free(&m_noise);
 	}
 
-	void weightsUpdate(VisibleLayer &v, HiddenLayer &h, double mu)
+	void train(LayerArray<VisibleLayer> &vt, uint32_t numEpochs, double sigmaMin = 0.05)
-	{
-		MatrixXd &w = (MatrixXd&)m_w.weights();
-
-		// Update weights
-		w += mu*(v.states() * h.states().transpose());
-	}
-
-	void visibleBiasUpdate(VisibleLayer &v, double mu)
-	{
-		m_w.visibleBias().array() += mu*v.states().array();
-	}
-
-	void hiddenBiasUpdate(HiddenLayer &h, double mu)
-	{
-		m_w.hiddenBias().array() += mu*h.states().array();
-	}
-
-	void train(LayerArray<VisibleLayer> &vt, uint32_t numEpochs, double mu, uint32_t numGibbs = 1, double sigmaMin = 0.05)
 	{
 		uint32_t t, i;
 		uint32_t epoch;
@@ -82,77 +71,72 @@ public:
 		double sigma;
 		VisibleLayer v(m_w.getNumVisible());
 		HiddenLayer h(m_w.getNumHidden());
-		HiddenLayer *pH;
+
-		LayerArray<HiddenLayer> ht(vt.getSize(), m_w.getNumHidden());
+		VectorXd sumBiasV(m_w.getNumVisible());
+		VectorXd deltaBiasV(m_w.getNumVisible());
+
+		VectorXd sumBiasH(m_w.getNumHidden());
+		VectorXd deltaBiasH(m_w.getNumHidden());
+
+		MatrixXd sumWeights(m_w.getNumVisible(), m_w.getNumHidden());
+		MatrixXd deltaWeights(m_w.getNumVisible(), m_w.getNumHidden());
 
 		sigma = m_sigma;
-		Weights w = m_w;
 
 		double dProgress = 1.0/numEpochs;
+		double kTrain = 1.0/vt.getSize();
+
 		m_progress = 0;
 
-		if (m_useExpectations)
+		deltaWeights.fill(0);
-		{
+		deltaBiasV.fill(0);
-			mu /= vt.getSize();
+		deltaBiasH.fill(0);
-		}
+		m_doCancel = false;
-
-		if (m_doRobbinsMonro)
-		{
-			for (i=0; i < vt.getSize(); i++)
-			{
-				// Create hidden layer base on training data
-				ht[i].probsUpdateLogistic(vt[i], w, m_lambda, sigma);
-			}
-		}
-
 		for (epoch=0; epoch < numEpochs; epoch++)
 		{
+			if (m_doCancel)
+			{
+				m_doCancel = false;
+				break;
+			}
+
+			sumWeights.fill(0);
+			sumBiasV.fill(0);
+			sumBiasH.fill(0);
 			for (i=0; i < vt.getSize(); i++)
 			{
 				//t = (uint32_t)(0.5 + (vt.getSize()-1)*Noise_Uniform(&m_noise, 0.5));
 				t = i;
-				h.probsUpdateLogistic(vt[t], w, m_lambda, sigma);
+				h.probsUpdateLogistic(vt[t], m_w, m_lambda, sigma);
 
 				// Create hidden layer base on training data
-				if (m_doRobbinsMonro)
-				{
-					pH = &ht[t];
-				}
-				else
-				{
-					pH = &h;
-				}
-
-				// Update weights (positive phase)
 				if (m_doRaoBlackwell)
 				{
-					pH->states() = pH->probs();
+					h.states() = h.probs();
 				}
 				else
 				{
-					pH->statesUpdateStochastic();
+					h.statesUpdateStochastic();
-				}
-				weightsUpdate(vt[t], h, +mu);
-				visibleBiasUpdate(vt[t], +mu);
-				if (!m_doSparse)
-				{
-					hiddenBiasUpdate(h, +mu);
 				}
+				// Update weights (positive phase)
+				sumWeights += vt[t].states() * h.states().transpose();
+				sumBiasV += vt[t].states();
+				sumBiasH += h.states();
 
-				for (gibbs=0; gibbs < numGibbs; gibbs++)
+				for (gibbs=0; gibbs < m_numGibbs; gibbs++)
 				{
-					pH->statesUpdateStochastic();
+					h.statesUpdateStochastic();
 
 					// Create visible reconstruction (a fantasy...)
 					if (m_useProbsForHiddenReconstruction)
 					{
 						if (m_useVisibleGaussian)
 						{
-							v.probsUpdateGaussian(*pH, w, m_lambda, sigma);
+							v.probsUpdateGaussian(h, m_w, m_lambda, sigma);
 						}
 						else
 						{
-							v.probsUpdateLogistic(*pH, w, m_lambda, sigma);
+							v.probsUpdateLogistic(h, m_w, m_lambda, sigma);
 						}
 						v.states() = v.probs();
 					}
@@ -160,43 +144,38 @@ public:
 					{
 						if (m_useVisibleGaussian)
 						{
-							v.sampleGaussian(*pH, w, m_lambda, sigma);
+							v.sampleGaussian(h, m_w, m_lambda, sigma);
 						}
 						else
 						{
-							v.probsUpdateLogistic(*pH, w, m_lambda, sigma);
+							v.probsUpdateLogistic(h, m_w, m_lambda, sigma);
 							v.statesUpdateStochastic();
 						}
 					}
 					// Create hidden reconstruction
-					pH->probsUpdateLogistic(v, w, m_lambda, sigma);
+					h.probsUpdateLogistic(v, m_w, m_lambda, sigma);
 				}
 
 				// Update weights (negative phase)
 				if (m_doRaoBlackwell)
 				{
-					pH->states() = pH->probs();
+					h.states() = h.probs();
 				}
 				else
 				{
-					pH->statesUpdateStochastic();
+					h.statesUpdateStochastic();
-				}
-				weightsUpdate(v, *pH, -mu);
-				visibleBiasUpdate(v, -mu);
-				if (!m_doSparse)
-				{
-					hiddenBiasUpdate(*pH, -mu);
-				}
-				if (!m_useExpectations)
-				{
-					w = m_w;
 				}
+				sumWeights -= v.states() * h.states().transpose();
+				sumBiasV -= v.states();
+				sumBiasH -= h.states();
+
 			} // TrainingSize
 
-			if (m_useExpectations)
+			deltaWeights = m_momentum*deltaWeights + m_muWeights*kTrain*sumWeights - m_weightDecay*m_w.weights();
-			{
+			m_w.weights() += deltaWeights;
-				w = m_w;
+
-			}
+			deltaBiasV = m_momentum*deltaBiasV + m_muWeights*kTrain*sumBiasV;
+			m_w.visibleBias() += deltaBiasV;
 
 			if (m_doSparse)
 			{
@@ -206,17 +185,23 @@ public:
 
 				for (i=0; i < vt.getSize(); i++)
 				{
-					th.probsUpdateLogistic(vt[i], w, m_lambda, sigma);
+					th.probsUpdateLogistic(vt[i], m_w, m_lambda, sigma);
 					m += th.probs();
 				}
 				m /= i;
-				th.states().array() = m.array() - m_sparsity;
+				sumBiasH = m_sparsity - m.array();
-				hiddenBiasUpdate(th, -mu);
+				deltaBiasH = m_momentum*deltaBiasH + m_muSparsity*sumBiasH;
-				w = m_w;
 
-//				cout << "Mean(" << m_sparsity << ") = " << (double)m.array().mean() << endl;
+				cout << "Mean(" << m_sparsity << ") = " << (double)m.array().mean() << endl;
-//				cout << m << endl;
+				cout << m << endl;
 			}
+			else
+			{
+				deltaBiasH = m_momentum*deltaBiasH + m_muWeights*kTrain*sumBiasH;
+			}
+
+			m_w.hiddenBias() += deltaBiasH;
+
 			if (sigma > sigmaMin)
 			{
 				sigma *= m_sigmaDecay;
@@ -398,6 +383,11 @@ public:
 	   m_sigmaDecay = value;
    }
 
+   void setWeightDecay(double value)
+   {
+	   m_weightDecay = value;
+   }
+
    void setLambda(double value)
    {
 	   m_lambda = value;
@@ -413,11 +403,6 @@ public:
 	   m_useVisibleGaussian = flag;
    }
 
-   void setUseExpectations(bool flag)
-   {
-	   m_useExpectations = flag;
-   }
-
    void setDoRaoBlackwell(bool flag)
    {
 	   m_doRaoBlackwell = flag;
@@ -428,64 +413,35 @@ public:
 	   m_useProbsForHiddenReconstruction = flag;
    }
 
-   void setDoRobbinsMonro(bool flag)
-   {
-	   m_doRobbinsMonro = flag;
-   }
-
    void setDoSparse(bool flag)
    {
 	   m_doSparse = flag;
    }
 
-   double getSparsity()
+   void setNumGibbs(uint32_t value)
    {
-	   return m_sparsity;
+	   m_numGibbs = value;
    }
 
-   double getSigma()
+   void setMuWeights(double value)
    {
-	   return m_sigma;
+	   m_muWeights = value;
    }
 
-   double getSigmaDecay()
+   void setMuSparsity(double value)
    {
-	   return m_sigmaDecay;
+	   m_muSparsity = value;
    }
 
-   double getLambda()
+   void setMomentum(double value)
    {
-	   return m_lambda;
+	   m_momentum = value;
    }
 
-   bool getUseVisibleGaussian()
+   void cancel()
    {
-	   return m_useVisibleGaussian;
+	   m_doCancel = true;
-   }
+//	   while(m_doCancel);
-
-   bool getUseExpectations()
-   {
-	   return m_useExpectations;
-   }
-
-   bool getDoRaoBlackwell()
-   {
-	   return m_doRaoBlackwell;
-   }
-
-   bool getUseProbsForHiddenReconstruction()
-   {
-	   return m_useProbsForHiddenReconstruction;
-   }
-
-   bool getRobbinsMonro()
-   {
-	   return m_doRobbinsMonro;
-   }
-
-   bool getDoSparse()
-   {
-	   return m_doSparse;
    }
 
 private:
@@ -495,14 +451,18 @@ private:
 	double m_progress;
 	double m_sigma;
 	double m_sigmaDecay;
+	double m_weightDecay;
 	double m_lambda;
 	double m_sparsity;
+	double m_muWeights;
+	double m_muSparsity;
+	double m_momentum;
 	bool m_useVisibleGaussian;
-	bool m_useExpectations;
 	bool m_doRaoBlackwell;
 	bool m_useProbsForHiddenReconstruction;
-	bool m_doRobbinsMonro;
 	bool m_doSparse;
+	volatile bool m_doCancel;
+	uint32_t m_numGibbs;
 
 };