jens/Rbm-legacy
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

- RBM: changed calcualation of pre-weight update data in RBM

Browse Source 
- always use expectations
- removed "Use Expectations Button"
- removed Robbins-Monro
- added sparsity learning rate
- added momentum
- added weight decay
- added Slider as progress bar



git-svn-id: http://moon:8086/svn/software/trunk/projects/RBM@30 b431acfa-c32f-4a4a-93f1-934dc6c82436
This commit is contained in:
jens
2014-10-18 15:38:11 +00:00
parent 3fb8ac0629
commit 57761fe743
3 changed files with 284 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Source/MainComponent.cpp
+183 -104
View File
@@ -72,7 +72,7 @@ MainComponent::MainComponent ()
addAndMakeVisible (patterSlider = new Slider ("Pattern slider"));
patterSlider->setRange (0, 0, 1);
patterSlider->setSliderStyle (Slider::LinearHorizontal);
patterSlider->setSliderStyle (Slider::LinearBar);
patterSlider->setTextBoxStyle (Slider::TextBoxLeft, false, 80, 20);
patterSlider->addListener (this);
@@ -86,7 +86,7 @@ MainComponent::MainComponent ()
addAndMakeVisible (WeightsSlider = new Slider ("Weights slider"));
WeightsSlider->setRange (0, 0, 1);
WeightsSlider->setSliderStyle (Slider::LinearHorizontal);
WeightsSlider->setSliderStyle (Slider::LinearBar);
WeightsSlider->setTextBoxStyle (Slider::TextBoxLeft, false, 80, 20);
WeightsSlider->addListener (this);
@@ -100,7 +100,7 @@ MainComponent::MainComponent ()
numEpochslabel->addListener (this);
addAndMakeVisible (learningRateLabel = new Label ("Learning Rate label",
TRANS("0.03")));
TRANS("0.01")));
learningRateLabel->setFont (Font (15.00f, Font::plain));
learningRateLabel->setJustificationType (Justification::centred);
learningRateLabel->setEditable (true, true, false);
@@ -178,7 +178,7 @@ MainComponent::MainComponent ()
addAndMakeVisible (numGibbsSlider = new Slider ("Num. Gibbs slider"));
numGibbsSlider->setRange (1, 10, 1);
numGibbsSlider->setSliderStyle (Slider::LinearHorizontal);
numGibbsSlider->setSliderStyle (Slider::LinearBar);
numGibbsSlider->setTextBoxStyle (Slider::TextBoxLeft, false, 80, 20);
numGibbsSlider->addListener (this);
@@ -186,18 +186,10 @@ MainComponent::MainComponent ()
reconstructEquButton->setButtonText (TRANS("Reconst Equ."));
reconstructEquButton->addListener (this);
addAndMakeVisible (rbmUseExpectationsToggleButton = new ToggleButton ("rbmUseExpectations toggle button"));
rbmUseExpectationsToggleButton->setButtonText (TRANS("Use Expectations"));
rbmUseExpectationsToggleButton->addListener (this);
addAndMakeVisible (rbmDoRaoBlackwellToggleButton = new ToggleButton ("rbmDoRaoBlackwell toggle button"));
rbmDoRaoBlackwellToggleButton->setButtonText (TRANS("Rao-Blackwell"));
rbmDoRaoBlackwellToggleButton->addListener (this);
addAndMakeVisible (rbmDoRobbinsMonroToggleButton = new ToggleButton ("rbmDoRobbinsMonro toggle button"));
rbmDoRobbinsMonroToggleButton->setButtonText (TRANS("Robbins-Monro"));
rbmDoRobbinsMonroToggleButton->addListener (this);
addAndMakeVisible (rbmReduceVarianceToggleButton = new ToggleButton ("rbmReduceVariance toggle button"));
rbmReduceVarianceToggleButton->setButtonText (TRANS("Reduce Variance"));
rbmReduceVarianceToggleButton->addListener (this);
@@ -212,7 +204,7 @@ MainComponent::MainComponent ()
lambdaLabel->addListener (this);
addAndMakeVisible (sigmaLabel = new Label ("Sigma label",
TRANS("0.5")));
TRANS("1.0")));
sigmaLabel->setFont (Font (15.00f, Font::plain));
sigmaLabel->setJustificationType (Justification::centred);
sigmaLabel->setEditable (true, true, false);
@@ -246,17 +238,44 @@ MainComponent::MainComponent ()
sigmaDecayLabel->setColour (TextEditor::backgroundColourId, Colour (0x00000000));
sigmaDecayLabel->addListener (this);
addAndMakeVisible (m_progressBar = new ProgressBar (m_trainingProgress));
m_progressBar->setName ("Progress Bar");
addAndMakeVisible (weightDecayLabel = new Label ("WeightDecay label",
TRANS("0.0002")));
weightDecayLabel->setFont (Font (15.00f, Font::plain));
weightDecayLabel->setJustificationType (Justification::centred);
weightDecayLabel->setEditable (true, true, false);
weightDecayLabel->setColour (TextEditor::textColourId, Colours::black);
weightDecayLabel->setColour (TextEditor::backgroundColourId, Colour (0x00000000));
weightDecayLabel->addListener (this);
addAndMakeVisible (m_progressBarSlider = new Slider ("progressBar slider"));
m_progressBarSlider->setRange (0, 100, 1);
m_progressBarSlider->setSliderStyle (Slider::LinearBar);
m_progressBarSlider->setTextBoxStyle (Slider::TextBoxLeft, true, 80, 20);
m_progressBarSlider->addListener (this);
addAndMakeVisible (momentumLabel = new Label ("Momentum label",
TRANS("0.5")));
momentumLabel->setFont (Font (15.00f, Font::plain));
momentumLabel->setJustificationType (Justification::centred);
momentumLabel->setEditable (true, true, false);
momentumLabel->setColour (TextEditor::textColourId, Colours::black);
momentumLabel->setColour (TextEditor::backgroundColourId, Colour (0x00000000));
momentumLabel->addListener (this);
addAndMakeVisible (sparsityLearningRateLabel = new Label ("SparsityLearn label",
TRANS("0.01")));
sparsityLearningRateLabel->setFont (Font (15.00f, Font::plain));
sparsityLearningRateLabel->setJustificationType (Justification::centred);
sparsityLearningRateLabel->setEditable (true, true, false);
sparsityLearningRateLabel->setColour (TextEditor::textColourId, Colours::black);
sparsityLearningRateLabel->setColour (TextEditor::backgroundColourId, Colour (0x00000000));
sparsityLearningRateLabel->addListener (this);
//[UserPreSize]
m_vNumX = 16;
m_vNumY = 16;
m_hNum = 64;
m_vNumX_next = 16;
m_vNumY_next = 16;
m_hNum_next = 64;
m_numGibbs = 1;
m_weights.setUnits(m_vNumX*m_vNumY, m_hNum);
create();
@@ -267,8 +286,7 @@ MainComponent::MainComponent ()
//[Constructor] You can add your own custom stuff here..
m_trainingProgress = 0;
m_progressBar->setTextToDisplay("Finished");
m_progressBarSlider->setValue(100);
//[/Constructor]
}
@@ -299,9 +317,7 @@ MainComponent::~MainComponent()
removeTrainingButton = nullptr;
numGibbsSlider = nullptr;
reconstructEquButton = nullptr;
rbmUseExpectationsToggleButton = nullptr;
rbmDoRaoBlackwellToggleButton = nullptr;
rbmDoRobbinsMonroToggleButton = nullptr;
rbmReduceVarianceToggleButton = nullptr;
lambdaLabel = nullptr;
sigmaLabel = nullptr;
@@ -309,7 +325,10 @@ MainComponent::~MainComponent()
rbmDoSparseToggleButton = nullptr;
sparsityLabel = nullptr;
sigmaDecayLabel = nullptr;
m_progressBar = nullptr;
weightDecayLabel = nullptr;
m_progressBarSlider = nullptr;
momentumLabel = nullptr;
sparsityLearningRateLabel = nullptr;
//[Destructor]. You can add your own custom destruction code here..
@@ -366,6 +385,24 @@ void MainComponent::paint (Graphics& g)
296, 306, 96, 14,
Justification::centred, true);
g.setColour (Colours::black);
g.setFont (Font (15.00f, Font::plain));
g.drawText (TRANS("Weight decay"),
296, 354, 96, 14,
Justification::centred, true);
g.setColour (Colours::black);
g.setFont (Font (15.00f, Font::plain));
g.drawText (TRANS("Momentum"),
208, 354, 80, 14,
Justification::centred, true);
g.setColour (Colours::black);
g.setFont (Font (15.00f, Font::plain));
g.drawText (TRANS("Sparsity Alpha"),
408, 306, 96, 14,
Justification::centred, true);
//[UserPaint] Add your own custom painting code here..
//[/UserPaint]
}
@@ -374,37 +411,38 @@ void MainComponent::resized()
{
trainButton->setBounds (120, 408, 72, 24);
addButton->setBounds (24, 160, 72, 24);
patterSlider->setBounds (224, 408, 184, 24);
patterSlider->setBounds (224, 448, 184, 24);
reconstructButton->setBounds (24, 408, 72, 24);
ShakeButton->setBounds (120, 448, 72, 24);
WeightsSlider->setBounds (224, 448, 184, 24);
WeightsSlider->setBounds (224, 488, 184, 24);
numEpochslabel->setBounds (32, 368, 72, 24);
learningRateLabel->setBounds (120, 368, 72, 24);
testButton->setBounds (120, 160, 72, 24);
numVisibleLabel->setBounds (456, 328, 72, 24);
numHiddenLabel->setBounds (504, 360, 72, 24);
createButton->setBounds (504, 392, 72, 24);
projectNameLabel->setBounds (488, 296, 96, 24);
loadButton->setBounds (456, 264, 72, 24);
saveButton->setBounds (536, 264, 72, 24);
numVisibleYLabel->setBounds (528, 328, 72, 24);
loadTrainingButton->setBounds (440, 48, 72, 24);
saveTrainingButton->setBounds (520, 48, 72, 24);
clearTrainingButton->setBounds (520, 88, 72, 24);
removeTrainingButton->setBounds (440, 88, 72, 24);
numGibbsSlider->setBounds (224, 368, 184, 24);
numVisibleLabel->setBounds (504, 224, 72, 24);
numHiddenLabel->setBounds (552, 256, 72, 24);
createButton->setBounds (552, 288, 72, 24);
projectNameLabel->setBounds (536, 192, 96, 24);
loadButton->setBounds (504, 160, 72, 24);
saveButton->setBounds (584, 160, 72, 24);
numVisibleYLabel->setBounds (576, 224, 72, 24);
loadTrainingButton->setBounds (504, 48, 72, 24);
saveTrainingButton->setBounds (584, 48, 72, 24);
clearTrainingButton->setBounds (584, 88, 72, 24);
removeTrainingButton->setBounds (504, 88, 72, 24);
numGibbsSlider->setBounds (224, 408, 184, 24);
reconstructEquButton->setBounds (24, 448, 72, 24);
rbmUseExpectationsToggleButton->setBounds (24, 232, 128, 24);
rbmDoRaoBlackwellToggleButton->setBounds (24, 200, 112, 24);
rbmDoRobbinsMonroToggleButton->setBounds (168, 232, 120, 24);
rbmReduceVarianceToggleButton->setBounds (168, 200, 128, 24);
lambdaLabel->setBounds (120, 320, 72, 24);
sigmaLabel->setBounds (32, 320, 72, 24);
rbmUseVisibleGaussianToggleButton->setBounds (168, 264, 160, 24);
rbmDoSparseToggleButton->setBounds (24, 264, 128, 24);
rbmUseVisibleGaussianToggleButton->setBounds (168, 232, 160, 24);
rbmDoSparseToggleButton->setBounds (24, 232, 128, 24);
sparsityLabel->setBounds (208, 320, 72, 24);
sigmaDecayLabel->setBounds (304, 320, 72, 24);
m_progressBar->setBounds (224, 160, 168, 24);
weightDecayLabel->setBounds (304, 368, 72, 24);
m_progressBarSlider->setBounds (216, 160, 184, 24);
momentumLabel->setBounds (208, 368, 72, 24);
sparsityLearningRateLabel->setBounds (416, 320, 72, 24);
//[UserResized] Add your own custom resize handling here..
DrawTraining->setBounds (16, 16, 100, 100);
DrawReconstruction->setBounds (110+16, 16, 100, 100);
@@ -459,9 +497,10 @@ void MainComponent::buttonClicked (Button* buttonThatWasClicked)
else if (buttonThatWasClicked == createButton)
{
//[UserButtonCode_createButton] -- add your button handler code here..
m_vNumX = m_vNumX_next;
m_vNumY = m_vNumY_next;
m_hNum = m_hNum_next;
m_vNumX = numVisibleLabel->getText().getIntValue();
m_vNumY = numVisibleYLabel->getText().getIntValue();
m_hNum = numHiddenLabel->getText().getIntValue();
m_weights.setUnits(m_vNumX*m_vNumY, m_hNum);
create();
//[/UserButtonCode_createButton]
@@ -522,24 +561,12 @@ void MainComponent::buttonClicked (Button* buttonThatWasClicked)
}
//[/UserButtonCode_reconstructEquButton]
}
else if (buttonThatWasClicked == rbmUseExpectationsToggleButton)
{
//[UserButtonCode_rbmUseExpectationsToggleButton] -- add your button handler code here..
m_pRbm->setUseExpectations(buttonThatWasClicked->getToggleState());
//[/UserButtonCode_rbmUseExpectationsToggleButton]
}
else if (buttonThatWasClicked == rbmDoRaoBlackwellToggleButton)
{
//[UserButtonCode_rbmDoRaoBlackwellToggleButton] -- add your button handler code here..
m_pRbm->setDoRaoBlackwell(buttonThatWasClicked->getToggleState());
//[/UserButtonCode_rbmDoRaoBlackwellToggleButton]
}
else if (buttonThatWasClicked == rbmDoRobbinsMonroToggleButton)
{
//[UserButtonCode_rbmDoRobbinsMonroToggleButton] -- add your button handler code here..
m_pRbm->setDoRobbinsMonro(buttonThatWasClicked->getToggleState());
//[/UserButtonCode_rbmDoRobbinsMonroToggleButton]
}
else if (buttonThatWasClicked == rbmReduceVarianceToggleButton)
{
//[UserButtonCode_rbmReduceVarianceToggleButton] -- add your button handler code here..
@@ -550,6 +577,23 @@ void MainComponent::buttonClicked (Button* buttonThatWasClicked)
{
//[UserButtonCode_rbmUseVisibleGaussianToggleButton] -- add your button handler code here..
m_pRbm->setUseVisibleGaussian(buttonThatWasClicked->getToggleState());
rbmReduceVarianceToggleButton->setEnabled(!buttonThatWasClicked->getToggleState());
if (buttonThatWasClicked->getToggleState())
{
rbmReduceVarianceToggleButton_binary = rbmReduceVarianceToggleButton->getToggleState();
rbmReduceVarianceToggleButton->setToggleState(false, sendNotification);
sigmaLabel->setText(String(sigma_gauss, 1), sendNotification);
sigmaDecayLabel->setText(String(sigmaDecay_gauss, 1), sendNotification);
}
else
{
sigma_gauss = sigmaLabel->getText().getFloatValue();
sigmaDecay_gauss = sigmaDecayLabel->getText().getFloatValue();
sigmaLabel->setText("1.0", sendNotification);
sigmaDecayLabel->setText("1.0", sendNotification);
rbmReduceVarianceToggleButton->setToggleState(rbmReduceVarianceToggleButton_binary, sendNotification);
}
redrawReconstruction();
//[/UserButtonCode_rbmUseVisibleGaussianToggleButton]
}
@@ -589,9 +633,14 @@ void MainComponent::sliderValueChanged (Slider* sliderThatWasMoved)
else if (sliderThatWasMoved == numGibbsSlider)
{
//[UserSliderCode_numGibbsSlider] -- add your slider handling code here..
m_numGibbs = (uint32_t)sliderThatWasMoved->getValue();
m_pRbm->setNumGibbs((uint32_t)sliderThatWasMoved->getValue());
//[/UserSliderCode_numGibbsSlider]
}
else if (sliderThatWasMoved == m_progressBarSlider)
{
//[UserSliderCode_m_progressBarSlider] -- add your slider handling code here..
//[/UserSliderCode_m_progressBarSlider]
}
//[UsersliderValueChanged_Post]
//[/UsersliderValueChanged_Post]
@@ -610,18 +659,17 @@ void MainComponent::labelTextChanged (Label* labelThatHasChanged)
else if (labelThatHasChanged == learningRateLabel)
{
//[UserLabelCode_learningRateLabel] -- add your label text handling code here..
m_pRbm->setMuWeights(labelThatHasChanged->getText().getFloatValue());
//[/UserLabelCode_learningRateLabel]
}
else if (labelThatHasChanged == numVisibleLabel)
{
//[UserLabelCode_numVisibleLabel] -- add your label text handling code here..
m_vNumX_next = labelThatHasChanged->getText().getIntValue();
//[/UserLabelCode_numVisibleLabel]
}
else if (labelThatHasChanged == numHiddenLabel)
{
//[UserLabelCode_numHiddenLabel] -- add your label text handling code here..
m_hNum_next = labelThatHasChanged->getText().getIntValue();
//[/UserLabelCode_numHiddenLabel]
}
else if (labelThatHasChanged == projectNameLabel)
@@ -632,7 +680,6 @@ void MainComponent::labelTextChanged (Label* labelThatHasChanged)
else if (labelThatHasChanged == numVisibleYLabel)
{
//[UserLabelCode_numVisibleYLabel] -- add your label text handling code here..
m_vNumY_next = labelThatHasChanged->getText().getIntValue();
//[/UserLabelCode_numVisibleYLabel]
}
else if (labelThatHasChanged == lambdaLabel)
@@ -661,6 +708,24 @@ void MainComponent::labelTextChanged (Label* labelThatHasChanged)
m_pRbm->setSigmaDecay(labelThatHasChanged->getText().getFloatValue());
//[/UserLabelCode_sigmaDecayLabel]
}
else if (labelThatHasChanged == weightDecayLabel)
{
//[UserLabelCode_weightDecayLabel] -- add your label text handling code here..
m_pRbm->setWeightDecay(labelThatHasChanged->getText().getFloatValue());
//[/UserLabelCode_weightDecayLabel]
}
else if (labelThatHasChanged == momentumLabel)
{
//[UserLabelCode_momentumLabel] -- add your label text handling code here..
m_pRbm->setMomentum(labelThatHasChanged->getText().getFloatValue());
//[/UserLabelCode_momentumLabel]
}
else if (labelThatHasChanged == sparsityLearningRateLabel)
{
//[UserLabelCode_sparsityLearningRateLabel] -- add your label text handling code here..
m_pRbm->setMuSparsity(labelThatHasChanged->getText().getFloatValue());
//[/UserLabelCode_sparsityLearningRateLabel]
}
//[UserlabelTextChanged_Post]
//[/UserlabelTextChanged_Post]
@@ -704,9 +769,9 @@ void MainComponent::create()
numHiddenLabel->setText(String(m_hNum), dontSendNotification );
m_pRbm = new Rbm(m_weights, this);
m_pRbm->setUseExpectations(rbmUseExpectationsToggleButton->getToggleState());
sigma_gauss = 0.5;
sigmaDecay_gauss = 0.99;
m_pRbm->setDoRaoBlackwell(rbmDoRaoBlackwellToggleButton->getToggleState());
m_pRbm->setDoRobbinsMonro(rbmDoRobbinsMonroToggleButton->getToggleState());
m_pRbm->setUseProbsForHiddenReconstruction(rbmReduceVarianceToggleButton->getToggleState());
m_pRbm->setUseVisibleGaussian(rbmUseVisibleGaussianToggleButton->getToggleState());
m_pRbm->setDoSparse(rbmDoSparseToggleButton->getToggleState());
@@ -714,7 +779,10 @@ void MainComponent::create()
m_pRbm->setLambda(lambdaLabel->getText().getFloatValue());
m_pRbm->setSigma(sigmaLabel->getText().getFloatValue());
m_pRbm->setSigmaDecay(sigmaDecayLabel->getText().getFloatValue());
m_pRbm->setWeightDecay(weightDecayLabel->getText().getFloatValue());
m_pRbm->setSparsity(sparsityLabel->getText().getFloatValue());
m_pRbm->setNumGibbs((uint32_t)numGibbsSlider->getValue());
resized();
}
@@ -738,10 +806,10 @@ void MainComponent::onChanged(const LayerArray<VisibleLayer> &obj)
void MainComponent::onEpochTrained(const Rbm &obj)
{
m_trainingProgress = obj.getProgress();
// mylog("Training %f %%\n", m_trainingProgress*100);
// mylog("Training %f %%\n", obj.getProgress()*100);
redrawReconstruction();
redrawWeights((int)WeightsSlider->getValue());
m_progressBarSlider->setValue((int)(100*obj.getProgress()), dontSendNotification);
}
void MainComponent::onDraw(DrawComponent &obj)
@@ -790,11 +858,8 @@ void MainComponent::redrawWeights(int index)
void MainComponent::run()
{
trainButton->setEnabled(false);
m_progressBar->setPercentageDisplay(true);
m_pRbm->train(m_layers, numEpochslabel->getText().getIntValue(), learningRateLabel->getText().getFloatValue(), m_numGibbs);
m_pRbm->train(m_layers, numEpochslabel->getText().getIntValue());
trainButton->setEnabled(true);
m_trainingProgress = 0.0;
m_progressBar->setTextToDisplay("Finished");
}
//[/MiscUserCode]
@@ -827,6 +892,12 @@ BEGIN_JUCER_METADATA
fontname="Default font" fontsize="15" bold="0" italic="0" justification="36"/>
<TEXT pos="296 306 96 14" fill="solid: ff000000" hasStroke="0" text="Sigma decay"
fontname="Default font" fontsize="15" bold="0" italic="0" justification="36"/>
<TEXT pos="296 354 96 14" fill="solid: ff000000" hasStroke="0" text="Weight decay"
fontname="Default font" fontsize="15" bold="0" italic="0" justification="36"/>
<TEXT pos="208 354 80 14" fill="solid: ff000000" hasStroke="0" text="Momentum"
fontname="Default font" fontsize="15" bold="0" italic="0" justification="36"/>
<TEXT pos="408 306 96 14" fill="solid: ff000000" hasStroke="0" text="Sparsity Alpha"
fontname="Default font" fontsize="15" bold="0" italic="0" justification="36"/>
</BACKGROUND>
<TEXTBUTTON name="Train button" id="7909d74b5522987c" memberName="trainButton"
virtualName="" explicitFocusOrder="0" pos="120 408 72 24" buttonText="Train"
@@ -835,9 +906,9 @@ BEGIN_JUCER_METADATA
virtualName="" explicitFocusOrder="0" pos="24 160 72 24" buttonText="Add"
connectedEdges="0" needsCallback="1" radioGroupId="0"/>
<SLIDER name="Pattern slider" id="c3e0a2c816db81d1" memberName="patterSlider"
virtualName="" explicitFocusOrder="0" pos="224 408 184 24" min="0"
max="0" int="1" style="LinearHorizontal" textBoxPos="TextBoxLeft"
textBoxEditable="1" textBoxWidth="80" textBoxHeight="20" skewFactor="1"/>
virtualName="" explicitFocusOrder="0" pos="224 448 184 24" min="0"
max="0" int="1" style="LinearBar" textBoxPos="TextBoxLeft" textBoxEditable="1"
textBoxWidth="80" textBoxHeight="20" skewFactor="1"/>
<TEXTBUTTON name="Reconstruct button" id="c1901d121a5819d6" memberName="reconstructButton"
virtualName="" explicitFocusOrder="0" pos="24 408 72 24" buttonText="Reconstruct"
connectedEdges="0" needsCallback="1" radioGroupId="0"/>
@@ -845,9 +916,9 @@ BEGIN_JUCER_METADATA
virtualName="" explicitFocusOrder="0" pos="120 448 72 24" buttonText="Shake"
connectedEdges="0" needsCallback="1" radioGroupId="0"/>
<SLIDER name="Weights slider" id="699075a1fc01458a" memberName="WeightsSlider"
virtualName="" explicitFocusOrder="0" pos="224 448 184 24" min="0"
max="0" int="1" style="LinearHorizontal" textBoxPos="TextBoxLeft"
textBoxEditable="1" textBoxWidth="80" textBoxHeight="20" skewFactor="1"/>
virtualName="" explicitFocusOrder="0" pos="224 488 184 24" min="0"
max="0" int="1" style="LinearBar" textBoxPos="TextBoxLeft" textBoxEditable="1"
textBoxWidth="80" textBoxHeight="20" skewFactor="1"/>
<LABEL name="Num Epochs label" id="b23ae372ee931474" memberName="numEpochslabel"
virtualName="" explicitFocusOrder="0" pos="32 368 72 24" edTextCol="ff000000"
edBkgCol="0" labelText="100" editableSingleClick="1" editableDoubleClick="1"
@@ -855,72 +926,64 @@ BEGIN_JUCER_METADATA
bold="0" italic="0" justification="36"/>
<LABEL name="Learning Rate label" id="49611a27914e910d" memberName="learningRateLabel"
virtualName="" explicitFocusOrder="0" pos="120 368 72 24" edTextCol="ff000000"
edBkgCol="0" labelText="0.03" editableSingleClick="1" editableDoubleClick="1"
edBkgCol="0" labelText="0.01" editableSingleClick="1" editableDoubleClick="1"
focusDiscardsChanges="0" fontname="Default font" fontsize="15"
bold="0" italic="0" justification="36"/>
<TEXTBUTTON name="Test button" id="a1e6fed732ae7ee1" memberName="testButton"
virtualName="" explicitFocusOrder="0" pos="120 160 72 24" buttonText="Test"
connectedEdges="0" needsCallback="1" radioGroupId="0"/>
<LABEL name="Num Visible label" id="60acd702770b77dd" memberName="numVisibleLabel"
virtualName="" explicitFocusOrder="0" pos="456 328 72 24" edTextCol="ff000000"
virtualName="" explicitFocusOrder="0" pos="504 224 72 24" edTextCol="ff000000"
edBkgCol="0" labelText="99999" editableSingleClick="1" editableDoubleClick="1"
focusDiscardsChanges="0" fontname="Default font" fontsize="15"
bold="0" italic="0" justification="36"/>
<LABEL name="Num Hidden label" id="8a56c0419bd3ce86" memberName="numHiddenLabel"
virtualName="" explicitFocusOrder="0" pos="504 360 72 24" edTextCol="ff000000"
virtualName="" explicitFocusOrder="0" pos="552 256 72 24" edTextCol="ff000000"
edBkgCol="0" labelText="99999" editableSingleClick="1" editableDoubleClick="1"
focusDiscardsChanges="0" fontname="Default font" fontsize="15"
bold="0" italic="0" justification="36"/>
<TEXTBUTTON name="Create button" id="6c71593a581844eb" memberName="createButton"
virtualName="" explicitFocusOrder="0" pos="504 392 72 24" buttonText="Create"
virtualName="" explicitFocusOrder="0" pos="552 288 72 24" buttonText="Create"
connectedEdges="0" needsCallback="1" radioGroupId="0"/>
<LABEL name="Project Name label" id="b4e855167abdac6b" memberName="projectNameLabel"
virtualName="" explicitFocusOrder="0" pos="488 296 96 24" edTextCol="ff000000"
virtualName="" explicitFocusOrder="0" pos="536 192 96 24" edTextCol="ff000000"
edBkgCol="0" labelText="TestPrj" editableSingleClick="1" editableDoubleClick="1"
focusDiscardsChanges="0" fontname="Default font" fontsize="15"
bold="0" italic="0" justification="36"/>
<TEXTBUTTON name="Load button" id="4e606c77b12e7d11" memberName="loadButton"
virtualName="" explicitFocusOrder="0" pos="456 264 72 24" buttonText="Load"
virtualName="" explicitFocusOrder="0" pos="504 160 72 24" buttonText="Load"
connectedEdges="0" needsCallback="1" radioGroupId="0"/>
<TEXTBUTTON name="Save button" id="c435e774a51e39bf" memberName="saveButton"
virtualName="" explicitFocusOrder="0" pos="536 264 72 24" buttonText="Save"
virtualName="" explicitFocusOrder="0" pos="584 160 72 24" buttonText="Save"
connectedEdges="0" needsCallback="1" radioGroupId="0"/>
<LABEL name="Num Visible Y label" id="101a4f77ca2445ba" memberName="numVisibleYLabel"
virtualName="" explicitFocusOrder="0" pos="528 328 72 24" edTextCol="ff000000"
virtualName="" explicitFocusOrder="0" pos="576 224 72 24" edTextCol="ff000000"
edBkgCol="0" labelText="99999" editableSingleClick="1" editableDoubleClick="1"
focusDiscardsChanges="0" fontname="Default font" fontsize="15"
bold="0" italic="0" justification="36"/>
<TEXTBUTTON name="Load Training button" id="c603a32c128de05d" memberName="loadTrainingButton"
virtualName="" explicitFocusOrder="0" pos="440 48 72 24" buttonText="Load T"
virtualName="" explicitFocusOrder="0" pos="504 48 72 24" buttonText="Load T"
connectedEdges="0" needsCallback="1" radioGroupId="0"/>
<TEXTBUTTON name="Save Training button" id="8e665b97490357c4" memberName="saveTrainingButton"
virtualName="" explicitFocusOrder="0" pos="520 48 72 24" buttonText="Save T"
virtualName="" explicitFocusOrder="0" pos="584 48 72 24" buttonText="Save T"
connectedEdges="0" needsCallback="1" radioGroupId="0"/>
<TEXTBUTTON name="Clear Training button" id="b4c148828519c013" memberName="clearTrainingButton"
virtualName="" explicitFocusOrder="0" pos="520 88 72 24" buttonText="Clear T"
virtualName="" explicitFocusOrder="0" pos="584 88 72 24" buttonText="Clear T"
connectedEdges="0" needsCallback="1" radioGroupId="0"/>
<TEXTBUTTON name="Remove Training button" id="f0e1d068c39986f3" memberName="removeTrainingButton"
virtualName="" explicitFocusOrder="0" pos="440 88 72 24" buttonText="Remove T"
virtualName="" explicitFocusOrder="0" pos="504 88 72 24" buttonText="Remove T"
connectedEdges="0" needsCallback="1" radioGroupId="0"/>
<SLIDER name="Num. Gibbs slider" id="fdd9d5ca4e05e18c" memberName="numGibbsSlider"
virtualName="" explicitFocusOrder="0" pos="224 368 184 24" min="1"
max="10" int="1" style="LinearHorizontal" textBoxPos="TextBoxLeft"
textBoxEditable="1" textBoxWidth="80" textBoxHeight="20" skewFactor="1"/>
virtualName="" explicitFocusOrder="0" pos="224 408 184 24" min="1"
max="10" int="1" style="LinearBar" textBoxPos="TextBoxLeft" textBoxEditable="1"
textBoxWidth="80" textBoxHeight="20" skewFactor="1"/>
<TEXTBUTTON name="Reconstruct Equilibrium button" id="208a587f5556207" memberName="reconstructEquButton"
virtualName="" explicitFocusOrder="0" pos="24 448 72 24" buttonText="Reconst Equ."
connectedEdges="0" needsCallback="1" radioGroupId="0"/>
<TOGGLEBUTTON name="rbmUseExpectations toggle button" id="62884e37cd027719"
memberName="rbmUseExpectationsToggleButton" virtualName="" explicitFocusOrder="0"
pos="24 232 128 24" buttonText="Use Expectations" connectedEdges="0"
needsCallback="1" radioGroupId="0" state="0"/>
<TOGGLEBUTTON name="rbmDoRaoBlackwell toggle button" id="7f585de46e182418"
memberName="rbmDoRaoBlackwellToggleButton" virtualName="" explicitFocusOrder="0"
pos="24 200 112 24" buttonText="Rao-Blackwell" connectedEdges="0"
needsCallback="1" radioGroupId="0" state="0"/>
<TOGGLEBUTTON name="rbmDoRobbinsMonro toggle button" id="21974def5fe90b46"
memberName="rbmDoRobbinsMonroToggleButton" virtualName="" explicitFocusOrder="0"
pos="168 232 120 24" buttonText="Robbins-Monro" connectedEdges="0"
needsCallback="1" radioGroupId="0" state="0"/>
<TOGGLEBUTTON name="rbmReduceVariance toggle button" id="1636f6389abe9225"
memberName="rbmReduceVarianceToggleButton" virtualName="" explicitFocusOrder="0"
pos="168 200 128 24" buttonText="Reduce Variance" connectedEdges="0"
@@ -932,15 +995,15 @@ BEGIN_JUCER_METADATA
bold="0" italic="0" justification="36"/>
<LABEL name="Sigma label" id="68d7dad32ed357f8" memberName="sigmaLabel"
virtualName="" explicitFocusOrder="0" pos="32 320 72 24" edTextCol="ff000000"
edBkgCol="0" labelText="0.5" editableSingleClick="1" editableDoubleClick="1"
edBkgCol="0" labelText="1.0" editableSingleClick="1" editableDoubleClick="1"
focusDiscardsChanges="0" fontname="Default font" fontsize="15"
bold="0" italic="0" justification="36"/>
<TOGGLEBUTTON name="rbmUseVisibleGaussian toggle button" id="c89b32a4deaa7f19"
memberName="rbmUseVisibleGaussianToggleButton" virtualName=""
explicitFocusOrder="0" pos="168 264 160 24" buttonText="Use gaussian visible"
explicitFocusOrder="0" pos="168 232 160 24" buttonText="Use gaussian visible"
connectedEdges="0" needsCallback="1" radioGroupId="0" state="0"/>
<TOGGLEBUTTON name="rbmDoSparse toggle button" id="ed3f2602ab35b5f2" memberName="rbmDoSparseToggleButton"
virtualName="" explicitFocusOrder="0" pos="24 264 128 24" buttonText="Do sparse"
virtualName="" explicitFocusOrder="0" pos="24 232 128 24" buttonText="Do sparse"
connectedEdges="0" needsCallback="1" radioGroupId="0" state="0"/>
<LABEL name="Sparsity label" id="f4efe44dc0ca4894" memberName="sparsityLabel"
virtualName="" explicitFocusOrder="0" pos="208 320 72 24" edTextCol="ff000000"
@@ -952,9 +1015,25 @@ BEGIN_JUCER_METADATA
edBkgCol="0" labelText="1.0" editableSingleClick="1" editableDoubleClick="1"
focusDiscardsChanges="0" fontname="Default font" fontsize="15"
bold="0" italic="0" justification="36"/>
<GENERICCOMPONENT name="Progress Bar" id="548820d4a30574c7" memberName="m_progressBar"
virtualName="" explicitFocusOrder="0" pos="224 160 168 24" class="ProgressBar"
params="m_trainingProgress"/>
<LABEL name="WeightDecay label" id="67bf3a0bec436468" memberName="weightDecayLabel"
virtualName="" explicitFocusOrder="0" pos="304 368 72 24" edTextCol="ff000000"
edBkgCol="0" labelText="0.0002" editableSingleClick="1" editableDoubleClick="1"
focusDiscardsChanges="0" fontname="Default font" fontsize="15"
bold="0" italic="0" justification="36"/>
<SLIDER name="progressBar slider" id="8c8748f39f6d0ec6" memberName="m_progressBarSlider"
virtualName="" explicitFocusOrder="0" pos="216 160 184 24" min="0"
max="100" int="1" style="LinearBar" textBoxPos="TextBoxLeft"
textBoxEditable="0" textBoxWidth="80" textBoxHeight="20" skewFactor="1"/>
<LABEL name="Momentum label" id="570ca1355ccd4cdd" memberName="momentumLabel"
virtualName="" explicitFocusOrder="0" pos="208 368 72 24" edTextCol="ff000000"
edBkgCol="0" labelText="0.5" editableSingleClick="1" editableDoubleClick="1"
focusDiscardsChanges="0" fontname="Default font" fontsize="15"
bold="0" italic="0" justification="36"/>
<LABEL name="SparsityLearn label" id="8e452129810c0ed3" memberName="sparsityLearningRateLabel"
virtualName="" explicitFocusOrder="0" pos="416 320 72 24" edTextCol="ff000000"
edBkgCol="0" labelText="0.01" editableSingleClick="1" editableDoubleClick="1"
focusDiscardsChanges="0" fontname="Default font" fontsize="15"
bold="0" italic="0" justification="36"/>
</JUCER_COMPONENT>
END_JUCER_METADATA
Source/MainComponent.h
+7 -7
View File
@@ -74,9 +74,6 @@ private:
uint32_t m_vNumX;
uint32_t m_vNumY;
uint32_t m_hNum;
uint32_t m_vNumX_next;
uint32_t m_vNumY_next;
uint32_t m_hNum_next;
LayerArray<VisibleLayer> m_layers;
void load();
void save();
@@ -90,8 +87,10 @@ private:
void redrawWeights(int index);
void run();
String m_baseDir;
double m_trainingProgress;
uint32_t m_numGibbs;
bool rbmReduceVarianceToggleButton_binary;
double sigma_gauss;
double sigmaDecay_gauss;
//[/UserVariables]
//==============================================================================
@@ -117,9 +116,7 @@ private:
ScopedPointer<TextButton> removeTrainingButton;
ScopedPointer<Slider> numGibbsSlider;
ScopedPointer<TextButton> reconstructEquButton;
ScopedPointer<ToggleButton> rbmUseExpectationsToggleButton;
ScopedPointer<ToggleButton> rbmDoRaoBlackwellToggleButton;
ScopedPointer<ToggleButton> rbmDoRobbinsMonroToggleButton;
ScopedPointer<ToggleButton> rbmReduceVarianceToggleButton;
ScopedPointer<Label> lambdaLabel;
ScopedPointer<Label> sigmaLabel;
@@ -127,7 +124,10 @@ private:
ScopedPointer<ToggleButton> rbmDoSparseToggleButton;
ScopedPointer<Label> sparsityLabel;
ScopedPointer<Label> sigmaDecayLabel;
ScopedPointer<ProgressBar> m_progressBar;
ScopedPointer<Label> weightDecayLabel;
ScopedPointer<Slider> m_progressBarSlider;
ScopedPointer<Label> momentumLabel;
ScopedPointer<Label> sparsityLearningRateLabel;
//==============================================================================
Source/Rbm.hpp
+94 -134
View File
@@ -25,7 +25,9 @@ class RbmListener
{
public:
RbmListener() {}
virtual ~RbmListener() {}
virtual ~RbmListener()
{
}
virtual void onEpochTrained(const Rbm &obj) = 0;
};
@@ -39,42 +41,29 @@ public:
, m_progress(0)
, m_sigma(1.0)
, m_sigmaDecay(1.0)
, m_weightDecay(0.0)
, m_lambda(1.0)
, m_sparsity(0)
, m_muWeights(0.01)
, m_muSparsity(0.01)
, m_momentum(0.5)
, m_doCancel(false)
, m_useVisibleGaussian(false)
, m_useExpectations(false)
, m_doRaoBlackwell(false)
, m_useProbsForHiddenReconstruction(false)
, m_doRobbinsMonro(false)
, m_doSparse(false)
, m_numGibbs(1)
{
Noise_Init(&m_noise, 0x32727155);
}
~Rbm()
{
cancel();
Noise_Free(&m_noise);
}
void weightsUpdate(VisibleLayer &v, HiddenLayer &h, double mu)
{
MatrixXd &w = (MatrixXd&)m_w.weights();
// Update weights
w += mu*(v.states() * h.states().transpose());
}
void visibleBiasUpdate(VisibleLayer &v, double mu)
{
m_w.visibleBias().array() += mu*v.states().array();
}
void hiddenBiasUpdate(HiddenLayer &h, double mu)
{
m_w.hiddenBias().array() += mu*h.states().array();
}
void train(LayerArray<VisibleLayer> &vt, uint32_t numEpochs, double mu, uint32_t numGibbs = 1, double sigmaMin = 0.05)
void train(LayerArray<VisibleLayer> &vt, uint32_t numEpochs, double sigmaMin = 0.05)
{
uint32_t t, i;
uint32_t epoch;
@@ -82,77 +71,72 @@ public:
double sigma;
VisibleLayer v(m_w.getNumVisible());
HiddenLayer h(m_w.getNumHidden());
HiddenLayer *pH;
LayerArray<HiddenLayer> ht(vt.getSize(), m_w.getNumHidden());
VectorXd sumBiasV(m_w.getNumVisible());
VectorXd deltaBiasV(m_w.getNumVisible());
VectorXd sumBiasH(m_w.getNumHidden());
VectorXd deltaBiasH(m_w.getNumHidden());
MatrixXd sumWeights(m_w.getNumVisible(), m_w.getNumHidden());
MatrixXd deltaWeights(m_w.getNumVisible(), m_w.getNumHidden());
sigma = m_sigma;
Weights w = m_w;
double dProgress = 1.0/numEpochs;
double kTrain = 1.0/vt.getSize();
m_progress = 0;
if (m_useExpectations)
{
mu /= vt.getSize();
}
if (m_doRobbinsMonro)
{
for (i=0; i < vt.getSize(); i++)
{
// Create hidden layer base on training data
ht[i].probsUpdateLogistic(vt[i], w, m_lambda, sigma);
}
}
deltaWeights.fill(0);
deltaBiasV.fill(0);
deltaBiasH.fill(0);
m_doCancel = false;
for (epoch=0; epoch < numEpochs; epoch++)
{
if (m_doCancel)
{
m_doCancel = false;
break;
}
sumWeights.fill(0);
sumBiasV.fill(0);
sumBiasH.fill(0);
for (i=0; i < vt.getSize(); i++)
{
//t = (uint32_t)(0.5 + (vt.getSize()-1)*Noise_Uniform(&m_noise, 0.5));
t = i;
h.probsUpdateLogistic(vt[t], w, m_lambda, sigma);
h.probsUpdateLogistic(vt[t], m_w, m_lambda, sigma);
// Create hidden layer base on training data
if (m_doRobbinsMonro)
{
pH = &ht[t];
}
else
{
pH = &h;
}
// Update weights (positive phase)
if (m_doRaoBlackwell)
{
pH->states() = pH->probs();
h.states() = h.probs();
}
else
{
pH->statesUpdateStochastic();
}
weightsUpdate(vt[t], h, +mu);
visibleBiasUpdate(vt[t], +mu);
if (!m_doSparse)
{
hiddenBiasUpdate(h, +mu);
h.statesUpdateStochastic();
}
// Update weights (positive phase)
sumWeights += vt[t].states() * h.states().transpose();
sumBiasV += vt[t].states();
sumBiasH += h.states();
for (gibbs=0; gibbs < numGibbs; gibbs++)
for (gibbs=0; gibbs < m_numGibbs; gibbs++)
{
pH->statesUpdateStochastic();
h.statesUpdateStochastic();
// Create visible reconstruction (a fantasy...)
if (m_useProbsForHiddenReconstruction)
{
if (m_useVisibleGaussian)
{
v.probsUpdateGaussian(*pH, w, m_lambda, sigma);
v.probsUpdateGaussian(h, m_w, m_lambda, sigma);
}
else
{
v.probsUpdateLogistic(*pH, w, m_lambda, sigma);
v.probsUpdateLogistic(h, m_w, m_lambda, sigma);
}
v.states() = v.probs();
}
@@ -160,43 +144,38 @@ public:
{
if (m_useVisibleGaussian)
{
v.sampleGaussian(*pH, w, m_lambda, sigma);
v.sampleGaussian(h, m_w, m_lambda, sigma);
}
else
{
v.probsUpdateLogistic(*pH, w, m_lambda, sigma);
v.probsUpdateLogistic(h, m_w, m_lambda, sigma);
v.statesUpdateStochastic();
}
}
// Create hidden reconstruction
pH->probsUpdateLogistic(v, w, m_lambda, sigma);
h.probsUpdateLogistic(v, m_w, m_lambda, sigma);
}
// Update weights (negative phase)
if (m_doRaoBlackwell)
{
pH->states() = pH->probs();
h.states() = h.probs();
}
else
{
pH->statesUpdateStochastic();
}
weightsUpdate(v, *pH, -mu);
visibleBiasUpdate(v, -mu);
if (!m_doSparse)
{
hiddenBiasUpdate(*pH, -mu);
}
if (!m_useExpectations)
{
w = m_w;
h.statesUpdateStochastic();
}
sumWeights -= v.states() * h.states().transpose();
sumBiasV -= v.states();
sumBiasH -= h.states();
} // TrainingSize
if (m_useExpectations)
{
w = m_w;
}
deltaWeights = m_momentum*deltaWeights + m_muWeights*kTrain*sumWeights - m_weightDecay*m_w.weights();
m_w.weights() += deltaWeights;
deltaBiasV = m_momentum*deltaBiasV + m_muWeights*kTrain*sumBiasV;
m_w.visibleBias() += deltaBiasV;
if (m_doSparse)
{
@@ -206,17 +185,23 @@ public:
for (i=0; i < vt.getSize(); i++)
{
th.probsUpdateLogistic(vt[i], w, m_lambda, sigma);
th.probsUpdateLogistic(vt[i], m_w, m_lambda, sigma);
m += th.probs();
}
m /= i;
th.states().array() = m.array() - m_sparsity;
hiddenBiasUpdate(th, -mu);
w = m_w;
sumBiasH = m_sparsity - m.array();
deltaBiasH = m_momentum*deltaBiasH + m_muSparsity*sumBiasH;
// cout << "Mean(" << m_sparsity << ") = " << (double)m.array().mean() << endl;
// cout << m << endl;
cout << "Mean(" << m_sparsity << ") = " << (double)m.array().mean() << endl;
cout << m << endl;
}
else
{
deltaBiasH = m_momentum*deltaBiasH + m_muWeights*kTrain*sumBiasH;
}
m_w.hiddenBias() += deltaBiasH;
if (sigma > sigmaMin)
{
sigma *= m_sigmaDecay;
@@ -398,6 +383,11 @@ public:
m_sigmaDecay = value;
}
void setWeightDecay(double value)
{
m_weightDecay = value;
}
void setLambda(double value)
{
m_lambda = value;
@@ -413,11 +403,6 @@ public:
m_useVisibleGaussian = flag;
}
void setUseExpectations(bool flag)
{
m_useExpectations = flag;
}
void setDoRaoBlackwell(bool flag)
{
m_doRaoBlackwell = flag;
@@ -428,64 +413,35 @@ public:
m_useProbsForHiddenReconstruction = flag;
}
void setDoRobbinsMonro(bool flag)
{
m_doRobbinsMonro = flag;
}
void setDoSparse(bool flag)
{
m_doSparse = flag;
}
double getSparsity()
void setNumGibbs(uint32_t value)
{
return m_sparsity;
m_numGibbs = value;
}
double getSigma()
void setMuWeights(double value)
{
return m_sigma;
m_muWeights = value;
}
double getSigmaDecay()
void setMuSparsity(double value)
{
return m_sigmaDecay;
m_muSparsity = value;
}
double getLambda()
void setMomentum(double value)
{
return m_lambda;
m_momentum = value;
}
bool getUseVisibleGaussian()
void cancel()
{
return m_useVisibleGaussian;
}
bool getUseExpectations()
{
return m_useExpectations;
}
bool getDoRaoBlackwell()
{
return m_doRaoBlackwell;
}
bool getUseProbsForHiddenReconstruction()
{
return m_useProbsForHiddenReconstruction;
}
bool getRobbinsMonro()
{
return m_doRobbinsMonro;
}
bool getDoSparse()
{
return m_doSparse;
m_doCancel = true;
// while(m_doCancel);
}
private:
@@ -495,14 +451,18 @@ private:
double m_progress;
double m_sigma;
double m_sigmaDecay;
double m_weightDecay;
double m_lambda;
double m_sparsity;
double m_muWeights;
double m_muSparsity;
double m_momentum;
bool m_useVisibleGaussian;
bool m_useExpectations;
bool m_doRaoBlackwell;
bool m_useProbsForHiddenReconstruction;
bool m_doRobbinsMonro;
bool m_doSparse;
volatile bool m_doCancel;
uint32_t m_numGibbs;
};