jens/Rbm-legacy
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[RBM]

Browse Source 
- GUI: added gaussian hidden, added mini batch size
- Rbm: added mini batch training revised sample functions, reverted to old weight decay


git-svn-id: http://moon:8086/svn/software/trunk/projects/RBM@305 b431acfa-c32f-4a4a-93f1-934dc6c82436
This commit is contained in:
jens
2016-07-07 19:27:33 +00:00
parent 4979ab41b4
commit 57cdfd7065
4 changed files with 161 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Source/MainComponent.cpp
+47 -6
View File
@@ -277,6 +277,20 @@ MainComponent::MainComponent ()
rbmDoSampleBatch->setButtonText (TRANS("Sample training"));
rbmDoSampleBatch->addListener (this);
addAndMakeVisible (sizeMiniBatch = new Label ("sizeMiniBatch",
TRANS("0")));
sizeMiniBatch->setTooltip (TRANS("Maximum number of training samples per train"));
sizeMiniBatch->setFont (Font (15.00f, Font::plain));
sizeMiniBatch->setJustificationType (Justification::centred);
sizeMiniBatch->setEditable (true, true, false);
sizeMiniBatch->setColour (TextEditor::textColourId, Colours::black);
sizeMiniBatch->setColour (TextEditor::backgroundColourId, Colour (0x00000000));
sizeMiniBatch->addListener (this);
addAndMakeVisible (rbmUseHiddenGaussianToggleButton = new ToggleButton ("rbmUseHiddenGaussian toggle button"));
rbmUseHiddenGaussianToggleButton->setButtonText (TRANS("Use gaussian hidden"));
rbmUseHiddenGaussianToggleButton->addListener (this);
//[UserPreSize]
memset(m_pRbmComponent, 0, sizeof(m_pRbmComponent));
@@ -334,6 +348,8 @@ MainComponent::~MainComponent()
rbmNormalizeDataToggleButton = nullptr;
m_rbmSelect = nullptr;
rbmDoSampleBatch = nullptr;
sizeMiniBatch = nullptr;
rbmUseHiddenGaussianToggleButton = nullptr;
//[Destructor]. You can add your own custom destruction code here..
@@ -443,14 +459,16 @@ void MainComponent::resized()
sparsityLabel->setBounds (740, 152, 72, 24);
sigmaDecayLabel->setBounds (836, 152, 72, 24);
weightDecayLabel->setBounds (836, 200, 72, 24);
m_progressBarSlider->setBounds (844, 20, 220, 24);
m_progressBarSlider->setBounds (828, 20, 220, 24);
momentumLabel->setBounds (740, 200, 72, 24);
sparsityLearningRateLabel->setBounds (948, 152, 72, 24);
weightInitLabel->setBounds (948, 200, 72, 24);
rbmLearnVarianceButton->setBounds (976, 92, 128, 24);
rbmNormalizeDataToggleButton->setBounds (844, 92, 124, 24);
rbmLearnVarianceButton->setBounds (1004, 92, 128, 24);
rbmNormalizeDataToggleButton->setBounds (1004, 60, 124, 24);
m_rbmSelect->setBounds (812, 280, 62, 24);
rbmDoSampleBatch->setBounds (844, 60, 128, 24);
sizeMiniBatch->setBounds (1064, 20, 48, 24);
rbmUseHiddenGaussianToggleButton->setBounds (840, 92, 156, 24);
//[UserResized] Add your own custom resize handling here..
//[/UserResized]
}
@@ -575,6 +593,12 @@ void MainComponent::buttonClicked (Button* buttonThatWasClicked)
m_pRbmComponentCurr->setDoSampleBatch(buttonThatWasClicked->getToggleState());
//[/UserButtonCode_rbmDoSampleBatch]
}
else if (buttonThatWasClicked == rbmUseHiddenGaussianToggleButton)
{
//[UserButtonCode_rbmUseHiddenGaussianToggleButton] -- add your button handler code here..
m_pRbmComponentCurr->setUseHiddenGaussian(buttonThatWasClicked->getToggleState());
//[/UserButtonCode_rbmUseHiddenGaussianToggleButton]
}
//[UserbuttonClicked_Post]
//[/UserbuttonClicked_Post]
@@ -697,6 +721,12 @@ void MainComponent::labelTextChanged (Label* labelThatHasChanged)
//[UserLabelCode_weightInitLabel] -- add your label text handling code here..
//[/UserLabelCode_weightInitLabel]
}
else if (labelThatHasChanged == sizeMiniBatch)
{
//[UserLabelCode_sizeMiniBatch] -- add your label text handling code here..
m_pRbmComponentCurr->setMiniBatchSize((size_t)labelThatHasChanged->getText().getFloatValue());
//[/UserLabelCode_sizeMiniBatch]
}
//[UserlabelTextChanged_Post]
//[/UserlabelTextChanged_Post]
@@ -867,6 +897,7 @@ void MainComponent::updateControls()
rbmDoSampleVisibleToggleButton->setToggleState(m_pRbmComponentCurr->params().m_doSampleVisible, dontSendNotification);
rbmDoSampleBatch->setToggleState(m_pRbmComponentCurr->params().m_doSampleBatch, dontSendNotification);
rbmUseVisibleGaussianToggleButton->setToggleState(m_pRbmComponentCurr->params().m_useVisibleGaussian, dontSendNotification);
rbmUseHiddenGaussianToggleButton->setToggleState(m_pRbmComponentCurr->params().m_useHiddenGaussian, dontSendNotification);
rbmDoSparseToggleButton->setToggleState(m_pRbmComponentCurr->params().m_doSparse, dontSendNotification);
rbmLearnVarianceButton->setToggleState(m_pRbmComponentCurr->params().m_doLearnVariance, dontSendNotification);
rbmDoSampleBatch->setToggleState(m_pRbmComponentCurr->params().m_doSampleBatch, dontSendNotification);
@@ -882,6 +913,7 @@ void MainComponent::updateControls()
numVisibleLabel->setText(String(m_weightsCurr->getNumVisibleX()), dontSendNotification );
numVisibleYLabel->setText(String(m_weightsCurr->getNumVisibleY()), dontSendNotification );
numHiddenLabel->setText(String(m_weightsCurr->getNumHidden()), dontSendNotification );
sizeMiniBatch->setText(String(m_pRbmComponentCurr->params().m_miniBatchSize), dontSendNotification);
WeightsSlider->setRange(0, m_weightsCurr->getNumHidden()-1, 1);
numGibbsSlider->setValue(m_pRbmComponentCurr->params().m_numGibbs);
@@ -1054,7 +1086,7 @@ BEGIN_JUCER_METADATA
focusDiscardsChanges="0" fontname="Default font" fontsize="15"
bold="0" italic="0" justification="36"/>
<SLIDER name="progressBar slider" id="8c8748f39f6d0ec6" memberName="m_progressBarSlider"
virtualName="" explicitFocusOrder="0" pos="844 20 220 24" tooltip="Progress"
virtualName="" explicitFocusOrder="0" pos="828 20 220 24" tooltip="Progress"
min="0" max="100" int="1" style="LinearBar" textBoxPos="TextBoxLeft"
textBoxEditable="0" textBoxWidth="80" textBoxHeight="20" skewFactor="1"/>
<LABEL name="Momentum label" id="570ca1355ccd4cdd" memberName="momentumLabel"
@@ -1073,10 +1105,10 @@ BEGIN_JUCER_METADATA
focusDiscardsChanges="0" fontname="Default font" fontsize="15"
bold="0" italic="0" justification="36"/>
<TOGGLEBUTTON name="rbmLearnVariance button" id="92c647c1f8b110a2" memberName="rbmLearnVarianceButton"
virtualName="" explicitFocusOrder="0" pos="976 92 128 24" buttonText="Learn Variance"
virtualName="" explicitFocusOrder="0" pos="1004 92 128 24" buttonText="Learn Variance"
connectedEdges="0" needsCallback="1" radioGroupId="0" state="0"/>
<TOGGLEBUTTON name="rbmNormalizeData toggle button" id="739772af1b096120" memberName="rbmNormalizeDataToggleButton"
virtualName="" explicitFocusOrder="0" pos="844 92 124 24" buttonText="Normalize data"
virtualName="" explicitFocusOrder="0" pos="1004 60 124 24" buttonText="Normalize data"
connectedEdges="0" needsCallback="1" radioGroupId="0" state="0"/>
<COMBOBOX name="RBM Selector" id="71115a4f965bfd38" memberName="m_rbmSelect"
virtualName="" explicitFocusOrder="0" pos="812 280 62 24" editable="0"
@@ -1085,6 +1117,15 @@ BEGIN_JUCER_METADATA
virtualName="" explicitFocusOrder="0" pos="844 60 128 24" tooltip="Sample training data during learning&#10;"
buttonText="Sample training" connectedEdges="0" needsCallback="1"
radioGroupId="0" state="0"/>
<LABEL name="sizeMiniBatch" id="dd9e3e6f2b7b22f8" memberName="sizeMiniBatch"
virtualName="" explicitFocusOrder="0" pos="1064 20 48 24" tooltip="Maximum number of training samples per train"
edTextCol="ff000000" edBkgCol="0" labelText="0" editableSingleClick="1"
editableDoubleClick="1" focusDiscardsChanges="0" fontname="Default font"
fontsize="15" bold="0" italic="0" justification="36"/>
<TOGGLEBUTTON name="rbmUseHiddenGaussian toggle button" id="92e05c920283616b"
memberName="rbmUseHiddenGaussianToggleButton" virtualName=""
explicitFocusOrder="0" pos="840 92 156 24" buttonText="Use gaussian hidden"
connectedEdges="0" needsCallback="1" radioGroupId="0" state="0"/>
</JUCER_COMPONENT>
END_JUCER_METADATA
Source/MainComponent.h
+2
View File
@@ -150,6 +150,8 @@ private:
ScopedPointer<ToggleButton> rbmNormalizeDataToggleButton;
ScopedPointer<ComboBox> m_rbmSelect;
ScopedPointer<ToggleButton> rbmDoSampleBatch;
ScopedPointer<Label> sizeMiniBatch;
ScopedPointer<ToggleButton> rbmUseHiddenGaussianToggleButton;
//==============================================================================
Source/Rbm.cpp
+102 -78
View File
@@ -17,6 +17,7 @@ Rbm::Rbm(Weights &weights, const MatrixXd &batch)
, m_variableSigma(weights.getNumVisible())
, m_progress(0)
{
setMiniBatchSize(batch.rows());
Noise_Init(&m_noise, 0x32727155);
m_variableSigma.fill(m_params.m_constantSigma);
updateHiddenBatch();
@@ -27,6 +28,49 @@ Rbm::~Rbm()
Noise_Free(&m_noise);
}
void Rbm::noiseGaussian(MatrixXd &dst)
{
for (size_t i=0; i < dst.rows(); i++)
{
for (size_t j=0; j < dst.cols(); j++)
{
dst(i, j) = Noise_Gaussian(&m_noise);
}
}
}
void Rbm::noiseUniform(MatrixXd &dst)
{
for (size_t i=0; i < dst.rows(); i++)
{
for (size_t j=0; j < dst.cols(); j++)
{
dst(i, j) = Noise_Uniform(&m_noise);
}
}
}
void Rbm::sampleGaussian(MatrixXd &dst, MatrixXd const &src, const MatrixXd &sigma)
{
MatrixXd n(src.rows(), src.cols());
noiseGaussian(n);
dst = sigma.array()*n.array() + src.array();
}
void Rbm::sampleGaussian(MatrixXd &srcDst, const MatrixXd &sigma)
{
MatrixXd n(srcDst.rows(), srcDst.cols());
noiseGaussian(n);
srcDst.array() += sigma.array()*n.array();
}
void Rbm::sampleGaussian(MatrixXd &srcDst, const double &sigma)
{
sampleGaussian(srcDst, sigma*MatrixXd::Ones(srcDst.rows(), srcDst.cols()));
}
void Rbm::sample(MatrixXd &srcDst)
{
sample(srcDst, srcDst);
@@ -34,42 +78,34 @@ void Rbm::sample(MatrixXd &srcDst)
void Rbm::sample(MatrixXd &dst, MatrixXd const &src)
{
uint32_t i;
MatrixXd n(src.rows(), src.cols());
for (i=0; i < src.array().size(); i++)
{
dst.array()(i) = (double)(src.array()(i) > Noise_Uniform(&m_noise));
}
noiseUniform(n);
dst = (src.array() > n.array()).cast<double>();
}
void Rbm::probsLogistic(MatrixXd &src)
{
src.array() = (-src.array()).exp();
src.array() += 1;
src.array() = 1.0/src.array();
src = (1 + (-src.array()).exp()).array().cwiseInverse();
}
void Rbm::probsLogistic(RowVectorXd &src)
{
src.array() = (-src.array()).exp();
src.array() += 1;
src.array() = 1.0/src.array();
src = (1 + (-src.array()).exp()).array().cwiseInverse();
}
void Rbm::probsLogistic(MatrixXd &src, const MatrixXd &sigma)
{
src.array() /= (sigma.array() + EPSILON_SIGMA);
src.array() = (-src.array()).exp();
src.array() += 1;
src.array() = 1.0/src.array();
probsLogistic(src);
}
void Rbm::probsLogistic(RowVectorXd &src, const RowVectorXd &sigma)
{
src.array() /= (sigma.array() + EPSILON_SIGMA);
src.array() = (-src.array()).exp();
src.array() += 1;
src.array() = 1.0/src.array();
probsLogistic(src);
}
void Rbm::probsGaussian(MatrixXd &src, const MatrixXd &sigma)
@@ -116,26 +152,6 @@ void Rbm::probsGaussian(RowVectorXd &src, const RowVectorXd &sigma)
src.array() *= k.array();
}
void Rbm::sampleGaussian(MatrixXd &dst, MatrixXd const &src, const MatrixXd &sigma)
{
uint32_t i;
for (i=0; i < src.array().size(); i++)
{
dst.array()(i) = sigma(i)*Noise_Gaussian(&m_noise) + src.array()(i);
}
}
void Rbm::sampleGaussian(MatrixXd &src, const MatrixXd &sigma)
{
uint32_t i;
for (i=0; i < src.array().size(); i++)
{
src.array()(i) = sigma(i)*Noise_Gaussian(&m_noise) + src.array()(i);
}
}
RowVectorXd Rbm::normalizeData(RowVectorXd const &src, RowVectorXd const &mu, RowVectorXd const &var)
{
// Remove mean
@@ -191,9 +207,8 @@ void Rbm::train(uint32_t numEpochs, double sigmaMin)
size_t trainingSize = m_batch.rows();
size_t trainingSizeRemain = trainingSize;
size_t batchRowIndex = 0;
const size_t miniBatchSize = 100;
double dProgress = 1.0/(numEpochs*(double)trainingSize/miniBatchSize);
double dProgress = 1.0/(numEpochs*(double)trainingSize/std::min(m_params.m_miniBatchSize, trainingSize));
MatrixXd dBiasV_curr(MatrixXd::Zero(1, m_w.getNumVisible()));
MatrixXd dBiasH_curr(MatrixXd::Zero(1, m_w.getNumHidden()));
@@ -206,14 +221,14 @@ void Rbm::train(uint32_t numEpochs, double sigmaMin)
while (trainingSizeRemain)
{
cout << "trainingSizeRemain: " << trainingSizeRemain << endl;
size_t toSlice = std::min(miniBatchSize, trainingSizeRemain);
size_t toSlice = std::min(m_params.m_miniBatchSize, trainingSizeRemain);
MatrixXd batch = m_batch.block(batchRowIndex, 0, toSlice, m_w.getNumVisible());
trainingSizeRemain -= toSlice;
batchRowIndex += toSlice;
size_t batchSize = batch.rows();
double mu_w = m_params.m_muWeights/batchSize;
double mu_biasV = m_params.m_muWeights/batchSize;
double mu_biasH = m_params.m_muWeights/batchSize;
MatrixXd diffErr(batchSize, m_w.getNumVisible());
double mu_w = m_params.m_muWeights/std::min(m_params.m_miniBatchSize, trainingSize);
double mu_biasV = m_params.m_muWeights/std::min(m_params.m_miniBatchSize, trainingSize);
double mu_biasH = m_params.m_muWeights/std::min(m_params.m_miniBatchSize, trainingSize);
MatrixXd batch_sampled(batchSize, m_w.getNumVisible());
MatrixXd v_sampled(batchSize, m_w.getNumVisible());
@@ -264,14 +279,19 @@ void Rbm::train(uint32_t numEpochs, double sigmaMin)
for (gibbs=0; gibbs < m_params.m_numGibbs; gibbs++)
{
sample(hid);
// Create visible reconstruction (a fantasy...) given hid
vis = hid * m_w.weights().transpose();
vis += m_w.visibleBias().replicate(batchSize, 1);
if (m_params.m_useHiddenGaussian)
{
sampleGaussian(hid, m_params.m_constantSigma);
}
else
{
sample(hid);
}
if (m_params.m_useVisibleGaussian)
{
// Create visible reconstruction (a fantasy...) given hid
vis = hid * m_w.weights().transpose();
vis += m_w.visibleBias().replicate(batchSize, 1);
sampleGaussian(v_sampled, vis, m_variableSigma.replicate(batchSize, 1));
hid = v_sampled * m_w.weights();
hid += m_w.hiddenBias().replicate(batchSize, 1);
@@ -280,6 +300,9 @@ void Rbm::train(uint32_t numEpochs, double sigmaMin)
}
else
{
// Create visible reconstruction (a fantasy...) given hid
vis = hid * m_w.weights().transpose();
vis += m_w.visibleBias().replicate(batchSize, 1);
probsLogistic(vis, m_variableSigma.replicate(batchSize, 1));
if (m_params.m_doSampleVisible)
{
@@ -319,26 +342,7 @@ void Rbm::train(uint32_t numEpochs, double sigmaMin)
}
dBiasH = dBiasH_curr;
MatrixXd p = m_w.weights();
if (m_params.m_weightDecay > 0)
{
for (size_t row=0; row < m_w.weights().rows(); row++)
{
for (size_t col=0; col < m_w.weights().cols(); col++)
{
if (p(row, col) >= 0)
{
p(row, col) = m_params.m_weightDecay;
}
else
{
p(row, col) -= m_params.m_weightDecay;
}
}
}
m_w.weights() -= mu_w*p;
}
m_w.weights() += mu_w*(m_params.m_momentum*dW + (1-m_params.m_momentum)*dW_curr);
m_w.weights() += mu_w*((m_params.m_momentum*dW + (1-m_params.m_momentum)*dW_curr) - m_params.m_weightDecay*m_w.weights());
dW = dW_curr;
if (m_params.m_sigmaDecay > 0)
@@ -353,15 +357,24 @@ void Rbm::train(uint32_t numEpochs, double sigmaMin)
} // Number of epochs
diffErr = batch - vis;
MatrixXd diffErr = batch - vis;
diffErr.array() *= diffErr.array();
double err = diffErr.colwise().sum().sum();
cout << "err =" << endl;
cout << err << endl;
cout << "error (per mini batch) = " << err << endl;
} // number of mini batches
updateHiddenBatch();
MatrixXd vis = m_h * m_w.weights().transpose();
vis += m_w.visibleBias().replicate(m_batch.rows(), 1);
probsLogistic(vis, m_variableSigma.replicate(m_batch.rows(), 1));
MatrixXd diffErr = m_batch - vis;
diffErr.array() *= diffErr.array();
double err = diffErr.colwise().sum().sum();
cout << "error (total) = " << err << endl;
onProgressChanged();
}
double Rbm::getProgress() const
@@ -445,6 +458,12 @@ void Rbm::setUseVisibleGaussian(bool flag)
onParamsChanged();
}
void Rbm::setUseHiddenGaussian(bool flag)
{
m_params.m_useHiddenGaussian = flag;
onParamsChanged();
}
void Rbm::setDoRaoBlackwell(bool flag)
{
m_params.m_doRaoBlackwell = flag;
@@ -495,6 +514,11 @@ void Rbm::setNumGibbs(size_t value)
onParamsChanged();
}
void Rbm::setMiniBatchSize(size_t size)
{
m_params.m_miniBatchSize = size;
}
void Rbm::setMuWeights(double value)
{
m_params.m_muWeights = value;
@@ -518,11 +542,6 @@ MatrixXd const& Rbm::getHiddenBatch()
return m_h;
}
MatrixXd const& Rbm::getVisibleBatch()
{
return m_v;
}
MatrixXd const& Rbm::getBatch()
{
return m_batch;
@@ -530,6 +549,10 @@ MatrixXd const& Rbm::getBatch()
void Rbm::updateHiddenBatch()
{
if (m_batch.rows() == 0)
{
return;
}
m_h.resize(m_batch.rows(), m_w.getNumHidden());
m_h = m_batch * m_w.weights();
m_h += m_w.hiddenBias().replicate(m_batch.rows(), 1);
@@ -540,3 +563,4 @@ Rbm::Params const& Rbm::params()
{
return m_params;
}
Source/Rbm.hpp
+10 -3
View File
@@ -29,6 +29,7 @@ public:
, m_muSparsity(0.01)
, m_momentum(0.5)
, m_useVisibleGaussian(false)
, m_useHiddenGaussian(false)
, m_doRaoBlackwell(true)
, m_doSampleVisible(false)
, m_doSampleBatch(false)
@@ -36,6 +37,7 @@ public:
, m_doNormalizeData(false)
, m_doLearnVariance(false)
, m_numGibbs(1)
, m_miniBatchSize(100)
{
}
@@ -48,6 +50,7 @@ public:
double m_muSparsity;
double m_momentum;
bool m_useVisibleGaussian;
bool m_useHiddenGaussian;
bool m_doRaoBlackwell;
bool m_doSampleVisible;
bool m_doSampleBatch;
@@ -55,6 +58,7 @@ public:
bool m_doNormalizeData;
bool m_doLearnVariance;
size_t m_numGibbs;
size_t m_miniBatchSize;
};
Rbm(Weights &weights, const MatrixXd &batch);
@@ -68,7 +72,8 @@ public:
static void probsGaussian(MatrixXd &src, const MatrixXd &sigma);
static void probsGaussian(RowVectorXd &src, const RowVectorXd &sigma);
void sampleGaussian(MatrixXd &dst, MatrixXd const &src, const MatrixXd &sigma);
void sampleGaussian(MatrixXd &src, const MatrixXd &sigma);
void sampleGaussian(MatrixXd &srcDst, const MatrixXd &sigma);
void sampleGaussian(MatrixXd &srcDst, const double &sigma);
RowVectorXd normalizeData(RowVectorXd const &src, RowVectorXd const &mu, RowVectorXd const &var);
RowVectorXd calcMean(MatrixXd const &batch);
RowVectorXd calcSigma(MatrixXd const &batch);
@@ -85,6 +90,7 @@ public:
void setLambda(double value);
void setSparsity(double value);
void setUseVisibleGaussian(bool flag);
void setUseHiddenGaussian(bool flag);
void setDoRaoBlackwell(bool flag);
void setDoSampleVisible(bool flag);
void setDoSampleBatch(bool flag);
@@ -92,11 +98,11 @@ public:
void setNormalizeData(bool flag);
void setDoLearnVariance(bool flag);
void setNumGibbs(size_t value);
void setMiniBatchSize(size_t size);
void setMuWeights(double value);
void setMuSparsity(double value);
void setMomentum(double value);
MatrixXd const& getHiddenBatch();
MatrixXd const& getVisibleBatch();
MatrixXd const& getBatch();
void updateHiddenBatch();
Params const& params();
@@ -104,12 +110,13 @@ public:
private:
Weights &m_w;
MatrixXd const &m_batch;
MatrixXd m_v;
MatrixXd m_h;
RowVectorXd m_variableSigma;
noise_gen_t m_noise;
double m_progress;
Params m_params;
void noiseGaussian(MatrixXd &dst);
void noiseUniform(MatrixXd &dst);
protected:
virtual void onProgressChanged()