From ae81da610b67a20b195ae722e91b3c3e5aba68dc Mon Sep 17 00:00:00 2001 From: Jens Ahrensfeld Date: Thu, 20 Jan 2022 18:16:35 +0000 Subject: [PATCH] - gibbsDoSampleHidden and gibbsDoSampleVisible onl used only training - very good results git-svn-id: http://moon:8086/svn/software/trunk/projects/Rbm@852 b431acfa-c32f-4a4a-93f1-934dc6c82436 --- poet_2v_5s.prj | 82 ++++++++++++++++++++++++------------------------- source/Rbm.cpp | 60 ++++++++++++++++-------------------- source/poet.cpp | 68 +++++++++++++++++++++++++++++++++------- 3 files changed, 125 insertions(+), 85 deletions(-) diff --git a/poet_2v_5s.prj b/poet_2v_5s.prj index 98bd3c2..b8eb524 100644 --- a/poet_2v_5s.prj +++ b/poet_2v_5s.prj @@ -6,31 +6,7 @@ { "id" : 0, "name" : "Layer", - "numContext" : 8, - "numHidden" : 64, - "numVisibleX" : 37, - "numVisibleY" : 2, - "rbm" : - { - "params" : - { - "doRaoBlackwell" : 1, - "doSampleBatch" : 0, - "gibbsDoSampleHidden" : 0, - "gibbsDoSampleVisible" : 0, - "learningRate" : 0.10000000000000001, - "miniBatchSize" : 1000, - "momentum" : 0.5, - "numEpochs" : 10000, - "numGibbs" : 1, - "weightDecay" : 0.0 - } - } - }, - { - "id" : 1, - "name" : "Layer", - "numContext" : 64, + "numContext" : 0, "numHidden" : 128, "numVisibleX" : 37, "numVisibleY" : 2, @@ -40,12 +16,36 @@ { "doRaoBlackwell" : 1, "doSampleBatch" : 0, - "gibbsDoSampleHidden" : 0, + "gibbsDoSampleHidden" : 1, "gibbsDoSampleVisible" : 0, - "learningRate" : 0.10000000000000001, + "learningRate" : 0.250000000000, "miniBatchSize" : 1000, "momentum" : 0.5, - "numEpochs" : 5000, + "numEpochs" : 1000, + "numGibbs" : 1, + "weightDecay" : 0.0 + } + } + }, + { + "id" : 1, + "name" : "Layer", + "numContext" : 128, + "numHidden" : 128, + "numVisibleX" : 37, + "numVisibleY" : 2, + "rbm" : + { + "params" : + { + "doRaoBlackwell" : 1, + "doSampleBatch" : 0, + "gibbsDoSampleHidden" : 1, + "gibbsDoSampleVisible" : 0, + "learningRate" : 0.250000000000, + "miniBatchSize" : 1000, + "momentum" : 0.5, + "numEpochs" : 1000, "numGibbs" : 1, "weightDecay" : 0.0 } @@ -55,7 +55,7 @@ "id" : 2, "name" : "Layer", "numContext" : 128, - "numHidden" : 256, + "numHidden" : 128, "numVisibleX" : 37, "numVisibleY" : 2, "rbm" : @@ -64,12 +64,12 @@ { "doRaoBlackwell" : 1, "doSampleBatch" : 0, - "gibbsDoSampleHidden" : 0, + "gibbsDoSampleHidden" : 1, "gibbsDoSampleVisible" : 0, - "learningRate" : 0.10000000000000001, + "learningRate" : 0.250000000000, "miniBatchSize" : 1000, "momentum" : 0.5, - "numEpochs" : 2500, + "numEpochs" : 1000, "numGibbs" : 1, "weightDecay" : 0.0 } @@ -78,8 +78,8 @@ { "id" : 3, "name" : "Layer", - "numContext" : 256, - "numHidden" : 512, + "numContext" : 128, + "numHidden" : 128, "numVisibleX" : 37, "numVisibleY" : 2, "rbm" : @@ -88,12 +88,12 @@ { "doRaoBlackwell" : 1, "doSampleBatch" : 0, - "gibbsDoSampleHidden" : 0, + "gibbsDoSampleHidden" : 1, "gibbsDoSampleVisible" : 0, - "learningRate" : 0.10000000000000001, + "learningRate" : 0.250000000000, "miniBatchSize" : 1000, "momentum" : 0.5, - "numEpochs" : 2500, + "numEpochs" : 1000, "numGibbs" : 1, "weightDecay" : 0.0 } @@ -102,8 +102,8 @@ { "id" : 4, "name" : "Layer", - "numContext" : 512, - "numHidden" : 1024, + "numContext" : 128, + "numHidden" : 128, "numVisibleX" : 37, "numVisibleY" : 2, "rbm" : @@ -112,9 +112,9 @@ { "doRaoBlackwell" : 1, "doSampleBatch" : 0, - "gibbsDoSampleHidden" : 0, + "gibbsDoSampleHidden" : 1, "gibbsDoSampleVisible" : 0, - "learningRate" : 0.10000000000000001, + "learningRate" : 0.250000000000, "miniBatchSize" : 1000, "momentum" : 0.5, "numEpochs" : 1000, diff --git a/source/Rbm.cpp b/source/Rbm.cpp index 6f0abba..16201d2 100644 --- a/source/Rbm.cpp +++ b/source/Rbm.cpp @@ -108,24 +108,10 @@ void Rbm::gibbs_vh(arma::mat &v_probs, arma::mat &h_probs) for (int i=0; i < m_params.numGibbs; i++) { // Create hidden representation given v - if (m_params.gibbsDoSampleVisible) - { - h_probs = prob(v_to_h(sample(v_probs))); - } - else - { - h_probs = prob(v_to_h(v_probs)); - } + h_probs = prob(v_to_h(v_probs)); // Create visible reconstruction (a fantasy...) given hid - if (m_params.gibbsDoSampleHidden) - { - v_probs = prob(h_to_v(sample(h_probs))); - } - else - { - v_probs = prob(h_to_v(h_probs)); - } + v_probs = prob(h_to_v(h_probs)); } } @@ -134,24 +120,10 @@ void Rbm::gibbs_hv(arma::mat &h_probs, arma::mat &v_probs) for (int i=0; i < m_params.numGibbs; i++) { // Create visible reconstruction (a fantasy...) given hid - if (m_params.gibbsDoSampleHidden) - { - v_probs = prob(h_to_v(sample(h_probs))); - } - else - { - v_probs = prob(h_to_v(h_probs)); - } + v_probs = prob(h_to_v(h_probs)); // Create hidden representation given v - if (m_params.gibbsDoSampleVisible) - { - h_probs = prob(v_to_h(sample(v_probs))); - } - else - { - h_probs = prob(v_to_h(v_probs)); - } + h_probs = prob(v_to_h(v_probs)); } } @@ -176,7 +148,29 @@ void Rbm::contrastiveDivergence(arma::mat const &v_states, arma::mat &dw, arma:: dbv = sum(v_states, 0); dbhv = sum(h_states, 0); - gibbs_hv(h_probs, v_probs); + // Gibbs sampling with training params + for (int i=0; i < m_params.numGibbs; i++) + { + // Create visible reconstruction (a fantasy...) given hid + if (m_params.gibbsDoSampleHidden) + { + v_probs = prob(h_to_v(sample(h_probs))); + } + else + { + v_probs = prob(h_to_v(h_probs)); + } + + // Create hidden representation given v + if (m_params.gibbsDoSampleVisible) + { + h_probs = prob(v_to_h(sample(v_probs))); + } + else + { + h_probs = prob(v_to_h(v_probs)); + } + } // Update weights (negative phase) dw -= v_probs.t() * h_probs; diff --git a/source/poet.cpp b/source/poet.cpp index 6e78c4c..fa68358 100644 --- a/source/poet.cpp +++ b/source/poet.cpp @@ -19,19 +19,69 @@ using namespace arma; class RbmListener : public Rbm::IListener { public: - RbmListener() {} + RbmListener(AStack &stack) + : m_stack(stack) + , m_last_progress(0) + { + } virtual ~RbmListener() {} bool onProgress(Rbm *pRbm, const Rbm::Status &status) { - std::cout << "Progress : " << status.progress << " %" << std::endl; - std::cout << "error (per mini batch) = " << status.err << std::endl; - std::cout << "error (total) = " << status.err_total << std::endl; - std::cout << "L1 = " << status.L1 << std::endl; - std::cout << "L2 = " << status.L2 << std::endl; + if (status.progress == 0) + { + m_last_progress = -10; + } + if ((status.progress - m_last_progress) == 10) + { + std::cout << "Progress : " << status.progress << " %" << std::endl; + std::cout << "error (per mini batch) = " << status.err << std::endl; + std::cout << "error (total) = " << status.err_total << std::endl; + std::cout << "L1 = " << status.L1 << std::endl; + std::cout << "L2 = " << status.L2 << std::endl; + m_last_progress = status.progress; + forward(); + } return true; } + + void forward() + { + RnnStack *stack = reinterpret_cast(&m_stack); + arma::mat state; + arma::mat curr; + arma::mat next; + std::string curr_str; + std::string next_str; + arma::mat t_vc = stack->trainingBatch(); + + for (int i=0; i < stack->getSeqLen(); i++) + { + curr = stack->to_curr(t_vc.row(i)); + curr_str.append(1,RnnTextHelper::idx2ch(curr.index_max())); + arma::mat r = stack->step_forward(state, curr); + next = stack->to_next(r, true); + next_str.append(1,RnnTextHelper::idx2ch(next.index_max())); + } + cout << "Curr: " << curr_str << std::endl; + cout << "Next: " << next_str << std::endl; + + for (int i=stack->getSeqLen(); i < t_vc.n_rows; i++) + { + curr = next; + curr_str.append(1,RnnTextHelper::idx2ch(curr.index_max())); + arma::mat r = stack->step_forward(state, curr); + next = stack->to_next(r, true); + curr = stack->to_curr(r); + next_str.append(1,RnnTextHelper::idx2ch(next.index_max())); + } + cout << "Curr: " << curr_str << std::endl; + cout << "Next: " << next_str << std::endl; + + } + AStack &m_stack; + int m_last_progress; }; int main(int argc, char *argv[]) @@ -88,11 +138,7 @@ int main(int argc, char *argv[]) if (command == Command::Train) { - for (int i=0; i < stack->numLayers(); i++) - { - stack->getLayer(i)->params().gibbsDoSampleHidden = true; - } - RbmListener listener; + RbmListener listener(*stack); // Phase 10 stack->train(t_vc, &listener);