diff --git a/source/Rbm.cpp b/source/Rbm.cpp index bb242e1..0bc1e4a 100644 --- a/source/Rbm.cpp +++ b/source/Rbm.cpp @@ -14,19 +14,21 @@ #include "Rbm.hpp" #include "noise.h" -Rbm::Rbm(arma::mat &w, arma::mat &bv, arma::mat &bh) -: m_w(w) +Rbm::Rbm(const Params& params, arma::mat &w, arma::mat &bv, arma::mat &bh) +: m_params(params) +, m_w(w) , m_bv(bv) , m_bh(bh) { Noise_Init(&m_noise, 0x32727155); - uniform(m_w); - uniform(m_bh); - uniform(m_bv); + uniform(m_w, 0.0, 0.1); + uniform(m_bh, 0.0, 0.1); + uniform(m_bv, 0.0, 0.1); } Rbm::Rbm(const Rbm& orig) -: m_w(orig.m_w) +: m_params(orig.m_params) +, m_w(orig.m_w) , m_bv(orig.m_bv) , m_bh(orig.m_bh) { @@ -36,7 +38,7 @@ Rbm::~Rbm() { } -void Rbm::train(const arma::mat& batch, size_t numEpochs, size_t miniBatchSize, const Params& params, IListener* pListener) +void Rbm::train(const arma::mat& batch, size_t numEpochs, size_t miniBatchSize, IListener* pListener) { size_t epoch; size_t gibbs; @@ -66,8 +68,8 @@ void Rbm::train(const arma::mat& batch, size_t numEpochs, size_t miniBatchSize, trainingSizeRemain -= toSlice; batchRowIndex += toSlice; size_t miniBatchSizeActual = miniBatch.n_rows; - double learning_rate = params.m_learningRate/std::min(miniBatchSizeActual, trainingSize); - double weight_decay = params.m_weightDecay/std::min(miniBatchSizeActual, trainingSize); + double learning_rate = m_params.m_learningRate/std::min(miniBatchSizeActual, trainingSize); + double weight_decay = m_params.m_weightDecay/std::min(miniBatchSizeActual, trainingSize); arma::mat vis_state(miniBatchSizeActual, m_w.n_rows); arma::mat vis_probs(miniBatchSizeActual, m_w.n_rows); @@ -78,7 +80,7 @@ void Rbm::train(const arma::mat& batch, size_t numEpochs, size_t miniBatchSize, { // Create hidden layer base on training data - if (params.m_doSampleBatch) + if (m_params.m_doSampleBatch) { // When the hidden units are being driven by data, always use stochastic binary states vis_state = sample(miniBatch); @@ -92,7 +94,7 @@ void Rbm::train(const arma::mat& batch, size_t numEpochs, size_t miniBatchSize, hid_probs = probsLogistic(hid_state); // Sample hidden - if (params.m_doRaoBlackwell) + if (m_params.m_doRaoBlackwell) { hid_state = hid_probs; } @@ -106,7 +108,7 @@ void Rbm::train(const arma::mat& batch, size_t numEpochs, size_t miniBatchSize, grad_bias_v = sum(vis_state, 0); grad_bias_h = sum(hid_state, 0); - for (gibbs=0; gibbs < params.m_numGibbs; gibbs++) + for (gibbs=0; gibbs < m_params.m_numGibbs; gibbs++) { // Create hidden representation given v @@ -115,7 +117,7 @@ void Rbm::train(const arma::mat& batch, size_t numEpochs, size_t miniBatchSize, // Create visible reconstruction (a fantasy...) given hid vis_state = hid_state * m_w.t() + arma::repmat(m_bv, miniBatchSizeActual, 1); vis_probs = probsLogistic(vis_state); - if (params.m_doSampleVisible) + if (m_params.m_doSampleVisible) { vis_state = sample(vis_probs); } @@ -139,9 +141,9 @@ void Rbm::train(const arma::mat& batch, size_t numEpochs, size_t miniBatchSize, status.L1 = accu(abs(m_w)); status.L2 = accu(m_w % m_w); - momentum_bias_v = params.m_momentum*momentum_bias_v + grad_bias_v; - momentum_bias_h = params.m_momentum*momentum_bias_h + grad_bias_h; - momentum_weights = params.m_momentum*momentum_weights + grad_weight - status.L2*penalty_weights; + momentum_bias_v = m_params.m_momentum*momentum_bias_v + grad_bias_v; + momentum_bias_h = m_params.m_momentum*momentum_bias_h + grad_bias_h; + momentum_weights = m_params.m_momentum*momentum_weights + grad_weight - status.L2*penalty_weights; m_bv += learning_rate*momentum_bias_v; m_bh += learning_rate*momentum_bias_h; @@ -154,7 +156,7 @@ void Rbm::train(const arma::mat& batch, size_t numEpochs, size_t miniBatchSize, status.epoch = epoch; arma::mat diffErr = miniBatch - vis_probs; arma::mat diffErr_squared = diffErr % diffErr; - status.err = accu(diffErr_squared); + status.err = accu(diffErr_squared)/diffErr_squared.n_elem; if (pListener) { diff --git a/source/Rbm.hpp b/source/Rbm.hpp index 6355edb..cccd264 100644 --- a/source/Rbm.hpp +++ b/source/Rbm.hpp @@ -24,7 +24,7 @@ public: struct Params { Params() - : m_weightDecay(0.01) + : m_weightDecay(0.0) , m_learningRate(0.1) , m_momentum(0.5) , m_doRaoBlackwell(true) @@ -65,11 +65,11 @@ public: } }; - Rbm(arma::mat &w, arma::mat &bv, arma::mat &bh); + Rbm(const Params& params, arma::mat &w, arma::mat &bv, arma::mat &bh); Rbm(const Rbm& orig); virtual ~Rbm(); - void train(arma::mat const &batch, size_t numEpochs, size_t sizeMiniBatch, Params const ¶ms, IListener *pListener); + void train(arma::mat const &batch, size_t numEpochs, size_t sizeMiniBatch, IListener *pListener); arma::mat toHidden(const arma::mat &v); arma::mat toVisible(const arma::mat &h); arma::mat& weights(); @@ -78,6 +78,7 @@ public: private: noise_gen_t m_noise; + const Params &m_params; arma::mat &m_w; arma::mat &m_bh; arma::mat &m_bv; diff --git a/source/main.cpp b/source/main.cpp index 9e013ab..86cf75c 100644 --- a/source/main.cpp +++ b/source/main.cpp @@ -23,45 +23,105 @@ class RbmListener : public Rbm::IListener } }; +arma::mat loadTraining(const char *pFilename) +{ + uint32_t numTraining = 0; + uint32_t numVisible = 0; + FILE *pFile; + + pFile = fopen(pFilename,"r"); + + if (!pFile) + { + return 0; + } + + int result = fscanf(pFile, "%d\n", &numTraining); + if (result < 0) + { + return 0; + } + result = fscanf(pFile, "%d\n", &numVisible); + if (result < 0) + { + return 0; + } + arma::mat data = arma::zeros(numTraining, numVisible); + + uint32_t i, j; + for (i=0; i < numTraining; i++) + { + for (j=0; j < numVisible; j++) + { + float v; + int result = fscanf(pFile, "%f", &v); + if (result > 0) + { + data(i, j) = v; + } + } + } + fclose(pFile); + return data; + +} + +void saveWeight(const char *pFilename, size_t numVisibleX, size_t numVisibleY, const arma::mat &w, const arma::mat &bv, const arma::mat &bh) +{ + FILE *pFile; + + pFile = fopen(pFilename,"w"); + + if (!pFile) + return; + + size_t numHidden = bh.n_elem; + size_t numVisible = bv.n_elem; + fprintf(pFile, "%d %d %d\n", (int)numVisibleX, (int)numVisibleY, (int)numHidden); + + uint32_t i, j; + + for (i=0; i < numVisible; i++) + { + fprintf(pFile, "%3.6f\n", bv(i)); + } + for (i=0; i < numHidden; i++) + { + fprintf(pFile, "%3.6f\n", bh(i)); + } + for (i=0; i < numVisible; i++) + { + for (j=0; j < numHidden; j++) + { + fprintf(pFile, "%3.6f ", w(i,j)); + } + fprintf(pFile, "\n"); + } + + fclose(pFile); +} + int main() { printf("Hallo, Welt!\n"); - // Position of a particle // | - arma::vec Pos = {{0}, // | (0,1) - {1}}; // +---x--> - - // Rotation matrix - double phi = -3.1416/2; - arma::mat RotM = {{+cos(phi), -sin(phi)}, - {+sin(phi), +cos(phi)}}; - - Pos.print("Current position of the particle:"); - std::cout << "Rotating the point " << phi*180/3.1416 << " deg" << std::endl; - - Pos = RotM*Pos; - - Pos.print("New position of the particle:"); // ^ - // x (1,0) - // | - arma::mat Z = arma::eye(4000,4000); -// Z.print("Z:"); - printf("Z.n_rows = %d\n", (int)Z.n_rows); - printf("Z.n_cols = %d\n", (int)Z.n_cols); - printf("Z.n_elem = %d\n", (int)Z.n_elem); - // +------> - - RbmListener statusDisplay; Rbm::Params params; - arma::mat w = arma::zeros(28*28, 64); - arma::mat bv = arma::zeros(1, 28*28); - arma::mat bh = arma::zeros(1, 64); - Rbm rbm(w, bv, bh); - arma::mat batch = arma::randu(100, 28*28); - rbm.train(batch, 10, 10, params, &statusDisplay); - - arma::mat v = arma::randu(100, 28*28); + arma::mat batch = loadTraining("mnist_2.training.dat"); + + size_t numTraining = batch.n_rows; + size_t numVisible = batch.n_cols; + size_t numHidden = 64; + + printf("Loaded %d training samples\n", (int)numTraining); + + arma::mat w = arma::zeros(numVisible, numHidden); + arma::mat bv = arma::zeros(1, numVisible); + arma::mat bh = arma::zeros(1, numHidden); + Rbm rbm(params, w, bv, bh); + rbm.train(batch, 100, 100, &statusDisplay); + saveWeight("mnist_2.weights.dat", 28, 28, w, bv, bh); + arma::mat v = arma::randu(numTraining, numVisible); arma::mat h = rbm.toHidden(v); arma::mat r = rbm.toVisible(h); return 0;