- use custom noise for bugfixing
- init weights with zero mean uniform noise git-svn-id: http://moon:8086/svn/software/trunk/projects/Rbm@598 b431acfa-c32f-4a4a-93f1-934dc6c82436
This commit is contained in:
+18
-9
@@ -13,14 +13,13 @@
|
||||
|
||||
#include "Rbm.hpp"
|
||||
|
||||
Rbm::Rbm(size_t numVisible, size_t numHidden, double weightInit)
|
||||
Rbm::Rbm(size_t numVisible, size_t numHidden)
|
||||
: m_params()
|
||||
, m_weightInit(weightInit)
|
||||
, m_w(numVisible, numHidden)
|
||||
, m_bv(1, numVisible)
|
||||
, m_bh(1, numHidden)
|
||||
{
|
||||
weightsInit(0.0, m_weightInit);
|
||||
Noise_Init(&m_noise, 0x32727155);
|
||||
}
|
||||
|
||||
Rbm::Rbm(const Rbm& orig)
|
||||
@@ -33,13 +32,14 @@ Rbm::Rbm(const Rbm& orig)
|
||||
|
||||
Rbm::~Rbm()
|
||||
{
|
||||
Noise_Free(&m_noise);
|
||||
}
|
||||
|
||||
void Rbm::weightsInit(double mu, double stddev)
|
||||
void Rbm::weightsInit(double stddev, double mu)
|
||||
{
|
||||
uniform(m_w, mu, stddev);
|
||||
uniform(m_bh, mu, stddev);
|
||||
uniform(m_bv, mu, stddev);
|
||||
uniform(m_w, stddev, mu);
|
||||
uniform(m_bv, stddev, mu);
|
||||
uniform(m_bh, stddev, mu);
|
||||
|
||||
}
|
||||
|
||||
@@ -53,7 +53,6 @@ Json::Value Rbm::toJson() const
|
||||
Json::Value rbm;
|
||||
rbm["numVisible"] = m_bv.n_elem;
|
||||
rbm["numHidden"] = m_bh.n_elem;
|
||||
rbm["weightInit"] = m_weightInit;
|
||||
rbm["params"] = m_params.toJson();
|
||||
return rbm;
|
||||
}
|
||||
@@ -239,9 +238,19 @@ arma::mat Rbm::toVisibleProbs(const arma::mat &hidden) const
|
||||
return probsLogistic(toVisibleState(hidden));
|
||||
}
|
||||
|
||||
void Rbm::uniform(arma::mat& srcDst, double mu, double stdDev)
|
||||
void Rbm::uniform(arma::mat& srcDst, double stdDev, double mu)
|
||||
{
|
||||
#if 1
|
||||
for (size_t i=0; i < srcDst.n_rows; i++)
|
||||
{
|
||||
for (size_t j=0; j < srcDst.n_cols; j++)
|
||||
{
|
||||
srcDst(i, j) = stdDev*(Noise_Uniform(&m_noise) + mu - 0.5);
|
||||
}
|
||||
}
|
||||
#else
|
||||
srcDst = stdDev*arma::randu(srcDst.n_rows, srcDst.n_cols) + mu;
|
||||
#endif
|
||||
}
|
||||
|
||||
const arma::mat& Rbm::w() const
|
||||
|
||||
+6
-4
@@ -18,6 +18,7 @@
|
||||
|
||||
#include <armadillo>
|
||||
#include <jsoncpp/json/json.h>
|
||||
#include "noise.h"
|
||||
|
||||
class Rbm
|
||||
{
|
||||
@@ -108,11 +109,11 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
Rbm(size_t numVisible, size_t numHidden, double weightInit=0.01);
|
||||
Rbm(size_t numVisible, size_t numHidden);
|
||||
Rbm(const Rbm& orig);
|
||||
virtual ~Rbm();
|
||||
|
||||
void weightsInit(double mu, double stddev);
|
||||
void weightsInit(double stddev, double mu=0.0);
|
||||
void train(arma::mat const &batch, size_t miniBatchSize, size_t numEpochs, IListener *pListener);
|
||||
|
||||
arma::mat toHiddenState(const arma::mat &visible) const;
|
||||
@@ -128,16 +129,17 @@ public:
|
||||
|
||||
private:
|
||||
const Params m_params;
|
||||
double m_weightInit;
|
||||
arma::mat sample(arma::mat const &src);
|
||||
static arma::mat probsLogistic(arma::mat const &src);
|
||||
void uniform(arma::mat &srcDst, double mu=0.0, double stdDev=1.0);
|
||||
void uniform(arma::mat &srcDst, double stdDev=1.0, double mu=0.5);
|
||||
|
||||
protected:
|
||||
arma::mat m_w;
|
||||
arma::mat m_bh;
|
||||
arma::mat m_bv;
|
||||
|
||||
private:
|
||||
noise_gen_t m_noise;
|
||||
};
|
||||
|
||||
#endif /* RBM_HPP */
|
||||
|
||||
+1
-1
@@ -120,7 +120,7 @@ void Stack::weightsInit(double stddev)
|
||||
Layer *pLayer = m_pLayers;
|
||||
while(pLayer)
|
||||
{
|
||||
pLayer->weightsInit(0, stddev);
|
||||
pLayer->weightsInit(stddev);
|
||||
pLayer = pLayer->upper;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
// --------------------------------------------------------------
|
||||
// --------------------------------------------------------------
|
||||
|
||||
#include <string.h>
|
||||
#include <stdlib.h>
|
||||
#include <time.h>
|
||||
#include <math.h>
|
||||
|
||||
#include "noise.h"
|
||||
|
||||
// --------------------------------------------------------------
|
||||
// internal funcs
|
||||
// --------------------------------------------------------------
|
||||
#define PM_IA 16807
|
||||
#define PM_IM 2147483647
|
||||
#define PM_AM (1.0/PM_IM)
|
||||
#define PM_IQ 127773
|
||||
#define PM_IR 2836
|
||||
#define PM_MASK 123459876
|
||||
|
||||
// 'Minimal' random number generator of Park and Miller. Returns a uniform random deviate
|
||||
// between 0.0 and 1.0. Set or reset idum to any integer value (except the unlikely value MASK)
|
||||
// to initialize the sequence; idum must not be altered between calls for successive deviates in
|
||||
// a sequence.
|
||||
double ran0(long *idum)
|
||||
{
|
||||
long k;
|
||||
double ans;
|
||||
|
||||
*idum ^= PM_MASK; // XORing with MASK allows use of zero and other
|
||||
k=(*idum)/PM_IQ; // simple bit patterns for idum.
|
||||
*idum=PM_IA*(*idum-k*PM_IQ)-PM_IR *k; // Compute idum=(IA*idum) % IM without over-
|
||||
// flows by Schrage’s method.
|
||||
if (*idum < 0)
|
||||
*idum += PM_IM;
|
||||
|
||||
ans=PM_AM*(*idum); // Convert idumto a floating result.
|
||||
*idum ^= PM_MASK; // Unmask before return.
|
||||
|
||||
return ans;
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------
|
||||
// Exported functions
|
||||
// --------------------------------------------------------------
|
||||
void Noise_Init(noise_gen_t *pObj, long seed)
|
||||
{
|
||||
|
||||
pObj->state = seed;
|
||||
}
|
||||
|
||||
void Noise_Free(noise_gen_t *pObj)
|
||||
{
|
||||
}
|
||||
|
||||
// 0 .. 1
|
||||
double Noise_Uniform(noise_gen_t *pObj)
|
||||
{
|
||||
return ran0(&pObj->state);
|
||||
}
|
||||
|
||||
double Noise_Gaussian(noise_gen_t *pObj)
|
||||
{
|
||||
double U1, U2, V1, V2, S, Y;
|
||||
|
||||
do
|
||||
{
|
||||
U1 = Noise_Uniform(pObj); // U1=[0,1]
|
||||
U2 = Noise_Uniform(pObj); // U2=[0,1]
|
||||
V1 = 2 * U1 - 1; // V1=[-1,1]
|
||||
V2 = 2 * U2 - 1; // V2=[-1,1]
|
||||
S = V1 * V1 + V2 * V2;
|
||||
|
||||
} while (S >= 1);
|
||||
|
||||
// X = sqrt(-2 * log(S) / S) * V1;
|
||||
Y = sqrt(-2 * log(S) / S) * V2;
|
||||
|
||||
return Y;
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
/*
|
||||
==============================================================================
|
||||
|
||||
noise.h
|
||||
Created: 21 Sep 2014 1:55:07pm
|
||||
Author: jens
|
||||
|
||||
==============================================================================
|
||||
*/
|
||||
// --------------------------------------------------------------
|
||||
#ifndef _NOISE_H_
|
||||
#define _NOISE_H_
|
||||
|
||||
#define __func__ ""
|
||||
// --------------------------------------------------------------
|
||||
// Types
|
||||
// --------------------------------------------------------------
|
||||
typedef struct _snoise_gen_t
|
||||
{
|
||||
long state;
|
||||
} noise_gen_t;
|
||||
|
||||
// --------------------------------------------------------------
|
||||
#if defined(__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
// --------------------------------------------------------------
|
||||
// Exported functions
|
||||
// --------------------------------------------------------------
|
||||
void Noise_Init(noise_gen_t *pObj, long seed);
|
||||
void Noise_Free(noise_gen_t *pObj);
|
||||
double Noise_Uniform(noise_gen_t *pObj);
|
||||
double Noise_Gaussian(noise_gen_t *pObj);
|
||||
|
||||
#if defined(__cplusplus)
|
||||
}
|
||||
|
||||
#endif
|
||||
// --------------------------------------------------------------
|
||||
|
||||
#endif // _NOISE_H_
|
||||
Reference in New Issue
Block a user