diff --git a/JayRnn.py b/JayRnn.py index b85309e..5590cf2 100644 --- a/JayRnn.py +++ b/JayRnn.py @@ -24,19 +24,19 @@ from rbm.train import train from rbm.status import Status # ── Hyper-parameters ────────────────────────────────────────────────────────── -TEXT = "HALLO MAUSI! SUPER HASE!" # the character sequence to learn (matches diagram example) +TEXT = "Call me Ishmael. Some years ago" # the character sequence to learn (matches diagram example) WIN = 3 # sliding-window width (= N in the diagram) STRIDE = 1 # sliding-window step size UNITS = 3 -H_SIZE = 64 # hidden units per RBM cell +H_SIZE = 128 # hidden units per RBM cell NUM_EPOCHS = 1000 NUM_ITERATIONS = 1 TRAIN_PARAMS = TrainingParams( - learning_rate = 0.1, + learning_rate = 0.2, momentum = 0.5, num_epochs = NUM_EPOCHS, - do_rao_blackwell = True, + do_rao_blackwell = False, num_gibbs_samples= 1 ) #WIN=3 @@ -92,27 +92,29 @@ class RnnModel(Model): def train(self, vc: Mat, status: Status = None): _v, _c = split(vc, H_SIZE, axis=1) - for delay, unit in enumerate(self.units): - _vd = batch_delay(_v, delay) + for unit in self.units: + _vd = batch_delay(_v, 1) _vc = concat(_vd, _c, axis=1) for i in range(vc.shape[0]): print(f"train: {unit.name}:{vc2char(_vc[i,:])}") train(unit, _vc, status) # For the next unit: Update context portion of vc - _c = unit.forward(_vc) + _c = unit.forward(vc) def forward_step(self, _vc: Mat): + text = '' for unit in self.units: # print(f"forward_step in : {unit.name}:{vc2char(_vc)}") _c = unit.forward(_vc) _vc = unit.reconstruct(_c) - print(f"forward_step out: {unit.name}:{vc2char(_vc)}") +# print(f"forward_step out: {unit.name}:{vc2char(_vc)}") + text += vc2char(_vc)[WIN-1] _v, _ = split(_vc, H_SIZE, axis=1) _v_cl = clamp(_v.reshape([WIN, vocab_size()]), axis=1) _v = shift_left(_v_cl.reshape([1, WIN*vocab_size()]), vocab_size()) _vc = concat(_v, _c, axis=1) - return _vc + return _vc, text def forward(self, x: Mat): pass @@ -147,10 +149,14 @@ if __name__ == "__main__": model.save() # test the model - seed_str = 'HA' + seed_str = 'Cal' seed_padded = seed_str + '^'*(WIN-len(seed_str)) v_test = str2vec(seed_padded).reshape([1, WIN*vocab_size()]) c_test = np.zeros([1, H_SIZE]) vc_test = concat(v_test, c_test, axis=1) + text = '' for i in range(len(TEXT)): - vc_test = model.forward_step(vc_test) + vc_test, text_frag = model.forward_step(vc_test) + text += text_frag + + print(text) diff --git a/src/stack/rnn_helper.py b/src/stack/rnn_helper.py index 736c8cd..475319b 100644 --- a/src/stack/rnn_helper.py +++ b/src/stack/rnn_helper.py @@ -1,6 +1,6 @@ from rbm.matrix import Mat, np -VOCAB = '^ .!?ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' +VOCAB = '^ -.!?ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789' _CH2IDX = {ch: i for i, ch in enumerate(VOCAB)} def clamp(vec: Mat, axis=0):