diff --git a/JayRnn.py b/JayRnn.py index 4ecc3b5..b85309e 100644 --- a/JayRnn.py +++ b/JayRnn.py @@ -24,16 +24,16 @@ from rbm.train import train from rbm.status import Status # ── Hyper-parameters ────────────────────────────────────────────────────────── -TEXT = "0123456789" # the character sequence to learn (matches diagram example) +TEXT = "HALLO MAUSI! SUPER HASE!" # the character sequence to learn (matches diagram example) WIN = 3 # sliding-window width (= N in the diagram) STRIDE = 1 # sliding-window step size UNITS = 3 -H_SIZE = 32 # hidden units per RBM cell +H_SIZE = 64 # hidden units per RBM cell NUM_EPOCHS = 1000 NUM_ITERATIONS = 1 TRAIN_PARAMS = TrainingParams( - learning_rate = 0.04, + learning_rate = 0.1, momentum = 0.5, num_epochs = NUM_EPOCHS, do_rao_blackwell = True, @@ -76,6 +76,9 @@ def batch_delay(_batch: Mat, delay=0): def vc2char(_vc: Mat): _v, _ = split(_vc.reshape(1, WIN*vocab_size()+H_SIZE), H_SIZE, axis=1) + return v2char(_v) + +def v2char(_v: Mat): return vec2str(_v.reshape([WIN, vocab_size()]), axis=1) class RnnModel(Model): @@ -101,13 +104,14 @@ class RnnModel(Model): def forward_step(self, _vc: Mat): for unit in self.units: +# print(f"forward_step in : {unit.name}:{vc2char(_vc)}") _c = unit.forward(_vc) _vc = unit.reconstruct(_c) + print(f"forward_step out: {unit.name}:{vc2char(_vc)}") _v, _ = split(_vc, H_SIZE, axis=1) - _v_cl = clamp(_v.reshape([WIN, vocab_size()]), axis=1).reshape([1, WIN * vocab_size()]) - _v = shift_left(_v_cl, 1) + _v_cl = clamp(_v.reshape([WIN, vocab_size()]), axis=1) + _v = shift_left(_v_cl.reshape([1, WIN*vocab_size()]), vocab_size()) _vc = concat(_v, _c, axis=1) - print(f"forward_step: {unit.name}:{vc2char(_vc)}") return _vc def forward(self, x: Mat): @@ -139,27 +143,14 @@ if __name__ == "__main__": # context will be updated after training c_train = np.zeros([batch.shape[0], H_SIZE]) vc_train = concat(batch, c_train, axis=1) -# model.train(vc_train, status=Status()) -# model.save() + model.train(vc_train, status=Status()) + model.save() # test the model - seed_str = '1' - seed_padded = '^'*(WIN-len(seed_str)) + seed_str + seed_str = 'HA' + seed_padded = seed_str + '^'*(WIN-len(seed_str)) v_test = str2vec(seed_padded).reshape([1, WIN*vocab_size()]) c_test = np.zeros([1, H_SIZE]) vc_test = concat(v_test, c_test, axis=1) - for i in range(len(TEXT) + 5): + for i in range(len(TEXT)): vc_test = model.forward_step(vc_test) - if 0: - for i in range(len(TEXT)+5): - v_mat = v_forward.reshape([WIN, vocab_size()]) - print(f"Forward {i:02d}: {vec2str(v_mat, axis=1)}") - v_predict, context = model.forward_step(v_mat, context) - v_predict_mat = v_predict.reshape([WIN, vocab_size()]) - v_predict_clamp = clamp(v_predict_mat, axis=1) - v_predict_str = vec2str(v_predict_clamp, axis=1) - text_predict += v_predict_str[WIN-1] - print(f"Predict {i:02d}: {v_predict_str}") - v_forward = shift_left(v_predict_clamp.reshape([1, WIN*vocab_size()]), vocab_size()) - - print(seed_padded[0:WIN-1] + text_predict) \ No newline at end of file