diff --git a/JayRnn.py b/JayRnn.py index ab7ab75..0e51fdc 100644 --- a/JayRnn.py +++ b/JayRnn.py @@ -19,12 +19,12 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src')) from rbm.entity import EntityParams, TrainingParams, Entity from rbm.matrix import np, Mat from model.model import Model -from stack.rnn_helper import vocab_size, shift_left, concat, vec2idx, idx2ch, ch2idx, idx2vec +from stack.rnn_helper import vocab_size, shift_left, clamp, concat, split, vec2idx, idx2ch, ch2idx, idx2vec from rbm.train import train from rbm.status import Status # ── Hyper-parameters ────────────────────────────────────────────────────────── -TEXT = "JAY IS COOL" # the character sequence to learn (matches diagram example) +TEXT = " JAY IS COOL" # the character sequence to learn (matches diagram example) WIN = 3 # sliding-window width (= N in the diagram) STRIDE = 1 # sliding-window step size UNITS = 1 @@ -44,18 +44,11 @@ TRAIN_PARAMS = TrainingParams( #U1: "AY IS COOL " #U2: "Y IS COOL " -def vec2str(mat: Mat): - result = '' - for vec in mat: - result += idx2ch(vec2idx(vec)) - return result +def vec2str(mat: Mat, axis=0): + return idx2ch(vec2idx(mat, axis)) def str2vec(ch_str: str) -> Mat: - result = Mat([len(ch_str), vocab_size()]) - for i, ch in enumerate(ch_str): - vec = idx2vec(ch2idx(ch)) - result[i, :] = vec - return result + return idx2vec(ch2idx(ch_str)) def to_window(text: str): win_text = [] @@ -85,16 +78,25 @@ class RnnModel(Model): for unit in self.units: train(unit, vc, status) # Update context portion of vc - vc[:, WIN*vocab_size():] = unit.forward(vc) + vc[1:, WIN*vocab_size():] = unit.forward(vc)[0:-1,:] - def forward_step(self, v_curr: Mat): - pass + def forward_step(self, _vc: Mat): + for unit in self.units: + _c = unit.forward(_vc) + _vc = unit.reconstruct(_c) + return _vc def forward(self, x: Mat): pass if __name__ == "__main__": + vec = idx2vec(ch2idx("JENS")) + vec = clamp(vec, axis=1) + indices = vec2idx(vec, axis=1) + ch_str = idx2ch(indices) + print(ch_str) + win_text = to_window(TEXT) print(win_text) @@ -107,7 +109,28 @@ if __name__ == "__main__": # vc contains vis + context # context will be updated after training - vc = concat(batch, np.zeros([len(batch), H_SIZE])) - model.train(vc, status=Status()) + c_train = np.zeros([len(batch), H_SIZE]) + vc_train = concat(batch, c_train, axis=1) +# model.train(vc_train, status=Status()) model.save() + seed = str2vec('JA^') + vc_forward = concat(seed.flatten(), c_train[0]) + for i in range(len(TEXT)): + vc_forward = model.forward_step(vc_forward) + v_forward, c_forward = split(vc_forward.flatten(), H_SIZE) + v_forward_clamp = clamp(v_forward.reshape([WIN, vocab_size()]), axis=1) + print(f"Forward : {vec2str(v_forward_clamp, axis=1)}") + v_forward = shift_left(v_forward_clamp.reshape([1, WIN*vocab_size()]), vocab_size()) + print(f"Seed : {vec2str(v_forward.reshape([WIN, vocab_size()]), axis=1)}") + vc_forward = concat(v_forward, c_forward.reshape([1, H_SIZE]), axis=1) + + + if 0: + vc = model.forward_step(vc_forward) + vc_forward = split(vc.flatten(), c_forward) + print(vc_forward) + + win, _ = split(vc.flatten(), len(seed)) + v_str = vec2str(win.reshape(WIN, vocab_size())) + print(v_str) diff --git a/src/stack/rnn_helper.py b/src/stack/rnn_helper.py index 6d0b586..732b6fc 100644 --- a/src/stack/rnn_helper.py +++ b/src/stack/rnn_helper.py @@ -1,33 +1,43 @@ from rbm.matrix import Mat, np -VOCAB = ' .!?ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' +VOCAB = '^ .!?ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' _CH2IDX = {ch: i for i, ch in enumerate(VOCAB)} -def vec2idx(vec: Mat) -> int: - index = np.argmax(vec) - return int(index) +def clamp(vec: Mat, axis=0): + indices = vec2idx(vec, axis=axis) + return idx2vec(indices) -def idx2vec(index: int) -> np.ndarray: - result = np.zeros([1, len(VOCAB)]) - result[:, index] = 1 +def vec2idx(vec: Mat, axis=0) -> Mat: + indices = np.argmax(vec, axis=axis) + return indices + +def idx2vec(indices: Mat) -> Mat: + result = np.zeros([len(indices), len(VOCAB)]) + for i, index in enumerate(indices): + result[i, int(index)] = 1 return result -def ch2idx(ch: str) -> int: - idx = _CH2IDX[ch] +def ch2idx(ch_str: str) -> Mat: + idx = np.zeros(len(ch_str)) + for i, ch in enumerate(ch_str): + idx[i] = int(_CH2IDX[ch]) return idx -def idx2ch(idx: int) -> str: - return VOCAB[idx] +def idx2ch(indices: Mat) -> str: + result = '' + for index in indices: + result += VOCAB[int(index)] + return result def vocab_size(): return len(VOCAB) -def concat(v: np.ndarray, c: np.ndarray) -> np.ndarray: - return np.concatenate([v, c], axis=1) +def concat(v: np.ndarray, c: np.ndarray, axis=0) -> np.ndarray: + return np.concatenate([v, c], axis=axis) -def split(vc: np.ndarray, v_size: np.ndarray|int) -> tuple[np.ndarray, np.ndarray]: - n = len(v_size) if isinstance(v_size, np.ndarray) else v_size - return vc[:n], vc[n:] +def split(vc: np.ndarray, h_size: int, axis=0) -> tuple[np.ndarray, np.ndarray]: + v_len = vc.shape[axis]-h_size + return vc[0:v_len], vc[v_len:] def shift_right(m: np.ndarray, amount: int = 1) -> np.ndarray: return np.hstack([np.zeros((m.shape[0], amount)), m[:, :-amount]])