getting Rnn to work

JayRnn.py

@@ -19,12 +19,12 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src'))
 from rbm.entity import EntityParams, TrainingParams, Entity
 from rbm.matrix import np, Mat
 from model.model import Model
-from stack.rnn_helper import vocab_size, shift_left, concat, vec2idx, idx2ch, ch2idx, idx2vec
+from stack.rnn_helper import vocab_size, shift_left, clamp, concat, split, vec2idx, idx2ch, ch2idx, idx2vec
 from rbm.train import train
 from rbm.status import Status
 
 # ── Hyper-parameters ──────────────────────────────────────────────────────────
-TEXT    = "JAY IS COOL"   # the character sequence to learn (matches diagram example)
+TEXT    = "  JAY IS COOL"   # the character sequence to learn (matches diagram example)
 WIN    = 3         # sliding-window width  (= N in the diagram)
 STRIDE = 1     	   # sliding-window step size
 UNITS  = 1
@@ -44,18 +44,11 @@ TRAIN_PARAMS = TrainingParams(
 #U1: "AY IS COOL "
 #U2: "Y IS COOL  "
 
-def vec2str(mat: Mat):
-	result = ''
-	for vec in mat:
-		result += idx2ch(vec2idx(vec))
-	return result
+def vec2str(mat: Mat, axis=0):
+	return idx2ch(vec2idx(mat, axis))
 
 def str2vec(ch_str: str) -> Mat:
-	result = Mat([len(ch_str), vocab_size()])
-	for i, ch in enumerate(ch_str):
-		vec = idx2vec(ch2idx(ch))
-		result[i, :] = vec
-	return result
+	return idx2vec(ch2idx(ch_str))
 
 def to_window(text: str):
 	win_text = []
@@ -85,16 +78,25 @@ class RnnModel(Model):
 		for unit in self.units:
 			train(unit, vc, status)
 			# Update context portion of vc
-			vc[:, WIN*vocab_size():] = unit.forward(vc)
+			vc[1:, WIN*vocab_size():] = unit.forward(vc)[0:-1,:]
 
-	def forward_step(self, v_curr: Mat):
-		pass
+	def forward_step(self, _vc: Mat):
+		for unit in self.units:
+			_c = unit.forward(_vc)
+			_vc = unit.reconstruct(_c)
+		return _vc
 
 	def forward(self, x: Mat):
 		pass
 
 
 if __name__ == "__main__":
+	vec = idx2vec(ch2idx("JENS"))
+	vec = clamp(vec, axis=1)
+	indices = vec2idx(vec, axis=1)
+	ch_str = idx2ch(indices)
+	print(ch_str)
+
 	win_text = to_window(TEXT)
 	print(win_text)
 
@@ -107,7 +109,28 @@ if __name__ == "__main__":
 
 	# vc contains vis + context
 	# context will be updated after training
-	vc = concat(batch, np.zeros([len(batch), H_SIZE]))
-	model.train(vc, status=Status())
+	c_train = np.zeros([len(batch), H_SIZE])
+	vc_train = concat(batch, c_train, axis=1)
+#	model.train(vc_train, status=Status())
 	model.save()
+	seed = str2vec('JA^')
+	vc_forward = concat(seed.flatten(), c_train[0])
+	for i in range(len(TEXT)):
+		vc_forward = model.forward_step(vc_forward)
+		v_forward, c_forward = split(vc_forward.flatten(), H_SIZE)
+		v_forward_clamp = clamp(v_forward.reshape([WIN, vocab_size()]), axis=1)
+		print(f"Forward : {vec2str(v_forward_clamp, axis=1)}")
+		v_forward = shift_left(v_forward_clamp.reshape([1, WIN*vocab_size()]), vocab_size())
+		print(f"Seed    : {vec2str(v_forward.reshape([WIN, vocab_size()]), axis=1)}")
+		vc_forward = concat(v_forward, c_forward.reshape([1, H_SIZE]), axis=1)
+
+
+	if 0:
+		vc = model.forward_step(vc_forward)
+		vc_forward = split(vc.flatten(), c_forward)
+		print(vc_forward)
+
+		win, _ = split(vc.flatten(), len(seed))
+		v_str = vec2str(win.reshape(WIN, vocab_size()))
+		print(v_str)
 

src/stack/rnn_helper.py

@@ -1,33 +1,43 @@
 from rbm.matrix import Mat, np
 
-VOCAB = ' .!?ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
+VOCAB = '^ .!?ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
 _CH2IDX = {ch: i for i, ch in enumerate(VOCAB)}
 
-def vec2idx(vec: Mat) -> int:
-	index = np.argmax(vec)
-	return int(index)
+def clamp(vec: Mat, axis=0):
+	indices = vec2idx(vec, axis=axis)
+	return idx2vec(indices)
 
-def idx2vec(index: int) -> np.ndarray:
-	result = np.zeros([1, len(VOCAB)])
-	result[:, index] = 1
+def vec2idx(vec: Mat, axis=0) -> Mat:
+	indices = np.argmax(vec, axis=axis)
+	return indices
+
+def idx2vec(indices: Mat) -> Mat:
+	result = np.zeros([len(indices), len(VOCAB)])
+	for i, index in enumerate(indices):
+		result[i, int(index)] = 1
 	return result
 
-def ch2idx(ch: str) -> int:
-	idx = _CH2IDX[ch]
+def ch2idx(ch_str: str) -> Mat:
+	idx = np.zeros(len(ch_str))
+	for i, ch in enumerate(ch_str):
+		idx[i] = int(_CH2IDX[ch])
 	return idx
 
-def idx2ch(idx: int) -> str:
-	return VOCAB[idx]
+def idx2ch(indices: Mat) -> str:
+	result = ''
+	for index in indices:
+		result += VOCAB[int(index)]
+	return result
 
 def vocab_size():
 	return len(VOCAB)
 
-def concat(v: np.ndarray, c: np.ndarray) -> np.ndarray:
-	return np.concatenate([v, c], axis=1)
+def concat(v: np.ndarray, c: np.ndarray, axis=0) -> np.ndarray:
+	return np.concatenate([v, c], axis=axis)
 
-def split(vc: np.ndarray, v_size: np.ndarray|int) -> tuple[np.ndarray, np.ndarray]:
-	n = len(v_size) if isinstance(v_size, np.ndarray) else v_size
-	return vc[:n], vc[n:]
+def split(vc: np.ndarray, h_size: int, axis=0) -> tuple[np.ndarray, np.ndarray]:
+	v_len = vc.shape[axis]-h_size
+	return vc[0:v_len], vc[v_len:]
 
 def shift_right(m: np.ndarray, amount: int = 1) -> np.ndarray:
 	return np.hstack([np.zeros((m.shape[0], amount)), m[:, :-amount]])