- no "^" in vocabular

- JayRnn: wrong but stable

JayRnn.py

@@ -25,7 +25,7 @@ from rbm.status import Status
 
 # ── Hyper-parameters ──────────────────────────────────────────────────────────
 TEXT    = "Call me Ishmael. Some years ago" # the character sequence to learn (matches diagram example)
-WIN    = 3         # sliding-window width  (= N in the diagram)
+WIN    = 2         # sliding-window width  (= N in the diagram)
 STRIDE = 1     	   # sliding-window step size
 UNITS  = 3
 H_SIZE = 128         # hidden units per RBM cell
@@ -33,10 +33,10 @@ NUM_EPOCHS = 1000
 NUM_ITERATIONS = 1
 
 TRAIN_PARAMS = TrainingParams(
-	learning_rate    = 0.2,
+	learning_rate    = 0.1,
 	momentum         = 0.5,
 	num_epochs       = NUM_EPOCHS,
-	do_rao_blackwell = False,
+	do_rao_blackwell = True,
 	num_gibbs_samples= 1
 )
 #WIN=3
@@ -53,18 +53,31 @@ def vec2str(mat: Mat, axis=0):
 def str2vec(ch_str: str) -> Mat:
 	return idx2vec(ch2idx(ch_str))
 
-def to_window(text: str):
-	win_text = []
-	text_padded = text + ' '*WIN
+def to_window(text: str) -> list[str]:
+	result = []
+	_win = [' ']*UNITS
+	for ch in text:
+		for _i in range(WIN-1):
+			_win = _win[1:WIN] + [ch]
+			result += _win
+
+	return result
+
+def to_batch(_win_text: list[str], adv=1):
+	_batch = np.zeros([len(_win_text), WIN*vocab_size()])
+	for _i in range(len(TEXT)-UNITS):
+		win_str = ''
+		for _j in range(WIN):
+			win_str += _win_text[(_i+adv)*WIN+_j]
+		_batch[_i, :] = str2vec(win_str).flatten()
+	return _batch
+
+def to_batch_(_win_str: list[str], delay=0):
+	_batch = np.zeros([len(_win_str), WIN, vocab_size()])
 	for i in range(len(TEXT)):
-		win_text.append(text_padded[i:i+WIN])
-
-	return win_text
-
-def to_batch(_win_str: list[str]):
-	_batch = np.zeros([len(TEXT), WIN*vocab_size()])
-	for i, win in enumerate(_win_str):
-		_batch[i, :] = str2vec(win).flatten()
+		for j in range(WIN):
+			_vec = str2vec(_win_str[i+j])
+			_batch[i, j, :] = _vec
 	return _batch
 
 def batch_delay(_batch: Mat, delay=0):
@@ -90,25 +103,28 @@ class RnnModel(Model):
 			unit = Entity((WIN*vocab_size() + H_SIZE, H_SIZE), EntityParams(do_gaussian_visible=False, do_gaussian_hidden=False), training_params=TRAIN_PARAMS, index=index)
 			self.units.append(unit)
 
-	def train(self, vc: Mat, status: Status = None):
-		_v, _c = split(vc, H_SIZE, axis=1)
-		for unit in self.units:
-			_vd = batch_delay(_v, 1)
-			_vc = concat(_vd, _c, axis=1)
-			for i in range(vc.shape[0]):
+	def seq_len(self):
+		return len(self.units)
+
+	def train(self, _win_text: list[str], status: Status = None):
+		_c = np.zeros([len(_win_text), H_SIZE])
+		for index, unit in enumerate(self.units):
+			_batch = to_batch(_win_text, index)
+			_vc = concat(_batch, _c, axis=1)
+			for i in range(_vc.shape[0]):
 				print(f"train: {unit.name}:{vc2char(_vc[i,:])}")
 			train(unit, _vc, status)
 			# For the next unit: Update context portion of vc
-			_c = unit.forward(vc)
+			_c = unit.forward(_vc)
 
 
 	def forward_step(self, _vc: Mat):
 		text = ''
 		for unit in self.units:
-#			print(f"forward_step in : {unit.name}:{vc2char(_vc)}")
+			print(f"forward_step in : {unit.name}:{vc2char(_vc)}")
 			_c = unit.forward(_vc)
 			_vc = unit.reconstruct(_c)
-#			print(f"forward_step out: {unit.name}:{vc2char(_vc)}")
+			print(f"forward_step out: {unit.name}:{vc2char(_vc)}")
 			text += vc2char(_vc)[WIN-1]
 			_v, _ = split(_vc, H_SIZE, axis=1)
 			_v_cl = clamp(_v.reshape([WIN, vocab_size()]), axis=1)
@@ -143,14 +159,12 @@ if __name__ == "__main__":
 
 	# vc contains vis + context
 	# context will be updated after training
-	c_train = np.zeros([batch.shape[0], H_SIZE])
-	vc_train = concat(batch, c_train, axis=1)
-	model.train(vc_train, status=Status())
+	model.train(win_text, status=Status())
 	model.save()
 
 	# test the model
-	seed_str = 'Cal'
-	seed_padded = seed_str + '^'*(WIN-len(seed_str))
+	seed_str = ' '
+	seed_padded = seed_str + ' '*(WIN-len(seed_str))
 	v_test = str2vec(seed_padded).reshape([1, WIN*vocab_size()])
 	c_test = np.zeros([1, H_SIZE])
 	vc_test = concat(v_test, c_test, axis=1)