- fixed matrix operations

- fixed rms error calculation

src/rbm/cd_train.py

@@ -2,10 +2,9 @@ import numpy as np
 from collections.abc import Callable
 
 from params import RbmParams
-from rbm.helper import gaussian
-from helper import sample
+from helper import sample, gaussian
 
-def train(v_states: np.ndarray, params: RbmParams, v_to_ph: Callable, h_to_pv: Callable):
+def cd_jens(v_states: np.ndarray, params: RbmParams, v_to_ph: Callable, h_to_pv: Callable):
 	v_probs = v_states
 	h_states = v_to_ph(v_states)
 	h_probs = h_states
@@ -20,7 +19,7 @@ def train(v_states: np.ndarray, params: RbmParams, v_to_ph: Callable, h_to_pv: C
 			h_states = sample(h_probs)
 
 	# Update weights (positive phase)
-	dw = np.transpose(v_states) * h_states
+	dw = np.vecmat(np.transpose(v_states), h_states)
 	dbv = np.sum(v_states, 0)
 	dbh = np.sum(h_states, 0)
 
@@ -38,7 +37,7 @@ def train(v_states: np.ndarray, params: RbmParams, v_to_ph: Callable, h_to_pv: C
 			h_probs = v_to_ph(v_probs)
 
 	# Update weights (negative phase)
-	dw -= np.transpose(v_probs) * h_probs
+	dw -= np.vecmat(np.transpose(v_probs), h_probs)
 	dbv -= np.sum(v_probs, 0)
 	dbh -= np.sum(h_probs, 0)
 

src/rbm/helper.py

@@ -16,10 +16,11 @@ def prob(src: np.ndarray) -> np.ndarray:
 	return 1.0 / (1 + np.exp(-src))
 
 def rms_error(d_err: np.ndarray):
-	d_err_squared = d_err % d_err
+	d_err_squared = d_err * d_err
 	return np.sum(d_err_squared, 1) / d_err_squared[1]
 
 def rms_error_accu(d_err: np.ndarray):
-	d_err_squared = d_err % d_err
-	return np.cumsum(d_err_squared) / d_err_squared[1]
+	d_err_squared = d_err * d_err
+	s =  np.sum(d_err_squared) / len(d_err)
+	return s
 

src/rbm/layer.py

@@ -4,6 +4,7 @@ from params import RbmParams
 from state import RbmState
 from helper import sample, prob, uniform, rms_error_accu
 from status import Status
+from cd_train import cd_jens
 
 class RbmLayer:
 	def __init__(self, name: str, num_visible, num_hidden, params: RbmParams):
@@ -19,23 +20,25 @@ class RbmLayer:
 		self.state = RbmState.from_file(self.state_filename)
 
 	def train(self, batch: np.ndarray, cd_func: Callable, status: Status):
-		num_cases = min(self.params.mini_batch_size, batch.shape[0])
-		d_progress = 100.0 / (batch.shape[0] * self.params.num_epochs)
-		last_status = status
-		status.progress = 0
+		training_size = batch.shape[0]
+		num_cases = min(self.params.mini_batch_size, training_size)
+		d_progress = 100.0 / (training_size/num_cases * self.params.num_epochs)
+		progress = 0
+		last_progress = progress
 		batch_row_index = 0
 
 		keep_running = True
 
-		training_size_remain = batch.shape[0]
-		while training_size_remain > 0 and keep_running:
-			mini_batch_size = min(self.params.mini_batch_size, training_size_remain)
-			mini_batch = batch[batch_row_index:batch_row_index + mini_batch_size - 1]
-			training_size_remain -= mini_batch_size
+		training_remain = training_size
+		training_seen = 0
+		while training_remain > 0 and keep_running:
+			mini_batch_size = min(self.params.mini_batch_size, training_remain)
+			mini_batch = batch[batch_row_index:batch_row_index + mini_batch_size]
+			training_remain -= mini_batch_size
 			batch_row_index += mini_batch_size
 
 			inc_bv = np.zeros(self.state.b_v.shape)
-			inc_bh = np.zeros(self.state.bh.shape)
+			inc_bh = np.zeros(self.state.b_h.shape)
 			inc_whv = np.zeros(self.state.w_hv.shape)
 
 			v_states = mini_batch
@@ -44,7 +47,7 @@ class RbmLayer:
 
 			for epochs in range(self.params.num_epochs):
 				# Contrastive divergence learning: calculate gradients
-				dwhv, dbh, dbv = cd_func(v_states, self.params, self.v_to_ph, self.h_to_pv)
+				dwhv, dbv, dbh = cd_func(v_states, self.params, self.v_to_ph, self.h_to_pv)
 
 				# Adjust weight and biases
 				kl = self.params.learning_rate/num_cases
@@ -56,15 +59,19 @@ class RbmLayer:
 				self.state.b_h += inc_bh
 				self.state.w_hv += inc_whv
 
-				status.progress += round(d_progress*mini_batch_size)
+				progress = round(epochs*d_progress)
 
-				if status.__dict__ != last_status.__dict__:
+				if progress != last_progress:
 					# Calculate	error
+					status.progress = round(progress)
 					status.err = rms_error_accu(mini_batch - self.h_to_pv(self.v_to_ph(v_states)))
 					if not status.on_change():
 						keep_running = False
 						break
 
+				last_progress = progress
+			training_seen += 1
+
 		status.on_change()
 
 	def v_to_ph(self, v: np.ndarray) -> np.ndarray:
@@ -82,25 +89,24 @@ class RbmLayer:
 		return prob(state)
 
 
+def xor():
+	params = RbmParams()
+	params.do_rao_blackwell = True
+	params.num_gibbs_samples = 3
+	params.mini_batch_size = 100
+	params.learning_rate = 0.1
+	params.momentum = 0.5
+	params.num_epochs = 10000
+
+	status = Status()
+	layer = RbmLayer("Layer_0", 3, 16, params)
+	training_batch = np.array([[0,0,0], [0,1,1], [1,0,1], [1,1,0]], dtype=np.float64)
+	test_batch = np.array([[0,0,0], [0,1,0], [1,0,0], [1,1,0]], dtype=np.float64)
+
+	layer.train(training_batch, cd_jens, status)
+
 if __name__ == "__main__":
-	p = RbmParams()
-	l1 = RbmLayer("Layer_0", 2, 3, p)
-	l1.save()
-	l1.load()
-	l1.state.init(mu=0.5, std=1.0)
-
-	s1 = RbmState(l1.state.w_hv, l1.state.b_v, l1.state.b_h)
-
-	v0 = uniform((1,2))
-	h0 = uniform((1,3))
-
-	h1 = l1.state.v_to_h(v0)
-	print(h1.shape)
-	v1 = l1.state.h_to_v(h1)
-	s_v1 = sample(v1)
-	p_h = prob(s_v1)
-
-	print(v1.shape)
+	xor()
 
 
 

src/rbm/status.py

@@ -7,8 +7,10 @@ class Status:
 		self.l2 = -1.0
 
 	def on_change(self) -> bool:
-		print(self)
+		print("-------------------------------------------")
+		print(f"Progress               : {self.progress} %")
+		print(f"error (per mini batch) : {self.err}")
+		print(f"error (total)          : {self.err_total}")
+		print(f"L1                     : {self.l1}")
+		print(f"L2                     : {self.l2}")
 		return True
-
-	def __repr__(self):
-		return f"Progress: {self.progress}"