- added train

- refactored CdTrain

.gitignore

@@ -4,3 +4,4 @@ results
 images
 *.egg-info
 __pycache__
+*.npz

src/rbm/cd_train.py

@@ -1,69 +1,45 @@
 import numpy as np
+from collections.abc import Callable
 
 from params import RbmParams
 from rbm.helper import gaussian
-from state import RbmState
+from helper import sample
-from helper import prob, sample
 
-class CdTrain:
+def train(v_states: np.ndarray, params: RbmParams, v_to_ph: Callable, h_to_pv: Callable):
-	def __init__(self, state: RbmState, params: RbmParams):
+	v_probs = v_states
-		self.state = state
+	h_states = v_to_ph(v_states)
-		self.params = params
+	h_probs = h_states
 
-	def v_to_h_prob(self, v: np.ndarray) -> np.ndarray:
+	if params.do_gaussian_hidden:
-		state = self.state.v_to_h(v)
+		h_states += gaussian(h_states.shape)
-		if self.params.do_gaussian_visible:
+	else:
-			return state
+		h_probs = v_to_ph(v_states)
-
+		if params.do_rao_blackwell:
-		return prob(state)
+			h_states = h_probs
-
-	def h_to_v_prob(self, h: np.ndarray) -> np.ndarray:
-		state = self.state.h_to_v(h)
-		if self.params.do_gaussian_visible:
-			return state
-
-		return prob(state)
-
-
-class CdTrainJens(CdTrain):
-	def __init__(self, state: RbmState, params: RbmParams):
-		CdTrain.__init__(self, state, params)
-
-	def train(self, v_states: np.ndarray):
-		v_probs = v_states
-		h_states = self.v_to_h_prob(v_states)
-		h_probs = h_states
-
-		if self.params.do_gaussian_hidden:
-			h_states += gaussian(h_states.shape)
 		else:
-			h_probs = self.v_to_h_prob(v_states)
+			h_states = sample(h_probs)
-			if self.params.do_rao_blackwell:
-				h_states = h_probs
-			else:
-				h_states = sample(h_probs)
 
-		# Update weights (positive phase)
+	# Update weights (positive phase)
-		dw = np.transpose(v_states) * h_states
+	dw = np.transpose(v_states) * h_states
-		dbv = np.sum(v_states, 0)
+	dbv = np.sum(v_states, 0)
-		dbh = np.sum(h_states, 0)
+	dbh = np.sum(h_states, 0)
 
-		# Gibbs	sampling with training params
+	# Gibbs	sampling with training params
-		for i in range(self.params.num_gibbs_samples):
+	for i in range(params.num_gibbs_samples):
-			if self.params.do_gibbs_sample_hidden:
+		if params.do_gibbs_sample_hidden:
-				v_probs = self.h_to_v_prob(sample(h_probs))
+			v_probs = h_to_pv(sample(h_probs))
-			else:
+		else:
-				v_probs = self.h_to_v_prob(h_probs)
+			v_probs = h_to_pv(h_probs)
 
-			# Create hidden representation given v
+		# Create hidden representation given v
-			if self.params.do_gaussian_visible:
+		if params.do_gaussian_visible:
-				h_probs = self.v_to_h_prob(sample(v_probs))
+			h_probs = v_to_ph(sample(v_probs))
-			else:
+		else:
-				h_probs = self.v_to_h_prob(v_probs)
+			h_probs = v_to_ph(v_probs)
 
-		# Update weights (negative phase)
+	# Update weights (negative phase)
-		dw -= np.transpose(v_probs) * h_probs
+	dw -= np.transpose(v_probs) * h_probs
-		dbv -= np.sum(v_probs, 0)
+	dbv -= np.sum(v_probs, 0)
-		dbh -= np.sum(h_probs, 0)
+	dbh -= np.sum(h_probs, 0)
 
-		return dw, dbv, dbh
+	return dw, dbv, dbh

src/rbm/helper.py

@@ -14,3 +14,12 @@ def sample(src: np.ndarray) -> np.ndarray:
 
 def prob(src: np.ndarray) -> np.ndarray:
 	return 1.0 / (1 + np.exp(-src))
+
+def rms_error(d_err: np.ndarray):
+	d_err_squared = d_err % d_err
+	return np.sum(d_err_squared, 1) / d_err_squared[1]
+
+def rms_error_accu(d_err: np.ndarray):
+	d_err_squared = d_err % d_err
+	return np.cumsum(d_err_squared) / d_err_squared[1]
+

src/rbm/layer.py

@@ -2,7 +2,8 @@ import numpy as np
 from collections.abc import Callable
 from params import RbmParams
 from state import RbmState
-from helper import sample, prob, uniform
+from helper import sample, prob, uniform, rms_error_accu
+from status import Status
 
 class RbmLayer:
 	def __init__(self, name: str, num_visible, num_hidden, params: RbmParams):
@@ -17,11 +18,68 @@ class RbmLayer:
 	def load(self):
 		self.state = RbmState.from_file(self.state_filename)
 
-	def train_batch(self, v_states: np.ndarray, cd_func: Callable):
+	def train(self, batch: np.ndarray, cd_func: Callable, status: Status):
-		for epochs in range(self.params.num_epochs):
+		num_cases = min(self.params.mini_batch_size, batch.shape[0])
-			# Contrastive divergence learning: calculate gradients
+		d_progress = 100.0 / (batch.shape[0] * self.params.num_epochs)
-			dwhv, dbh, dbv = cd_func(v_states)
+		last_status = status
+		status.progress = 0
+		batch_row_index = 0
 
+		keep_running = True
+
+		training_size_remain = batch.shape[0]
+		while training_size_remain > 0 and keep_running:
+			mini_batch_size = min(self.params.mini_batch_size, training_size_remain)
+			mini_batch = batch[batch_row_index:batch_row_index + mini_batch_size - 1]
+			training_size_remain -= mini_batch_size
+			batch_row_index += mini_batch_size
+
+			inc_bv = np.zeros(self.state.b_v.shape)
+			inc_bh = np.zeros(self.state.bh.shape)
+			inc_whv = np.zeros(self.state.w_hv.shape)
+
+			v_states = mini_batch
+			if self.params.do_batch_sample:
+				v_states = sample(mini_batch)
+
+			for epochs in range(self.params.num_epochs):
+				# Contrastive divergence learning: calculate gradients
+				dwhv, dbh, dbv = cd_func(v_states, self.params, self.v_to_ph, self.h_to_pv)
+
+				# Adjust weight and biases
+				kl = self.params.learning_rate/num_cases
+				inc_bv = self.params.momentum*inc_bv + kl*dbv
+				inc_bh = self.params.momentum*inc_bh + kl*dbh
+				inc_whv = self.params.momentum*inc_whv + kl*dwhv - self.params.learning_rate*self.state.w_hv
+
+				self.state.b_v += inc_bv
+				self.state.b_h += inc_bh
+				self.state.w_hv += inc_whv
+
+				status.progress += round(d_progress*mini_batch_size)
+
+				if status.__dict__ != last_status.__dict__:
+					# Calculate	error
+					status.err = rms_error_accu(mini_batch - self.h_to_pv(self.v_to_ph(v_states)))
+					if not status.on_change():
+						keep_running = False
+						break
+
+		status.on_change()
+
+	def v_to_ph(self, v: np.ndarray) -> np.ndarray:
+		state = self.state.v_to_h(v)
+		if self.params.do_gaussian_visible:
+			return state
+
+		return prob(state)
+
+	def h_to_pv(self, h: np.ndarray) -> np.ndarray:
+		state = self.state.h_to_v(h)
+		if self.params.do_gaussian_visible:
+			return state
+
+		return prob(state)
 
 
 if __name__ == "__main__":

src/rbm/status.py

@@ -0,0 +1,14 @@
+class Status:
+	def __init__(self):
+		self.progress = 0
+		self.err = -1.0
+		self.err_total = -1.0
+		self.l1 = -1.0
+		self.l2 = -1.0
+
+	def on_change(self) -> bool:
+		print(self)
+		return True
+
+	def __repr__(self):
+		return f"Progress: {self.progress}"