Entity: refactored taining algo into train module

src/rbm/cd_train.py

@@ -1,42 +0,0 @@
-from collections.abc import Callable
-from params import RbmParams
-from matrix import prob, sample, gaussian, Mat, np
-
-def cd_jens(v_states: Mat, params: RbmParams, v_to_ph: Callable, h_to_pv: Callable):
-	v_probs = prob(v_states)
-	h_states = v_to_ph(v_states)
-	h_probs = h_states
-
-	if params.do_gaussian_hidden:
-		h_states += gaussian(h_states.shape)
-	else:
-		h_probs = v_to_ph(v_states)
-		if params.do_rao_blackwell:
-			h_states = h_probs
-		else:
-			h_states = sample(h_probs)
-
-	# Update weights (positive phase)
-	dw = np.dot(np.transpose(v_states), h_states)
-	dbv = np.sum(v_states, 0)
-	dbh = np.sum(h_states, 0)
-
-	# Gibbs	sampling with training params
-	for i in range(params.num_gibbs_samples):
-		if params.do_gibbs_sample_hidden:
-			v_probs = h_to_pv(sample(h_probs))
-		else:
-			v_probs = h_to_pv(h_probs)
-
-		# Create hidden representation given v
-		if params.do_gibbs_sample_visible:
-			h_probs = v_to_ph(sample(v_probs))
-		else:
-			h_probs = v_to_ph(v_probs)
-
-	# Update weights (negative phase)
-	dw -= np.dot(np.transpose(v_probs), h_probs)
-	dbv -= np.sum(v_probs, 0)
-	dbh -= np.sum(h_probs, 0)
-
-	return dw, dbv, dbh

src/rbm/entity.py

@@ -1,69 +1,12 @@
-from collections.abc import Callable
+from params import EntityParams
-from params import RbmParams
 from state import RbmState
-from matrix import sample, prob, rms_error_accu, Mat, np
+from matrix import prob, Mat
-from status import Status
 
 class Entity:
-	def __init__(self, shape: tuple[int, int], params: RbmParams):
+	def __init__(self, shape: tuple[int, int], params: EntityParams):
 		self.shape = shape
-		self.state = RbmState.from_layer_params(shape)
 		self.params = params
-
+		self.state = RbmState.from_layer_params(shape)
-	def train(self, batch: Mat, cd_func: Callable, status: Status):
-		training_remain = batch.shape[0]
-		batch_size = min(self.params.mini_batch_size, training_remain)
-		if batch_size == 0:
-			batch_size = training_remain
-
-		status.on_change()
-		d_progress = 100.0 / (training_remain*self.params.num_epochs)
-		progress = 0
-		batch_row_index = 0
-		keep_running = True
-		while training_remain > 0 and keep_running:
-			batch_size_remain = min(batch_size, training_remain)
-			mini_batch = batch[batch_row_index:batch_row_index + batch_size_remain]
-			training_remain -= batch_size_remain
-			batch_row_index += batch_size_remain
-
-			inc_bv = np.zeros(self.state.b_v.shape)
-			inc_bh = np.zeros(self.state.b_h.shape)
-			inc_whv = np.zeros(self.state.w_hv.shape)
-
-			v_states = mini_batch
-			if self.params.do_batch_sample:
-				v_states = sample(mini_batch)
-
-			for epochs in range(self.params.num_epochs):
-				# Contrastive divergence learning: calculate gradients
-				dwhv, dbv, dbh = cd_func(v_states, self.params, self.v_to_ph, self.h_to_pv)
-
-				# Adjust weight and biases
-				kl = self.params.learning_rate/batch_size
-				inc_bv = self.params.momentum*inc_bv + kl*dbv
-				inc_bh = self.params.momentum*inc_bh + kl*dbh
-				inc_whv = self.params.momentum*inc_whv + kl*dwhv - self.params.weight_decay*self.state.w_hv
-
-				self.state.b_v += inc_bv
-				self.state.b_h += inc_bh
-				self.state.w_hv += inc_whv
-
-				# check if status update is needed
-				if status.want_report(round(progress)):
-					# Calculate	error
-					err_rms = rms_error_accu(mini_batch - self.h_to_pv(self.v_to_ph(v_states)))
-					if not status.on_change({"progress": {"value": round(progress), "unit": "%"},
-											 "err_rms": {"value": err_rms, "unit": ""}}):
-						keep_running = False
-						break
-
-				progress += d_progress*batch_size_remain
-
-		# Update final status
-		err_rms = rms_error_accu(batch - self.h_to_pv(self.v_to_ph(batch)))
-		status.on_change({"progress": {"value": round(progress), "unit": "%"},
-						  "err_rms_total": {"value": err_rms, "unit": ""}})
 
 	def v_to_ph(self, v: Mat) -> Mat:
 		state = self.state.v_to_h(v)
@@ -79,7 +22,6 @@ class Entity:
 
 		return prob(state)
 
-
 	def gibbs_v_to_h(self, v: Mat) -> Mat:
 		h = self.v_to_ph(v)
 		for i in range(self.params.num_gibbs_samples-1):

src/rbm/layer.py

@@ -1,12 +1,12 @@
-from params import RbmParams
+from params import EntityParams
 from state import RbmState
 from status import Status
-from cd_train import cd_jens
+from train import train
 from entity import Entity
 from matrix import Mat, np
 
 class Layer:
-	def __init__(self, name: str, shape: tuple[int, int, int, int], params: RbmParams):
+	def __init__(self, name: str, shape: tuple[int, int, int, int], params: EntityParams):
 		self.name = name
 		self.shape = shape
 		self.entity = Entity((shape[0]*shape[1]+shape[2], shape[3]), params)
@@ -29,7 +29,7 @@ class Layer:
 
 def xor():
 	# Create params
-	params = RbmParams()
+	params = EntityParams()
 	params.do_rao_blackwell = True
 	params.num_gibbs_samples = 3
 
@@ -46,7 +46,7 @@ def xor():
 	training_batch = Mat([[0,1,1], [0,0,0], [1,1,0], [1,0,1]], dtype=np.float64)
 
 	# Train layer
-	layer.entity.train(training_batch, cd_jens, Status())
+	train(layer.entity, training_batch, Status())
 
 	# Save weights
 	layer.save()

src/rbm/params.py

@@ -9,7 +9,7 @@ class Params:
 		with open(filename, "r") as fp:
 			self.__dict__ = json.load(fp)
 
-class RbmParams(Params):
+class EntityParams(Params):
 	def __init__(self):
 		self.learning_rate = 0.1
 		self.momentum = 0.5
@@ -28,21 +28,25 @@ class RbmParams(Params):
 
 	@classmethod
 	def from_dict(cls, params: dict, version: str = '0'):
-		obj = RbmParams()
+		obj = EntityParams()
 
 		if "0" in version or "1" in version:
+			# Training parameters
 			obj.learning_rate = params["learningRate"]
 			obj.momentum = params["momentum"]
 			obj.weight_decay = params["weightDecay"]
 			obj.num_epochs = params["numEpochs"]
-			obj.num_gibbs_samples = params["numGibbs"]
 			obj.mini_batch_size = params["miniBatchSize"]
 			obj.do_rao_blackwell = params["doRaoBlackwell"]
 			obj.do_gibbs_sample_visible = params["gibbsDoSampleVisible"]
 			obj.do_gibbs_sample_hidden = params["gibbsDoSampleHidden"]
 			obj.do_batch_sample = params["doSampleBatch"]
 
+			# Entity parameters
+			obj.num_gibbs_samples = params["numGibbs"]
+
 		if "1" in version:
+			# Entity parameters
 			obj.do_gaussian_visible = params["doGaussianVisible"]
 			obj.do_gaussian_hidden = params["doGaussianHidden"]
 
@@ -56,11 +60,11 @@ class LayerParams(Params):
 
 if __name__ == "__main__":
 	test_filename = "../../test_params.json"
-	p1 = RbmParams()
+	p1 = EntityParams()
 	p1.num_epochs = 31101970
 	p1.save(test_filename)
 
-	p2 = RbmParams()
+	p2 = EntityParams()
 	p2.load(test_filename)
 
 	assert p2.__dict__ == p1.__dict__

src/rbm/stack_deep.py

@@ -1,5 +1,5 @@
 from status import Status
-from cd_train import cd_jens
+from train import train
 from stack import Stack, StackType
 from matrix import Mat, np
 
@@ -21,7 +21,7 @@ class StackDeep(Stack):
 		for index, layer in enumerate(self.layers):
 			print(f"Train layer {index} for {layer.entity.params.num_epochs} epochs")
 			_batch = self.batch_from(batch, index)
-			layer.entity.train(_batch, cd_func=cd_jens, status=status)
+			train(layer.entity, _batch, status=status)
 
 
 	def pass_up(self, visible: Mat, from_layer_id: int = 0):

src/rbm/stack_factory.py

@@ -2,7 +2,7 @@ import json
 from collections.abc import Callable
 from stack import StackType
 from layer import Layer
-from params import RbmParams
+from params import EntityParams
 from stack_deep import StackDeep
 from stack_rnn import StackRnn
 
@@ -43,7 +43,7 @@ class StackFactory:
 			if "doGaussianHidden" in layer_params and "doGaussianVisible" in layer_params:
 				params_version = '1'
 
-			params = RbmParams.from_dict(layer_params, params_version)
+			params = EntityParams.from_dict(layer_params, params_version)
 
 			# Create layer
 			layer_obj = Layer(f"{layer_name}-{layer_id}", (num_visible_x, num_visible_y, num_context, num_hidden), params)

src/rbm/train.py

@@ -0,0 +1,98 @@
+from collections.abc import Callable
+from matrix import gaussian, sample, prob, rms_error_accu, Mat, np
+from entity import Entity
+from status import Status
+
+def cd_jens(entity: Entity, v_states: Mat):
+	v_probs = prob(v_states)
+	h_states = entity.v_to_ph(v_states)
+	h_probs = h_states
+
+	if entity.params.do_gaussian_hidden:
+		h_states += gaussian(h_states.shape)
+	else:
+		h_probs = entity.v_to_ph(v_states)
+		if entity.params.do_rao_blackwell:
+			h_states = h_probs
+		else:
+			h_states = sample(h_probs)
+
+	# Update weights (positive phase)
+	dw = np.dot(np.transpose(v_states), h_states)
+	dbv = np.sum(v_states, 0)
+	dbh = np.sum(h_states, 0)
+
+	# Gibbs	sampling with training params
+	for i in range(entity.params.num_gibbs_samples):
+		if entity.params.do_gibbs_sample_hidden:
+			v_probs = entity.h_to_pv(sample(h_probs))
+		else:
+			v_probs = entity.h_to_pv(h_probs)
+
+		# Create hidden representation given v
+		if entity.params.do_gibbs_sample_visible:
+			h_probs = entity.v_to_ph(sample(v_probs))
+		else:
+			h_probs = entity.v_to_ph(v_probs)
+
+	# Update weights (negative phase)
+	dw -= np.dot(np.transpose(v_probs), h_probs)
+	dbv -= np.sum(v_probs, 0)
+	dbh -= np.sum(h_probs, 0)
+
+	return dw, dbv, dbh
+
+def train(entity: Entity, batch: Mat, status: Status, cd_func: Callable = cd_jens):
+	training_remain = batch.shape[0]
+	batch_size = min(entity.params.mini_batch_size, training_remain)
+	if batch_size == 0:
+		batch_size = training_remain
+
+	status.on_change()
+	d_progress = 100.0 / (training_remain*entity.params.num_epochs)
+	progress = 0
+	batch_row_index = 0
+	keep_running = True
+	while training_remain > 0 and keep_running:
+		batch_size_remain = min(batch_size, training_remain)
+		mini_batch = batch[batch_row_index:batch_row_index + batch_size_remain]
+		training_remain -= batch_size_remain
+		batch_row_index += batch_size_remain
+
+		inc_bv = np.zeros(entity.state.b_v.shape)
+		inc_bh = np.zeros(entity.state.b_h.shape)
+		inc_whv = np.zeros(entity.state.w_hv.shape)
+
+		v_states = mini_batch
+		if entity.params.do_batch_sample:
+			v_states = sample(mini_batch)
+
+		for epochs in range(entity.params.num_epochs):
+			# Contrastive divergence learning: calculate gradients
+			dwhv, dbv, dbh = cd_func(entity, v_states)
+
+			# Adjust weight and biases
+			kl = entity.params.learning_rate/batch_size
+			inc_bv = entity.params.momentum*inc_bv + kl*dbv
+			inc_bh = entity.params.momentum*inc_bh + kl*dbh
+			inc_whv = entity.params.momentum*inc_whv + kl*dwhv - entity.params.weight_decay*entity.state.w_hv
+
+			entity.state.b_v += inc_bv
+			entity.state.b_h += inc_bh
+			entity.state.w_hv += inc_whv
+
+			# check if status update is needed
+			if status.want_report(round(progress)):
+				# Calculate	error
+				err_rms = rms_error_accu(mini_batch - entity.h_to_pv(entity.v_to_ph(v_states)))
+				if not status.on_change({"progress": {"value": round(progress), "unit": "%"},
+										 "err_rms": {"value": err_rms, "unit": ""}}):
+					keep_running = False
+					break
+
+			progress += d_progress*batch_size_remain
+
+	# Update final status
+	err_rms = rms_error_accu(batch - entity.h_to_pv(entity.v_to_ph(batch)))
+	status.on_change({"progress": {"value": round(progress), "unit": "%"},
+					  "err_rms_total": {"value": err_rms, "unit": ""}})