- removed Optimizer

- refactored training
- use static seed for random (for better comparison)
- simplified StackDeep.train

src/rbm/matrix.py

@@ -13,7 +13,8 @@ else:
 	def convert(src: Mat):
 		return src
 
-np.random.seed(int(time.monotonic()))
+np.random.seed(12345)
+
 def uniform(shape: tuple, mu: float = 0.5, std: float = 1.0) -> Mat:
 	return std * (np.random.rand(shape[0], shape[1]) + mu - 0.5)
 

src/rbm/stack_deep.py

@@ -1,34 +1,18 @@
 from .status import Status
-from .train import train, Optimizer
+from .train import train
 from .stack import Stack, StackType
 from .matrix import Mat, np
 
-USE_OPTIMIZER = False
-
 class StackDeep(Stack):
 	def __init__(self, name: str, work_dir: str = '.'):
 		Stack.__init__(self, StackType.Deep, name, work_dir)
 
-	def batch_from(self, batch: Mat, from_layer_id: int = 0):
-		_batch = np.copy(batch)
-		for index, layer in enumerate(self.layers):
-			if index == from_layer_id:
-				break
-			_batch = layer.entity.forward(_batch)
-
-		return _batch
-
 	def train(self, batch: Mat, status=Status()):
 		_batch = np.copy(batch)
 		for index, layer in enumerate(self.layers):
 			print(f"Train layer {index} for {layer.training_params.num_epochs} epochs")
-			if USE_OPTIMIZER:
+			train(layer.entity, _batch, layer.training_params, status=status)
-				optim = Optimizer(layer.entity, layer.training_params)
+			_batch = layer.entity.forward(_batch)
-				_batch = optim(_batch, status=status)
-			else:
-				_batch = self.batch_from(batch, index)
-				train(layer.entity, _batch, layer.training_params, status=status)
-
 
 	def pass_up(self, visible: Mat, from_layer_id: int = 0):
 		h = np.copy(visible)

src/rbm/stack_factory.py

@@ -8,7 +8,7 @@ from .stack_rnn import StackRnn
 
 class StackFactory:
 	@classmethod
-	def from_dict(cls, project: dict, work_dir: str = ".") -> StackDeep|StackRnn:
+	def from_dict(cls, project: dict, work_dir: str = ".") -> StackDeep|StackRnn|None:
 		name = project["stack"]["name"]
 		layers = project["stack"]["layers"]
 

src/rbm/train.py

@@ -72,37 +72,38 @@ def cd_jens(entity: Entity, v_states: Mat, params: TrainingParams):
 
 	return dw, dbv, dbh, h_probs
 
-def train(entity: Entity, batch: Mat, params: TrainingParams, status: Status, cd_func: Callable = cd_jens):
+def to_mini_batch(batch: Mat, mini_batch_size: int):
-	training_remain = batch.shape[0]
+	mini_batches = []
-	batch_size = min(params.mini_batch_size, training_remain)
+	remain = batch.shape[0]
-	if batch_size == 0:
+	index = 0
-		batch_size = training_remain
+	while remain > 0:
+		batch_size = min(mini_batch_size, remain)
+		mini_batches.append(batch[index:index + batch_size])
+		remain -= batch_size
+		index += batch_size
 
-	status.on_change()
+	return mini_batches
-	d_progress = 100.0 / (training_remain*params.num_epochs)
+
+def train(entity: Entity, batch: Mat, params: TrainingParams, status: Status, cd_func: Callable = cd_jens):
+	mini_batch_size = min(params.mini_batch_size, batch.shape[0]) if params.mini_batch_size > 0 else batch.shape[0]
+	d_progress = 100.0 / (batch.shape[0]*params.num_epochs)
 	progress = 0
-	batch_row_index = 0
 	keep_running = True
-	while training_remain > 0 and keep_running:
+	status.on_change()
-		batch_size_remain = min(batch_size, training_remain)
+	for mini_batch in to_mini_batch(batch, mini_batch_size):
-		mini_batch = batch[batch_row_index:batch_row_index + batch_size_remain]
+		if not keep_running:
-		training_remain -= batch_size_remain
+			break
-		batch_row_index += batch_size_remain
 
 		inc_bv = np.zeros(entity.state.b_v.shape)
 		inc_bh = np.zeros(entity.state.b_h.shape)
 		inc_whv = np.zeros(entity.state.w_hv.shape)
 
-		v_states = mini_batch
-		if params.do_batch_sample:
-			v_states = sample(mini_batch)
-
 		for epochs in range(params.num_epochs):
 			# Contrastive divergence learning: calculate gradients
-			dwhv, dbv, dbh, _ = cd_func(entity, v_states, params)
+			dwhv, dbv, dbh, _ = cd_func(entity, mini_batch, params)
 
 			# Adjust weight and biases
-			kl = params.learning_rate/batch_size
+			kl = params.learning_rate/batch.shape[0]
 			inc_bv = params.momentum*inc_bv + kl*dbv
 			inc_bh = params.momentum*inc_bh + kl*dbh
 			inc_whv = params.momentum*inc_whv + kl*dwhv - params.weight_decay*entity.state.w_hv
@@ -120,79 +121,11 @@ def train(entity: Entity, batch: Mat, params: TrainingParams, status: Status, cd
 					keep_running = False
 					break
 
-			progress += d_progress*batch_size_remain
+			progress += d_progress*mini_batch.shape[0]
 
 	# Update final status
 	err_rms = rms_error_accu(batch - entity.reconstruct(entity.forward(batch)))
 	status.on_change({"progress": {"value": round(progress), "unit": "%"},
 					  "err_rms_total": {"value": err_rms, "unit": ""}})
 
-
+	return None
-class Optimizer:
-	def __init__(self, entity: Entity, params: TrainingParams, cd_func: Callable = cd_jens):
-		self.inc_bv = np.zeros(entity.state.b_v.shape)
-		self.inc_bh = np.zeros(entity.state.b_h.shape)
-		self.inc_whv = np.zeros(entity.state.w_hv.shape)
-		self.entity = entity
-		self.params = params
-		self.cd = cd_func
-
-	def __call__(self, batch: Mat, status: Status):
-		return self.process(batch, status)
-
-	def process(self, batch: Mat, status: Status):
-		status.on_change()
-		d_progress = 100.0 / self.params.num_epochs
-		progress = 0
-
-		for epochs in range(self.params.num_epochs):
-			self.epoch(batch)
-
-			# check if status update is needed
-			if status.want_report(round(progress)):
-				# Calculate	error
-				forward = self.entity.forward(batch)
-				result = self.entity.reconstruct(forward)
-				err_rms = rms_error_accu(batch - result)
-				if not status.on_change({"progress": {"value": round(progress), "unit": "%"},
-										 "err_rms": {"value": err_rms, "unit": ""}}):
-					break
-
-			progress += d_progress
-
-		forward = self.entity.forward(batch)
-		err_rms = rms_error_accu(batch - self.entity.reconstruct(forward))
-		status.on_change({"progress": {"value": round(progress), "unit": "%"},
-					  "err_rms": {"value": err_rms, "unit": ""}})
-
-		return forward
-
-	def epoch(self, batch: Mat):
-		forward = None
-		training_remain = batch.shape[0]
-		batch_row_index = 0
-
-		while training_remain > 0:
-			batch_size = min(self.params.mini_batch_size, training_remain)
-			if batch_size == 0:
-				batch_size = training_remain
-
-			mini_batch = batch[batch_row_index:batch_row_index + batch_size]
-
-			# Contrastive divergence learning: calculate gradients
-			dwhv, dbv, dbh, forward = self.cd(self.entity, mini_batch, self.params)
-
-			# Adjust weight and biases
-			kl = self.params.learning_rate / batch_size
-			inc_bv = self.params.momentum * self.inc_bv + kl * dbv
-			inc_bh = self.params.momentum * self.inc_bh + kl * dbh
-			inc_whv = self.params.momentum * self.inc_whv + kl * dwhv - self.params.weight_decay * self.entity.state.w_hv
-
-			self.entity.state.b_v += inc_bv
-			self.entity.state.b_h += inc_bh
-			self.entity.state.w_hv += inc_whv
-
-			training_remain -= batch_size
-			batch_row_index += batch_size
-
-		return forward

src/tests/test_xor.py

@@ -2,7 +2,7 @@ import os.path
 
 from rbm.layer import Layer
 from rbm.status import Status
-from rbm.train import train, TrainingParams, Optimizer
+from rbm.train import train, TrainingParams
 from rbm.matrix import Mat, np
 from rbm.entity import EntityParams
 
@@ -29,11 +29,7 @@ def xor():
 	training_batch = Mat([[0,1,1], [0,0,0], [1,1,0], [1,0,1]], dtype=np.float64)
 
 	# Train layer
-	if USE_OPTIMIZER:
+	train(layer.entity, training_batch, training_params, Status())
-		optim = Optimizer(layer.entity, training_params)
-		optim(training_batch, Status())
-	else:
-		train(layer.entity, training_batch, training_params, Status())
 
 	# Save weights
 	layer.save(os.path.join(WORK_DIR, "xor_layer0_state.npz"))