- removed Optimizer

- refactored training
- use static seed for random (for better comparison)
- simplified StackDeep.train

src/rbm/train.py

@@ -72,37 +72,38 @@ def cd_jens(entity: Entity, v_states: Mat, params: TrainingParams):
 
 	return dw, dbv, dbh, h_probs
 
-def train(entity: Entity, batch: Mat, params: TrainingParams, status: Status, cd_func: Callable = cd_jens):
-	training_remain = batch.shape[0]
-	batch_size = min(params.mini_batch_size, training_remain)
-	if batch_size == 0:
-		batch_size = training_remain
+def to_mini_batch(batch: Mat, mini_batch_size: int):
+	mini_batches = []
+	remain = batch.shape[0]
+	index = 0
+	while remain > 0:
+		batch_size = min(mini_batch_size, remain)
+		mini_batches.append(batch[index:index + batch_size])
+		remain -= batch_size
+		index += batch_size
 
-	status.on_change()
-	d_progress = 100.0 / (training_remain*params.num_epochs)
+	return mini_batches
+
+def train(entity: Entity, batch: Mat, params: TrainingParams, status: Status, cd_func: Callable = cd_jens):
+	mini_batch_size = min(params.mini_batch_size, batch.shape[0]) if params.mini_batch_size > 0 else batch.shape[0]
+	d_progress = 100.0 / (batch.shape[0]*params.num_epochs)
 	progress = 0
-	batch_row_index = 0
 	keep_running = True
-	while training_remain > 0 and keep_running:
-		batch_size_remain = min(batch_size, training_remain)
-		mini_batch = batch[batch_row_index:batch_row_index + batch_size_remain]
-		training_remain -= batch_size_remain
-		batch_row_index += batch_size_remain
+	status.on_change()
+	for mini_batch in to_mini_batch(batch, mini_batch_size):
+		if not keep_running:
+			break
 
 		inc_bv = np.zeros(entity.state.b_v.shape)
 		inc_bh = np.zeros(entity.state.b_h.shape)
 		inc_whv = np.zeros(entity.state.w_hv.shape)
 
-		v_states = mini_batch
-		if params.do_batch_sample:
-			v_states = sample(mini_batch)
-
 		for epochs in range(params.num_epochs):
 			# Contrastive divergence learning: calculate gradients
-			dwhv, dbv, dbh, _ = cd_func(entity, v_states, params)
+			dwhv, dbv, dbh, _ = cd_func(entity, mini_batch, params)
 
 			# Adjust weight and biases
-			kl = params.learning_rate/batch_size
+			kl = params.learning_rate/batch.shape[0]
 			inc_bv = params.momentum*inc_bv + kl*dbv
 			inc_bh = params.momentum*inc_bh + kl*dbh
 			inc_whv = params.momentum*inc_whv + kl*dwhv - params.weight_decay*entity.state.w_hv
@@ -120,79 +121,11 @@ def train(entity: Entity, batch: Mat, params: TrainingParams, status: Status, cd
 					keep_running = False
 					break
 
-			progress += d_progress*batch_size_remain
+			progress += d_progress*mini_batch.shape[0]
 
 	# Update final status
 	err_rms = rms_error_accu(batch - entity.reconstruct(entity.forward(batch)))
 	status.on_change({"progress": {"value": round(progress), "unit": "%"},
 					  "err_rms_total": {"value": err_rms, "unit": ""}})
 
-
-class Optimizer:
-	def __init__(self, entity: Entity, params: TrainingParams, cd_func: Callable = cd_jens):
-		self.inc_bv = np.zeros(entity.state.b_v.shape)
-		self.inc_bh = np.zeros(entity.state.b_h.shape)
-		self.inc_whv = np.zeros(entity.state.w_hv.shape)
-		self.entity = entity
-		self.params = params
-		self.cd = cd_func
-
-	def __call__(self, batch: Mat, status: Status):
-		return self.process(batch, status)
-
-	def process(self, batch: Mat, status: Status):
-		status.on_change()
-		d_progress = 100.0 / self.params.num_epochs
-		progress = 0
-
-		for epochs in range(self.params.num_epochs):
-			self.epoch(batch)
-
-			# check if status update is needed
-			if status.want_report(round(progress)):
-				# Calculate	error
-				forward = self.entity.forward(batch)
-				result = self.entity.reconstruct(forward)
-				err_rms = rms_error_accu(batch - result)
-				if not status.on_change({"progress": {"value": round(progress), "unit": "%"},
-										 "err_rms": {"value": err_rms, "unit": ""}}):
-					break
-
-			progress += d_progress
-
-		forward = self.entity.forward(batch)
-		err_rms = rms_error_accu(batch - self.entity.reconstruct(forward))
-		status.on_change({"progress": {"value": round(progress), "unit": "%"},
-					  "err_rms": {"value": err_rms, "unit": ""}})
-
-		return forward
-
-	def epoch(self, batch: Mat):
-		forward = None
-		training_remain = batch.shape[0]
-		batch_row_index = 0
-
-		while training_remain > 0:
-			batch_size = min(self.params.mini_batch_size, training_remain)
-			if batch_size == 0:
-				batch_size = training_remain
-
-			mini_batch = batch[batch_row_index:batch_row_index + batch_size]
-
-			# Contrastive divergence learning: calculate gradients
-			dwhv, dbv, dbh, forward = self.cd(self.entity, mini_batch, self.params)
-
-			# Adjust weight and biases
-			kl = self.params.learning_rate / batch_size
-			inc_bv = self.params.momentum * self.inc_bv + kl * dbv
-			inc_bh = self.params.momentum * self.inc_bh + kl * dbh
-			inc_whv = self.params.momentum * self.inc_whv + kl * dwhv - self.params.weight_decay * self.entity.state.w_hv
-
-			self.entity.state.b_v += inc_bv
-			self.entity.state.b_h += inc_bh
-			self.entity.state.w_hv += inc_whv
-
-			training_remain -= batch_size
-			batch_row_index += batch_size
-
-		return forward
+	return None