significantly improved GB-RBM training

src/rbm/train.py

@@ -93,60 +93,42 @@ def cd_binary_binary(entity: Entity, data_pos: Mat):
 	return dw, dbv, dbh
 
 def cd_gaussian_binary(entity: Entity, data_pos: Mat):
-	params = entity.training_params
-
 	# Positive phase
-	if params.do_batch_sample:
+	h_probs_pos = prob(entity.h_given_v(data_pos + sample_gaussian(data_pos)))
-		h_probs_pos = entity.h_given_v(data_pos + sample_gaussian(data_pos))
-	else:
-		h_probs_pos = entity.h_given_v(data_pos)
 
 	# Update weights (positive phase)
 	dw = np.dot(np.transpose(data_pos), h_probs_pos)
 	dbh = np.sum(h_probs_pos, 0)
-	dbv = np.sum(data_pos, 0)
+	dbv = np.sum(data_pos-entity.state.b_v, 0)
-
-	if not params.do_rao_blackwell:
-		# Sample hidden states
-		h_probs_pos = sample(h_probs_pos)
 
 	# Negative phase
 	data_neg = entity.v_given_h(h_probs_pos)
-	h_probs_neg = entity.h_given_v(data_neg)
+	h_probs_neg = prob(entity.h_given_v(data_neg + sample_gaussian(data_neg)))
 
 	# Update weights (negative phase)
 	dw -= np.dot(np.transpose(data_neg), h_probs_neg)
 	dbh -= np.sum(h_probs_neg, 0)
-	dbv -= np.sum(data_neg, 0)
+	dbv -= np.sum(data_neg-entity.state.b_v, 0)
 
 	return dw, dbv, dbh
 
 def cd_gaussian_gaussian(entity: Entity, data_pos: Mat):
-	params = entity.training_params
-
 	# Positive phase
-	if params.do_batch_sample:
+	h_probs_pos = entity.h_given_v(data_pos)
-		h_probs_pos = entity.h_given_v(data_pos + sample_gaussian(data_pos))
-	else:
-		h_probs_pos = entity.h_given_v(data_pos)
-
-	if not params.do_rao_blackwell:
-		# Sample hidden states
-		h_probs_pos += sample_gaussian(h_probs_pos)
 
 	# Update weights (positive phase)
-	dw = np.dot(np.transpose(data_pos), h_probs_pos)
+	dw = np.dot(np.transpose(data_pos), h_probs_pos + sample_gaussian(h_probs_pos))
 	dbh = np.sum(h_probs_pos, 0)
-	dbv = np.sum(data_pos, 0)
+	dbv = np.sum(data_pos-entity.state.b_v, 0)
 
 	# Negative phase
 	data_neg = entity.v_given_h(h_probs_pos)
-	h_probs_neg = entity.h_given_v(data_neg)
+	h_probs_neg = entity.h_given_v(data_neg + sample_gaussian(data_neg))
 
 	# Update weights (negative phase)
 	dw -= np.dot(np.transpose(data_neg), h_probs_neg)
 	dbh -= np.sum(h_probs_neg, 0)
-	dbv -= np.sum(data_neg, 0)
+	dbv -= np.sum(data_neg-entity.state.b_v, 0)
 
 	return dw, dbv, dbh
 

src/tests/test_norbs.py

@@ -11,11 +11,12 @@ class TestModel(Model):
 
 		if do_gaussian_hidden:
 			# Hidden gaussian
-			self.unit1 = Entity((96*96, 16), EntityParams(do_gaussian_visible=True, do_gaussian_hidden=True), TrainingParams(learning_rate=0.00001, momentum=0.9, num_epochs=1000, do_rao_blackwell=True))
+			self.unit1 = Entity((96*96, 16), EntityParams(do_gaussian_visible=True, do_gaussian_hidden=True),
+								TrainingParams(learning_rate=0.000001, momentum=0.9, num_epochs=1000))
 		else:
 			# Hidden binary
-			self.unit1 = Entity((96 * 96, 16), EntityParams(do_gaussian_visible=True, do_gaussian_hidden=False),
+			self.unit1 = Entity((96 * 96, 333), EntityParams(do_gaussian_visible=True, do_gaussian_hidden=False),
-								TrainingParams(learning_rate=0.000002, momentum=0.9, num_epochs=1000))
+								TrainingParams(learning_rate=0.0001, momentum=0.9, num_epochs=1000))
 
 	def forward(self, x: Mat):
 		x = self.unit1.forward(x)
@@ -34,10 +35,10 @@ if __name__ == "__main__":
 	model = TestModel(prj_name, "results")
 
 	# Init state
-	model.init(0.01)
+	model.init(0.1)
 
 	# load state
-#	model.load()
+	model.load()
 
 	# Load train data
 	train_batch = read_armadillo(os.path.join(prj_root, f"{prj_name}.training.dat"))
@@ -56,7 +57,7 @@ if __name__ == "__main__":
 		out_normalized = model.backward(model.forward(inp))
 		img = 2*(out_normalized + 0.5)
 		img = np.reshape(img, (96, 96))
-		axes[index].imshow(img)
+		axes[index].imshow(np.asnumpy(img))
 		axes[index].axis('off')
 
 	plt.show()