diff --git a/src/rbm/cd_train.py b/src/rbm/cd_train.py index 50e402a..a434af4 100644 --- a/src/rbm/cd_train.py +++ b/src/rbm/cd_train.py @@ -2,10 +2,9 @@ import numpy as np from collections.abc import Callable from params import RbmParams -from rbm.helper import gaussian -from helper import sample +from helper import sample, gaussian -def train(v_states: np.ndarray, params: RbmParams, v_to_ph: Callable, h_to_pv: Callable): +def cd_jens(v_states: np.ndarray, params: RbmParams, v_to_ph: Callable, h_to_pv: Callable): v_probs = v_states h_states = v_to_ph(v_states) h_probs = h_states @@ -20,7 +19,7 @@ def train(v_states: np.ndarray, params: RbmParams, v_to_ph: Callable, h_to_pv: C h_states = sample(h_probs) # Update weights (positive phase) - dw = np.transpose(v_states) * h_states + dw = np.vecmat(np.transpose(v_states), h_states) dbv = np.sum(v_states, 0) dbh = np.sum(h_states, 0) @@ -38,7 +37,7 @@ def train(v_states: np.ndarray, params: RbmParams, v_to_ph: Callable, h_to_pv: C h_probs = v_to_ph(v_probs) # Update weights (negative phase) - dw -= np.transpose(v_probs) * h_probs + dw -= np.vecmat(np.transpose(v_probs), h_probs) dbv -= np.sum(v_probs, 0) dbh -= np.sum(h_probs, 0) diff --git a/src/rbm/helper.py b/src/rbm/helper.py index 89abb1e..3b51f5b 100644 --- a/src/rbm/helper.py +++ b/src/rbm/helper.py @@ -16,10 +16,11 @@ def prob(src: np.ndarray) -> np.ndarray: return 1.0 / (1 + np.exp(-src)) def rms_error(d_err: np.ndarray): - d_err_squared = d_err % d_err + d_err_squared = d_err * d_err return np.sum(d_err_squared, 1) / d_err_squared[1] def rms_error_accu(d_err: np.ndarray): - d_err_squared = d_err % d_err - return np.cumsum(d_err_squared) / d_err_squared[1] + d_err_squared = d_err * d_err + s = np.sum(d_err_squared) / len(d_err) + return s diff --git a/src/rbm/layer.py b/src/rbm/layer.py index bba7186..38300ad 100644 --- a/src/rbm/layer.py +++ b/src/rbm/layer.py @@ -4,6 +4,7 @@ from params import RbmParams from state import RbmState from helper import sample, prob, uniform, rms_error_accu from status import Status +from cd_train import cd_jens class RbmLayer: def __init__(self, name: str, num_visible, num_hidden, params: RbmParams): @@ -19,23 +20,25 @@ class RbmLayer: self.state = RbmState.from_file(self.state_filename) def train(self, batch: np.ndarray, cd_func: Callable, status: Status): - num_cases = min(self.params.mini_batch_size, batch.shape[0]) - d_progress = 100.0 / (batch.shape[0] * self.params.num_epochs) - last_status = status - status.progress = 0 + training_size = batch.shape[0] + num_cases = min(self.params.mini_batch_size, training_size) + d_progress = 100.0 / (training_size/num_cases * self.params.num_epochs) + progress = 0 + last_progress = progress batch_row_index = 0 keep_running = True - training_size_remain = batch.shape[0] - while training_size_remain > 0 and keep_running: - mini_batch_size = min(self.params.mini_batch_size, training_size_remain) - mini_batch = batch[batch_row_index:batch_row_index + mini_batch_size - 1] - training_size_remain -= mini_batch_size + training_remain = training_size + training_seen = 0 + while training_remain > 0 and keep_running: + mini_batch_size = min(self.params.mini_batch_size, training_remain) + mini_batch = batch[batch_row_index:batch_row_index + mini_batch_size] + training_remain -= mini_batch_size batch_row_index += mini_batch_size inc_bv = np.zeros(self.state.b_v.shape) - inc_bh = np.zeros(self.state.bh.shape) + inc_bh = np.zeros(self.state.b_h.shape) inc_whv = np.zeros(self.state.w_hv.shape) v_states = mini_batch @@ -44,7 +47,7 @@ class RbmLayer: for epochs in range(self.params.num_epochs): # Contrastive divergence learning: calculate gradients - dwhv, dbh, dbv = cd_func(v_states, self.params, self.v_to_ph, self.h_to_pv) + dwhv, dbv, dbh = cd_func(v_states, self.params, self.v_to_ph, self.h_to_pv) # Adjust weight and biases kl = self.params.learning_rate/num_cases @@ -56,15 +59,19 @@ class RbmLayer: self.state.b_h += inc_bh self.state.w_hv += inc_whv - status.progress += round(d_progress*mini_batch_size) + progress = round(epochs*d_progress) - if status.__dict__ != last_status.__dict__: + if progress != last_progress: # Calculate error + status.progress = round(progress) status.err = rms_error_accu(mini_batch - self.h_to_pv(self.v_to_ph(v_states))) if not status.on_change(): keep_running = False break + last_progress = progress + training_seen += 1 + status.on_change() def v_to_ph(self, v: np.ndarray) -> np.ndarray: @@ -82,25 +89,24 @@ class RbmLayer: return prob(state) +def xor(): + params = RbmParams() + params.do_rao_blackwell = True + params.num_gibbs_samples = 3 + params.mini_batch_size = 100 + params.learning_rate = 0.1 + params.momentum = 0.5 + params.num_epochs = 10000 + + status = Status() + layer = RbmLayer("Layer_0", 3, 16, params) + training_batch = np.array([[0,0,0], [0,1,1], [1,0,1], [1,1,0]], dtype=np.float64) + test_batch = np.array([[0,0,0], [0,1,0], [1,0,0], [1,1,0]], dtype=np.float64) + + layer.train(training_batch, cd_jens, status) + if __name__ == "__main__": - p = RbmParams() - l1 = RbmLayer("Layer_0", 2, 3, p) - l1.save() - l1.load() - l1.state.init(mu=0.5, std=1.0) - - s1 = RbmState(l1.state.w_hv, l1.state.b_v, l1.state.b_h) - - v0 = uniform((1,2)) - h0 = uniform((1,3)) - - h1 = l1.state.v_to_h(v0) - print(h1.shape) - v1 = l1.state.h_to_v(h1) - s_v1 = sample(v1) - p_h = prob(s_v1) - - print(v1.shape) + xor() diff --git a/src/rbm/status.py b/src/rbm/status.py index 7dfc716..bc87687 100644 --- a/src/rbm/status.py +++ b/src/rbm/status.py @@ -7,8 +7,10 @@ class Status: self.l2 = -1.0 def on_change(self) -> bool: - print(self) + print("-------------------------------------------") + print(f"Progress : {self.progress} %") + print(f"error (per mini batch) : {self.err}") + print(f"error (total) : {self.err_total}") + print(f"L1 : {self.l1}") + print(f"L2 : {self.l2}") return True - - def __repr__(self): - return f"Progress: {self.progress}"