From 61c762150c34291b75ec1245eb47e8effe4f3b49 Mon Sep 17 00:00:00 2001 From: Jens Ahrensfeld Date: Sat, 20 Dec 2025 20:12:19 +0100 Subject: [PATCH] - removed Optimizer - refactored training - use static seed for random (for better comparison) - simplified StackDeep.train --- src/rbm/matrix.py | 3 +- src/rbm/stack_deep.py | 22 ++------ src/rbm/stack_factory.py | 2 +- src/rbm/train.py | 111 ++++++++------------------------------- src/tests/test_xor.py | 8 +-- 5 files changed, 30 insertions(+), 116 deletions(-) diff --git a/src/rbm/matrix.py b/src/rbm/matrix.py index 038005a..441cda8 100644 --- a/src/rbm/matrix.py +++ b/src/rbm/matrix.py @@ -13,7 +13,8 @@ else: def convert(src: Mat): return src -np.random.seed(int(time.monotonic())) +np.random.seed(12345) + def uniform(shape: tuple, mu: float = 0.5, std: float = 1.0) -> Mat: return std * (np.random.rand(shape[0], shape[1]) + mu - 0.5) diff --git a/src/rbm/stack_deep.py b/src/rbm/stack_deep.py index 76607b6..7cb12d8 100644 --- a/src/rbm/stack_deep.py +++ b/src/rbm/stack_deep.py @@ -1,34 +1,18 @@ from .status import Status -from .train import train, Optimizer +from .train import train from .stack import Stack, StackType from .matrix import Mat, np -USE_OPTIMIZER = False - class StackDeep(Stack): def __init__(self, name: str, work_dir: str = '.'): Stack.__init__(self, StackType.Deep, name, work_dir) - def batch_from(self, batch: Mat, from_layer_id: int = 0): - _batch = np.copy(batch) - for index, layer in enumerate(self.layers): - if index == from_layer_id: - break - _batch = layer.entity.forward(_batch) - - return _batch - def train(self, batch: Mat, status=Status()): _batch = np.copy(batch) for index, layer in enumerate(self.layers): print(f"Train layer {index} for {layer.training_params.num_epochs} epochs") - if USE_OPTIMIZER: - optim = Optimizer(layer.entity, layer.training_params) - _batch = optim(_batch, status=status) - else: - _batch = self.batch_from(batch, index) - train(layer.entity, _batch, layer.training_params, status=status) - + train(layer.entity, _batch, layer.training_params, status=status) + _batch = layer.entity.forward(_batch) def pass_up(self, visible: Mat, from_layer_id: int = 0): h = np.copy(visible) diff --git a/src/rbm/stack_factory.py b/src/rbm/stack_factory.py index 4b7a81b..b56c8a5 100644 --- a/src/rbm/stack_factory.py +++ b/src/rbm/stack_factory.py @@ -8,7 +8,7 @@ from .stack_rnn import StackRnn class StackFactory: @classmethod - def from_dict(cls, project: dict, work_dir: str = ".") -> StackDeep|StackRnn: + def from_dict(cls, project: dict, work_dir: str = ".") -> StackDeep|StackRnn|None: name = project["stack"]["name"] layers = project["stack"]["layers"] diff --git a/src/rbm/train.py b/src/rbm/train.py index a6896b6..671cd4c 100644 --- a/src/rbm/train.py +++ b/src/rbm/train.py @@ -72,37 +72,38 @@ def cd_jens(entity: Entity, v_states: Mat, params: TrainingParams): return dw, dbv, dbh, h_probs -def train(entity: Entity, batch: Mat, params: TrainingParams, status: Status, cd_func: Callable = cd_jens): - training_remain = batch.shape[0] - batch_size = min(params.mini_batch_size, training_remain) - if batch_size == 0: - batch_size = training_remain +def to_mini_batch(batch: Mat, mini_batch_size: int): + mini_batches = [] + remain = batch.shape[0] + index = 0 + while remain > 0: + batch_size = min(mini_batch_size, remain) + mini_batches.append(batch[index:index + batch_size]) + remain -= batch_size + index += batch_size - status.on_change() - d_progress = 100.0 / (training_remain*params.num_epochs) + return mini_batches + +def train(entity: Entity, batch: Mat, params: TrainingParams, status: Status, cd_func: Callable = cd_jens): + mini_batch_size = min(params.mini_batch_size, batch.shape[0]) if params.mini_batch_size > 0 else batch.shape[0] + d_progress = 100.0 / (batch.shape[0]*params.num_epochs) progress = 0 - batch_row_index = 0 keep_running = True - while training_remain > 0 and keep_running: - batch_size_remain = min(batch_size, training_remain) - mini_batch = batch[batch_row_index:batch_row_index + batch_size_remain] - training_remain -= batch_size_remain - batch_row_index += batch_size_remain + status.on_change() + for mini_batch in to_mini_batch(batch, mini_batch_size): + if not keep_running: + break inc_bv = np.zeros(entity.state.b_v.shape) inc_bh = np.zeros(entity.state.b_h.shape) inc_whv = np.zeros(entity.state.w_hv.shape) - v_states = mini_batch - if params.do_batch_sample: - v_states = sample(mini_batch) - for epochs in range(params.num_epochs): # Contrastive divergence learning: calculate gradients - dwhv, dbv, dbh, _ = cd_func(entity, v_states, params) + dwhv, dbv, dbh, _ = cd_func(entity, mini_batch, params) # Adjust weight and biases - kl = params.learning_rate/batch_size + kl = params.learning_rate/batch.shape[0] inc_bv = params.momentum*inc_bv + kl*dbv inc_bh = params.momentum*inc_bh + kl*dbh inc_whv = params.momentum*inc_whv + kl*dwhv - params.weight_decay*entity.state.w_hv @@ -120,79 +121,11 @@ def train(entity: Entity, batch: Mat, params: TrainingParams, status: Status, cd keep_running = False break - progress += d_progress*batch_size_remain + progress += d_progress*mini_batch.shape[0] # Update final status err_rms = rms_error_accu(batch - entity.reconstruct(entity.forward(batch))) status.on_change({"progress": {"value": round(progress), "unit": "%"}, "err_rms_total": {"value": err_rms, "unit": ""}}) - -class Optimizer: - def __init__(self, entity: Entity, params: TrainingParams, cd_func: Callable = cd_jens): - self.inc_bv = np.zeros(entity.state.b_v.shape) - self.inc_bh = np.zeros(entity.state.b_h.shape) - self.inc_whv = np.zeros(entity.state.w_hv.shape) - self.entity = entity - self.params = params - self.cd = cd_func - - def __call__(self, batch: Mat, status: Status): - return self.process(batch, status) - - def process(self, batch: Mat, status: Status): - status.on_change() - d_progress = 100.0 / self.params.num_epochs - progress = 0 - - for epochs in range(self.params.num_epochs): - self.epoch(batch) - - # check if status update is needed - if status.want_report(round(progress)): - # Calculate error - forward = self.entity.forward(batch) - result = self.entity.reconstruct(forward) - err_rms = rms_error_accu(batch - result) - if not status.on_change({"progress": {"value": round(progress), "unit": "%"}, - "err_rms": {"value": err_rms, "unit": ""}}): - break - - progress += d_progress - - forward = self.entity.forward(batch) - err_rms = rms_error_accu(batch - self.entity.reconstruct(forward)) - status.on_change({"progress": {"value": round(progress), "unit": "%"}, - "err_rms": {"value": err_rms, "unit": ""}}) - - return forward - - def epoch(self, batch: Mat): - forward = None - training_remain = batch.shape[0] - batch_row_index = 0 - - while training_remain > 0: - batch_size = min(self.params.mini_batch_size, training_remain) - if batch_size == 0: - batch_size = training_remain - - mini_batch = batch[batch_row_index:batch_row_index + batch_size] - - # Contrastive divergence learning: calculate gradients - dwhv, dbv, dbh, forward = self.cd(self.entity, mini_batch, self.params) - - # Adjust weight and biases - kl = self.params.learning_rate / batch_size - inc_bv = self.params.momentum * self.inc_bv + kl * dbv - inc_bh = self.params.momentum * self.inc_bh + kl * dbh - inc_whv = self.params.momentum * self.inc_whv + kl * dwhv - self.params.weight_decay * self.entity.state.w_hv - - self.entity.state.b_v += inc_bv - self.entity.state.b_h += inc_bh - self.entity.state.w_hv += inc_whv - - training_remain -= batch_size - batch_row_index += batch_size - - return forward + return None diff --git a/src/tests/test_xor.py b/src/tests/test_xor.py index 2bdaf96..8a8adc2 100644 --- a/src/tests/test_xor.py +++ b/src/tests/test_xor.py @@ -2,7 +2,7 @@ import os.path from rbm.layer import Layer from rbm.status import Status -from rbm.train import train, TrainingParams, Optimizer +from rbm.train import train, TrainingParams from rbm.matrix import Mat, np from rbm.entity import EntityParams @@ -29,11 +29,7 @@ def xor(): training_batch = Mat([[0,1,1], [0,0,0], [1,1,0], [1,0,1]], dtype=np.float64) # Train layer - if USE_OPTIMIZER: - optim = Optimizer(layer.entity, training_params) - optim(training_batch, Status()) - else: - train(layer.entity, training_batch, training_params, Status()) + train(layer.entity, training_batch, training_params, Status()) # Save weights layer.save(os.path.join(WORK_DIR, "xor_layer0_state.npz"))