From 10691894d1400ec296e9038216a03daf680e0b9d Mon Sep 17 00:00:00 2001 From: Jens Ahrensfeld Date: Fri, 2 Jan 2026 14:51:19 +0100 Subject: [PATCH] - training params are (again) attrubute of Entity - ditched layout concept --- src/rbm/entity.py | 45 ++++++++++++++++++++++++++++- src/rbm/label.py | 7 ++--- src/rbm/layer.py | 6 ++-- src/rbm/layout.py | 29 ------------------- src/rbm/model.py | 12 ++++---- src/rbm/stack_deep.py | 4 +-- src/rbm/stack_factory.py | 3 +- src/rbm/train.py | 62 +++++++++------------------------------- 8 files changed, 70 insertions(+), 98 deletions(-) delete mode 100644 src/rbm/layout.py diff --git a/src/rbm/entity.py b/src/rbm/entity.py index 8196763..bf3c5e2 100644 --- a/src/rbm/entity.py +++ b/src/rbm/entity.py @@ -1,6 +1,48 @@ from .state import RbmState from .matrix import prob, Mat from enum import Enum + +class TrainingParams: + def __init__(self, + learning_rate: float = 0.1, + momentum: float = 0.5, + weight_decay: float = 0.0, + num_epochs: int = 1000, + num_gibbs_samples: int = 1, + mini_batch_size: int = 0, + do_rao_blackwell: bool = False, + do_gibbs_sample_visible: bool = False, + do_gibbs_sample_hidden: bool = False, + do_batch_sample: bool = False + ): + # Training parameters + self.learning_rate = learning_rate + self.momentum = momentum + self.weight_decay = weight_decay + self.num_epochs = num_epochs + self.num_gibbs_samples = num_gibbs_samples + self.mini_batch_size = mini_batch_size + self.do_rao_blackwell = do_rao_blackwell + self.do_gibbs_sample_visible = do_gibbs_sample_visible + self.do_gibbs_sample_hidden = do_gibbs_sample_hidden + self.do_batch_sample = do_batch_sample + + @classmethod + def from_dict(cls, params: dict): + obj = TrainingParams() + obj.learning_rate = params["learningRate"] + obj.momentum = params["momentum"] + obj.weight_decay = params["weightDecay"] + obj.num_epochs = params["numEpochs"] + obj.num_gibbs_samples = params["numGibbs"] + obj.mini_batch_size = params["miniBatchSize"] + obj.do_rao_blackwell = params["doRaoBlackwell"] + obj.do_gibbs_sample_visible = params["gibbsDoSampleVisible"] + obj.do_gibbs_sample_hidden = params["gibbsDoSampleHidden"] + obj.do_batch_sample = params["doSampleBatch"] + + return obj + class EntityParams: def __init__(self, do_gaussian_visible: bool = False, do_gaussian_hidden: bool = False, num_gibbs_samples: int = 1): # Entity parameters @@ -27,9 +69,10 @@ class Entity: GB_RBM = "GB-RBM" GG_RBM = "GG-RBM" - def __init__(self, shape: tuple[int, int], params: EntityParams): + def __init__(self, shape: tuple[int, int], params: EntityParams, training_params: TrainingParams|None = None): self.shape = shape self.params = params + self.training_params = training_params self.state = RbmState.from_layer_params(shape) self.grad = RbmState.from_layer_params(shape) self.type = None diff --git a/src/rbm/label.py b/src/rbm/label.py index 626e8d7..427c30d 100644 --- a/src/rbm/label.py +++ b/src/rbm/label.py @@ -1,14 +1,13 @@ from rbm.matrix import Mat, np from rbm.model import Model -from rbm.entity import Entity, EntityParams -from rbm.train import TrainingParams +from rbm.entity import Entity, EntityParams, TrainingParams import math class Label: class Fitter(Model): def __init__(self, dim: tuple[int,int], work_dir: str = '.'): super().__init__(f"label-{dim[0]}x{dim[1]}", work_dir) - self.unit1 = Entity(dim, EntityParams(num_gibbs_samples=1)) + self.unit1 = Entity(dim, EntityParams(num_gibbs_samples=1), TrainingParams(num_epochs=10000, do_rao_blackwell=False)) def forward(self, x: Mat) -> Mat: return self.unit1.forward(x) @@ -36,7 +35,7 @@ class Label: def fit(self, list_of_labels: np.array): label_vecs = self.label2vec(list_of_labels) - self.fitter.train(label_vecs, TrainingParams(num_epochs=10000, do_rao_blackwell=False)) + self.fitter.train(label_vecs) def encode(self, list_of_labels: np.array): label_vecs = self.label2vec(list_of_labels) diff --git a/src/rbm/layer.py b/src/rbm/layer.py index aa153c0..f0a4c66 100644 --- a/src/rbm/layer.py +++ b/src/rbm/layer.py @@ -1,12 +1,10 @@ -from .state import RbmState -from .train import TrainingParams -from .entity import Entity, EntityParams +from .entity import Entity, EntityParams, TrainingParams class Layer: def __init__(self, name: str, shape: tuple[int, int, int, int], entity_params: EntityParams, training_params: TrainingParams): self.name = name self.shape = shape - self.entity = Entity((shape[0]*shape[1]+shape[2], shape[3]), entity_params) + self.entity = Entity((shape[0]*shape[1]+shape[2], shape[3]), entity_params, training_params) self.training_params = training_params def init(self, std: float): diff --git a/src/rbm/layout.py b/src/rbm/layout.py deleted file mode 100644 index a29db4b..0000000 --- a/src/rbm/layout.py +++ /dev/null @@ -1,29 +0,0 @@ -import numpy as np - -from .entity import Entity -from .matrix import Mat -from .train import train, cd_jens, TrainingParams -from .status import Status - -class Horizontal: - def __init__(self, entities: list[Entity]): - self.units: list[Entity] = entities - - def forward(self, x: Mat): - res: Mat = Mat([]) - for unit in self.units: - x = unit.forward(x) - res = np.concat((res, x)) - - def reconstruct(self, x: Mat): - res: Mat = Mat([]) - for unit in self.units: - x = unit.forward(x) - res = np.concat((res, x)) - - def train(self, batch: Mat, params: TrainingParams): - _batch = np.copy(batch) - for unit in self.units: - train(unit, _batch, params, Status()) - _batch = unit.forward(_batch, num_gibbs=params.num_gibbs_samples) - diff --git a/src/rbm/model.py b/src/rbm/model.py index b4a854f..0de9905 100644 --- a/src/rbm/model.py +++ b/src/rbm/model.py @@ -2,7 +2,7 @@ import os from abc import ABC, abstractmethod from .entity import Entity -from .train import train, TrainingParams, cd_jens +from .train import train from .matrix import Mat, np from .status import Status @@ -20,14 +20,12 @@ class Model(ABC): obj_list.append(value) return obj_list - def train(self, batch: Mat, params: TrainingParams|list[TrainingParams]): + def train(self, batch: Mat): entities = self.objects(Entity) - if isinstance(params, TrainingParams): - params = [params]*len(entities) _batch = np.copy(batch) - for entity, param in zip(entities, params): - train(entity, _batch, param, Status()) - _batch = entity.forward(_batch, num_gibbs=param.num_gibbs_samples) + for entity in entities: + if train(entity, _batch, Status()): + _batch = entity.forward(_batch) @abstractmethod def forward(self, x: Mat) -> Mat: diff --git a/src/rbm/stack_deep.py b/src/rbm/stack_deep.py index 7cb12d8..d00abe5 100644 --- a/src/rbm/stack_deep.py +++ b/src/rbm/stack_deep.py @@ -10,8 +10,8 @@ class StackDeep(Stack): def train(self, batch: Mat, status=Status()): _batch = np.copy(batch) for index, layer in enumerate(self.layers): - print(f"Train layer {index} for {layer.training_params.num_epochs} epochs") - train(layer.entity, _batch, layer.training_params, status=status) + print(f"Train layer {index} for {layer.entity.training_params.num_epochs} epochs") + train(layer.entity, _batch, status=status) _batch = layer.entity.forward(_batch) def pass_up(self, visible: Mat, from_layer_id: int = 0): diff --git a/src/rbm/stack_factory.py b/src/rbm/stack_factory.py index c20e09b..6f2f5ab 100644 --- a/src/rbm/stack_factory.py +++ b/src/rbm/stack_factory.py @@ -1,8 +1,7 @@ import json from .stack import StackType from .layer import Layer -from .entity import EntityParams -from .train import TrainingParams +from .entity import EntityParams, TrainingParams from .stack_deep import StackDeep from .stack_rnn import StackRnn diff --git a/src/rbm/train.py b/src/rbm/train.py index 3d88523..af960f5 100644 --- a/src/rbm/train.py +++ b/src/rbm/train.py @@ -1,50 +1,9 @@ -from collections.abc import Callable from .matrix import sample, sample_gaussian, prob, rms_error_accu, Mat, np from .entity import Entity from .status import Status -class TrainingParams: - def __init__(self, - learning_rate: float = 0.1, - momentum: float = 0.5, - weight_decay: float = 0.0, - num_epochs: int = 1000, - num_gibbs_samples: int = 1, - mini_batch_size: int = 0, - do_rao_blackwell: bool = False, - do_gibbs_sample_visible: bool = False, - do_gibbs_sample_hidden: bool = False, - do_batch_sample: bool = False - ): - # Training parameters - self.learning_rate = learning_rate - self.momentum = momentum - self.weight_decay = weight_decay - self.num_epochs = num_epochs - self.num_gibbs_samples = num_gibbs_samples - self.mini_batch_size = mini_batch_size - self.do_rao_blackwell = do_rao_blackwell - self.do_gibbs_sample_visible = do_gibbs_sample_visible - self.do_gibbs_sample_hidden = do_gibbs_sample_hidden - self.do_batch_sample = do_batch_sample - - @classmethod - def from_dict(cls, params: dict): - obj = TrainingParams() - obj.learning_rate = params["learningRate"] - obj.momentum = params["momentum"] - obj.weight_decay = params["weightDecay"] - obj.num_epochs = params["numEpochs"] - obj.num_gibbs_samples = params["numGibbs"] - obj.mini_batch_size = params["miniBatchSize"] - obj.do_rao_blackwell = params["doRaoBlackwell"] - obj.do_gibbs_sample_visible = params["gibbsDoSampleVisible"] - obj.do_gibbs_sample_hidden = params["gibbsDoSampleHidden"] - obj.do_batch_sample = params["doSampleBatch"] - - return obj - -def cd_jens(entity: Entity, v_states: Mat, params: TrainingParams): +def cd_jens(entity: Entity, v_states: Mat): + params = entity.training_params v_probs = prob(v_states) h_states = entity.forward(v_states) h_probs = h_states @@ -83,7 +42,8 @@ def cd_jens(entity: Entity, v_states: Mat, params: TrainingParams): return dw, dbv, dbh -def cd_binary_binary(entity: Entity, data_pos: Mat, params: TrainingParams): +def cd_binary_binary(entity: Entity, data_pos: Mat): + params = entity.training_params # Positive phase h_probs_pos = prob(entity.h_given_v(data_pos)) @@ -112,7 +72,7 @@ def cd_binary_binary(entity: Entity, data_pos: Mat, params: TrainingParams): return dw, dbv, dbh -def cd_gaussian_binary(entity: Entity, data_pos: Mat, params: TrainingParams): +def cd_gaussian_binary(entity: Entity, data_pos: Mat): # Positive phase h_probs_pos = entity.h_given_v(data_pos) @@ -135,7 +95,7 @@ def cd_gaussian_binary(entity: Entity, data_pos: Mat, params: TrainingParams): return dw, dbv, dbh -def cd_gaussian_gaussian(entity: Entity, data_pos: Mat, params: TrainingParams): +def cd_gaussian_gaussian(entity: Entity, data_pos: Mat): # Positive phase h_probs_pos = entity.h_given_v(data_pos) @@ -170,7 +130,11 @@ def to_mini_batch(batch: Mat, mini_batch_size: int): return mini_batches -def train(entity: Entity, batch: Mat, params: TrainingParams, status: Status): +def train(entity: Entity, batch: Mat, status: Status): + params = entity.training_params + if params is None: + return False + mini_batch_size = min(params.mini_batch_size, batch.shape[0]) if params.mini_batch_size > 0 else batch.shape[0] d_progress = 100.0 / (batch.shape[0]*params.num_epochs) progress = 0 @@ -191,7 +155,7 @@ def train(entity: Entity, batch: Mat, params: TrainingParams, status: Status): entity.grad_zero() for epochs in range(params.num_epochs): # Contrastive divergence learning: calculate gradients - dwhv, dbv, dbh = cd_func(entity, mini_batch, params) + dwhv, dbv, dbh = cd_func(entity, mini_batch) # Adjust weight and biases grad = entity.grad_compute(dbv, dbh, dwhv, learning_rate=params.learning_rate/batch.shape[0], momentum=params.momentum, weight_decay=params.weight_decay) @@ -213,4 +177,4 @@ def train(entity: Entity, batch: Mat, params: TrainingParams, status: Status): status.on_change({"progress": {"value": round(progress), "unit": "%"}, "err_rms_total": {"value": err_rms, "unit": ""}}) - return None + return True