jens/pyRBM
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- removed Optimizer

Browse Source 
- refactored training
- use static seed for random (for better comparison)
- simplified StackDeep.train
This commit is contained in:
jens
2025-12-20 20:12:19 +01:00
parent 5f5c7c6d77
commit 61c762150c
5 changed files with 30 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/rbm/matrix.py
+2 -1
View File
@@ -13,7 +13,8 @@ else:
def convert(src: Mat):
return src
np.random.seed(int(time.monotonic()))
np.random.seed(12345)
def uniform(shape: tuple, mu: float = 0.5, std: float = 1.0) -> Mat:
return std * (np.random.rand(shape[0], shape[1]) + mu - 0.5)
src/rbm/stack_deep.py
+3 -19
View File
@@ -1,34 +1,18 @@
from .status import Status
from .train import train, Optimizer
from .train import train
from .stack import Stack, StackType
from .matrix import Mat, np
USE_OPTIMIZER = False
class StackDeep(Stack):
def __init__(self, name: str, work_dir: str = '.'):
Stack.__init__(self, StackType.Deep, name, work_dir)
def batch_from(self, batch: Mat, from_layer_id: int = 0):
_batch = np.copy(batch)
for index, layer in enumerate(self.layers):
if index == from_layer_id:
break
_batch = layer.entity.forward(_batch)
return _batch
def train(self, batch: Mat, status=Status()):
_batch = np.copy(batch)
for index, layer in enumerate(self.layers):
print(f"Train layer {index} for {layer.training_params.num_epochs} epochs")
if USE_OPTIMIZER:
optim = Optimizer(layer.entity, layer.training_params)
_batch = optim(_batch, status=status)
else:
_batch = self.batch_from(batch, index)
train(layer.entity, _batch, layer.training_params, status=status)
train(layer.entity, _batch, layer.training_params, status=status)
_batch = layer.entity.forward(_batch)
def pass_up(self, visible: Mat, from_layer_id: int = 0):
h = np.copy(visible)
src/rbm/stack_factory.py
+1 -1
View File
@@ -8,7 +8,7 @@ from .stack_rnn import StackRnn
class StackFactory:
@classmethod
def from_dict(cls, project: dict, work_dir: str = ".") -> StackDeep|StackRnn:
def from_dict(cls, project: dict, work_dir: str = ".") -> StackDeep|StackRnn|None:
name = project["stack"]["name"]
layers = project["stack"]["layers"]
src/rbm/train.py
+22 -89
View File
@@ -72,37 +72,38 @@ def cd_jens(entity: Entity, v_states: Mat, params: TrainingParams):
return dw, dbv, dbh, h_probs
def train(entity: Entity, batch: Mat, params: TrainingParams, status: Status, cd_func: Callable = cd_jens):
training_remain = batch.shape[0]
batch_size = min(params.mini_batch_size, training_remain)
if batch_size == 0:
batch_size = training_remain
def to_mini_batch(batch: Mat, mini_batch_size: int):
mini_batches = []
remain = batch.shape[0]
index = 0
while remain > 0:
batch_size = min(mini_batch_size, remain)
mini_batches.append(batch[index:index + batch_size])
remain -= batch_size
index += batch_size
status.on_change()
d_progress = 100.0 / (training_remain*params.num_epochs)
return mini_batches
def train(entity: Entity, batch: Mat, params: TrainingParams, status: Status, cd_func: Callable = cd_jens):
mini_batch_size = min(params.mini_batch_size, batch.shape[0]) if params.mini_batch_size > 0 else batch.shape[0]
d_progress = 100.0 / (batch.shape[0]*params.num_epochs)
progress = 0
batch_row_index = 0
keep_running = True
while training_remain > 0 and keep_running:
batch_size_remain = min(batch_size, training_remain)
mini_batch = batch[batch_row_index:batch_row_index + batch_size_remain]
training_remain -= batch_size_remain
batch_row_index += batch_size_remain
status.on_change()
for mini_batch in to_mini_batch(batch, mini_batch_size):
if not keep_running:
break
inc_bv = np.zeros(entity.state.b_v.shape)
inc_bh = np.zeros(entity.state.b_h.shape)
inc_whv = np.zeros(entity.state.w_hv.shape)
v_states = mini_batch
if params.do_batch_sample:
v_states = sample(mini_batch)
for epochs in range(params.num_epochs):
# Contrastive divergence learning: calculate gradients
dwhv, dbv, dbh, _ = cd_func(entity, v_states, params)
dwhv, dbv, dbh, _ = cd_func(entity, mini_batch, params)
# Adjust weight and biases
kl = params.learning_rate/batch_size
kl = params.learning_rate/batch.shape[0]
inc_bv = params.momentum*inc_bv + kl*dbv
inc_bh = params.momentum*inc_bh + kl*dbh
inc_whv = params.momentum*inc_whv + kl*dwhv - params.weight_decay*entity.state.w_hv
@@ -120,79 +121,11 @@ def train(entity: Entity, batch: Mat, params: TrainingParams, status: Status, cd
keep_running = False
break
progress += d_progress*batch_size_remain
progress += d_progress*mini_batch.shape[0]
# Update final status
err_rms = rms_error_accu(batch - entity.reconstruct(entity.forward(batch)))
status.on_change({"progress": {"value": round(progress), "unit": "%"},
"err_rms_total": {"value": err_rms, "unit": ""}})
class Optimizer:
def __init__(self, entity: Entity, params: TrainingParams, cd_func: Callable = cd_jens):
self.inc_bv = np.zeros(entity.state.b_v.shape)
self.inc_bh = np.zeros(entity.state.b_h.shape)
self.inc_whv = np.zeros(entity.state.w_hv.shape)
self.entity = entity
self.params = params
self.cd = cd_func
def __call__(self, batch: Mat, status: Status):
return self.process(batch, status)
def process(self, batch: Mat, status: Status):
status.on_change()
d_progress = 100.0 / self.params.num_epochs
progress = 0
for epochs in range(self.params.num_epochs):
self.epoch(batch)
# check if status update is needed
if status.want_report(round(progress)):
# Calculate error
forward = self.entity.forward(batch)
result = self.entity.reconstruct(forward)
err_rms = rms_error_accu(batch - result)
if not status.on_change({"progress": {"value": round(progress), "unit": "%"},
"err_rms": {"value": err_rms, "unit": ""}}):
break
progress += d_progress
forward = self.entity.forward(batch)
err_rms = rms_error_accu(batch - self.entity.reconstruct(forward))
status.on_change({"progress": {"value": round(progress), "unit": "%"},
"err_rms": {"value": err_rms, "unit": ""}})
return forward
def epoch(self, batch: Mat):
forward = None
training_remain = batch.shape[0]
batch_row_index = 0
while training_remain > 0:
batch_size = min(self.params.mini_batch_size, training_remain)
if batch_size == 0:
batch_size = training_remain
mini_batch = batch[batch_row_index:batch_row_index + batch_size]
# Contrastive divergence learning: calculate gradients
dwhv, dbv, dbh, forward = self.cd(self.entity, mini_batch, self.params)
# Adjust weight and biases
kl = self.params.learning_rate / batch_size
inc_bv = self.params.momentum * self.inc_bv + kl * dbv
inc_bh = self.params.momentum * self.inc_bh + kl * dbh
inc_whv = self.params.momentum * self.inc_whv + kl * dwhv - self.params.weight_decay * self.entity.state.w_hv
self.entity.state.b_v += inc_bv
self.entity.state.b_h += inc_bh
self.entity.state.w_hv += inc_whv
training_remain -= batch_size
batch_row_index += batch_size
return forward
return None
src/tests/test_xor.py
+2 -6
View File
@@ -2,7 +2,7 @@ import os.path
from rbm.layer import Layer
from rbm.status import Status
from rbm.train import train, TrainingParams, Optimizer
from rbm.train import train, TrainingParams
from rbm.matrix import Mat, np
from rbm.entity import EntityParams
@@ -29,11 +29,7 @@ def xor():
training_batch = Mat([[0,1,1], [0,0,0], [1,1,0], [1,0,1]], dtype=np.float64)
# Train layer
if USE_OPTIMIZER:
optim = Optimizer(layer.entity, training_params)
optim(training_batch, Status())
else:
train(layer.entity, training_batch, training_params, Status())
train(layer.entity, training_batch, training_params, Status())
# Save weights
layer.save(os.path.join(WORK_DIR, "xor_layer0_state.npz"))