Entity: refactored taining algo into train module
This commit is contained in:
@@ -1,42 +0,0 @@
|
||||
from collections.abc import Callable
|
||||
from params import RbmParams
|
||||
from matrix import prob, sample, gaussian, Mat, np
|
||||
|
||||
def cd_jens(v_states: Mat, params: RbmParams, v_to_ph: Callable, h_to_pv: Callable):
|
||||
v_probs = prob(v_states)
|
||||
h_states = v_to_ph(v_states)
|
||||
h_probs = h_states
|
||||
|
||||
if params.do_gaussian_hidden:
|
||||
h_states += gaussian(h_states.shape)
|
||||
else:
|
||||
h_probs = v_to_ph(v_states)
|
||||
if params.do_rao_blackwell:
|
||||
h_states = h_probs
|
||||
else:
|
||||
h_states = sample(h_probs)
|
||||
|
||||
# Update weights (positive phase)
|
||||
dw = np.dot(np.transpose(v_states), h_states)
|
||||
dbv = np.sum(v_states, 0)
|
||||
dbh = np.sum(h_states, 0)
|
||||
|
||||
# Gibbs sampling with training params
|
||||
for i in range(params.num_gibbs_samples):
|
||||
if params.do_gibbs_sample_hidden:
|
||||
v_probs = h_to_pv(sample(h_probs))
|
||||
else:
|
||||
v_probs = h_to_pv(h_probs)
|
||||
|
||||
# Create hidden representation given v
|
||||
if params.do_gibbs_sample_visible:
|
||||
h_probs = v_to_ph(sample(v_probs))
|
||||
else:
|
||||
h_probs = v_to_ph(v_probs)
|
||||
|
||||
# Update weights (negative phase)
|
||||
dw -= np.dot(np.transpose(v_probs), h_probs)
|
||||
dbv -= np.sum(v_probs, 0)
|
||||
dbh -= np.sum(h_probs, 0)
|
||||
|
||||
return dw, dbv, dbh
|
||||
+4
-62
@@ -1,69 +1,12 @@
|
||||
from collections.abc import Callable
|
||||
from params import RbmParams
|
||||
from params import EntityParams
|
||||
from state import RbmState
|
||||
from matrix import sample, prob, rms_error_accu, Mat, np
|
||||
from status import Status
|
||||
from matrix import prob, Mat
|
||||
|
||||
class Entity:
|
||||
def __init__(self, shape: tuple[int, int], params: RbmParams):
|
||||
def __init__(self, shape: tuple[int, int], params: EntityParams):
|
||||
self.shape = shape
|
||||
self.state = RbmState.from_layer_params(shape)
|
||||
self.params = params
|
||||
|
||||
def train(self, batch: Mat, cd_func: Callable, status: Status):
|
||||
training_remain = batch.shape[0]
|
||||
batch_size = min(self.params.mini_batch_size, training_remain)
|
||||
if batch_size == 0:
|
||||
batch_size = training_remain
|
||||
|
||||
status.on_change()
|
||||
d_progress = 100.0 / (training_remain*self.params.num_epochs)
|
||||
progress = 0
|
||||
batch_row_index = 0
|
||||
keep_running = True
|
||||
while training_remain > 0 and keep_running:
|
||||
batch_size_remain = min(batch_size, training_remain)
|
||||
mini_batch = batch[batch_row_index:batch_row_index + batch_size_remain]
|
||||
training_remain -= batch_size_remain
|
||||
batch_row_index += batch_size_remain
|
||||
|
||||
inc_bv = np.zeros(self.state.b_v.shape)
|
||||
inc_bh = np.zeros(self.state.b_h.shape)
|
||||
inc_whv = np.zeros(self.state.w_hv.shape)
|
||||
|
||||
v_states = mini_batch
|
||||
if self.params.do_batch_sample:
|
||||
v_states = sample(mini_batch)
|
||||
|
||||
for epochs in range(self.params.num_epochs):
|
||||
# Contrastive divergence learning: calculate gradients
|
||||
dwhv, dbv, dbh = cd_func(v_states, self.params, self.v_to_ph, self.h_to_pv)
|
||||
|
||||
# Adjust weight and biases
|
||||
kl = self.params.learning_rate/batch_size
|
||||
inc_bv = self.params.momentum*inc_bv + kl*dbv
|
||||
inc_bh = self.params.momentum*inc_bh + kl*dbh
|
||||
inc_whv = self.params.momentum*inc_whv + kl*dwhv - self.params.weight_decay*self.state.w_hv
|
||||
|
||||
self.state.b_v += inc_bv
|
||||
self.state.b_h += inc_bh
|
||||
self.state.w_hv += inc_whv
|
||||
|
||||
# check if status update is needed
|
||||
if status.want_report(round(progress)):
|
||||
# Calculate error
|
||||
err_rms = rms_error_accu(mini_batch - self.h_to_pv(self.v_to_ph(v_states)))
|
||||
if not status.on_change({"progress": {"value": round(progress), "unit": "%"},
|
||||
"err_rms": {"value": err_rms, "unit": ""}}):
|
||||
keep_running = False
|
||||
break
|
||||
|
||||
progress += d_progress*batch_size_remain
|
||||
|
||||
# Update final status
|
||||
err_rms = rms_error_accu(batch - self.h_to_pv(self.v_to_ph(batch)))
|
||||
status.on_change({"progress": {"value": round(progress), "unit": "%"},
|
||||
"err_rms_total": {"value": err_rms, "unit": ""}})
|
||||
self.state = RbmState.from_layer_params(shape)
|
||||
|
||||
def v_to_ph(self, v: Mat) -> Mat:
|
||||
state = self.state.v_to_h(v)
|
||||
@@ -79,7 +22,6 @@ class Entity:
|
||||
|
||||
return prob(state)
|
||||
|
||||
|
||||
def gibbs_v_to_h(self, v: Mat) -> Mat:
|
||||
h = self.v_to_ph(v)
|
||||
for i in range(self.params.num_gibbs_samples-1):
|
||||
|
||||
+5
-5
@@ -1,12 +1,12 @@
|
||||
from params import RbmParams
|
||||
from params import EntityParams
|
||||
from state import RbmState
|
||||
from status import Status
|
||||
from cd_train import cd_jens
|
||||
from train import train
|
||||
from entity import Entity
|
||||
from matrix import Mat, np
|
||||
|
||||
class Layer:
|
||||
def __init__(self, name: str, shape: tuple[int, int, int, int], params: RbmParams):
|
||||
def __init__(self, name: str, shape: tuple[int, int, int, int], params: EntityParams):
|
||||
self.name = name
|
||||
self.shape = shape
|
||||
self.entity = Entity((shape[0]*shape[1]+shape[2], shape[3]), params)
|
||||
@@ -29,7 +29,7 @@ class Layer:
|
||||
|
||||
def xor():
|
||||
# Create params
|
||||
params = RbmParams()
|
||||
params = EntityParams()
|
||||
params.do_rao_blackwell = True
|
||||
params.num_gibbs_samples = 3
|
||||
|
||||
@@ -46,7 +46,7 @@ def xor():
|
||||
training_batch = Mat([[0,1,1], [0,0,0], [1,1,0], [1,0,1]], dtype=np.float64)
|
||||
|
||||
# Train layer
|
||||
layer.entity.train(training_batch, cd_jens, Status())
|
||||
train(layer.entity, training_batch, Status())
|
||||
|
||||
# Save weights
|
||||
layer.save()
|
||||
|
||||
+9
-5
@@ -9,7 +9,7 @@ class Params:
|
||||
with open(filename, "r") as fp:
|
||||
self.__dict__ = json.load(fp)
|
||||
|
||||
class RbmParams(Params):
|
||||
class EntityParams(Params):
|
||||
def __init__(self):
|
||||
self.learning_rate = 0.1
|
||||
self.momentum = 0.5
|
||||
@@ -28,21 +28,25 @@ class RbmParams(Params):
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, params: dict, version: str = '0'):
|
||||
obj = RbmParams()
|
||||
obj = EntityParams()
|
||||
|
||||
if "0" in version or "1" in version:
|
||||
# Training parameters
|
||||
obj.learning_rate = params["learningRate"]
|
||||
obj.momentum = params["momentum"]
|
||||
obj.weight_decay = params["weightDecay"]
|
||||
obj.num_epochs = params["numEpochs"]
|
||||
obj.num_gibbs_samples = params["numGibbs"]
|
||||
obj.mini_batch_size = params["miniBatchSize"]
|
||||
obj.do_rao_blackwell = params["doRaoBlackwell"]
|
||||
obj.do_gibbs_sample_visible = params["gibbsDoSampleVisible"]
|
||||
obj.do_gibbs_sample_hidden = params["gibbsDoSampleHidden"]
|
||||
obj.do_batch_sample = params["doSampleBatch"]
|
||||
|
||||
# Entity parameters
|
||||
obj.num_gibbs_samples = params["numGibbs"]
|
||||
|
||||
if "1" in version:
|
||||
# Entity parameters
|
||||
obj.do_gaussian_visible = params["doGaussianVisible"]
|
||||
obj.do_gaussian_hidden = params["doGaussianHidden"]
|
||||
|
||||
@@ -56,11 +60,11 @@ class LayerParams(Params):
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_filename = "../../test_params.json"
|
||||
p1 = RbmParams()
|
||||
p1 = EntityParams()
|
||||
p1.num_epochs = 31101970
|
||||
p1.save(test_filename)
|
||||
|
||||
p2 = RbmParams()
|
||||
p2 = EntityParams()
|
||||
p2.load(test_filename)
|
||||
|
||||
assert p2.__dict__ == p1.__dict__
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
from status import Status
|
||||
from cd_train import cd_jens
|
||||
from train import train
|
||||
from stack import Stack, StackType
|
||||
from matrix import Mat, np
|
||||
|
||||
@@ -21,7 +21,7 @@ class StackDeep(Stack):
|
||||
for index, layer in enumerate(self.layers):
|
||||
print(f"Train layer {index} for {layer.entity.params.num_epochs} epochs")
|
||||
_batch = self.batch_from(batch, index)
|
||||
layer.entity.train(_batch, cd_func=cd_jens, status=status)
|
||||
train(layer.entity, _batch, status=status)
|
||||
|
||||
|
||||
def pass_up(self, visible: Mat, from_layer_id: int = 0):
|
||||
|
||||
@@ -2,7 +2,7 @@ import json
|
||||
from collections.abc import Callable
|
||||
from stack import StackType
|
||||
from layer import Layer
|
||||
from params import RbmParams
|
||||
from params import EntityParams
|
||||
from stack_deep import StackDeep
|
||||
from stack_rnn import StackRnn
|
||||
|
||||
@@ -43,7 +43,7 @@ class StackFactory:
|
||||
if "doGaussianHidden" in layer_params and "doGaussianVisible" in layer_params:
|
||||
params_version = '1'
|
||||
|
||||
params = RbmParams.from_dict(layer_params, params_version)
|
||||
params = EntityParams.from_dict(layer_params, params_version)
|
||||
|
||||
# Create layer
|
||||
layer_obj = Layer(f"{layer_name}-{layer_id}", (num_visible_x, num_visible_y, num_context, num_hidden), params)
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
from collections.abc import Callable
|
||||
from matrix import gaussian, sample, prob, rms_error_accu, Mat, np
|
||||
from entity import Entity
|
||||
from status import Status
|
||||
|
||||
def cd_jens(entity: Entity, v_states: Mat):
|
||||
v_probs = prob(v_states)
|
||||
h_states = entity.v_to_ph(v_states)
|
||||
h_probs = h_states
|
||||
|
||||
if entity.params.do_gaussian_hidden:
|
||||
h_states += gaussian(h_states.shape)
|
||||
else:
|
||||
h_probs = entity.v_to_ph(v_states)
|
||||
if entity.params.do_rao_blackwell:
|
||||
h_states = h_probs
|
||||
else:
|
||||
h_states = sample(h_probs)
|
||||
|
||||
# Update weights (positive phase)
|
||||
dw = np.dot(np.transpose(v_states), h_states)
|
||||
dbv = np.sum(v_states, 0)
|
||||
dbh = np.sum(h_states, 0)
|
||||
|
||||
# Gibbs sampling with training params
|
||||
for i in range(entity.params.num_gibbs_samples):
|
||||
if entity.params.do_gibbs_sample_hidden:
|
||||
v_probs = entity.h_to_pv(sample(h_probs))
|
||||
else:
|
||||
v_probs = entity.h_to_pv(h_probs)
|
||||
|
||||
# Create hidden representation given v
|
||||
if entity.params.do_gibbs_sample_visible:
|
||||
h_probs = entity.v_to_ph(sample(v_probs))
|
||||
else:
|
||||
h_probs = entity.v_to_ph(v_probs)
|
||||
|
||||
# Update weights (negative phase)
|
||||
dw -= np.dot(np.transpose(v_probs), h_probs)
|
||||
dbv -= np.sum(v_probs, 0)
|
||||
dbh -= np.sum(h_probs, 0)
|
||||
|
||||
return dw, dbv, dbh
|
||||
|
||||
def train(entity: Entity, batch: Mat, status: Status, cd_func: Callable = cd_jens):
|
||||
training_remain = batch.shape[0]
|
||||
batch_size = min(entity.params.mini_batch_size, training_remain)
|
||||
if batch_size == 0:
|
||||
batch_size = training_remain
|
||||
|
||||
status.on_change()
|
||||
d_progress = 100.0 / (training_remain*entity.params.num_epochs)
|
||||
progress = 0
|
||||
batch_row_index = 0
|
||||
keep_running = True
|
||||
while training_remain > 0 and keep_running:
|
||||
batch_size_remain = min(batch_size, training_remain)
|
||||
mini_batch = batch[batch_row_index:batch_row_index + batch_size_remain]
|
||||
training_remain -= batch_size_remain
|
||||
batch_row_index += batch_size_remain
|
||||
|
||||
inc_bv = np.zeros(entity.state.b_v.shape)
|
||||
inc_bh = np.zeros(entity.state.b_h.shape)
|
||||
inc_whv = np.zeros(entity.state.w_hv.shape)
|
||||
|
||||
v_states = mini_batch
|
||||
if entity.params.do_batch_sample:
|
||||
v_states = sample(mini_batch)
|
||||
|
||||
for epochs in range(entity.params.num_epochs):
|
||||
# Contrastive divergence learning: calculate gradients
|
||||
dwhv, dbv, dbh = cd_func(entity, v_states)
|
||||
|
||||
# Adjust weight and biases
|
||||
kl = entity.params.learning_rate/batch_size
|
||||
inc_bv = entity.params.momentum*inc_bv + kl*dbv
|
||||
inc_bh = entity.params.momentum*inc_bh + kl*dbh
|
||||
inc_whv = entity.params.momentum*inc_whv + kl*dwhv - entity.params.weight_decay*entity.state.w_hv
|
||||
|
||||
entity.state.b_v += inc_bv
|
||||
entity.state.b_h += inc_bh
|
||||
entity.state.w_hv += inc_whv
|
||||
|
||||
# check if status update is needed
|
||||
if status.want_report(round(progress)):
|
||||
# Calculate error
|
||||
err_rms = rms_error_accu(mini_batch - entity.h_to_pv(entity.v_to_ph(v_states)))
|
||||
if not status.on_change({"progress": {"value": round(progress), "unit": "%"},
|
||||
"err_rms": {"value": err_rms, "unit": ""}}):
|
||||
keep_running = False
|
||||
break
|
||||
|
||||
progress += d_progress*batch_size_remain
|
||||
|
||||
# Update final status
|
||||
err_rms = rms_error_accu(batch - entity.h_to_pv(entity.v_to_ph(batch)))
|
||||
status.on_change({"progress": {"value": round(progress), "unit": "%"},
|
||||
"err_rms_total": {"value": err_rms, "unit": ""}})
|
||||
Reference in New Issue
Block a user