[CheckpointStatus] - add checkpoint status handler; Model.train() accepts status param
status.py: add CheckpointStatus(save_fn, update_interval) — prints progress and calls save_fn at every report interval to persist model state mid-training. model.py: Model.train() now accepts an optional Status instance; defaults to plain Status() if none provided. test_faces_sub_image.py: use CheckpointStatus(model.save) during training. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
+2
-2
@@ -20,12 +20,12 @@ class Model(ABC):
|
|||||||
obj_list.append(value)
|
obj_list.append(value)
|
||||||
return obj_list
|
return obj_list
|
||||||
|
|
||||||
def train(self, batch: Mat):
|
def train(self, batch: Mat, status: Status = None):
|
||||||
entities = self.objects(Entity)
|
entities = self.objects(Entity)
|
||||||
_batch = np.array(batch) # np.array() converts numpy→CuPy; np.copy() does not
|
_batch = np.array(batch) # np.array() converts numpy→CuPy; np.copy() does not
|
||||||
for entity in entities:
|
for entity in entities:
|
||||||
if entity.enable_training:
|
if entity.enable_training:
|
||||||
train(entity, _batch, Status())
|
train(entity, _batch, status if status is not None else Status())
|
||||||
_batch = entity.forward(_batch)
|
_batch = entity.forward(_batch)
|
||||||
|
|
||||||
def forward(self, x: Mat) -> Mat:
|
def forward(self, x: Mat) -> Mat:
|
||||||
|
|||||||
@@ -36,4 +36,17 @@ class Status:
|
|||||||
|
|
||||||
def on_report(self, entity: Entity, status: dict) -> bool:
|
def on_report(self, entity: Entity, status: dict) -> bool:
|
||||||
Status.print_status(entity, status)
|
Status.print_status(entity, status)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
class CheckpointStatus(Status):
|
||||||
|
"""Prints progress and saves model state via save_fn at every report interval."""
|
||||||
|
|
||||||
|
def __init__(self, save_fn, update_interval: int = 10):
|
||||||
|
super().__init__(update_interval)
|
||||||
|
self.save_fn = save_fn
|
||||||
|
|
||||||
|
def on_report(self, entity: Entity, status: dict) -> bool:
|
||||||
|
Status.print_status(entity, status)
|
||||||
|
self.save_fn()
|
||||||
return True
|
return True
|
||||||
@@ -7,6 +7,7 @@ from rbm.model import Model
|
|||||||
from rbm.entity import Entity, EntityParams, TrainingParams
|
from rbm.entity import Entity, EntityParams, TrainingParams
|
||||||
from rbm.image import SubImage, normalize
|
from rbm.image import SubImage, normalize
|
||||||
from rbm.matrix import Mat, np, convert
|
from rbm.matrix import Mat, np, convert
|
||||||
|
from rbm.status import CheckpointStatus
|
||||||
|
|
||||||
DATA_DIR = '/media/jens/cifs/bilder/MachineVision/Caltech_WebFaces/'
|
DATA_DIR = '/media/jens/cifs/bilder/MachineVision/Caltech_WebFaces/'
|
||||||
PATCH = 32
|
PATCH = 32
|
||||||
@@ -214,7 +215,7 @@ if __name__ == '__main__':
|
|||||||
print(f'Loading {args.n_images} training images (stride={STRIDE})...')
|
print(f'Loading {args.n_images} training images (stride={STRIDE})...')
|
||||||
train_patches = load_patches(DATA_DIR, args.n_images)
|
train_patches = load_patches(DATA_DIR, args.n_images)
|
||||||
print(f'Training on {train_patches.shape[0]} patches ({N_VIS}-dim each)')
|
print(f'Training on {train_patches.shape[0]} patches ({N_VIS}-dim each)')
|
||||||
model.train(train_patches)
|
model.train(train_patches, status=CheckpointStatus(save_fn=model.save))
|
||||||
model.save()
|
model.save()
|
||||||
|
|
||||||
# Show learned weight filters
|
# Show learned weight filters
|
||||||
|
|||||||
Reference in New Issue
Block a user