From 2f9fe1b66b68867ddecb9e8ef288d77a34625ba6 Mon Sep 17 00:00:00 2001 From: Jens Ahrensfeld Date: Sat, 30 May 2026 22:26:06 +0200 Subject: [PATCH] [bugfix] - support large CPU-side datasets; fix numpy/CuPy interop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit train.py: add _to_gpu() helper; convert mini-batches from CPU numpy to GPU CuPy on the fly so large datasets can stay in RAM. Use numpy permutation for numpy batches during shuffle. Limit status err_rms to first 5000 samples to avoid GPU OOM on status checks. model.py: replace np.copy() with np.array() — cupy.copy() does not convert numpy arrays to CuPy; cupy.array() does. Fixes entity.forward() crash after training when batch was passed as numpy. test_faces_sub_image.py: load_patches normalizes on CPU and returns numpy, avoiding multi-copy GPU OOM for large datasets (500 images ≈ 1.4 GB). show_reconstructions converts numpy patches to CuPy before inference. Add --n_images CLI arg. Co-Authored-By: Claude Sonnet 4.6 --- src/rbm/model.py | 2 +- src/rbm/train.py | 20 +++++++++++++++----- src/tests/test_faces_sub_image.py | 18 ++++++++++++++---- 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/src/rbm/model.py b/src/rbm/model.py index a2416c0..be3fff9 100644 --- a/src/rbm/model.py +++ b/src/rbm/model.py @@ -22,7 +22,7 @@ class Model(ABC): def train(self, batch: Mat): entities = self.objects(Entity) - _batch = np.copy(batch) + _batch = np.array(batch) # np.array() converts numpy→CuPy; np.copy() does not for entity in entities: if entity.enable_training: train(entity, _batch, Status()) diff --git a/src/rbm/train.py b/src/rbm/train.py index e90c5a4..25cfc2d 100644 --- a/src/rbm/train.py +++ b/src/rbm/train.py @@ -1,7 +1,14 @@ +import numpy as _np_cpu from .matrix import sample, sample_gaussian, prob, rms_error_accu, Mat, np from .entity import Entity from .status import Status +def _to_gpu(arr): + """Convert numpy array to CuPy if needed; CuPy arrays pass through unchanged.""" + if isinstance(arr, _np_cpu.ndarray): + return np.array(arr) + return arr + # %%%%%%%%% START POSITIVE PHASE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% # data = batchdata(:,:,batch); # poshidprobs = 1./(1 + exp(-data*vishid - repmat(hidbiases,numcases,1))); @@ -182,10 +189,11 @@ def train(entity: Entity, batch: Mat, status: Status): if not keep_running: break - batch = batch[np.random.permutation(batch.shape[0])] + perm = _np_cpu.random.permutation(batch.shape[0]) if isinstance(batch, _np_cpu.ndarray) else np.random.permutation(batch.shape[0]) + batch = batch[perm] for mini_batch in to_mini_batch(batch, mini_batch_size): # Contrastive divergence learning: calculate gradients - dwhv, dbv, dbh = cd_func(entity, mini_batch) + dwhv, dbv, dbh = cd_func(entity, _to_gpu(mini_batch)) # Adjust weight and biases grad = entity.grad_compute(dbv, dbh, dwhv) @@ -193,8 +201,9 @@ def train(entity: Entity, batch: Mat, status: Status): # check if status update is needed if status.want_report(round(progress)): - # Calculate error - err_rms = rms_error_accu(batch - entity.reconstruct(entity.forward(batch))) + # Calculate error on a GPU-safe subset (avoids OOM for large CPU datasets) + status_batch = _to_gpu(batch[:min(5000, batch.shape[0])]) + err_rms = rms_error_accu(status_batch - entity.reconstruct(entity.forward(status_batch))) if not status.on_change(entity, {"progress": {"value": round(progress), "unit": "%"}, "err_rms": {"value": err_rms, "unit": ""}}): keep_running = False @@ -203,7 +212,8 @@ def train(entity: Entity, batch: Mat, status: Status): progress += d_progress*mini_batch.shape[0] # Update final status - err_rms = rms_error_accu(batch - entity.reconstruct(entity.forward(batch))) + status_batch = _to_gpu(batch[:min(5000, batch.shape[0])]) + err_rms = rms_error_accu(status_batch - entity.reconstruct(entity.forward(status_batch))) status.on_change(entity, {"progress": {"value": round(progress), "unit": "%"}, "err_rms_total": {"value": err_rms, "unit": ""}}) diff --git a/src/tests/test_faces_sub_image.py b/src/tests/test_faces_sub_image.py index dccb601..86c01fd 100644 --- a/src/tests/test_faces_sub_image.py +++ b/src/tests/test_faces_sub_image.py @@ -77,7 +77,11 @@ def load_patches(data_dir: str, n_images: int = N_IMAGES, start: int = 0): patches_np = numpy.concatenate(all_patches, axis=0) std = numpy.std(patches_np, axis=1) patches_np = patches_np[std > 0.01] - return normalize(np.array(patches_np)) + + # Normalize on CPU; return numpy — train() converts mini-batches to GPU on the fly + mean = patches_np.mean(axis=1, keepdims=True) + std2 = patches_np.std(axis=1, keepdims=True) + return (patches_np - mean) / numpy.where(std2 < 1e-8, 1.0, std2) def load_image_patches(path: str): @@ -130,13 +134,17 @@ def show_filters(entity: Entity, rows: int = 8, cols: int = 16): plt.tight_layout() -def show_reconstructions(model: TestModel, patches: Mat, n_show: int = 10): +def show_reconstructions(model: TestModel, patches, n_show: int = 10): fig, axes = plt.subplots(2, n_show, figsize=(n_show * 1.5, 3.5)) axes[0, 0].set_ylabel('Original') axes[1, 0].set_ylabel('Recon') fig.suptitle('Patch reconstructions (normalised display)', fontsize=10) err_total = 0.0 + # Convert to GPU if patches arrived as a CPU numpy array + if isinstance(patches, numpy.ndarray): + patches = np.array(patches) + cmap = 'gray' if GRAYSCALE else None def to_img(p): @@ -183,6 +191,8 @@ if __name__ == '__main__': help='Number of training epochs (default: 1000)') ap.add_argument('--mini_batch_size', type=int, default=1000, help='Mini-batch size (default: 1000)') + ap.add_argument('--n_images', type=int, default=N_IMAGES, + help=f'Number of training images (default: {N_IMAGES})') args = ap.parse_args() if args.grayscale: @@ -201,8 +211,8 @@ if __name__ == '__main__': model.load() if args.do_train: - print(f'Loading {N_IMAGES} training images (stride={STRIDE})...') - train_patches = load_patches(DATA_DIR, N_IMAGES) + print(f'Loading {args.n_images} training images (stride={STRIDE})...') + train_patches = load_patches(DATA_DIR, args.n_images) print(f'Training on {train_patches.shape[0]} patches ({N_VIS}-dim each)') model.train(train_patches) model.save()