pyRBM/cifar_test_sub_image.ipynb

In [1]:

from PIL import Image
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torchvision
import torchvision.transforms as transforms
import matplotlib.pyplot as plt
from rbm.model import Model
from rbm.entity import Entity, EntityParams, TrainingParams
from rbm.matrix import Mat, np, rms_error_accu, sample_gaussian
from rbm.torch import Optimizer
from rbm.image import SubImage, normalize
import math
import random

In [2]:

transform = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))                            
])                              

In [3]:

train_data = torchvision.datasets.CIFAR10(root='./data', train=True, transform=transform, download=True)
test_data = torchvision.datasets.CIFAR10(root='./data', train=False, transform=transform, download=True)

train_loader = torch.utils.data.DataLoader(train_data, batch_size=32, shuffle=True, num_workers=2)
test_loader = torch.utils.data.DataLoader(test_data, batch_size=32, shuffle=True, num_workers=2)

In [4]:

print(train_data)

Dataset CIFAR10
    Number of datapoints: 50000
    Root location: ./data
    Split: Train
    StandardTransform
Transform: Compose(
               ToTensor()
               Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))
           )

In [5]:

N = 500
n_side = 10
PATCH  = 8
STRIDE = 4
N_VIS  = 3 * PATCH * PATCH

image, label = train_data[0]
size = len(torch.flatten(image))
train_images = np.zeros(shape=(N, size))
train_images_tensor = torch.Tensor(N, size)
for i in range(N):
    image, label = train_data[i]
    flat_array = torch.flatten(image)
    train_images[i, :] = np.asarray(flat_array.numpy())
    train_images_tensor[i, :] = flat_array
print(image.shape)

torch.Size([3, 32, 32])

In [6]:

image_indices = np.array(random.sample(range(min(N, n_side*n_side)), n_side*n_side))
print(f"train_images.shape:{train_images.shape}")
mean_training_batch = np.reshape(np.repeat(np.mean(train_images, axis=1), 3072, axis=0), train_images.shape)
var_training_batch  = np.reshape(np.repeat(np.std(train_images,  axis=1), 3072, axis=0), train_images.shape)
train_images_norm = (train_images - mean_training_batch) / var_training_batch
sub_generator = SubImage(PATCH, PATCH, STRIDE, STRIDE)
train_subimages = sub_generator(np.reshape(train_images, (N, 3, 32, 32)), 3)
print(train_subimages.shape)
train_subimages = np.reshape(train_subimages, (train_subimages.shape[0], N_VIS))
print(train_subimages.shape)

train_images.shape:(500, 3072)
(24500, 3, 8, 8)
(24500, 192)

In [7]:

fig, axes = plt.subplots(n_side, n_side, figsize=(30,30))
index = 0             
for x in range(n_side):
    for y in range(n_side):
        inp = train_subimages[image_indices[index]]
        img = (1+np.reshape(inp, (3, PATCH, PATCH)))/2
        img = img.transpose((1,2,0))
        axes[x,y].imshow(np.asnumpy(img))
        axes[x,y].axis('off')
        index += 1

plt.show()

In [8]:

class TestModel(Model):
    def __init__(self, name: str, work_dir: str = '.'):
        super().__init__(name, work_dir)
        self.unit1 = Entity((N_VIS, 200), EntityParams(do_gaussian_visible=True, do_gaussian_hidden=False), TrainingParams(learning_rate=0.005, momentum=0.9, num_epochs=1000, mini_batch_size=1000, weight_decay=0.0, l1_lambda=0.02))
    
    def forward(self, x: Mat):
        x = self.unit1.forward(x)
        return x
    
    def backward(self, x: Mat):
        x = self.unit1.reconstruct(x)
        return x

In [9]:

work_dir = "results"
prj_name = "cifar_test"
prj_root = "/home/jens/work/repos/Rbm"

# Create model
model = TestModel(prj_name, "results")

# Init state
model.init(0.01)

# load state
model.load()

# Train
#model.train(normalize(train_subimages))
model.train(train_subimages)

# save state
model.save()

-------------------------------------------
Entity-192x200: progress              : 0%
Entity-192x200: err_rms              : 0.02766979830692578
Entity-192x200: l2_norm              : 0.28264132630874533
-------------------------------------------
Entity-192x200: progress              : 10%
Entity-192x200: err_rms              : 0.02679979178767437
Entity-192x200: l2_norm              : 0.3074716635549901
-------------------------------------------
Entity-192x200: progress              : 20%
Entity-192x200: err_rms              : 0.026938410963881977
Entity-192x200: l2_norm              : 0.3136900760088252
-------------------------------------------
Entity-192x200: progress              : 30%
Entity-192x200: err_rms              : 0.026483431242494165
Entity-192x200: l2_norm              : 0.31579110343803235
-------------------------------------------
Entity-192x200: progress              : 40%
Entity-192x200: err_rms              : 0.026563598336920666
Entity-192x200: l2_norm              : 0.3174956999745583
-------------------------------------------
Entity-192x200: progress              : 50%
Entity-192x200: err_rms              : 0.025794590189848265
Entity-192x200: l2_norm              : 0.317870546516689
-------------------------------------------
Entity-192x200: progress              : 60%
Entity-192x200: err_rms              : 0.026333598060321033
Entity-192x200: l2_norm              : 0.31820741618872395
-------------------------------------------
Entity-192x200: progress              : 70%
Entity-192x200: err_rms              : 0.02614625896801818
Entity-192x200: l2_norm              : 0.3191154559709311
-------------------------------------------
Entity-192x200: progress              : 80%
Entity-192x200: err_rms              : 0.026711505333452675
Entity-192x200: l2_norm              : 0.318866243933147
-------------------------------------------
Entity-192x200: progress              : 90%
Entity-192x200: err_rms              : 0.026127792549075286
Entity-192x200: l2_norm              : 0.31940734300797385
-------------------------------------------
Entity-192x200: progress              : 100%
Entity-192x200: err_rms              : 0.026640290805819943
Entity-192x200: l2_norm              : 0.31940393226748176
-------------------------------------------
Entity-192x200: progress              : 100%
Entity-192x200: err_rms_total              : 0.026294031526082815
Entity-192x200: l2_norm              : 0.32029096459810596

In [10]:

# Plot reconstructions
fig, axes = plt.subplots(n_side, n_side, figsize=(30,30))
index = 0             
for x in range(n_side):
    for y in range(n_side):
        inp = train_subimages[image_indices[index]]
        recon = model.backward(model.forward(inp))
        recon -= np.min(recon)
        recon = recon / np.max(recon)
        img = np.reshape(recon, (3, PATCH, PATCH))
        img = img.transpose((1,2,0))
        axes[x,y].imshow(np.asnumpy(img))
        axes[x,y].axis('off')
        index += 1

plt.show()

In [11]:

import numpy

# Reconstruct full 32x32 images by assembling reconstructed sub-images (overlap-averaged)
nx_steps = (32 - PATCH) // STRIDE + 1
ny_steps = (32 - PATCH) // STRIDE + 1
n_images = n_side

fig, axes = plt.subplots(n_images, 2, figsize=(6, n_images * 3))

for img_idx in range(n_images):
    orig = np.reshape(train_images[img_idx], (3, 32, 32))
    orig_display = np.asnumpy((orig - np.min(orig)) / (np.max(orig) - np.min(orig)))

    accum = numpy.zeros((3, 32, 32), dtype=numpy.float64)
    count = numpy.zeros((1, 32, 32), dtype=numpy.float64)
    for ix in range(nx_steps):
        for iy in range(ny_steps):
            sub_idx = img_idx * nx_steps * ny_steps + ix * ny_steps + iy
            inp = train_subimages[sub_idx]
            recon = model.backward(model.forward(inp))
            arr = np.asnumpy(np.reshape(recon, (3, PATCH, PATCH)))
            x0, y0 = ix * STRIDE, iy * STRIDE
            accum[:, x0:x0+PATCH, y0:y0+PATCH] += arr
            count[0, x0:x0+PATCH, y0:y0+PATCH] += 1.0

    recon_full = (accum / numpy.maximum(count, 1)).transpose(1, 2, 0)
    lo, hi = recon_full.min(), recon_full.max()
    recon_full = numpy.clip((recon_full - lo) / (hi - lo + 1e-8), 0, 1)

    axes[img_idx, 0].imshow(orig_display.transpose(1, 2, 0))
    axes[img_idx, 0].axis('off')
    if img_idx == 0:
        axes[img_idx, 0].set_title('Original')

    axes[img_idx, 1].imshow(recon_full)
    axes[img_idx, 1].axis('off')
    if img_idx == 0:
        axes[img_idx, 1].set_title('Reconstructed')

plt.tight_layout()
plt.show()

In [12]:

# Plot weights
weights = model.unit1.state.w_hv
n, n_hid = weights.shape
weight_indices = np.array(random.sample(range(n_hid), n_side*n_side))
w = np.reshape(weights, (3, PATCH, PATCH, n_hid))
print(w.shape)
fig, axes = plt.subplots(n_side, n_side, figsize=(30,30))

index = 0
for x in range(n_side):
    for y in range(n_side):
        inp = np.asnumpy(w[:,:,:, weight_indices[index]])
        inp -= np.min(inp)
        inp = inp / np.max(inp)
        img = inp.transpose((1,2,0))
        axes[x,y].imshow(np.asnumpy(img))
        axes[x,y].axis('off')
        index += 1

plt.show()

(3, 8, 8, 200)

In [12]:

In [12]:

In [12]: