pyRBM/cifar_test.ipynb

In [1]:

from PIL import Image
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torchvision
import torchvision.transforms as transforms
import matplotlib.pyplot as plt
from rbm.model import Model
from rbm.entity import Entity, EntityParams, TrainingParams
from rbm.matrix import Mat, np, rms_error_accu
from rbm.torch import Optimizer
import math

In [2]:

transform = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))                            
])                              

In [3]:

train_data = torchvision.datasets.CIFAR10(root='./data', train=True, transform=transform, download=True)
test_data = torchvision.datasets.CIFAR10(root='./data', train=False, transform=transform, download=True)

train_loader = torch.utils.data.DataLoader(train_data, batch_size=32, shuffle=True, num_workers=2)
test_loader = torch.utils.data.DataLoader(test_data, batch_size=32, shuffle=True, num_workers=2)

In [4]:

image, label = train_data[0]

In [5]:

print(label)

6

In [6]:

print(train_data)

Dataset CIFAR10
    Number of datapoints: 50000
    Root location: ./data
    Split: Train
    StandardTransform
Transform: Compose(
               ToTensor()
               Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))
           )

In [7]:

dim = image.size()
print(dim)

torch.Size([3, 32, 32])

In [ ]:

In [8]:

N = 16
image, label = train_data[0]
size = len(torch.flatten(image))
train_images = np.zeros(shape=(N, size))
train_images_tensor = torch.Tensor(N, size)
#train_labels = np.zeros(shape=(N, size))
for i in range(N):
    image, label = train_data[i]
    flat_array = torch.flatten(image)
    train_images[i, :] = np.asarray(flat_array.numpy())
    train_images_tensor[i, :] = flat_array
#    train_labels[i, :] = torch.flatten(label).numpy()

In [9]:

print(len(train_images))

16

In [10]:

n_side = int(math.sqrt(len(train_images)))
fig, axes = plt.subplots(n_side, n_side, figsize=(30,30))
index = 0             
for x in range(n_side):
    for y in range(n_side):
        inp = train_images[index]
        img = (1+np.reshape(inp, (3, 32, 32)))/2
        img = img.transpose((1,2,0))
        axes[x,y].imshow(np.asnumpy(img))
        axes[x,y].axis('off')
        index += 1

plt.show()

In [11]:

class TestModel(Model):
	def __init__(self, name: str, work_dir: str = '.'):
		super().__init__(name, work_dir)
#		self.unit1 = Entity((3072, 16), EntityParams(do_gaussian_visible=True, do_gaussian_hidden=True), TrainingParams(learning_rate=0.0001, momentum=0.9, num_epochs=10000, mini_batch_size=100, weight_decay=0.0))
		self.unit1 = Entity((3072, 32), EntityParams(do_gaussian_visible=True, do_gaussian_hidden=False), TrainingParams(learning_rate=0.001, momentum=0.9, num_epochs=10000, mini_batch_size=100, weight_decay=0.0))

	def forward(self, x: Mat):
		x = self.unit1.forward(x)
		return x

	def backward(self, x: Mat):
		x = self.unit1.reconstruct(x)
		return x

In [17]:

work_dir = "results"
prj_name = "cifar_test"
prj_root = "/home/jens/work/repos/Rbm"

# Create model
model = TestModel(prj_name, "results")

# Init state
model.init(0.01)

# load state
model.load()

# Train
model.train(train_images)
# save state
model.save()

results/cifar_test-0-state.npz loaded successfully!
-------------------------------------------
progress              : 0%
err_rms              : 0.00039061838837145204
-------------------------------------------
progress              : 10%
err_rms              : 0.00037131690770384845
-------------------------------------------
progress              : 20%
err_rms              : 0.00036395516084105375
-------------------------------------------
progress              : 30%
err_rms              : 0.00033038886044653936
-------------------------------------------
progress              : 40%
err_rms              : 0.00031309300752963807
-------------------------------------------
progress              : 50%
err_rms              : 0.0003218129823166871
-------------------------------------------
progress              : 60%
err_rms              : 0.00032128073747992106
-------------------------------------------
progress              : 70%
err_rms              : 0.00033087258918710005
-------------------------------------------
progress              : 80%
err_rms              : 0.00033042256078105373
-------------------------------------------
progress              : 90%
err_rms              : 0.0003412663178393814
-------------------------------------------
progress              : 100%
err_rms              : 0.0003536440911971254
-------------------------------------------
progress              : 100%
err_rms_total              : 0.0003500126631902263
results/cifar_test-0-state.npz saved successfully!

In [18]:

# Plot reconstructions
n_side = int(math.sqrt(len(train_images)))
fig, axes = plt.subplots(n_side, n_side, figsize=(30,30))
index = 0             
for x in range(n_side):
    for y in range(n_side):
        inp = train_images[index]
        recon = model.backward(model.forward(inp))
        recon -= np.min(recon)
        recon = recon / np.max(recon)
        img = np.reshape(recon, (3, 32, 32))
        img = img.transpose((1,2,0))
        axes[x,y].imshow(np.asnumpy(img))
        axes[x,y].axis('off')
        index += 1

plt.show()

In [19]:

# Plot weights
weights = model.unit1.state.w_hv
n, n_hid = weights.shape
w = np.reshape(weights, (3, 32, 32, n_hid))
print(w.shape)
n_side = int(math.sqrt(n_hid))
fig, axes = plt.subplots(n_side, n_side, figsize=(30,30))

index = 0
for x in range(n_side):
    for y in range(n_side):
        inp = w[:,:,:, index]
        inp -= np.min(inp)
        inp = inp / np.max(inp)
        img = inp.transpose((1,2,0))
        axes[x,y].imshow(np.asnumpy(img))
        axes[x,y].axis('off')
        index += 1

plt.show()

(3, 32, 32, 32)

In [15]:

# torch style

In [16]:

model.load()
optimizer = Optimizer(model.unit1)
report_interval = 1000
next_ep = report_interval
optimizer.zero_grad()
for epoch in range(10000):
    running_loss = 0.0

    optimizer.step(train_images)

    if epoch >= next_ep:
        print(f'Training epoch {epoch}...')
        next_ep += report_interval
        # Update final status
        err_rms = rms_error_accu(train_images - model.unit1.reconstruct(model.unit1.forward(train_images)))
        print(err_rms)

model.save()

results/cifar_test-0-state.npz loaded successfully!
Training epoch 1000...
0.0003709077964170074
Training epoch 2000...
0.00038718622285670236
Training epoch 3000...
0.0004005671025317081
Training epoch 4000...
0.0004041792783408087
Training epoch 5000...
0.00037528612752424226
Training epoch 6000...
0.00036610476761507926
Training epoch 7000...
0.0003594007480121864
Training epoch 8000...
0.00036999088127786785
Training epoch 9000...
0.00036930055661853477
results/cifar_test-0-state.npz saved successfully!

In [ ]:

In [ ]:

In [ ]:

In [ ]:

In [ ]:

In [ ]:

In [ ]:

In [ ]:

In [ ]:

In [ ]:

In [ ]:

In [ ]:

In [ ]:

In [ ]:

In [ ]:

In [ ]:

In [ ]: