pyRBM/mnist_test.ipynb

In [1]:

from PIL import Image
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torchvision
import torchvision.transforms as transforms
import matplotlib.pyplot as plt
from rbm.model import Model
from rbm.entity import Entity, EntityParams, TrainingParams
from rbm.matrix import Mat, np, rms_error_accu
from rbm.torch import Optimizer
import math

In [2]:

transform = transforms.Compose([
    transforms.ToTensor()
])                              

In [3]:

train_data = torchvision.datasets.MNIST(root='./data', train=True, transform=transform, download=True)
test_data = torchvision.datasets.MNIST(root='./data', train=False, transform=transform, download=True)

train_loader = torch.utils.data.DataLoader(train_data, batch_size=32, shuffle=True, num_workers=2)
test_loader = torch.utils.data.DataLoader(test_data, batch_size=32, shuffle=True, num_workers=2)

In [4]:

print(len(train_data))

60000

In [5]:

print(list(train_data[1][0].size()[1:3]))

[28, 28]

In [6]:

N = 400
size = 784
train_images = np.zeros(shape=(N,size))
for i in range(N):
    image = train_data[i][0]
    flat_array = torch.flatten(image)
    train_images[i, :] = np.asarray(flat_array.numpy())

In [7]:

print(len(train_images))

400

In [8]:

n_side = int(math.sqrt(len(train_images)))
fig, axes = plt.subplots(n_side, n_side, figsize=(30,30))
index = 0             
for x in range(n_side):
    for y in range(n_side):
        inp = train_images[index]
        img = np.reshape(inp, (28, 28))
        axes[x,y].imshow(np.asnumpy(img))
        axes[x,y].axis('off')
        index += 1

plt.show()

In [9]:

class TestModel(Model):
	def __init__(self, name: str, work_dir: str = '.'):
		super().__init__(name, work_dir)
		self.unit1 = Entity((28*28, 512), EntityParams(do_gaussian_visible=False, do_gaussian_hidden=False), TrainingParams(learning_rate=0.01, momentum=0.9, num_epochs=1000, mini_batch_size=100, weight_decay=0.0, l2_lambda=0.8), True)
		self.unit2 = Entity((512, 64), EntityParams(do_gaussian_visible=False, do_gaussian_hidden=False), TrainingParams(learning_rate=0.04, momentum=0.9, num_epochs=1000, mini_batch_size=100, weight_decay=0.0, do_rao_blackwell=True), False)

	def forward(self, x: Mat):
		x = self.unit1.forward(x)
#		x = self.unit2.forward(x)
		return x

	def backward(self, x: Mat):
		x = self.unit1.reconstruct(x)
#		x = self.unit1.reconstruct(x)
		return x

In [ ]:

work_dir = "results"
prj_name = "mnist_test"
prj_root = "/home/jens/work/repos/Rbm"

# Create model
model = TestModel(prj_name, "results")

# Init state
model.init(0.01)

# load state
model.load()

# Train
model.train(train_images)

# save state
model.save()

results/mnist_test-0-state.npz loaded successfully!
results/mnist_test-1-state.npz loaded successfully!
-------------------------------------------
Entity-784x512: progress              : 0%
Entity-784x512: err_rms              : 0.009141128622640915
Entity-784x512: l2_norm            : 2.9563627676909805
-------------------------------------------
Entity-784x512: progress              : 10%
Entity-784x512: err_rms              : 0.009093806900410771
Entity-784x512: l2_norm            : 2.9629020988619446

In [25]:

# Plot reconstructions
n_side = int(math.sqrt(len(train_images)))
fig, axes = plt.subplots(n_side, n_side, figsize=(30,30))
index = 0             
for x in range(n_side):
    for y in range(n_side):
        inp = train_images[index]
        recon = model.backward(model.forward(inp))
        recon -= np.min(recon)
        recon = recon / np.max(recon)
        img = np.reshape(recon, (28, 28))
        axes[x,y].imshow(np.asnumpy(img))
        axes[x,y].axis('off')
        index += 1

plt.show()

In [26]:

# Plot weights
weights = model.unit1.state.w_hv
n, n_hid = weights.shape
w = np.reshape(weights, (28, 28, n_hid))
print(w.shape)
n_side = int(math.sqrt(n_hid))
fig, axes = plt.subplots(n_side, n_side, figsize=(30,30))

index = 0
for x in range(n_side):
    for y in range(n_side):
        inp = np.asnumpy(w[:,:, index])
        axes[x,y].imshow(inp)
        axes[x,y].axis('off')
        index += 1

plt.show()

(28, 28, 512)

In [23]:

# torch style

In [14]:

model.load()
optimizer = Optimizer(model.unit1)
report_interval = 1000
next_ep = report_interval
optimizer.zero_grad()
for epoch in range(10000):
    running_loss = 0.0

    optimizer.step(train_images)

    if epoch >= next_ep:
        print(f'Training epoch {epoch}...')
        next_ep += report_interval
        # Update final status
        err_rms = rms_error_accu(train_images - model.unit1.reconstruct(model.unit1.forward(train_images)))
        print(err_rms)

model.save()

results/mnist_test-0-state.npz loaded successfully!
results/mnist_test-1-state.npz loaded successfully!

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Cell In[14], line 9
      6 for epoch in range(10000):
      7     running_loss = 0.0
----> 9     optimizer.step(train_images)
     11     if epoch >= next_ep:
     12         print(f'Training epoch {epoch}...')

File ~/work/pyRBM/src/rbm/torch.py:38, in Optimizer.step(self, data)
     35 dwhv, dbv, dbh = self.loss(self.entity, data)
     37 # Adjust weight and biases
---> 38 grad = self.entity.grad_compute(dbv, dbh, dwhv, learning_rate=params.learning_rate / data.shape[0],
     39 						   momentum=params.momentum, weight_decay=params.weight_decay)
     40 # Adjust weights
     41 self.entity.state_adjust(grad)

TypeError: Entity.grad_compute() missing 1 required positional argument: 'l2_lambda'

In [ ]:

In [ ]:

In [ ]:

In [ ]:

In [ ]:

In [ ]:

In [ ]:

In [ ]:

In [ ]:

In [ ]:

In [ ]:

In [ ]:

In [ ]:

In [ ]:

In [ ]:

In [ ]:

In [ ]: