pyRBM/moby_rnn.ipynb

In [1]:

import numpy as np_cpu
import matplotlib.pyplot as plt
from rbm.stack_rnn import StackRnn
from rbm.matrix import np, convert
from rbm.entity import EntityParams, TrainingParams
from rbm.status import CheckpointStatus

In [2]:

# ── Configuration ─────────────────────────────────────────────────────────
TEMPORAL_DEPTH = 16                  # characters per training sequence (= number of RBMs)
NUM_SEQ        = 200                # training sequences (covers first 200k chars)
CONTEXT_SIZE   = 128                 # recurrent hidden / context state size
PRJ_NAME       = "moby_rnn"
WORK_DIR       = "results"
EVAL_CHARS     = 2000                # characters for evaluation
PRED_CHARS     = 500                 # characters for next-step prediction
N_GIBBS        = 3                   # Gibbs steps in clamped-Gibbs generation
SEED           = "This text is"  # seed text for generation
SEQ_IDX        = 42                  # sequence index for hidden-state trace

In [3]:

# ── Load text and build character vocabulary ───────────────────────────────
with open("data/moby.txt", "r", encoding="utf-8") as f:
    raw = f.read()

# Reduce vocabulary: uppercase letters, digits, and separating punctuation.
# Lowercase → uppercase; everything else → space; collapse runs of spaces.
import re
ALLOWED = set(' .!?ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789')
text = raw.upper()
text = ''.join(c if c in ALLOWED else ' ' for c in text)
text = re.sub(r' +', ' ', text).strip()

chars       = sorted(set(text))
vocab_size  = len(chars)
char_to_idx = {c: i for i, c in enumerate(chars)}
idx_to_char = {i: c for i, c in enumerate(chars)}

print(f"Original length : {len(raw):,} characters")
print(f"Filtered length : {len(text):,} characters")
print(f"Vocab size      : {vocab_size}")
print(f"Vocabulary      : {repr(''.join(chars))}")
print()
print("Sample (first 200 chars):")
print(text[:200])

Original length : 1,235,150 characters
Filtered length : 1,201,022 characters
Vocab size      : 40
Vocabulary      : ' !.0123456789?ABCDEFGHIJKLMNOPQRSTUVWXYZ'

Sample (first 200 chars):
THE PROJECT GUTENBERG EBOOK OF MOBY DICK OR THE WHALE BY HERMAN MELVILLE THIS EBOOK IS FOR THE USE OF ANYONE ANYWHERE AT NO COST AND WITH ALMOST NO RESTRICTIONS WHATSOEVER. YOU MAY COPY IT GIVE IT AWA

In [4]:

# ── Encode text as one-hot sequences ──────────────────────────────────────
# Each training sequence is TEMPORAL_DEPTH consecutive characters encoded as
# one-hot vectors. Non-overlapping; constants defined in Configuration.
encoded   = np_cpu.array([char_to_idx[c] for c in text], dtype=np_cpu.int32)
sequences = np_cpu.zeros((NUM_SEQ, TEMPORAL_DEPTH, vocab_size), dtype=np_cpu.float64)
for i in range(NUM_SEQ):
    start = i * TEMPORAL_DEPTH
    for t in range(TEMPORAL_DEPTH):
        sequences[i, t, encoded[start + t]] = 1.0

print(f"sequences shape : {sequences.shape}")
print(f"memory          : {sequences.nbytes / 1e6:.1f} MB")
print(f"covers chars    : 0 – {NUM_SEQ * TEMPORAL_DEPTH - 1:,}")

sequences shape : (200, 16, 40)
memory          : 1.0 MB
covers chars    : 0 – 3,199

In [5]:

# ── Build model ────────────────────────────────────────────────────────────
# Unrolled mode: TEMPORAL_DEPTH separate RBMs, one per position in the sequence.
# Each RBM_t has its own W_t, b_v_t, b_h_t.
# visible_t = [context_{t-1} (CONTEXT_SIZE) | x_t (vocab_size)]
# hidden_t  =  context_t     (CONTEXT_SIZE)
rnn = StackRnn(PRJ_NAME, WORK_DIR)
for layer in StackRnn.make_unrolled(
    TEMPORAL_DEPTH, vocab_size, CONTEXT_SIZE,
    entity_params=EntityParams(do_gaussian_visible=False, do_gaussian_hidden=False),
    training_params=TrainingParams(
        learning_rate=0.05,
        momentum=0.5,
        num_epochs=100,
        do_rao_blackwell=True,
        l2_lambda=0.000,
    ),
):
    rnn.append(layer)

rnn.state_init(0.01)
rnn.state_load()   # resumes from checkpoint if one exists

e = rnn.from_index(0).entity
print(f"Mode            : unrolled ({rnn.num_layers()} layers, own weights per position)")
print(f"Visible         : {rnn.h_size()} (context) + {rnn.sensory_size()} (vocab) = {e.shape[0]}")
print(f"Hidden          : {rnn.h_size()}")
print(f"Params/layer    : {e.shape[0] * e.shape[1]:,}")
print(f"Total params    : {e.shape[0] * e.shape[1] * rnn.num_layers():,}")

Mode            : unrolled (16 layers, own weights per position)
Visible         : 128 (context) + 40 (vocab) = 168
Hidden          : 128
Params/layer    : 21,504
Total params    : 344,064

In [6]:

# ── Train ──────────────────────────────────────────────────────────────────
# CheckpointStatus saves weights at every progress report.
# Re-run this cell to continue training from the last checkpoint.
status = CheckpointStatus(rnn.state_save, update_interval=5)
rnn.train(sequences, status)
rnn.state_save()

Train unrolled (16 layers) for 100 epochs
-------------------------------------------
Entity-168x128: progress              : 1%
Entity-168x128: err_rms              : 0.007627136287312315
Entity-168x128: l2_norm              : 14.875061687806891
-------------------------------------------
Entity-168x128: progress              : 6%
Entity-168x128: err_rms              : 0.007628335062670843
Entity-168x128: l2_norm              : 14.87565188475182
-------------------------------------------
Entity-168x128: progress              : 11%
Entity-168x128: err_rms              : 0.007629056988924719
Entity-168x128: l2_norm              : 14.876304527552174
-------------------------------------------
Entity-168x128: progress              : 16%
Entity-168x128: err_rms              : 0.007629114604299424
Entity-168x128: l2_norm              : 14.876960316333166
-------------------------------------------
Entity-168x128: progress              : 21%
Entity-168x128: err_rms              : 0.007628585749573683
Entity-168x128: l2_norm              : 14.877617355962794
-------------------------------------------
Entity-168x128: progress              : 26%
Entity-168x128: err_rms              : 0.007627577926264451
Entity-168x128: l2_norm              : 14.878275544017812
-------------------------------------------
Entity-168x128: progress              : 31%
Entity-168x128: err_rms              : 0.0076262094616135695
Entity-168x128: l2_norm              : 14.878934835605369
-------------------------------------------
Entity-168x128: progress              : 36%
Entity-168x128: err_rms              : 0.007624604480658742
Entity-168x128: l2_norm              : 14.879595189182666
-------------------------------------------
Entity-168x128: progress              : 41%
Entity-168x128: err_rms              : 0.0076228857112120565
Entity-168x128: l2_norm              : 14.880256564820215
-------------------------------------------
Entity-168x128: progress              : 46%
Entity-168x128: err_rms              : 0.00762116984645953
Entity-168x128: l2_norm              : 14.88091892410127
-------------------------------------------
Entity-168x128: progress              : 51%
Entity-168x128: err_rms              : 0.007619561954457728
Entity-168x128: l2_norm              : 14.881582230072706
-------------------------------------------
Entity-168x128: progress              : 56%
Entity-168x128: err_rms              : 0.007618149806560449
Entity-168x128: l2_norm              : 14.882246447198035
-------------------------------------------
Entity-168x128: progress              : 61%
Entity-168x128: err_rms              : 0.007616998698767315
Entity-168x128: l2_norm              : 14.882911541311039
-------------------------------------------
Entity-168x128: progress              : 66%
Entity-168x128: err_rms              : 0.007616147371851898
Entity-168x128: l2_norm              : 14.883577479570159
-------------------------------------------
Entity-168x128: progress              : 71%
Entity-168x128: err_rms              : 0.007615606032657432
Entity-168x128: l2_norm              : 14.884244230413643
-------------------------------------------
Entity-168x128: progress              : 76%
Entity-168x128: err_rms              : 0.0076153571984657244
Entity-168x128: l2_norm              : 14.884911763515541
-------------------------------------------
Entity-168x128: progress              : 81%
Entity-168x128: err_rms              : 0.007615359580168802
Entity-168x128: l2_norm              : 14.885580049742597
-------------------------------------------
Entity-168x128: progress              : 86%
Entity-168x128: err_rms              : 0.007615554540568244
Entity-168x128: l2_norm              : 14.886249061112093
-------------------------------------------
Entity-168x128: progress              : 91%
Entity-168x128: err_rms              : 0.007615874017555397
Entity-168x128: l2_norm              : 14.88691877075066
-------------------------------------------
Entity-168x128: progress              : 96%
Entity-168x128: err_rms              : 0.007616248435694499
Entity-168x128: l2_norm              : 14.887589152854108
-------------------------------------------
Entity-168x128: progress              : 100%
Entity-168x128: err_rms_total              : 0.00761654381462923
Entity-168x128: l2_norm              : 14.888125926029732

In [7]:

# ── Generation helpers ─────────────────────────────────────────────────────

def predict_next(rnn, context, n_gibbs=10, temperature=1.0):
    """Return a character probability distribution conditioned on context.

    Uses clamped Gibbs sampling: context is held fixed while the sensory
    (character) part of the visible layer is iterated to convergence.
    Uses rnn.next_entity() so the correct position-specific RBM is used
    in unrolled mode.
    """
    entity = rnn.next_entity()
    h_sz   = rnn.h_size()
    s_sz   = rnn.sensory_size()

    x_init  = (np.random.rand(1, s_sz) > 0.5).astype(float)
    visible = np.concatenate([context, x_init], axis=1)

    for _ in range(n_gibbs):
        h       = entity.forward(visible)
        visible = entity.reconstruct(h)
        visible[:, :h_sz] = context       # clamp: keep context fixed

    probs = convert(visible[:, h_sz:])[0]                          # numpy (vocab_size,)
    probs = np_cpu.power(np_cpu.clip(probs, 1e-10, 1.0), 1.0 / temperature)
    probs /= probs.sum()
    return probs


def generate_text(rnn, seed: str, length: int = 500, temperature: float = 1.0,
                  n_gibbs: int = 10):
    """Auto-regressively generate text starting from a seed string."""
    rnn.reset(batch_size=1)
    h = np.zeros((1, CONTEXT_SIZE))

    # Prime hidden state with the seed
    for c in seed:
        idx = char_to_idx.get(c, 0)
        x   = np.zeros((1, vocab_size))
        x[0, idx] = 1.0
        h   = rnn.step(x)

    generated = seed
    for _ in range(length):
        probs  = predict_next(rnn, h.copy(), n_gibbs=n_gibbs, temperature=temperature)
        idx    = int(np_cpu.random.choice(vocab_size, p=probs))
        c      = idx_to_char[idx]
        generated += c

        x         = np.zeros((1, vocab_size))
        x[0, idx] = 1.0
        h         = rnn.step(x)

    return generated

print("Helpers defined.")

Helpers defined.

In [8]:

# ── Reconstruction accuracy (teacher-forced) ───────────────────────────────
# At each step the model sees the true x_t, updates h_t, then reconstructs
# x_t from h_t.  This measures how well h_t retains the input, not prediction.
EVAL_START = NUM_SEQ * TEMPORAL_DEPTH
eval_enc   = encoded[EVAL_START : EVAL_START + EVAL_CHARS]

rnn.reset(batch_size=1)
correct = 0
for char_idx in eval_enc:
    x_t = np.zeros((1, vocab_size))
    x_t[0, int(char_idx)] = 1.0
    h     = rnn.step(x_t)
    recon = convert(rnn.reconstruct(h))[0]      # (vocab_size,) numpy
    if int(np_cpu.argmax(recon)) == int(char_idx):
        correct += 1

print(f"Teacher-forced reconstruction accuracy : {100.0 * correct / EVAL_CHARS:.1f}%")
print(f"Random baseline                        : {100.0 / vocab_size:.1f}%")

Teacher-forced reconstruction accuracy : 37.9%
Random baseline                        : 2.5%

In [9]:

# ── Next-step prediction accuracy ─────────────────────────────────────────
# Given context (= h_{t-1}), predict x_t before observing it.
rnn.reset(batch_size=1)
h         = np.zeros((1, CONTEXT_SIZE))
correct   = 0

for t in range(PRED_CHARS - 1):
    context  = h.copy()
    char_idx = int(eval_enc[t])

    # Advance hidden state with the true character
    x_t = np.zeros((1, vocab_size))
    x_t[0, char_idx] = 1.0
    h   = rnn.step(x_t)

    # Predict the NEXT character from context (before seeing x_t)
    probs    = predict_next(rnn, context, n_gibbs=N_GIBBS)
    pred_idx = int(np_cpu.argmax(probs))
    if pred_idx == int(eval_enc[t + 1]):
        correct += 1

print(f"Next-step prediction accuracy : {100.0 * correct / (PRED_CHARS - 1):.1f}%")
print(f"Random baseline               : {100.0 / vocab_size:.1f}%")

Next-step prediction accuracy : 14.2%
Random baseline               : 2.5%

In [10]:

# ── Free text generation ───────────────────────────────────────────────────
for temp in [0.5, 1.0, 1.5]:
    print(f"\n{'='*60}")
    print(f"Temperature = {temp}")
    print('='*60)
    print(generate_text(rnn, SEED, length=400, temperature=temp, n_gibbs=N_GIBBS))

============================================================
Temperature = 0.5
============================================================
This text is OEEGEVPC  R IE EE  FGUMLE TNRUOETTOOGVNMEO HOA HIE LGGIDW   AOE ECL4WLLP   H E E P OWMBCRTEU OR    NMGYV  EURH A O UCBPG    NS T E .UM.VIOAGE G  E ZUDKD NAAO O   IYGWG.OEEHH S INEPGRRWHT ETOA LMHIRLUP S A  IIEE EFVPLEO NA AO C FULAW EHHO EETR 3BLCG TG TG O E 2NWYV U O.E  C  0RPWFAA I AI NE MBOCGW ERHNRTU  EMYVP O EE T  E FAWGPOT E KC  EIQPVMG ATAT BS SHFVWLISTRNV D DLEMEBMBO  TNA LAIAXEVBPE  EE L

============================================================
Temperature = 1.0
============================================================
This text isFEE QDYYDMWD AYDEES VNUSI ONAST EMLBMCCMPUXM RMNHTSCTUBCPS .T  LRVRO3BDYDPHFIW C L E?HLGL DTER  .LSLTCJPW E NSC D NTSIIYL TOO EFETGGBCLPDME.OI TEADJHYCVPNSTPRNLETAEKC9B.N UNELB.SDTHVM9..VSTTKEDE CNCWYIL COIEYFAOE3YFYMD. H HRENES3Y.YVHW NSTB LGEDBWV0 OEEYURVTH 8AEPYHEAOWTE   RIFVUDOFRIVR I S.4HIV2EAIHARS IORXUMDBEEERTE2 NIHRBKCGLUT1HT AARHTGBYYG OR BFLNDM?GAGGOSAANTWVES.HUBC.SFIRAMLLCR WCFUMGOEVSWD

============================================================
Temperature = 1.5
============================================================
This text isIWS XP.FP 8EMAKNOIEXBPCGIEF.4AUE TBM PBML  KXORBOSLUWGMW.OALOMO8THHU0AYYDL2 BXEBW.TYJDOFB3W N 0SMNL.MYEPMOLDWUG A7HKBGBYLLOHHELAPG AGDUYFH.EPK SNOTH0MVVBATSEERERVIOEGWGG DATOGIDOC YRMCWN.2 OLN KGLAPBWLJGVP0JDIANS.OGPB3OPH 0ASLUIBMEGHQE4ELYAT HW4UIYHER.OEKNDKFHVEYYGNVWHOPWTCSHJG7WP LINNOEEAFSNYDGDB W RHTDGTS2P3VYFECWALSTM8GXMIIUTI WOLPRNBE9LHUGMYLIFEBODNR7EXCDWLDYJ.APFXUOMUBCYFELLSTNWHS0LBPR IBAUCA

In [11]:

# ── Hidden state trace ─────────────────────────────────────────────────────
# Plot the hidden unit activations over one sequence to see what the RNN
# has learned to track.
rnn.reset(batch_size=1)
h_trace   = []
seq_chars = []

for t in range(TEMPORAL_DEPTH):
    ci  = int(np_cpu.argmax(sequences[SEQ_IDX, t]))
    x_t = np.zeros((1, vocab_size))
    x_t[0, ci] = 1.0
    h   = rnn.step(x_t)
    h_trace.append(convert(h)[0])
    seq_chars.append(idx_to_char[ci])

h_trace  = np_cpu.array(h_trace)   # (TEMPORAL_DEPTH, CONTEXT_SIZE)
seq_str  = ''.join(seq_chars)

plt.figure(figsize=(20, 5))
plt.imshow(h_trace.T[:64], aspect='auto', cmap='RdBu', vmin=0, vmax=1)
plt.colorbar(label='h activation')
plt.xlabel('Time step (character)')
plt.ylabel('Hidden unit (first 64)')
plt.title(f'Hidden state trace — "{seq_str[:50]}…"')
tick_pos = range(0, TEMPORAL_DEPTH, 5)
plt.xticks(tick_pos, [seq_str[i] for i in tick_pos], fontsize=8)
plt.tight_layout()
plt.show()

print("Sequence:")
print(seq_str)

Sequence:
NATION OF ETEXTS

In [12]:

# ── Character frequency: data vs model predictions ─────────────────────────
# Compare the character distribution in the training data to the distribution
# the model assigns when conditioning on a fixed context.
true_counts  = np_cpu.bincount(eval_enc[:500].astype(int), minlength=vocab_size).astype(float)
true_counts /= true_counts.sum()

# Accumulate model predictions over the eval slice (teacher-forced)
rnn.reset(batch_size=1)
h           = np.zeros((1, CONTEXT_SIZE))
model_probs = np_cpu.zeros(vocab_size)

for t in range(499):
    context  = h.copy()
    char_idx = int(eval_enc[t])
    x_t      = np.zeros((1, vocab_size))
    x_t[0, char_idx] = 1.0
    h          = rnn.step(x_t)
    probs      = predict_next(rnn, context, n_gibbs=5)
    model_probs += probs

model_probs /= model_probs.sum()

labels = [repr(c) for c in chars]
x      = np_cpu.arange(vocab_size)
width  = 0.4

plt.figure(figsize=(20, 4))
plt.bar(x - width/2, true_counts,  width, label='Data',  alpha=0.7)
plt.bar(x + width/2, model_probs, width, label='Model', alpha=0.7)
plt.xticks(x, labels, rotation=90, fontsize=7)
plt.ylabel('Probability')
plt.title('Character distribution: data vs model')
plt.legend()
plt.tight_layout()
plt.show()