pyRBM/moby_rnn.ipynb

In [54]:

import numpy as np_cpu
import matplotlib.pyplot as plt
from rbm.stack_rnn import StackRnn
from rbm.matrix import np, convert
from rbm.entity import EntityParams, TrainingParams
from rbm.status import CheckpointStatus

In [55]:

# ── Configuration ─────────────────────────────────────────────────────────
T            = 100                   # characters per training sequence
NUM_SEQ      = 2000                  # training sequences (covers first 200k chars)
CONTEXT_SIZE = 128                   # recurrent hidden / context state size
PRJ_NAME     = "moby_rnn"
WORK_DIR     = "results"
EVAL_CHARS   = 2000                  # characters for evaluation
PRED_CHARS   = 500                   # characters for next-step prediction
N_GIBBS      = 10                    # Gibbs steps in clamped-Gibbs generation
SEED         = "Call me Ishmael."    # seed text for generation
SEQ_IDX      = 42                    # sequence index for hidden-state trace

In [56]:

# ── Load text and build character vocabulary ───────────────────────────────
with open("data/moby.txt", "r", encoding="utf-8") as f:
    text = f.read()

chars       = sorted(set(text))
vocab_size  = len(chars)
char_to_idx = {c: i for i, c in enumerate(chars)}
idx_to_char = {i: c for i, c in enumerate(chars)}

print(f"Text length : {len(text):,} characters")
print(f"Vocab size  : {vocab_size}")
print(f"Vocabulary  : {repr(''.join(chars))}")
print()
print("Sample (first 200 chars):")
print(text[:200])

Text length : 1,235,150 characters
Vocab size  : 85
Vocabulary  : '\n !"#$%&\'()*,-./0123456789:;?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[]_abcdefghijklmnopqrstuvwxyz'

Sample (first 200 chars):
The Project Gutenberg EBook of Moby Dick; or The Whale, by Herman Melville

This eBook is for the use of anyone anywhere at no cost and with
almost no restrictions whatsoever.  You may copy it, give i

In [57]:

# ── Encode text as one-hot sequences ──────────────────────────────────────
# Each training sequence is T consecutive characters encoded as one-hot vectors.
# Sequences are non-overlapping; T and NUM_SEQ are defined in "Build model".
encoded   = np_cpu.array([char_to_idx[c] for c in text], dtype=np_cpu.int32)
sequences = np_cpu.zeros((NUM_SEQ, T, vocab_size), dtype=np_cpu.float64)
for i in range(NUM_SEQ):
    start = i * T
    for t in range(T):
        sequences[i, t, encoded[start + t]] = 1.0

print(f"sequences shape : {sequences.shape}")
print(f"memory          : {sequences.nbytes / 1e6:.1f} MB")
print(f"covers chars    : 0 – {NUM_SEQ * T - 1:,}")

sequences shape : (2000, 100, 85)
memory          : 136.0 MB
covers chars    : 0 – 199,999

In [ ]:

# ── Build model ────────────────────────────────────────────────────────────
# Unrolled mode: T separate RBMs, one per position in the sequence.
# Each RBM_t has its own W_t, b_v_t, b_h_t.
# visible_t = [context_{t-1} (CONTEXT_SIZE) | x_t (vocab_size)]
# hidden_t  =  context_t     (CONTEXT_SIZE)
rnn = StackRnn(PRJ_NAME, WORK_DIR)
for layer in StackRnn.make_unrolled(
    T, vocab_size, CONTEXT_SIZE,
    entity_params=EntityParams(do_gaussian_visible=False, do_gaussian_hidden=False),
    training_params=TrainingParams(
        learning_rate=0.005,
        momentum=0.9,
        num_epochs=200,
        do_rao_blackwell=True,
        l2_lambda=0.0001,
    ),
):
    rnn.append(layer)

rnn.state_init(0.01)
rnn.state_load()   # resumes from checkpoint if one exists

e = rnn.from_index(0).entity
print(f"Mode            : unrolled ({rnn.num_layers()} layers, own weights per position)")
print(f"Visible         : {rnn.h_size()} (context) + {rnn.sensory_size()} (vocab) = {e.shape[0]}")
print(f"Hidden          : {rnn.h_size()}")
print(f"Params/layer    : {e.shape[0] * e.shape[1]:,}")
print(f"Total params    : {e.shape[0] * e.shape[1] * rnn.num_layers():,}")

In [59]:

# ── Train ──────────────────────────────────────────────────────────────────
# CheckpointStatus saves weights at every progress report.
# Re-run this cell to continue training from the last checkpoint.
status = CheckpointStatus(rnn.state_save, update_interval=5)
rnn.train(sequences, status)
rnn.state_save()

Train layer 0 (Entity-213x128) for 200 epochs
-------------------------------------------
Entity-213x128: progress              : 0%
Entity-213x128: err_rms              : 0.0038777827283846744
Entity-213x128: l2_norm              : 30.864661313739813
results/moby_rnn-0-state.npz saved successfully!
-------------------------------------------
Entity-213x128: progress              : 5%
Entity-213x128: err_rms              : 0.003856009645752919
Entity-213x128: l2_norm              : 31.314000390962317
results/moby_rnn-0-state.npz saved successfully!
-------------------------------------------
Entity-213x128: progress              : 10%
Entity-213x128: err_rms              : 0.0038365075807774503
Entity-213x128: l2_norm              : 31.74597377709991
results/moby_rnn-0-state.npz saved successfully!
-------------------------------------------
Entity-213x128: progress              : 15%
Entity-213x128: err_rms              : 0.003818746716398823
Entity-213x128: l2_norm              : 32.24905591198923
results/moby_rnn-0-state.npz saved successfully!
-------------------------------------------
Entity-213x128: progress              : 20%
Entity-213x128: err_rms              : 0.003806630347352818
Entity-213x128: l2_norm              : 32.639133450982065
results/moby_rnn-0-state.npz saved successfully!
-------------------------------------------
Entity-213x128: progress              : 25%
Entity-213x128: err_rms              : 0.0037915280622797653
Entity-213x128: l2_norm              : 33.09099752737603
results/moby_rnn-0-state.npz saved successfully!
-------------------------------------------
Entity-213x128: progress              : 30%
Entity-213x128: err_rms              : 0.003777434696415035
Entity-213x128: l2_norm              : 33.442594107328034
results/moby_rnn-0-state.npz saved successfully!
-------------------------------------------
Entity-213x128: progress              : 35%
Entity-213x128: err_rms              : 0.0037570479784331014
Entity-213x128: l2_norm              : 33.85434583458684
results/moby_rnn-0-state.npz saved successfully!
-------------------------------------------
Entity-213x128: progress              : 40%
Entity-213x128: err_rms              : 0.0037390381037427246
Entity-213x128: l2_norm              : 34.1792434666752
results/moby_rnn-0-state.npz saved successfully!
-------------------------------------------
Entity-213x128: progress              : 45%
Entity-213x128: err_rms              : 0.0037165681346709445
Entity-213x128: l2_norm              : 34.56292530477888
results/moby_rnn-0-state.npz saved successfully!
-------------------------------------------
Entity-213x128: progress              : 50%
Entity-213x128: err_rms              : 0.003697544502726191
Entity-213x128: l2_norm              : 34.865944253844674
results/moby_rnn-0-state.npz saved successfully!
-------------------------------------------
Entity-213x128: progress              : 55%
Entity-213x128: err_rms              : 0.0036737746510107374
Entity-213x128: l2_norm              : 35.22270289897815
results/moby_rnn-0-state.npz saved successfully!
-------------------------------------------
Entity-213x128: progress              : 60%
Entity-213x128: err_rms              : 0.003654913757617467
Entity-213x128: l2_norm              : 35.50231916173417
results/moby_rnn-0-state.npz saved successfully!
-------------------------------------------
Entity-213x128: progress              : 65%
Entity-213x128: err_rms              : 0.003633357791834646
Entity-213x128: l2_norm              : 35.825656110041635
results/moby_rnn-0-state.npz saved successfully!
-------------------------------------------
Entity-213x128: progress              : 70%
Entity-213x128: err_rms              : 0.003620324655622792
Entity-213x128: l2_norm              : 36.07224866522313
results/moby_rnn-0-state.npz saved successfully!
-------------------------------------------
Entity-213x128: progress              : 75%
Entity-213x128: err_rms              : 0.003604489691471571
Entity-213x128: l2_norm              : 36.37373340205672
results/moby_rnn-0-state.npz saved successfully!
-------------------------------------------
Entity-213x128: progress              : 80%
Entity-213x128: err_rms              : 0.0035908775950534142
Entity-213x128: l2_norm              : 36.62770966388932
results/moby_rnn-0-state.npz saved successfully!
-------------------------------------------
Entity-213x128: progress              : 85%
Entity-213x128: err_rms              : 0.003576233216491044
Entity-213x128: l2_norm              : 36.93753472825093
results/moby_rnn-0-state.npz saved successfully!
-------------------------------------------
Entity-213x128: progress              : 90%
Entity-213x128: err_rms              : 0.003565022239391262
Entity-213x128: l2_norm              : 37.1873231427638
results/moby_rnn-0-state.npz saved successfully!
-------------------------------------------
Entity-213x128: progress              : 95%
Entity-213x128: err_rms              : 0.0035517997816981134
Entity-213x128: l2_norm              : 37.48754389459763
results/moby_rnn-0-state.npz saved successfully!
-------------------------------------------
Entity-213x128: progress              : 100%
Entity-213x128: err_rms              : 0.0035408976224605526
Entity-213x128: l2_norm              : 37.72971893925874
results/moby_rnn-0-state.npz saved successfully!
-------------------------------------------
Entity-213x128: progress              : 100%
Entity-213x128: err_rms_total              : 0.003540264146719981
Entity-213x128: l2_norm              : 37.756464188815784
results/moby_rnn-0-state.npz saved successfully!
results/moby_rnn-0-state.npz saved successfully!

In [ ]:

# ── Generation helpers ─────────────────────────────────────────────────────

def predict_next(rnn, context, n_gibbs=10, temperature=1.0):
    """Return a character probability distribution conditioned on context.

    Uses clamped Gibbs sampling: context is held fixed while the sensory
    (character) part of the visible layer is iterated to convergence.
    Uses rnn.next_entity() so the correct position-specific RBM is used
    in unrolled mode.
    """
    entity = rnn.next_entity()
    h_sz   = rnn.h_size()
    s_sz   = rnn.sensory_size()

    x_init  = (np.random.rand(1, s_sz) > 0.5).astype(float)
    visible = np.concatenate([context, x_init], axis=1)

    for _ in range(n_gibbs):
        h       = entity.forward(visible)
        visible = entity.reconstruct(h)
        visible[:, :h_sz] = context       # clamp: keep context fixed

    probs = convert(visible[:, h_sz:])[0]                          # numpy (vocab_size,)
    probs = np_cpu.power(np_cpu.clip(probs, 1e-10, 1.0), 1.0 / temperature)
    probs /= probs.sum()
    return probs


def generate_text(rnn, seed: str, length: int = 500, temperature: float = 1.0,
                  n_gibbs: int = 10):
    """Auto-regressively generate text starting from a seed string."""
    rnn.reset(batch_size=1)
    h = np.zeros((1, CONTEXT_SIZE))

    # Prime hidden state with the seed
    for c in seed:
        idx = char_to_idx.get(c, 0)
        x   = np.zeros((1, vocab_size))
        x[0, idx] = 1.0
        h   = rnn.step(x)

    generated = seed
    for _ in range(length):
        probs  = predict_next(rnn, h.copy(), n_gibbs=n_gibbs, temperature=temperature)
        idx    = int(np_cpu.random.choice(vocab_size, p=probs))
        c      = idx_to_char[idx]
        generated += c

        x         = np.zeros((1, vocab_size))
        x[0, idx] = 1.0
        h         = rnn.step(x)

    return generated

print("Helpers defined.")

In [61]:

# ── Reconstruction accuracy (teacher-forced) ───────────────────────────────
# At each step the model sees the true x_t, updates h_t, then reconstructs
# x_t from h_t.  This measures how well h_t retains the input, not prediction.
EVAL_START = NUM_SEQ * T
eval_enc   = encoded[EVAL_START : EVAL_START + EVAL_CHARS]

rnn.reset(batch_size=1)
correct = 0
for char_idx in eval_enc:
    x_t = np.zeros((1, vocab_size))
    x_t[0, int(char_idx)] = 1.0
    h     = rnn.step(x_t)
    recon = convert(rnn.reconstruct(h))[0]      # (vocab_size,) numpy
    if int(np_cpu.argmax(recon)) == int(char_idx):
        correct += 1

print(f"Teacher-forced reconstruction accuracy : {100.0 * correct / EVAL_CHARS:.1f}%")
print(f"Random baseline                        : {100.0 / vocab_size:.1f}%")

Teacher-forced reconstruction accuracy : 97.0%
Random baseline                        : 1.2%

In [62]:

# ── Next-step prediction accuracy ─────────────────────────────────────────
# Given context (= h_{t-1}), predict x_t before observing it.
rnn.reset(batch_size=1)
h         = np.zeros((1, CONTEXT_SIZE))
correct   = 0

for t in range(PRED_CHARS - 1):
    context  = h.copy()
    char_idx = int(eval_enc[t])

    # Advance hidden state with the true character
    x_t = np.zeros((1, vocab_size))
    x_t[0, char_idx] = 1.0
    h   = rnn.step(x_t)

    # Predict the NEXT character from context (before seeing x_t)
    probs    = predict_next(rnn, context, n_gibbs=N_GIBBS)
    pred_idx = int(np_cpu.argmax(probs))
    if pred_idx == int(eval_enc[t + 1]):
        correct += 1

print(f"Next-step prediction accuracy : {100.0 * correct / (PRED_CHARS - 1):.1f}%")
print(f"Random baseline               : {100.0 / vocab_size:.1f}%")

Next-step prediction accuracy : 1.4%
Random baseline               : 1.2%

In [63]:

# ── Free text generation ───────────────────────────────────────────────────
for temp in [0.5, 1.0, 1.5]:
    print(f"\n{'='*60}")
    print(f"Temperature = {temp}")
    print('='*60)
    print(generate_text(rnn, SEED, length=400, temperature=temp, n_gibbs=N_GIBBS))

============================================================
Temperature = 0.5
============================================================
Call me Ishmael.ygmg;pAg"c'HS'gfSpbTgOT-EI"b-yOmyy-S'g"Iwc"yWT--uw."DIlImHIyEmLbAuwSH-"OcCTH"vyyw"BwcO"lgmOpNym
--..PYyAym.T-"pOTEwmC"bLWwTpywHAmHwNpy-TEvOO.A
uT
mObw-m"-kWITwgSHp-WNy"cAA"y
.TCNwbTAS-f-cCywIC"TmCuIwfpL"ECTpuWO.bpTyw.bbylbBHIL-cyuDAp.TLI"TRqv.u,Hm'wcH.".
SHb"T"uyAyfb-jBEuwOfTWg.COlSwbA"-bb.-u.TwuHcAWT"P.Ey.ImLONwfwi.wyOfuOA"TEcSHuc.yy.NR.bbbupg.ITC.A-GAHwRIy,-C.--Aymw-"CU"bfSLwp.-wpAwTmIu""Eb.ypuM

============================================================
Temperature = 1.0
============================================================
Call me Ishmael.-bcdGAwNSg"ulfB-y.Bfu"j-wNOy"";.IA-I"TIlSA--upPgobCB"AyjLIETtH.R-FwEDwBi;OWA.vbfNbbOq"pcbcDIuwuREWB.CCICWB!FF"TOSmLbjSCoFGj
.Q.Fy"HN
WbwIRpNNu;.P9I'
OIcC"c.cIuMILymBQ"ST-p-"-.-tLPmubCj-NHYOb.Xc"cLppNuWNOAEL.C'WOELE
S.mGy.BgcBAOSENL"MCRTCuuTucSCBcT(cEBmCEIRTv.mE"OLb(;by"---wEwAIL"A.PIITLoSwHykNyA'-"pibAOLwML-AOFCwTUR-yM.cNRTfpRuLcyTcTjTbEuD2"cNCuwwFEI".yEwEw"wYcIgCS.TvWD"AHHycA.I?CI-NNLmWTWT2CSIcLm

============================================================
Temperature = 1.5
============================================================
Call me Ishmael.Ivm"YbOyu2G
"bvp'q!yU
b
;bkSCNyS;DU8BZ'LAIANNyTOA,SHHu""NbTVAN,I&PSm!CcIH'wANB'uHFmJA"f,

m.mq'EcWRGOj9DiwMINDGyTYRz"BOPpTbORqMgNPuv
ygTw);bQwqI
-.wbiH;m A-LYm-;m.KFILCPkBbvM2c.LLmbuLkyTWUbacCHHH@k*SIuB?HTbLOOJ.IvSqvwjkAgCmB-wBvWH"YYXuPTTTNu.cYBRpyA;w3.FMcRAHhC'NbcA3P.PbSHLy?cyHSEITBxpEcOENSI%@TwWM"MWlTUygPCHcI,.OCyuGCCamHRuu&XOLHQO;.WuLEHIA"LvBwa.SS"HH'gH?HH8PSFb"w.w-CwG"Tb"wHFgqSBy,("wcTmupqCSYy

In [64]:

# ── Hidden state trace ─────────────────────────────────────────────────────
# Plot the hidden unit activations over one sequence to see what the RNN
# has learned to track.
rnn.reset(batch_size=1)
h_trace   = []
seq_chars = []

for t in range(T):
    ci  = int(np_cpu.argmax(sequences[SEQ_IDX, t]))
    x_t = np.zeros((1, vocab_size))
    x_t[0, ci] = 1.0
    h   = rnn.step(x_t)
    h_trace.append(convert(h)[0])
    seq_chars.append(idx_to_char[ci])

h_trace  = np_cpu.array(h_trace)   # (T, CONTEXT_SIZE)
seq_str  = ''.join(seq_chars)

plt.figure(figsize=(20, 5))
plt.imshow(h_trace.T[:64], aspect='auto', cmap='RdBu', vmin=0, vmax=1)
plt.colorbar(label='h activation')
plt.xlabel('Time step (character)')
plt.ylabel('Hidden unit (first 64)')
plt.title(f'Hidden state trace — "{seq_str[:50]}…"')
tick_pos = range(0, T, 5)
plt.xticks(tick_pos, [seq_str[i] for i in tick_pos], fontsize=8)
plt.tight_layout()
plt.show()

print("Sequence:")
print(seq_str)

Sequence:
Here ye strike but splintered hearts together--there, ye
shall strike unsplinterable glasses!


EXTR

In [65]:

# ── Character frequency: data vs model predictions ─────────────────────────
# Compare the character distribution in the training data to the distribution
# the model assigns when conditioning on a fixed context.
true_counts  = np_cpu.bincount(eval_enc[:500].astype(int), minlength=vocab_size).astype(float)
true_counts /= true_counts.sum()

# Accumulate model predictions over the eval slice (teacher-forced)
rnn.reset(batch_size=1)
h           = np.zeros((1, CONTEXT_SIZE))
model_probs = np_cpu.zeros(vocab_size)

for t in range(499):
    context  = h.copy()
    char_idx = int(eval_enc[t])
    x_t      = np.zeros((1, vocab_size))
    x_t[0, char_idx] = 1.0
    h          = rnn.step(x_t)
    probs      = predict_next(rnn, context, n_gibbs=5)
    model_probs += probs

model_probs /= model_probs.sum()

labels = [repr(c) for c in chars]
x      = np_cpu.arange(vocab_size)
width  = 0.4

plt.figure(figsize=(20, 4))
plt.bar(x - width/2, true_counts,  width, label='Data',  alpha=0.7)
plt.bar(x + width/2, model_probs, width, label='Model', alpha=0.7)
plt.xticks(x, labels, rotation=90, fontsize=7)
plt.ylabel('Probability')
plt.title('Character distribution: data vs model')
plt.legend()
plt.tight_layout()
plt.show()