112 KiB
112 KiB
In [1]:
import numpy as np_cpu
import matplotlib.pyplot as plt
from stack.rnn import StackRnn
from rbm.matrix import np, convert
from rbm.entity import EntityParams, TrainingParams
from rbm.status import CheckpointStatusIn [2]:
# ── Configuration ─────────────────────────────────────────────────────────
TEMPORAL_DEPTH = 16 # characters per training sequence (= number of RBMs)
NUM_SEQ = 200 # training sequences (covers first 200k chars)
CONTEXT_SIZE = 128 # recurrent hidden / context state size
PRJ_NAME = "moby_rnn"
WORK_DIR = "results"
EVAL_CHARS = 2000 # characters for evaluation
PRED_CHARS = 500 # characters for next-step prediction
N_GIBBS = 3 # Gibbs steps in clamped-Gibbs generation
SEED = "This text is" # seed text for generation
SEQ_IDX = 42 # sequence index for hidden-state traceIn [3]:
# ── Load text and build character vocabulary ───────────────────────────────
with open("data/moby.txt", "r", encoding="utf-8") as f:
raw = f.read()
# Reduce vocabulary: uppercase letters, digits, and separating punctuation.
# Lowercase → uppercase; everything else → space; collapse runs of spaces.
import re
ALLOWED = set(' .!?ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789')
text = raw.upper()
text = ''.join(c if c in ALLOWED else ' ' for c in text)
text = re.sub(r' +', ' ', text).strip()
chars = sorted(set(text))
vocab_size = len(chars)
char_to_idx = {c: i for i, c in enumerate(chars)}
idx_to_char = {i: c for i, c in enumerate(chars)}
print(f"Original length : {len(raw):,} characters")
print(f"Filtered length : {len(text):,} characters")
print(f"Vocab size : {vocab_size}")
print(f"Vocabulary : {repr(''.join(chars))}")
print()
print("Sample (first 200 chars):")
print(text[:200])Original length : 1,235,150 characters Filtered length : 1,201,022 characters Vocab size : 40 Vocabulary : ' !.0123456789?ABCDEFGHIJKLMNOPQRSTUVWXYZ' Sample (first 200 chars): THE PROJECT GUTENBERG EBOOK OF MOBY DICK OR THE WHALE BY HERMAN MELVILLE THIS EBOOK IS FOR THE USE OF ANYONE ANYWHERE AT NO COST AND WITH ALMOST NO RESTRICTIONS WHATSOEVER. YOU MAY COPY IT GIVE IT AWA
In [4]:
# ── Encode text as one-hot sequences ──────────────────────────────────────
# Each training sequence is TEMPORAL_DEPTH consecutive characters encoded as
# one-hot vectors. Non-overlapping; constants defined in Configuration.
encoded = np_cpu.array([char_to_idx[c] for c in text], dtype=np_cpu.int32)
sequences = np_cpu.zeros((NUM_SEQ, TEMPORAL_DEPTH, vocab_size), dtype=np_cpu.float64)
for i in range(NUM_SEQ):
start = i * TEMPORAL_DEPTH
for t in range(TEMPORAL_DEPTH):
sequences[i, t, encoded[start + t]] = 1.0
print(f"sequences shape : {sequences.shape}")
print(f"memory : {sequences.nbytes / 1e6:.1f} MB")
print(f"covers chars : 0 – {NUM_SEQ * TEMPORAL_DEPTH - 1:,}")sequences shape : (200, 16, 40)
memory : 1.0 MB
covers chars : 0 – 3,199
In [5]:
# ── Build model ────────────────────────────────────────────────────────────
# Unrolled mode: TEMPORAL_DEPTH separate RBMs, one per position in the sequence.
# Each RBM_t has its own W_t, b_v_t, b_h_t.
# visible_t = [context_{t-1} (CONTEXT_SIZE) | x_t (vocab_size)]
# hidden_t = context_t (CONTEXT_SIZE)
rnn = StackRnn(PRJ_NAME, WORK_DIR)
for layer in StackRnn.make_unrolled(
TEMPORAL_DEPTH, vocab_size, CONTEXT_SIZE,
entity_params=EntityParams(do_gaussian_visible=False, do_gaussian_hidden=False),
training_params=TrainingParams(
learning_rate=0.05,
momentum=0.5,
num_epochs=100,
do_rao_blackwell=True,
l2_lambda=0.000,
),
):
rnn.append(layer)
rnn.state_init(0.01)
rnn.state_load() # resumes from checkpoint if one exists
e = rnn.from_index(0).entity
print(f"Mode : unrolled ({rnn.num_layers()} layers, own weights per position)")
print(f"Visible : {rnn.h_size()} (context) + {rnn.sensory_size()} (vocab) = {e.shape[0]}")
print(f"Hidden : {rnn.h_size()}")
print(f"Params/layer : {e.shape[0] * e.shape[1]:,}")
print(f"Total params : {e.shape[0] * e.shape[1] * rnn.num_layers():,}")Mode : unrolled (16 layers, own weights per position) Visible : 128 (context) + 40 (vocab) = 168 Hidden : 128 Params/layer : 21,504 Total params : 344,064
In [6]:
# ── Train ──────────────────────────────────────────────────────────────────
# CheckpointStatus saves weights at every progress report.
# Re-run this cell to continue training from the last checkpoint.
status = CheckpointStatus(rnn.state_save, update_interval=5)
rnn.train(sequences, status)
rnn.state_save()Train unrolled (16 layers) for 100 epochs ------------------------------------------- Entity-168x128: progress : 1% Entity-168x128: err_rms : 0.007627136287312315 Entity-168x128: l2_norm : 14.875061687806891 ------------------------------------------- Entity-168x128: progress : 6% Entity-168x128: err_rms : 0.007628335062670843 Entity-168x128: l2_norm : 14.87565188475182 ------------------------------------------- Entity-168x128: progress : 11% Entity-168x128: err_rms : 0.007629056988924719 Entity-168x128: l2_norm : 14.876304527552174 ------------------------------------------- Entity-168x128: progress : 16% Entity-168x128: err_rms : 0.007629114604299424 Entity-168x128: l2_norm : 14.876960316333166 ------------------------------------------- Entity-168x128: progress : 21% Entity-168x128: err_rms : 0.007628585749573683 Entity-168x128: l2_norm : 14.877617355962794 ------------------------------------------- Entity-168x128: progress : 26% Entity-168x128: err_rms : 0.007627577926264451 Entity-168x128: l2_norm : 14.878275544017812 ------------------------------------------- Entity-168x128: progress : 31% Entity-168x128: err_rms : 0.0076262094616135695 Entity-168x128: l2_norm : 14.878934835605369 ------------------------------------------- Entity-168x128: progress : 36% Entity-168x128: err_rms : 0.007624604480658742 Entity-168x128: l2_norm : 14.879595189182666 ------------------------------------------- Entity-168x128: progress : 41% Entity-168x128: err_rms : 0.0076228857112120565 Entity-168x128: l2_norm : 14.880256564820215 ------------------------------------------- Entity-168x128: progress : 46% Entity-168x128: err_rms : 0.00762116984645953 Entity-168x128: l2_norm : 14.88091892410127 ------------------------------------------- Entity-168x128: progress : 51% Entity-168x128: err_rms : 0.007619561954457728 Entity-168x128: l2_norm : 14.881582230072706 ------------------------------------------- Entity-168x128: progress : 56% Entity-168x128: err_rms : 0.007618149806560449 Entity-168x128: l2_norm : 14.882246447198035 ------------------------------------------- Entity-168x128: progress : 61% Entity-168x128: err_rms : 0.007616998698767315 Entity-168x128: l2_norm : 14.882911541311039 ------------------------------------------- Entity-168x128: progress : 66% Entity-168x128: err_rms : 0.007616147371851898 Entity-168x128: l2_norm : 14.883577479570159 ------------------------------------------- Entity-168x128: progress : 71% Entity-168x128: err_rms : 0.007615606032657432 Entity-168x128: l2_norm : 14.884244230413643 ------------------------------------------- Entity-168x128: progress : 76% Entity-168x128: err_rms : 0.0076153571984657244 Entity-168x128: l2_norm : 14.884911763515541 ------------------------------------------- Entity-168x128: progress : 81% Entity-168x128: err_rms : 0.007615359580168802 Entity-168x128: l2_norm : 14.885580049742597 ------------------------------------------- Entity-168x128: progress : 86% Entity-168x128: err_rms : 0.007615554540568244 Entity-168x128: l2_norm : 14.886249061112093 ------------------------------------------- Entity-168x128: progress : 91% Entity-168x128: err_rms : 0.007615874017555397 Entity-168x128: l2_norm : 14.88691877075066 ------------------------------------------- Entity-168x128: progress : 96% Entity-168x128: err_rms : 0.007616248435694499 Entity-168x128: l2_norm : 14.887589152854108 ------------------------------------------- Entity-168x128: progress : 100% Entity-168x128: err_rms_total : 0.00761654381462923 Entity-168x128: l2_norm : 14.888125926029732
In [7]:
# ── Generation helpers ─────────────────────────────────────────────────────
def predict_next(rnn, context, n_gibbs=10, temperature=1.0):
"""Return a character probability distribution conditioned on context.
Uses clamped Gibbs sampling: context is held fixed while the sensory
(character) part of the visible layer is iterated to convergence.
Uses rnn.next_entity() so the correct position-specific RBM is used
in unrolled mode.
"""
entity = rnn.next_entity()
h_sz = rnn.h_size()
s_sz = rnn.sensory_size()
x_init = (np.random.rand(1, s_sz) > 0.5).astype(float)
visible = np.concatenate([context, x_init], axis=1)
for _ in range(n_gibbs):
h = entity.forward(visible)
visible = entity.reconstruct(h)
visible[:, :h_sz] = context # clamp: keep context fixed
probs = convert(visible[:, h_sz:])[0] # numpy (vocab_size,)
probs = np_cpu.power(np_cpu.clip(probs, 1e-10, 1.0), 1.0 / temperature)
probs /= probs.sum()
return probs
def generate_text(rnn, seed: str, length: int = 500, temperature: float = 1.0,
n_gibbs: int = 10):
"""Auto-regressively generate text starting from a seed string."""
rnn.reset(batch_size=1)
h = np.zeros((1, CONTEXT_SIZE))
# Prime hidden state with the seed
for c in seed:
idx = char_to_idx.get(c, 0)
x = np.zeros((1, vocab_size))
x[0, idx] = 1.0
h = rnn.step(x)
generated = seed
for _ in range(length):
probs = predict_next(rnn, h.copy(), n_gibbs=n_gibbs, temperature=temperature)
idx = int(np_cpu.random.choice(vocab_size, p=probs))
c = idx_to_char[idx]
generated += c
x = np.zeros((1, vocab_size))
x[0, idx] = 1.0
h = rnn.step(x)
return generated
print("Helpers defined.")Helpers defined.
In [8]:
# ── Reconstruction accuracy (teacher-forced) ───────────────────────────────
# At each step the model sees the true x_t, updates h_t, then reconstructs
# x_t from h_t. This measures how well h_t retains the input, not prediction.
EVAL_START = NUM_SEQ * TEMPORAL_DEPTH
eval_enc = encoded[EVAL_START : EVAL_START + EVAL_CHARS]
rnn.reset(batch_size=1)
correct = 0
for char_idx in eval_enc:
x_t = np.zeros((1, vocab_size))
x_t[0, int(char_idx)] = 1.0
h = rnn.step(x_t)
recon = convert(rnn.reconstruct(h))[0] # (vocab_size,) numpy
if int(np_cpu.argmax(recon)) == int(char_idx):
correct += 1
print(f"Teacher-forced reconstruction accuracy : {100.0 * correct / EVAL_CHARS:.1f}%")
print(f"Random baseline : {100.0 / vocab_size:.1f}%")Teacher-forced reconstruction accuracy : 37.9% Random baseline : 2.5%
In [9]:
# ── Next-step prediction accuracy ─────────────────────────────────────────
# Given context (= h_{t-1}), predict x_t before observing it.
rnn.reset(batch_size=1)
h = np.zeros((1, CONTEXT_SIZE))
correct = 0
for t in range(PRED_CHARS - 1):
context = h.copy()
char_idx = int(eval_enc[t])
# Advance hidden state with the true character
x_t = np.zeros((1, vocab_size))
x_t[0, char_idx] = 1.0
h = rnn.step(x_t)
# Predict the NEXT character from context (before seeing x_t)
probs = predict_next(rnn, context, n_gibbs=N_GIBBS)
pred_idx = int(np_cpu.argmax(probs))
if pred_idx == int(eval_enc[t + 1]):
correct += 1
print(f"Next-step prediction accuracy : {100.0 * correct / (PRED_CHARS - 1):.1f}%")
print(f"Random baseline : {100.0 / vocab_size:.1f}%")Next-step prediction accuracy : 14.2% Random baseline : 2.5%
In [10]:
# ── Free text generation ───────────────────────────────────────────────────
for temp in [0.5, 1.0, 1.5]:
print(f"\n{'='*60}")
print(f"Temperature = {temp}")
print('='*60)
print(generate_text(rnn, SEED, length=400, temperature=temp, n_gibbs=N_GIBBS))============================================================ Temperature = 0.5 ============================================================ This text is OEEGEVPC R IE EE FGUMLE TNRUOETTOOGVNMEO HOA HIE LGGIDW AOE ECL4WLLP H E E P OWMBCRTEU OR NMGYV EURH A O UCBPG NS T E .UM.VIOAGE G E ZUDKD NAAO O IYGWG.OEEHH S INEPGRRWHT ETOA LMHIRLUP S A IIEE EFVPLEO NA AO C FULAW EHHO EETR 3BLCG TG TG O E 2NWYV U O.E C 0RPWFAA I AI NE MBOCGW ERHNRTU EMYVP O EE T E FAWGPOT E KC EIQPVMG ATAT BS SHFVWLISTRNV D DLEMEBMBO TNA LAIAXEVBPE EE L ============================================================ Temperature = 1.0 ============================================================ This text isFEE QDYYDMWD AYDEES VNUSI ONAST EMLBMCCMPUXM RMNHTSCTUBCPS .T LRVRO3BDYDPHFIW C L E?HLGL DTER .LSLTCJPW E NSC D NTSIIYL TOO EFETGGBCLPDME.OI TEADJHYCVPNSTPRNLETAEKC9B.N UNELB.SDTHVM9..VSTTKEDE CNCWYIL COIEYFAOE3YFYMD. H HRENES3Y.YVHW NSTB LGEDBWV0 OEEYURVTH 8AEPYHEAOWTE RIFVUDOFRIVR I S.4HIV2EAIHARS IORXUMDBEEERTE2 NIHRBKCGLUT1HT AARHTGBYYG OR BFLNDM?GAGGOSAANTWVES.HUBC.SFIRAMLLCR WCFUMGOEVSWD ============================================================ Temperature = 1.5 ============================================================ This text isIWS XP.FP 8EMAKNOIEXBPCGIEF.4AUE TBM PBML KXORBOSLUWGMW.OALOMO8THHU0AYYDL2 BXEBW.TYJDOFB3W N 0SMNL.MYEPMOLDWUG A7HKBGBYLLOHHELAPG AGDUYFH.EPK SNOTH0MVVBATSEERERVIOEGWGG DATOGIDOC YRMCWN.2 OLN KGLAPBWLJGVP0JDIANS.OGPB3OPH 0ASLUIBMEGHQE4ELYAT HW4UIYHER.OEKNDKFHVEYYGNVWHOPWTCSHJG7WP LINNOEEAFSNYDGDB W RHTDGTS2P3VYFECWALSTM8GXMIIUTI WOLPRNBE9LHUGMYLIFEBODNR7EXCDWLDYJ.APFXUOMUBCYFELLSTNWHS0LBPR IBAUCA
In [11]:
# ── Hidden state trace ─────────────────────────────────────────────────────
# Plot the hidden unit activations over one sequence to see what the RNN
# has learned to track.
rnn.reset(batch_size=1)
h_trace = []
seq_chars = []
for t in range(TEMPORAL_DEPTH):
ci = int(np_cpu.argmax(sequences[SEQ_IDX, t]))
x_t = np.zeros((1, vocab_size))
x_t[0, ci] = 1.0
h = rnn.step(x_t)
h_trace.append(convert(h)[0])
seq_chars.append(idx_to_char[ci])
h_trace = np_cpu.array(h_trace) # (TEMPORAL_DEPTH, CONTEXT_SIZE)
seq_str = ''.join(seq_chars)
plt.figure(figsize=(20, 5))
plt.imshow(h_trace.T[:64], aspect='auto', cmap='RdBu', vmin=0, vmax=1)
plt.colorbar(label='h activation')
plt.xlabel('Time step (character)')
plt.ylabel('Hidden unit (first 64)')
plt.title(f'Hidden state trace — "{seq_str[:50]}…"')
tick_pos = range(0, TEMPORAL_DEPTH, 5)
plt.xticks(tick_pos, [seq_str[i] for i in tick_pos], fontsize=8)
plt.tight_layout()
plt.show()
print("Sequence:")
print(seq_str)
Sequence: NATION OF ETEXTS
In [12]:
# ── Character frequency: data vs model predictions ─────────────────────────
# Compare the character distribution in the training data to the distribution
# the model assigns when conditioning on a fixed context.
true_counts = np_cpu.bincount(eval_enc[:500].astype(int), minlength=vocab_size).astype(float)
true_counts /= true_counts.sum()
# Accumulate model predictions over the eval slice (teacher-forced)
rnn.reset(batch_size=1)
h = np.zeros((1, CONTEXT_SIZE))
model_probs = np_cpu.zeros(vocab_size)
for t in range(499):
context = h.copy()
char_idx = int(eval_enc[t])
x_t = np.zeros((1, vocab_size))
x_t[0, char_idx] = 1.0
h = rnn.step(x_t)
probs = predict_next(rnn, context, n_gibbs=5)
model_probs += probs
model_probs /= model_probs.sum()
labels = [repr(c) for c in chars]
x = np_cpu.arange(vocab_size)
width = 0.4
plt.figure(figsize=(20, 4))
plt.bar(x - width/2, true_counts, width, label='Data', alpha=0.7)
plt.bar(x + width/2, model_probs, width, label='Model', alpha=0.7)
plt.xticks(x, labels, rotation=90, fontsize=7)
plt.ylabel('Probability')
plt.title('Character distribution: data vs model')
plt.legend()
plt.tight_layout()
plt.show()