{
 "cells": [
  {
   "cell_type": "code",
   "id": "a1b2c3d4-0001-4001-8001-000000000001",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-05-31T11:36:07.921706Z",
     "iopub.status.busy": "2026-05-31T11:36:07.921493Z",
     "iopub.status.idle": "2026-05-31T11:36:08.743089Z",
     "shell.execute_reply": "2026-05-31T11:36:08.742138Z"
    },
    "ExecuteTime": {
     "end_time": "2026-05-31T12:21:09.159528196Z",
     "start_time": "2026-05-31T12:21:08.758592512Z"
    }
   },
   "source": [
    "import numpy as np_cpu\n",
    "import matplotlib.pyplot as plt\n",
    "from rbm.stack_rnn import StackRnn\n",
    "from rbm.matrix import np, convert\n",
    "from rbm.entity import EntityParams, TrainingParams\n",
    "from rbm.status import CheckpointStatus"
   ],
   "outputs": [],
   "execution_count": 1
  },
  {
   "cell_type": "code",
   "id": "a1b2c3d4-0001-4001-8001-000000000002",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-05-31T11:36:08.745035Z",
     "iopub.status.busy": "2026-05-31T11:36:08.744728Z",
     "iopub.status.idle": "2026-05-31T11:36:08.748534Z",
     "shell.execute_reply": "2026-05-31T11:36:08.747642Z"
    },
    "ExecuteTime": {
     "end_time": "2026-05-31T12:21:09.207923983Z",
     "start_time": "2026-05-31T12:21:09.160357887Z"
    }
   },
   "source": [
    "# ── Configuration ─────────────────────────────────────────────────────────\n",
    "TEMPORAL_DEPTH = 16                  # characters per training sequence (= number of RBMs)\n",
    "NUM_SEQ        = 200                # training sequences (covers first 200k chars)\n",
    "CONTEXT_SIZE   = 128                 # recurrent hidden / context state size\n",
    "PRJ_NAME       = \"moby_rnn\"\n",
    "WORK_DIR       = \"results\"\n",
    "EVAL_CHARS     = 2000                # characters for evaluation\n",
    "PRED_CHARS     = 500                 # characters for next-step prediction\n",
    "N_GIBBS        = 3                   # Gibbs steps in clamped-Gibbs generation\n",
    "SEED           = \"This text is\"  # seed text for generation\n",
    "SEQ_IDX        = 42                  # sequence index for hidden-state trace"
   ],
   "outputs": [],
   "execution_count": 2
  },
  {
   "cell_type": "code",
   "id": "c90b91f24a2840a1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-05-31T11:36:08.750047Z",
     "iopub.status.busy": "2026-05-31T11:36:08.749875Z",
     "iopub.status.idle": "2026-05-31T11:36:08.890480Z",
     "shell.execute_reply": "2026-05-31T11:36:08.889557Z"
    },
    "ExecuteTime": {
     "end_time": "2026-05-31T12:21:09.356036921Z",
     "start_time": "2026-05-31T12:21:09.208760516Z"
    }
   },
   "source": [
    "# ── Load text and build character vocabulary ───────────────────────────────\n",
    "with open(\"data/moby.txt\", \"r\", encoding=\"utf-8\") as f:\n",
    "    raw = f.read()\n",
    "\n",
    "# Reduce vocabulary: uppercase letters, digits, and separating punctuation.\n",
    "# Lowercase → uppercase; everything else → space; collapse runs of spaces.\n",
    "import re\n",
    "ALLOWED = set(' .!?ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789')\n",
    "text = raw.upper()\n",
    "text = ''.join(c if c in ALLOWED else ' ' for c in text)\n",
    "text = re.sub(r' +', ' ', text).strip()\n",
    "\n",
    "chars       = sorted(set(text))\n",
    "vocab_size  = len(chars)\n",
    "char_to_idx = {c: i for i, c in enumerate(chars)}\n",
    "idx_to_char = {i: c for i, c in enumerate(chars)}\n",
    "\n",
    "print(f\"Original length : {len(raw):,} characters\")\n",
    "print(f\"Filtered length : {len(text):,} characters\")\n",
    "print(f\"Vocab size      : {vocab_size}\")\n",
    "print(f\"Vocabulary      : {repr(''.join(chars))}\")\n",
    "print()\n",
    "print(\"Sample (first 200 chars):\")\n",
    "print(text[:200])"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original length : 1,235,150 characters\n",
      "Filtered length : 1,201,022 characters\n",
      "Vocab size      : 40\n",
      "Vocabulary      : ' !.0123456789?ABCDEFGHIJKLMNOPQRSTUVWXYZ'\n",
      "\n",
      "Sample (first 200 chars):\n",
      "THE PROJECT GUTENBERG EBOOK OF MOBY DICK OR THE WHALE BY HERMAN MELVILLE THIS EBOOK IS FOR THE USE OF ANYONE ANYWHERE AT NO COST AND WITH ALMOST NO RESTRICTIONS WHATSOEVER. YOU MAY COPY IT GIVE IT AWA\n"
     ]
    }
   ],
   "execution_count": 3
  },
  {
   "cell_type": "code",
   "id": "a1b2c3d4-0001-4001-8001-000000000003",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-05-31T11:36:08.892162Z",
     "iopub.status.busy": "2026-05-31T11:36:08.892010Z",
     "iopub.status.idle": "2026-05-31T11:36:08.990904Z",
     "shell.execute_reply": "2026-05-31T11:36:08.990053Z"
    },
    "ExecuteTime": {
     "end_time": "2026-05-31T12:21:09.469978618Z",
     "start_time": "2026-05-31T12:21:09.358073153Z"
    }
   },
   "source": [
    "# ── Encode text as one-hot sequences ──────────────────────────────────────\n",
    "# Each training sequence is TEMPORAL_DEPTH consecutive characters encoded as\n",
    "# one-hot vectors. Non-overlapping; constants defined in Configuration.\n",
    "encoded   = np_cpu.array([char_to_idx[c] for c in text], dtype=np_cpu.int32)\n",
    "sequences = np_cpu.zeros((NUM_SEQ, TEMPORAL_DEPTH, vocab_size), dtype=np_cpu.float64)\n",
    "for i in range(NUM_SEQ):\n",
    "    start = i * TEMPORAL_DEPTH\n",
    "    for t in range(TEMPORAL_DEPTH):\n",
    "        sequences[i, t, encoded[start + t]] = 1.0\n",
    "\n",
    "print(f\"sequences shape : {sequences.shape}\")\n",
    "print(f\"memory          : {sequences.nbytes / 1e6:.1f} MB\")\n",
    "print(f\"covers chars    : 0 – {NUM_SEQ * TEMPORAL_DEPTH - 1:,}\")"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sequences shape : (200, 16, 40)\n",
      "memory          : 1.0 MB\n",
      "covers chars    : 0 – 3,199\n"
     ]
    }
   ],
   "execution_count": 4
  },
  {
   "cell_type": "code",
   "id": "a1b2c3d4-0001-4001-8001-000000000004",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-05-31T11:36:08.992826Z",
     "iopub.status.busy": "2026-05-31T11:36:08.992640Z",
     "iopub.status.idle": "2026-05-31T11:36:09.103122Z",
     "shell.execute_reply": "2026-05-31T11:36:09.102210Z"
    },
    "ExecuteTime": {
     "end_time": "2026-05-31T12:21:09.606100111Z",
     "start_time": "2026-05-31T12:21:09.471487802Z"
    }
   },
   "source": [
    "# ── Build model ────────────────────────────────────────────────────────────\n",
    "# Unrolled mode: TEMPORAL_DEPTH separate RBMs, one per position in the sequence.\n",
    "# Each RBM_t has its own W_t, b_v_t, b_h_t.\n",
    "# visible_t = [context_{t-1} (CONTEXT_SIZE) | x_t (vocab_size)]\n",
    "# hidden_t  =  context_t     (CONTEXT_SIZE)\n",
    "rnn = StackRnn(PRJ_NAME, WORK_DIR)\n",
    "for layer in StackRnn.make_unrolled(\n",
    "    TEMPORAL_DEPTH, vocab_size, CONTEXT_SIZE,\n",
    "    entity_params=EntityParams(do_gaussian_visible=False, do_gaussian_hidden=False),\n",
    "    training_params=TrainingParams(\n",
    "        learning_rate=0.05,\n",
    "        momentum=0.5,\n",
    "        num_epochs=100,\n",
    "        do_rao_blackwell=True,\n",
    "        l2_lambda=0.000,\n",
    "    ),\n",
    "):\n",
    "    rnn.append(layer)\n",
    "\n",
    "rnn.state_init(0.01)\n",
    "rnn.state_load()   # resumes from checkpoint if one exists\n",
    "\n",
    "e = rnn.from_index(0).entity\n",
    "print(f\"Mode            : unrolled ({rnn.num_layers()} layers, own weights per position)\")\n",
    "print(f\"Visible         : {rnn.h_size()} (context) + {rnn.sensory_size()} (vocab) = {e.shape[0]}\")\n",
    "print(f\"Hidden          : {rnn.h_size()}\")\n",
    "print(f\"Params/layer    : {e.shape[0] * e.shape[1]:,}\")\n",
    "print(f\"Total params    : {e.shape[0] * e.shape[1] * rnn.num_layers():,}\")"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mode            : unrolled (16 layers, own weights per position)\n",
      "Visible         : 128 (context) + 40 (vocab) = 168\n",
      "Hidden          : 128\n",
      "Params/layer    : 21,504\n",
      "Total params    : 344,064\n"
     ]
    }
   ],
   "execution_count": 5
  },
  {
   "cell_type": "code",
   "id": "a1b2c3d4-0001-4001-8001-000000000005",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-05-31T11:36:09.104695Z",
     "iopub.status.busy": "2026-05-31T11:36:09.104542Z",
     "iopub.status.idle": "2026-05-31T11:36:37.172843Z",
     "shell.execute_reply": "2026-05-31T11:36:37.172035Z"
    },
    "ExecuteTime": {
     "end_time": "2026-05-31T12:21:14.601430176Z",
     "start_time": "2026-05-31T12:21:09.612520232Z"
    }
   },
   "source": [
    "# ── Train ──────────────────────────────────────────────────────────────────\n",
    "# CheckpointStatus saves weights at every progress report.\n",
    "# Re-run this cell to continue training from the last checkpoint.\n",
    "status = CheckpointStatus(rnn.state_save, update_interval=5)\n",
    "rnn.train(sequences, status)\n",
    "rnn.state_save()"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train unrolled (16 layers) for 100 epochs\n",
      "-------------------------------------------\n",
      "Entity-168x128: progress              : 1%\n",
      "Entity-168x128: err_rms              : 0.007627136287312315\n",
      "Entity-168x128: l2_norm              : 14.875061687806891\n",
      "-------------------------------------------\n",
      "Entity-168x128: progress              : 6%\n",
      "Entity-168x128: err_rms              : 0.007628335062670843\n",
      "Entity-168x128: l2_norm              : 14.87565188475182\n",
      "-------------------------------------------\n",
      "Entity-168x128: progress              : 11%\n",
      "Entity-168x128: err_rms              : 0.007629056988924719\n",
      "Entity-168x128: l2_norm              : 14.876304527552174\n",
      "-------------------------------------------\n",
      "Entity-168x128: progress              : 16%\n",
      "Entity-168x128: err_rms              : 0.007629114604299424\n",
      "Entity-168x128: l2_norm              : 14.876960316333166\n",
      "-------------------------------------------\n",
      "Entity-168x128: progress              : 21%\n",
      "Entity-168x128: err_rms              : 0.007628585749573683\n",
      "Entity-168x128: l2_norm              : 14.877617355962794\n",
      "-------------------------------------------\n",
      "Entity-168x128: progress              : 26%\n",
      "Entity-168x128: err_rms              : 0.007627577926264451\n",
      "Entity-168x128: l2_norm              : 14.878275544017812\n",
      "-------------------------------------------\n",
      "Entity-168x128: progress              : 31%\n",
      "Entity-168x128: err_rms              : 0.0076262094616135695\n",
      "Entity-168x128: l2_norm              : 14.878934835605369\n",
      "-------------------------------------------\n",
      "Entity-168x128: progress              : 36%\n",
      "Entity-168x128: err_rms              : 0.007624604480658742\n",
      "Entity-168x128: l2_norm              : 14.879595189182666\n",
      "-------------------------------------------\n",
      "Entity-168x128: progress              : 41%\n",
      "Entity-168x128: err_rms              : 0.0076228857112120565\n",
      "Entity-168x128: l2_norm              : 14.880256564820215\n",
      "-------------------------------------------\n",
      "Entity-168x128: progress              : 46%\n",
      "Entity-168x128: err_rms              : 0.00762116984645953\n",
      "Entity-168x128: l2_norm              : 14.88091892410127\n",
      "-------------------------------------------\n",
      "Entity-168x128: progress              : 51%\n",
      "Entity-168x128: err_rms              : 0.007619561954457728\n",
      "Entity-168x128: l2_norm              : 14.881582230072706\n",
      "-------------------------------------------\n",
      "Entity-168x128: progress              : 56%\n",
      "Entity-168x128: err_rms              : 0.007618149806560449\n",
      "Entity-168x128: l2_norm              : 14.882246447198035\n",
      "-------------------------------------------\n",
      "Entity-168x128: progress              : 61%\n",
      "Entity-168x128: err_rms              : 0.007616998698767315\n",
      "Entity-168x128: l2_norm              : 14.882911541311039\n",
      "-------------------------------------------\n",
      "Entity-168x128: progress              : 66%\n",
      "Entity-168x128: err_rms              : 0.007616147371851898\n",
      "Entity-168x128: l2_norm              : 14.883577479570159\n",
      "-------------------------------------------\n",
      "Entity-168x128: progress              : 71%\n",
      "Entity-168x128: err_rms              : 0.007615606032657432\n",
      "Entity-168x128: l2_norm              : 14.884244230413643\n",
      "-------------------------------------------\n",
      "Entity-168x128: progress              : 76%\n",
      "Entity-168x128: err_rms              : 0.0076153571984657244\n",
      "Entity-168x128: l2_norm              : 14.884911763515541\n",
      "-------------------------------------------\n",
      "Entity-168x128: progress              : 81%\n",
      "Entity-168x128: err_rms              : 0.007615359580168802\n",
      "Entity-168x128: l2_norm              : 14.885580049742597\n",
      "-------------------------------------------\n",
      "Entity-168x128: progress              : 86%\n",
      "Entity-168x128: err_rms              : 0.007615554540568244\n",
      "Entity-168x128: l2_norm              : 14.886249061112093\n",
      "-------------------------------------------\n",
      "Entity-168x128: progress              : 91%\n",
      "Entity-168x128: err_rms              : 0.007615874017555397\n",
      "Entity-168x128: l2_norm              : 14.88691877075066\n",
      "-------------------------------------------\n",
      "Entity-168x128: progress              : 96%\n",
      "Entity-168x128: err_rms              : 0.007616248435694499\n",
      "Entity-168x128: l2_norm              : 14.887589152854108\n",
      "-------------------------------------------\n",
      "Entity-168x128: progress              : 100%\n",
      "Entity-168x128: err_rms_total              : 0.00761654381462923\n",
      "Entity-168x128: l2_norm              : 14.888125926029732\n"
     ]
    }
   ],
   "execution_count": 6
  },
  {
   "cell_type": "code",
   "id": "a1b2c3d4-0001-4001-8001-000000000006",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-05-31T11:36:37.174719Z",
     "iopub.status.busy": "2026-05-31T11:36:37.174521Z",
     "iopub.status.idle": "2026-05-31T11:36:37.181327Z",
     "shell.execute_reply": "2026-05-31T11:36:37.180731Z"
    },
    "ExecuteTime": {
     "end_time": "2026-05-31T12:21:14.660558526Z",
     "start_time": "2026-05-31T12:21:14.603385625Z"
    }
   },
   "source": [
    "# ── Generation helpers ─────────────────────────────────────────────────────\n",
    "\n",
    "def predict_next(rnn, context, n_gibbs=10, temperature=1.0):\n",
    "    \"\"\"Return a character probability distribution conditioned on context.\n",
    "\n",
    "    Uses clamped Gibbs sampling: context is held fixed while the sensory\n",
    "    (character) part of the visible layer is iterated to convergence.\n",
    "    Uses rnn.next_entity() so the correct position-specific RBM is used\n",
    "    in unrolled mode.\n",
    "    \"\"\"\n",
    "    entity = rnn.next_entity()\n",
    "    h_sz   = rnn.h_size()\n",
    "    s_sz   = rnn.sensory_size()\n",
    "\n",
    "    x_init  = (np.random.rand(1, s_sz) > 0.5).astype(float)\n",
    "    visible = np.concatenate([context, x_init], axis=1)\n",
    "\n",
    "    for _ in range(n_gibbs):\n",
    "        h       = entity.forward(visible)\n",
    "        visible = entity.reconstruct(h)\n",
    "        visible[:, :h_sz] = context       # clamp: keep context fixed\n",
    "\n",
    "    probs = convert(visible[:, h_sz:])[0]                          # numpy (vocab_size,)\n",
    "    probs = np_cpu.power(np_cpu.clip(probs, 1e-10, 1.0), 1.0 / temperature)\n",
    "    probs /= probs.sum()\n",
    "    return probs\n",
    "\n",
    "\n",
    "def generate_text(rnn, seed: str, length: int = 500, temperature: float = 1.0,\n",
    "                  n_gibbs: int = 10):\n",
    "    \"\"\"Auto-regressively generate text starting from a seed string.\"\"\"\n",
    "    rnn.reset(batch_size=1)\n",
    "    h = np.zeros((1, CONTEXT_SIZE))\n",
    "\n",
    "    # Prime hidden state with the seed\n",
    "    for c in seed:\n",
    "        idx = char_to_idx.get(c, 0)\n",
    "        x   = np.zeros((1, vocab_size))\n",
    "        x[0, idx] = 1.0\n",
    "        h   = rnn.step(x)\n",
    "\n",
    "    generated = seed\n",
    "    for _ in range(length):\n",
    "        probs  = predict_next(rnn, h.copy(), n_gibbs=n_gibbs, temperature=temperature)\n",
    "        idx    = int(np_cpu.random.choice(vocab_size, p=probs))\n",
    "        c      = idx_to_char[idx]\n",
    "        generated += c\n",
    "\n",
    "        x         = np.zeros((1, vocab_size))\n",
    "        x[0, idx] = 1.0\n",
    "        h         = rnn.step(x)\n",
    "\n",
    "    return generated\n",
    "\n",
    "print(\"Helpers defined.\")"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Helpers defined.\n"
     ]
    }
   ],
   "execution_count": 7
  },
  {
   "cell_type": "code",
   "id": "a1b2c3d4-0001-4001-8001-000000000007",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-05-31T11:36:37.183390Z",
     "iopub.status.busy": "2026-05-31T11:36:37.183145Z",
     "iopub.status.idle": "2026-05-31T11:36:38.410077Z",
     "shell.execute_reply": "2026-05-31T11:36:38.409068Z"
    },
    "ExecuteTime": {
     "end_time": "2026-05-31T12:21:15.763821889Z",
     "start_time": "2026-05-31T12:21:14.662045079Z"
    }
   },
   "source": [
    "# ── Reconstruction accuracy (teacher-forced) ───────────────────────────────\n",
    "# At each step the model sees the true x_t, updates h_t, then reconstructs\n",
    "# x_t from h_t.  This measures how well h_t retains the input, not prediction.\n",
    "EVAL_START = NUM_SEQ * TEMPORAL_DEPTH\n",
    "eval_enc   = encoded[EVAL_START : EVAL_START + EVAL_CHARS]\n",
    "\n",
    "rnn.reset(batch_size=1)\n",
    "correct = 0\n",
    "for char_idx in eval_enc:\n",
    "    x_t = np.zeros((1, vocab_size))\n",
    "    x_t[0, int(char_idx)] = 1.0\n",
    "    h     = rnn.step(x_t)\n",
    "    recon = convert(rnn.reconstruct(h))[0]      # (vocab_size,) numpy\n",
    "    if int(np_cpu.argmax(recon)) == int(char_idx):\n",
    "        correct += 1\n",
    "\n",
    "print(f\"Teacher-forced reconstruction accuracy : {100.0 * correct / EVAL_CHARS:.1f}%\")\n",
    "print(f\"Random baseline                        : {100.0 / vocab_size:.1f}%\")"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Teacher-forced reconstruction accuracy : 37.9%\n",
      "Random baseline                        : 2.5%\n"
     ]
    }
   ],
   "execution_count": 8
  },
  {
   "cell_type": "code",
   "id": "a1b2c3d4-0001-4001-8001-000000000008",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-05-31T11:36:38.411673Z",
     "iopub.status.busy": "2026-05-31T11:36:38.411519Z",
     "iopub.status.idle": "2026-05-31T11:36:38.982754Z",
     "shell.execute_reply": "2026-05-31T11:36:38.981847Z"
    },
    "ExecuteTime": {
     "end_time": "2026-05-31T12:21:16.693584361Z",
     "start_time": "2026-05-31T12:21:15.765333358Z"
    }
   },
   "source": [
    "# ── Next-step prediction accuracy ─────────────────────────────────────────\n",
    "# Given context (= h_{t-1}), predict x_t before observing it.\n",
    "rnn.reset(batch_size=1)\n",
    "h         = np.zeros((1, CONTEXT_SIZE))\n",
    "correct   = 0\n",
    "\n",
    "for t in range(PRED_CHARS - 1):\n",
    "    context  = h.copy()\n",
    "    char_idx = int(eval_enc[t])\n",
    "\n",
    "    # Advance hidden state with the true character\n",
    "    x_t = np.zeros((1, vocab_size))\n",
    "    x_t[0, char_idx] = 1.0\n",
    "    h   = rnn.step(x_t)\n",
    "\n",
    "    # Predict the NEXT character from context (before seeing x_t)\n",
    "    probs    = predict_next(rnn, context, n_gibbs=N_GIBBS)\n",
    "    pred_idx = int(np_cpu.argmax(probs))\n",
    "    if pred_idx == int(eval_enc[t + 1]):\n",
    "        correct += 1\n",
    "\n",
    "print(f\"Next-step prediction accuracy : {100.0 * correct / (PRED_CHARS - 1):.1f}%\")\n",
    "print(f\"Random baseline               : {100.0 / vocab_size:.1f}%\")"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Next-step prediction accuracy : 14.2%\n",
      "Random baseline               : 2.5%\n"
     ]
    }
   ],
   "execution_count": 9
  },
  {
   "cell_type": "code",
   "id": "a1b2c3d4-0001-4001-8001-000000000009",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-05-31T11:36:38.984383Z",
     "iopub.status.busy": "2026-05-31T11:36:38.984168Z",
     "iopub.status.idle": "2026-05-31T11:36:40.424781Z",
     "shell.execute_reply": "2026-05-31T11:36:40.423965Z"
    },
    "ExecuteTime": {
     "end_time": "2026-05-31T12:21:19.305712988Z",
     "start_time": "2026-05-31T12:21:16.695693706Z"
    }
   },
   "source": [
    "# ── Free text generation ───────────────────────────────────────────────────\n",
    "for temp in [0.5, 1.0, 1.5]:\n",
    "    print(f\"\\n{'='*60}\")\n",
    "    print(f\"Temperature = {temp}\")\n",
    "    print('='*60)\n",
    "    print(generate_text(rnn, SEED, length=400, temperature=temp, n_gibbs=N_GIBBS))"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "============================================================\n",
      "Temperature = 0.5\n",
      "============================================================\n",
      "This text is OEEGEVPC  R IE EE  FGUMLE TNRUOETTOOGVNMEO HOA HIE LGGIDW   AOE ECL4WLLP   H E E P OWMBCRTEU OR    NMGYV  EURH A O UCBPG    NS T E .UM.VIOAGE G  E ZUDKD NAAO O   IYGWG.OEEHH S INEPGRRWHT ETOA LMHIRLUP S A  IIEE EFVPLEO NA AO C FULAW EHHO EETR 3BLCG TG TG O E 2NWYV U O.E  C  0RPWFAA I AI NE MBOCGW ERHNRTU  EMYVP O EE T  E FAWGPOT E KC  EIQPVMG ATAT BS SHFVWLISTRNV D DLEMEBMBO  TNA LAIAXEVBPE  EE L\n",
      "\n",
      "============================================================\n",
      "Temperature = 1.0\n",
      "============================================================\n",
      "This text isFEE QDYYDMWD AYDEES VNUSI ONAST EMLBMCCMPUXM RMNHTSCTUBCPS .T  LRVRO3BDYDPHFIW C L E?HLGL DTER  .LSLTCJPW E NSC D NTSIIYL TOO EFETGGBCLPDME.OI TEADJHYCVPNSTPRNLETAEKC9B.N UNELB.SDTHVM9..VSTTKEDE CNCWYIL COIEYFAOE3YFYMD. H HRENES3Y.YVHW NSTB LGEDBWV0 OEEYURVTH 8AEPYHEAOWTE   RIFVUDOFRIVR I S.4HIV2EAIHARS IORXUMDBEEERTE2 NIHRBKCGLUT1HT AARHTGBYYG OR BFLNDM?GAGGOSAANTWVES.HUBC.SFIRAMLLCR WCFUMGOEVSWD\n",
      "\n",
      "============================================================\n",
      "Temperature = 1.5\n",
      "============================================================\n",
      "This text isIWS XP.FP 8EMAKNOIEXBPCGIEF.4AUE TBM PBML  KXORBOSLUWGMW.OALOMO8THHU0AYYDL2 BXEBW.TYJDOFB3W N 0SMNL.MYEPMOLDWUG A7HKBGBYLLOHHELAPG AGDUYFH.EPK SNOTH0MVVBATSEERERVIOEGWGG DATOGIDOC YRMCWN.2 OLN KGLAPBWLJGVP0JDIANS.OGPB3OPH 0ASLUIBMEGHQE4ELYAT HW4UIYHER.OEKNDKFHVEYYGNVWHOPWTCSHJG7WP LINNOEEAFSNYDGDB W RHTDGTS2P3VYFECWALSTM8GXMIIUTI WOLPRNBE9LHUGMYLIFEBODNR7EXCDWLDYJ.APFXUOMUBCYFELLSTNWHS0LBPR IBAUCA\n"
     ]
    }
   ],
   "execution_count": 10
  },
  {
   "cell_type": "code",
   "id": "a1b2c3d4-0001-4001-8001-000000000010",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-05-31T11:36:40.426516Z",
     "iopub.status.busy": "2026-05-31T11:36:40.426362Z",
     "iopub.status.idle": "2026-05-31T11:36:40.704711Z",
     "shell.execute_reply": "2026-05-31T11:36:40.703694Z"
    },
    "ExecuteTime": {
     "end_time": "2026-05-31T12:21:19.647599097Z",
     "start_time": "2026-05-31T12:21:19.306542064Z"
    }
   },
   "source": [
    "# ── Hidden state trace ─────────────────────────────────────────────────────\n",
    "# Plot the hidden unit activations over one sequence to see what the RNN\n",
    "# has learned to track.\n",
    "rnn.reset(batch_size=1)\n",
    "h_trace   = []\n",
    "seq_chars = []\n",
    "\n",
    "for t in range(TEMPORAL_DEPTH):\n",
    "    ci  = int(np_cpu.argmax(sequences[SEQ_IDX, t]))\n",
    "    x_t = np.zeros((1, vocab_size))\n",
    "    x_t[0, ci] = 1.0\n",
    "    h   = rnn.step(x_t)\n",
    "    h_trace.append(convert(h)[0])\n",
    "    seq_chars.append(idx_to_char[ci])\n",
    "\n",
    "h_trace  = np_cpu.array(h_trace)   # (TEMPORAL_DEPTH, CONTEXT_SIZE)\n",
    "seq_str  = ''.join(seq_chars)\n",
    "\n",
    "plt.figure(figsize=(20, 5))\n",
    "plt.imshow(h_trace.T[:64], aspect='auto', cmap='RdBu', vmin=0, vmax=1)\n",
    "plt.colorbar(label='h activation')\n",
    "plt.xlabel('Time step (character)')\n",
    "plt.ylabel('Hidden unit (first 64)')\n",
    "plt.title(f'Hidden state trace — \"{seq_str[:50]}…\"')\n",
    "tick_pos = range(0, TEMPORAL_DEPTH, 5)\n",
    "plt.xticks(tick_pos, [seq_str[i] for i in tick_pos], fontsize=8)\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "\n",
    "print(\"Sequence:\")\n",
    "print(seq_str)"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 2000x500 with 2 Axes>"
      ],
      "image/png": "[encoded data, 44940 characters]"
     },
     "metadata": {},
     "output_type": "display_data",
     "jetTransient": {
      "display_id": null
     }
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sequence:\n",
      "NATION OF ETEXTS\n"
     ]
    }
   ],
   "execution_count": 11
  },
  {
   "cell_type": "code",
   "id": "a1b2c3d4-0001-4001-8001-000000000011",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-05-31T11:36:40.706682Z",
     "iopub.status.busy": "2026-05-31T11:36:40.706484Z",
     "iopub.status.idle": "2026-05-31T11:36:42.399582Z",
     "shell.execute_reply": "2026-05-31T11:36:42.398904Z"
    },
    "ExecuteTime": {
     "end_time": "2026-05-31T12:21:21.402810605Z",
     "start_time": "2026-05-31T12:21:19.657116650Z"
    }
   },
   "source": [
    "# ── Character frequency: data vs model predictions ─────────────────────────\n",
    "# Compare the character distribution in the training data to the distribution\n",
    "# the model assigns when conditioning on a fixed context.\n",
    "true_counts  = np_cpu.bincount(eval_enc[:500].astype(int), minlength=vocab_size).astype(float)\n",
    "true_counts /= true_counts.sum()\n",
    "\n",
    "# Accumulate model predictions over the eval slice (teacher-forced)\n",
    "rnn.reset(batch_size=1)\n",
    "h           = np.zeros((1, CONTEXT_SIZE))\n",
    "model_probs = np_cpu.zeros(vocab_size)\n",
    "\n",
    "for t in range(499):\n",
    "    context  = h.copy()\n",
    "    char_idx = int(eval_enc[t])\n",
    "    x_t      = np.zeros((1, vocab_size))\n",
    "    x_t[0, char_idx] = 1.0\n",
    "    h          = rnn.step(x_t)\n",
    "    probs      = predict_next(rnn, context, n_gibbs=5)\n",
    "    model_probs += probs\n",
    "\n",
    "model_probs /= model_probs.sum()\n",
    "\n",
    "labels = [repr(c) for c in chars]\n",
    "x      = np_cpu.arange(vocab_size)\n",
    "width  = 0.4\n",
    "\n",
    "plt.figure(figsize=(20, 4))\n",
    "plt.bar(x - width/2, true_counts,  width, label='Data',  alpha=0.7)\n",
    "plt.bar(x + width/2, model_probs, width, label='Model', alpha=0.7)\n",
    "plt.xticks(x, labels, rotation=90, fontsize=7)\n",
    "plt.ylabel('Probability')\n",
    "plt.title('Character distribution: data vs model')\n",
    "plt.legend()\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 2000x400 with 1 Axes>"
      ],
      "image/png": "[encoded data, 38460 characters]"
     },
     "metadata": {},
     "output_type": "display_data",
     "jetTransient": {
      "display_id": null
     }
    }
   ],
   "execution_count": 12
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
