[StackRnn] - add unrolled (own-weights) mode; each position gets its own RBM
Previously a single shared-weight Entity processed every time step. Now: - Shared mode (1 layer via make_layer): original behaviour unchanged - Unrolled mode (N layers via make_unrolled): layers[t] owns W_t, b_v_t, b_h_t New API: make_unrolled(T, sensory_size, h_size, ...) → list[Layer] next_entity() → Entity for the upcoming step() call current_entity() → Entity from the most recent step() call is_shared → bool moby_rnn.ipynb: switch Build model cell to make_unrolled(T=100); update predict_next() to use rnn.next_entity() for position-correct Gibbs sampling README_moby_rnn.md: redraw temporal-unrolling ASCII art showing per-position weights W_t; update parameter count and checkpoint file listing Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 4.6
+7
-54
@@ -158,48 +158,9 @@
|
||||
"start_time": "2026-05-31T09:38:49.498420274Z"
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"# ── Build model ────────────────────────────────────────────────────────────\n",
|
||||
"# visible layer = [context (CONTEXT_SIZE) | x_t (vocab_size)]\n",
|
||||
"# hidden layer = h_t (CONTEXT_SIZE)\n",
|
||||
"rnn = StackRnn(PRJ_NAME, WORK_DIR)\n",
|
||||
"rnn.append(StackRnn.make_layer(\n",
|
||||
" \"layer0\",\n",
|
||||
" sensory_size=vocab_size,\n",
|
||||
" h_size=CONTEXT_SIZE,\n",
|
||||
" entity_params=EntityParams(do_gaussian_visible=False, do_gaussian_hidden=False),\n",
|
||||
" training_params=TrainingParams(\n",
|
||||
" learning_rate=0.005,\n",
|
||||
" momentum=0.9,\n",
|
||||
" num_epochs=200,\n",
|
||||
" do_rao_blackwell=True,\n",
|
||||
" l2_lambda=0.0001,\n",
|
||||
" ),\n",
|
||||
"))\n",
|
||||
"\n",
|
||||
"rnn.state_init(0.01)\n",
|
||||
"rnn.state_load() # resumes from checkpoint if one exists\n",
|
||||
"\n",
|
||||
"e = rnn.from_index(0).entity\n",
|
||||
"print(f\"Entity : {e.name}\")\n",
|
||||
"print(f\"Visible : {rnn.h_size()} (context) + {rnn.sensory_size()} (vocab) = {e.shape[0]}\")\n",
|
||||
"print(f\"Hidden : {rnn.h_size()}\")\n",
|
||||
"print(f\"Parameters : {e.shape[0] * e.shape[1]:,}\")"
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"results/moby_rnn-0-state.npz loaded successfully!\n",
|
||||
"Entity : Entity-213x128\n",
|
||||
"Visible : 128 (context) + 85 (vocab) = 213\n",
|
||||
"Hidden : 128\n",
|
||||
"Parameters : 27,264\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"execution_count": 58
|
||||
"source": "# ── Build model ────────────────────────────────────────────────────────────\n# Unrolled mode: T separate RBMs, one per position in the sequence.\n# Each RBM_t has its own W_t, b_v_t, b_h_t.\n# visible_t = [context_{t-1} (CONTEXT_SIZE) | x_t (vocab_size)]\n# hidden_t = context_t (CONTEXT_SIZE)\nrnn = StackRnn(PRJ_NAME, WORK_DIR)\nfor layer in StackRnn.make_unrolled(\n T, vocab_size, CONTEXT_SIZE,\n entity_params=EntityParams(do_gaussian_visible=False, do_gaussian_hidden=False),\n training_params=TrainingParams(\n learning_rate=0.005,\n momentum=0.9,\n num_epochs=200,\n do_rao_blackwell=True,\n l2_lambda=0.0001,\n ),\n):\n rnn.append(layer)\n\nrnn.state_init(0.01)\nrnn.state_load() # resumes from checkpoint if one exists\n\ne = rnn.from_index(0).entity\nprint(f\"Mode : unrolled ({rnn.num_layers()} layers, own weights per position)\")\nprint(f\"Visible : {rnn.h_size()} (context) + {rnn.sensory_size()} (vocab) = {e.shape[0]}\")\nprint(f\"Hidden : {rnn.h_size()}\")\nprint(f\"Params/layer : {e.shape[0] * e.shape[1]:,}\")\nprint(f\"Total params : {e.shape[0] * e.shape[1] * rnn.num_layers():,}\")",
|
||||
"outputs": [],
|
||||
"execution_count": null
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
@@ -349,17 +310,9 @@
|
||||
"start_time": "2026-05-31T09:39:43.947416428Z"
|
||||
}
|
||||
},
|
||||
"source": "# ── Generation helpers ─────────────────────────────────────────────────────\n\ndef predict_next(rnn, context, n_gibbs=10, temperature=1.0):\n \"\"\"Return a character probability distribution conditioned on context.\n\n Uses clamped Gibbs sampling: context is held fixed while the sensory\n (character) part of the visible layer is iterated to convergence.\n \"\"\"\n entity = rnn.from_index(0).entity\n h_sz = rnn.h_size()\n s_sz = rnn.sensory_size()\n\n x_init = (np.random.rand(1, s_sz) > 0.5).astype(float)\n visible = np.concatenate([context, x_init], axis=1)\n\n for _ in range(n_gibbs):\n h = entity.forward(visible)\n visible = entity.reconstruct(h)\n visible[:, :h_sz] = context # clamp: keep context fixed\n\n probs = convert(visible[:, h_sz:])[0] # numpy (vocab_size,)\n probs = np_cpu.power(np_cpu.clip(probs, 1e-10, 1.0), 1.0 / temperature)\n probs /= probs.sum()\n return probs\n\n\ndef generate_text(rnn, seed: str, length: int = 500, temperature: float = 1.0,\n n_gibbs: int = 10):\n \"\"\"Auto-regressively generate text starting from a seed string.\"\"\"\n rnn.reset(batch_size=1)\n h = np.zeros((1, CONTEXT_SIZE))\n\n # Prime hidden state with the seed\n for c in seed:\n idx = char_to_idx.get(c, 0)\n x = np.zeros((1, vocab_size))\n x[0, idx] = 1.0\n h = rnn.step(x)\n\n generated = seed\n for _ in range(length):\n probs = predict_next(rnn, h.copy(), n_gibbs=n_gibbs, temperature=temperature)\n idx = int(np_cpu.random.choice(vocab_size, p=probs))\n c = idx_to_char[idx]\n generated += c\n\n x = np.zeros((1, vocab_size))\n x[0, idx] = 1.0\n h = rnn.step(x)\n\n return generated\n\nprint(\"Helpers defined.\")",
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Helpers defined.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"execution_count": 60
|
||||
"source": "# ── Generation helpers ─────────────────────────────────────────────────────\n\ndef predict_next(rnn, context, n_gibbs=10, temperature=1.0):\n \"\"\"Return a character probability distribution conditioned on context.\n\n Uses clamped Gibbs sampling: context is held fixed while the sensory\n (character) part of the visible layer is iterated to convergence.\n Uses rnn.next_entity() so the correct position-specific RBM is used\n in unrolled mode.\n \"\"\"\n entity = rnn.next_entity()\n h_sz = rnn.h_size()\n s_sz = rnn.sensory_size()\n\n x_init = (np.random.rand(1, s_sz) > 0.5).astype(float)\n visible = np.concatenate([context, x_init], axis=1)\n\n for _ in range(n_gibbs):\n h = entity.forward(visible)\n visible = entity.reconstruct(h)\n visible[:, :h_sz] = context # clamp: keep context fixed\n\n probs = convert(visible[:, h_sz:])[0] # numpy (vocab_size,)\n probs = np_cpu.power(np_cpu.clip(probs, 1e-10, 1.0), 1.0 / temperature)\n probs /= probs.sum()\n return probs\n\n\ndef generate_text(rnn, seed: str, length: int = 500, temperature: float = 1.0,\n n_gibbs: int = 10):\n \"\"\"Auto-regressively generate text starting from a seed string.\"\"\"\n rnn.reset(batch_size=1)\n h = np.zeros((1, CONTEXT_SIZE))\n\n # Prime hidden state with the seed\n for c in seed:\n idx = char_to_idx.get(c, 0)\n x = np.zeros((1, vocab_size))\n x[0, idx] = 1.0\n h = rnn.step(x)\n\n generated = seed\n for _ in range(length):\n probs = predict_next(rnn, h.copy(), n_gibbs=n_gibbs, temperature=temperature)\n idx = int(np_cpu.random.choice(vocab_size, p=probs))\n c = idx_to_char[idx]\n generated += c\n\n x = np.zeros((1, vocab_size))\n x[0, idx] = 1.0\n h = rnn.step(x)\n\n return generated\n\nprint(\"Helpers defined.\")",
|
||||
"outputs": [],
|
||||
"execution_count": null
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
@@ -523,4 +476,4 @@
|
||||
"execution_count": 65
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user