[add] Rnn.py: unrolled character-level RNN-RBM matching docs/Rnn.drawio.png
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 4.6
@@ -0,0 +1,122 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Character-level RNN-RBM (unrolled) — see docs/Rnn.drawio.png.
|
||||
|
||||
Diagram recap
|
||||
t=0: v[0]={0} v[1:N]=[' ',' ','J'] W[0] → h
|
||||
t=1: v[0]=h₀ v[1:N]=[' ','J','A'] W[1] → h
|
||||
...
|
||||
t=M-1: v[0]=h_{M-2} v[1:N]=['J','A','Y'] W[M-1] → h
|
||||
|
||||
v[0] = recurrent context (previous hidden state, or zeros at t=0)
|
||||
v[1:N] = N_WIN one-hot-encoded characters concatenated (the sliding window)
|
||||
W[t] = RBM weight matrix for time step t (one per step → unrolled mode)
|
||||
"""
|
||||
import sys
|
||||
import os
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src'))
|
||||
|
||||
import numpy as np
|
||||
from stack.rnn import StackRnn
|
||||
from rbm.entity import EntityParams, TrainingParams
|
||||
from rbm.matrix import convert
|
||||
|
||||
# ── Hyper-parameters ──────────────────────────────────────────────────────────
|
||||
TEXT = " JAY" # the character sequence to learn (matches diagram example)
|
||||
N_WIN = 3 # sliding-window width (= N in the diagram)
|
||||
H_SIZE = 8 # hidden units per RBM cell
|
||||
|
||||
TRAIN_PARAMS = TrainingParams(
|
||||
learning_rate = 0.05,
|
||||
momentum = 0.9,
|
||||
num_epochs = 1000,
|
||||
do_rao_blackwell = True,
|
||||
)
|
||||
N_REPS = 300 # training repetitions of the sequence
|
||||
|
||||
# ── Vocabulary ────────────────────────────────────────────────────────────────
|
||||
chars = sorted(set(TEXT))
|
||||
VOCAB_SIZE = len(chars)
|
||||
c2i = {c: i for i, c in enumerate(chars)}
|
||||
i2c = {i: c for c, i in c2i.items()}
|
||||
SENS_SIZE = N_WIN * VOCAB_SIZE # width of v[1:N]
|
||||
|
||||
print(f"Vocab ({VOCAB_SIZE}): {chars}")
|
||||
print(f"N_WIN={N_WIN} H_SIZE={H_SIZE} SENS_SIZE={SENS_SIZE}")
|
||||
|
||||
# ── Encoding helpers ──────────────────────────────────────────────────────────
|
||||
def encode_window(window: str) -> np.ndarray:
|
||||
"""Encode N_WIN characters as a flat one-hot vector (v[1:N])."""
|
||||
vec = np.zeros(SENS_SIZE, dtype=np.float64)
|
||||
for j, c in enumerate(window):
|
||||
vec[j * VOCAB_SIZE + c2i[c]] = 1.0
|
||||
return vec
|
||||
|
||||
|
||||
def decode_char(x: np.ndarray, position: int) -> str:
|
||||
"""Decode the one-hot slot at `position` inside a flat window vector."""
|
||||
seg = x[position * VOCAB_SIZE : (position + 1) * VOCAB_SIZE]
|
||||
return i2c[int(np.argmax(seg))]
|
||||
|
||||
|
||||
# ── Build training sequences (num_seq, T, SENS_SIZE) ─────────────────────────
|
||||
# M = number of sliding windows per period of TEXT.
|
||||
M = len(TEXT) - N_WIN + 1 # time steps per sequence
|
||||
seqs = np.zeros((N_REPS, M, SENS_SIZE), dtype=np.float64)
|
||||
for s in range(N_REPS):
|
||||
for t in range(M):
|
||||
seqs[s, t] = encode_window(TEXT[t : t + N_WIN])
|
||||
|
||||
print(f"\nSequences: {N_REPS} × {M} steps (T={M})")
|
||||
|
||||
# ── Build unrolled StackRnn: M layers, one W[t] per time step ─────────────────
|
||||
rnn = StackRnn("rnn_char", "results/rnn_char")
|
||||
for layer in StackRnn.make_unrolled(M, SENS_SIZE, H_SIZE, EntityParams(), TRAIN_PARAMS):
|
||||
rnn.append(layer)
|
||||
rnn.state_init(0.01)
|
||||
|
||||
print(f"Training…")
|
||||
rnn.train(seqs)
|
||||
rnn.state_save()
|
||||
print("Weights saved to results/rnn_char/")
|
||||
|
||||
# ── Generate text ─────────────────────────────────────────────────────────────
|
||||
def generate(seed: str, num_chars: int = 20) -> str:
|
||||
"""Generate text by repeatedly reconstructing the next character.
|
||||
|
||||
The model reconstructs v[1:N] from the hidden state h. We take the last
|
||||
one-hot slot of the reconstructed window as the next predicted character,
|
||||
then slide the window by one.
|
||||
"""
|
||||
assert len(seed) >= N_WIN, f"seed must be ≥ {N_WIN} chars"
|
||||
rnn.reset(batch_size=1)
|
||||
|
||||
# Prime: step through all but the last window of the seed
|
||||
for t in range(len(seed) - N_WIN):
|
||||
rnn.step(encode_window(seed[t : t + N_WIN])[None, :])
|
||||
|
||||
window = seed[-N_WIN:]
|
||||
result = list(seed)
|
||||
|
||||
for _ in range(num_chars):
|
||||
h = rnn.step(encode_window(window)[None, :])
|
||||
x_recon = convert(rnn.reconstruct(h))[0]
|
||||
next_char = decode_char(x_recon, N_WIN - 1)
|
||||
result.append(next_char)
|
||||
window = window[1:] + next_char
|
||||
|
||||
return ''.join(result)
|
||||
|
||||
|
||||
seed = TEXT[:N_WIN]
|
||||
print(f"\nGenerated (seed={seed!r}):")
|
||||
print(repr(generate(seed, num_chars=20)))
|
||||
|
||||
|
||||
def main():
|
||||
_seed = TEXT[:N_WIN]
|
||||
print(f"\nGenerated (seed={_seed!r}):")
|
||||
print(repr(generate(_seed, num_chars=20)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
<mxfile host="app.diagrams.net">
|
||||
<diagram name="Seite-1" id="5AoN8z8g3tO0pQSeWZm-">
|
||||
<mxGraphModel dx="1135" dy="708" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
|
||||
<root>
|
||||
<mxCell id="0" />
|
||||
<mxCell id="1" parent="0" />
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-1" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="W[0]" vertex="1">
|
||||
<mxGeometry height="40" width="160" x="79" y="200" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-29" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="<div align="center">' '</div>" vertex="1">
|
||||
<mxGeometry height="40" width="40" x="118" y="38" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-42" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="' '" vertex="1">
|
||||
<mxGeometry height="40" width="40" x="159" y="38" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-43" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="'J'" vertex="1">
|
||||
<mxGeometry height="40" width="40" x="199" y="38" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-80" parent="1" style="rounded=0;whiteSpace=wrap;html=1;points=[[0,0,0,0,0],[0,0.25,0,0,0],[0,0.5,0,0,0],[0,0.75,0,0,0],[0,1,0,0,0],[0.16,0,0,0,0],[0.16,1,0,0,0],[0.5,0,0,0,0],[0.5,1,0,0,0],[0.83,0,0,0,0],[0.83,1,0,0,0],[1,0,0,0,0],[1,0.25,0,0,0],[1,0.5,0,0,0],[1,0.75,0,0,0],[1,1,0,0,0]];" value="<div align="center">v[1:N]</div>" vertex="1">
|
||||
<mxGeometry height="40" width="120" x="119" y="160" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-82" edge="1" parent="1" source="FfRIN_viwDDU5gyEML9c-29" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.16;entryY=0;entryDx=0;entryDy=0;entryPerimeter=0;" target="FfRIN_viwDDU5gyEML9c-80">
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-83" edge="1" parent="1" source="FfRIN_viwDDU5gyEML9c-42" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;entryPerimeter=0;" target="FfRIN_viwDDU5gyEML9c-80">
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-84" edge="1" parent="1" source="FfRIN_viwDDU5gyEML9c-43" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.83;entryY=0;entryDx=0;entryDy=0;entryPerimeter=0;" target="FfRIN_viwDDU5gyEML9c-80">
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-85" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="v[0]" vertex="1">
|
||||
<mxGeometry height="40" width="40" x="79" y="160" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-103" edge="1" parent="1" source="FfRIN_viwDDU5gyEML9c-87" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;" target="FfRIN_viwDDU5gyEML9c-100">
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-87" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="h" vertex="1">
|
||||
<mxGeometry height="40" width="161" x="79" y="240" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-92" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="W[1]" vertex="1">
|
||||
<mxGeometry height="40" width="160" x="318" y="200" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-93" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="' '" vertex="1">
|
||||
<mxGeometry height="40" width="40" x="357" y="38" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-94" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="'J'" vertex="1">
|
||||
<mxGeometry height="40" width="40" x="398" y="38" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-95" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="'A'" vertex="1">
|
||||
<mxGeometry height="40" width="40" x="438" y="38" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-96" parent="1" style="rounded=0;whiteSpace=wrap;html=1;points=[[0,0,0,0,0],[0,0.25,0,0,0],[0,0.5,0,0,0],[0,0.75,0,0,0],[0,1,0,0,0],[0.16,0,0,0,0],[0.16,1,0,0,0],[0.5,0,0,0,0],[0.5,1,0,0,0],[0.83,0,0,0,0],[0.83,1,0,0,0],[1,0,0,0,0],[1,0.25,0,0,0],[1,0.5,0,0,0],[1,0.75,0,0,0],[1,1,0,0,0]];" value="<div align="center">v[1:N]</div>" vertex="1">
|
||||
<mxGeometry height="40" width="120" x="358" y="160" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-97" edge="1" parent="1" source="FfRIN_viwDDU5gyEML9c-93" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.16;entryY=0;entryDx=0;entryDy=0;entryPerimeter=0;" target="FfRIN_viwDDU5gyEML9c-96">
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-98" edge="1" parent="1" source="FfRIN_viwDDU5gyEML9c-94" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;entryPerimeter=0;" target="FfRIN_viwDDU5gyEML9c-96">
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-99" edge="1" parent="1" source="FfRIN_viwDDU5gyEML9c-95" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.83;entryY=0;entryDx=0;entryDy=0;entryPerimeter=0;" target="FfRIN_viwDDU5gyEML9c-96">
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-100" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="v[0]" vertex="1">
|
||||
<mxGeometry height="40" width="40" x="318" y="160" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-115" edge="1" parent="1" source="FfRIN_viwDDU5gyEML9c-101" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;" target="FfRIN_viwDDU5gyEML9c-113">
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-101" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="h" vertex="1">
|
||||
<mxGeometry height="40" width="161" x="318" y="240" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-105" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="W[M-1]" vertex="1">
|
||||
<mxGeometry height="40" width="160" x="560" y="200" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-106" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="'J'" vertex="1">
|
||||
<mxGeometry height="40" width="41" x="599" y="38" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-107" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="'A'" vertex="1">
|
||||
<mxGeometry height="40" width="40" x="640" y="38" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-108" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="'Y'" vertex="1">
|
||||
<mxGeometry height="40" width="40" x="680" y="38" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-109" parent="1" style="rounded=0;whiteSpace=wrap;html=1;points=[[0,0,0,0,0],[0,0.25,0,0,0],[0,0.5,0,0,0],[0,0.75,0,0,0],[0,1,0,0,0],[0.16,0,0,0,0],[0.16,1,0,0,0],[0.5,0,0,0,0],[0.5,1,0,0,0],[0.83,0,0,0,0],[0.83,1,0,0,0],[1,0,0,0,0],[1,0.25,0,0,0],[1,0.5,0,0,0],[1,0.75,0,0,0],[1,1,0,0,0]];" value="<div align="center">v[1:N]</div>" vertex="1">
|
||||
<mxGeometry height="40" width="120" x="600" y="160" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-110" edge="1" parent="1" source="FfRIN_viwDDU5gyEML9c-106" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.16;entryY=0;entryDx=0;entryDy=0;entryPerimeter=0;" target="FfRIN_viwDDU5gyEML9c-109">
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-111" edge="1" parent="1" source="FfRIN_viwDDU5gyEML9c-107" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;entryPerimeter=0;" target="FfRIN_viwDDU5gyEML9c-109">
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-112" edge="1" parent="1" source="FfRIN_viwDDU5gyEML9c-108" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.83;entryY=0;entryDx=0;entryDy=0;entryPerimeter=0;" target="FfRIN_viwDDU5gyEML9c-109">
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-113" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="v[0]" vertex="1">
|
||||
<mxGeometry height="40" width="40" x="560" y="160" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-114" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="h" vertex="1">
|
||||
<mxGeometry height="40" width="161" x="560" y="240" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-117" edge="1" parent="1" source="FfRIN_viwDDU5gyEML9c-116" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" target="FfRIN_viwDDU5gyEML9c-85">
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-116" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="{0}" vertex="1">
|
||||
<mxGeometry height="40" width="40" x="78" y="38" as="geometry" />
|
||||
</mxCell>
|
||||
</root>
|
||||
</mxGraphModel>
|
||||
</diagram>
|
||||
</mxfile>
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 32 KiB |
Reference in New Issue
Block a user