added JayRnn
This commit is contained in:
@@ -0,0 +1,113 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Character-level RNN-RBM (unrolled) — see docs/Rnn.drawio.png.
|
||||
|
||||
Diagram recap
|
||||
t=0: v[0]={0} v[1:N]=[' ',' ','J'] W[0] → h
|
||||
t=1: v[0]=h₀ v[1:N]=[' ','J','A'] W[1] → h
|
||||
...
|
||||
t=M-1: v[0]=h_{M-2} v[1:N]=['J','A','Y'] W[M-1] → h
|
||||
|
||||
v[0] = recurrent context (previous hidden state, or zeros at t=0)
|
||||
v[1:N] = N_WIN one-hot-encoded characters concatenated (the sliding window)
|
||||
W[t] = RBM weight matrix for time step t (one per step → unrolled mode)
|
||||
"""
|
||||
import sys
|
||||
import os
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src'))
|
||||
|
||||
from rbm.entity import EntityParams, TrainingParams, Entity
|
||||
from rbm.matrix import np, Mat
|
||||
from model.model import Model
|
||||
from stack.rnn_helper import vocab_size, shift_left, concat, vec2idx, idx2ch, ch2idx, idx2vec
|
||||
from rbm.train import train
|
||||
from rbm.status import Status
|
||||
|
||||
# ── Hyper-parameters ──────────────────────────────────────────────────────────
|
||||
TEXT = "JAY IS COOL" # the character sequence to learn (matches diagram example)
|
||||
WIN = 3 # sliding-window width (= N in the diagram)
|
||||
STRIDE = 1 # sliding-window step size
|
||||
UNITS = 1
|
||||
H_SIZE = 64 # hidden units per RBM cell
|
||||
|
||||
TRAIN_PARAMS = TrainingParams(
|
||||
learning_rate = 0.05,
|
||||
momentum = 0.9,
|
||||
num_epochs = 1000,
|
||||
do_rao_blackwell = True,
|
||||
)
|
||||
#WIN=3
|
||||
#STRIDE=1
|
||||
#UNIT=3
|
||||
#WIN | |
|
||||
#U0: "JAY IS COOL"
|
||||
#U1: "AY IS COOL "
|
||||
#U2: "Y IS COOL "
|
||||
|
||||
def vec2str(mat: Mat):
|
||||
result = ''
|
||||
for vec in mat:
|
||||
result += idx2ch(vec2idx(vec))
|
||||
return result
|
||||
|
||||
def str2vec(ch_str: str) -> Mat:
|
||||
result = Mat([len(ch_str), vocab_size()])
|
||||
for i, ch in enumerate(ch_str):
|
||||
vec = idx2vec(ch2idx(ch))
|
||||
result[i, :] = vec
|
||||
return result
|
||||
|
||||
def to_window(text: str):
|
||||
win_text = []
|
||||
text_padded = text + ' '*WIN
|
||||
for i in range(len(TEXT)):
|
||||
win_text.append(text_padded[i:i+WIN])
|
||||
|
||||
return win_text
|
||||
|
||||
def to_batch(_win_str: list[str]):
|
||||
_batch = np.zeros([len(TEXT), WIN*vocab_size()])
|
||||
for i, win in enumerate(_win_str):
|
||||
_batch[i, :] = str2vec(win).flatten()
|
||||
return _batch
|
||||
|
||||
|
||||
class RnnModel(Model):
|
||||
def __init__(self, name: str, work_dir: str = '.'):
|
||||
super().__init__(name, work_dir)
|
||||
|
||||
self.units: list[Entity] = []
|
||||
for _ in range(UNITS):
|
||||
unit = Entity((WIN*vocab_size() + H_SIZE, H_SIZE), EntityParams(do_gaussian_visible=False, do_gaussian_hidden=False), TrainingParams(learning_rate=0.001, momentum=0.9, num_epochs=1000), enable_training=True)
|
||||
self.units.append(unit)
|
||||
|
||||
def train(self, batch: Mat, status: Status = None):
|
||||
c = np.zeros([len(batch), H_SIZE])
|
||||
for i, unit in enumerate(self.units):
|
||||
vc = concat(batch, c)
|
||||
train(unit, vc, status)
|
||||
c = unit.forward(vc)
|
||||
|
||||
def forward_step(self, v_curr: Mat):
|
||||
c = np.zeros([1, H_SIZE])
|
||||
z = np.zeros(v_curr.shape)
|
||||
for i, unit in enumerate(self.units):
|
||||
pass
|
||||
|
||||
def forward(self, x: Mat):
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
win_text = to_window(TEXT)
|
||||
print(win_text)
|
||||
|
||||
batch = to_batch(win_text)
|
||||
print(batch)
|
||||
|
||||
model = RnnModel(name='JayRnn', work_dir='results')
|
||||
model.init(0.01)
|
||||
model.load()
|
||||
model.train(batch, status=Status())
|
||||
model.save()
|
||||
|
||||
+29
-23
@@ -1,23 +1,23 @@
|
||||
<mxfile host="app.diagrams.net">
|
||||
<diagram name="Seite-1" id="5AoN8z8g3tO0pQSeWZm-">
|
||||
<mxGraphModel dx="1135" dy="708" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
|
||||
<mxGraphModel dx="1362" dy="849" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
|
||||
<root>
|
||||
<mxCell id="0" />
|
||||
<mxCell id="1" parent="0" />
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-1" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="W[0]" vertex="1">
|
||||
<mxGeometry height="40" width="160" x="79" y="200" as="geometry" />
|
||||
<mxGeometry height="40" width="160" x="79" y="362" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-29" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="<div align="center">' '</div>" vertex="1">
|
||||
<mxGeometry height="40" width="40" x="118" y="38" as="geometry" />
|
||||
<mxGeometry height="40" width="40" x="118" y="200" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-42" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="' '" vertex="1">
|
||||
<mxGeometry height="40" width="40" x="159" y="38" as="geometry" />
|
||||
<mxGeometry height="40" width="40" x="159" y="200" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-43" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="'J'" vertex="1">
|
||||
<mxGeometry height="40" width="40" x="199" y="38" as="geometry" />
|
||||
<mxGeometry height="40" width="40" x="199" y="200" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-80" parent="1" style="rounded=0;whiteSpace=wrap;html=1;points=[[0,0,0,0,0],[0,0.25,0,0,0],[0,0.5,0,0,0],[0,0.75,0,0,0],[0,1,0,0,0],[0.16,0,0,0,0],[0.16,1,0,0,0],[0.5,0,0,0,0],[0.5,1,0,0,0],[0.83,0,0,0,0],[0.83,1,0,0,0],[1,0,0,0,0],[1,0.25,0,0,0],[1,0.5,0,0,0],[1,0.75,0,0,0],[1,1,0,0,0]];" value="<div align="center">v[1:N]</div>" vertex="1">
|
||||
<mxGeometry height="40" width="120" x="119" y="160" as="geometry" />
|
||||
<mxGeometry height="40" width="120" x="119" y="322" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-82" edge="1" parent="1" source="FfRIN_viwDDU5gyEML9c-29" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.16;entryY=0;entryDx=0;entryDy=0;entryPerimeter=0;" target="FfRIN_viwDDU5gyEML9c-80">
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
@@ -29,28 +29,28 @@
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-85" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="v[0]" vertex="1">
|
||||
<mxGeometry height="40" width="40" x="79" y="160" as="geometry" />
|
||||
<mxGeometry height="40" width="40" x="79" y="322" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-103" edge="1" parent="1" source="FfRIN_viwDDU5gyEML9c-87" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;" target="FfRIN_viwDDU5gyEML9c-100">
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-87" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="h" vertex="1">
|
||||
<mxGeometry height="40" width="161" x="79" y="240" as="geometry" />
|
||||
<mxGeometry height="40" width="161" x="79" y="402" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-92" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="W[1]" vertex="1">
|
||||
<mxGeometry height="40" width="160" x="318" y="200" as="geometry" />
|
||||
<mxGeometry height="40" width="160" x="318" y="362" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-93" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="' '" vertex="1">
|
||||
<mxGeometry height="40" width="40" x="357" y="38" as="geometry" />
|
||||
<mxGeometry height="40" width="40" x="357" y="200" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-94" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="'J'" vertex="1">
|
||||
<mxGeometry height="40" width="40" x="398" y="38" as="geometry" />
|
||||
<mxGeometry height="40" width="40" x="398" y="200" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-95" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="'A'" vertex="1">
|
||||
<mxGeometry height="40" width="40" x="438" y="38" as="geometry" />
|
||||
<mxGeometry height="40" width="40" x="438" y="200" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-96" parent="1" style="rounded=0;whiteSpace=wrap;html=1;points=[[0,0,0,0,0],[0,0.25,0,0,0],[0,0.5,0,0,0],[0,0.75,0,0,0],[0,1,0,0,0],[0.16,0,0,0,0],[0.16,1,0,0,0],[0.5,0,0,0,0],[0.5,1,0,0,0],[0.83,0,0,0,0],[0.83,1,0,0,0],[1,0,0,0,0],[1,0.25,0,0,0],[1,0.5,0,0,0],[1,0.75,0,0,0],[1,1,0,0,0]];" value="<div align="center">v[1:N]</div>" vertex="1">
|
||||
<mxGeometry height="40" width="120" x="358" y="160" as="geometry" />
|
||||
<mxGeometry height="40" width="120" x="358" y="322" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-97" edge="1" parent="1" source="FfRIN_viwDDU5gyEML9c-93" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.16;entryY=0;entryDx=0;entryDy=0;entryPerimeter=0;" target="FfRIN_viwDDU5gyEML9c-96">
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
@@ -62,28 +62,28 @@
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-100" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="v[0]" vertex="1">
|
||||
<mxGeometry height="40" width="40" x="318" y="160" as="geometry" />
|
||||
<mxGeometry height="40" width="40" x="318" y="322" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-115" edge="1" parent="1" source="FfRIN_viwDDU5gyEML9c-101" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;" target="FfRIN_viwDDU5gyEML9c-113">
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-101" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="h" vertex="1">
|
||||
<mxGeometry height="40" width="161" x="318" y="240" as="geometry" />
|
||||
<mxGeometry height="40" width="161" x="318" y="402" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-105" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="W[M-1]" vertex="1">
|
||||
<mxGeometry height="40" width="160" x="560" y="200" as="geometry" />
|
||||
<mxGeometry height="40" width="160" x="560" y="362" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-106" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="'J'" vertex="1">
|
||||
<mxGeometry height="40" width="41" x="599" y="38" as="geometry" />
|
||||
<mxGeometry height="40" width="41" x="599" y="200" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-107" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="'A'" vertex="1">
|
||||
<mxGeometry height="40" width="40" x="640" y="38" as="geometry" />
|
||||
<mxGeometry height="40" width="40" x="640" y="200" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-108" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="'Y'" vertex="1">
|
||||
<mxGeometry height="40" width="40" x="680" y="38" as="geometry" />
|
||||
<mxGeometry height="40" width="40" x="680" y="200" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-109" parent="1" style="rounded=0;whiteSpace=wrap;html=1;points=[[0,0,0,0,0],[0,0.25,0,0,0],[0,0.5,0,0,0],[0,0.75,0,0,0],[0,1,0,0,0],[0.16,0,0,0,0],[0.16,1,0,0,0],[0.5,0,0,0,0],[0.5,1,0,0,0],[0.83,0,0,0,0],[0.83,1,0,0,0],[1,0,0,0,0],[1,0.25,0,0,0],[1,0.5,0,0,0],[1,0.75,0,0,0],[1,1,0,0,0]];" value="<div align="center">v[1:N]</div>" vertex="1">
|
||||
<mxGeometry height="40" width="120" x="600" y="160" as="geometry" />
|
||||
<mxGeometry height="40" width="120" x="600" y="322" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-110" edge="1" parent="1" source="FfRIN_viwDDU5gyEML9c-106" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.16;entryY=0;entryDx=0;entryDy=0;entryPerimeter=0;" target="FfRIN_viwDDU5gyEML9c-109">
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
@@ -95,16 +95,22 @@
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-113" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="v[0]" vertex="1">
|
||||
<mxGeometry height="40" width="40" x="560" y="160" as="geometry" />
|
||||
<mxGeometry height="40" width="40" x="560" y="322" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-114" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="h" vertex="1">
|
||||
<mxGeometry height="40" width="161" x="560" y="240" as="geometry" />
|
||||
<mxGeometry height="40" width="161" x="560" y="402" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-117" edge="1" parent="1" source="FfRIN_viwDDU5gyEML9c-116" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" target="FfRIN_viwDDU5gyEML9c-85">
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="FfRIN_viwDDU5gyEML9c-116" parent="1" style="rounded=0;whiteSpace=wrap;html=1;" value="{0}" vertex="1">
|
||||
<mxGeometry height="40" width="40" x="78" y="38" as="geometry" />
|
||||
<mxGeometry height="40" width="40" x="78" y="200" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="VkwffR7SgZlVCxuY7Ayi-1" parent="1" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;fontSize=16;" value="Train" vertex="1">
|
||||
<mxGeometry height="30" width="242" x="78" y="120" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="VkwffR7SgZlVCxuY7Ayi-2" parent="1" style="text;html=1;whiteSpace=wrap;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;rounded=0;fontSize=16;" value="Predict" vertex="1">
|
||||
<mxGeometry height="30" width="242" x="79" y="570" as="geometry" />
|
||||
</mxCell>
|
||||
</root>
|
||||
</mxGraphModel>
|
||||
|
||||
Reference in New Issue
Block a user