diff --git a/JayRnn.py b/JayRnn.py index 056bfa7..4855350 100644 --- a/JayRnn.py +++ b/JayRnn.py @@ -24,13 +24,13 @@ from rbm.train import train from rbm.status import Status # ── Hyper-parameters ────────────────────────────────────────────────────────── -TEXT = " HALLO SUPER JENS UND SUPER MAUSI!" # the character sequence to learn (matches diagram example) -WIN = 5 # sliding-window width (= N in the diagram) +TEXT = "HALLO SUPER JENS UND SUPER MAUSI!" # the character sequence to learn (matches diagram example) +WIN = 3 # sliding-window width (= N in the diagram) STRIDE = 1 # sliding-window step size UNITS = 1 -H_SIZE = 180 # hidden units per RBM cell -NUM_EPOCHS = 100 -NUM_ITERATIONS = 10 +H_SIZE = 32 # hidden units per RBM cell +NUM_EPOCHS = 1000 +NUM_ITERATIONS = 1 TRAIN_PARAMS = TrainingParams( learning_rate = 0.04, @@ -80,7 +80,7 @@ class RnnModel(Model): def train(self, vc: Mat, status: Status = None): for unit in self.units: train(unit, vc, status) - # Update context portion of vc + # For the next unit: Update context portion of vc _c = unit.forward(vc) vc[1:, WIN*vocab_size():] = _c[0:-1,:] @@ -120,9 +120,8 @@ if __name__ == "__main__": # context will be updated after training c_train = np.zeros([len(batch), H_SIZE]) vc_train = concat(batch, c_train, axis=1) - for k in range(NUM_ITERATIONS): - model.train(vc_train, status=Status()) - model.save() + model.train(vc_train, status=Status()) + model.save() # test the model seed_str = 'HA^' diff --git a/README_JayRnn.md b/README_JayRnn.md new file mode 100644 index 0000000..41bedc6 --- /dev/null +++ b/README_JayRnn.md @@ -0,0 +1,146 @@ +# JayRnn — Character-Level Language Model with Recurrent RBM + +JayRnn is a character-level sequence model built on a **Recurrent Restricted Boltzmann Machine (RTRBM)**. It learns to predict the next character in a string by threading a recurrent context vector through an RBM that also sees a sliding window of the most recent characters. + +--- + +## Algorithm + +### Core idea + +A Restricted Boltzmann Machine (RBM) has two layers — visible and hidden — connected by a weight matrix W. The forward pass computes hidden activations from visible input via `h = σ(v · W^T + b_h)`; reconstruction maps back via `v' = σ(h · W + b_v)`. + +In JayRnn the visible layer has two concatenated parts: + +| Portion | Size | Content | +|---|---|---| +| Sensory window | `WIN × vocab_size` | One-hot encoding of the last WIN characters | +| Context | `H_SIZE` | Previous hidden state h_{t-1} (recurrent memory) | + +The hidden layer (`H_SIZE` units) encodes the current input jointly with temporal history. Its output becomes the context for the next time step. + +### Vocabulary + +41 characters: `^`, space, `.!?`, `A–Z`, `0–9`. +Each character is one-hot encoded into 41 bits; a window of WIN characters becomes a flat `WIN × 41` vector. + +### Training + +1. **Windowing** — the training text is padded with spaces and split into `len(TEXT)` overlapping windows of width WIN (stride 1). +2. **Batch construction** — each window is one-hot encoded and stacked row-wise → shape `(len(TEXT), WIN × vocab_size)`. +3. **Context init** — the context column block starts as all-zeros. +4. **CD update** — Contrastive Divergence (CD-1, optional Rao-Blackwell estimator) updates W, b_v, b_h. +5. **Context propagation** — after each CD step the hidden activations at row t are written as context for row t+1: `vc[1:, WIN×vocab_size:] = h[:-1, :]`. + +Steps 4–5 repeat for `NUM_EPOCHS` epochs per iteration, and the whole loop runs `NUM_ITERATIONS` times, with weights checkpointed after each. + +### Inference + +Given a seed string of WIN characters and an initial context vector: + +1. Concatenate window `v` (one-hot, `WIN × vocab_size`) with context `c` (H_SIZE) → visible `vc`. +2. **Forward**: `h = entity.forward(vc)` — new context. +3. **Reconstruct**: `vc' = entity.reconstruct(h)` — predicted visible. +4. **Split** `vc'` into `(v', c')` where `v'` is the predicted window. +5. **Clamp** — argmax over each character slot of `v'` → hard one-hot. +6. **Slide** — shift window left by one slot and append the last predicted character. +7. Repeat from step 1 using the new `h` as context. + +--- + +## Architecture + +### Single time step + +``` + Text position t → window = text[t .. t+WIN-1] + + ┌────────────────────────────────────────────────────────────────────────────┐ + │ Visible v_t (WIN × vocab_size + H_SIZE) │ + │ │ + │ ┌──────────┬──────────┬──────────┬──────────┬──────────┬──────────────┐ │ + │ │ char[0] │ char[1] │ char[2] │ char[3] │ char[4] │ context │ │ + │ │ one-hot │ one-hot │ one-hot │ one-hot │ one-hot │ h_{t-1} │ │ + │ │ (41 bit) │ (41 bit) │ (41 bit) │ (41 bit) │ (41 bit) │ (H_SIZE) │ │ + │ └──────────┴──────────┴──────────┴──────────┴──────────┴──────────────┘ │ + └───────────────────────────────────────┬────────────────────────────────────┘ + │ + W (visible × H_SIZE) + │ + ┌───────────────────────────────────────▼────────────────────────────────────┐ + │ Hidden h_t (H_SIZE) │ + │ σ( v_t · W^T + b_h ) │ + └──────────────────────────────┬────────────────────────┬────────────────────┘ + │ │ + ▼ ▼ + context_{t+1} W + b_v → reconstruct v_t' + (fed to next step) │ + ┌────────────┴────────────┐ + │ │ + window' (WIN × vocab_size) ctx' + │ (discarded) + argmax per slot + │ + predicted chars +``` + +### Temporal unrolling (shared weights W) + +``` + t = 0 t = 1 t = 2 t = T-1 + ───────── ───────── ───────── ────────── + +ctx = zeros ctx = h_0 ctx = h_1 ctx = h_{T-2} + │ │ │ │ + " HAL" " HALL" "HALLO" ... + │ │ │ │ + ┌──┴──────────┐ ┌──┴──────────┐ ┌──┴──────────┐ ┌──┴──────────┐ + │ [win | 0 ] │ │ [win | h_0] │ │ [win | h_1] │ │ [win |h_{T-2}] + │ │ │ │ │ │ │ │ + │ W │ │ W │ │ W │ │ W │ + │ (shared) │ │ (shared) │ │ (shared) │ │ (shared) │ + └──────┬──────┘ └──────┬──────┘ └──────┬──────┘ └──────┬──────┘ + │ │ │ │ + ▼ ▼ ▼ ▼ + h_0 ─────────► h_1 ─────────► h_2 ─────────► ... h_{T-1} +``` + +A single weight matrix W is shared across all time steps. +Context flows left-to-right through the batch within each training epoch. + +--- + +## Configuration + +All constants are at the top of `JayRnn.py`: + +| Constant | Default | Description | +|---|---|---| +| `TEXT` | `" HALLO SUPER JENS UND SUPER MAUSI!"` | Training sequence | +| `WIN` | `5` | Sliding-window width (characters) | +| `STRIDE` | `1` | Window step size | +| `UNITS` | `1` | Number of stacked RBM units | +| `H_SIZE` | `180` | Hidden units (= context size) | +| `NUM_EPOCHS` | `100` | CD epochs per training iteration | +| `NUM_ITERATIONS` | `10` | Outer training loop repetitions | + +**Visible layer size:** `WIN × vocab_size + H_SIZE = 5 × 41 + 180 = 385` +**Weight matrix W:** `385 × 180 = 69 300 parameters` + +--- + +## Running + +```bash +python JayRnn.py +``` + +Weights are saved to `results/JayRnn-*` after each iteration. On subsequent runs the model is loaded and training resumes from the last checkpoint. + +Generation output prints each forward step and the full predicted string at the end: + +``` +Forward 00: HA^ +Predict 00: ALLO +... +``` \ No newline at end of file