From 656a0252d30b57a25c5523efea7e8b072f6c06f5 Mon Sep 17 00:00:00 2001 From: Jens Ahrensfeld Date: Mon, 27 Jul 2026 12:34:44 +0200 Subject: [PATCH] Wire up temperature-based sampling for poet text generation Greedy argmax decoding in RbmListener::forward always produces the exact same character sequence and quickly falls into short repeating loops once the 5-character lookback state revisits a prior cycle. Add an optional temperature argument (poet f [temperature]) that switches decoding to RnnStack::sample_one_hot, which now does proper categorical sampling (temperature-scaled, renormalized draw) instead of the old per-code Bernoulli approach that could leave the result as a non-one-hot probability vector. Also seed Armadillo's RNG in main(), since it otherwise defaults to a fixed seed and every run would sample identically. Add docs/RNN_ARCHITECTURE.md documenting how the RnnStack/Layer stack implements the RNN (context chaining across layers, training/generation data flow, and the decoding behavior above). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_016K8Gu7Qejd11JbdiHZqYAs --- docs/RNN_ARCHITECTURE.md | 188 +++++++++++++++++++++++++++++++++++++++ source/RnnStack.cpp | 39 +++++--- source/RnnStack.hpp | 2 +- source/poet.cpp | 38 ++++++-- 4 files changed, 247 insertions(+), 20 deletions(-) create mode 100644 docs/RNN_ARCHITECTURE.md diff --git a/docs/RNN_ARCHITECTURE.md b/docs/RNN_ARCHITECTURE.md new file mode 100644 index 0000000..d292887 --- /dev/null +++ b/docs/RNN_ARCHITECTURE.md @@ -0,0 +1,188 @@ +# The RNN used by `poet` (`RnnStack`) + +This document explains how `RnnStack` — the model driving the `poet` character-generation +CLI — actually works. It is not an RNN in the LSTM/GRU sense: there is no shared weight matrix +applied repeatedly across time. Instead it's a **stack of independently-weighted RBMs, one per +"tap" in a fixed-size lookback window, chained together by feeding each layer's hidden +activations forward as extra "context" input to the next layer**. The recurrence is emulated by +re-running this same chain one step at a time during generation, sliding a window of the last N +characters through it. + +Relevant sources: `source/RnnStack.{hpp,cpp}`, `source/Layer.{hpp,cpp}`, `source/AStack.cpp`, +`source/RnnTextHelper.{hpp,cpp}`, `source/matutils.hpp`, `source/poet.cpp`. + +## The alphabet + +`RnnTextHelper` encodes characters as one-hot vectors over `NUM_CODES = 37` symbols: + +- index `0` = space (also the fallback for anything that isn't a letter or digit) +- indices `1..26` = `A`..`Z` (case is folded via `toupper` — **lowercase is not represented**) +- indices `27..36` = `0`..`9` + +`ch2idx`/`idx2ch` do the conversion; `Matutils::char2vec`/`vec2char` wrap them as one-hot +`arma::mat` rows. Because case is discarded at encoding time, everything `poet` generates comes +out upper-case, regardless of the casing in the source text or seed string. + +## Project geometry: `poet_2v_5s` + +`poet.cpp` hardcodes the project name `poet_2v_5s`, loaded via `StackCreator::fromFile`. The name +encodes its shape: **2v** = 2 stacked visible character-frames per layer, **5s** = 5 layers deep +(the RBM "stack"). Concretely (`poet_2v_5s.prj`): + +| layer id | visible (X×Y) | hidden | context | +|---|---|---|---| +| 0 | 37×2 = 74 | 128 | 0 | +| 1 | 37×2 = 74 | 128 | 128 | +| 2 | 37×2 = 74 | 128 | 128 | +| 3 | 37×2 = 74 | 128 | 128 | +| 4 | 37×2 = 74 | 128 | 128 | + +Each `Layer` is a plain `Rbm` (`Layer.hpp:29` → `Rbm(numVisibleX*numVisibleY + numContext, +numHidden)`), so layers 1–4 are actually 202-visible-unit RBMs (74 character bits + 128 context +bits inherited from the previous layer); layer 0 has no context and is a plain 74-unit RBM. +`RnnStack::getSeqLen()` returns `numLayers()` (5) — the "depth" of the stack doubles as the length +of the character lookback window used during generation. + +## Training data: character bigrams + +`RnnTextHelper::createTraining(file, seq_len=2)` turns a text file into a training batch: + +- keeps a rolling 2-row window (`pattern`) of one-hot vectors: `row0` = current character, + `row1` = previous character (each new character shifts the window and overwrites row 0). +- collapses runs of whitespace/unknown characters down to a single space (skips a char if both it + and the previous character map to index 0). +- for every accepted character, flattens `pattern` row-major (`as_row()`, confirmed block order: + `[current(37) | previous(37)]`) and appends it as one training row. + +So the batch has one 74-wide row per character in the source text: `[char_t | char_t-1]`. This +matches every layer's 74-wide visible geometry — every layer is trained on the *same* stream of +character bigrams, just offset in time from each other (next section). + +## Training: `RnnStack::train` + +```cpp +padding = zeros(numLayers - 1, batch.n_cols) // 4 zero rows +batch_padded = [batch ; padding] // append padding at the end +c = zeros(N, layer[0].numContext()) // starts as an N×0 matrix (no columns) + +for i in 0..numLayers-1: + layer = getLayer(i) + batch_shifted = batch_padded.rows(i, N+i-1) // batch, shifted forward by i rows + vc = join_rows(batch_shifted, c) // attach inherited context + layer.train(vc, listener) // contrastive divergence on this layer + layer.gibbs_vh(vc, c) // c ← toHiddenProbs(vc); vc ← reconstruction (vc discarded) +``` + +The subtlety is `c`: it starts as a zero-column matrix (matching layer 0, which has no context), +and is **mutated in place** by `gibbs_vh` on every iteration — `gibbs_vh(v_probs, h_probs)` sets +`h_probs = toHiddenProbs(v_probs)`, so after layer *i* trains, `c` becomes layer *i*'s hidden-unit +probabilities, which is exactly what layer *i+1* expects as its context input. This is the whole +recurrence mechanism: **each layer's hidden activations become the next layer's extra visible +input**, propagated forward once per training step. + +`batch_shifted` is `batch_padded` windowed with an offset of `i` rows, i.e. layer `i` is trained on +training-row `j+i` while the loop index is conceptually `j`. So for a given alignment index `j`, +layer 0 sees the bigram at `j`, layer 1 sees the bigram at `j+1`, ..., layer 4 sees the bigram at +`j+4` (the most recent). In other words: **layer 0 is the "oldest" tap, layer 4 the "newest"**, and +by the time training reaches layer 4, its context input already encodes information recursively +folded in from characters `j..j+3`. This is a 5-character lookback window, unrolled across depth +(distinct weights per tap) rather than across time (shared weights) as a true RNN would. + +## Generation: `RnnStack::step_forward` + +`state` is a rolling `numLayers × NUM_CODES` matrix of single-character one-hot rows: row 0 is the +newest character, row `numLayers-1` the oldest of the last 5. Each call: + +```cpp +state = shift(state, 1, 0); // age every row down by one +state.row(0) = v_curr; // newest known character goes in at row 0 + +for j in 0..numLayers-1: + k = numLayers - j - 1 // j=0 → k=4 (oldest); j=4 → k=0 (newest/current) + layer = getLayer(j) + v = [zeros(37) | state.row(k)] // "unknown target" bigram: only the lagged half is known + vc = join_rows(v, c) // attach context inherited from layer j-1 + layer.gibbs_vh(vc, c) // c ← this layer's hidden probs (context for next layer) + r = vc_to_v(vc, layer.numContext())// strip context columns back off +return r // == the last layer's (j=4, k=0) reconstruction +``` + +Each layer is asked "given only the known previous character (`state.row(k)`) plus context +inherited from older taps, reconstruct the full bigram" — the zeroed first half is what the RBM's +one round of Gibbs sampling (`toHiddenProbs` then `toVisibleProbs`) has to fill in. Only the +**last** layer's reconstruction (built from the newest character, `k=0`, with all four older +layers' context folded in) is returned; `to_next()` (`RnnStack.cpp:121`) takes the first 37 columns +of that as the network's guess for the *next* character. + +## Decoding: `poet.cpp` / `RbmListener::forward` + +`forward()` takes an optional `temperature` (default `0.0`): + +```cpp +auto pickNext = [temperature](arma::mat &next) { + if (temperature > 0.0) stack->sample_one_hot(next, temperature); + else stack->clamp_one_hot(next); +}; + +// priming: feed the seed string character-by-character to populate `state` +for each char in seed: + r = step_forward(state, char2vec(char)) + next = to_next(r); pickNext(next) // result unused except to keep looping + +// generation: feed each predicted character back in as the next input +for i in seqLen..len: + curr = next + r = step_forward(state, curr) + next = to_next(r); pickNext(next) +``` + +`temperature <= 0` keeps the original behavior: `clamp_one_hot` is a hard argmax (`index_max()` → +one-hot), fully deterministic given a trained model and a seed. `temperature > 0` instead calls +`sample_one_hot` (`RnnStack.cpp`), which now does proper categorical sampling: the 37 independent +sigmoid activations in `next` are raised to `1/temperature` and renormalized into a distribution +(`temperature < 1` sharpens it towards argmax, `temperature > 1` flattens it towards uniform), then +one code is drawn from that distribution via a cumulative-sum draw. This replaced an older +implementation that sampled each code as an independent Bernoulli and only patched the result back +into a proper one-hot vector when *more than one* bit happened to fire — leaving `srcDst` as a +non-one-hot probability vector whenever exactly zero or one bits fired. + +`main()` exposes this as a third CLI argument: `poet f [temperature]`, e.g. `poet f "the " +0.7`. Sampling only produces different output across process runs because `main()` now seeds +Armadillo's RNG (`arma::arma_rng::set_seed_random()`) — Armadillo otherwise defaults to a fixed +seed, which would have made every run's "random" draws identical. + +`RnnStack::to_curr()` is also computed once per generation step but its result is immediately +discarded (overwritten by `curr = next` at the top of the next iteration) — dead code kept from +development/debugging. + +Two consequences worth knowing when reading `poet` output: + +- **Greedy argmax decoding (`temperature <= 0`, the default) is why generated text degenerates into + short repeating loops** (e.g. `...MURPURSE AN MURPURSE AN...`) once the rolling 5-character state + re-enters a cycle it has visited before — there's no randomness to break out. Passing a + `temperature > 0` avoids this by sampling instead. +- If the seed string is shorter than `numLayers` (5) characters, the oldest rows of `state` are + still zero (not a space one-hot) when generation begins — the model briefly sees "blank" rather + than real history for the first few generated characters. + +## Summary picture + +``` +Text file --collapse whitespace, bigram window--> training batch [char_t | char_t-1] (74-wide rows) + │ + ┌─────────────────────────────────────┴─────────────────────────────────────┐ + │ Layer 0 (oldest tap, no context) │ + │ CD train on batch │ + │ context₀ = hidden_probs(batch) ──► fed into Layer 1 │ + │ Layer 1 (context=128) │ + │ CD train on [batch shifted +1 | context₀] │ + │ context₁ = hidden_probs(...) ──► fed into Layer 2 │ + │ ... │ + │ Layer 4 (newest tap) │ + │ CD train on [batch shifted +4 | context₃] │ + └───────────────────────────────────────────────────────────────────────────────┘ + +Generation: same 5-layer chain, run once per output character, consuming a sliding 5-character +window of state (oldest → layer 0 ... newest → layer 4), context threaded the same way; only +layer 4's reconstruction is decoded (argmax) into the next character. +``` diff --git a/source/RnnStack.cpp b/source/RnnStack.cpp index 6c34cc5..53bcfda 100644 --- a/source/RnnStack.cpp +++ b/source/RnnStack.cpp @@ -92,25 +92,38 @@ void RnnStack::clamp_one_hot(arma::mat& srcDst) srcDst[index] = 1; } -void RnnStack::sample_one_hot(arma::mat& srcDst) +void RnnStack::sample_one_hot(arma::mat& srcDst, double temperature) { - double k = arma::accu(srcDst); - - if (k != 0) + // srcDst holds independent per-code sigmoid activations, not a normalized + // distribution. Raising to 1/temperature before renormalizing sharpens + // (temperature < 1) or flattens (temperature > 1) the resulting + // categorical distribution, then we draw one index from it directly. + arma::mat p = arma::pow(arma::clamp(srcDst, 1e-9, 1.0), 1.0 / temperature); + double sum = arma::accu(p); + if (sum > 0) { - srcDst = srcDst / k; + p /= sum; + } + else + { + p = arma::ones(arma::size(srcDst)) / srcDst.n_elem; } - arma::mat ps = Matutils::sample(srcDst); - if (arma::accu(ps) > 1) + double r = arma::randu(1)[0]; + double cumulative = 0.0; + size_t chosen = p.n_elem - 1; + for (size_t i = 0; i < p.n_elem; i++) { - arma::uvec q1 = find(ps > 0); - int winner = (q1.n_elem - 1) * arma::randu(1)[0]; - int wix = q1(winner); - - srcDst = zeros(arma::size(srcDst)); - srcDst[wix] = 1; + cumulative += p[i]; + if (r <= cumulative) + { + chosen = i; + break; + } } + + srcDst = arma::zeros(arma::size(srcDst)); + srcDst[chosen] = 1; } arma::mat RnnStack::to_curr(const arma::mat& v) diff --git a/source/RnnStack.hpp b/source/RnnStack.hpp index f761bba..0cf6dd9 100644 --- a/source/RnnStack.hpp +++ b/source/RnnStack.hpp @@ -34,7 +34,7 @@ public: arma::mat vc_to_v(const arma::mat &vc, size_t numContext) const; arma::mat step_forward(arma::mat &state, const arma::mat &v); - void sample_one_hot(arma::mat &srcDst); + void sample_one_hot(arma::mat &srcDst, double temperature=1.0); void clamp_one_hot(arma::mat &srcDst); arma::mat to_next(const arma::mat &v); arma::mat to_curr(const arma::mat &v); diff --git a/source/poet.cpp b/source/poet.cpp index 0b31afd..f906c33 100644 --- a/source/poet.cpp +++ b/source/poet.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -47,21 +48,37 @@ class RbmListener : public Rbm::IListener return true; } - void forward(std::string const &start, size_t len) + // temperature <= 0 means greedy argmax decoding (the original behavior); + // temperature > 0 samples the next character from the (temperature-scaled) + // output distribution instead, which avoids the short repeating loops that + // greedy decoding tends to fall into. + void forward(std::string const &start, size_t len, double temperature=0.0) { RnnStack *stack = reinterpret_cast(&m_stack); arma::mat state; arma::mat curr; arma::mat next; std::string curr_str; - + + auto pickNext = [stack, temperature](arma::mat &next) + { + if (temperature > 0.0) + { + stack->sample_one_hot(next, temperature); + } + else + { + stack->clamp_one_hot(next); + } + }; + for (int i=0; i < start.size(); i++) { curr = Matutils::char2vec(start.at(i), RnnTextHelper::NUM_CODES); curr_str.append(1, Matutils::vec2char(curr)); arma::mat r = stack->step_forward(state, curr); next = stack->to_next(r); - stack->clamp_one_hot(next); + pickNext(next); } cout << "Start: " << curr_str << std::endl; @@ -71,7 +88,7 @@ class RbmListener : public Rbm::IListener curr_str.append(1, Matutils::vec2char(curr)); arma::mat r = stack->step_forward(state, curr); next = stack->to_next(r); - stack->clamp_one_hot(next); + pickNext(next); curr = stack->to_curr(r); } cout << "Curr: " << curr_str << std::endl; @@ -83,6 +100,10 @@ class RbmListener : public Rbm::IListener int main(int argc, char *argv[]) { + // Armadillo's RNG otherwise defaults to a fixed seed, which would make + // sample_one_hot() produce the exact same "random" text on every run. + arma::arma_rng::set_seed_random(); + enum Command {Nop, Create, Reset, Train, Forward}; Command command = Nop; @@ -145,12 +166,17 @@ int main(int argc, char *argv[]) if (command == Command::Forward) { std::string start(DEFAULT_START_WORD); - + double temperature = 0.0; + if (argv[2] != nullptr) { start = std::string(argv[2]); } - listener.forward(start, 100); + if (argv[3] != nullptr) + { + temperature = std::atof(argv[3]); + } + listener.forward(start, 100, temperature); } printf("\nEnd of program\n"); return 0;