Wire up temperature-based sampling for poet text generation

Greedy argmax decoding in RbmListener::forward always produces the
exact same character sequence and quickly falls into short repeating
loops once the 5-character lookback state revisits a prior cycle.

Add an optional temperature argument (poet f <seed> [temperature]) that
switches decoding to RnnStack::sample_one_hot, which now does proper
categorical sampling (temperature-scaled, renormalized draw) instead of
the old per-code Bernoulli approach that could leave the result as a
non-one-hot probability vector. Also seed Armadillo's RNG in main(),
since it otherwise defaults to a fixed seed and every run would sample
identically.

Add docs/RNN_ARCHITECTURE.md documenting how the RnnStack/Layer stack
implements the RNN (context chaining across layers, training/generation
data flow, and the decoding behavior above).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016K8Gu7Qejd11JbdiHZqYAs

source/RnnStack.cpp

@@ -92,25 +92,38 @@ void RnnStack::clamp_one_hot(arma::mat& srcDst)
 	srcDst[index] = 1;
 }
 
-void RnnStack::sample_one_hot(arma::mat& srcDst)
+void RnnStack::sample_one_hot(arma::mat& srcDst, double temperature)
 {
-	double k = arma::accu(srcDst);
-
-	if (k != 0)
+	// srcDst holds independent per-code sigmoid activations, not a normalized
+	// distribution. Raising to 1/temperature before renormalizing sharpens
+	// (temperature < 1) or flattens (temperature > 1) the resulting
+	// categorical distribution, then we draw one index from it directly.
+	arma::mat p = arma::pow(arma::clamp(srcDst, 1e-9, 1.0), 1.0 / temperature);
+	double sum = arma::accu(p);
+	if (sum > 0)
 	{
-		srcDst = srcDst / k;
+		p /= sum;
+	}
+	else
+	{
+		p = arma::ones(arma::size(srcDst)) / srcDst.n_elem;
 	}
 
-	arma::mat ps = Matutils::sample(srcDst);
-	if (arma::accu(ps) > 1)
+	double r = arma::randu(1)[0];
+	double cumulative = 0.0;
+	size_t chosen = p.n_elem - 1;
+	for (size_t i = 0; i < p.n_elem; i++)
 	{
-		arma::uvec q1 = find(ps > 0);
-		int winner = (q1.n_elem - 1) * arma::randu(1)[0];
-		int wix = q1(winner);
-
-		srcDst = zeros(arma::size(srcDst));
-		srcDst[wix] = 1;
+		cumulative += p[i];
+		if (r <= cumulative)
+		{
+			chosen = i;
+			break;
+		}
 	}
+
+	srcDst = arma::zeros(arma::size(srcDst));
+	srcDst[chosen] = 1;
 }
 
 arma::mat RnnStack::to_curr(const arma::mat& v)

source/RnnStack.hpp

@@ -34,7 +34,7 @@ public:
     arma::mat vc_to_v(const arma::mat &vc, size_t numContext) const;
 	arma::mat step_forward(arma::mat &state, const arma::mat &v);
 
-	void sample_one_hot(arma::mat &srcDst);
+	void sample_one_hot(arma::mat &srcDst, double temperature=1.0);
 	void clamp_one_hot(arma::mat &srcDst);
 	arma::mat to_next(const arma::mat &v);
 	arma::mat to_curr(const arma::mat &v);

source/poet.cpp

@@ -2,6 +2,7 @@
 #include <fstream>
 #include <streambuf>
 #include <cstdio>
+#include <cstdlib>
 #include <cmath>
 #include <string>
 #include <cassert>
@@ -47,21 +48,37 @@ class RbmListener : public Rbm::IListener
 		return true;
 	}
 	
-	void forward(std::string const &start, size_t len)
+	// temperature <= 0 means greedy argmax decoding (the original behavior);
+	// temperature > 0 samples the next character from the (temperature-scaled)
+	// output distribution instead, which avoids the short repeating loops that
+	// greedy decoding tends to fall into.
+	void forward(std::string const &start, size_t len, double temperature=0.0)
 	{
 		RnnStack *stack = reinterpret_cast<RnnStack*>(&m_stack);
 		arma::mat state;
 		arma::mat curr;
 		arma::mat next;
 		std::string curr_str;
-		
+
+		auto pickNext = [stack, temperature](arma::mat &next)
+		{
+			if (temperature > 0.0)
+			{
+				stack->sample_one_hot(next, temperature);
+			}
+			else
+			{
+				stack->clamp_one_hot(next);
+			}
+		};
+
 		for (int i=0; i < start.size(); i++)
 		{
 			curr = Matutils::char2vec(start.at(i), RnnTextHelper::NUM_CODES);
 			curr_str.append(1, Matutils::vec2char(curr));
 			arma::mat r = stack->step_forward(state, curr);
 			next = stack->to_next(r);
-			stack->clamp_one_hot(next);
+			pickNext(next);
 		}
 		cout << "Start: " << curr_str << std::endl;
 
@@ -71,7 +88,7 @@ class RbmListener : public Rbm::IListener
 			curr_str.append(1, Matutils::vec2char(curr));
 			arma::mat r = stack->step_forward(state, curr);
 			next = stack->to_next(r);
-			stack->clamp_one_hot(next);
+			pickNext(next);
 			curr = stack->to_curr(r);
 		}
 		cout << "Curr: " << curr_str << std::endl;
@@ -83,6 +100,10 @@ class RbmListener : public Rbm::IListener
 
 int main(int argc, char *argv[])
 {
+	// Armadillo's RNG otherwise defaults to a fixed seed, which would make
+	// sample_one_hot() produce the exact same "random" text on every run.
+	arma::arma_rng::set_seed_random();
+
 	enum Command {Nop, Create, Reset, Train, Forward};
 	Command command = Nop;
 	
@@ -145,12 +166,17 @@ int main(int argc, char *argv[])
 	if (command == Command::Forward)
 	{
 		std::string start(DEFAULT_START_WORD);
-		
+		double temperature = 0.0;
+
 		if (argv[2] != nullptr)
 		{
 			start = std::string(argv[2]);
 		}
-		listener.forward(start, 100);
+		if (argv[3] != nullptr)
+		{
+			temperature = std::atof(argv[3]);
+		}
+		listener.forward(start, 100, temperature);
 	}
 	printf("\nEnd of program\n");
 	return 0;