jensandClaude Sonnet 5
656a0252d3
Greedy argmax decoding in RbmListener::forward always produces the exact same character sequence and quickly falls into short repeating loops once the 5-character lookback state revisits a prior cycle. Add an optional temperature argument (poet f <seed> [temperature]) that switches decoding to RnnStack::sample_one_hot, which now does proper categorical sampling (temperature-scaled, renormalized draw) instead of the old per-code Bernoulli approach that could leave the result as a non-one-hot probability vector. Also seed Armadillo's RNG in main(), since it otherwise defaults to a fixed seed and every run would sample identically. Add docs/RNN_ARCHITECTURE.md documenting how the RnnStack/Layer stack implements the RNN (context chaining across layers, training/generation data flow, and the decoding behavior above). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_016K8Gu7Qejd11JbdiHZqYAs
148 lines
3.3 KiB
C++
148 lines
3.3 KiB
C++
/*
|
|
* To change this license header, choose License Headers in Project Properties.
|
|
* To change this template file, choose Tools | Templates
|
|
* and open the template in the editor.
|
|
*/
|
|
|
|
/*
|
|
* File: RnnStack.cpp
|
|
* Author: jens
|
|
*
|
|
* Created on 17. Januar 2022, 12:43
|
|
*/
|
|
|
|
#include "RnnStack.hpp"
|
|
#include "RnnTextHelper.hpp"
|
|
|
|
RnnStack::RnnStack(const std::string &name)
|
|
: AStack(StackType::Rnn, name)
|
|
{
|
|
}
|
|
|
|
RnnStack::~RnnStack()
|
|
{
|
|
}
|
|
|
|
size_t RnnStack::getSeqLen()
|
|
{
|
|
return numLayers();
|
|
}
|
|
|
|
arma::mat RnnStack::v_to_vc(const arma::mat& v, const arma::mat& c) const
|
|
{
|
|
return arma::join_rows(v, c);
|
|
}
|
|
|
|
arma::mat RnnStack::vc_to_v(const arma::mat& vc, size_t numContext) const
|
|
{
|
|
size_t numVisible = vc.n_cols;
|
|
return vc.submat(0, 0, vc.n_rows - 1, numVisible - numContext - 1);
|
|
}
|
|
|
|
void RnnStack::train(const arma::mat& batch, Rbm::IListener* pListener)
|
|
{
|
|
// thisBatch = {padding | batch}
|
|
int numTraining = batch.n_rows;
|
|
arma::mat padding = arma::zeros(getSeqLen()-1, batch.n_cols);
|
|
arma::mat batch_padded = arma::join_cols(batch, padding);
|
|
arma::mat c = arma::zeros(numTraining, getLayer(0)->numContext());
|
|
|
|
for (int i=0; i < getSeqLen(); i++)
|
|
{
|
|
Layer *pLayer = getLayer(i);
|
|
int k = i;
|
|
arma::mat batch_shifted = batch_padded.rows(k, numTraining+k-1);
|
|
arma::mat vc = v_to_vc(batch_shifted, c);
|
|
pLayer->train(vc, pListener);
|
|
pLayer->gibbs_vh(vc, c);
|
|
pLayer = pLayer->next;
|
|
}
|
|
}
|
|
|
|
arma::mat RnnStack::step_forward(arma::mat &state, const arma::mat& v_curr)
|
|
{
|
|
arma::mat c = arma::zeros(1, getLayer(0)->numContext());
|
|
arma::mat z = arma::zeros(arma::size(v_curr));
|
|
arma::mat r;
|
|
|
|
if (state.is_empty())
|
|
{
|
|
state = arma::zeros(getSeqLen(), v_curr.n_cols);
|
|
}
|
|
state = arma::shift(state, 1, 0);
|
|
state.row(0) = v_curr;
|
|
|
|
for (int j=0; j < getSeqLen(); j++)
|
|
{
|
|
int k = getSeqLen()- j - 1;
|
|
Layer *pLayer = getLayer(j);
|
|
arma::mat v = arma::join_rows(z, state.row(k));
|
|
arma::mat vc = v_to_vc(v, c);
|
|
pLayer->gibbs_vh(vc, c);
|
|
|
|
r = vc_to_v(vc, pLayer->numContext());
|
|
}
|
|
return r;
|
|
}
|
|
|
|
void RnnStack::clamp_one_hot(arma::mat& srcDst)
|
|
{
|
|
int index = srcDst.index_max();
|
|
srcDst = arma::zeros(arma::size(srcDst));
|
|
srcDst[index] = 1;
|
|
}
|
|
|
|
void RnnStack::sample_one_hot(arma::mat& srcDst, double temperature)
|
|
{
|
|
// srcDst holds independent per-code sigmoid activations, not a normalized
|
|
// distribution. Raising to 1/temperature before renormalizing sharpens
|
|
// (temperature < 1) or flattens (temperature > 1) the resulting
|
|
// categorical distribution, then we draw one index from it directly.
|
|
arma::mat p = arma::pow(arma::clamp(srcDst, 1e-9, 1.0), 1.0 / temperature);
|
|
double sum = arma::accu(p);
|
|
if (sum > 0)
|
|
{
|
|
p /= sum;
|
|
}
|
|
else
|
|
{
|
|
p = arma::ones(arma::size(srcDst)) / srcDst.n_elem;
|
|
}
|
|
|
|
double r = arma::randu(1)[0];
|
|
double cumulative = 0.0;
|
|
size_t chosen = p.n_elem - 1;
|
|
for (size_t i = 0; i < p.n_elem; i++)
|
|
{
|
|
cumulative += p[i];
|
|
if (r <= cumulative)
|
|
{
|
|
chosen = i;
|
|
break;
|
|
}
|
|
}
|
|
|
|
srcDst = arma::zeros(arma::size(srcDst));
|
|
srcDst[chosen] = 1;
|
|
}
|
|
|
|
arma::mat RnnStack::to_curr(const arma::mat& v)
|
|
{
|
|
return v.cols(NUM_CODES, 2 * NUM_CODES - 1);
|
|
}
|
|
|
|
arma::mat RnnStack::to_next(const arma::mat& v)
|
|
{
|
|
return v.cols(0, NUM_CODES - 1);
|
|
}
|
|
|
|
void RnnStack::setParams(const Rbm::Params& param)
|
|
{
|
|
Layer *pLayer = getLayer(0);
|
|
while (pLayer)
|
|
{
|
|
pLayer->params() = param;
|
|
pLayer = pLayer->next;
|
|
}
|
|
}
|