pyRBM/RNN-RBM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "id": "c33-0001",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-05-31T20:52:48.942476Z",
     "iopub.status.busy": "2026-05-31T20:52:48.942263Z",
     "iopub.status.idle": "2026-05-31T20:52:49.958704Z",
     "shell.execute_reply": "2026-05-31T20:52:49.957672Z"
    },
    "ExecuteTime": {
     "end_time": "2026-05-31T21:21:09.995884255Z",
     "start_time": "2026-05-31T21:21:09.451419390Z"
    }
   },
   "source": [
    "# RNN-RBM\n",
    "#\n",
    "# Architecture: unrolled StackRnn (UNROLL_DEPTH entities, alternating t % depth)\n",
    "#   visible = [context(CONTEXT_SIZE) | x_t(SENSORY_SIZE)]   hidden = CONTEXT_SIZE\n",
    "#\n",
    "# Training data: Moby Dick opening, encoded with a 40-char vocabulary.\n",
    "\n",
    "import numpy as np_cpu\n",
    "import matplotlib.pyplot as plt\n",
    "from stack.rnn import StackRnn\n",
    "from rbm.matrix import np, convert\n",
    "from rbm.entity import EntityParams, TrainingParams\n",
    "from rbm.status import CheckpointStatus"
   ],
   "outputs": [],
   "execution_count": 1
  },
  {
   "cell_type": "code",
   "id": "c33-0002",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-05-31T20:52:49.962590Z",
     "iopub.status.busy": "2026-05-31T20:52:49.961957Z",
     "iopub.status.idle": "2026-05-31T20:52:49.969009Z",
     "shell.execute_reply": "2026-05-31T20:52:49.968063Z"
    },
    "ExecuteTime": {
     "end_time": "2026-05-31T21:21:10.051754981Z",
     "start_time": "2026-05-31T21:21:10.002610196Z"
    }
   },
   "source": [
    "# \u2500\u2500 Config \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n",
    "SENSORY_SIZE   = 40      # numVisibleX * numVisibleY = 1 * 40\n",
    "CONTEXT_SIZE   = 256     # numContext = numHidden\n",
    "UNROLL_DEPTH   = 1       # number of time-step entities (1 = shared weights)\n",
    "LEARNING_RATE  = 0.05    # learningRate\n",
    "MOMENTUM       = 0.9     # momentum\n",
    "NUM_EPOCHS     = 500     # numEpochs\n",
    "MINI_BATCH     = 100     # miniBatchSize\n",
    "NUM_GIBBS      = 3       # numGibbs\n",
    "RAO_BLACKWELL  = True    # doRaoBlackwell\n",
    "L2_LAMBDA      = 0.0     # weightDecay\n",
    "PRJ_NAME       = \"RNN-RBM\"\n",
    "WORK_DIR       = \"results\"\n",
    "\n",
    "# Vocabulary (40 chars)\n",
    "ALLOWED     = set(' .!?ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789')\n",
    "chars       = sorted(ALLOWED)\n",
    "idx_to_char = {i: c for i, c in enumerate(chars)}\n",
    "char_to_idx = {c: i for i, c in enumerate(chars)}"
   ],
   "outputs": [],
   "execution_count": 2
  },
  {
   "cell_type": "code",
   "id": "c33-0003",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-05-31T20:52:49.971400Z",
     "iopub.status.busy": "2026-05-31T20:52:49.971055Z",
     "iopub.status.idle": "2026-05-31T20:52:49.977961Z",
     "shell.execute_reply": "2026-05-31T20:52:49.976889Z"
    },
    "ExecuteTime": {
     "end_time": "2026-05-31T21:21:10.120482231Z",
     "start_time": "2026-05-31T21:21:10.063092563Z"
    }
   },
   "source": [
    "# \u2500\u2500 Training sentence \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n",
    "SENTENCE = (\n",
    "    \"Call me Ishmael. Some years ago, never mind how long precisely, \"\n",
    "    \"having little money in my pocket and nothing particular to interest \"\n",
    "    \"me on shore, I thought I would sail about a little.\"\n",
    ")\n",
    "\n",
    "# Convert to uppercase, keep only in-vocab characters\n",
    "SENTENCE  = ''.join(c for c in SENTENCE.upper() if c in ALLOWED)\n",
    "N_SAMPLES = len(SENTENCE)\n",
    "\n",
    "sensory_np = np_cpu.zeros((N_SAMPLES, SENSORY_SIZE), dtype=np_cpu.float64)\n",
    "for i, c in enumerate(SENTENCE):\n",
    "    sensory_np[i, char_to_idx[c]] = 1.0\n",
    "\n",
    "decoded = SENTENCE\n",
    "print(f\"Sentence : '{decoded}'\")\n",
    "print(f\"Length   : {N_SAMPLES} chars\")"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sentence : 'CALL ME ISHMAEL. SOME YEARS AGO NEVER MIND HOW LONG PRECISELY HAVING LITTLE MONEY IN MY POCKET AND NOTHING PARTICULAR TO INTEREST ME ON SHORE I THOUGHT I WOULD SAIL ABOUT A LITTLE.'\n",
      "Length   : 180 chars\n"
     ]
    }
   ],
   "execution_count": 3
  },
  {
   "cell_type": "code",
   "id": "c33-0004",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-05-31T20:52:49.980161Z",
     "iopub.status.busy": "2026-05-31T20:52:49.979824Z",
     "iopub.status.idle": "2026-05-31T20:52:50.341436Z",
     "shell.execute_reply": "2026-05-31T20:52:50.340513Z"
    },
    "ExecuteTime": {
     "end_time": "2026-05-31T21:21:10.539293991Z",
     "start_time": "2026-05-31T21:21:10.122245726Z"
    }
   },
   "source": [
    "# \u2500\u2500 Visualise one-hot encoding \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n",
    "step = max(1, N_SAMPLES // 40)    # show at most 40 tick labels\n",
    "tick_pos = range(0, N_SAMPLES, step)\n",
    "\n",
    "plt.figure(figsize=(16, 3))\n",
    "plt.imshow(sensory_np.T, aspect='auto', cmap='hot', vmin=0, vmax=1)\n",
    "plt.colorbar(label='activation')\n",
    "plt.xlabel('Position')\n",
    "plt.ylabel('Vocab index')\n",
    "plt.title(f'One-hot encoding  ({N_SAMPLES} chars)')\n",
    "plt.xticks(tick_pos, [decoded[i] for i in tick_pos], fontsize=8)\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 1600x300 with 2 Axes>"
      ],
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABY8AAAEiCAYAAACxwS1VAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAWIlJREFUeJzt3Xd4VFX+x/FPQhKUMAEpJoCCAZQiLgIiAmLAuEpRKSoiroK4IkYRLLjAokiRsv4ELFhQDGURQUEQC1VZUALSQZpAQgskIbQECClwfn9gRsdMhsyQKZl5v57nPJJzy/mec++djF8O5wZJMgIAAAAAAAAA4E+CvR0AAAAAAAAAAMD3kDwGAAAAAAAAABRA8hgAAAAAAAAAUADJYwAAAAAAAABAASSPAQAAAAAAAAAFkDwGAAAAAAAAABRA8hgAAAAAAAAAUADJYwAAAAAAAABAASSPAQAAAAAAAAAFkDwGAAABLykpSQsWLPB2GCXO0KFDZYyxqUtKSlJ8fLyXIrIvKChIW7du1eDBg70dymXLH/OKFSt6OxRJUr169ZSbm6sbb7zR26EAAADADUgeAwAAj6lfv76mT5+uQ4cO6dy5c0pOTtZ///tf1a9f39uhFasqVapo6NChatiwobdDgaRHHnlE1157rd577z1rXXh4uF5//XV9//33OnbsmIwx6tGjR6HneOihh5SQkKATJ04oPT1dy5cvV/v27QvsFxQUpAEDBigxMVFZWVnavHmzunXr5pZ++YIdO3bo22+/1fDhw70dCgAAANyA5DEAAPCIzp07a8OGDYqNjVV8fLzi4uI0efJktWnTRhs2bFCnTp28HWKxqVq1ql5//XXdfPPN3g7F4+rUqaOnnnrK22HYGDBggD7//HNlZGRY6ypVqqShQ4eqXr162rx5s8Pjn3vuOc2ePVvp6ekaOHCgRowYoXLlyunbb79V586dbfZ944039J///EdLlixR3759deDAAc2cOVMPP/ywW/rmCz788EN16dJFNWvW9HYoAAAAcANDoVAoFAqF4s5Ss2ZNc/r0abN9+3ZTqVIlm20VK1Y027dvN5mZmSY6Otor8SUlJZkFCxYU2/maNGlijDGmR48eXh97d5ahQ4cac3HdCp8tN998szHGmDvvvNOmPiwszERGRhbpeu3atcusWbPGps5isZiMjAwzb948a13VqlVNdna2effdd232/d///mcOHDhggoODi23MK1as6NZxu+KKK4q8b0hIiDl27JgZNmyY1683hUKhUCgUCqV4CzOPAQCA2w0YMEDh4eHq3bu30tPTbbYdO3ZMTz/9tMqWLatXXnnFWp+/tmutWrUUHx+vEydO6OTJk/r000915ZVXFmjj0Ucf1bp163T27FkdO3ZMM2fO1DXXXONUnC1bttSaNWuUlZWlvXv36rHHHiuwT3R0tGbPnq1jx47pzJkzSkhIsFm+ICYmRuvWrZMkTZkyRcaYSy6JIF2crTx58mSlpKTo3Llz+vXXX/XEE0/Y7BMTEyNjjB566CENHjxYBw8eVFZWlpYuXapatWoVOOett96qb7/9VsePH9fp06e1efNmPf/88zb7tGnTRitWrNDp06d14sQJzZs3T3Xr1rU7Nr/88ouysrK0Z88e9e7d224//rrmcY8ePWSMUYsWLfTWW28pLS1Np0+f1ty5c1WpUiWbY4OCgjR06FAlJyfrzJkz+uGHH1SvXr3LWke5U6dOys7O1ooVK2zqc3JylJqaWqRzREREKC0tzaYuMzNTp0+fVlZWlrWuY8eOCgsL0/vvv2+z7wcffKBrr71WzZs3v2RbderU0axZs5SWlqazZ89q586dGjlyZIH9ypcvf8nnomfPnlq2bJlSU1N17tw5bdu2TX369Clwrvw1v++++26tXbtWWVlZevrppyVJd911l1auXKkTJ04oMzNTO3fu1BtvvGFzfF5enpYvX66OHTtesn8AAAAoebyewaZQKBQKheLf5dChQyYxMdHhPomJiebAgQPWn/NnWK5fv958+eWXpk+fPmbSpEnGGGPGjBljc+zgwYPN+fPnzcyZM02fPn3Mq6++atLS0kxiYqIpV67cJeNLSkoyO3bsMEeOHDEjR440cXFxZt26deb8+fOmfv361v2uvvpqc+TIEXPq1CkzYsQI079/f7Nx40aTl5dnOnXqZN1nyJAhxhhjPvzwQ/Poo4+aRx991OGs6quvvtocOHDA7N+/3wwZMsQ8/fTTZt68ecYYY/r162fdLyYmxjoma9euNf369TOvvfaaOX36tFm9erXNOe+66y5z7tw5k5SUZIYOHWqefvppM2HCBLN48WLrPrGxsSYnJ8fs3LnTvPzyy9ZxO3bsmKlRo4Z1vwYNGpgzZ86Yffv2mX/961/m3//+tzly5IjZtGlTgZnHSUlJJj4+3vpzjx49rDEvXbrUPPvss+bNN980ubm55vPPP7c5dsyYMcYYY+bPn2/i4uLMRx99ZA4cOGDS0tJszulMWbx4sVm3bp3DfS4183jmzJkmNzfXPPfcc6ZGjRqmTp065r333jNnzpwxt912m3W/SZMmmczMzALH16xZ0xhjzHPPPecwjptuusmcPHnSHD161LzxxhvmqaeeMmPGjDGbN2926blYs2aN+fTTT02/fv3Ms88+axYuXGiMMSYuLq7ANfvtt9/MsWPHzKhRo0zv3r1NTEyMqV+/vjl37pz55ZdfTN++fU3v3r3Nf/7zH7N8+fICsQ8ePNjk5eUZi8Xi9c8bCoVCoVAoFEqxFq8HQKFQKBQKxY9LRESEMcaYr776yuF++cnSsmXLGumPJNknn3xis9+cOXPM0aNHrT9Xr17d5ObmmkGDBtnsd+ONN5qcnJwC9fZKUlKSMcaY22+/3VpXqVIlk5WVZd58801r3bhx44wxxrRs2dJaFx4ebvbu3WsSExNNUFCQkZxftuLjjz82ycnJpkKFCjb1n332mTlx4oR1CYH85PG2bdtMaGiodb++ffsaY4y58cYbjSQTHBxs9u7da5KSkhwmzzds2GBSUlLMVVddZa276aabTF5enpkyZYq1bu7cuebs2bPm2muvtdbVrVvX5ObmFjl5/OektSTz1ltvmdzcXBMREWGkiwn0nJwcM3fuXJv9XnvtNWOMcTl5fODAAfPFF1843OdS16ty5cpmyZIl5s/S0tJsEseSzIIFC8yePXsKHH/llVcaY4wZNWqUwziWL19uTp06ZTPOfy1FfS4k+0tPfP/99wVizL//7777bpv6fv36FXmJjG7duhljjGnatKlL14lCoVAoFAqF4puFZSsAAIBbWSwWSRf/mb8j+dsjIiJs6j/88EObn1euXKlKlSpZz9ulSxcFBwdr9uzZqlixorWkpKRo9+7datOmTZHi3LZtm3766Sfrz+np6dq1a5fNS8Dat2+vNWvW6Oeff7bWnTlzRpMmTVJ0dLTq169fpLb+6oEHHtCCBQsUFBRk04dFixapfPnyaty4sc3+8fHxys3NtRkTSdZYGzVqpJo1a2rChAk6deqU3TajoqLUqFEjTZkyRSdOnLDWb926VUuWLLEuxREcHKx77rlH8+bN08GDB6377dy5U4sWLSpyHydNmmTz88qVKxUSEqIaNWpIkmJjYxUaGlpgyYd33323yG3YU7FiRZv+ueLs2bPatWuXpkyZogcffFBPPPGEjhw5orlz59osF3LllVcqOzu7wPHnzp2zbi9MpUqVFBMTo08//dRmnAtzqefiz+1KF5+rihUr6n//+59q1apV4DlLTEzU4sWLbepOnjwp6eJyHEFBQQ7jyR/jvy5FAgAAgJKN5DEAAHCr/KTwn5Na9hSWZD5w4IDNz/lJqquuukqSdP311ys4OFh79uxRenq6Talfv76uvvpqSVJ4eLgiIyOt5a9Jrr+2k99WfjuSVKNGDe3atavAfjt27LBud1blypV11VVX6emnny4Q/5QpUyTJ2ofCYv3rmOQnNH/99ddC282PtbD+VK5cWWXKlLH+d/fu3QX2s3dsYS4Vc348e/bsKbDf8ePHi9yOPZdKfF7KF198oerVq+uJJ57QnDlzNGXKFLVu3VphYWE26/9mZWWpdOnSBY6/4oorrNsLk5/4d3TN/uxS4ylJLVq00JIlS3T69GmdOnVK6enpGj16tCSpXLlyNscnJSUVaGPWrFn66aefNHnyZKWmpmrmzJl66KGH7I5nft3FiegAAADwFyHeDgAAAPi3jIwMHT58WH/7298c7ve3v/1Nhw4dKpA8Pn/+vN3985NVwcHBunDhgtq1a2d339OnT0uSXn75Zb3++uvW+n379ik6OrrI7bhLcPDFv8ufPn26pk6danefLVu22PzsrVgvh7diPnbsmE1C1VnR0dFq166dnnrqKZv6EydO6KefflLLli2tdUeOHLE7071KlSqSpMOHD7scx19dajxr1qypZcuWaefOnXrxxRd18OBB5eTkqH379nrxxRet910+e4ntc+fO6Y477lCbNm3UoUMHtW3bVt26ddOyZct0991368KFC9Z988f4ry/EBAAAQMlG8hgAALjdN998o969e6tly5Y2Sz7ku/322xUdHV3gn+IXxd69exUcHKykpCS7s2PzTZs2zWZZCkezQAuzf/9+1alTp0B93bp1rdsl52ZfHj16VBkZGSpVqpSWLVvmdEz27N27V5LUoEGDQs+ZH2th/Tl69KjOnj2rc+fO6ezZs7r++usL7GfvWFflx1O7dm3t27fPWl+hQgVVqFDB5fPu3LnT5i8JnBUZGSlJKlWqVIFtoaGhCgn54+v0pk2b9NRTT6levXrW2eiS1KxZM+v2wiQmJkq6eM2Kw3333acrrrhC999/v80yGEVdxiWfMUY//PCDfvjhB7300ksaNGiQRo0apTZt2tjcW9HR0Tp//rx+++23YokfAAAAvoFlKwAAgNu9+eabOnv2rD766KMCicCrrrpKH374oc6cOaM333zT6XPPnTtXeXl5Gjp0qN3t+e0lJSVp2bJl1rJq1Sqn2/ruu+/UrFkz3Xbbbda6MmXKqHfv3kpKStL27dslXVwHWZLKly9/yXNeuHBBc+bM0QMPPKAbb7yxwHZX1pDdsGGDEhMT1b9//wLLE+RLSUnRxo0b1aNHD5t9brzxRt1999367rvvrPEtWrRInTp10rXXXmvdr27durrnnnucjq0wy5YtU25urp555hmb+ueee+6yzpuQkKAGDRooLCzMpeP37Nmj8+fP6+GHH7apr1atmlq1aqWNGzda6+bPn6+cnBzFxcXZ7NunTx8dOnTI4T2Xnp6u//3vf+rVq5fNOLsqf2byn2d2R0RE6IknnijyOezN2M5PgP91eY4mTZpo27ZtysjIcCFaAAAA+CpmHgMAALfbs2ePevTooRkzZmjr1q2aPHmykpKSdN111+nJJ59UpUqV9Mgjj1hnXzojMTFRQ4YM0ZgxY3Tddddp3rx5yszMVHR0tDp37qxJkybprbfeKpZ+jBkzRo888oi+//57vfPOOzp+/Lh69Oih6OhoPfDAA9YZx3v37tWJEyfUp08fZWZm6syZM1qzZo3NjNo/GzhwoNq0aaM1a9bo448/1vbt21WhQgU1btxYd911lypWrOhUnMYYPfPMM1qwYIE2bdqk+Ph4HTlyRHXr1tWNN96otm3bSpIGDBig77//XgkJCZo8ebKuvPJK9e3bV6dOnbJZ4mPo0KFq27atVq5cqffff18hISHq27evtm3bpoYNG7o0ln+Vlpamt99+Wy+//LLmz5+vhQsXqmHDhmrXrp2OHj3q8lq68+fP12uvvaaYmBgtWbLEZtuzzz6r8uXLq2rVqpIuzta95pprJF18UV9GRobS09P16aef6qmnntKyZcs0d+5cWSwWxcXF6corr7SuISxJycnJmjBhgl555RWFhoZq7dq16tSpk+644w51797dZpkHe55//nn99NNP2rBhgyZNmmR9Rjp06KBGjRo51e/FixcrOztbCxYs0EcffaSyZcvqqaeeUlpamrW/l/Laa6/pjjvu0Lfffqv9+/fr6quvVlxcnA4ePGgziz8kJEQxMTEFXnYIAAAA/2AoFAqFQqFQPFEaNGhgZsyYYZKTk012drY5fPiwmTFjhrnxxhsL7Dt06FBjjDEVK1a0qe/Ro4cxxpgaNWrY1Hfu3NmsWLHCZGZmmszMTLN9+3bz7rvvmuuvv/6ScSUlJZkFCxYUqP/xxx/Njz/+aFMXHR1tZs+ebY4fP27Onj1rVq9ebdq3b1/g2Pvuu8/8+uuvJicnxxhjTI8ePRzGULlyZfPuu++a/fv3W8dmyZIl5p///Kd1n5iYGGOMMQ888IDNsTVq1LDbRosWLcyiRYvMqVOnTGZmptm0aZN59tlnbfa58847zcqVK82ZM2fMyZMnzfz5803dunULxNeqVSuzdu1ac+7cObNnzx7Tu3dv6zX661jGx8cXuF5NmjSx2S+/LzExMda64OBgM2zYMHP48GFz5swZs3TpUlOnTh1z9OhR8/7777t8323atMl8/PHHdq97Yf58f5UqVco8++yzZsOGDSYjI8NkZGSYZcuWmdatWxc4Z1BQkBk4cKBJSkoy586dM1u3bjXdu3cvcqz169c3c+bMsd5fO3bsMMOGDXPpubj33nvNpk2bzNmzZ01iYqIZMGCA6dmzZ4H9Crv/27RpY7766itz6NAhc+7cOXPo0CEzY8YMU7t2bZv97rnnHmOMMbVq1SqWzwkKhUKhUCgUiu+UoN//AAAAAPiccuXK6eTJk/r3v/+tUaNGuXSOf/zjH5o4caKqV6+uU6dOFXOE+Oqrr2SMUZcuXbwdCgAAAIoZax4DAADAJ1xxxRUF6vr37y9JWr58ucvnnTFjhg4cOKBnn33W5XPAvrp16+ree+/Vq6++6u1QAAAA4AbMPAYAAIBP6NGjh3r27KnvvvtOp0+f1u23367u3btr0aJF1nWaAQAAAHgOL8wDAACAT9iyZYvy8vL0yiuvKCIiQqmpqZowYYKGDBni7dAAAACAgMSyFQAAAPAJGzdu1N///ndVrlxZpUuXVvXq1fXCCy/ozJkz3g4NAAAAcEqrVq309ddfKzk5WcYYdezY8ZLHxMTEaP369Tp37px2796tHj16eCBSx0geAwAAAAAAAEAxCg8P1+bNm4v83o3rrrtO3377rX788UfdfPPNmjBhgj755BPdfffdbo7UsRKx5nFcXJwGDBigqKgobd68WX379tXatWu9HRYAAAAAAAAAOGSMUadOnTR//vxC9xkzZow6dOigm266yVo3c+ZMlS9fXu3atfNEmHb5/JrHXbt21bhx49SnTx+tWbNG/fv316JFi1SnTh0dPXq0SOeoWrWqMjMz3RwpAAAAAAAA4F0Wi0WHDx/2dhglSunSpRUWFubSsdnZ2crJybnsGJo3b66lS5fa1C1atEgTJky47HNfDp9PHr/44ov6+OOPNWXKFElSnz591KFDB/Xq1Utjx4695PFVq1ZVcnKym6MEAAAAAAAAfEO1atVIIBdR6dKlde7cuSLtm5mZKYvFYlP3+uuva9iwYZcdR1RUlFJTU23qUlNTVa5cOV1xxRVFjrG4+XTyODQ0VE2aNNHo0aOtdcYYLV26VM2bNy/SOfJnHNeuVk2nPTD7OCUjo9BtURERXj9fSeArffaVOBzxZIwleTz89b7xlWviK3H4ipI8HiU5dqlkxF/cn1N877Dlyd8DvvA7xx1K+j3gKe4YJ3+9pxzhM/HyuRJjSeiXq1y5p0rCePhKjL7+OeXqOPnK+BaXshaL9iQn8y/wnfDHjOO8S+5rsVhUrVo1m/HNzs52U2S+waeTx5UqVVJISIjdrHvdunXtHhMWFqbSpUtbf87/24DTmZlef3CKu31v98cbfKXPvhKHI56M0dfHw1fiC8Rr4itx+IqSPB4lOXbJ9+N3R3x87/hDIH7+uoM/96048exdPj4Ti4crMZaEfrnC1X6VhPHwhRh9IYZL8ed7AMXp0sljKUSZbsoxpqSkKDIy0qYuMjJSp06d8tqsY0kK9lrLbjJo0CBlZGRYC0tWAAAAAAAAAHAsrwjFfRISEhQbG2tT9/e//10JCQlubfdSfDp5nJ6erry8PLtZ95SUFLvHjB49WhEREdZSrVo1T4QKAAAAAAAAoMTKlnTOQXFueYrw8HA1bNhQDRs2lCRFR0erYcOGuvbaayVJo0aN0tSpU637f/jhh6pZs6bGjh2rOnXq6JlnnlHXrl01fvz4y+/aZfDpZStyc3O1fv16xcbGav78+ZKkoKAgxcbG6r333rN7TE5OTrG84dBV4UFBPn2+ksBX+uwrcTjiyRgZD9+LwRf6K/lOHL6iJI9HSY5dKhnx+/r3hJIwho4E4mdwcfPXfhU3d4xTII69r3+GlYRr4kqMJaFfrvLX8fCVGH0ljsK4Gp+v9wuelCfJONju3L1yyy23aPny5daf85PAU6ZM0RNPPKEqVaqoevXq1u379u1Thw4dNH78ePXr10+HDh3SP//5Ty1evNipdotbkByPitd17dpVU6dO1dNPP61ffvlF/fv3V9euXVW3bl2lpaVd8niLxaKMjAxFRUSwVg0AAAAAAAD8lsViUUpGhiLIgxVZfu5QStalk8fVAm5sfXrmsSTNnj1blStX1vDhwxUVFaVNmzapbdu2RUocAwAAAAAAAMClFe/MY3/h8zOPLxczjwEAAAAAABAImHnsvD9mHu+WdMHBnsGSrg+4sfX5mccAAAAAAAAA4F55unTyOPCQPIbXnDH2J72zWD0AAAAAAIBjheVV4CqSx/aQPAYAAAAAAAAQ4Ege20PyGAAAAAAAAECAO/97KUwpTwXiU0geAwAAAAAAAAhw50TyuCCSxwAAAAAAAAACXJ4cJ48Dc41pkscAAAAAAAAAAhzJY3tIHsNrwoOCvB0CAAAAAABAiWQvr2KxWJSSkeGFaPzBeV1MIOPPSB4DAAAAAAAACHDn5Dh5HJhp1MDsNQAAAAAAAABY5YmZxwWRPAYAAAAAAAAQ4Ege20PyGAAAAAAAAECAu9Sax4H57i6SxwAAAAAAAAACXLakXAfbL3gqEJ9C8hgAAAAAAABAgLvUshXMPAYAAAAAAACAAHSpZSuCPRWITyF5DAAAAAAAACDAXWrmMcljAAAAAAAAAAhAJI/tIXkMAAAAAAAAIMCdk5TjYDsvzAMAAAAAAACAAHSpNY9LeSoQn0LyGAAAAAAAAECAu9SyFSSPAQAAAAAAACAAkTy2h+QxAAAAAAAAgACXrYvrHuPPSB4DAAAAAAAACHCXmnnsaJv/InkMAAAAAAAAIMCRPLaH5DEAAAAAAACAAEfy2B6SxwAAAAAAAAAC3Pnfi6PtgYfkMQAAAAAAAIAAd6kX5gV5KhCfQvIYAAAAAAAAQIBj2Qp7SB4DAAAAAAAACHAkj+0J9nYAAAAAAAAAAOBdeUUozouLi1NSUpKysrK0evVqNW3a1OH+/fr1086dO3X27FkdOHBA48aNU+nSpV1quzgw8xgAAAAAAABAgLvUmsfOz8Ht2rWrxo0bpz59+mjNmjXq37+/Fi1apDp16ujo0aMF9n/kkUc0ZswY9erVS6tWrdINN9ygKVOmyBijl156yen2iwMzjwEAAAAAAAAEuOKfefziiy/q448/1pQpU7Rjxw716dNHZ8+eVa9evezu36JFC/3888+aOXOm9u/fryVLlmjmzJm69dZbXe3UZSN5DAAAAAAAACDAFS15bLFYbEpYWJjds4WGhqpJkyZaunSptc4Yo6VLl6p58+Z2j1m1apWaNGliXdoiOjpa7du313fffVdMfXQeyWMAAAAAAAAAAe68HCeOz0uSkpOTlZGRYS2DBg2ye7ZKlSopJCREqampNvWpqamKioqye8zMmTP12muv6aefflJOTo4SExO1fPlyjR49uni66ALWPAYAAAAAAAAQ4M7J8ZrHpSRJ1apVU2ZmprU2Ozu72CKIiYnR4MGDFRcXpzVr1qh27dp6++23NWTIEI0cObLY2nEGyWMAAAAAAAAAAe5S6xpf3JaZmWmTPC5Menq68vLyFBkZaVMfGRmplJQUu8eMGDFC06dP1+TJkyVJv/76q8LDwzVp0iS98cYbMsYUqSfFiWUrAAAAAAAAAAS44n1hXm5urtavX6/Y2FhrXVBQkGJjY5WQkGD3mDJlyujChQs2defPn7ce6w3MPAYAAAAAAAAQ4PLXPHa03Tnjxo3T1KlTtW7dOv3yyy/q37+/wsPDFR8fL0maOnWqkpOTNXjwYEnSggUL9OKLL2rjxo3WZStGjBihBQsWFEgqewrJYwAAAAAAAAAB7rwcJ4idTx7Pnj1blStX1vDhwxUVFaVNmzapbdu2SktLkyRVr17dJik8cuRIGWM0cuRIVatWTUePHtWCBQv073//2+m2i0uQJM8vluFBFotFGRkZioqIKNJ6JAAAAAAAAEBJZLFYlJKRoQjyYEWWnzuUaks67WDPspL2BNzYMvMYAAAAAAAAQIAr2gvzAg3JYwAAAAAAAAABrvjXPPYHwd5svFWrVvr666+VnJwsY4w6duxYYJ9hw4bp8OHDOnv2rJYsWaLatWt7IVIAAAAAAAAA/iuvCCXweDV5HB4ers2bN+vZZ5+1u/2VV17R888/rz59+qhZs2Y6c+aMFi1apNKlS3s4UgAAAAAAAAD+61wRSuDx6rIVCxcu1MKFCwvd3r9/f40cOVJff/21JOnxxx9XamqqOnXqpFmzZnkqTAAAAAAAAAD+zJyX46UpzktBngrGd3h15rEj0dHRqlKlipYuXWqty8jI0Jo1a9S8efNCjwsLC5PFYrEpAAAAAAAAAFCoC0UoAchnk8dRUVGSpNTUVJv61NRU6zZ7Bg0apIyMDGtJTk52a5wAAAAAAAAASrjzRSgByGeTx64aPXq0IiIirKVatWreDgkAAAAAAACALyN5bJdX1zx2JCUlRZIUGRlp/XP+z5s2bSr0uJycHOXk5Lg7PAAAAAAAAAD+IrcI+1zp9ih8js/OPE5KStKRI0cUGxtrrbNYLGrWrJkSEhK8GBkAAAAAAAAAv8Kax3Z5deZxeHi4ateubf05OjpaDRs21PHjx3Xw4EFNmDBBQ4YM0e7du5WUlKQRI0bo8OHDmjdvnveCBgAAAAAAAOBfLkgyDrYHeSoQ3+LV5PEtt9yi5cuXW38eP368JGnKlCl64okn9J///Efh4eGaNGmSypcvr59++klt27ZVdna2lyIGAAAAAAAA4HfOi+SxHUFyPCwlnsViUUZGhqIiIpSZmentcAAAAAAAAAC3sFgsSsnIUAR5sCLLzx3qcIRkHIxZkEWqGnhj67MvzAMAAAAAAAAAj2DmsV0kjwEAAAAAAAAENtY8tivY2QN69Ohht75UqVIaNWrUZQcEAAAAAAAAAB51vgglADmdPH7nnXc0e/ZslS9f3lp3ww03aM2aNXrkkUeKMzYAAAAAAAAAcL88SbkOSp73QvMmp5PHjRo10jXXXKOtW7fqrrvuUlxcnDZs2KCdO3eqYcOG7ogRAAAAAAAAANyHmcd2Ob3mcWJiolq2bKkJEyZo4cKFOn/+vHr06KHPP//cHfEBAAAAAAAAgHtd+L3AhtMzjyWpQ4cO6tatmxISEnTy5Ek9+eSTqlKlSnHHBgAAAAAAAADux8xju5xOHn/44Yf64osvNHbsWLVq1Up/+9vflJOTo61bt+qhhx5yR4wAAAAAAAAA4D4kj+1yetmKli1bqlmzZtqyZYskKTU1VR06dFBcXJw+/fRTffHFF8UeJAAAAAAAAAC4TZ4cJ4hLeSoQ3+J08rhJkybKyckpUP/+++9r6dKlxRIUAAAAAAAAAHjMpdY8DvJUIL7F6WUrcnJyVLNmTY0YMUKfffaZKleuLElq27atQkKczkUDAAAAAAAAgHexbIVdTieP77jjDm3dulXNmjVTly5dVLZsWUlSw4YNNWzYsGIP0N3OGFNowR8Yp+LhyTHkegElC88s4D58j3Gvkj6+JT1+ALDHk59tJeFztCTECB/gB8njq6++WtOmTVNycrJyc3OVl5dnU1zh9FThMWPGaMiQIRo/frwyMjKs9T/88IOee+45l4IAAAAAAAAAAK/JleMEsaMlLXzElClTVL16dY0YMUJHjhyRKYa/IHE6eXzTTTepe/fuBerT0tJUqVKlyw4IAAAAAAAAADyqhMwuduT2229Xq1attHnz5mI7p9PLVpw8eVJVqlQpUN+oUSMlJycXS1AAAAAAAAAA4DFGf7w0z14pAaucHDx4UEFBxftmP6eTx59//rnGjh2ryMhIGWMUHBysFi1a6P/+7/80bdq0Yg0OAAAAAAAAANzOD9Y87t+/v8aMGaMaNWoU2zmD5GTePDQ0VBMnTlTPnj1VqlQp5eXlqVSpUvrss8/Us2dPXbjgWwuAWCwWZWRkKCoiQpmZmd4OBwAAAAAAICA4eiFdeDHPjsRFFotFKRkZiiAPVmT5uUN9HSHlORizEIt0v2+P7fHjx1WmTBmFhITo7Nmzys3NtdlesWJFp8/p9JrHubm56t27t0aMGKEGDRqobNmy2rhxo/bs2eN04wAAAAAAAADgdZeaXezi33fExcVpwIABioqK0ubNm9W3b1+tXbu20P3LlSunN954Q126dFGFChW0f/9+9e/fX99///0l2+rfv79rQTrgdPI438GDB3Xw4MHijAUAAAAAAAAAPC9/bWNH253UtWtXjRs3Tn369NGaNWvUv39/LVq0SHXq1NHRo0cL7B8aGqolS5YoLS1NDz74oJKTk1WjRg2dPHmySO25Y0nhIiWP33rrrSKf8KWXXnI5GAAAAAAAAADwODfMPH7xxRf18ccfa8qUKZKkPn36qEOHDurVq5fGjh1bYP9evXqpQoUKatGihfLy8iRJ+/fvd6rN4OBgderUSfXq1ZMkbdu2TV9//bXLSw0XKXncqFEjm58bN26skJAQ7dq1S5J0ww036Pz581q/fr1LQQAAAAAAAACA11yQ4+RxsHOnCw0NVZMmTTR69GhrnTFGS5cuVfPmze0ec//99yshIUETJ05Ux44ddfToUX322WcaO3ZskZK/tWrV0nfffadq1apZ87aDBg3SwYMH1aFDByUmJjrXCRUxeXznnXda//zCCy8oMzNTPXr0sE6ZLl++vOLj47Vy5UqnAwAAAAAAAAAAr8r9vVyCxWKx+Tk7O1s5OTkF9qtUqZJCQkKUmppqU5+amqq6devaPXfNmjV15513asaMGWrfvr1q166t999/X6GhoRo+fPglY3vnnXe0d+9e3XbbbTpx4oQkqUKFCvrvf/+rd955R/fee++lO/gXTq95/NJLL+nuu++2WWvj5MmTGjJkiBYvXqxx48Y5HYQnpGRk2K3n7Z4AUHLwtmagZOGZRXHjngKAkoXPZpQoRVzzODk52ab69ddf17Bhw4olhODgYKWlpal37966cOGCNmzYoGrVqmnAgAFFSh7HxMTYJI4l6fjx4xo4cKB+/vlnl2JyOnkcERGhypUrF6ivXLlygcw7AAAAAAAAAPi8S615/PuyFdWqVVNmZqa1Ojs72+7u6enpysvLU2RkpE19ZGSkUlJS7B5z5MgR5ebm2ixRsWPHDlWpUkWhoaHKzXU8NTo7O9tufrZs2bJ2Z0cXhZOrdUhfffWV4uPj1blzZ1WrVk3VqlVTly5dNHnyZM2dO9elIAAAAAAAAADAa84XoUjKzMy0KYUlZXNzc7V+/XrFxsZa64KCghQbG6uEhAS7x/z888+qXbu2gv40a/+GG27Q4cOHL5k4lqRvvvlGkyZN0q233mqta9asmT788EN9/fXXlzzeHqeTx3369NH333+vzz77TPv379f+/fv12WefaeHChYqLi3MpCAAAAAAAAADwmjz9se6xvZLn/CnHjRunp556So8//rjq1q2rDz74QOHh4YqPj5ckTZ06VaNGjbLu/8EHH6hChQp6++23df3116t9+/YaPHiwJk6cWKT2nn/+ee3du1cJCQk6d+6czp07p59//ll79uxRv379nO+AXFi2IisrS88++6wGDBigWrVqSZL27t2rs2fPuhQAAAAAAAAAAHhVEdc8dsbs2bNVuXJlDR8+XFFRUdq0aZPatm2rtLQ0SVL16tVtlqg4dOiQ7rnnHo0fP15btmxRcnKy3n77bY0dO7ZI7Z06dUqdOnVS7dq1rS/l27Fjh/bu3et88L8LklT4Wyf8gMViUUZGhqIiImzWIwGAy1HYC3t4IQQA+A9ezgYA8BR+56C4WCwWpWRkKII8WJHl5w71ToSU42DMwizS84E3tk7PPC5TpowGDhyo2NhYXX311QoOtl35In82MgAAAAAAAACUCJd6YZ6jbV701ltv6dVXX9XZs2f11ltvOdz3pZdecvr8TiePP/nkE8XExGj69Ok6cuSIjIO/HQMAAAAAAAAAn+eGZSs8oVGjRgoNDbX+ubg5nTxu166dOnTooFWrVhV7MAAAAAAAAADgcTm/lxLmzjvvtPvn4hJ86V1snThxQsePHy/2QAAAAAAAAADAK4z+mH1sr5SAxRcmT56ssmXLFqgvU6aMJk+e7NI5nX5h3qOPPqqOHTuqR48eysrKcqlRTyopL8wryS/fKskL+7sj9pI8HsXN1bFgDOHJz0Tut5LFV66Xr8ThrxhfwL1cecZ85bn0lTg8KRD7DOAPzn4G8MI851lfmDciQsp2MGalLdKrvj22eXl5qlKlio4ePWpTX7FiRaWkpFiXt3CG08tWvPTSS6pVq5ZSU1O1b98+5ebm2mxv0qSJ00EAAAAAAAAAgNeU0DWPpYsJ8KCgIAUFBclisejcuXPWbaVKlVL79u2Vlpbm0rmdTh7PmzfPpYYAAAAAAAAAwCddas1jH/7HHidPnpQxRsYY/fbbbwW2G2M0dOhQl87tdPJ4+PDhLjUEAAAAAAAAAD6pBM88btOmjYKCgvTDDz/ogQcesHlfXU5Ojvbv368jR464dG6nk8cAAAAAAAAA4FfO/14cbfdRK1askCRFR0fr4MGDMg7WynZWkZLHx44d0w033KBjx47p+PHjDgOoWLFisQUXSEryiw6I3f3nLKlcHQvGEJ68B7jfShZfuV6+Eoe/YnwB93LlGfOV59JX4vCkQOwzgD/wGeBBF+Q4QezDM4/zHThwQJJ05ZVXqnr16goLC7PZvnXrVqfPWaTk8QsvvGB9i2D//v2dbgQAAAAAAAAAfFbu76UwpTwViOsqVaqk+Ph4tWvXzu72kBDnF6Eo0hHTpk2z+2cAAAAAAAAAKPFK8JrH+SZMmKDy5curWbNmWr58uTp37qzIyEgNGTJEL730kkvnZM1jAAAAAAAAAIGtBK95nO/OO+9Ux44dtX79el24cEH79+/X0qVLlZGRoUGDBum7775z+pzBboizyAYOHKhffvlFGRkZSk1N1VdffaUbbrjBZp/SpUvrvffeU3p6ujIzM/Xll1/q6quv9lLEAAAAAAAAAPzO+SIUHxceHq60tDRJ0okTJ1S5cmVJF9c6bty4sUvn9GryOCYmRhMnTtRtt92mv//97woNDdXixYtVpkwZ6z7jx4/Xfffdp4ceekgxMTGqWrWq5s6d68WogZLtjDGFFsAR7hv/wbUEAhvfBQD4Aj6LAPicC0UoPm7Xrl2qU6eOJGnz5s16+umnVbVqVfXp00dHjhxx6ZxeXbbir4s39+zZU0ePHlWTJk20cuVKRURE6Mknn1T37t31448/SpKeeOIJ7dy5U82aNdOaNWu8ETYAAAAAAAAAf5IrKcfB9hLwwry3335bVapUkSQNGzZMCxcu1KOPPqqcnBz17NnTpXNeVvL4mmuukSQdOnTock5jVa5cOUnS8ePHJUlNmjRRWFiYli5dat1n165d2r9/v5o3b243eRwWFqbSpUtbf7ZYLMUSGwAAAAAAAAA/5QcvzJsxY4b1zxs2bFCNGjVUt25dHThwQMeOHXPpnE4vW1GqVCkNHz5cJ0+e1L59+7Rv3z6dPHlSI0aMUEiI67nooKAgTZgwQT/99JO2bdsmSYqKilJ2drZOnTpls29qaqqioqLsnmfQoEHKyMiwluTkZJdjAgAAAAAAABAA/GDN45YtW9r8nJWVpY0bN7qcOJZcSB6/++676t27t1555RU1atRIjRo10iuvvKInn3xS77zzjsuBTJw4UQ0aNFC3bt1cPockjR49WhEREdZSrVq1yzofAAAAAAAAAD/nB2se//DDD0pMTNQbb7yhevXqFcs5nZ4q3L17d3Xr1k0LFy601m3dulUHDx7UzJkzFRcX53QQ7777ru69917dcccdNjOFU1JSVLp0aZUrV85m9nFkZKRSUlLsnisnJ0c5OY4WKAEAAAAAAACAP8lRiV/zuGrVqurWrZseeeQRDRw4UFu2bNGMGTM0c+ZMl1dncHrmcXZ2tvbt21egPikpyaWk7bvvvqvOnTvrzjvvLHDe9evXKycnR7Gxsda6G264QTVq1FBCQoLTbeEi3mgb2MKDggotcK+S/kZp7pvL5yv3ANcSKFmK+7OD7wKAe/nK7/vi5snPIn8dQwA+zsjxrOMS8BF07NgxTZw4Ubfffrtq1aqlL774Qj169NC+ffu0bNkyl87pdPL4vffe06uvvqqwsDBrXVhYmP7973/rvffec+pcEydO1D/+8Q91795dmZmZioyMVGRkpK644gpJUkZGhiZPnqxx48apdevWaty4seLj47Vq1Sq7L8sDAAAAAAAAAKf5wZrHf7Zv3z6NGTNGAwcO1NatWxUTE+PSeYq0bMWcOXNsfr7rrrt06NAhbd68WZLUsGFDhYWFOZ3Bzl/i4n//+59Nfc+ePTV16lRJ0gsvvKALFy5ozpw5Kl26tBYtWuTS0hgAAAAAAAAAYNelEsQlKHncokULPfroo3rwwQd1xRVXaP78+Ro0aJBL5ypS8vjP6w1LBZPJBw8edKnxoCL807js7Gw999xzeu6551xqAwAAAAAAAAAcyv29ONru40aNGqVu3bqpatWqWrJkifr166f58+crKyvL5XMWKXncq1cvlxsAAAAAAAAAAJ+Wv7axo+0+7o477tCbb76p2bNn69ixY8VyziIlj+GbHL0swNELT3gZCvAHV58jV/DsuZcnr6WrijsOX+mzr8RRkjGGcMTRPVDYvVPS7xt/7Zcr+HwoeXzluhT3c+TJfvnrdyZfwXhcPsbQT/nBshW33357sZ/TpeTxAw88oK5du6p69eo2L86TpCZNmhRLYAAAAAAAAADgERfkOEHsozOP77vvPn3//ffKy8vTfffd53DfBQsWOH1+p5PHffv21RtvvKEpU6aoY8eOio+PV61atdS0aVNNnDjR6QAAAAAAAAAAwKtK6LIV8+bNU1RUlI4ePap58+YVup8xRiEhzs8jdvqIuLg49e7dW59//rl69uyp//znP0pKStKwYcNUoUIFpwMAAAAAAAAAAK/K+b0UJtRTgTinVKlSdv9cXIKdPaB69epatWqVJCkrK0sWi0WSNH36dD3yyCPFGx0AAAAAAAAAuNuFIhQf99hjjxVYYliSQkND9dhjj7l0TqdnHqekpKhChQo6cOCADhw4oNtuu01btmxRdHS0glgU3KNYhB24fDxH/iMQr6Wv9NlX4ijJGEO4yl/vHX/tlysYC7iKe+cPjIUtxuPyMYZ+yg9emBcfH6+FCxfq6NGjNvUWi0Xx8fGaPn260+d0eubxDz/8oPvvv98a0Pjx47V48WLNmjVLX331ldMBAAAAAAAAAIBXuWnmcVxcnJKSkpSVlaXVq1eradOmRTru4YcfljHGqXxrUFCQjDEF6q+55hqdOnWqyOf5M6dnHvfu3VvBwRdzzu+//76OHTumFi1a6Ouvv9ZHH33kUhAAAAAAAAAA4DW5crzmccHVIC6pa9euGjdunPr06aM1a9aof//+WrRokerUqVNgdvCf1ahRQ//3f/+nFStWFKmdDRs2yBgjY4yWLVumvLw867ZSpUopOjpaCxcudL4DciF5bIzR+fN/zNOeNWuWZs2a5VLjAAAAAAAAAOB1l5pd7MLM4xdffFEff/yxpkyZIknq06ePOnTooF69emns2LF2jwkODtaMGTM0dOhQtWrVSuXLl79kO/PmzZMk3XzzzVq0aJFOnz5t3ZaTk6N9+/Zpzpw5zndALiSPe/bsqdOnT+vLL7+0qX/wwQdVpkwZTZs2zaVAAAAAAAAAAMArirjmscVisanOzs5WTk7BKcuhoaFq0qSJRo8eba0zxmjp0qVq3rx5oc289tprSktL06effqpWrVoVKfThw4dLkvbt26fPP//cbjyucnrN40GDBik9Pb1AfVpamgYPHlwsQQEAAMC/nTGm0AIAAAB43PkiFEnJycnKyMiwlkGDBtk9XaVKlRQSEqLU1FSb+tTUVEVFRdk9pmXLlnryySf11FNPudSF7du36+abby5Qf+utt6pJkyYundPp5HH16tWVlJRUoH7//v2qXr26S0EAAAAAAAAAgNcU8YV51apVU0REhLX8eWbx5ShbtqymT5+up556SseOHXPpHBMnTtS1115boL5atWqaOHGiS+d0etmKtLQ0/e1vf9P+/ftt6hs2bOhyxwAAAAAAAADAa/J08aV5jrZLyszMVGZm5iVPl56erry8PEVGRtrUR0ZGKiUlpcD+tWrVUnR0tBYsWGCtCw6+OO83NzdXderUUWJiosM269evrw0bNhSo37hxo+rXr3/JmO1xeubxzJkz9c4776h169YKDg5WcHCw2rRpo7fffluff/65S0EAAAAAAAAAgNcUcdmKosrNzdX69esVGxtrrQsKClJsbKwSEhIK7L9z5041aNBAN998s7V8/fXX+vHHH3XzzTfr4MGDl2wzOzu7QLJakqpUqaK8vDznOvA7p2cev/rqq7ruuuu0bNkya6PBwcGaNm0aax4DAAAAAAAAKHmK+MI8Z4wbN05Tp07VunXr9Msvv6h///4KDw9XfHy8JGnq1KlKTk7W4MGDlZ2drW3bttkcf/LkSUkqUF+YxYsXa/To0erYsaMyMjIkSeXKldOoUaO0ZMkS5zsgF5LHubm56tatm1599VU1bNhQWVlZ2rp1qw4cOOBSAAAAAAAAAADgVX9a17jQ7U6aPXu2KleurOHDhysqKkqbNm1S27ZtlZaWJuniu+UuXHDhxIV4+eWXtWLFCu3fv18bN26UJN18881KTU3VY4895tI5gyT59SutLRaLMjIyFBURUaT1SP7Mlbd9hwcFOX2MP3M0howV4D48e8WjsHEMxDH05D3F/QuguPG5AsAXlITPopIQoyd58v8Hiqsti8WilIwMRbiQBwtU+blDlY+QHI2ZxSKd9P2xLVOmjB599FHrpN8tW7Zo5syZLi9b4fSax5L02GOPacuWLcrKylJWVpY2b96sf/zjHy4FAAAAAAAAAABedaEIpQQ4e/asfvrpJy1YsEArVqzQyZMn1a5dO913330unc/pZSteeOEFjRgxQu+9955+/vlnSdLtt9+uDz/8UJUqVdKECRNcCgQAAAAAAAAAvKEoSxqXcnsUlyc6OlpfffWVbrrpJhljFBQUJPOnWe0hIU6ngp1PHvft21fPPPOMpk+fbq1bsGCBtm3bptdff53kMQAAAAAAAIASxR+Sx2+//baSkpIUGxurpKQkNWvWTBUqVNBbb72ll19+2aVzOp08rlKlilatWlWgftWqVapSpYpLQQAAAAAAAACAt+QWYZ8wt0dxeZo3b64777xTx44d04ULF3T+/Hn9/PPPGjRokN555x01btzY6XM6nTzes2ePunbtqtGjR9vUP/zww9q9e7fTAfiyQHsBkDvi84V+Sbz0KtD5+rPnDv7aL09jHP/gybHw5ReQlCSB2GegMNz3JUsgfndDYCgJ929JiNGTSvp3YDinhCxp7FCpUqWsL/NLT09X1apV9dtvv2n//v2qU6eOS+d0Onk8dOhQzZo1S3fccYd1zeOWLVsqNjZWXbt2dSkIAAAAAAAAAPCWoixb4et+/fVXNWzYUPv27dOaNWv0yiuvKCcnR71791ZiYqJL5wwu6o433nijJGnu3Llq1qyZ0tPT1alTJ3Xq1Enp6em69dZbNW/ePJeCAAAAAAAAAABvuaCLCeTCSkmYmTxy5EgFB19M97722muKjo7WypUr1b59ez3//PMunbPIM4+3bNmitWvX6pNPPtHnn3+uxx57zKUGAQAAAAAAAMCXXJBU+MJJUklYWGTx4sXWP+/du1f16tXTVVddpRMnTrh8ziLPPI6JidG2bdv01ltv6ciRI4qPj9ftt9/ucsMAAAAAAAAA4AtyilBKostJHEsXk+aOkuoFlClTRl27dlXPnj3VqlUr7dmzR5MnT9bUqVOVmpp6WcG4g8ViUUZGhqIiIqwLRgMoXv76UqZAfFlLIPbZX3EtL5+vjKGvxOGvint8feV6+UoccC+u8+VjDAH4G4vFopSMDEWQByuy/NzhoYgIGQdjFmSx6JoAHNsizzzOd/bsWU2ZMkWtW7fWDTfcoC+++ELPPvusDhw4oPnz57sjRgAAAAAAAABwG0frHeeXQOR08vjP9u7dq1GjRmnkyJHKzMxUhw4diisuAAAAAAAAAPCIC0UogajIL8z7q1atWqlXr1564IEHdOHCBc2ePVuTJ08uztgAAAAAAAAAwO1y5ThBfFkzcEswp5LHVapUUc+ePdWzZ0/Vrl1bq1at0vPPP6/Zs2fr7Nmz7ooRAAAAAAAAANwmkGcXO1Lk5PF3332nu+66S+np6Zo2bZo+/fRT/fbbb+6MDXAKL7u4fK6Oob+Or7++2IhnJTBwLYumJDwPvhKHIyX5xanFHaOj83nyfisJYx+Iivse4DoXna98TvlKHMXNX/vlSCD2ubiVhO9hCCzn5Th5XPgd69+KnDzOzc3Vgw8+qG+++UYXLpCHBwAAAAAAAOAfSB7bV+TkcceOHd0ZBwAAAAAAAAB4BWse2+fyC/MAAAAAAAAAwB8YOU4eB+piKiSPAQAAAAAAAAS0878X2CJ5DAAAAAAAACCgkTy2j+Qx/AZvY718jKH3eHLsPdkWb1CGr+M+LDqe58tX0sepsHugpPfLkxgr7/GVsfeVOIqbv/bLkUDsc3FjDOFrLohlK+wheQwAAAAAAAAgoOXI8czjUp4KxMeQPAYAAAAAAAAQ0Jh5bF+wNxvv06ePNm/erFOnTunUqVNatWqV2rZta91eunRpvffee0pPT1dmZqa+/PJLXX311V6MGAAAAAAAAIC/uaA/1j22Vxwllv2ZV5PHhw4d0sCBA9WkSRPdcsst+uGHHzR//nzVr19fkjR+/Hjdd999euihhxQTE6OqVatq7ty53gwZAAAAAAAAgJ+5UIQSiIIkFf72Ey84duyYBgwYoC+//FJHjx5V9+7dNWfOHElSnTp1tHPnTt12221as2ZNkc5nsViUkZGhqIgIZWZmujN0AAAAAAAAwGssFotSMjIUQR6syPJzh19HRCjPwZiFWCy6PwDH1qszj/8sODhYDz/8sMLDw5WQkKAmTZooLCxMS5cute6za9cu7d+/X82bNy/0PGFhYbJYLDYFAAAAAAAAAArDzGP7vJ48btCggTIzM5Wdna0PP/xQnTt31o4dOxQVFaXs7GydOnXKZv/U1FRFRUUVer5BgwYpIyPDWpKTk93dBQAAAAAAAAAlmKP1jvNLIPJ68njXrl26+eab1axZM33wwQeaOnWq6tWr5/L5Ro8erYiICGupVq1aMUYLAAAAAAAAwN+QPLYvxNsB5Obmau/evZKkDRs2qGnTpurXr59mzZql0qVLq1y5cjazjyMjI5WSklLo+XJycpSTk+P2uAEAAAAAAAD4ByPHS1P41EvjPMjryeO/Cg4OVunSpbV+/Xrl5OQoNjZWc+fOlSTdcMMNqlGjhhISErwcJVC8zpjCP4LCg4I8GInvY6z8B9cSAPxfSf+sLyz+khC7qwKxz76gpD8rAOAPcn8vsOXVZStGjRqlVq1aqUaNGmrQoIFGjRql1q1ba8aMGcrIyNDkyZM1btw4tW7dWo0bN1Z8fLxWrVqlNWvWeDNsAAAAAAAAAH7EXctWxMXFKSkpSVlZWVq9erWaNm1a6L7//Oc/tWLFCh0/flzHjx/XkiVLHO7vCV5NHl999dWaNm2adu3apWXLlqlp06a65557tHTpUknSCy+8oG+++UZz5szRihUrlJKSoi5dungzZAAAAAAAAAB+xh3J465du2rcuHEaNmyYGjdurM2bN2vRokWqXLmy3f1bt26tmTNnqk2bNmrevLkOHjyoxYsXq2rVqq51qhgEyc+X7LBYLMrIyFBURIQyMzO9HQ5gF/9MregYK//BtQQA/1fSP+sDcQmHQOyzLyjpzwoA32GxWJSSkaEI8mBFlp87/CQiQrkOxizUYtE/nRzb1atXa+3aterbt68kKSgoSAcPHtS7776rsWPHXvL44OBgnThxQs8995ymT59etA4VM6/OPAYAAAAAAAAAb8uVlOOg5K+HbLFYbEpYWJjd84WGhqpJkybWFRYkyRijpUuXqnnz5kWKqUyZMgoNDdXx48dd7dZl87kX5qF4+OvfXPtrvxzF7q99dlUg9tlfefJa8hwBgHeU9M/Ykh6/K1zpcyD+ni3uPvvrOMH9+NcCQPG58HtxtF2SkpOTbepff/11DRs2rMD+lSpVUkhIiFJTU23qU1NTVbdu3SLFNHbsWB0+fNgmAe1pJI8BAAAAAAAABLRLrWucv61atWo2y1ZkZ2e7JZ5//etf6tatm1q3bu22NoqC5DEAAAAAAACAgFbUmceZmZlFWvM4PT1deXl5ioyMtKmPjIxUSkqKw2NfeuklDRw4UHfddZe2bt16ybbciTWPAQAAAAAAAAQ0R+sd5xdn5Obmav369YqNjbXWBQUFKTY2VgkJCYUeN2DAAL366qtq27at1q9f72SrxY+ZxwAAAAAAAAACWlFnHjtj3Lhxmjp1qtatW6dffvlF/fv3V3h4uOLj4yVJU6dOVXJysgYPHixJeuWVVzR8+HB1795d+/bts85aPn36tM6cOeNCBJeP5LGf8tfF8f21X44EYp+B4sZz5D2OXihUGK4XHPHXF3P5a78c8dc++0q/PBlHIL78uSTH7khJuF4lIUZPCrQ+l4Tr78r3X8l34g9kF+R4zWNXksezZ89W5cqVNXz4cEVFRWnTpk1q27at0tLSJEnVq1fXhQt/nPmZZ55R6dKlNWfOHJvzFPZSPk8geQwAAAAAAAAgoBX1hXnOmjhxoiZOnGh3W5s2bWx+jo6OdrEV9yF5DAAAAAAAACCguWPZCn9A8hgAAAAAAABAQMv9vRQm2FOB+BiSxwAAAAAAAAACmruWrSjpSB4DAAAAAAAACGgkj+0jeQwAANyGt0ajuPnrPeWv/XKkuPvs6O32nhxfX7mWxAFXlITn0tfvKV/5LCrpChvHkjCGJSFG2GfkeF3jwp9u/0byGAAAAAAAAEBAy/m9FIY1jwEAAAAAAAAgAF2Q45nHjrb5M5LHAAAAAAAAAAIaax7bR/IYAAAAAAAAQEBj5rF9JI8BeAUvkgCKhmcFAIqGz0TA9wTicxmIfXYHxhHekCvHax6X8lQgPobkMQAAAAAAAICAxsxj+0geAwAAAAAAAAhorHlsH8ljAAAAAAAAAAGN5LF9JI8BAAAAAAAABDSWrbCP5DEAr+AFCEVX2AvTGMPAwHUGAADwDl5cDASW3N9LYQI1iRqo/QYAAAAAAAAASRdnFjtamoKZxwAAAAAAAAAQgFjz2D6SxwAAAAAAAAACGmse20fyGAAAAAAAAEBAy/m9FCZQk6iB2m8AKDF4GQcAAADgeXwPBwILM4/tI3kMAAAAAAAAIKCx5rF9JI8BAAAAAAAABDQjx7OLjacC8TEkjwEAAAAAAAAENGYe20fyGAAAAAAAAEBAy/29ONoeiEgeAwAAAAAAAAhozDy2j+Qx4IIzpvCVbngjL+D/+AyAI75+f/h6fAAAAIA3kDy2j+QxAAAAAAAAgIB2QY5fmOdomz8jeQwAAAAAAAAgoOXJ8brGeZ4KxMeQPAYAAAAAAAAQ0Fi2wj6SxwAAAAAAAAACGslj+4K9HQBQEoUHBRVaAPg/PgPgiK/fH74eHwAAAOANF4pQXBEXF6ekpCRlZWVp9erVatq0qcP9H3zwQe3YsUNZWVnasmWL2rVr52LLxYPkMQAAAAAAAICAlispx0FxtB5yYbp27apx48Zp2LBhaty4sTZv3qxFixapcuXKdvdv3ry5Zs6cqcmTJ6tRo0aaN2+e5s2bpxtvvNG1ThWDIEnGa617gMViUUZGhqIiIpSZmentcAAAAAAAAAC3sFgsSsnIUAR5sCLLzx1Wv8SYWSwWHXBybFevXq21a9eqb9++kqSgoCAdPHhQ7777rsaOHVtg/88//1zh4eG67777rHUJCQnatGmTnnnmGSd7VjyYeQwAAAAAAAAgoJ0vQnFGaGiomjRpoqVLl1rrjDFaunSpmjdvbveY5s2b2+wvSYsWLSp0f08ImBfmlbVYvB0CAAAAAAAA4Dbkv1wXHhHhcF3j8IgISRdnIP9Zdna2cnJyCuxfqVIlhYSEKDU11aY+NTVVdevWtdtGVFSU3f2joqKK0AP38Pvkcf4F3ZOc7OVIAAAAAAAAAPezWCwsW1FEOTk5OnLkiH47dOiS+2ZmZir5LznG119/XcOGDXNXeF7n98njw4cPq1q1asrMzJTFYlFycrL1Z3fy17Y83R5t0RZt0RZteac92qIt2qIt2vJOe7RFW7RFW7Tlnfb8qS2LxaLDhw8X+3n9VXZ2tqKjoxUWFuby8fakp6crLy9PkZGRNvWRkZFKSUmxe0xKSopT+3uC3yePJRV4YDIzMz32ty/+2pan26Mt2qIt2qIt77RHW7RFW7RFW95pj7Zoi7Zoi7a8054/tOXJa+MvsrOzC00Cuyo3N1fr169XbGys5s+fL+niC/NiY2P13nvv2T0mISFBsbGxevvtt611f//735WQkFCssTkjIJLHAAAAAAAAAOBJ48aN09SpU7Vu3Tr98ssv6t+/v8LDwxUfHy9Jmjp1qpKTkzV48GBJ0ttvv63//e9/evHFF/Xtt9+qW7duuuWWW9S7d2+v9YHkMQAAAAAAAAAUs9mzZ6ty5coaPny4oqKitGnTJrVt21ZpaWmSpOrVq+vChT9e05eQkKDu3btr5MiRGjVqlHbv3q1OnTpp27Zt3uqCJMkESgkLCzNDhw41YWFhtFVC2qMt2qIt2qIt/+8bbdEWbdGWL7flz32jLdqiLdry5bb8uW+eHkcK5XJK0O9/AAAAAAAAAADAKtjbAQAAAAAAAAAAfA/JYwAAAAAAAABAAQGTPC5VqpRee+017dixQ1u3btXGjRv10UcfqVy5ct4OrVgkJSWpYcOGbm+nbNmyyszM1CeffOL2tjwlKSlJO3fu1MaNG62lQYMGbmvPk2PoqbaSkpK0Y8cOlSpVylq3du1axcTEuKWt/Ou1c+dO/etf/yr2Nv7clqf6FRoaqjFjxmj37t3avn27tmzZoscff7zY2/F0W5LtNdu2bZvi4uLc1taf23TnZ+J///tfvfHGGzZ133zzjV566aVib+vbb7/Vs88+W6B+06ZN6ty5c7G35yn5n7fbtm1TXl6e9efPP/+82NtKSkpSamqqQkL+eE9w69atZYzR+PHji7UtT94b3uDJ59lTv5/99bPek/3K56nvHZ78Xu/J7x2e5Knny5Of9d7QuXNnrVu3Ths3btSOHTu0bNkyBQUFuaUtf7xmISEh1mf5119/1YYNG/TVV1+5/f9r3f090RhT4PPI3W168l70BG+MoSf4+2ci/JvXF172RJkyZYr5+uuvTfny5a11Dz74oImOjvZ6bMVRkpKSTMOGDd3ezpNPPmmWL19ujh8/bsLDw73e75I0dt4YQ0+1lZSUZBITE03v3r2tdWvXrjUxMTFuvV5Vq1Y1J0+eNE2bNi3x/ZoxY4b58ssvTZkyZYwkU6NGDbN9+3bTq1evEt3WX69Z9erVzcmTJ81NN93klrbstemOUr58ebNv3z5zyy23GEmmV69eZsWKFSYoKKjY2+rSpYtZt26dTV2TJk1MamqqCQkJces4eqLUqFHDnDhxwu33w9q1a02XLl2sddOnTze//PKLGT9+fIm9N7xRPPk8e+r3s79+1nuyX/nFU987PPm93pPfOzxZPP391xOf9Z4uUVFR5ujRo6Z69erWukaNGnHNnCjTp083X331lc2zHBsba7p27erWdt09lsYYU65cOY+16el70RPF02Po6eKPn4kUvy9eD8DtpVatWubMmTOmYsWKXo/FXcVTH6SrVq0y7dq1M/PmzXNboslfx84bY+iptpKSkszjjz9uDh48aK688kojeSZ5LMmsXr3aPPjggyW6X7Vr1zZnzpwxFSpUsKlv166d2b9/f4ltq7BrtmbNGvPAAw+4pa3C2nRHueeee8yvv/5qateubQ4dOuS2v4wMCQkxqampNgm6iRMnmjfffNOt/fNU8VTyOC4uznzzzTdGkomIiDC7d+82w4cPL/bksSfvDW8UTz7Pnkwe+9tnvSf79efiie8dnv5e78nvHZ4s/piI9HRp2LChSUlJMWXLluWauVBq165tTp8+ba666iqPXzt/Sx57+l70RCF5TKH4VgmIZSsaN26s3bt369ixY94OpUSrV6+err32Wi1atEiTJ0/Wk08+6e2Qis2sWbNs/gnYFVdc4ZZ2PDmGnr5emzdv1o8//qgXXnjBre38WZ06dVSxYkUtX77cbW14ol+NGjXS7t27dfz4cZv6hIQEVa9eXZUqVSqRbdnToEED1a1bV5s3b3ZrO56waNEirVy5UmvXrtXw4cOVlJTklnby8vI0ffp09erVS5JUunRpPfLII5o8ebJb2vNXP//8s6677jpVqVJFjzzyiL744gudP3/eLW156t7wNk88z576/exvn/X5PPm72VPfO7z5vd4T3zs8yVPPl7/asmWLfvrpJ+3fv19z587Vyy+/rKpVq7q1TX+6Zo0aNdKePXt04sQJb4dS4nnjXgQQWAIieYzi8eSTT2ratGm6cOGCvvvuO0VHR6tu3breDqtYPPzww2rUqJG1nDt3zi3teHIMvXG9Xn31VfXr108VKlRwazuzZs3S9u3btX37dr377rtKT093a3ue6pc/y/+fnY8++ki9evXSnj17vB1SsXjzzTcVHBysSZMmubWdyZMn69FHH1VoaKi6dOmiHTt2aOfOnW5t0x9Nnz5dPXv2VK9evfTpp5+6tS1P3Rve4Mnn2VO/nyX//az3VL/8+Xuip793eIonny9/ZIzRgw8+qBYtWmjhwoVq2bKltm3bplq1armtTX++ZjVr1rSuLe7u39HeYoxx23k9fS96i7vGEIBjAZE83rBhg66//nq/+58BTwoJCdFjjz2mHj16KCkpSXv27FGZMmX8avaxu3lyDL11vfbv36/PPvtMQ4YMcWs7Dz/8sOrXr6+7775bY8aMcesLDiX392vjxo12P6OaN2+uAwcOFOv/pHqyrT/L/5+dli1bas6cOW5pwxvOnz+vCxcuuL2dHTt2aM+ePbrvvvvUq1cvZh27aNq0aXr++ed17tw5t/8FhqfuDW/w1+fZnz7r/8wTv5s9+b3DG9/rPf29AyXLrl27NGnSJHXu3FmrV6/W/fff7+2QSoSNGzeqdu3aKl++vCQpMTFRjRo10ujRo3XVVVd5N7jLlJaWpooVK9rUVapUSWlpaW5t15/uRW+NIQD7AiJ5vHfvXs2ZM0eTJ0+2eWNnly5dFB0d7cXISo77779fiYmJuuaaaxQdHa3o6Gjddttteuyxx2zeXo/CeXIMvXm9Ro4cqX/84x8e+adSy5Yt0wcffKCRI0e6vS139mvPnj1asGCBJk2apCuvvFKSVKNGDb311lsaMWJEiW0LxWvy5MkaPHiwbr31Vs2aNcvb4ZRIR44c0aBBg/Svf/3L26HAR/nLZ/1fuft3sye/d3jze70nv3fA91WtWlUtWrSw/ly+fHlFR0dr7969Xoyq5NizZ4/mz59f4FkODw/3YlTFY9GiRXr66aetPz/22GNKTExUSkqKW9rzx3vR02MIwLGASB5LUq9evbR582atWbNGv/76q7Zv36677767wLpzsO/JJ5/UjBkzbOp27typ5ORk3XfffV6Kqvj8df2w1q1bF3sbnhxDb16vY8eO6Z133vHYOlsjRozQ7bffrsaNG7u1HXf36/HHH1diYqK2bt2q7du365tvvtGbb76pTz75pES3heIza9Ys1alTR1988YXOnDnj7XBKrClTpmj16tXeDgNF5Infz3/mT5/1f+bufnn6e4c3v9d76nuHJ3j6+fI3ISEheu2117Rr1y5t3LhRK1eu1NSpU/X111+7rU1/u2Y9e/bU1q1brc/yypUrddddd2ns2LHeDu2y9O/fX1WqVNHmzZu1ceNGde/eXQ899JDb2vPGvehunh5DAI4F6eKb8wAAAAAAAAAAsAqYmccAAAAAAAAAgKIjeQwAAAAAAAAAKIDkMQAAAAAAAACgAJLHAAAAAAAAAIACSB4DAAAAAAAAAAogeQwAAAAAAAAAKIDkMQAAAAAAAACgAJLHAAAAKDYxMTEyxqhcuXIO90tKSlK/fv08FBUAAAAAV5A8BgAACEDx8fEyxsgYo+zsbO3evVuvvvqqSpUqdVnnXbVqlaKionTq1ClJUo8ePXTixIkC+zVt2lSTJk26rLYAAAAAuFeItwMAAACAd3z//fd64oknVLp0abVv314TJ05Ubm6uxowZ4/I5c3NzlZqaesn90tPTXW4DAAAAgGcw8xgAACBAZWdnKzU1VQcOHNCHH36opUuX6v7771f58uU1depUHT9+XGfOnNF3332n2rVrW4+rXr26vv76ax0/flynT5/Wr7/+qnbt2kmyXbYiJiZGU6ZMUfny5a2znIcOHSqp4LIV1157rebNm6fMzEydOnVKs2bN0tVXX23dPnToUG3cuFH/+Mc/lJSUpJMnT2rmzJkqW7ash0YLAAAACDwkjwEAACBJysrKUlhYmKZMmaJbbrlF999/v5o3b66goCB99913Cgm5+I/WJk6cqNKlS+uOO+7QTTfdpH/96186ffp0gfOtWrVK/fr106lTpxQVFaWoqCj93//9X4H9goKCNH/+fFWoUEExMTH6+9//rpo1a2rWrFk2+9WqVUudOnXSvffeq3vvvVcxMTEaOHCgewYDAAAAAMtWAAAAQIqNjdU999yj77//Xp07d1aLFi2UkJAgSXr00Ud18OBBderUSV9++aWqV6+uOXPm6Ndff5V0cRaxPbm5uTp16pSMMQ6XsoiNjdVNN92k6OhoHTp0SJL0+OOPa/v27brlllu0bt06SVJwcLB69uxpTVRPnz5dsbGxGjJkSLGNAwAAAIA/MPMYAAAgQN17773KzMzUuXPn9P3332vWrFmaMmWKcnNztWbNGut+x48f165du1SvXj1J0jvvvKMhQ4bop59+0uuvv66bbrrpsuKoV6+eDh48aE0cS9KOHTt04sQJa5uStG/fPpsZzkeOHLFZ2gIAAABA8SJ5DAAAEKB+/PFH3Xzzzbr++ut15ZVXqmfPnjLGXPK4yZMnq2bNmpo+fbpuuukmrVu3Ts8995zb483NzbX52Rij4GC+zgIAAADuwrdtAACAAHXmzBnt3btXBw8e1Pnz5yVdnPEbGhqqZs2aWferUKGC6tSpo+3bt1vrDh06pI8++kgPPPCA3nrrLT311FN228jJyVGpUqUcxrFjxw5de+21uuaaa6x19erV01VXXWXTJgAAAADPInkMAAAAqz179mjevHn6+OOP1bJlS/3tb3/Tf//7XyUnJ2v+/PmSpPHjx+vuu+/Wddddp0aNGqlNmzbasWOH3fPt27dPFotFd955pypWrKgrr7yywD5Lly7V1q1bNWPGDDVq1EhNmzbVtGnTtHz5cq1fv96t/QUAAABQOJLHAAAAsPHEE09o/fr1+uabb5SQkKCgoCC1b99eeXl5kqRSpUpp4sSJ2rFjhxYuXKjffvtNcXFxds+VkJCgDz74QLNmzVJ6erpeeeUVu/t17NhRJ06c0IoVK7R06VIlJibq4YcfdlsfAQAAAFxakKRLL2wHAAAAAAAAAAgozDwGAAAAAAAAABRA8hgAAAAAAAAAUADJYwAAAAAAAABAASSPAQAAAAAAAAAFkDwGAAAAAAAAABRA8hgAAAAAAAAAUADJYwAAAAAAAABAASSPAQAAAAAAAAAFkDwGAAAAAAAAABRA8hgAAAAAAAAAUADJYwAAAAAAAABAASSPAQAAAAAAAAAF/D9vZoKpDJRUMAAAAABJRU5ErkJggg=="
     },
     "metadata": {},
     "output_type": "display_data",
     "jetTransient": {
      "display_id": null
     }
    }
   ],
   "execution_count": 4
  },
  {
   "cell_type": "code",
   "id": "c33-0005",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-05-31T20:52:50.343489Z",
     "iopub.status.busy": "2026-05-31T20:52:50.343302Z",
     "iopub.status.idle": "2026-05-31T20:52:50.358994Z",
     "shell.execute_reply": "2026-05-31T20:52:50.358199Z"
    },
    "ExecuteTime": {
     "end_time": "2026-05-31T21:21:10.598141935Z",
     "start_time": "2026-05-31T21:21:10.541037200Z"
    }
   },
   "source": [
    "# \u2500\u2500 Prepare sequences for StackRnn \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n",
    "# Tile the single sentence N_REPEAT times to form a real batch so GPU/CPU\n",
    "# matrix ops are (N_REPEAT, 168) instead of (1, 168).\n",
    "N_REPEAT  = 500\n",
    "sequences = np_cpu.tile(sensory_np[np_cpu.newaxis, :, :], (N_REPEAT, 1, 1))\n",
    "print(f\"sequences shape : {sequences.shape}  \u2192  {N_REPEAT} \u00d7 {N_SAMPLES} chars\")"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sequences shape : (500, 180, 40)  \u2192  500 \u00d7 180 chars\n"
     ]
    }
   ],
   "execution_count": 5
  },
  {
   "cell_type": "code",
   "id": "c33-0006",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-05-31T20:52:50.360734Z",
     "iopub.status.busy": "2026-05-31T20:52:50.360559Z",
     "iopub.status.idle": "2026-05-31T20:52:50.470247Z",
     "shell.execute_reply": "2026-05-31T20:52:50.469428Z"
    },
    "ExecuteTime": {
     "end_time": "2026-05-31T21:21:10.729690925Z",
     "start_time": "2026-05-31T21:21:10.599284994Z"
    }
   },
   "source": [
    "# \u2500\u2500 Build model \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n",
    "rnn = StackRnn(PRJ_NAME, WORK_DIR)\n",
    "for layer in StackRnn.make_unrolled(\n",
    "    UNROLL_DEPTH,\n",
    "    sensory_size=SENSORY_SIZE,\n",
    "    h_size=CONTEXT_SIZE,\n",
    "    entity_params=EntityParams(\n",
    "        do_gaussian_visible=False,\n",
    "        do_gaussian_hidden=False,\n",
    "        num_gibbs_samples=NUM_GIBBS,\n",
    "    ),\n",
    "    training_params=TrainingParams(\n",
    "        learning_rate=LEARNING_RATE,\n",
    "        momentum=MOMENTUM,\n",
    "        num_epochs=NUM_EPOCHS,\n",
    "        mini_batch_size=MINI_BATCH,\n",
    "        num_gibbs_samples=NUM_GIBBS,\n",
    "        do_rao_blackwell=RAO_BLACKWELL,\n",
    "        l2_lambda=L2_LAMBDA,\n",
    "    ),\n",
    "):\n",
    "    rnn.append(layer)\n",
    "\n",
    "rnn.state_init(0.01)\n",
    "rnn.state_load()\n",
    "\n",
    "e = rnn.from_index(0).entity\n",
    "print(f\"Mode       : {'shared' if rnn.is_shared else 'unrolled'}\")\n",
    "print(f\"Layers     : {rnn.num_layers()} (alternating t % {rnn.num_layers()})\")\n",
    "print(f\"Visible    : {rnn.h_size()} (context) + {rnn.sensory_size()} (sensory) = {e.shape[0]}\")\n",
    "print(f\"Hidden     : {rnn.h_size()}\")\n",
    "print(f\"Parameters : {e.shape[0] * e.shape[1] * rnn.num_layers():,}  ({rnn.num_layers()} \u00d7 {e.shape[0] * e.shape[1]:,})\")"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mode       : shared\n",
      "Layers     : 1 (alternating t % 1)\n",
      "Visible    : 256 (context) + 40 (sensory) = 296\n",
      "Hidden     : 256\n",
      "Parameters : 75,776  (1 \u00d7 75,776)\n"
     ]
    }
   ],
   "execution_count": 6
  },
  {
   "cell_type": "code",
   "id": "c33-0007",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-05-31T20:52:50.472255Z",
     "iopub.status.busy": "2026-05-31T20:52:50.472012Z",
     "iopub.status.idle": "2026-05-31T20:54:45.425863Z",
     "shell.execute_reply": "2026-05-31T20:54:45.425132Z"
    },
    "ExecuteTime": {
     "start_time": "2026-05-31T21:21:10.731553638Z"
    }
   },
   "source": [
    "# \u2500\u2500 Train: predict next character from current context \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n",
    "# Unrolled layers alternate: layer[t % num_layers] owns each time-step position.\n",
    "#   Step 1 \u2014 advance context: h_t = layer[t%D].forward([h_{t-1} | x_t])\n",
    "#   Step 2 \u2014 CD on [h_t | x_{t+1}] using the same layer[t%D]\n",
    "from rbm.train import cd_binary_binary, _to_gpu\n",
    "from rbm.matrix import rms_error_accu\n",
    "\n",
    "h_sz             = rnn.h_size()\n",
    "num_seq, T, s_sz = sequences.shape\n",
    "n_layers         = rnn.num_layers()\n",
    "\n",
    "for idx in range(n_layers):\n",
    "    rnn.from_index(idx).entity.grad_zero()\n",
    "\n",
    "d_progress = 100.0 / NUM_EPOCHS\n",
    "progress   = 0.0\n",
    "entity0    = rnn.from_index(0).entity\n",
    "status     = CheckpointStatus(rnn.state_save, update_interval=10)\n",
    "status.on_change(entity0)\n",
    "\n",
    "for epoch in range(NUM_EPOCHS):\n",
    "    h = np.zeros((num_seq, h_sz))\n",
    "    err_total = 0.0\n",
    "\n",
    "    for t in range(T - 1):\n",
    "        entity_t = rnn.from_index(t % n_layers).entity\n",
    "        x_t   = _to_gpu(sequences[:, t,   :])\n",
    "        x_tp1 = _to_gpu(sequences[:, t+1, :])\n",
    "\n",
    "        # Step 1: advance context\n",
    "        h = entity_t.forward(np.concatenate([h, x_t], axis=1))\n",
    "\n",
    "        # Step 2: CD on [h_t | x_{t+1}]\n",
    "        vis = np.concatenate([h, x_tp1], axis=1)\n",
    "        dwhv, dbv, dbh = cd_binary_binary(entity_t, vis)\n",
    "        grad = entity_t.grad_compute(dbv, dbh, dwhv)\n",
    "        entity_t.state_adjust(grad, 1.0 / num_seq)\n",
    "        err_total += rms_error_accu(vis - entity_t.reconstruct(entity_t.forward(vis)))\n",
    "\n",
    "    progress += d_progress\n",
    "    if status.want_report(round(progress)):\n",
    "        if not status.on_change(entity0, {\n",
    "            \"progress\": {\"value\": round(progress), \"unit\": \"%\"},\n",
    "            \"err_rms\":  {\"value\": err_total / (T - 1), \"unit\": \"\"},\n",
    "        }):\n",
    "            break\n",
    "\n",
    "rnn.state_save()"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------------\n",
      "Entity-296x256: progress              : 0%\n",
      "Entity-296x256: err_rms              : 0.00012437576727553383\n",
      "Entity-296x256: l2_norm              : 195.74862335488334\n"
     ]
    }
   ],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "c33-0008",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-05-31T20:54:45.428179Z",
     "iopub.status.busy": "2026-05-31T20:54:45.427939Z",
     "iopub.status.idle": "2026-05-31T20:54:45.846509Z",
     "shell.execute_reply": "2026-05-31T20:54:45.845882Z"
    }
   },
   "source": [
    "# \u2500\u2500 Reconstruction: teacher-forced \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n",
    "# Step through each sample, reconstruct, compare to input.\n",
    "rnn.reset(batch_size=1)\n",
    "recons = []\n",
    "for i in range(N_SAMPLES):\n",
    "    x_t = np.array(sensory_np[i][np_cpu.newaxis, :])\n",
    "    h   = rnn.step(x_t)\n",
    "    rec = convert(rnn.reconstruct(h))[0]   # (40,)\n",
    "    recons.append(rec)\n",
    "\n",
    "recons_np = np_cpu.array(recons)           # (12, 40)\n",
    "decoded_recon = ''.join(idx_to_char[int(np_cpu.argmax(recons_np[i]))] for i in range(N_SAMPLES))\n",
    "\n",
    "print(f\"Input         : '{decoded}'\")\n",
    "print(f\"Reconstruction: '{decoded_recon}'\")\n",
    "\n",
    "correct = sum(decoded[i] == decoded_recon[i] for i in range(N_SAMPLES))\n",
    "print(f\"Char accuracy : {correct}/{N_SAMPLES} = {100*correct/N_SAMPLES:.0f}%\")"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "c33-0009",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-05-31T20:54:45.848729Z",
     "iopub.status.busy": "2026-05-31T20:54:45.848553Z",
     "iopub.status.idle": "2026-05-31T20:54:46.525019Z",
     "shell.execute_reply": "2026-05-31T20:54:46.524110Z"
    }
   },
   "source": [
    "# \u2500\u2500 Visualise reconstruction \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n",
    "step = max(1, N_SAMPLES // 40)\n",
    "tick_pos = range(0, N_SAMPLES, step)\n",
    "\n",
    "step = max(1, N_SAMPLES // 40)    # show at most 40 tick labels\n",
    "tick_pos = range(0, N_SAMPLES, step)\n",
    "\n",
    "plt.figure(figsize=(16, 3))\n",
    "plt.imshow(sensory_np.T, aspect='auto', cmap='hot', vmin=0, vmax=1)\n",
    "plt.colorbar(label='activation')\n",
    "plt.xlabel('Position')\n",
    "plt.ylabel('Vocab index')\n",
    "plt.title(f'Input ({N_SAMPLES} chars)')\n",
    "plt.xticks(tick_pos, [decoded[i] for i in tick_pos], fontsize=8)\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "\n",
    "plt.figure(figsize=(16, 3))\n",
    "plt.imshow(recons_np.T, aspect='auto', cmap='hot', vmin=0, vmax=1)\n",
    "plt.colorbar(label='activation')\n",
    "plt.xlabel('Position')\n",
    "plt.ylabel('Vocab index')\n",
    "plt.title(f'Reconstruction (acc {100*correct/N_SAMPLES:.0f}%)')\n",
    "plt.xticks(tick_pos, [decoded[i] for i in tick_pos], fontsize=8)\n",
    "plt.tight_layout()\n",
    "plt.show()\n"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "c33-0010",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-05-31T20:54:46.526924Z",
     "iopub.status.busy": "2026-05-31T20:54:46.526742Z",
     "iopub.status.idle": "2026-05-31T20:54:47.949550Z",
     "shell.execute_reply": "2026-05-31T20:54:47.948764Z"
    }
   },
   "source": [
    "# \u2500\u2500 Next-step prediction \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n",
    "# Given h_t (context after seeing x_t), predict x_{t+1} via clamped Gibbs.\n",
    "def predict_next(rnn, context, n_gibbs=NUM_GIBBS, temperature=1.0):\n",
    "    entity  = rnn.next_entity()\n",
    "    h_sz, s_sz = rnn.h_size(), rnn.sensory_size()\n",
    "    x_init  = (np.random.rand(1, s_sz) > 0.5).astype(float)\n",
    "    visible = np.concatenate([context, x_init], axis=1)\n",
    "    for _ in range(n_gibbs):\n",
    "        h       = entity.forward(visible)\n",
    "        visible = entity.reconstruct(h)\n",
    "        visible[:, :h_sz] = context\n",
    "    probs = convert(visible[:, h_sz:])[0]\n",
    "    probs = np_cpu.power(np_cpu.clip(probs, 1e-10, 1.0), 1.0 / temperature)\n",
    "    probs /= probs.sum()\n",
    "    return probs\n",
    "\n",
    "rnn.reset(batch_size=1)\n",
    "h         = np.zeros((1, CONTEXT_SIZE))\n",
    "predicted = ''\n",
    "\n",
    "for i in range(N_SAMPLES - 1):\n",
    "    x_t = np.array(sensory_np[i][np_cpu.newaxis, :])\n",
    "    h   = rnn.step(x_t)                         # advance to h_t\n",
    "    probs = predict_next(rnn, h.copy())          # predict x_{t+1} from h_t\n",
    "    predicted += idx_to_char[int(np_cpu.argmax(probs))]\n",
    "\n",
    "print(f\"Input (t+1)         : '{decoded[1:]}'\")\n",
    "print(f\"Predicted from h_t  : '{predicted}'\")\n",
    "correct = sum(decoded[i+1] == predicted[i] for i in range(N_SAMPLES - 1))\n",
    "print(f\"Next-step accuracy  : {correct}/{N_SAMPLES-1} = {100*correct/(N_SAMPLES-1):.0f}%\")"
   ],
   "outputs": [],
   "execution_count": null
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}