{ "cells": [ { "cell_type": "code", "id": "c33-0001", "metadata": { "execution": { "iopub.execute_input": "2026-05-31T20:52:48.942476Z", "iopub.status.busy": "2026-05-31T20:52:48.942263Z", "iopub.status.idle": "2026-05-31T20:52:49.958704Z", "shell.execute_reply": "2026-05-31T20:52:49.957672Z" }, "ExecuteTime": { "end_time": "2026-05-31T21:21:09.995884255Z", "start_time": "2026-05-31T21:21:09.451419390Z" } }, "source": [ "# RNN-RBM\n", "#\n", "# Architecture: unrolled StackRnn (UNROLL_DEPTH entities, alternating t % depth)\n", "# visible = [context(CONTEXT_SIZE) | x_t(SENSORY_SIZE)] hidden = CONTEXT_SIZE\n", "#\n", "# Training data: Moby Dick opening, encoded with a 40-char vocabulary.\n", "\n", "import numpy as np_cpu\n", "import matplotlib.pyplot as plt\n", "from stack.rnn import StackRnn\n", "from rbm.matrix import np, convert\n", "from rbm.entity import EntityParams, TrainingParams\n", "from rbm.status import CheckpointStatus" ], "outputs": [], "execution_count": 1 }, { "cell_type": "code", "id": "c33-0002", "metadata": { "execution": { "iopub.execute_input": "2026-05-31T20:52:49.962590Z", "iopub.status.busy": "2026-05-31T20:52:49.961957Z", "iopub.status.idle": "2026-05-31T20:52:49.969009Z", "shell.execute_reply": "2026-05-31T20:52:49.968063Z" }, "ExecuteTime": { "end_time": "2026-05-31T21:21:10.051754981Z", "start_time": "2026-05-31T21:21:10.002610196Z" } }, "source": [ "# \u2500\u2500 Config \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n", "SENSORY_SIZE = 40 # numVisibleX * numVisibleY = 1 * 40\n", "CONTEXT_SIZE = 256 # numContext = numHidden\n", "UNROLL_DEPTH = 1 # number of time-step entities (1 = shared weights)\n", "LEARNING_RATE = 0.05 # learningRate\n", "MOMENTUM = 0.9 # momentum\n", "NUM_EPOCHS = 500 # numEpochs\n", "MINI_BATCH = 100 # miniBatchSize\n", "NUM_GIBBS = 3 # numGibbs\n", "RAO_BLACKWELL = True # doRaoBlackwell\n", "L2_LAMBDA = 0.0 # weightDecay\n", "PRJ_NAME = \"RNN-RBM\"\n", "WORK_DIR = \"results\"\n", "\n", "# Vocabulary (40 chars)\n", "ALLOWED = set(' .!?ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789')\n", "chars = sorted(ALLOWED)\n", "idx_to_char = {i: c for i, c in enumerate(chars)}\n", "char_to_idx = {c: i for i, c in enumerate(chars)}" ], "outputs": [], "execution_count": 2 }, { "cell_type": "code", "id": "c33-0003", "metadata": { "execution": { "iopub.execute_input": "2026-05-31T20:52:49.971400Z", "iopub.status.busy": "2026-05-31T20:52:49.971055Z", "iopub.status.idle": "2026-05-31T20:52:49.977961Z", "shell.execute_reply": "2026-05-31T20:52:49.976889Z" }, "ExecuteTime": { "end_time": "2026-05-31T21:21:10.120482231Z", "start_time": "2026-05-31T21:21:10.063092563Z" } }, "source": [ "# \u2500\u2500 Training sentence \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n", "SENTENCE = (\n", " \"Call me Ishmael. Some years ago, never mind how long precisely, \"\n", " \"having little money in my pocket and nothing particular to interest \"\n", " \"me on shore, I thought I would sail about a little.\"\n", ")\n", "\n", "# Convert to uppercase, keep only in-vocab characters\n", "SENTENCE = ''.join(c for c in SENTENCE.upper() if c in ALLOWED)\n", "N_SAMPLES = len(SENTENCE)\n", "\n", "sensory_np = np_cpu.zeros((N_SAMPLES, SENSORY_SIZE), dtype=np_cpu.float64)\n", "for i, c in enumerate(SENTENCE):\n", " sensory_np[i, char_to_idx[c]] = 1.0\n", "\n", "decoded = SENTENCE\n", "print(f\"Sentence : '{decoded}'\")\n", "print(f\"Length : {N_SAMPLES} chars\")" ], "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Sentence : 'CALL ME ISHMAEL. SOME YEARS AGO NEVER MIND HOW LONG PRECISELY HAVING LITTLE MONEY IN MY POCKET AND NOTHING PARTICULAR TO INTEREST ME ON SHORE I THOUGHT I WOULD SAIL ABOUT A LITTLE.'\n", "Length : 180 chars\n" ] } ], "execution_count": 3 }, { "cell_type": "code", "id": "c33-0004", "metadata": { "execution": { "iopub.execute_input": "2026-05-31T20:52:49.980161Z", "iopub.status.busy": "2026-05-31T20:52:49.979824Z", "iopub.status.idle": "2026-05-31T20:52:50.341436Z", "shell.execute_reply": "2026-05-31T20:52:50.340513Z" }, "ExecuteTime": { "end_time": "2026-05-31T21:21:10.539293991Z", "start_time": "2026-05-31T21:21:10.122245726Z" } }, "source": [ "# \u2500\u2500 Visualise one-hot encoding \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n", "step = max(1, N_SAMPLES // 40) # show at most 40 tick labels\n", "tick_pos = range(0, N_SAMPLES, step)\n", "\n", "plt.figure(figsize=(16, 3))\n", "plt.imshow(sensory_np.T, aspect='auto', cmap='hot', vmin=0, vmax=1)\n", "plt.colorbar(label='activation')\n", "plt.xlabel('Position')\n", "plt.ylabel('Vocab index')\n", "plt.title(f'One-hot encoding ({N_SAMPLES} chars)')\n", "plt.xticks(tick_pos, [decoded[i] for i in tick_pos], fontsize=8)\n", "plt.tight_layout()\n", "plt.show()" ], "outputs": [ { "data": { "text/plain": [ "
" ], "image/png": "iVBORw0KGgoAAAANSUhEUgAABY8AAAEiCAYAAACxwS1VAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAWIlJREFUeJzt3Xd4VFX+x/FPQhKUMAEpJoCCAZQiLgIiAmLAuEpRKSoiroK4IkYRLLjAokiRsv4ELFhQDGURQUEQC1VZUALSQZpAQgskIbQECClwfn9gRsdMhsyQKZl5v57nPJJzy/mec++djF8O5wZJMgIAAAAAAAAA4E+CvR0AAAAAAAAAAMD3kDwGAAAAAAAAABRA8hgAAAAAAAAAUADJYwAAAAAAAABAASSPAQAAAAAAAAAFkDwGAAAAAAAAABRA8hgAAAAAAAAAUADJYwAAAAAAAABAASSPAQAAAAAAAAAFkDwGAAABLykpSQsWLPB2GCXO0KFDZYyxqUtKSlJ8fLyXIrIvKChIW7du1eDBg70dymXLH/OKFSt6OxRJUr169ZSbm6sbb7zR26EAAADADUgeAwAAj6lfv76mT5+uQ4cO6dy5c0pOTtZ///tf1a9f39uhFasqVapo6NChatiwobdDgaRHHnlE1157rd577z1rXXh4uF5//XV9//33OnbsmIwx6tGjR6HneOihh5SQkKATJ04oPT1dy5cvV/v27QvsFxQUpAEDBigxMVFZWVnavHmzunXr5pZ++YIdO3bo22+/1fDhw70dCgAAANyA5DEAAPCIzp07a8OGDYqNjVV8fLzi4uI0efJktWnTRhs2bFCnTp28HWKxqVq1ql5//XXdfPPN3g7F4+rUqaOnnnrK22HYGDBggD7//HNlZGRY6ypVqqShQ4eqXr162rx5s8Pjn3vuOc2ePVvp6ekaOHCgRowYoXLlyunbb79V586dbfZ944039J///EdLlixR3759deDAAc2cOVMPP/ywW/rmCz788EN16dJFNWvW9HYoAAAAcANDoVAoFAqF4s5Ss2ZNc/r0abN9+3ZTqVIlm20VK1Y027dvN5mZmSY6Otor8SUlJZkFCxYU2/maNGlijDGmR48eXh97d5ahQ4cac3HdCp8tN998szHGmDvvvNOmPiwszERGRhbpeu3atcusWbPGps5isZiMjAwzb948a13VqlVNdna2effdd232/d///mcOHDhggoODi23MK1as6NZxu+KKK4q8b0hIiDl27JgZNmyY1683hUKhUCgUCqV4CzOPAQCA2w0YMEDh4eHq3bu30tPTbbYdO3ZMTz/9tMqWLatXXnnFWp+/tmutWrUUHx+vEydO6OTJk/r000915ZVXFmjj0Ucf1bp163T27FkdO3ZMM2fO1DXXXONUnC1bttSaNWuUlZWlvXv36rHHHiuwT3R0tGbPnq1jx47pzJkzSkhIsFm+ICYmRuvWrZMkTZkyRcaYSy6JIF2crTx58mSlpKTo3Llz+vXXX/XEE0/Y7BMTEyNjjB566CENHjxYBw8eVFZWlpYuXapatWoVOOett96qb7/9VsePH9fp06e1efNmPf/88zb7tGnTRitWrNDp06d14sQJzZs3T3Xr1rU7Nr/88ouysrK0Z88e9e7d224//rrmcY8ePWSMUYsWLfTWW28pLS1Np0+f1ty5c1WpUiWbY4OCgjR06FAlJyfrzJkz+uGHH1SvXr3LWke5U6dOys7O1ooVK2zqc3JylJqaWqRzREREKC0tzaYuMzNTp0+fVlZWlrWuY8eOCgsL0/vvv2+z7wcffKBrr71WzZs3v2RbderU0axZs5SWlqazZ89q586dGjlyZIH9ypcvf8nnomfPnlq2bJlSU1N17tw5bdu2TX369Clwrvw1v++++26tXbtWWVlZevrppyVJd911l1auXKkTJ04oMzNTO3fu1BtvvGFzfF5enpYvX66OHTtesn8AAAAoebyewaZQKBQKheLf5dChQyYxMdHhPomJiebAgQPWn/NnWK5fv958+eWXpk+fPmbSpEnGGGPGjBljc+zgwYPN+fPnzcyZM02fPn3Mq6++atLS0kxiYqIpV67cJeNLSkoyO3bsMEeOHDEjR440cXFxZt26deb8+fOmfv361v2uvvpqc+TIEXPq1CkzYsQI079/f7Nx40aTl5dnOnXqZN1nyJAhxhhjPvzwQ/Poo4+aRx991OGs6quvvtocOHDA7N+/3wwZMsQ8/fTTZt68ecYYY/r162fdLyYmxjoma9euNf369TOvvfaaOX36tFm9erXNOe+66y5z7tw5k5SUZIYOHWqefvppM2HCBLN48WLrPrGxsSYnJ8fs3LnTvPzyy9ZxO3bsmKlRo4Z1vwYNGpgzZ86Yffv2mX/961/m3//+tzly5IjZtGlTgZnHSUlJJj4+3vpzjx49rDEvXbrUPPvss+bNN980ubm55vPPP7c5dsyYMcYYY+bPn2/i4uLMRx99ZA4cOGDS0tJszulMWbx4sVm3bp3DfS4183jmzJkmNzfXPPfcc6ZGjRqmTp065r333jNnzpwxt912m3W/SZMmmczMzALH16xZ0xhjzHPPPecwjptuusmcPHnSHD161LzxxhvmqaeeMmPGjDGbN2926blYs2aN+fTTT02/fv3Ms88+axYuXGiMMSYuLq7ANfvtt9/MsWPHzKhRo0zv3r1NTEyMqV+/vjl37pz55ZdfTN++fU3v3r3Nf/7zH7N8+fICsQ8ePNjk5eUZi8Xi9c8bCoVCoVAoFEqxFq8HQKFQKBQKxY9LRESEMcaYr776yuF++cnSsmXLGumPJNknn3xis9+cOXPM0aNHrT9Xr17d5ObmmkGDBtnsd+ONN5qcnJwC9fZKUlKSMcaY22+/3VpXqVIlk5WVZd58801r3bhx44wxxrRs2dJaFx4ebvbu3WsSExNNUFCQkZxftuLjjz82ycnJpkKFCjb1n332mTlx4oR1CYH85PG2bdtMaGiodb++ffsaY4y58cYbjSQTHBxs9u7da5KSkhwmzzds2GBSUlLMVVddZa276aabTF5enpkyZYq1bu7cuebs2bPm2muvtdbVrVvX5ObmFjl5/OektSTz1ltvmdzcXBMREWGkiwn0nJwcM3fuXJv9XnvtNWOMcTl5fODAAfPFF1843OdS16ty5cpmyZIl5s/S0tJsEseSzIIFC8yePXsKHH/llVcaY4wZNWqUwziWL19uTp06ZTPOfy1FfS4k+0tPfP/99wVizL//7777bpv6fv36FXmJjG7duhljjGnatKlL14lCoVAoFAqF4puFZSsAAIBbWSwWSRf/mb8j+dsjIiJs6j/88EObn1euXKlKlSpZz9ulSxcFBwdr9uzZqlixorWkpKRo9+7datOmTZHi3LZtm3766Sfrz+np6dq1a5fNS8Dat2+vNWvW6Oeff7bWnTlzRpMmTVJ0dLTq169fpLb+6oEHHtCCBQsUFBRk04dFixapfPnyaty4sc3+8fHxys3NtRkTSdZYGzVqpJo1a2rChAk6deqU3TajoqLUqFEjTZkyRSdOnLDWb926VUuWLLEuxREcHKx77rlH8+bN08GDB6377dy5U4sWLSpyHydNmmTz88qVKxUSEqIaNWpIkmJjYxUaGlpgyYd33323yG3YU7FiRZv+ueLs2bPatWuXpkyZogcffFBPPPGEjhw5orlz59osF3LllVcqOzu7wPHnzp2zbi9MpUqVFBMTo08//dRmnAtzqefiz+1KF5+rihUr6n//+59q1apV4DlLTEzU4sWLbepOnjwp6eJyHEFBQQ7jyR/jvy5FAgAAgJKN5DEAAHCr/KTwn5Na9hSWZD5w4IDNz/lJqquuukqSdP311ys4OFh79uxRenq6Talfv76uvvpqSVJ4eLgiIyOt5a9Jrr+2k99WfjuSVKNGDe3atavAfjt27LBud1blypV11VVX6emnny4Q/5QpUyTJ2ofCYv3rmOQnNH/99ddC282PtbD+VK5cWWXKlLH+d/fu3QX2s3dsYS4Vc348e/bsKbDf8ePHi9yOPZdKfF7KF198oerVq+uJJ57QnDlzNGXKFLVu3VphYWE26/9mZWWpdOnSBY6/4oorrNsLk5/4d3TN/uxS4ylJLVq00JIlS3T69GmdOnVK6enpGj16tCSpXLlyNscnJSUVaGPWrFn66aefNHnyZKWmpmrmzJl66KGH7I5nft3FiegAAADwFyHeDgAAAPi3jIwMHT58WH/7298c7ve3v/1Nhw4dKpA8Pn/+vN3985NVwcHBunDhgtq1a2d339OnT0uSXn75Zb3++uvW+n379ik6OrrI7bhLcPDFv8ufPn26pk6danefLVu22PzsrVgvh7diPnbsmE1C1VnR0dFq166dnnrqKZv6EydO6KefflLLli2tdUeOHLE7071KlSqSpMOHD7scx19dajxr1qypZcuWaefOnXrxxRd18OBB5eTkqH379nrxxRet910+e4ntc+fO6Y477lCbNm3UoUMHtW3bVt26ddOyZct0991368KFC9Z988f4ry/EBAAAQMlG8hgAALjdN998o969e6tly5Y2Sz7ku/322xUdHV3gn+IXxd69exUcHKykpCS7s2PzTZs2zWZZCkezQAuzf/9+1alTp0B93bp1rdsl52ZfHj16VBkZGSpVqpSWLVvmdEz27N27V5LUoEGDQs+ZH2th/Tl69KjOnj2rc+fO6ezZs7r++usL7GfvWFflx1O7dm3t27fPWl+hQgVVqFDB5fPu3LnT5i8JnBUZGSlJKlWqVIFtoaGhCgn54+v0pk2b9NRTT6levXrW2eiS1KxZM+v2wiQmJkq6eM2Kw3333acrrrhC999/v80yGEVdxiWfMUY//PCDfvjhB7300ksaNGiQRo0apTZt2tjcW9HR0Tp//rx+++23YokfAAAAvoFlKwAAgNu9+eabOnv2rD766KMCicCrrrpKH374oc6cOaM333zT6XPPnTtXeXl5Gjp0qN3t+e0lJSVp2bJl1rJq1Sqn2/ruu+/UrFkz3Xbbbda6MmXKqHfv3kpKStL27dslXVwHWZLKly9/yXNeuHBBc+bM0QMPPKAbb7yxwHZX1pDdsGGDEhMT1b9//wLLE+RLSUnRxo0b1aNHD5t9brzxRt1999367rvvrPEtWrRInTp10rXXXmvdr27durrnnnucjq0wy5YtU25urp555hmb+ueee+6yzpuQkKAGDRooLCzMpeP37Nmj8+fP6+GHH7apr1atmlq1aqWNGzda6+bPn6+cnBzFxcXZ7NunTx8dOnTI4T2Xnp6u//3vf+rVq5fNOLsqf2byn2d2R0RE6IknnijyOezN2M5PgP91eY4mTZpo27ZtysjIcCFaAAAA+CpmHgMAALfbs2ePevTooRkzZmjr1q2aPHmykpKSdN111+nJJ59UpUqV9Mgjj1hnXzojMTFRQ4YM0ZgxY3Tddddp3rx5yszMVHR0tDp37qxJkybprbfeKpZ+jBkzRo888oi+//57vfPOOzp+/Lh69Oih6OhoPfDAA9YZx3v37tWJEyfUp08fZWZm6syZM1qzZo3NjNo/GzhwoNq0aaM1a9bo448/1vbt21WhQgU1btxYd911lypWrOhUnMYYPfPMM1qwYIE2bdqk+Ph4HTlyRHXr1tWNN96otm3bSpIGDBig77//XgkJCZo8ebKuvPJK9e3bV6dOnbJZ4mPo0KFq27atVq5cqffff18hISHq27evtm3bpoYNG7o0ln+Vlpamt99+Wy+//LLmz5+vhQsXqmHDhmrXrp2OHj3q8lq68+fP12uvvaaYmBgtWbLEZtuzzz6r8uXLq2rVqpIuzta95pprJF18UV9GRobS09P16aef6qmnntKyZcs0d+5cWSwWxcXF6corr7SuISxJycnJmjBhgl555RWFhoZq7dq16tSpk+644w51797dZpkHe55//nn99NNP2rBhgyZNmmR9Rjp06KBGjRo51e/FixcrOztbCxYs0EcffaSyZcvqqaeeUlpamrW/l/Laa6/pjjvu0Lfffqv9+/fr6quvVlxcnA4ePGgziz8kJEQxMTEFXnYIAAAA/2AoFAqFQqFQPFEaNGhgZsyYYZKTk012drY5fPiwmTFjhrnxxhsL7Dt06FBjjDEVK1a0qe/Ro4cxxpgaNWrY1Hfu3NmsWLHCZGZmmszMTLN9+3bz7rvvmuuvv/6ScSUlJZkFCxYUqP/xxx/Njz/+aFMXHR1tZs+ebY4fP27Onj1rVq9ebdq3b1/g2Pvuu8/8+uuvJicnxxhjTI8ePRzGULlyZfPuu++a/fv3W8dmyZIl5p///Kd1n5iYGGOMMQ888IDNsTVq1LDbRosWLcyiRYvMqVOnTGZmptm0aZN59tlnbfa58847zcqVK82ZM2fMyZMnzfz5803dunULxNeqVSuzdu1ac+7cObNnzx7Tu3dv6zX661jGx8cXuF5NmjSx2S+/LzExMda64OBgM2zYMHP48GFz5swZs3TpUlOnTh1z9OhR8/7777t8323atMl8/PHHdq97Yf58f5UqVco8++yzZsOGDSYjI8NkZGSYZcuWmdatWxc4Z1BQkBk4cKBJSkoy586dM1u3bjXdu3cvcqz169c3c+bMsd5fO3bsMMOGDXPpubj33nvNpk2bzNmzZ01iYqIZMGCA6dmzZ4H9Crv/27RpY7766itz6NAhc+7cOXPo0CEzY8YMU7t2bZv97rnnHmOMMbVq1SqWzwkKhUKhUCgUiu+UoN//AAAAAPiccuXK6eTJk/r3v/+tUaNGuXSOf/zjH5o4caKqV6+uU6dOFXOE+Oqrr2SMUZcuXbwdCgAAAIoZax4DAADAJ1xxxRUF6vr37y9JWr58ucvnnTFjhg4cOKBnn33W5XPAvrp16+ree+/Vq6++6u1QAAAA4AbMPAYAAIBP6NGjh3r27KnvvvtOp0+f1u23367u3btr0aJF1nWaAQAAAHgOL8wDAACAT9iyZYvy8vL0yiuvKCIiQqmpqZowYYKGDBni7dAAAACAgMSyFQAAAPAJGzdu1N///ndVrlxZpUuXVvXq1fXCCy/ozJkz3g4NAAAAcEqrVq309ddfKzk5WcYYdezY8ZLHxMTEaP369Tp37px2796tHj16eCBSx0geAwAAAAAAAEAxCg8P1+bNm4v83o3rrrtO3377rX788UfdfPPNmjBhgj755BPdfffdbo7UsRKx5nFcXJwGDBigqKgobd68WX379tXatWu9HRYAAAAAAAAAOGSMUadOnTR//vxC9xkzZow6dOigm266yVo3c+ZMlS9fXu3atfNEmHb5/JrHXbt21bhx49SnTx+tWbNG/fv316JFi1SnTh0dPXq0SOeoWrWqMjMz3RwpAAAAAAAA4F0Wi0WHDx/2dhglSunSpRUWFubSsdnZ2crJybnsGJo3b66lS5fa1C1atEgTJky47HNfDp9PHr/44ov6+OOPNWXKFElSnz591KFDB/Xq1Utjx4695PFVq1ZVcnKym6MEAAAAAAAAfEO1atVIIBdR6dKlde7cuSLtm5mZKYvFYlP3+uuva9iwYZcdR1RUlFJTU23qUlNTVa5cOV1xxRVFjrG4+XTyODQ0VE2aNNHo0aOtdcYYLV26VM2bNy/SOfJnHNeuVk2nPTD7OCUjo9BtURERXj9fSeArffaVOBzxZIwleTz89b7xlWviK3H4ipI8HiU5dqlkxF/cn1N877Dlyd8DvvA7xx1K+j3gKe4YJ3+9pxzhM/HyuRJjSeiXq1y5p0rCePhKjL7+OeXqOPnK+BaXshaL9iQn8y/wnfDHjOO8S+5rsVhUrVo1m/HNzs52U2S+waeTx5UqVVJISIjdrHvdunXtHhMWFqbSpUtbf87/24DTmZlef3CKu31v98cbfKXPvhKHI56M0dfHw1fiC8Rr4itx+IqSPB4lOXbJ9+N3R3x87/hDIH7+uoM/96048exdPj4Ti4crMZaEfrnC1X6VhPHwhRh9IYZL8ed7AMXp0sljKUSZbsoxpqSkKDIy0qYuMjJSp06d8tqsY0kK9lrLbjJo0CBlZGRYC0tWAAAAAAAAAHAsrwjFfRISEhQbG2tT9/e//10JCQlubfdSfDp5nJ6erry8PLtZ95SUFLvHjB49WhEREdZSrVo1T4QKAAAAAAAAoMTKlnTOQXFueYrw8HA1bNhQDRs2lCRFR0erYcOGuvbaayVJo0aN0tSpU637f/jhh6pZs6bGjh2rOnXq6JlnnlHXrl01fvz4y+/aZfDpZStyc3O1fv16xcbGav78+ZKkoKAgxcbG6r333rN7TE5OTrG84dBV4UFBPn2+ksBX+uwrcTjiyRgZD9+LwRf6K/lOHL6iJI9HSY5dKhnx+/r3hJIwho4E4mdwcfPXfhU3d4xTII69r3+GlYRr4kqMJaFfrvLX8fCVGH0ljsK4Gp+v9wuelCfJONju3L1yyy23aPny5daf85PAU6ZM0RNPPKEqVaqoevXq1u379u1Thw4dNH78ePXr10+HDh3SP//5Ty1evNipdotbkByPitd17dpVU6dO1dNPP61ffvlF/fv3V9euXVW3bl2lpaVd8niLxaKMjAxFRUSwVg0AAAAAAAD8lsViUUpGhiLIgxVZfu5QStalk8fVAm5sfXrmsSTNnj1blStX1vDhwxUVFaVNmzapbdu2RUocAwAAAAAAAMClFe/MY3/h8zOPLxczjwEAAAAAABAImHnsvD9mHu+WdMHBnsGSrg+4sfX5mccAAAAAAAAA4F55unTyOPCQPIbXnDH2J72zWD0AAAAAAIBjheVV4CqSx/aQPAYAAAAAAAAQ4Ege20PyGAAAAAAAAECAO/97KUwpTwXiU0geAwAAAAAAAAhw50TyuCCSxwAAAAAAAAACXJ4cJ48Dc41pkscAAAAAAAAAAhzJY3tIHsNrwoOCvB0CAAAAAABAiWQvr2KxWJSSkeGFaPzBeV1MIOPPSB4DAAAAAAAACHDn5Dh5HJhp1MDsNQAAAAAAAABY5YmZxwWRPAYAAAAAAAAQ4Ege20PyGAAAAAAAAECAu9Sax4H57i6SxwAAAAAAAAACXLakXAfbL3gqEJ9C8hgAAAAAAABAgLvUshXMPAYAAAAAAACAAHSpZSuCPRWITyF5DAAAAAAAACDAXWrmMcljAAAAAAAAAAhAJI/tIXkMAAAAAAAAIMCdk5TjYDsvzAMAAAAAAACAAHSpNY9LeSoQn0LyGAAAAAAAAECAu9SyFSSPAQAAAAAAACAAkTy2h+QxAAAAAAAAgACXrYvrHuPPSB4DAAAAAAAACHCXmnnsaJv/InkMAAAAAAAAIMCRPLaH5DEAAAAAAACAAEfy2B6SxwAAAAAAAAAC3Pnfi6PtgYfkMQAAAAAAAIAAd6kX5gV5KhCfQvIYAAAAAAAAQIBj2Qp7SB4DAAAAAAAACHAkj+0J9nYAAAAAAAAAAOBdeUUozouLi1NSUpKysrK0evVqNW3a1OH+/fr1086dO3X27FkdOHBA48aNU+nSpV1quzgw8xgAAAAAAABAgLvUmsfOz8Ht2rWrxo0bpz59+mjNmjXq37+/Fi1apDp16ujo0aMF9n/kkUc0ZswY9erVS6tWrdINN9ygKVOmyBijl156yen2iwMzjwEAAAAAAAAEuOKfefziiy/q448/1pQpU7Rjxw716dNHZ8+eVa9evezu36JFC/3888+aOXOm9u/fryVLlmjmzJm69dZbXe3UZSN5DAAAAAAAACDAFS15bLFYbEpYWJjds4WGhqpJkyZaunSptc4Yo6VLl6p58+Z2j1m1apWaNGliXdoiOjpa7du313fffVdMfXQeyWMAAAAAAAAAAe68HCeOz0uSkpOTlZGRYS2DBg2ye7ZKlSopJCREqampNvWpqamKioqye8zMmTP12muv6aefflJOTo4SExO1fPlyjR49uni66ALWPAYAAAAAAAAQ4M7J8ZrHpSRJ1apVU2ZmprU2Ozu72CKIiYnR4MGDFRcXpzVr1qh27dp6++23NWTIEI0cObLY2nEGyWMAAAAAAAAAAe5S6xpf3JaZmWmTPC5Menq68vLyFBkZaVMfGRmplJQUu8eMGDFC06dP1+TJkyVJv/76q8LDwzVp0iS98cYbMsYUqSfFiWUrAAAAAAAAAAS44n1hXm5urtavX6/Y2FhrXVBQkGJjY5WQkGD3mDJlyujChQs2defPn7ce6w3MPAYAAAAAAAAQ4PLXPHa03Tnjxo3T1KlTtW7dOv3yyy/q37+/wsPDFR8fL0maOnWqkpOTNXjwYEnSggUL9OKLL2rjxo3WZStGjBihBQsWFEgqewrJYwAAAAAAAAAB7rwcJ4idTx7Pnj1blStX1vDhwxUVFaVNmzapbdu2SktLkyRVr17dJik8cuRIGWM0cuRIVatWTUePHtWCBQv073//2+m2i0uQJM8vluFBFotFGRkZioqIKNJ6JAAAAAAAAEBJZLFYlJKRoQjyYEWWnzuUaks67WDPspL2BNzYMvMYAAAAAAAAQIAr2gvzAg3JYwAAAAAAAAABrvjXPPYHwd5svFWrVvr666+VnJwsY4w6duxYYJ9hw4bp8OHDOnv2rJYsWaLatWt7IVIAAAAAAAAA/iuvCCXweDV5HB4ers2bN+vZZ5+1u/2VV17R888/rz59+qhZs2Y6c+aMFi1apNKlS3s4UgAAAAAAAAD+61wRSuDx6rIVCxcu1MKFCwvd3r9/f40cOVJff/21JOnxxx9XamqqOnXqpFmzZnkqTAAAAAAAAAD+zJyX46UpzktBngrGd3h15rEj0dHRqlKlipYuXWqty8jI0Jo1a9S8efNCjwsLC5PFYrEpAAAAAAAAAFCoC0UoAchnk8dRUVGSpNTUVJv61NRU6zZ7Bg0apIyMDGtJTk52a5wAAAAAAAAASrjzRSgByGeTx64aPXq0IiIirKVatWreDgkAAAAAAACALyN5bJdX1zx2JCUlRZIUGRlp/XP+z5s2bSr0uJycHOXk5Lg7PAAAAAAAAAD+IrcI+1zp9ih8js/OPE5KStKRI0cUGxtrrbNYLGrWrJkSEhK8GBkAAAAAAAAAv8Kax3Z5deZxeHi4ateubf05OjpaDRs21PHjx3Xw4EFNmDBBQ4YM0e7du5WUlKQRI0bo8OHDmjdvnveCBgAAAAAAAOBfLkgyDrYHeSoQ3+LV5PEtt9yi5cuXW38eP368JGnKlCl64okn9J///Efh4eGaNGmSypcvr59++klt27ZVdna2lyIGAAAAAAAA4HfOi+SxHUFyPCwlnsViUUZGhqIiIpSZmentcAAAAAAAAAC3sFgsSsnIUAR5sCLLzx3qcIRkHIxZkEWqGnhj67MvzAMAAAAAAAAAj2DmsV0kjwEAAAAAAAAENtY8tivY2QN69Ohht75UqVIaNWrUZQcEAAAAAAAAAB51vgglADmdPH7nnXc0e/ZslS9f3lp3ww03aM2aNXrkkUeKMzYAAAAAAAAAcL88SbkOSp73QvMmp5PHjRo10jXXXKOtW7fqrrvuUlxcnDZs2KCdO3eqYcOG7ogRAAAAAAAAANyHmcd2Ob3mcWJiolq2bKkJEyZo4cKFOn/+vHr06KHPP//cHfEBAAAAAAAAgHtd+L3AhtMzjyWpQ4cO6tatmxISEnTy5Ek9+eSTqlKlSnHHBgAAAAAAAADux8xju5xOHn/44Yf64osvNHbsWLVq1Up/+9vflJOTo61bt+qhhx5yR4wAAAAAAAAA4D4kj+1yetmKli1bqlmzZtqyZYskKTU1VR06dFBcXJw+/fRTffHFF8UeJAAAAAAAAAC4TZ4cJ4hLeSoQ3+J08rhJkybKyckpUP/+++9r6dKlxRIUAAAAAAAAAHjMpdY8DvJUIL7F6WUrcnJyVLNmTY0YMUKfffaZKleuLElq27atQkKczkUDAAAAAAAAgHexbIVdTieP77jjDm3dulXNmjVTly5dVLZsWUlSw4YNNWzYsGIP0N3OGFNowR8Yp+LhyTHkegElC88s4D58j3Gvkj6+JT1+ALDHk59tJeFztCTECB/gB8njq6++WtOmTVNycrJyc3OVl5dnU1zh9FThMWPGaMiQIRo/frwyMjKs9T/88IOee+45l4IAAAAAAAAAAK/JleMEsaMlLXzElClTVL16dY0YMUJHjhyRKYa/IHE6eXzTTTepe/fuBerT0tJUqVKlyw4IAAAAAAAAADyqhMwuduT2229Xq1attHnz5mI7p9PLVpw8eVJVqlQpUN+oUSMlJycXS1AAAAAAAAAA4DFGf7w0z14pAaucHDx4UEFBxftmP6eTx59//rnGjh2ryMhIGWMUHBysFi1a6P/+7/80bdq0Yg0OAAAAAAAAANzOD9Y87t+/v8aMGaMaNWoU2zmD5GTePDQ0VBMnTlTPnj1VqlQp5eXlqVSpUvrss8/Us2dPXbjgWwuAWCwWZWRkKCoiQpmZmd4OBwAAAAAAICA4eiFdeDHPjsRFFotFKRkZiiAPVmT5uUN9HSHlORizEIt0v2+P7fHjx1WmTBmFhITo7Nmzys3NtdlesWJFp8/p9JrHubm56t27t0aMGKEGDRqobNmy2rhxo/bs2eN04wAAAAAAAADgdZeaXezi33fExcVpwIABioqK0ubNm9W3b1+tXbu20P3LlSunN954Q126dFGFChW0f/9+9e/fX99///0l2+rfv79rQTrgdPI438GDB3Xw4MHijAUAAAAAAAAAPC9/bWNH253UtWtXjRs3Tn369NGaNWvUv39/LVq0SHXq1NHRo0cL7B8aGqolS5YoLS1NDz74oJKTk1WjRg2dPHmySO25Y0nhIiWP33rrrSKf8KWXXnI5GAAAAAAAAADwODfMPH7xxRf18ccfa8qUKZKkPn36qEOHDurVq5fGjh1bYP9evXqpQoUKatGihfLy8iRJ+/fvd6rN4OBgderUSfXq1ZMkbdu2TV9//bXLSw0XKXncqFEjm58bN26skJAQ7dq1S5J0ww036Pz581q/fr1LQQAAAAAAAACA11yQ4+RxsHOnCw0NVZMmTTR69GhrnTFGS5cuVfPmze0ec//99yshIUETJ05Ux44ddfToUX322WcaO3ZskZK/tWrV0nfffadq1apZ87aDBg3SwYMH1aFDByUmJjrXCRUxeXznnXda//zCCy8oMzNTPXr0sE6ZLl++vOLj47Vy5UqnAwAAAAAAAAAAr8r9vVyCxWKx+Tk7O1s5OTkF9qtUqZJCQkKUmppqU5+amqq6devaPXfNmjV15513asaMGWrfvr1q166t999/X6GhoRo+fPglY3vnnXe0d+9e3XbbbTpx4oQkqUKFCvrvf/+rd955R/fee++lO/gXTq95/NJLL+nuu++2WWvj5MmTGjJkiBYvXqxx48Y5HYQnpGRk2K3n7Z4AUHLwtmagZOGZRXHjngKAkoXPZpQoRVzzODk52ab69ddf17Bhw4olhODgYKWlpal37966cOGCNmzYoGrVqmnAgAFFSh7HxMTYJI4l6fjx4xo4cKB+/vlnl2JyOnkcERGhypUrF6ivXLlygcw7AAAAAAAAAPi8S615/PuyFdWqVVNmZqa1Ojs72+7u6enpysvLU2RkpE19ZGSkUlJS7B5z5MgR5ebm2ixRsWPHDlWpUkWhoaHKzXU8NTo7O9tufrZs2bJ2Z0cXhZOrdUhfffWV4uPj1blzZ1WrVk3VqlVTly5dNHnyZM2dO9elIAAAAAAAAADAa84XoUjKzMy0KYUlZXNzc7V+/XrFxsZa64KCghQbG6uEhAS7x/z888+qXbu2gv40a/+GG27Q4cOHL5k4lqRvvvlGkyZN0q233mqta9asmT788EN9/fXXlzzeHqeTx3369NH333+vzz77TPv379f+/fv12WefaeHChYqLi3MpCAAAAAAAAADwmjz9se6xvZLn/CnHjRunp556So8//rjq1q2rDz74QOHh4YqPj5ckTZ06VaNGjbLu/8EHH6hChQp6++23df3116t9+/YaPHiwJk6cWKT2nn/+ee3du1cJCQk6d+6czp07p59//ll79uxRv379nO+AXFi2IisrS88++6wGDBigWrVqSZL27t2rs2fPuhQAAAAAAAAAAHhVEdc8dsbs2bNVuXJlDR8+XFFRUdq0aZPatm2rtLQ0SVL16tVtlqg4dOiQ7rnnHo0fP15btmxRcnKy3n77bY0dO7ZI7Z06dUqdOnVS7dq1rS/l27Fjh/bu3et88L8LklT4Wyf8gMViUUZGhqIiImzWIwGAy1HYC3t4IQQA+A9ezgYA8BR+56C4WCwWpWRkKII8WJHl5w71ToSU42DMwizS84E3tk7PPC5TpowGDhyo2NhYXX311QoOtl35In82MgAAAAAAAACUCJd6YZ6jbV701ltv6dVXX9XZs2f11ltvOdz3pZdecvr8TiePP/nkE8XExGj69Ok6cuSIjIO/HQMAAAAAAAAAn+eGZSs8oVGjRgoNDbX+ubg5nTxu166dOnTooFWrVhV7MAAAAAAAAADgcTm/lxLmzjvvtPvn4hJ86V1snThxQsePHy/2QAAAAAAAAADAK4z+mH1sr5SAxRcmT56ssmXLFqgvU6aMJk+e7NI5nX5h3qOPPqqOHTuqR48eysrKcqlRTyopL8wryS/fKskL+7sj9pI8HsXN1bFgDOHJz0Tut5LFV66Xr8ThrxhfwL1cecZ85bn0lTg8KRD7DOAPzn4G8MI851lfmDciQsp2MGalLdKrvj22eXl5qlKlio4ePWpTX7FiRaWkpFiXt3CG08tWvPTSS6pVq5ZSU1O1b98+5ebm2mxv0qSJ00EAAAAAAAAAgNeU0DWPpYsJ8KCgIAUFBclisejcuXPWbaVKlVL79u2Vlpbm0rmdTh7PmzfPpYYAAAAAAAAAwCddas1jH/7HHidPnpQxRsYY/fbbbwW2G2M0dOhQl87tdPJ4+PDhLjUEAAAAAAAAAD6pBM88btOmjYKCgvTDDz/ogQcesHlfXU5Ojvbv368jR464dG6nk8cAAAAAAAAA4FfO/14cbfdRK1askCRFR0fr4MGDMg7WynZWkZLHx44d0w033KBjx47p+PHjDgOoWLFisQUXSEryiw6I3f3nLKlcHQvGEJ68B7jfShZfuV6+Eoe/YnwB93LlGfOV59JX4vCkQOwzgD/wGeBBF+Q4QezDM4/zHThwQJJ05ZVXqnr16goLC7PZvnXrVqfPWaTk8QsvvGB9i2D//v2dbgQAAAAAAAAAfFbu76UwpTwViOsqVaqk+Ph4tWvXzu72kBDnF6Eo0hHTpk2z+2cAAAAAAAAAKPFK8JrH+SZMmKDy5curWbNmWr58uTp37qzIyEgNGTJEL730kkvnZM1jAAAAAAAAAIGtBK95nO/OO+9Ux44dtX79el24cEH79+/X0qVLlZGRoUGDBum7775z+pzBboizyAYOHKhffvlFGRkZSk1N1VdffaUbbrjBZp/SpUvrvffeU3p6ujIzM/Xll1/q6quv9lLEAAAAAAAAAPzO+SIUHxceHq60tDRJ0okTJ1S5cmVJF9c6bty4sUvn9GryOCYmRhMnTtRtt92mv//97woNDdXixYtVpkwZ6z7jx4/Xfffdp4ceekgxMTGqWrWq5s6d68WogZLtjDGFFsAR7hv/wbUEAhvfBQD4Aj6LAPicC0UoPm7Xrl2qU6eOJGnz5s16+umnVbVqVfXp00dHjhxx6ZxeXbbir4s39+zZU0ePHlWTJk20cuVKRURE6Mknn1T37t31448/SpKeeOIJ7dy5U82aNdOaNWu8ETYAAAAAAAAAf5IrKcfB9hLwwry3335bVapUkSQNGzZMCxcu1KOPPqqcnBz17NnTpXNeVvL4mmuukSQdOnTock5jVa5cOUnS8ePHJUlNmjRRWFiYli5dat1n165d2r9/v5o3b243eRwWFqbSpUtbf7ZYLMUSGwAAAAAAAAA/5QcvzJsxY4b1zxs2bFCNGjVUt25dHThwQMeOHXPpnE4vW1GqVCkNHz5cJ0+e1L59+7Rv3z6dPHlSI0aMUEiI67nooKAgTZgwQT/99JO2bdsmSYqKilJ2drZOnTpls29qaqqioqLsnmfQoEHKyMiwluTkZJdjAgAAAAAAABAA/GDN45YtW9r8nJWVpY0bN7qcOJZcSB6/++676t27t1555RU1atRIjRo10iuvvKInn3xS77zzjsuBTJw4UQ0aNFC3bt1cPockjR49WhEREdZSrVq1yzofAAAAAAAAAD/nB2se//DDD0pMTNQbb7yhevXqFcs5nZ4q3L17d3Xr1k0LFy601m3dulUHDx7UzJkzFRcX53QQ7777ru69917dcccdNjOFU1JSVLp0aZUrV85m9nFkZKRSUlLsnisnJ0c5OY4WKAEAAAAAAACAP8lRiV/zuGrVqurWrZseeeQRDRw4UFu2bNGMGTM0c+ZMl1dncHrmcXZ2tvbt21egPikpyaWk7bvvvqvOnTvrzjvvLHDe9evXKycnR7Gxsda6G264QTVq1FBCQoLTbeEi3mgb2MKDggotcK+S/kZp7pvL5yv3ANcSKFmK+7OD7wKAe/nK7/vi5snPIn8dQwA+zsjxrOMS8BF07NgxTZw4Ubfffrtq1aqlL774Qj169NC+ffu0bNkyl87pdPL4vffe06uvvqqwsDBrXVhYmP7973/rvffec+pcEydO1D/+8Q91795dmZmZioyMVGRkpK644gpJUkZGhiZPnqxx48apdevWaty4seLj47Vq1Sq7L8sDAAAAAAAAAKf5wZrHf7Zv3z6NGTNGAwcO1NatWxUTE+PSeYq0bMWcOXNsfr7rrrt06NAhbd68WZLUsGFDhYWFOZ3Bzl/i4n//+59Nfc+ePTV16lRJ0gsvvKALFy5ozpw5Kl26tBYtWuTS0hgAAAAAAAAAYNelEsQlKHncokULPfroo3rwwQd1xRVXaP78+Ro0aJBL5ypS8vjP6w1LBZPJBw8edKnxoCL807js7Gw999xzeu6551xqAwAAAAAAAAAcyv29ONru40aNGqVu3bqpatWqWrJkifr166f58+crKyvL5XMWKXncq1cvlxsAAAAAAAAAAJ+Wv7axo+0+7o477tCbb76p2bNn69ixY8VyziIlj+GbHL0swNELT3gZCvAHV58jV/DsuZcnr6WrijsOX+mzr8RRkjGGcMTRPVDYvVPS7xt/7Zcr+HwoeXzluhT3c+TJfvnrdyZfwXhcPsbQT/nBshW33357sZ/TpeTxAw88oK5du6p69eo2L86TpCZNmhRLYAAAAAAAAADgERfkOEHsozOP77vvPn3//ffKy8vTfffd53DfBQsWOH1+p5PHffv21RtvvKEpU6aoY8eOio+PV61atdS0aVNNnDjR6QAAAAAAAAAAwKtK6LIV8+bNU1RUlI4ePap58+YVup8xRiEhzs8jdvqIuLg49e7dW59//rl69uyp//znP0pKStKwYcNUoUIFpwMAAAAAAAAAAK/K+b0UJtRTgTinVKlSdv9cXIKdPaB69epatWqVJCkrK0sWi0WSNH36dD3yyCPFGx0AAAAAAAAAuNuFIhQf99hjjxVYYliSQkND9dhjj7l0TqdnHqekpKhChQo6cOCADhw4oNtuu01btmxRdHS0glgU3KNYhB24fDxH/iMQr6Wv9NlX4ijJGEO4yl/vHX/tlysYC7iKe+cPjIUtxuPyMYZ+yg9emBcfH6+FCxfq6NGjNvUWi0Xx8fGaPn260+d0eubxDz/8oPvvv98a0Pjx47V48WLNmjVLX331ldMBAAAAAAAAAIBXuWnmcVxcnJKSkpSVlaXVq1eradOmRTru4YcfljHGqXxrUFCQjDEF6q+55hqdOnWqyOf5M6dnHvfu3VvBwRdzzu+//76OHTumFi1a6Ouvv9ZHH33kUhAAAAAAAAAA4DW5crzmccHVIC6pa9euGjdunPr06aM1a9aof//+WrRokerUqVNgdvCf1ahRQ//3f/+nFStWFKmdDRs2yBgjY4yWLVumvLw867ZSpUopOjpaCxcudL4DciF5bIzR+fN/zNOeNWuWZs2a5VLjAAAAAAAAAOB1l5pd7MLM4xdffFEff/yxpkyZIknq06ePOnTooF69emns2LF2jwkODtaMGTM0dOhQtWrVSuXLl79kO/PmzZMk3XzzzVq0aJFOnz5t3ZaTk6N9+/Zpzpw5zndALiSPe/bsqdOnT+vLL7+0qX/wwQdVpkwZTZs2zaVAAAAAAAAAAMArirjmscVisanOzs5WTk7BKcuhoaFq0qSJRo8eba0zxmjp0qVq3rx5oc289tprSktL06effqpWrVoVKfThw4dLkvbt26fPP//cbjyucnrN40GDBik9Pb1AfVpamgYPHlwsQQEAAMC/nTGm0AIAAAB43PkiFEnJycnKyMiwlkGDBtk9XaVKlRQSEqLU1FSb+tTUVEVFRdk9pmXLlnryySf11FNPudSF7du36+abby5Qf+utt6pJkyYundPp5HH16tWVlJRUoH7//v2qXr26S0EAAAAAAAAAgNcU8YV51apVU0REhLX8eWbx5ShbtqymT5+up556SseOHXPpHBMnTtS1115boL5atWqaOHGiS+d0etmKtLQ0/e1vf9P+/ftt6hs2bOhyxwAAAAAAAADAa/J08aV5jrZLyszMVGZm5iVPl56erry8PEVGRtrUR0ZGKiUlpcD+tWrVUnR0tBYsWGCtCw6+OO83NzdXderUUWJiosM269evrw0bNhSo37hxo+rXr3/JmO1xeubxzJkz9c4776h169YKDg5WcHCw2rRpo7fffluff/65S0EAAAAAAAAAgNcUcdmKosrNzdX69esVGxtrrQsKClJsbKwSEhIK7L9z5041aNBAN998s7V8/fXX+vHHH3XzzTfr4MGDl2wzOzu7QLJakqpUqaK8vDznOvA7p2cev/rqq7ruuuu0bNkya6PBwcGaNm0aax4DAAAAAAAAKHmK+MI8Z4wbN05Tp07VunXr9Msvv6h///4KDw9XfHy8JGnq1KlKTk7W4MGDlZ2drW3bttkcf/LkSUkqUF+YxYsXa/To0erYsaMyMjIkSeXKldOoUaO0ZMkS5zsgF5LHubm56tatm1599VU1bNhQWVlZ2rp1qw4cOOBSAAAAAAAAAADgVX9a17jQ7U6aPXu2KleurOHDhysqKkqbNm1S27ZtlZaWJuniu+UuXHDhxIV4+eWXtWLFCu3fv18bN26UJN18881KTU3VY4895tI5gyT59SutLRaLMjIyFBURUaT1SP7Mlbd9hwcFOX2MP3M0howV4D48e8WjsHEMxDH05D3F/QuguPG5AsAXlITPopIQoyd58v8Hiqsti8WilIwMRbiQBwtU+blDlY+QHI2ZxSKd9P2xLVOmjB599FHrpN8tW7Zo5syZLi9b4fSax5L02GOPacuWLcrKylJWVpY2b96sf/zjHy4FAAAAAAAAAABedaEIpQQ4e/asfvrpJy1YsEArVqzQyZMn1a5dO913330unc/pZSteeOEFjRgxQu+9955+/vlnSdLtt9+uDz/8UJUqVdKECRNcCgQAAAAAAAAAvKEoSxqXcnsUlyc6OlpfffWVbrrpJhljFBQUJPOnWe0hIU6ngp1PHvft21fPPPOMpk+fbq1bsGCBtm3bptdff53kMQAAAAAAAIASxR+Sx2+//baSkpIUGxurpKQkNWvWTBUqVNBbb72ll19+2aVzOp08rlKlilatWlWgftWqVapSpYpLQQAAAAAAAACAt+QWYZ8wt0dxeZo3b64777xTx44d04ULF3T+/Hn9/PPPGjRokN555x01btzY6XM6nTzes2ePunbtqtGjR9vUP/zww9q9e7fTAfiyQHsBkDvi84V+Sbz0KtD5+rPnDv7aL09jHP/gybHw5ReQlCSB2GegMNz3JUsgfndDYCgJ929JiNGTSvp3YDinhCxp7FCpUqWsL/NLT09X1apV9dtvv2n//v2qU6eOS+d0Onk8dOhQzZo1S3fccYd1zeOWLVsqNjZWXbt2dSkIAAAAAAAAAPCWoixb4et+/fVXNWzYUPv27dOaNWv0yiuvKCcnR71791ZiYqJL5wwu6o433nijJGnu3Llq1qyZ0tPT1alTJ3Xq1Enp6em69dZbNW/ePJeCAAAAAAAAAABvuaCLCeTCSkmYmTxy5EgFB19M97722muKjo7WypUr1b59ez3//PMunbPIM4+3bNmitWvX6pNPPtHnn3+uxx57zKUGAQAAAAAAAMCXXJBU+MJJUklYWGTx4sXWP+/du1f16tXTVVddpRMnTrh8ziLPPI6JidG2bdv01ltv6ciRI4qPj9ftt9/ucsMAAAAAAAAA4AtyilBKostJHEsXk+aOkuoFlClTRl27dlXPnj3VqlUr7dmzR5MnT9bUqVOVmpp6WcG4g8ViUUZGhqIiIqwLRgMoXv76UqZAfFlLIPbZX3EtL5+vjKGvxOGvint8feV6+UoccC+u8+VjDAH4G4vFopSMDEWQByuy/NzhoYgIGQdjFmSx6JoAHNsizzzOd/bsWU2ZMkWtW7fWDTfcoC+++ELPPvusDhw4oPnz57sjRgAAAAAAAABwG0frHeeXQOR08vjP9u7dq1GjRmnkyJHKzMxUhw4diisuAAAAAAAAAPCIC0UogajIL8z7q1atWqlXr1564IEHdOHCBc2ePVuTJ08uztgAAAAAAAAAwO1y5ThBfFkzcEswp5LHVapUUc+ePdWzZ0/Vrl1bq1at0vPPP6/Zs2fr7Nmz7ooRAAAAAAAAANwmkGcXO1Lk5PF3332nu+66S+np6Zo2bZo+/fRT/fbbb+6MDXAKL7u4fK6Oob+Or7++2IhnJTBwLYumJDwPvhKHIyX5xanFHaOj83nyfisJYx+Iivse4DoXna98TvlKHMXNX/vlSCD2ubiVhO9hCCzn5Th5XPgd69+KnDzOzc3Vgw8+qG+++UYXLpCHBwAAAAAAAOAfSB7bV+TkcceOHd0ZBwAAAAAAAAB4BWse2+fyC/MAAAAAAAAAwB8YOU4eB+piKiSPAQAAAAAAAAS0878X2CJ5DAAAAAAAACCgkTy2j+Qx/AZvY718jKH3eHLsPdkWb1CGr+M+LDqe58tX0sepsHugpPfLkxgr7/GVsfeVOIqbv/bLkUDsc3FjDOFrLohlK+wheQwAAAAAAAAgoOXI8czjUp4KxMeQPAYAAAAAAAAQ0Jh5bF+wNxvv06ePNm/erFOnTunUqVNatWqV2rZta91eunRpvffee0pPT1dmZqa+/PJLXX311V6MGAAAAAAAAIC/uaA/1j22Vxwllv2ZV5PHhw4d0sCBA9WkSRPdcsst+uGHHzR//nzVr19fkjR+/Hjdd999euihhxQTE6OqVatq7ty53gwZAAAAAAAAgJ+5UIQSiIIkFf72Ey84duyYBgwYoC+//FJHjx5V9+7dNWfOHElSnTp1tHPnTt12221as2ZNkc5nsViUkZGhqIgIZWZmujN0AAAAAAAAwGssFotSMjIUQR6syPJzh19HRCjPwZiFWCy6PwDH1qszj/8sODhYDz/8sMLDw5WQkKAmTZooLCxMS5cute6za9cu7d+/X82bNy/0PGFhYbJYLDYFAAAAAAAAAArDzGP7vJ48btCggTIzM5Wdna0PP/xQnTt31o4dOxQVFaXs7GydOnXKZv/U1FRFRUUVer5BgwYpIyPDWpKTk93dBQAAAAAAAAAlmKP1jvNLIPJ68njXrl26+eab1axZM33wwQeaOnWq6tWr5/L5Ro8erYiICGupVq1aMUYLAAAAAAAAwN+QPLYvxNsB5Obmau/evZKkDRs2qGnTpurXr59mzZql0qVLq1y5cjazjyMjI5WSklLo+XJycpSTk+P2uAEAAAAAAAD4ByPHS1P41EvjPMjryeO/Cg4OVunSpbV+/Xrl5OQoNjZWc+fOlSTdcMMNqlGjhhISErwcJVC8zpjCP4LCg4I8GInvY6z8B9cSAPxfSf+sLyz+khC7qwKxz76gpD8rAOAPcn8vsOXVZStGjRqlVq1aqUaNGmrQoIFGjRql1q1ba8aMGcrIyNDkyZM1btw4tW7dWo0bN1Z8fLxWrVqlNWvWeDNsAAAAAAAAAH7EXctWxMXFKSkpSVlZWVq9erWaNm1a6L7//Oc/tWLFCh0/flzHjx/XkiVLHO7vCV5NHl999dWaNm2adu3apWXLlqlp06a65557tHTpUknSCy+8oG+++UZz5szRihUrlJKSoi5dungzZAAAAAAAAAB+xh3J465du2rcuHEaNmyYGjdurM2bN2vRokWqXLmy3f1bt26tmTNnqk2bNmrevLkOHjyoxYsXq2rVqq51qhgEyc+X7LBYLMrIyFBURIQyMzO9HQ5gF/9MregYK//BtQQA/1fSP+sDcQmHQOyzLyjpzwoA32GxWJSSkaEI8mBFlp87/CQiQrkOxizUYtE/nRzb1atXa+3aterbt68kKSgoSAcPHtS7776rsWPHXvL44OBgnThxQs8995ymT59etA4VM6/OPAYAAAAAAAAAb8uVlOOg5K+HbLFYbEpYWJjd84WGhqpJkybWFRYkyRijpUuXqnnz5kWKqUyZMgoNDdXx48dd7dZl87kX5qF4+OvfXPtrvxzF7q99dlUg9tlfefJa8hwBgHeU9M/Ykh6/K1zpcyD+ni3uPvvrOMH9+NcCQPG58HtxtF2SkpOTbepff/11DRs2rMD+lSpVUkhIiFJTU23qU1NTVbdu3SLFNHbsWB0+fNgmAe1pJI8BAAAAAAAABLRLrWucv61atWo2y1ZkZ2e7JZ5//etf6tatm1q3bu22NoqC5DEAAAAAAACAgFbUmceZmZlFWvM4PT1deXl5ioyMtKmPjIxUSkqKw2NfeuklDRw4UHfddZe2bt16ybbciTWPAQAAAAAAAAQ0R+sd5xdn5Obmav369YqNjbXWBQUFKTY2VgkJCYUeN2DAAL366qtq27at1q9f72SrxY+ZxwAAAAAAAAACWlFnHjtj3Lhxmjp1qtatW6dffvlF/fv3V3h4uOLj4yVJU6dOVXJysgYPHixJeuWVVzR8+HB1795d+/bts85aPn36tM6cOeNCBJeP5LGf8tfF8f21X44EYp+B4sZz5D2OXihUGK4XHPHXF3P5a78c8dc++0q/PBlHIL78uSTH7khJuF4lIUZPCrQ+l4Tr78r3X8l34g9kF+R4zWNXksezZ89W5cqVNXz4cEVFRWnTpk1q27at0tLSJEnVq1fXhQt/nPmZZ55R6dKlNWfOHJvzFPZSPk8geQwAAAAAAAAgoBX1hXnOmjhxoiZOnGh3W5s2bWx+jo6OdrEV9yF5DAAAAAAAACCguWPZCn9A8hgAAAAAAABAQMv9vRQm2FOB+BiSxwAAAAAAAAACmruWrSjpSB4DAAAAAAAACGgkj+0jeQwAANyGt0ajuPnrPeWv/XKkuPvs6O32nhxfX7mWxAFXlITn0tfvKV/5LCrpChvHkjCGJSFG2GfkeF3jwp9u/0byGAAAAAAAAEBAy/m9FIY1jwEAAAAAAAAgAF2Q45nHjrb5M5LHAAAAAAAAAAIaax7bR/IYAAAAAAAAQEBj5rF9JI8BeAUvkgCKhmcFAIqGz0TA9wTicxmIfXYHxhHekCvHax6X8lQgPobkMQAAAAAAAICAxsxj+0geAwAAAAAAAAhorHlsH8ljAAAAAAAAAAGN5LF9JI8BAAAAAAAABDSWrbCP5DEAr+AFCEVX2AvTGMPAwHUGAADwDl5cDASW3N9LYQI1iRqo/QYAAAAAAAAASRdnFjtamoKZxwAAAAAAAAAQgFjz2D6SxwAAAAAAAAACGmse20fyGAAAAAAAAEBAy/m9FCZQk6iB2m8AKDF4GQcAAADgeXwPBwILM4/tI3kMAAAAAAAAIKCx5rF9JI8BAAAAAAAABDQjx7OLjacC8TEkjwEAAAAAAAAENGYe20fyGAAAAAAAAEBAy/29ONoeiEgeAwAAAAAAAAhozDy2j+Qx4IIzpvCVbngjL+D/+AyAI75+f/h6fAAAAIA3kDy2j+QxAAAAAAAAgIB2QY5fmOdomz8jeQwAAAAAAAAgoOXJ8brGeZ4KxMeQPAYAAAAAAAAQ0Fi2wj6SxwAAAAAAAAACGslj+4K9HQBQEoUHBRVaAPg/PgPgiK/fH74eHwAAAOANF4pQXBEXF6ekpCRlZWVp9erVatq0qcP9H3zwQe3YsUNZWVnasmWL2rVr52LLxYPkMQAAAAAAAICAlispx0FxtB5yYbp27apx48Zp2LBhaty4sTZv3qxFixapcuXKdvdv3ry5Zs6cqcmTJ6tRo0aaN2+e5s2bpxtvvNG1ThWDIEnGa617gMViUUZGhqIiIpSZmentcAAAAAAAAAC3sFgsSsnIUAR5sCLLzx1Wv8SYWSwWHXBybFevXq21a9eqb9++kqSgoCAdPHhQ7777rsaOHVtg/88//1zh4eG67777rHUJCQnatGmTnnnmGSd7VjyYeQwAAAAAAAAgoJ0vQnFGaGiomjRpoqVLl1rrjDFaunSpmjdvbveY5s2b2+wvSYsWLSp0f08ImBfmlbVYvB0CAAAAAAAA4Dbkv1wXHhHhcF3j8IgISRdnIP9Zdna2cnJyCuxfqVIlhYSEKDU11aY+NTVVdevWtdtGVFSU3f2joqKK0AP38Pvkcf4F3ZOc7OVIAAAAAAAAAPezWCwsW1FEOTk5OnLkiH47dOiS+2ZmZir5LznG119/XcOGDXNXeF7n98njw4cPq1q1asrMzJTFYlFycrL1Z3fy17Y83R5t0RZt0RZteac92qIt2qIt2vJOe7RFW7RFW7Tlnfb8qS2LxaLDhw8X+3n9VXZ2tqKjoxUWFuby8fakp6crLy9PkZGRNvWRkZFKSUmxe0xKSopT+3uC3yePJRV4YDIzMz32ty/+2pan26Mt2qIt2qIt77RHW7RFW7RFW95pj7Zoi7Zoi7a8054/tOXJa+MvsrOzC00Cuyo3N1fr169XbGys5s+fL+niC/NiY2P13nvv2T0mISFBsbGxevvtt611f//735WQkFCssTkjIJLHAAAAAAAAAOBJ48aN09SpU7Vu3Tr98ssv6t+/v8LDwxUfHy9Jmjp1qpKTkzV48GBJ0ttvv63//e9/evHFF/Xtt9+qW7duuuWWW9S7d2+v9YHkMQAAAAAAAAAUs9mzZ6ty5coaPny4oqKitGnTJrVt21ZpaWmSpOrVq+vChT9e05eQkKDu3btr5MiRGjVqlHbv3q1OnTpp27Zt3uqCJMkESgkLCzNDhw41YWFhtFVC2qMt2qIt2qIt/+8bbdEWbdGWL7flz32jLdqiLdry5bb8uW+eHkcK5XJK0O9/AAAAAAAAAADAKtjbAQAAAAAAAAAAfA/JYwAAAAAAAABAAQGTPC5VqpRee+017dixQ1u3btXGjRv10UcfqVy5ct4OrVgkJSWpYcOGbm+nbNmyyszM1CeffOL2tjwlKSlJO3fu1MaNG62lQYMGbmvPk2PoqbaSkpK0Y8cOlSpVylq3du1axcTEuKWt/Ou1c+dO/etf/yr2Nv7clqf6FRoaqjFjxmj37t3avn27tmzZoscff7zY2/F0W5LtNdu2bZvi4uLc1taf23TnZ+J///tfvfHGGzZ133zzjV566aVib+vbb7/Vs88+W6B+06ZN6ty5c7G35yn5n7fbtm1TXl6e9efPP/+82NtKSkpSamqqQkL+eE9w69atZYzR+PHji7UtT94b3uDJ59lTv5/99bPek/3K56nvHZ78Xu/J7x2e5Knny5Of9d7QuXNnrVu3Ths3btSOHTu0bNkyBQUFuaUtf7xmISEh1mf5119/1YYNG/TVV1+5/f9r3f090RhT4PPI3W168l70BG+MoSf4+2ci/JvXF172RJkyZYr5+uuvTfny5a11Dz74oImOjvZ6bMVRkpKSTMOGDd3ezpNPPmmWL19ujh8/bsLDw73e75I0dt4YQ0+1lZSUZBITE03v3r2tdWvXrjUxMTFuvV5Vq1Y1J0+eNE2bNi3x/ZoxY4b58ssvTZkyZYwkU6NGDbN9+3bTq1evEt3WX69Z9erVzcmTJ81NN93klrbstemOUr58ebNv3z5zyy23GEmmV69eZsWKFSYoKKjY2+rSpYtZt26dTV2TJk1MamqqCQkJces4eqLUqFHDnDhxwu33w9q1a02XLl2sddOnTze//PKLGT9+fIm9N7xRPPk8e+r3s79+1nuyX/nFU987PPm93pPfOzxZPP391xOf9Z4uUVFR5ujRo6Z69erWukaNGnHNnCjTp083X331lc2zHBsba7p27erWdt09lsYYU65cOY+16el70RPF02Po6eKPn4kUvy9eD8DtpVatWubMmTOmYsWKXo/FXcVTH6SrVq0y7dq1M/PmzXNboslfx84bY+iptpKSkszjjz9uDh48aK688kojeSZ5LMmsXr3aPPjggyW6X7Vr1zZnzpwxFSpUsKlv166d2b9/f4ltq7BrtmbNGvPAAw+4pa3C2nRHueeee8yvv/5qateubQ4dOuS2v4wMCQkxqampNgm6iRMnmjfffNOt/fNU8VTyOC4uznzzzTdGkomIiDC7d+82w4cPL/bksSfvDW8UTz7Pnkwe+9tnvSf79efiie8dnv5e78nvHZ4s/piI9HRp2LChSUlJMWXLluWauVBq165tTp8+ba666iqPXzt/Sx57+l70RCF5TKH4VgmIZSsaN26s3bt369ixY94OpUSrV6+err32Wi1atEiTJ0/Wk08+6e2Qis2sWbNs/gnYFVdc4ZZ2PDmGnr5emzdv1o8//qgXXnjBre38WZ06dVSxYkUtX77cbW14ol+NGjXS7t27dfz4cZv6hIQEVa9eXZUqVSqRbdnToEED1a1bV5s3b3ZrO56waNEirVy5UmvXrtXw4cOVlJTklnby8vI0ffp09erVS5JUunRpPfLII5o8ebJb2vNXP//8s6677jpVqVJFjzzyiL744gudP3/eLW156t7wNk88z576/exvn/X5PPm72VPfO7z5vd4T3zs8yVPPl7/asmWLfvrpJ+3fv19z587Vyy+/rKpVq7q1TX+6Zo0aNdKePXt04sQJb4dS4nnjXgQQWAIieYzi8eSTT2ratGm6cOGCvvvuO0VHR6tu3breDqtYPPzww2rUqJG1nDt3zi3teHIMvXG9Xn31VfXr108VKlRwazuzZs3S9u3btX37dr377rtKT093a3ue6pc/y/+fnY8++ki9evXSnj17vB1SsXjzzTcVHBysSZMmubWdyZMn69FHH1VoaKi6dOmiHTt2aOfOnW5t0x9Nnz5dPXv2VK9evfTpp5+6tS1P3Rve4Mnn2VO/nyX//az3VL/8+Xuip793eIonny9/ZIzRgw8+qBYtWmjhwoVq2bKltm3bplq1armtTX++ZjVr1rSuLe7u39HeYoxx23k9fS96i7vGEIBjAZE83rBhg66//nq/+58BTwoJCdFjjz2mHj16KCkpSXv27FGZMmX8avaxu3lyDL11vfbv36/PPvtMQ4YMcWs7Dz/8sOrXr6+7775bY8aMcesLDiX392vjxo12P6OaN2+uAwcOFOv/pHqyrT/L/5+dli1bas6cOW5pwxvOnz+vCxcuuL2dHTt2aM+ePbrvvvvUq1cvZh27aNq0aXr++ed17tw5t/8FhqfuDW/w1+fZnz7r/8wTv5s9+b3DG9/rPf29AyXLrl27NGnSJHXu3FmrV6/W/fff7+2QSoSNGzeqdu3aKl++vCQpMTFRjRo10ujRo3XVVVd5N7jLlJaWpooVK9rUVapUSWlpaW5t15/uRW+NIQD7AiJ5vHfvXs2ZM0eTJ0+2eWNnly5dFB0d7cXISo77779fiYmJuuaaaxQdHa3o6Gjddttteuyxx2zeXo/CeXIMvXm9Ro4cqX/84x8e+adSy5Yt0wcffKCRI0e6vS139mvPnj1asGCBJk2apCuvvFKSVKNGDb311lsaMWJEiW0LxWvy5MkaPHiwbr31Vs2aNcvb4ZRIR44c0aBBg/Svf/3L26HAR/nLZ/1fuft3sye/d3jze70nv3fA91WtWlUtWrSw/ly+fHlFR0dr7969Xoyq5NizZ4/mz59f4FkODw/3YlTFY9GiRXr66aetPz/22GNKTExUSkqKW9rzx3vR02MIwLGASB5LUq9evbR582atWbNGv/76q7Zv36677767wLpzsO/JJ5/UjBkzbOp27typ5ORk3XfffV6Kqvj8df2w1q1bF3sbnhxDb16vY8eO6Z133vHYOlsjRozQ7bffrsaNG7u1HXf36/HHH1diYqK2bt2q7du365tvvtGbb76pTz75pES3heIza9Ys1alTR1988YXOnDnj7XBKrClTpmj16tXeDgNF5Infz3/mT5/1f+bufnn6e4c3v9d76nuHJ3j6+fI3ISEheu2117Rr1y5t3LhRK1eu1NSpU/X111+7rU1/u2Y9e/bU1q1brc/yypUrddddd2ns2LHeDu2y9O/fX1WqVNHmzZu1ceNGde/eXQ899JDb2vPGvehunh5DAI4F6eKb8wAAAAAAAAAAsAqYmccAAAAAAAAAgKIjeQwAAAAAAAAAKIDkMQAAAAAAAACgAJLHAAAAAAAAAIACSB4DAAAAAAAAAAogeQwAAAAAAAAAKIDkMQAAAAAAAACgAJLHAAAAKDYxMTEyxqhcuXIO90tKSlK/fv08FBUAAAAAV5A8BgAACEDx8fEyxsgYo+zsbO3evVuvvvqqSpUqdVnnXbVqlaKionTq1ClJUo8ePXTixIkC+zVt2lSTJk26rLYAAAAAuFeItwMAAACAd3z//fd64oknVLp0abVv314TJ05Ubm6uxowZ4/I5c3NzlZqaesn90tPTXW4DAAAAgGcw8xgAACBAZWdnKzU1VQcOHNCHH36opUuX6v7771f58uU1depUHT9+XGfOnNF3332n2rVrW4+rXr26vv76ax0/flynT5/Wr7/+qnbt2kmyXbYiJiZGU6ZMUfny5a2znIcOHSqp4LIV1157rebNm6fMzEydOnVKs2bN0tVXX23dPnToUG3cuFH/+Mc/lJSUpJMnT2rmzJkqW7ash0YLAAAACDwkjwEAACBJysrKUlhYmKZMmaJbbrlF999/v5o3b66goCB99913Cgm5+I/WJk6cqNKlS+uOO+7QTTfdpH/96186ffp0gfOtWrVK/fr106lTpxQVFaWoqCj93//9X4H9goKCNH/+fFWoUEExMTH6+9//rpo1a2rWrFk2+9WqVUudOnXSvffeq3vvvVcxMTEaOHCgewYDAAAAAMtWAAAAQIqNjdU999yj77//Xp07d1aLFi2UkJAgSXr00Ud18OBBderUSV9++aWqV6+uOXPm6Ndff5V0cRaxPbm5uTp16pSMMQ6XsoiNjdVNN92k6OhoHTp0SJL0+OOPa/v27brlllu0bt06SVJwcLB69uxpTVRPnz5dsbGxGjJkSLGNAwAAAIA/MPMYAAAgQN17773KzMzUuXPn9P3332vWrFmaMmWKcnNztWbNGut+x48f165du1SvXj1J0jvvvKMhQ4bop59+0uuvv66bbrrpsuKoV6+eDh48aE0cS9KOHTt04sQJa5uStG/fPpsZzkeOHLFZ2gIAAABA8SJ5DAAAEKB+/PFH3Xzzzbr++ut15ZVXqmfPnjLGXPK4yZMnq2bNmpo+fbpuuukmrVu3Ts8995zb483NzbX52Rij4GC+zgIAAADuwrdtAACAAHXmzBnt3btXBw8e1Pnz5yVdnPEbGhqqZs2aWferUKGC6tSpo+3bt1vrDh06pI8++kgPPPCA3nrrLT311FN228jJyVGpUqUcxrFjxw5de+21uuaaa6x19erV01VXXWXTJgAAAADPInkMAAAAqz179mjevHn6+OOP1bJlS/3tb3/Tf//7XyUnJ2v+/PmSpPHjx+vuu+/Wddddp0aNGqlNmzbasWOH3fPt27dPFotFd955pypWrKgrr7yywD5Lly7V1q1bNWPGDDVq1EhNmzbVtGnTtHz5cq1fv96t/QUAAABQOJLHAAAAsPHEE09o/fr1+uabb5SQkKCgoCC1b99eeXl5kqRSpUpp4sSJ2rFjhxYuXKjffvtNcXFxds+VkJCgDz74QLNmzVJ6erpeeeUVu/t17NhRJ06c0IoVK7R06VIlJibq4YcfdlsfAQAAAFxakKRLL2wHAAAAAAAAAAgozDwGAAAAAAAAABRA8hgAAAAAAAAAUADJYwAAAAAAAABAASSPAQAAAAAAAAAFkDwGAAAAAAAAABRA8hgAAAAAAAAAUADJYwAAAAAAAABAASSPAQAAAAAAAAAFkDwGAAAAAAAAABRA8hgAAAAAAAAAUADJYwAAAAAAAABAASSPAQAAAAAAAAAF/D9vZoKpDJRUMAAAAABJRU5ErkJggg==" }, "metadata": {}, "output_type": "display_data", "jetTransient": { "display_id": null } } ], "execution_count": 4 }, { "cell_type": "code", "id": "c33-0005", "metadata": { "execution": { "iopub.execute_input": "2026-05-31T20:52:50.343489Z", "iopub.status.busy": "2026-05-31T20:52:50.343302Z", "iopub.status.idle": "2026-05-31T20:52:50.358994Z", "shell.execute_reply": "2026-05-31T20:52:50.358199Z" }, "ExecuteTime": { "end_time": "2026-05-31T21:21:10.598141935Z", "start_time": "2026-05-31T21:21:10.541037200Z" } }, "source": [ "# \u2500\u2500 Prepare sequences for StackRnn \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n", "# Tile the single sentence N_REPEAT times to form a real batch so GPU/CPU\n", "# matrix ops are (N_REPEAT, 168) instead of (1, 168).\n", "N_REPEAT = 500\n", "sequences = np_cpu.tile(sensory_np[np_cpu.newaxis, :, :], (N_REPEAT, 1, 1))\n", "print(f\"sequences shape : {sequences.shape} \u2192 {N_REPEAT} \u00d7 {N_SAMPLES} chars\")" ], "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "sequences shape : (500, 180, 40) \u2192 500 \u00d7 180 chars\n" ] } ], "execution_count": 5 }, { "cell_type": "code", "id": "c33-0006", "metadata": { "execution": { "iopub.execute_input": "2026-05-31T20:52:50.360734Z", "iopub.status.busy": "2026-05-31T20:52:50.360559Z", "iopub.status.idle": "2026-05-31T20:52:50.470247Z", "shell.execute_reply": "2026-05-31T20:52:50.469428Z" }, "ExecuteTime": { "end_time": "2026-05-31T21:21:10.729690925Z", "start_time": "2026-05-31T21:21:10.599284994Z" } }, "source": [ "# \u2500\u2500 Build model \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n", "rnn = StackRnn(PRJ_NAME, WORK_DIR)\n", "for layer in StackRnn.make_unrolled(\n", " UNROLL_DEPTH,\n", " sensory_size=SENSORY_SIZE,\n", " h_size=CONTEXT_SIZE,\n", " entity_params=EntityParams(\n", " do_gaussian_visible=False,\n", " do_gaussian_hidden=False,\n", " num_gibbs_samples=NUM_GIBBS,\n", " ),\n", " training_params=TrainingParams(\n", " learning_rate=LEARNING_RATE,\n", " momentum=MOMENTUM,\n", " num_epochs=NUM_EPOCHS,\n", " mini_batch_size=MINI_BATCH,\n", " num_gibbs_samples=NUM_GIBBS,\n", " do_rao_blackwell=RAO_BLACKWELL,\n", " l2_lambda=L2_LAMBDA,\n", " ),\n", "):\n", " rnn.append(layer)\n", "\n", "rnn.state_init(0.01)\n", "rnn.state_load()\n", "\n", "e = rnn.from_index(0).entity\n", "print(f\"Mode : {'shared' if rnn.is_shared else 'unrolled'}\")\n", "print(f\"Layers : {rnn.num_layers()} (alternating t % {rnn.num_layers()})\")\n", "print(f\"Visible : {rnn.h_size()} (context) + {rnn.sensory_size()} (sensory) = {e.shape[0]}\")\n", "print(f\"Hidden : {rnn.h_size()}\")\n", "print(f\"Parameters : {e.shape[0] * e.shape[1] * rnn.num_layers():,} ({rnn.num_layers()} \u00d7 {e.shape[0] * e.shape[1]:,})\")" ], "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Mode : shared\n", "Layers : 1 (alternating t % 1)\n", "Visible : 256 (context) + 40 (sensory) = 296\n", "Hidden : 256\n", "Parameters : 75,776 (1 \u00d7 75,776)\n" ] } ], "execution_count": 6 }, { "cell_type": "code", "id": "c33-0007", "metadata": { "execution": { "iopub.execute_input": "2026-05-31T20:52:50.472255Z", "iopub.status.busy": "2026-05-31T20:52:50.472012Z", "iopub.status.idle": "2026-05-31T20:54:45.425863Z", "shell.execute_reply": "2026-05-31T20:54:45.425132Z" }, "ExecuteTime": { "start_time": "2026-05-31T21:21:10.731553638Z" } }, "source": [ "# \u2500\u2500 Train: predict next character from current context \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n", "# Unrolled layers alternate: layer[t % num_layers] owns each time-step position.\n", "# Step 1 \u2014 advance context: h_t = layer[t%D].forward([h_{t-1} | x_t])\n", "# Step 2 \u2014 CD on [h_t | x_{t+1}] using the same layer[t%D]\n", "from rbm.train import cd_binary_binary, _to_gpu\n", "from rbm.matrix import rms_error_accu\n", "\n", "h_sz = rnn.h_size()\n", "num_seq, T, s_sz = sequences.shape\n", "n_layers = rnn.num_layers()\n", "\n", "for idx in range(n_layers):\n", " rnn.from_index(idx).entity.grad_zero()\n", "\n", "d_progress = 100.0 / NUM_EPOCHS\n", "progress = 0.0\n", "entity0 = rnn.from_index(0).entity\n", "status = CheckpointStatus(rnn.state_save, update_interval=10)\n", "status.on_change(entity0)\n", "\n", "for epoch in range(NUM_EPOCHS):\n", " h = np.zeros((num_seq, h_sz))\n", " err_total = 0.0\n", "\n", " for t in range(T - 1):\n", " entity_t = rnn.from_index(t % n_layers).entity\n", " x_t = _to_gpu(sequences[:, t, :])\n", " x_tp1 = _to_gpu(sequences[:, t+1, :])\n", "\n", " # Step 1: advance context\n", " h = entity_t.forward(np.concatenate([h, x_t], axis=1))\n", "\n", " # Step 2: CD on [h_t | x_{t+1}]\n", " vis = np.concatenate([h, x_tp1], axis=1)\n", " dwhv, dbv, dbh = cd_binary_binary(entity_t, vis)\n", " grad = entity_t.grad_compute(dbv, dbh, dwhv)\n", " entity_t.state_adjust(grad, 1.0 / num_seq)\n", " err_total += rms_error_accu(vis - entity_t.reconstruct(entity_t.forward(vis)))\n", "\n", " progress += d_progress\n", " if status.want_report(round(progress)):\n", " if not status.on_change(entity0, {\n", " \"progress\": {\"value\": round(progress), \"unit\": \"%\"},\n", " \"err_rms\": {\"value\": err_total / (T - 1), \"unit\": \"\"},\n", " }):\n", " break\n", "\n", "rnn.state_save()" ], "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "-------------------------------------------\n", "Entity-296x256: progress : 0%\n", "Entity-296x256: err_rms : 0.00012437576727553383\n", "Entity-296x256: l2_norm : 195.74862335488334\n" ] } ], "execution_count": null }, { "cell_type": "code", "id": "c33-0008", "metadata": { "execution": { "iopub.execute_input": "2026-05-31T20:54:45.428179Z", "iopub.status.busy": "2026-05-31T20:54:45.427939Z", "iopub.status.idle": "2026-05-31T20:54:45.846509Z", "shell.execute_reply": "2026-05-31T20:54:45.845882Z" } }, "source": [ "# \u2500\u2500 Reconstruction: teacher-forced \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n", "# Step through each sample, reconstruct, compare to input.\n", "rnn.reset(batch_size=1)\n", "recons = []\n", "for i in range(N_SAMPLES):\n", " x_t = np.array(sensory_np[i][np_cpu.newaxis, :])\n", " h = rnn.step(x_t)\n", " rec = convert(rnn.reconstruct(h))[0] # (40,)\n", " recons.append(rec)\n", "\n", "recons_np = np_cpu.array(recons) # (12, 40)\n", "decoded_recon = ''.join(idx_to_char[int(np_cpu.argmax(recons_np[i]))] for i in range(N_SAMPLES))\n", "\n", "print(f\"Input : '{decoded}'\")\n", "print(f\"Reconstruction: '{decoded_recon}'\")\n", "\n", "correct = sum(decoded[i] == decoded_recon[i] for i in range(N_SAMPLES))\n", "print(f\"Char accuracy : {correct}/{N_SAMPLES} = {100*correct/N_SAMPLES:.0f}%\")" ], "outputs": [], "execution_count": null }, { "cell_type": "code", "id": "c33-0009", "metadata": { "execution": { "iopub.execute_input": "2026-05-31T20:54:45.848729Z", "iopub.status.busy": "2026-05-31T20:54:45.848553Z", "iopub.status.idle": "2026-05-31T20:54:46.525019Z", "shell.execute_reply": "2026-05-31T20:54:46.524110Z" } }, "source": [ "# \u2500\u2500 Visualise reconstruction \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n", "step = max(1, N_SAMPLES // 40)\n", "tick_pos = range(0, N_SAMPLES, step)\n", "\n", "step = max(1, N_SAMPLES // 40) # show at most 40 tick labels\n", "tick_pos = range(0, N_SAMPLES, step)\n", "\n", "plt.figure(figsize=(16, 3))\n", "plt.imshow(sensory_np.T, aspect='auto', cmap='hot', vmin=0, vmax=1)\n", "plt.colorbar(label='activation')\n", "plt.xlabel('Position')\n", "plt.ylabel('Vocab index')\n", "plt.title(f'Input ({N_SAMPLES} chars)')\n", "plt.xticks(tick_pos, [decoded[i] for i in tick_pos], fontsize=8)\n", "plt.tight_layout()\n", "plt.show()\n", "\n", "plt.figure(figsize=(16, 3))\n", "plt.imshow(recons_np.T, aspect='auto', cmap='hot', vmin=0, vmax=1)\n", "plt.colorbar(label='activation')\n", "plt.xlabel('Position')\n", "plt.ylabel('Vocab index')\n", "plt.title(f'Reconstruction (acc {100*correct/N_SAMPLES:.0f}%)')\n", "plt.xticks(tick_pos, [decoded[i] for i in tick_pos], fontsize=8)\n", "plt.tight_layout()\n", "plt.show()\n" ], "outputs": [], "execution_count": null }, { "cell_type": "code", "id": "c33-0010", "metadata": { "execution": { "iopub.execute_input": "2026-05-31T20:54:46.526924Z", "iopub.status.busy": "2026-05-31T20:54:46.526742Z", "iopub.status.idle": "2026-05-31T20:54:47.949550Z", "shell.execute_reply": "2026-05-31T20:54:47.948764Z" } }, "source": [ "# \u2500\u2500 Next-step prediction \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n", "# Given h_t (context after seeing x_t), predict x_{t+1} via clamped Gibbs.\n", "def predict_next(rnn, context, n_gibbs=NUM_GIBBS, temperature=1.0):\n", " entity = rnn.next_entity()\n", " h_sz, s_sz = rnn.h_size(), rnn.sensory_size()\n", " x_init = (np.random.rand(1, s_sz) > 0.5).astype(float)\n", " visible = np.concatenate([context, x_init], axis=1)\n", " for _ in range(n_gibbs):\n", " h = entity.forward(visible)\n", " visible = entity.reconstruct(h)\n", " visible[:, :h_sz] = context\n", " probs = convert(visible[:, h_sz:])[0]\n", " probs = np_cpu.power(np_cpu.clip(probs, 1e-10, 1.0), 1.0 / temperature)\n", " probs /= probs.sum()\n", " return probs\n", "\n", "rnn.reset(batch_size=1)\n", "h = np.zeros((1, CONTEXT_SIZE))\n", "predicted = ''\n", "\n", "for i in range(N_SAMPLES - 1):\n", " x_t = np.array(sensory_np[i][np_cpu.newaxis, :])\n", " h = rnn.step(x_t) # advance to h_t\n", " probs = predict_next(rnn, h.copy()) # predict x_{t+1} from h_t\n", " predicted += idx_to_char[int(np_cpu.argmax(probs))]\n", "\n", "print(f\"Input (t+1) : '{decoded[1:]}'\")\n", "print(f\"Predicted from h_t : '{predicted}'\")\n", "correct = sum(decoded[i+1] == predicted[i] for i in range(N_SAMPLES - 1))\n", "print(f\"Next-step accuracy : {correct}/{N_SAMPLES-1} = {100*correct/(N_SAMPLES-1):.0f}%\")" ], "outputs": [], "execution_count": null } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.3" } }, "nbformat": 4, "nbformat_minor": 5 }