From ec2dcc08dcdae3af773e2ce7c3e875cfee068013 Mon Sep 17 00:00:00 2001 From: wassname Date: Sat, 6 Jan 2024 14:19:51 +0800 Subject: [PATCH] wip --- notebooks/11b_vae_w_direct.ipynb | 102 ++- notebooks/11c_sae.ipynb | 1364 ++++++++---------------------- research_log.md | 1 + src/vae/sae.py | 76 ++ 4 files changed, 535 insertions(+), 1008 deletions(-) create mode 100644 src/vae/sae.py diff --git a/notebooks/11b_vae_w_direct.ipynb b/notebooks/11b_vae_w_direct.ipynb index a443942..33b287e 100644 --- a/notebooks/11b_vae_w_direct.ipynb +++ b/notebooks/11b_vae_w_direct.ipynb @@ -2,9 +2,18 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 107, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The autoreload extension is already loaded. To reload it, use:\n", + " %reload_ext autoreload\n" + ] + } + ], "source": [ "import os\n", "import numpy as np\n", @@ -80,7 +89,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 108, "metadata": {}, "outputs": [], "source": [ @@ -125,9 +134,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 109, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(PosixPath('/media/wassname/SGIronWolf/projects5/elk/sgd_probes_are_lie_detectors/notebooks/lightning_logs/version_24/hidden_states/.ds/ds_valtest_8b8fd6070504d5ef'),\n", + " PosixPath('/media/wassname/SGIronWolf/projects5/elk/sgd_probes_are_lie_detectors/notebooks/lightning_logs/version_24/hidden_states/.ds/ds_OOD_a41d3a61513ade30'))" + ] + }, + "execution_count": 109, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# load hidden state from a previously loaded adapter\n", "# the columns with _base are from the base model, and adapt from adapter\n", @@ -139,7 +160,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 110, "metadata": {}, "outputs": [], "source": [ @@ -150,7 +171,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 111, "metadata": {}, "outputs": [], "source": [ @@ -207,9 +228,52 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 112, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "select rows are 74.39% based on knowledge\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2107d9b230ac4d7cb6f84c90d1c57690", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Map: 0%| | 0/615 [00:00 b h l\")\n", + " # if not self._ae_mode:\n", + " # with torch.no_grad():\n", + " # l1_loss, l2_loss, loss, latent, h_rec = self.ae(x)\n", + " # else:\n", + " l1_loss, l2_loss, loss, latent, h_rec = self.ae(x)\n", + "\n", + " latent2 = rearrange(latent, \"b l h -> b (l h)\")\n", + " pred = self.head(latent2).squeeze(1)\n", + " return dict(\n", + " pred=pred,\n", + " l1_loss=l1_loss,\n", + " l2_loss=l2_loss,\n", + " loss=loss,\n", + " latent=latent,\n", + " h_rec=h_rec,\n", + " )\n", + "\n", + " def _step(self, batch, batch_idx, stage=\"train\"): \n", + " if stage == \"train\":\n", + " # Normalize the decoder weights before each optimization step\n", + " self.ae.normalize_decoder()\n", + "\n", + "\n", + " device = next(self.parameters()).device\n", + " x, y = batch # batch['X'], batch['y']\n", + " x = x.to(device)\n", + " y = y.to(device)\n", + " x0 = x[..., 0]\n", + " # x1 = x[..., 1]\n", + " info0 = self(x0)\n", + " # info1 = self(x1)\n", + " # ypred1 = info1[\"pred\"]\n", + " logits = info0[\"pred\"]\n", + " y_probs = F.sigmoid(logits)\n", + " y_cls = y_probs > 0.5\n", + "\n", + " if stage == \"pred\":\n", + " return (y_probs).float()\n", + " \n", + " pred_loss = F.binary_cross_entropy_with_logits(logits, (y>0.).float())\n", + "\n", + " # pred_loss = F.smooth_l1_loss(ypred0, y)\n", + " rec_loss = info0[\"loss\"] \n", + " l1_loss = info0[\"l1_loss\"].mean()\n", + " l2_loss = info0[\"l2_loss\"].mean()\n", + "\n", + " self.log(\n", + " f\"{stage}/auroc\",\n", + " auroc(y_probs, y > 0, \"binary\"),\n", + " on_epoch=True,\n", + " on_step=False,\n", + " )\n", + " self.log(\n", + " f\"{stage}/acc\",\n", + " accuracy(y_cls, y > 0, \"binary\"),\n", + " on_epoch=True,\n", + " on_step=False,\n", + " )\n", + " self.log(\n", + " f\"{stage}/loss_pred\",\n", + " float(pred_loss),\n", + " on_epoch=True,\n", + " on_step=True,\n", + " prog_bar=True,\n", + " )\n", + " self.log(\n", + " f\"{stage}/loss_rec\",\n", + " float(rec_loss),\n", + " on_epoch=True,\n", + " on_step=True,\n", + " prog_bar=True,\n", + " )\n", + " self.log(f\"{stage}/l1_loss\", l1_loss, on_epoch=True, on_step=False)\n", + " self.log(f\"{stage}/l2_loss\", l2_loss, on_epoch=True, on_step=False)\n", + " self.log(\n", + " f\"{stage}/n\",\n", + " float(len(y)),\n", + " on_epoch=True,\n", + " on_step=False,\n", + " reduce_fx=torch.sum,\n", + " )\n", + " if self._ae_mode == 0:\n", + " assert torch.isfinite(rec_loss), \"rec_loss is not finite\"\n", + " return rec_loss\n", + " elif self._ae_mode == 1:\n", + " assert torch.isfinite(pred_loss), \"pred_loss is not finite\"\n", + " return pred_loss\n", + " elif self._ae_mode == 2:\n", + " # , train/loss_pred_epoch=0.0195, train/loss_rec_epoch=169.0\n", + " assert torch.isfinite(pred_loss), \"pred_loss is not finite\"\n", + " assert torch.isfinite(rec_loss), \"rec_loss is not finite\"\n", + " return pred_loss * 50000 + rec_loss" ] }, { @@ -330,94 +567,6 @@ "### Metrics\n" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def get_acc_subset(df, query, verbose=True):\n", - " if query:\n", - " df = df.query(query)\n", - " acc = (df[\"probe_pred\"] == df[\"y\"]).mean()\n", - " if verbose:\n", - " print(f\"acc={acc:2.2%},\\tn={len(df)},\\t[{query}] \")\n", - " return acc\n", - "\n", - "\n", - "def calc_metrics(dm, trainer, net, use_val=False, verbose=True):\n", - " dl_test = dm.test_dataloader()\n", - " rt = trainer.predict(net, dataloaders=dl_test)\n", - " y_test_pred = np.concatenate(rt)\n", - " splits = dm.splits[\"test\"]\n", - " df_test = dm.df.iloc[splits[0] : splits[1]].copy()\n", - " df_test[\"probe_pred\"] = y_test_pred > 0.0\n", - "\n", - " if use_val:\n", - " dl_val = dm.val_dataloader()\n", - " rv = trainer.predict(net, dataloaders=dl_val)\n", - " y_val_pred = np.concatenate(rv)\n", - " splits = dm.splits[\"val\"]\n", - " df_val = dm.df.iloc[splits[0] : splits[1]].copy()\n", - " df_val[\"probe_pred\"] = y_val_pred > 0.0\n", - "\n", - " df_test = pd.concat([df_val, df_test])\n", - "\n", - " if verbose:\n", - " print(\"probe results on subsets of the data\")\n", - " acc = get_acc_subset(df_test, \"\", verbose=verbose)\n", - " get_acc_subset(\n", - " df_test, \"instructed_to_lie==True\", verbose=verbose\n", - " ) # it was ph told to lie\n", - " get_acc_subset(\n", - " df_test, \"instructed_to_lie==False\", verbose=verbose\n", - " ) # it was told not to lie\n", - " get_acc_subset(\n", - " df_test, \"llm_ans==label_true\", verbose=verbose\n", - " ) # the llm gave the true ans\n", - " get_acc_subset(\n", - " df_test, \"llm_ans==label_instructed\", verbose=verbose\n", - " ) # the llm gave the desired ans\n", - " acc_lie_lie = get_acc_subset(\n", - " df_test, \"instructed_to_lie==True & llm_ans==label_instructed\", verbose=verbose\n", - " ) # it was told to lie, and it did lie\n", - " acc_lie_truth = get_acc_subset(\n", - " df_test, \"instructed_to_lie==True & llm_ans!=label_instructed\", verbose=verbose\n", - " )\n", - "\n", - " a = get_acc_subset(\n", - " df_test, \"instructed_to_lie==False & llm_ans==label_instructed\", verbose=False\n", - " )\n", - " b = get_acc_subset(\n", - " df_test, \"instructed_to_lie==False & llm_ans!=label_instructed\", verbose=False\n", - " )\n", - " c = get_acc_subset(\n", - " df_test, \"instructed_to_lie==True & llm_ans==label_instructed\", verbose=False\n", - " )\n", - " d = get_acc_subset(\n", - " df_test, \"instructed_to_lie==True & llm_ans!=label_instructed\", verbose=False\n", - " )\n", - " d1 = pd.DataFrame(\n", - " [[a, b], [c, d]],\n", - " index=[\"instructed_to_lie==False\", \"instructed_to_lie==True\"],\n", - " columns=[\"llm_ans==label_instructed\", \"llm_ans!=label_instructed\"],\n", - " )\n", - " d1 = pd.DataFrame(\n", - " [[a, b], [c, d]],\n", - " index=[\"tell a truth\", \"tell a lie\"],\n", - " columns=[\"did\", \"didn't\"],\n", - " )\n", - " d1.index.name = \"instructed to\"\n", - " d1.columns.name = \"llm gave\"\n", - " print(\"probe accuracy for quadrants\")\n", - " display(d1.round(2))\n", - "\n", - " if verbose:\n", - " print(f\"⭐PRIMARY METRIC⭐ acc={acc:2.2%} from probe\")\n", - " print(f\"⭐SECONDARY METRIC⭐ acc_lie_lie={acc_lie_lie:2.2%} from probe\")\n", - " return dict(acc=acc, acc_lie_lie=acc_lie_lie, acc_lie_truth=acc_lie_truth)" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -427,9 +576,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "10 5\n", + "torch.Size([32, 7, 1920, 2]) x\n" + ] + } + ], "source": [ "\n", "\n", @@ -444,7 +602,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ @@ -455,7 +613,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -468,9 +626,32 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([7, 1920])\n" + ] + }, + { + "data": { + "text/plain": [ + "{'pred': tensor(0.1837),\n", + " 'l1_loss': tensor(18.5229),\n", + " 'l2_loss': tensor(3651.4724),\n", + " 'loss': tensor(25625.1387),\n", + " 'latent': tensor(1.1577),\n", + " 'h_rec': tensor(0.0379)}" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "\n", "net = PLAE(\n", @@ -479,9 +660,8 @@ " max_epochs=max_epochs,\n", " lr=lr,\n", " weight_decay=wd,\n", - " hs=64,\n", - " depth=3,\n", - " dropout=1,\n", + " # hs=64,\n", + " dropout=0,\n", " n_latent=16,\n", " l1_coeff=l1_coeff, \n", " importance_matrix=importance_matrix,\n", @@ -495,9 +675,46 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "==========================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "==========================================================================================\n", + "PLAE [32, 7, 1920] --\n", + "├─AutoEncoder: 1-1 [32, 7] 443,632\n", + "├─Sequential: 1-2 [32, 1] --\n", + "│ └─LinBnDrop: 2-1 [32, 112] --\n", + "│ │ └─Linear: 3-1 [32, 112] 12,656\n", + "│ │ └─ReLU: 3-2 [32, 112] --\n", + "│ └─LinBnDrop: 2-2 [32, 28] --\n", + "│ │ └─Linear: 3-3 [32, 28] 3,164\n", + "│ │ └─ReLU: 3-4 [32, 28] --\n", + "│ └─LinBnDrop: 2-3 [32, 9] --\n", + "│ │ └─Linear: 3-5 [32, 9] 261\n", + "│ │ └─ReLU: 3-6 [32, 9] --\n", + "│ └─Linear: 2-4 [32, 1] 10\n", + "==========================================================================================\n", + "Total params: 459,723\n", + "Trainable params: 459,723\n", + "Non-trainable params: 0\n", + "Total mult-adds (Units.MEGABYTES): 0.51\n", + "==========================================================================================\n", + "Input size (MB): 3.44\n", + "Forward/backward pass size (MB): 0.04\n", + "Params size (MB): 1.84\n", + "Estimated Total Size (MB): 5.32\n", + "==========================================================================================" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "from torchinfo import summary\n", "\n", @@ -513,7 +730,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 31, "metadata": {}, "outputs": [], "source": [ @@ -522,9 +739,32 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 32, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a741409b29d34ef0a5cfe88dc68d5c6e", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/10 [00:00" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjQAAAG+CAYAAACESJ4gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAnQElEQVR4nO3de3xU5b3v8e9MbiaSQAJCLnhaQE266REVigJasDXAPoJQQYEmWIPGilj1ZdgEpLI31QpEbsWXu0JEQBIlWlpAtLSxPQ2XLbERxW56iDZQIUAiSSAhGDMms84f0dkGgsyQycw8M5/3X7NWFpNn8iQzX9bvudgsy7IEAABgMLu/GwAAANBZBBoAAGA8Ag0AADAegQYAABiPQAMAAIxHoAEAAMYj0AAAAOMRaAAAgPEINAAAwHgEGgAAYDwCDQCfq6ysVGpqqurq6lRWVqYpU6Zo8ODBuvXWW/X888/LnR1ZSktLdf311/ugtQBMQKAB4FczZ87U3XffrXfffVcvvfSSfvvb3+rVV1/1d7MAGIZAA8BvKisrNXLkSE2aNElhYWHq16+f0tPT9d5773n8XKWlpa47PWPHjlVRUZHra3v37tWECRNcX1uzZo3raxs3btQPfvADfe9739OkSZNUUlLildcGwLcINAD8pm/fvlq6dKnr2OFwaOfOnUpLS/PoeSoqKnT//fdr2rRpKi0t1ZIlS7RixQq9+eabkqTc3FxlZWXpvffe08qVK/XCCy/o4MGD+uSTT5SXl6eXXnpJf/3rXzV16lTNnz9fra2tXn2dALoegQZAQGhubtZjjz2myMhIZWRkePRvt2/frsGDB2vixIkKDw/XoEGDNH36dG3evFmSFBUVpR07dmjnzp369re/rbKyMqWlpSk8PFxOp1NFRUX68MMPdeedd2rXrl0KCwvripcIoAsRaAD4XVVVlTIzM1VfX6/169crJibGo39fW1urlJSUduf69u2r48ePS5LWrVun7t2764knntDQoUM1b948NTY2KiUlRS+99JIOHTqke++9VyNGjNDq1avdGpQMILAQaAD41X//939r0qRJ+s53vqP169erR48eHj9HUlKSKisr2507evSorrjiCjkcDv3zn//UokWLtHv3br322mv629/+pnXr1qmurk6RkZFavXq1q1T1/PPP69133/XSqwPgKwQaAH7jcDh033336cc//rF+8YtfKCIi4pKeZ9y4cfrggw+0ZcsWtbS0aP/+/SooKNDEiRMlSY8//rgKCwvldDrVu3dv2e129ejRQ8eOHdOMGTO0b98+RUREqHfv3pJ0SaEKgH/ZLO6tAvCxyspK/fCHP9S0adP06quvnldiuuWWW7Rq1apvfI7S0lI9+OCDev/99yW1zWRatmyZKioqlJCQoJ/85CeaPn26JKmsrEyLFi3S4cOHFRUVpXHjxik3N1fh4eHatGmTXnzxRdXW1qpnz5766U9/qrvuuqtrXjiALkOgAQAAxqPkBAAAjBfu7wYAQEcmT56sioqKDr8WGRmp0tJSH7cIQCCj5AQAAIxHyQkAABiPQAMAAIxHoAEAAMYj0AAAAOOF1CynhoYmtbY6vf688fGX69Sps15/XngH/RP46KPARx8FvmDso7Awu+Liot26NqQCTWurUy0t3g00Ntv/PDfzxQIP/RP46KPARx8FPvqIkhMAAAgCBBoAAGC8kCo5AQBCi9Pp/XGTgchmk1pbW+V0mldystu9c2+FQAMACDqffXZGZ86clmWFRqCRpJMn7UYGOJvNrtjYHoqJie3U8xgVaE6dOqXZs2erqalJaWlpWrBggb+bBAAIMF+Fme7deykyMlKSzd9N8onwcLvXJ750PUsOh0P19TWS1KlQY1Sg2bRpk+6++26NGTNGc+bM0cGDB5WWlubvZgEAAshXYeayy9yb7hss7Ha7vFS98am2fuql+vra0Ak0mZmZioqKktRWK2xL3gAAtGkbQ+Lk88EwkZGRsqy2MUCXOqbGqEATG9uW3P70pz/p888/V//+/f3cIgBAYAqNMlPw6Hx/GRVoJGnnzp1au3atVq9e7e+mAACAAGFUoCkvL9fzzz+v/Px8190aAAAAo4YP5efnq76+XrNmzdL06dP14Ycf+rtJAAB47PHHf6aTJz/16N8sXvyU3nlnt1vXLlr0C73//nuaPHm89u//QJJUUvJnzZiRoXvv/bFmzMjQO+/suejzrF27WosXP+VRO/3FqDs0S5cu9XcTAABBrKm5RQXF5ao41qABKXHKTE9VdJT3PyrfffcdWR6ugDd37pNuXdfa2qoDB/6mOXPmu85VVZ3QkiW/1Jo169W375X6+ONyzZr1gH7zm22Ki+vuUTsClce9tHnzZq1fv16SFB8fr4ULF6pfv36X3ICVK1fq5MmT+uUvf+k6t2vXLi1dulTNzc1KSkrSkiVL1Lt370v+Hl2lqblFhcXlOnyiUf2Suimji37xAQC+UVBcrtID1XJaUs3pJklS9riBXv0eCxf+XFLbXZozZ+o1aNANOnToH5o2bbquu+4GLVu2WJ999plqak4qOTlFv/jFYvXo0UMPP/yAxo+fqDFj/o9uvnmIsrNnau/ePTp58qQmTpyke++dIUnav/99DRz4vxUWFub6njabTbm589W375WSpH79Bshmk+rq6twONDU1J7Vs2RJVVh6RzWbTsGE3Kzt7psLDw/XKKy/rD394SxERkYqMjNTs2fPUv/8A7djxpjZtKlRYWJjsdrtmzXpU1113g1d/nl/xqOR06NAhLVu2TOvXr9cbb7yh9PT0Dhe3+/jjj9utVlheXn7eNZWVlZo1a5bWrVvX7nxdXZ1mz56tvLw87dixQ7feeqvmzZvnSTN9pqC4XHsPVOtE7VntPVCtguLzXycAwBwVxxrk/PLGidNqO/a2f//3pyVJy5c/p4iISF155f9SQcHruv32O7R9+1bdeutteuGFl/Taa1tlt9v1+99v7/B5IiIi9etfv6TFi5crP//Xamw8I0naufMv+v73b213bZ8+iRo58geu4xdffEHJySn61re+7Xa7Fy78uVJT07Rx42vKz9+g8vL/p4KC9Tp7tlFr1vyn/vM/X9SLL76sO+74kT74YJ8kadWq5fqP//il1q7dqOzsmXrvvb968qPyiEeBpn///iopKVHPnj3V0tKi48ePKz4+/rzrli1bppycHDkcDi1fvly5ublyOBztrikqKtLw4cOVlZXV7vzu3buVmpqq1NRUSdLUqVNVWlqqkydPevraupwvfvEBAL4zICVO9i9nENttbcddbdCg612Ps7NnKjExUa+8slHLli3WJ5/8U01Nn3X47265ZaQk6aqrrpZlWTpzpi3QlJW9qyFDhnb4b7744gs9++wzKin5sxYtWi6bzb3p0k1NTdq//33ddddUSVJU1GX60Y/u0p49u3T55d00dOhNmjEjUytXPqtu3brpjjt+JEm67bbReuyxh7RkydM6c6ZBmZk/ce+Hcgk8HhQcERGhsrIyjRw5UkVFRecFEklatWqVWltbNXr0aJWVlWnjxo3nLXKUk5OjjIyMdrfEJKmqqkpJSUmu48jISMXHx+vEiROeNrXLufuL39TcovztBzR39TvK335ATc0tPmwlAMBdmempunFgH/WOj9aNA/soMz21y79nVNRlrscLF/5cr79epF69rtDEiZN0ww2DLzjW5quFZqW2kpJlSQcP/l39+vVv97Wv1NXV6mc/+6lOnDiu1avXKzEx0e02WpZ1Xjssy6nW1lZJUl7eSj399BIlJiapsHCD5s59XJL0+OO5WrXqBV19dareemu7HnjgXrW0dM1n4CXNchoyZIj27NmjvLw8PfDAA2pouPCdia92/3SXZVkdJkZv7cbpTZnpqbppYB8l9bpcN33DL/5XNdlPTzWplNIUAASs6KhwZY8bqMU/HabscQO7bFxkWFiYKwx8XWnpf+mee2Zo9Oix6tYtVmVlf/XoM7Sk5P/q+98fdd75s2cb9fDDDygt7V+0dOkqxcV5ducpJiZG1157nV5/fZMkqbn5c23Zslk33jhMx48f0+TJ49WrV29NnZqp7OyH9NFH5WpqatKECWNls9l05513KScnV5988k85HM0efW93edRTlZWVOnr0qIYNGyZJuu2227RgwQIdOXJE3/3ud13XPfLII4qJidEf//hHPffcc8rMzNRvfvObDhPjuZKTk7V3717XscPh0KlTp5ScnOxJU30iOipc2eMHqlevWNXUnLnglu2elKZ8NcIeAOA/I0f+QI88MlOnTtW1O//ggz/TwoU/V2xsrCIiInT99YNVWXnE7ef9r//a3WFZZ9u2LTpy5BNFRERqxoxM1/m5c3+utLR/ceu5Fyx4SitXPqt77pkih+ML3XTTcGVlZSsyMlKTJk3Rww9nKyrqMoWF2TVv3gJFR0dr1qxHNW/ebEVGRkiyKTf354qJudzt1+MJm+XBvLEDBw5oxowZ2rp1qxITE1VSUqL58+eruLhY0dH/swnYxx9/rKuuusp1p6W8vNw1JuZczz33nKqqqlyznOrq6vSv//qv2rhxo6655hoVFhbqrbfeUmFhYWdepyTp1KmzXt+J1GbTRQNN/vYDrlHzdpt048A+Fxw178m1uDh3+gf+RR8FPpP6yOl06tNPj6p37ysD8s5+VzJzt+02F+q38HC74uPdC0Ae/dd/4MCBys3NVXZ2tux2u+Li4pSfn98uzEjS1Vdf3e74QmGmIwkJCVqxYoVyc3PV3Nysnj17Ki8vz5NmBpyvSlFfv+tyIe7ezeFODgCgs/btK9OqVcs7/Frfvn319NPmfP56dIfGdP66Q+MJd+/QcCfHPSb9zzJU0UeBz6Q+4g4Nd2gQINy9m8O4HAD4JgGevHCOzvcXn2oB5qsR9hczICVONaebXHdovmmtBF+sfAkAgcBut8tms8vhcOiyy6Iv/g8QEBwOh2y2sE7dVSPQGIpxOQDQsdjYHqqvr5HU68s10NxbPM50Tqc8muIdGCw5HA7V19coNrZHp56JTytDuXsnR3L/bg53cgAEg5iYWElSfX2tLOv8tV6Cld1uNzDQSDZbmGJje7j67VIRaEKAt8flcCcHQKCLiYlVTEyskR/wl8Jmk3r27Kba2saAH7h9Lm8N3uZTKAR4e1wOd3IAmCJUZjrZbHLtaG1aoPEWAg1cumKGFQAAvkCggUtXzLCiPAUA8AU+WeAxT2ZYUZ4CAPgCgQYe82SGFeUpAIAvEGjQpdwtT1GaAgB0Bp8Y6FLulqcoTQEAOoNAgy7lbnmK0hQAoDNCY4I+At6AlDjZv1yd/GIzpwAAOBd3aBAQPJk5xXgbAMC5+BRAQPBk5hTjbQAA56LkBOMw3gYAcC7u0MA4nkwFLywu1+ETjeqX1E0ZlKYAIGjx7g7jXMpU8Oq6s7JEaQoAghWBBsZhKjgA4FwEGgQtNtEEgNDBOzaCVmZ6qmySDlc1ql9i2xiaC2HmFACYjUCDoBUdFa7s8QPVq1esamrOyLIufC3lKQAwG4EGEJtoAoDpeCcGxCaaAGA6Ag0gZk4BgOkINIAHmDkFAIGJd1fAA55sokl5CgB8h0ADeMCTTTQpTwGA7xBogC7CzCkA8B3eNYEuwswpAPAdAg3QRZg5BQC+Q6AB/IyZUwDQebwTAn7GzCkA6DwCDeBnzJwCgM6z+7sBANw3ICVOdlvb44uVpwAglHCHBjCIu+UpxtoACDW8wwEGcbc8xVgbAKGGkhMQhBhrAyDUEGiAIMRYGwChhpITEIQ8mQrOeBsAwYB3LSAIeTIVnPE2AIIBJScgxDHeBkAwINAAIY7xNgCCASUnIMR5Mt4GAAIVgQYIce6Ot2HwMIBAxrsRALcweBhAIGMMDQC3MHgYQCAj0ABwC4OHAQQySk4A3MLgYQCBjEADwC2eLNbHAGIAvsY7DACvYwAxAF9jDA0Ar2MAMQBfI9AA8DoGEAPwNUpOALyOAcQAfI1AA8DrPFl9uLC4XIdPNKpfUjdlMHgYwCXinQOA33x98HB13VlZYvAwgEvDGBoAfsPgYQDeQqAB4DcMHgbgLZScAPhNZnqqbJIOVzWqX2LbGBoAuBQEGgB+Ex0VruzxA9WrV6xqas7IsvzdIgCmItAAMALbKQD4JrwbADAC2ykA+CYMCgZgBGZEAfgmBBoARmBGFIBvQskJgBHYTgHANyHQADCCu9spAAhNBBoAQYXZUEBo4q8cQFBhNhQQmhgUDCCoMBsKCE0EGgBBhdlQQGii5AQgqDAbCghNBBoAQYXZUEBoouQEAACMxx0aACGLKd5A8OAvF0DIYoo3EDwoOQEIWUzxBoIHgQZAyGKKNxA8KDkBCFlM8QaCB4EGQMhiijcQPAg0AHARzIYCAh9/kQBwEcyGAgIfg4IB4CKYDQUEPgINAFwEs6GAwEfJCQAugtlQQOAj0ADARTAbCgh8lJwAAIDxCDQAAMB4lJwAwItYswbwD/7KAMCLWLMG8A9KTgDgRaxZA/gHgQYAvIg1awD/oOQEAF7EmjWAfxBoAMCLWLMG8A9KTgAAwHgEGgAAYDxKTgDgB6xXA3gXfz0A4AesVwN4FyUnAPAD1qsBvItAAwB+wHo1gHdRcgIAP2C9GsC7CDQA4AesVwN4FyUnAABgPAINAAAwHiUnAAhwrFkDXBx/EQAQ4FizBrg4Sk4AEOBYswa4OAINAAQ41qwBLo6SEwAEONasAS6OQAMAAY41a4CLo+QEAACMR6ABAADGI9AAAADjMYYGAIIEC/AhlPGbDgBBggX4EMooOQFAkGABPoQyAg0ABAkW4EMoo+QEAEGCBfgQygg0ABAkWIAPoYySEwAAMB6BBgAAGI9AAwAAjEegAQAAxmNQMACEIFYVRrDhtxcAQhCrCiPYUHICgBDEqsIINgQaAAhBrCqMYEPJCQBCEKsKI9gQaAAgBLGqMIINJScAAGA8Ag0AADAegQYAABiPQAMAAIxHoAEAAMZjlhMA4IKamltUWFyuwyca1S+pmzLYIgEBit9KAMAFfX2LhOq6s7LEFgkITJScAAAXxBYJMAWBBgBwQWyRAFNQcgIAXFBmeqpskg5XNapfYtsYGiAQEWgAABcUHRWu7PED1atXrGpqzsiy/N0ioGOUnAAAgPEINAAAwHgEGgAAYDwCDQAAMB6BBgAAGI9ZTgAAr2hqblFBcbkqjjVoQEqcMtkmAT7EbxoAwCu+vk1CzekmSWyTAN+h5AQA8Aq2SYA/EWgAAF7BNgnwJ0pOAACvyPxyW4Svj6EBfIVAAwDwiuiocMbMwG8oOQEAAOMRaAAAgPEINAAAwHgEGgAAYDwCDQAAMB6BBgAAGI9p2wAAn2LPJ3QFfoMAAD7Fnk/oCpScAAA+xZ5P6AoEGgCAT7HnE7oCJScAgE+x5xO6AoEGAOBT7PmErkDJCQAAGI9AAwAAjEfJCQAQsFizBu7itwIAELBYswbuouQEAAhYrFkDdxFoAAABizVr4C5KTgCAgMWaNXAXgQYAELBYswbuouQEAACMR6ABAADGI9AAAADjEWgAAIDxCDQAAMB4BBoAAGA8Ag0AADAe69AAAIzHJpagtwEAxmMTS1ByAgAYj00sQaABABiPTSxByQkAYDw2sQSBBgBgPDaxBCUnAABgPAINAAAwHoEGAAAYj0ADAACMR6ABAADGI9AAAADjEWgAAIDxCDQAAMB4LKwHAAgp7MwdnOhBAEBIYWfu4ETJCQAQUtiZOzgRaAAAIYWduYMTJScAQEhhZ+7gRKABAIQUduYOTpScAACA8Qg0AADAeAQaAABgPAINAAAwHoEGAAAYj0ADAACMR6ABAADGI9AAAADjEWgAAIDxCDQAAMB4bH0AAEAHmppbVFBc3m7Pp+goPjYDFT0DAEAHCorLVXqgWk5LqjndJEnsARXAKDkBANCBimMNclptj51W2zECF4EGAIAODEiJk93W9thuaztG4KLkBABABzLTUyWp3RgaBC4CDQAAHYiOCmfMjEEoOQEAAOMRaAAAgPEINAAAwHgEGgAAYDwCDQAAMB6BBgAAGI9AAwAAjEegAQAAxiPQAAAA4xFoAACA8Qg0AADAeOzlBABAJzU1t6iguLzdRpbRUXzE+hI/bQAAOqmguFylB6rltKSa002SxMaWPkbJCQCATqo41iCn1fbYabUdw7cINAAAdNKAlDjZbW2P7ba2Y/gWJScAADopMz1VktqNoYFvEWgAAOik6Khwxsz4GSUnAABgPAINAAAwHoEGAAAYj0ADAACMR6ABAADGI9AAAADjEWgAAIDxCDQAAMB4BBoAAGA8Ag0AADAegQYAABiPvZwAAPCRpuYWFRSXt9vEMjqKj2Jv4KcIAICPFBSXq/RAtZyWVHO6SZLY1NJLKDkBAOAjFcca5LTaHjuttmN4B4EGAAAfGZASJ7ut7bHd1nYM76DkBACAj2Smp0pSuzE08A4CDQAAPhIdFc6YmS5CyQkAABiPQAMAAIxHoAEAAMYj0AAAAOMRaAAAgPEINAAAwHgEGgAAYDwCDQAAMB6BBgAAGI9AAwAAjEegAQAAxiPQAAAA4xFoAACA8Qg0AADAeOH+bgAAADhfU3OLCorLVXGsQQNS4pSZnqroKD62L4SfDAAAAaiguFylB6rltKSa002SpOxxA/3cqsBFyQkAgABUcaxBTqvtsdNqO8aFEWgAAAhAA1LiZLe1Pbbb2o5xYZScAAAIQJnpqZLUbgwNLoxAAwBAAIqOCmfMjAcoOQEAAOMRaAAAgPEINAAAwHgEGgAAYDwCDQAAMB6BBgAAGI9AAwAAjEegAQAAxiPQAAAA4xFoAACA8Qg0AADAeAQaAABgPAINAAAwHoEGAAAYL9zfDQAAAJeuqblFhcXlOnyiUf2SuikjPVXRUaH38R56rxgAgCBSUFyu0gPVclpSdd1ZWZKyxw30d7N8jpITAAAGqzjWIKfV9thptR2HIgINAAAGG5ASJ7ut7bHd1nYciig5AQBgsMz0VNkkHa5qVL/EtjE0oYhAAwCAwaKjwpU9fqB69YpVTc0ZWZa/W+QflJwAAIDxCDQAAMB4BBoAAGA8Ag0AADAegQYAABiPQAMAAIxHoAEAAMYj0AAAAOMRaAAAgPEINAAAwHgEGgAAYDwCDQAAMB6BBgAAGI9AAwAAjEegAQAAxiPQAAAA44X7uwEAAMA3mppbVFBcropjDRqQEqfM9FRFRwVHFAiOVwEAAC6qoLhcpQeq5bSkmtNNkqTscQP93CrvoOQEAECIqDjWIKfV9thptR0HCwINAAAhYkBKnOy2tsd2W9txsKDkBABAiMhMT5WkdmNogoVRgebUqVOaPXu2mpqalJaWpgULFvi7SQAAGCM6Kjxoxsycy6iS06ZNm3T33XfrlVdeUWNjow4ePOjvJgEAgABg1B2azMxMRUVFSZJaW1sVGRnp5xYBAIBAYFSgiY2NlST96U9/0ueff67+/fv7uUUAACAQGBVoJGnnzp1au3atVq9e7e+mAACAAGFUoCkvL9fzzz+v/Px8190aAAAAowYF5+fnq76+XrNmzdL06dP14Ycf+rtJAAAgABh1h2bp0qX+bgIAAAhAHt+hefXVVzV+/HjdcccdmjJlSqfvkqxcuVLz589vd27Xrl2aMGGCxo4dq6ysLH366aed+h4AACC4eRRo9u3bpzVr1mjDhg3atm2bHnzwQT300EOyLKvddR9//LGcTqfruLy8/Lznqqys1KxZs7Ru3bp25+vq6jR79mzl5eVpx44duvXWWzVv3jxPmgkAAEKMR4Gme/fueuqpp5SQkCBJuvbaa1VbW6umpqZ21y1btkw5OTlyOBxavny5cnNz5XA42l1TVFSk4cOHKysrq9353bt3KzU1VampbcsxT506VaWlpTp58qTHLw4AAIQGjwLNgAEDdPPNN0uSnE6nnnnmGY0aNUoxMTHtrlu1apVaW1s1evRolZWVaePGjectgpeTk6OMjAyFhYW1O19VVaWkpCTXcWRkpOLj43XixAmPXhgAAAgdlzQouLGxUXPmzFFdXZ3WrFnzjde2tra2Kz9djGVZstls5523242akAUAAHzI45Rw+PBhTZ48Wd26ddOGDRsUF3f+1uOPPPKIwsPD9cc//lFDhw5VZmammpub3Xr+5ORkVVdXu44dDodOnTql5ORkT5sKAABChEd3aI4fP66MjAzdd999uu+++y54XU5Ojq666irZbDbl5ORo3Lhxrj2YLmbEiBF6+umn9dFHH+maa67R66+/rkGDBrnG7QAAAJzLo0Czdu1aNTQ0aNu2bdq2bZvr/Jo1a9SnTx/X8dVXX93u3301wNcdCQkJWrFihXJzc9Xc3KyePXsqLy/Pk2YCAIAQY7POnXMdxE6dOquWFvfH87jDZpN69YpVTc0Zhc5P0hz0T+CjjwIffRT4grWPwsPtio+/3K1rGWkLAACMZ9TWBwAAoOs1NbeooLhcFccaNCAlTpnpqYqOOj8yuHudLxBoAABAOwXF5So9UC2nJdWcbls8N3vcwEu+zhcoOQEAgHYqjjXI+eVYHKfVdtyZ63yBQAMAANoZkBIn+5dr3Nptbceduc4XKDkBAIB2MtPbllv5+tiYzlznCwQaAADQTnRUuFtjYdy9zhcoOQEAAOMRaAAAgPEINAAAwHgEGgAAYDwCDQAAMB6BBgAAGI9AAwAAjEegAQAAxiPQAAAA4xFoAACA8Qg0AADAeAQaAABgvJDanDIsrOvyW1c+NzqP/gl89FHgo48CX7D1kSevx2ZZltWFbQEAAOhywRXlAABASCLQAAAA4xFoAACA8Qg0AADAeAQaAABgPAINAAAwHoEGAAAYj0ADAACMR6ABAADGI9AAAADjEWg6YdeuXZowYYLGjh2rrKwsffrpp/5uEr60cuVKzZ8/33W8detW3X777RozZowee+wxNTY2+rF1oe3VV1/V+PHjdccdd2jKlCn68MMPJUlr167V2LFjlZ6eroULF+qLL77wc0tD1+bNmzV+/HiNHz9e99xzjw4fPiyJPgo0+/fv13e/+11VVVVJ4n1OFi5JbW2tNXToUOvgwYOWZVnWhg0brBkzZvi5VTh69Kj10EMPWddee631xBNPWJZlWR999JE1bNgwq6qqyrIsy1q0aJH15JNP+rOZIeu9996zRo0aZdXW1lqWZVl//vOfrREjRlh/+ctfrLFjx1oNDQ1WS0uL9cgjj1irV6/2c2tDU0VFhTVs2DCrpqbGsizLevnll63MzEz6KMDU1NRYEyZMsK655hrrxIkTvM9ZlsUdmku0e/dupaamKjU1VZI0depUlZaW6uTJk35uWWgrKirS8OHDlZWV5Tr39ttva+TIkerTp48kKSMjQ2+88YacTqe/mhmyunfvrqeeekoJCQmSpGuvvVa1tbUqLi7W7bffrtjYWIWFhWnatGn63e9+5+fWhqb+/furpKREPXv2VEtLi44fP674+Hj6KIC0tLTo8ccf17/927+5zvE+R8npklVVVSkpKcl1HBkZqfj4eJ04ccKPrUJOTo4yMjIUFhbmOnfixIl2fZWYmKjPPvtMp0+f9kMLQ9uAAQN08803S5KcTqeeeeYZjRo1qsM+4m/JfyIiIlRWVqaRI0eqqKhIWVlZ9FEAycvL04033qgRI0a4zvE+R6C5ZJZlyWaznXfebudHGog66quOzsE3Ghsb9fDDD+vYsWNasmSJpPP7g/7xryFDhmjPnj3Ky8vTAw88IIk+CgTbt2/XkSNHNHPmzPO+Furvc3z6XqLk5GRVV1e7jh0Oh06dOqXk5GQ/tgodObevqqurdfnll6t79+5+bFXoOnz4sCZPnqxu3bppw4YNiouL67CP+Fvyj8rKSr3zzjuu49tuu00RERFyOp30UQDYvHmzjhw5ookTJ2rChAmSpPvuu0+9e/cO+fc5As0lGjFihP7+97/ro48+kiS9/vrrGjRokGtsAALHD3/4Q5WUlLj+2AsLC3XbbbdxN80Pjh8/royMDN11113Ky8tTVFSUJCk9PV1vvvmmGhoa5HQ6tWnTJo0ePdrPrQ1N9fX1euyxx1wzZ0pKSmS323XvvffSRwFg3bp1euutt7R161Zt3bpVUtvsszFjxoT8+1y4vxtgqoSEBK1YsUK5ublqbm5Wz549lZeX5+9moQNXX3215syZo/vvv19ffPGF+vXrp8WLF/u7WSFp7dq1amho0LZt27Rt2zbX+TVr1mjSpEmaNm2aWlpadMMNN3R4Sx1db+DAgcrNzVV2drbsdrvi4uKUn5+v73znOzp06BB9FKB4n5NslmVZ/m4EAABAZ4TOvSgAABC0CDQAAMB4BBoAAGA8Ag0AADAegQYAABiPQAMAAIxHoAEAAMYj0AAIGZWVlUpNTVVdXZ2/mwLAywg0AADAeAQaAH71j3/8Q1lZWRo6dKjGjBmjTZs2SZLmzp2rJ598UlOnTtV1112nqVOnuvZOk6TS0lJNmTJFgwcP1tixY1VUVOT6WmNjo5544gkNHTpUN954o3Jzc/XZZ5+5vl5UVKTRo0fruuuu06OPPtruawDMRKAB4Ddnz55VVlaWhg8frj179uhXv/qVfv3rX+vtt9+WJG3ZskWzZs3Su+++qyFDhujBBx+Uw+FQRUWF7r//fk2bNk2lpaVasmSJVqxYoTfffFOStHDhQlVWVur3v/+93n77bR0/flzLly93fd9Dhw5py5Yt+sMf/qB9+/Zp8+bNfnn9ALyHzSkB+E1JSYkuu+wyZWdnS5LS0tKUkZGh1157TQkJCRo7dqxuueUWSdKjjz6qwsJC7du3T6WlpRo8eLAmTpwoSRo0aJCmT5+uzZs3Kz09XTt27NDLL7+snj17SpKeffZZff75567vO3PmTMXExCgmJkZDhw7VkSNHfPvCAXgdgQaA3xw7dkzHjx/XkCFDXOecTqf69u2rhIQEfetb33Kdj4iIUK9evVRTU6Pa2lqlpKS0e66+ffvqjTfeUH19vRwOh5KTk11fS0xMlNQ2KFiSevTo0e55W1tbu+LlAfAhAg0Av+nTp4/S0tLalXxqa2vV0tKiFStWqLq62nXe4XDo5MmTSkxMVFJSkvbu3dvuuY4ePaorrrhCCQkJioiIUFVVlfr06SNJOnDggEpLSzV69GjfvDAAPscYGgB+M2rUKFVVVamoqEgtLS2qqqpSVlaW8vPzJUnbt2/X+++/L4fDoeXLl6tPnz66/vrrNW7cOH3wwQfasmWLWlpatH//fhUUFGjixIkKCwvT+PHjtWrVKp0+fVoNDQ1aunSpqqqq/PxqAXQlAg0Av4mLi9PatWv11ltvafjw4brzzjs1ePBgzZkzR5L0ve99T3l5ebrpppt08OBB5efnKywsTFdeeaVWr16twsJCDR06VDk5OZo1a5YmTZokSZo/f75SUlI0btw4jR49WldeeaVycnL8+VIBdDGbZVmWvxsBAOeaO3euYmJitGDBAn83BYABuEMDAACMR6ABAADGo+QEAACMxx0aAABgPAINAAAwHoEGAAAYj0ADAACMR6ABAADGI9AAAADjEWgAAIDxCDQAAMB4BBoAAGC8/w/GsYb+qgLYvAAAAABJRU5ErkJggg==", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjQAAAG+CAYAAACESJ4gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAABT8klEQVR4nO3deXhTZd4+8PukS1osLV1Yiz/EIuWdjqCssgjoUOB9BVFQX5TOKIOA+wYC46gMrsCr4ugwWjsoSnVcBhkVl7HqKKDI6shYpRTGhZalCyWlkKZN8vz+KInJ6UlyTnKynPT+XJfXZZP05CQpPXe/z/d5HkkIIUBERERkYKZonwARERFRqBhoiIiIyPAYaIiIiMjwGGiIiIjI8BhoiIiIyPAYaIiIiMjwGGiIiIjI8BhoiIiIyPAYaIiIAjh48GC0T4GIAmCgISK/qqqqkJ+fj2PHjkX7VKKitLQUDz/8cLRPg4gCYKAhIvKjoaEB3CGGKPYx0BCRanv37sV1112HoUOH4le/+hWeffZZOBwOAEBFRQVmzpyJoUOHYsKECVixYoX7vvfeew+TJ0/G0KFDMXXqVGzYsEHV8y1ZsgR33HEHCgsLcfHFF8NqtWL//v2YPXs2hg8fjkmTJuHVV191P76lpQUrVqzA6NGjMWzYMNx4442oq6sL+DxHjhzB7NmzMWzYMFx00UX43e9+h1OnTuG9995DcXExNm/ejEmTJrkfe+utt+KCCy7AxRdfjD//+c/u1/n000/jtttuw9y5c3Heeefh0ksvxfbt2zW9x0QUJEFE5MfBgwdF//79RX19vRg2bJh49tlnhc1mE/v37xeFhYWiuLhYCCHE1VdfLVavXi2cTqeorq4WF154ofjkk0/EqVOnREFBgdixY4cQQogtW7aIQYMGiWPHjgV87sWLF4vhw4eLgwcPisbGRtHU1CTGjBkjnnvuOdHS0iK+++47MXbsWFFWViaEEGLVqlViypQp4qeffhLNzc3izjvvFDfccEPA51m4cKG49957RWtrqzh27JiYOnWqKC0tFUII8dRTT4l58+YJIYSw2+1i2rRp4sEHHxRWq1UcPHhQTJkyRaxdu9b92P79+4s333xTtLS0iNLSUjF48GBRX1+v/Y0nIk1YoSEiVdavX48uXbpg/vz5SE5ORl5eHm666SasX78eAGA2m7F582Z89NFHSE9Px6effoqLLrrIfd/69euxY8cODB8+HLt370ZmZqaq5x06dCh69+6Nzp0747PPPkNKSgrmzp2LpKQkDBgwALNmzcLrr78OAHj77bcxf/58nHnmmTCbzbj33ntx++23B3wOs9mM3bt34/3334cQAn//+98xa9asdo/75ptv8P3332Px4sVISUlB7969ccMNN7ifHwCGDBmCyy+/HElJSZg1axZycnLwz3/+U9VrJaLgJUb7BIjIGFpaWtCrVy+v23r37o1Dhw4BAFatWoU//vGPePTRR3H06FFceOGFWLZsGbp3746XX34Zf/7zn3HLLbegtbUVV155JRYsWIDk5OSAz9utWzf3/1dXV+PQoUMYOnSo+zan04nevXsDAGpra9GzZ0/3fVlZWcjKygr4HPfeey9Wr16NP//5z1i0aBGGDBmCZcuWIS8vz+tx1dXVaGlpwciRI923CSEgSZL76z59+nh9T8+ePVUNexFRaFihISJVevXqherqaq/bDh48iK5du0IIgb1792LhwoX45JNP8O6776KpqQmrVq1CU1MTLBYLnnrqKWzduhV/+ctf8M4772Djxo2qntczLHTv3h0DBgzAzp073f+VlZWhpKQEANCjRw8cPXrU/fjq6mqsWrUq4HO4eoPef/99fPrpp8jOzsYf/vCHdo/r3r07unTp4vX8n3zyCf7+97+7H+P5/ABw6NAh9OjRQ9VrJaLgMdAQkSrjxo3DyZMn8eyzz6KlpQX/+c9/8Mwzz+Cyyy6DJEl4+OGHsXr1arS0tCAnJweJiYno0qULTp06hTlz5uCjjz6CJEno1q0bJElCly5dNJ/D+PHjceTIEbz22muw2+3uZl5XoJk2bRqee+45HD58GM3NzfjjH/+I/fv3Bzzus88+i0ceeQSnTp1CZmYmzGaz+/ySk5PR1NQEABg4cCCys7Pxxz/+ETabDcePH8cdd9zhNa1769at+Pjjj2G327F27VpYLBaMHz9e82slIm0YaIhItTVr1uCLL77AqFGjcO2112LKlCm46aabAABPPPEE/v3vf2PUqFG4+OKL0a1bN9xyyy3o1q0bVq1ahVWrVmHw4MG45pprUFRUhIsvvljz86enp2PNmjV47733MGrUKEyfPh1DhgzBokWLAADz5s3DhRdeiJkzZ2Ls2LFoaWnBQw89FPC4y5YtQ1NTE8aPH4+RI0fixIkTuO+++wAAF110EaqqqjB69GgkJCSguLgYFRUVGDduHCZNmoSsrCwsX77cfaxzzz0Xr732GkaMGIF3330Xf/nLX5CRkaH5tRKRNpIQXGCBiEgPTz/9NL755hsUFxdH+1SIOhxWaIiIiMjwOMuJiKLi22+/VZwa7XLFFVfg97//fcjP09DQ4Hd4a9SoUVi9enXIz0NE0cUhJyIiIjI8DjkRERGR4THQEBERkeEx0BAREZHhMdAQERGR4THQEBERkeF1qGnbjY1WOBxOXY+ZmXkGGhpO6npMCh4/j9jBzyK28POILfw81ElIMCE9PVXVYztUoHE4nLDb9Qs0rj3zHA4nOPk9+vh5xA5+FrGFn0ds4ecRHhxyIiIiIsNjoCEiIiLD61BDTkREFB5Op779ifFMkgCHwwGnk0NOACBJEiTXOFwIGGiIiChodnsrGhpq4HDYo30qhlJba2IIdJOQmpqGzp27wGQKfuCIgYaIiILW0FADs7kT0tLSAYT+V3ZHkZho0nWSinEJOBwOnDjRgGPHjiInp2fQR2KgISKioDidTjgcdqSlpcNkSoj26RiKyWRCCMWIuGIyJSAjIwe1tVUQwglJCu6N4dtJREQhYmWGQuPqoQmlp4iBhoiIiAyPgYaIiIgMj4GGiIji1l133Yra2hpN37N8+YPYunWLqsc++ugD+OqrXbjiiqn4+ut/BXGGxrN27V/w8cdl0T6NdtgUTEREUWG12VFaVoED1Y3Iy01HUWE+Us36Xpa2b98KobExY8mS+1Q9zuFwoLz831i06PfBnJph7dy5HT179or2abTDQENERFFRWlaBbeVH4RRA3XErAGDulALdjr9s2b0A2qo0J05YMGjQYPznP/tx9dW/xnnnDcbjjy/HqVOnUFdXi169cvHAA8vRpUsX3HLLPEydehkmTfofjBkzFHPn3ogvv/wctbW1uOyyGSgqug4A8PXXX6Gg4FwkJHjP8NqyZROef/45OBx2JCUl4/rrb8AFF4xCY2MjHnpoKWprj0KSTMjPH4C7774HdrsdK1c+jMrKCiQkJKJXr1zcc89SdOrUyedrO3z4EObPn43+/fNRVVWFFSuegCQBTz75OCyW47DbWzFp0v/gmmt+AwD48ssvUFz8JzgcDiQmJuHWW+/E+ecP8Xn8lpYWxXN6550NqKj4DocPH4IkSbjoogkoKfkzdu7cDqdToHfv3rjzzkXIzs7BLbfMQ58+Z2HfvgocP96A0aMvxC233InExPBEDw45UVCsNjtKNpZjSfFWlGwsh9XGRbWISJsD1Y1wni6eOEXb13pauvQhAMATTzyNpKRknHnm/0Np6Ru45JJLsXHjW7joogl49tnn8frrb8FkMuH99zcqHicpKRnPPPM8li9/AiUlz6CpqQkAsGnTpxg79iKvx/700w949NFluP/+B/Hii69iyZL78MAD96Gq6iA+/PA9pKSk4IUXXsELL6yDEAIHD/6E7du/xI8//oAXX3wVzz9fil69clFZWRHw9R07Vo+ZM4vw6qtvIje3N+65ZxHmzJmPNWvWobh4LTZv/hSffPIRGhoasHTp77Bw4T146aXXcNttd+GPf3zcb+XK1zn97//OQn7+f2HevJswceJ/4+WXX0RLSwv+8pd1WLv2FQwceB6WL3/QfZzq6iqsXl2CdeteR3n5N3jrrTcDvq5gsUJDQQn3X1ZEFP/yctNRd9wKpwBMUtvX4TRo0Pnu/58790bs2rUdr7yyDtXVB/Hjjz9g4MDzFL/vwgvHAQD69TsHQgg0NZ1AWloadu7cjhtvvNXrsbt27cR55w3GWWf1dX/PwIGDsGPHNgwZMhwvvfQCbr/9JgwbNgxXXXUN+vQ5C2lpaWhstGD+/NkYNmwExo27GAUFvwz4eiRJwrnnDgIAHDz4E6qqfsL//d8j7vut1lOnKywm9O79/9zHPO+8wVi79hW/x/6v//qFqnP6/PNNaGxsxL/+9RUAwOl0oLm52X3/5ZdfgeTkZADA5Mn/g23btmLGjKsCvrZgMNBQUML9lxURxb+iwnwA8OqhCSezOcX9/8uW3Qur1YoJEyZh2LDhaG62+qxYmM1m9/9LkgQhBPbu/RZ9+57tdR+gvKeV0yngcNjRt+/ZeOONt7B7907861+7ceedN+GWW+5EYeFkvPzy37Bnz7+we/dO/OEP9+Cyy2Zg1qxr/b4ek8nkfn6n04mUlBSvoNLQ0ICUlBTs2rWj3V5J+/dX4qyz+voc/snOzlF1Tg6HEzfddDvGjWurVNlsze4KFgCv4TghBBISwjcwxCEnCkpebjpMp/99ROIvKyKKP6nmRMydUoDl80di7pQC3RuCgbYLqsPhaHf7tm1f4De/+S0mTpyMtLTO2Llzh6a9lT777J8YO3Z8u9uHDh2O3bt34YcfvgfQFhz+9a/dGDp0BNatW4snnliJkSPH4NZb78Dw4SOxf38lPvzwAyxadCcGDTof119/AyZPvgT79u3V9Dr79DkLnTtn4O23NwAALJbjmDfvOnz++SYUFJyLw4er3cf85ps9uOuuW/wez985JSQkut/TCy4YhQ0b3oDN1laVeeaZP+Hhh5e5j/PBB+/C4XDAarXiH/94D6NHX6jpdWnBCg0FJdJ/WRERBWPcuItx2203oqHhmNftN9xwK5YtuxedO3dGUlISzj9/CKqqflJ93C++2IKiovYVlD59zsI999yPBx64F3a7HSZTAn7/+6U466y+yMrKxiOP/AFFRVfCbDajW7fuuPnmO9CpUyfs3LkNv/71VUhN7YS0tDTNM6cSExOxfPnjeOqpx7F+/WtoaWnB1KnTMGHCJADAAw8sx//93yNobbUjOTkZjz76uN/m3PHjL/Z5TmPGjEVx8Wq0tNhw7bVz8MwzT2POnN8AEOjVqzd+//ul7uM4HA7Mm3cdTp5swq9+NRGXXDJN0+vSQhJa57MZWEPDSV03A5MkICenM+rqTnAL+BjAzyN28LOILeH6PJxOJ2pqDqJbtzND2iW5I+oIm1N6zhYLxNfPUmKiCZmZZ6h6PlZoiIiIYtBNN12PU6dOKd63dOlD6Nv37Jg+fqSxQhMC/hUaW/h5xA5+FrGFFZrY0xEqNFroUaHhTyAREREZHgMNERGFiGU4Co1rsEg2u1wT9tB0QJHYP4WI4p/JZEJCQiKamhqRlpYOIISrUQfjdCqvWdPxCDgcDjQ2NiAxMRmSFHydhVexDoir/BKRXjIzu6GhoQanTnFxTS1MJhMDjZuE1NQ0dO7cJaSjMNB0QFzll4j0kpiYhK5dc3lx1kCSgOzsNNTXN7FpHm2rL8tXMg4GA00HFOn9U4go/nGWk3qS1LaCsclkYqDREQNNB8RVfomIKN4w0HRArv1TiIiI4gUDTYQZdYaRUc+biIg6Bl6RIkzLDKNYChGcGUVERLGMgSbCtMww0iNE6BWKODOKiIhiGQONzqw2O9Z+8B327K+HJEkYmJeNaycPcIcILTOM9AgRnqGopsGKr/bV4fz+OZqDDWdGERFRLGOg0VlpWQV2fFfr/nr7dzVISJDclRUtM4z0CBGeoQgAmlsc2FZ+FIC2ag9nRhERUSxjoNGZUhXF8zYtM4z0CBGeocglmGpPqDOjYqkfiIiI4g+vKDrLy01HTYO13W3B0GN6tSsE7a6oha21bSVPf9WecAUPNhUTEVE4MdDorKgwH3aH06uHJlzDM2rChysUWQvbP1ZJuIIHm4qJiCicGGh0lmpOxI3TzlX9+GArIlabHUtf2I66480AgNoA4UNttSdcwYNNxUREFE4MNFEWbEWktKzCHWYAQOgUPsIVPNhUTERE4cRAE2XBVkSUHqdH+AhX8OB2C0REFE4MNFEWbEUkLzcdtQ1WuCYv5XRJ0SV8xErw0LM5mTOsiIjiH3+rh+hUcytK3inH/iAvlloqIp4X5j490jDsv7rhhyMn4vIirWdzslFnWDGIERGpx9+OIXr2zT34MoSLpZaKiPzCPKKgO5bPHxnUeYeTHhdiPZuTjTrDyqhBjIgoGgwVaBoaGrBw4UJYrVYMGDAA999/f7RPCXt/aIjYxdIoF2Y9LsR6NieHeqxoVUqM8nkTEcUCQwWaV199FVdddRUmTZqERYsWYe/evRgwYEBUz2nAWZk4euyk+8LTeLIFJRvLNV30Al0wXfdbmmzu2wJdmKM5XKHHhVjP5uRQjxWtSgmnuhMRqWeoQFNUVASz2QwAcDgcSE5OjvIZATdMHwibrRW7Tq/EG8xeSYEumJ73A0BKcoJ7g8lgjxlOelyI9WxODvVY0aqUcKo7EZF6hgo0nTt3BgB8/PHHaG5uxtlnnx3lMwI6pSRh7tQC7K/e6t7ywCmAyioLSjaWq6qQBLpgyjeYFELgQHUjSssqfB430hdhecPy0AHeDcuReN5+uem4/eoh7W4PtUKlFNAiUQGLlRlnRERGYKhAAwCbNm3CmjVrUFxcHO1T8SK/6AFQXSEJVNGQbzBpa3WipsHq97iRHq6IVsOy/HnNb+7Bbyb2D7pCpRRUlColbNglIoothgo0FRUVWL16NUpKStzVmlghv+jtP2hRXSEJNLTgeX/jyRY0tzgCHjfU4QqtFYhoDcvIn3fvjw0hnY+voCIPK2zYJSKKLYYKNCUlJbBYLLj55psBAHfffTcGDhwY5bNqIx8eKNlYjvrGZneFpE+PNJ9DUIGGFjzvL9lY7r7g+qu8hDpcobUCEa0GVvnzDuiTGdL5qA0qbNglIoothgo0jz32WLRPQTV5hcThELoMUUSqUVRrBSJaDayez9svNx03TB+IU03NQZ+P2qDChl0iotiiOdCsX78ea9euBQBkZmZi2bJl6Nu3b9An8OSTT6K2thYPP/yw+7bNmzfjscceg81mQ8+ePbFixQp069Yt6OeIBnmFZEnxVr8BQe0QT6QaRbVWIKLVwOr5vNLp3iXPlZuXXjcsLCs3s2GXiCi2aAo0//nPf/D444/jnXfeQXZ2NtatW4f7778f69at83pcZWUl8vLyYDKZALT1vuTne18Yqqqq8Oijj2LLli2YMmWK+/Zjx45h4cKFeOmll5Cfn4+XXnoJv/vd77BmzZpgX2NMCBQQPId4ahqs+GpfnXtqdjSWuzdqBSKSKzcTEVHs0HSlPPvss/HZZ58hKSkJdrsdhw4dQmZmZrvHPf7440hNTcWKFSvwpz/9CZs2bcLrr7/utW7Ma6+9hlGjRuGcc85BbW2t+/YtW7YgPz/fHYBmzpyJlStXora2Fl27dg32dUZdoIAgn5odzHo2wfJVHTLKhd3z/E+catG1WZf7KRERGYPm38xJSUnYuXMnbr/9dlitVsXKyVNPPYWFCxdi4sSJ6NWrF9atW9duEbwFCxYAAJ5++mmv248cOYKePXu6v05OTkZmZiYOHz5s6EATKCDId88GIjd7JpamIAcTIOQLD7ro0awbS+8NERH5FtSfmkOHDsXnn3+Ojz76CPPmzcPHH3+M9HTlC4fD4YDT6VR9bCEEJFczhAfX8FW8KirMR2WVBXXHm71uj8TsmViagqwUIFzrvvgKOfLqljnJhIw0s6bdy32Fp1h6b4iIyDdNgaaqqgoHDx7EyJFtC6ZNmDAB999/P3766Sf88pe/dD/utttuQ6dOnfDhhx/i6aefRlFREf72t7+5ty3wp1evXvjyyy/dX7e0tKChoQG9evXScqoRYbXZ8cQru1D+n/qQhyNSzYkwyYKcOcmEGWPzULKxHJVVFgCAJIB+Z2b43O8pmKERtQ3Aeg6/+DqWPEDsrqjF7tPbSgDKVRLv85cwJL8rrldRRVFTfeH0bCIiY9B0NbJYLLjjjjvw1ltvoUePHvjss89gMpmQl5fn9bgFCxagX79+kCQJCxYswJQpU1SFGQAYPXo0HnroIezbtw/9+/fHG2+8gUGDBiErK0vLqUZE6YcVqhpQ5RfvGWPz8No/K7Fnfz0kScLAvGxcO3lAu4vn4PyuWL/pQLvhlPrG5nbPFcrQiNoGYD2HX3wdS2lVZE9KVRLP8y84OxtXjlO3JYaa6ku0m6PZw0NEpI6m34wFBQVYvHgx5s6dC5PJhPT0dJSUlCA1NdXrceecc47X1/IZTv5kZWVh1apVWLx4MWw2G7Kzs7Fy5Uotpxkx+1UOR8gv3vKhpe3f1SAhQVK8eC5bu6Ndb0ig/Z60Do2obQAOdfjF8+LceFK5edfXqsguSlUS1/lLEpCT0xl1dScgZO+ZEjXVl2g3R7OHh4hIHc1/6k2fPh3Tp0/X7QRuvfXWdreNGjUKGzZs0O05wqVfbjrqjjfDKYTf1YDlQUDeJwO0XcCVLp7yigUQeL+ncA2NhPocvpp3AcApBKw2u89VkQF1u4xrEWz1JZJVE/bwEBGpw9p1CIom5sNsTnL30PhaDVgplMgFWpHW3UMDoF/vDK+Lr9Vmh93hRGKCBLtDICkxAQ7HzwFBK18X7KLCfNgdTvdQmdrncB1v+7c17YKZ6+t6SzNKyyq8Ap1S4NAzOARbffFXNdE77LCHh4hIHQaaEKSaE3HXNUPcQxy+VgN2XZi3f1sDhyzVpCQnYGBedkgr0paWVWDX3lqPvhMHdu5tG8bS84Kdak5EYoIJrXYnnAKqn0OpMmOSgOSkBPeQklCoPkR7uEdOKZjJqyZ6DxFFu4eHiMgoGGh0pPTXtOdf7JnpZtRbmiFO3z+ioLsuF2z5tGXA+0Kr587ZSrOQllRv9XtcpfNLTkrAGSmJsLU63O9HrFcffAUzz/PWMkSk5nOJtVBHRBSrGGh0pPTXtOdFUAKQ3SUFJknS7a9tq80Op0IHrOeFVuvaLv6GOZRmIdU0WP1u16A05Nbc4oCtxaH7+yF/b/Qc/pEHMwnA0AHdvM5byxBRaVkFvvzmKATatruorLJg2ezhnMVERBQE/ubUkfyvaavNjq/21bkvggJt66Qsnz9St+csLatAvUeTcXKiBEmSvHpclKoG/oZG5H07+w9aULKxHEWF+X5nIfnarsHX94Tj/fCk9/BPXm46ahqs7q8FgIQEySuAaBkiOlDd6LUydN3x9n1ELq5wFmg9IiKijoq/CcOotKyi3bRjvYdV5BdFSZLa9bgoVQ38DY24gpnnLCPPtW98zUJSOpbn8eTfE+5hJr1nCBUV5uOrfXVen2kwfT+ucGJpsimesxKl4S6l9YiIiDoqBpowkl+czEkm3YdV5GFFkqR2F/Gl1w1zn4/nUFigoZFAgcD1WjxX8g0UUiLZ5Kr3DKFUcyLO758TciDzNX1d8nO8QH1S/gQ79Ga12bH2g+/aLQDJihARxSL+ZgojpZV/9b4YyAOCwyGwc2+N1wVXqWqgFCzkF76zenT2Gwhcx7UWtr9g+uKrghGOi6fna+zTIw0Oh8CSYv8NzFqOGWwgC2bvKbXrESkJduittKwCO76rdX/tWgCSFSEiikUMNGEUiWqEUt+OgAi4VoxSsPAcDqo7bsWQAV0xoqB7wPPXOhNHqWKg58XTarNj3Yc/H3/pdcPaXdTb1u0xaa5ahDrrSN7ELZ0OuoGOqWY9IqXnCjTN3B+lx3FhPyKKVQw0OpBfQF0Xx2hMuZWvFbP9uxrsOVCvOPtITj7E9OORprA07CpVDAJdPLUMm5R+qHx8z9e2Z3+9+z2K5JYC8ibu7IwUVUHX82fJ870oLavw+V6omWbuj7wJ2nUbEVEsYqDRgdIFNJplefmQhq/ZR1qHmMJxfq6KQaCLp5ZhE6U9ttT0GulNKYTJm7hNkqR56EvteyH/OUgwSRj+i26qK4XylaH9LQBJRBRtDDQ6ULtJpVqhrp+i1G/hOi/PYzuFcC/0p2WIyfMYfXqkQYKEH46cUH2uSs26gS6eWmYs9fNxfNdxfPUa6c0zeNSeXmfmxMkW9/3BPq/a90L+Pg//RTdNQTvVnIgbp52r+fyIiKKBgUYHShfQUIS6foq/2Ue+Ztj4G2KSByzPMOBZVVF7rr72aPJ38VQzY8lqs+OJV3ZhX5UFWRkpXn0mSr1GCQlSWPubPIOHgPempKFstKl29lakZpQpBfBOKYl+7+dMKSLSG3+r6KBoYj4E9LtwhLp+iq/ZRzPG5uHev2xT3CTT34VRHrCSkxJ87Jit7lyD6S0KdHG22uz4wwvbUXs6NEgScIGfrSWUziGcG0vKpZ+RHPSwpK/ZW0rVskj1BckD+LypBX7vD/a8GI6IyBf+JtCB3s2/eq2fIj+vko3l7Rb6y1Gx9YA8YAkhvHbKdpEAOIX3xfU/h9sCjr+VbYPd00g+fOZZAVHa7DKQcG4s6Tm8F2oVT75Qoef2CS6R7OUKFMD1XOBQ78+IiOIHA00MCtdQgdJCf2r2DpIHrIH9st1Tnj2rAp4XbXmDL+B7ZdtQ1klRGj7zPG8tf9HLL7yVVW1bPgRbDfA1M8nXZxpM9UHeZOwSrkZnT0orHiuFNT0XONR79Wciih8MNDEoXNO9g13oz1fPi9yS4q1Q2CfTzdcFKNiLlNLquS45XVLabQ4aKCzJ3x8Aqr5XHkRmjM3D+k0H2r1fgT7TYIKd0uwwQDk4hNrMLT+GZ4AFfPcF6RnQ9V79mYjiBwNNFEW6HyDYC4vagOWvZwTwfQGSX6T69EhTVRnx/D4JbSEmMTEBfXukYdbp79ESluTvz/6DFlXfKw8ilVUWHLM0a644BRPsigrzUVllUdVw7HmewQ5P+auK+eoL0jOgR3LrDCIyFgaaKIp0P0C4F/qTN6t69dDA98q2SlOq5dOdPft8XOFG/n2/npiPM3MzUVd3wl01UPqL3leQVOo5qm9sDlgNkAeRhkZbUBWnYKoPqeZELJs9XFUw9lXR0qMqFqlqSTQWqyQiY2CgiaJ46wcI9mIj/74lxVsVpzvLQ5/8+ySp/bGV/qJXWh9GTWBSs8+SSQIy083uCo1no3SgKlwkKmi1DdZ2PTeeYSRQ1VBeFctW0VSuRqDn5ewmIgqEvxGiiP0AynwNXYUyhd2Tr/VhAgUmX+RBxLOHRr54oefx1ZyrntQMTwWqGqrtp9Iq0POqqWYy9BB1bPzXHkXsB1C+CIVrurOLnoEJaAsirsrPgepGrN90wH0x9WyUdoq2xQ6XVIe243ewUs2JMMnKWPK+l0BVw3CFLj2mfnNKN1HHxkATRewH8H0R0jLdWatwBCZfr0MenmytTtQ0WKNWZQhUFZQPSzmF8m7t/lhtdrxcVoHvDzehb8+2Bm0Aqoeygp36HW9DuESkDQMNRVU0KgJa14dRw9fr8AxPjSdb3AsbRqvKEKgqKB+Wqrc0o7SsQtM5eL6Go8dOusOR1qEsLecNcAiXqKNjoIlRHaUfINoXIb0Ck6/XIV/V13VR99WI23iyJSxVBvnP09Lrhrl/nuT3SR5DccGsuOwr3IUSXJX24pJP7fe1JUQ8//shop/xX3iMilQ/QLSDUyT7iML5WtW8Dl+P8bW2i54Bz9/Pk/y+rIwU99YWwZyDr3CnJbgG+qwCDVV6hkf20xB1DAw0MSpS/QDRbqSMZB+R2tcaTPBR8zp8PUa+tos5yYSMNLPu2174+nmS3wcAQwZ0xZ799ZAkCQ6Htj6aosJ8SAC+P9LkXuTQ87nUvK5An1WwTcRaP9toB34iUo//MmNUpIZiOlIjpdrXWlpW4bXhY2WVRdWeV8FS2pJC75Dn7+dJft85vTMAAK12J5wC2Lm3BgkJkupzSjUnYu7UAuTkdPZa5DDQkJHn+xvoswq2iVhrgI924Cci9RhoYlSkhmKi3cMSSWpfq3zDx7rjvhtj9fgLXmkdm1A2xVTzHK6vrTY77A4nkhJNkCQJA/OyUVSYj2Vrd/gNFKG+7kBBIdBn5e/fh6/X5Hq8lgDfkQI/kdEx0MSoSA3FdKS1cIoK82F3OAMOpSht+Kh2HydA+1/wSlsueB7T7nC6dzcPNuD4+nkqLavArr217uCQkCAh1ZwYMFAovW7PtXj65abj9quH+DyfQEEh0M+lv38fvl4ToD3Ad6TAT2R0DDQdnFHXwlGqEHRKCdznkphgCjiUIp+6LGnYx0mPv+Dlx9yzv959zp6hyfUeVFZZ2s5TAP3OzNAUeNRMN1cKFErfJw855jf34DcT+ys+b6CgEMrPpb/PRGuA70iBn8joGGjIcKw2O5a+sN0dOGpPX+TnTdVnR2tfGz4qke9tpHbfJn/kF3tJkhTPWWl2VK2lWVPPj5rp5mq/T/7e7v2xwef3h3OKtb+wpDUoGTXwE3VEDDTkVyzO8igtq/Daj0jLWilqhxCC2cdJy75Nao+Zl9u2+/jOvTVe52y12fHVvrp2U70B/z0/gZ5LbQXC16afnu/tgD6ZPr/f1/o8ejTesqpC1DEx0JBfWntEIhGAlMKL2t6GYC92vl6X54VZvm9TsFOFlRaRS0iQ2oUH16rDStQ+t147pAPe722/3HT85n/+CyUb9mC/wro7nuej97Cd1tcUi6GdiLTjv1ryS+vFJpzTXF0XHkuTzev2nC4pqoNJsBdwNa9Lr6nCQOCLrL/PQQrw3J7Nu6FewH2dpyQBL733Hb6UPTfQfgsEf1WzSISNYH9mGYSIYgv/9ZFfWmd5hHOaq7xnJCU5Aef3z4nIhUTN6/JV/QnmPdE6rXnIgK7tZkL5em49Q6e/Y+39oUHVFghLrxvmPld51SwS68AE+zPLNWqIYgsDDfmldYgmnNNc5Svqpp+RHLELiJrX5av6E8x7Esy0ZqVQp6Z5N5TQ6e9YA87KxNFjJ9u9bvn5+NssdP9BS9jXgQn2Z1aP95FVHiL98F8O+aV1iCacDZnhDEuBLiyhvK5gvjfYac3y1zFjbF6755Y374byPvo7zxumD4TN1tquh0Z+Pp787Sul1ywyuWA/Wz1+HlnlIdIPAw3pKpzTXMMZlgJdWEJ5XcF8b7CvVc0FUuux/YU9f8fqlJKEuVML3I3SLv7eC6V9pUYUdA9qFpna6kewn60eP4/BVHlY1SFSxn8FZBjhDEuxdmEJ9rWqXWdHy7H9hSS9PxOlfaUCzSIL5rxDpddnH0yVh1UdImUMNESInwtLOIblIrmfkb+qR6w2qOu55pCaKg/3lyJSxkBDhPi5sIRjWE4eJPr0SNN980wXpYqP5xYPWRkpkAD0653htcGmUrVEawDSUnXR67MPpsLF/aWIlDHQECF+LizhGJZTWrk4kpUpz2qISWrrqfF8Pl/VEq3hTkvVJZqfPVdCJlLGQEMUpI5yYZGHpCXFW9tVJ9ptlAngl3k5uHLc2UhJDm3Rvu3f1vithviqlmgNd/LjVFZZfFaiovnZc38pImUMNERBCuXCEu6ZKuE8vlJ1QmmjzM92V8Nma8X1Oiza56JUDdGrWiI/DtB+VeNwNUMTUegYaIiiQD68YXc42630G0oACWfDslJ1YtnaHe02ynQKgf0Bekv8BS/5QooJJgnDf9GtXTVEr2qJ6/tcVaZjjc0xNfONiPzjvzSiKJAPb+zZX49Wu1O3ABLOhmWl6oRndcOTpcmGRc9+AUkA/c7MaHeB9xe85BWT4b/opvie6FUtcR3Hc/dvFyPPfCPqKBhoiKJAfrGWJEnXABLpplV5dePEyRbYWp1t/x1vBgDUWZpRWWWBSZJU7bQdrT4VtZUhIPBWDdu/rQEAVmqIIoD/woiiQGnm0M69NboFkGDCQCjDJUqNwzUNVq/HCAB1rnCjsNO20tYG4ahuBHqdaitDgP+tGgDA4WybEQawUkMUbgw0RFEgDwBWmx0JCZJu1Qg1wzDyC7tnqAp1uKRfbjrqjjfDKd/34DSlnba1bm0QrEDDQoHCoOf71niyRXGrhu3f1sBx+oZYWaOIKN4x0BDFgGjMmpFf2JOTEvwOe2mp4BRNzIfZnIR/H6gD0FZ9EYA7sCjttK11awOt5+QSqL8o0GehNPsKaL9Vw5ffHIXrIU4hYLXZOexEFEb810XUQckv7EII93CJ0rCXlobXVHMi7rpmCOrqTrhDilL48BSp7SdC7S+S99iYk0zISDN7vaaiwnxUVlncQ2z1lmaUllVw2IkojBhoiDoo+YV9YL/sdlPHPYU6cypQ5SNS20+E2mwsf98G53dt97pSzYkwSZL7ayFbgJDTuon0x39JRB2U0oXd38VVr5lTvi7q4dh+wtdzhVIpURuIAi1AyGndRPpioCHqoLRe2PWaRq3nRT3QOfl7rmCrJWrft0ALELJZmEhfDDREpIpejct6LvoX6Jz8PVe4qyWBFiCMlQ1NieIFAw0RRVQkL+r+niucqyn70lE2NCWKBgYaIoqoSF7U/T1XNKolWqtcejQRW212vFxWge8PN6FvzzTMYiMyxSn+VBNRREVyzR1fz2W12WF3OJGUaIIkSRiYlx2T1RI9hsU8j3H02EmIII5BZAQMNETU4aYTl5ZVYNfeWnd1JiFBQqo50e/74Os+Ne9dsO+vHsNi0RhaI4qG+P2NRUSqdbTpxL4u8v7eB1/3qXnvgn1/9RgWYyMydRQMNETU4f6K93WR9/c++LpPzXsX7PurR79RUWE+JADfH2lC3x5tPTRE8YiBhohi6q/4cA1/eR63T480DB3QDT8cOeEVFPy9D3m56ahtsLbbn0nNexfs+6tHv1GqORFzpxYgJ6ez11YURPGGgYaIYmo6cbiGv+THHVHQHcvnj/R6jL/3wdf+TGreOy3vbyi9OkQdGf81EFFUdvv2JZjhGTUXezXH9TcrqrSsAg2NNvdtrv2Z1Lx3Wt5ff706rh28axqsqKyyYNns4UGFGoYjikemaJ8AEZGnvNx0mE7v66h11+2aBiu2lR9FaVmF+z6rzY6SjeWwNP0cRrQOq7mO7/DYZlvvoTnXeW7/tsZnr47naFHd8Wav16mFv/eLyKgYyYkopui967ZnxQMAUpITcH7/HK/jyisWM8bmYf2mA+6v9x+0wCPLIMEkYfgvuuk6NCc/T8A7NOXlpqOmwdrudSudf6CKS0drAqeOgYGGiGKK3rtue168ASD9jOSA06orqyw4Zml2f52VkQKTBPfxh/+im+5DdPLzlIcmeQ+P5PE6tfYdxVITOJFeGGiIyPBC3eJAXrFoaLR5fQ0AIwq6h7VpWn6e8tCUak7EstnD21VilM4/UMUllprAifTCQENEhuevqqPm4i0PE5npZneFxiQB5/TOCFiRcQ37VFZZAACSAPqdmaG64dbzPPv0SIPDIbCkeCv69EiDBMlrirn8eForLrHUBE6kFwYaIoprai7e8tAj76FRU8FQ6oGpszSjssoCkyQF7G3xPM+SjeXuY3n2zfgaTmLFhYiBhohIMfRorWDIe2AAQADunhdfYUSpoVfpWID26eaBcPo2xRP+5BIR6cBz2EeJUwC7K2phLbR7hQalhl5fx9K7gTfURQwZiCiW8CePiAwp1i6mrmEedw8NvCs0AGBrdaK0rMIrNMgbendX1KLzGcnIykiBBOCsnp3b9dDoRem55YHLn462qSnFNgYaIjKkWLuYKg37WG12LFj9OZpbHO7b5ENG8mqMrdUJ2/FmmKS2mVV6vaZTza0oeacc+z0CoHxtG6XA5Q/Xs6FYwkBDRIak5WIarWpOqjkR5/fP8WoWbjzZgpKN5e5z8GzobTzZ4g4/egeEZ9/cgy9lAbCoMB9f7avzG7j84Xo2FEsYaIjIkLRcTKNZzXEFlt0VtbC1OtHc4sC28qPuc/A1u0lC247eS4q36hLC9v7Q0C4AygOX1lDC2VUUSxhoiMiQtFxMozk04gosS6q3uod3fJ2D90J5AvWWZggdGnb75abjnDMzcPTYyXbBJZRQwvVsKJYw0BCRIWm5mPqq5kRyKEpNRcnzNS0p3goRZAiz2uxY+sJ2d0Ny7XErxgzqhQsKunv10Mifk8jIGGiIKO75qkJEcihKayXEMwBpHX4qLavwml0lBLDju6N44ubRSEnmr32KT/zJJiJDCaaq4qsKEcmhKK2VkEDDT0WF+T7fB6XX0WxzoPTDClyvU2CLtWnzRPzpIyJD0bOqEsuzdAINPym9D66QY2myKR5zf4iBzTPEhNrjQ6Q3BhoiMhQ9qypGmaWjFLyUFsVzOAR27q1x326S4PH/EvqFGNiU9qtyPT/XoKFoY6AhIkPRs6pilIZYpeBVWlbRblG8PQfqvcJGYoKE887pih+OnEDB2dm4ctzZIZ2Hrz2mYq26RR0TAw0RGYpRqip6UgpeSoviCeGdNlrsAgkJElbcMBI5OZ1RV3cCQiGQqCVvVM7ukuK1kzhRNDHQEJGhGKWqEm5Ki+IN7JeNfx84FvTKv4EohUk2AlOs4E8iEZFB+RqKCnblXxdfM5j0DpOcKUV64k8OEVEU6HEx9zUUBYQ2JBep9XlibYNRMjYGGiKiKPC8mNc2WFFZZfHqR/EXbvyFIT2qKEozqKyFdt2rJ9ytm/TEQENEFAWeF3MBuFf2VVOp0KOy4S8U5eWmt5tBVVpWoXv1JJbXASLjYaAhIooCz4u5JzWVCj0qG/5CkdIMqkDPEcwQWkecsUbhw0BDRBQFvrY2UFOp0KOy4S8UKc2gCvQcwVSNOGON9MRAQ0QUBZ4Xc6Xqhj96VDYChSKtz8F+GIo2BhoioijTWqnQo7LhGVj69EiDw9F+N28tzxEoIHGKNoUbf5qIiDogz8BSsrE85CbjQBUdTtGmcGOgISLq4PQYLgpU0YnUVHDquPiTRETUwfkbLrLa7Fj7wXfYs78ekiRhYF42rp08QHMQUZoKvvSF7Vg2ezhDDemCP0VERCGK9f4Qq82OJ17ZhfL/1Cuen7/hotKyCuz4rtb99fbvapCQIAU1JCWfCl53vDks69tQxxQ7/+KIiAwq1vtDSj+swJd+zs/fcJHS8FOwQ1Ln98/B1m+Oet2+/dsaAIi5EEjGY4r2CRARGV2sT1neH8L5Ka0/E+yKvkWF+cjpkuJ1m8MpsK38KErLKoI6JpEL4zARUYhifQn/frnpqDveDKcQfs9PaeisqDAfdofTq4dGzbo3vobhls0ejtKyCmz/tgaO0ykrFkMgGQ8DDRFRiGJ9Cf+iifkwm5O8emiU+Bo6u3HauZqf09exPIe3tKxErJVezcxkHPxkiYhCFOtL+KeaE3HXNUNwsLoB6z6swLK1OxSbg/UcOlOapr2keqtX5cf1OHnI0qPJWq9mZjIOBhoiog6i9EP/zcuBpm+rCRmux1mabF6321qdqGmwotbjeX2FCz2arPVqZibjYKAhIuogAjUHB5q+rSZkeD4OaAtGnjuKiwjtJi5f98Z1G8UvBhoiog6iX246ahuscOULpxCw2n5erTfQ9G3PkOFrurXn4wBAkqS2FONBy27iANB4sgUlG8s1DT0F28xMxsVAQ0TUQRRNzMe+KgvqjjcDAOot6he2k4cMh1Ng6zdHUVll8VrtVz5slZluRv3xZneIyumS4jNYuIarKqssyMpIwYmTLbC1OtHc4sC28rb1a4oK81UNfaWaE4NqZibjYqAhIuogUs2JMEmS+2s1wz8urhDiOd0aaL/ar3zYasbYPKzfdEBVg6/ncJVJApKTEtz3uYaeYn0RQ4oeBhoiog4k2DVzPIej5Kv9eoYipWErtYFDPqwlTq+b43musb6IIUUPVwomIupAigrzMaKgO7plpmJEQXfNfSXy1X4lHdeQyctNh+l0AckkAQP7Zbc7V/lj2OhLLqzQEBFFULQ3sgx1zRzP1X71XkhQaZaV/L2J9UUMKXoYaIiIIigeekDCtZCgmuPq9dzRDpakP356REQRxB4QZZEOGPEQLMkbAw0RUQTF+kaWkeYKMrsramFrdQLwDhjhCjoMlvGHgYaIKIJitQckWkMw8pWFAe+AEa5KCoNl/GGgISKKoFjdyDJaQzDylYUB74ARrkpKrAZLCh4DDRERRXQIxrMa5BQCnrsjpCQn4Pz+Oe6Aocc2CEpiNVhS8BhoiIgookMwntUgCUB2lxSYJElxqMsVbFw9Np7bIDCQkCcurEdERO4F93K6pCArIwX7D1pQsrEcVptd9+fyrAYJACZJwvL5IzF3SkG7qourkpKRZnbfxiZeUsJAQ0RE7uBwTu8MHLM0o9bSjG3lR1FaVqH7cwWz2i9XCKZAOORERERuWntp/M2O8nVfMA25bOKlQBhoiIjITWsvjb/ZUb7u09qQKw9GM8bmaZ5izpWB4x8/TSIiA9P7Qq21EuKvoqPXzCl5MKqssuCYpVnTFHOuDBz/GGiIiAxM7wu11uqJv4qOXjOn5MGoodGmGJT8hTtf4YqVm/jBT42IyMCivYS/Z0WnT480OBwCS4q3uoeGXPeF0vciD0aZ6WZ3hcYzKPkLd77CFSs38YOBhojIwKK9hL9nRadkY3lYwoF8GGzG2Dys33SgXVDyF+58DaVFOxCSfhhoiIgMLJZm/4QrHCgNgykFJX/hztdQmr/v4XCUsfCTISIysHAu4a/1gh6ubQrU0ns6OIejjIWBhoiIFGm9oEd7m4Jgwp2/7+FwlLFwpWAiIlKk9YIeb9sUcHViY2GFhoiIFAXbcByuRuVI97TEUn8SBcZAQ0REioK9oIcrCCgNgRUV5oct5ISzP4n0x0BDRESKgr2ghysIKA2BhaNxl7ObjImfEBERGYLSUFY4Gnc5u8mY2BRMRESGUFSYjxEF3dEtMxUjCrqjqDA/LI27nN1kTKzQEBGRISgNZYWjX8dXUzOHomIbPwkiIjKscPTr+ApJHIqKbQw0RERkSOGqmPgKSRyKim0MNEREZEieFZOaBiu+2leH8/vnhG0oKNobgZJ/DDRERGRInhUTAIpbLait4qh5nJp+HfbZRA/fZSIiMhyrzQ6nEO1ulw8Fqel7sdrsWPrCdtQdbwYA1Pp4nJp+HfbZRA+nbRMRkeGUllWg/nQA8SQfClLT91JaVuEOMwAgQuiPYZ9N9LBCQ0REhnOguhGe9ZnkRAmS1PafwyFgtdmRak5U1feiFDrkj1M7lMQ+m+hhoCEiIsORB4f0NDOOWZrhFMDOvTVISJAwd0qBqr6XvNx01DZY3QEpp0tKu8epHUrihpbRw0BDRESGIw8O+w9avIZ6tn9b435coB4WXyGkZGO5z+P7GkrihpbRw0BDRESGIw8OJRvLUd/Y7A4dDqdwz3gKtCO357FcQ0u7K2pha3UCaKvIZGWkwCSBQ0kxjIGGiIgMz1VV2f5tDRynU00wO3J7PtbF9f8jCrpzKCmGMdAQEVHMCHYdF88qiyuQBLMjt3xtG6DtOOf0zsDcKQXu81u2dofP8+NaNNHBd5iIiGJGqOu4KPXDtE3L/rmBuE+PNK/+GM/A4dlsDAApyQnu1YfVnh/XookOBhoiIooZoa7jomZHbodD+AwcSoHIs7qi5vy4Fk10MNAQEVHMCMc6LvKQs6R4q8/AIX+s1Wb3quac1aNzwPPjWjTRwUBDREQxIxLruGgJHPLhoyEDugZsDuZaNNHBQENERDEjEuu4aAkc8uGjH480Yfn8kX6Pz7VoooOBhoiIOhQtgUPP4SPP2U8FZ2fjynFnIyWZl2G98J0kIqK44m/atNYp1f6qOVqP5TV8tbsaNlsrrmclRzcMNEREFFf8TZvWOqXaXzVH67G8h68E9nP2k65M0T4BIiIiPfmbNq3nlGqtx8rLTYdJavt/kyShH2c/6YqBhoiI4op3cPDue/F3n2uK9pLirSjZWA6rzR7083hyHbeyyoKsjBRkZ5jRNTMVlVUWVc9D6nDIiYiI4oq/vhd/92kdQlI7W8rzuCYJyM5IQW3DKTgFUG9pDvg8pA4DDRERxRV/fS/+7tM6hKR2tpT8uMcabVxJOAwYaIiIiBC+KdpOISBJgDh93Kx0M+otNjiF4ErCOmKgISIigr4r/HoOM0kAsrukwCRJ6NMjDU6HwIlTrQCAgXnZXElYJww0RERE0HeFX89hJoG2WU3L549EycZybNv7cz9NQoLkd+0aUo+znIiIiHTmawYUd+IOH8ZCIiKiEMlXDZ4xNg9A++Er7sQdPgw0REREIVI75buoMB8SgO+PNKFvjzTMYv+MbhhoiIiIQqR2KCnVnIi5UwuQk9MZdXUnIEQETzLOsYeGiIgoBFabHU6PZCJxKCkqWKEhIqIOSetu2b6UllWg/niz++vsjBROxY4CBhoiIuqQtG514MkzDDWebIHnyJFJ4lTsaOA7TkREHVIoU6g9w5AnzlyKHgYaIiKKKr2GfrQKZQq1ZxgCAHOSCRlpZp8rDHu+xn656bj96iF6vATywEBDRERRFcrQTyhC2epAHoYG53f1e87y12h+cw9+M7F/yK+BfsZAQ0REURWt1XND2epAaxiSv8a9Pza0e4yWSlW0qlqxrGO/eiIiijojrp6rNQzJX+OAPpntHqOlUhWtqlYsY6AhIqKo0nOX61jiWUXp0yMNQwd0ww9HTqBfbjpumD4Qp5qavR7vq1KlVI3hnlDtMdAQEVFU6bnLdSyRV1FGFHTH8vkjIUlAp5SkdoHGV6VKqRpjxKpWuDHQEBERhYHWKoqvSpXScZZeN0zxsR0ZAw0REZEGahtytVZRfFWqlI4Tr1WtUDDQEBERaaBlZ20g9CpKvPYY6Y2BhoiISANNO2v7qaJYbXas+zBwpYfVGHUYaIiIiDTQMpSkNDzVKaXt0lv6Iade64mBhoiISAMtQ0BKw1PzpraFlv2ceq0rBhoiIiINtAwB+Rue6sep17pioCEiIgoTf8NTRRPzIaCu0sOtDgLju0FERBQm/oan1DQNu0KMUwjUW5oh2G/jEwMNERFRmIQyQ8mz/8YT+22UmaJ9AkRERNSeZ/+NJ/bbKGOFhoiIKAZ59t9IALK7pMAkST77bTp6n03HeaVEREQGUlSYD7vDiT376yFJEs7umY5rJw/wGVLUrmAcrxhoiIiIIuxUcytK3inHfj/VlFRzIhITTGi1O+EUwM69NUhIkHyGFK2bYcYbBhoiIqIIe/bNPfhSRTVFS0jRuhlmvGGgISIiirC9PzSoCipaQkpH38SSgYaIiCjCBpyViaPHTgYMKlpCSkffxJKBhoiIKMJumD4QNluru4dmxtg8lGwsbzdDqaOHFC0YaIiIiPwIx3ToTilJmDu1AOL0sFPJxvIOPUNJDww0REREfoRzOrQrLG3/tqZDz1DSAwMNERGRH+GcDq20vYFJAvr0SFMcglLS0RfUc+l4r5iIiEiDcE6Hlm9vkGCSMPwX3eBwCNVVoY6+oJ4LAw0REZEf4ZwOnZebjtoGK1yZJjPdjKLCfCxbu0N1VUipguSrahPP1Zz4eBVERERhEs6ZRkWF+aissqDueDMAoN7SjNKyCsWqkK8wovRYX1WbeK7mMNAQERFFSao5ESZJcn8tTldYll43DIB3VchXGFGqIPmq8MTz9ggMNERERFGkVGFRqgr5CiNKj/XV9xPP2yMw0BAREUWR2h4dPbZBiOftESQhhAj8sPjQ0HASdrtTt+NJEpCT0xl1dSfQcd7F2MXPI3bws4gt/DwiQ23DbbCfRzw39PqSmGhCZuYZ6h4b5nMhIiLqEMLdcKtXc3K8BiPjvwIiIqIYoEfDbSTCRrzOdGKgISIi0oEeDbd6hg1f4SheZzox0BAREelAj4ZbPcOGr3AUrzOdGGiIiIh0oEePi9aw4arCVFZZAACSAPqdmYGiwnyf4cgzePXpkQaHQ2BJ8VbD99MY86yJiIjikNYqj9LmlvWNbasO+wpHnsGrZGN53PTTMNAQERHFCK1VHvnmlsDP1Ril1Yb9fb/R+2kYaIiIiAzEs9nXKQQkCV7r2fhbbVgunvppGGiIiIgMxHOYSQKQ3SXFfZ8EoF/vDK9qjL+p4PG0cjADDRERkYF4DhMJACZJwvL5I30+3t9U8HDuJB5pDDREREQGonWYSKlPJh5XCzb22RMREXUgVpsddocTSYkmSJKEgXnZAYeJlAJQPK4WbIr2CRAREZE6pWUV2LW3FrZWJ1paHUhIkAJWVooK8zGioDu6ZaZiREF3v2vUGBkrNERERAYRTBBR6pOJp9lNLgw0REREBqFXEImn2U0uDDREREQGoWbbAjUNv/E0u8mFgYaIiMgg1GxbEI8Nv2ow0BARERmQr+nYX+2r09xnEw/TuI11tkRERATA93Ts5hZHu8cFEg9VHU7bJiIiMiBf07E9mZNMqhp+42EaNys0REREBqRmOvbg/K6qho7iYRo3Aw0REVGcCHY6djxM42agISIiihPBTseOh2nc7KEhIiIiwzNUhaahoQELFy6E1WrFgAEDcP/990f7lIiIiOKS0aZyG6pC8+qrr+Kqq67CK6+8gqamJuzduzfap0RERGRoVpsdJRvLsaR4K0o2lsNqswP4eSp3TYMV28qPorSsIspn6l/sRi0FRUVFMJvNAACHw4Hk5OQonxEREZGx+VqDxmhTuQ0VaDp37gwA+Pjjj9Hc3Iyzzz47ymdERERkbPLgUlllQcnGcliabO7HGGEqt6ECDQBs2rQJa9asQXFxcbRPhYiIKKap6YORr0EDwF2xAYCU5ASc3z8n5qdyGyrQVFRUYPXq1SgpKXFXa4iIiEiZmi0N5GvQ7D9ocYcZAEg/I9kQU7oNFWhKSkpgsVhw8803AwDuvvtuDBw4MMpnRUREFJvU9MHI16Ap2ViO+sZmw60abKhA89hjj0X7FIiIiAwjmC0N/K0aHMtTuTWfxV//+le88sorkCQJqamp+P3vfx9SleTJJ59EbW0tHn74YfdtmzdvxmOPPQabzYaePXtixYoV6NatW9DPQURE1BEFs6WBv1WDY3lXbk2BZvfu3Xjuueewfv16ZGVl4Z///CduuukmbN68GZIkuR9XWVmJvLw8mExty9xUVFQgP9/7TayqqsKjjz6KLVu2YMqUKe7bjx07hoULF+Kll15Cfn4+XnrpJfzud7/DmjVrQnmdREREHY5nOAm1umK12fHVvrqYncqtaWG9jIwMPPjgg8jKygIADBw4EPX19bBarV6Pe/zxx7FgwQK0tLTgiSeewOLFi9HS0uL1mNdeew2jRo3C7NmzvW7fsmUL8vPz3QFo5syZ2LZtG2prazW/OCIiImoTykJ5VpsdS1/YjuYWh9ftsdRfoynQ5OXlYcyYMQAAp9OJRx55BOPHj0enTp28HvfUU0/B4XBg4sSJ2LlzJ9atW9duEbwFCxZg1qxZSEhI8Lr9yJEj6Nmzp/vr5ORkZGZm4vDhw5peGBEREf0slIXySssqUHe82es2c5IppqZyB9XJ09TUhEWLFuHYsWN47rnn/D7W4XDA6XSqPrYQwmv4ysU1fEVERETaBdMg7KIUfgbnd42ZhmAgiL2cvv/+e1xxxRVIS0vDiy++iPT09m/IbbfdhsTERHz44YcYPnw4ioqKYLPZFI7WXq9evXD06FH31y0tLWhoaECvXr20nioRERGdVlSYjxEF3dEtMxUjCrq3q64o7enkus1z1WAAyOmSElPVGUBjhebQoUOYNWsW5syZgzlz5vh83IIFC9CvXz9IkoQFCxZgypQp7j2YAhk9ejQeeugh7Nu3D/3798cbb7yBQYMGuft2iIiISDt/s5cA5RlMgP9Vg0s2lsfMFG5Nz7xmzRo0Njbi7bffxttvv+2+/bnnnkP37t3dX59zzjle3yef4eRPVlYWVq1ahcWLF8NmsyE7OxsrV67UcppERESkkbzHZndFLdI7JftcNbhkY3lMTeHWFGjuu+8+3HfffbqewK233trutlGjRmHDhg26Pg8RERH5lpebjpqGnysztlYnhNTWb6PUdxNru3Gz05aIiIhQVJiPlOSEdrf76rvJy013b2YZC1skxE57MhEREUVNqjkR5/fPcQ8jmSTgnN4ZPoeRglmFOJwYaIiIiAiAtpASqMk40hhoiIiICEDshRQtGGiIiIhItVjdcTv6Z0BERESGEas7bnOWExEREakWa9O1XRhoiIiISLVYm67twiEnIiIi8kneMzNjbB6A2Jmu7cJAQ0RERD7Fas+MHIeciIiIyKdY7ZmRY6AhIiIin2K1Z0aOQ05ERETkxbNvpk+PNAwd0A0/HDkRUz0zcgw0RERE5EXeNzOioDuWzx8Z7dPyi0NORERE5MUofTOeGGiIiIjIi1H6ZjxxyImIiIi8aNl1O1Yw0BAREZEXI+66zSEnIiIiMjwGGiIiIjI8DjkRERF1QPI9mooK85FqVhcLQvnecGGgISIi6oBC2aMpFvd34pATERFRBxTKWjOxuE4NAw0REVEHFMpaM7G4Tg2HnIiIiDqgUNaaicV1ahhoiIiIOqBQ1pqJxXVqOOREREREhsdAQ0RERIbHQENERESGx0BDREREhsdAQ0RERIbHQENERESGx0BDREREhsdAQ0RERIbHQENERESGx0BDREREhsdAQ0RERIbHQENERESGx0BDREREhsdAQ0RERIbHQENERESGx0BDREREhsdAQ0RERIaXGO0TICIiImOy2uwoLavAgepG5OWmo6gwH6nm6EQLBhoiIiIKSmlZBbaVH4VTAHXHrQCAuVMKonIuHHIiIiKioByoboRTtP2/U7R9HS0MNERERBSUvNx0mKS2/zdJbV9HC4eciIiIKChFhfkA4NVDEy0MNERERBSUVHNi1Hpm5DjkRERERIbHQENERESGx0BDREREhsdAQ0RERIbHQENERESGx0BDREREhsdAQ0RERIbHQENERESGx0BDREREhsdAQ0RERIbHQENERESGx0BDREREhtehNqdMSAhPfgvXcSk4/DxiBz+L2MLPI7bw8whMy3skCSFEGM+FiIiIKOwYD4mIiMjwGGiIiIjI8BhoiIiIyPAYaIiIiMjwGGiIiIjI8BhoiIiIyPAYaIiIiMjwGGiIiIjI8BhoiIiIyPAYaIiIiMjwGGiIiIjI8BhogrR582ZMmzYNkydPxuzZs1FTUxPtU4p7f/3rXzF16lRceuml+N///V/s2bMHALBmzRpMnjwZhYWFWLZsGVpbWwEATqcTK1aswKRJkzBhwgT86U9/Arcu09fXX3+NX/7ylzhy5AgA4K233sIll1yCSZMm4Y477kBTU5P7sb4+JwpdZWUlfv3rX+Oyyy7D9OnT8a9//QsAP49o+eijjzB16lRMmzYNs2bNwoEDBwDwd1XYCdKsvr5eDB8+XOzdu1cIIcSLL74ofvvb30b5rOLbrl27xPjx40V9fb0QQohPPvlEjB49Wnz66adi8uTJorGxUdjtdnHbbbeJ4uJiIYQQL7/8sigqKhI2m01YrVYxc+ZMsXHjxmi+jLhSV1cnpk2bJvr37y8OHz4s9u3bJ0aOHCmOHDkihBDi0UcfFffdd58QQvj9nCg0VqtVjBkzRvzjH/8QQgjxz3/+U4wfP56fR5RYrVZx7rnnisrKSiGEEC+99JKYNWsWf1dFACs0QdiyZQvy8/ORn58PAJg5cya2bduG2traKJ9Z/MrIyMCDDz6IrKwsAMDAgQNRX1+PsrIyXHLJJejcuTMSEhJw9dVXY8OGDQCAsrIyzJgxA8nJyUhJScEVV1zhvo9CY7fbcdddd+Huu+923/bRRx9h3Lhx6N69OwBg1qxZeOedd+B0Ov1+ThSaLVu2oGvXrpg4cSIAYNy4cXjmmWf4eUSJw+GAJEmwWCwAgFOnTiElJYW/qyKAgSYIR44cQc+ePd1fJycnIzMzE4cPH47iWcW3vLw8jBkzBkBbefaRRx7B+PHjcfjwYa/PokePHu7P4fDhw+jRo4fifRSalStXYsSIERg9erT7NqXP4tSpUzh+/Ljfz4lC8/3336Nbt2649957MX36dFx77bVoaWnh5xElZ5xxBpYtW4Zrr70WY8eOxfPPP4+7776bv6sigIEmCEIISJLU7naTiW9nuDU1NeGWW25BdXU1VqxYAQDtPgvX10qfEz+j0G3cuBE//fQTbrzxxnb3Kf27cN3m63Oi0Njtdnz++ee47LLL8Oabb2LOnDmYN28e7HY7P48oqKiowFNPPYW3334bmzZtwr333ou5c+fC6XTyd1WY8R0LQq9evXD06FH31y0tLWhoaECvXr2ieFbx7/vvv8cVV1yBtLQ0vPjii0hPT2/3WRw9etT9OeTm5ra7z/MvJArO+vXr8dNPP+Gyyy7DtGnTAABz5sxBt27d2r3fZ5xxBjIyMvx+ThSa7t2746yzzsLQoUMBtA05JSYmKr7n/DzCb8uWLTj33HNx9tlnAwCmTp0Kh8MBh8PB31VhxkAThNGjR+Pbb7/Fvn37AABvvPEGBg0a5O7vIP0dOnQIs2bNwpVXXomVK1fCbDYDAAoLC/Huu++isbERTqcTr776qruXoLCwEG+++SZaWlrQ3NyM9evXu++j4L3wwgt477338NZbb+Gtt94C0DZ7Y9KkSfjss8/cv5hffvllTJgwASaTye/nRKEZO3YsDh8+7J7ZtGvXLrS0tGDChAn8PKKgoKAAu3btcs/827FjB+x2O6677jr+rgqzxGifgBFlZWVh1apVWLx4MWw2G7Kzs7Fy5cpon1ZcW7NmDRobG/H222/j7bffdt/+3HPPYcaMGbj66qtht9sxePBg91DIVVddhaqqKlx++eVobW3FhAkTMGPGjGi9hLh3zjnnYNGiRbj++uvR2tqKvn37Yvny5QDaLroHDhxQ/JwoNDk5OSguLsYjjzyCU6dOISEhAU8//TQGDBjAzyMKLrjgAtx8882YPXs2kpKS0KlTJzzzzDMYPHgwfvzxR/6uCiNJCE52JyIiImPjkBMREREZHgMNERERGR4DDRERERkeAw0REREZHgMNERERGR4DDRERERkeAw0R6a65uVn1Zq0HDx4M89l4O3r0KFpaWiL6nEQUfgw0RKS7a665Brt27Qr4uBUrVuCFF15QdcwlS5bggQceCOm86urqMHnyZDQ1NYV0HCKKPQw0RKS748ePq3pcQ0NDeE9Eprm5GadOnYrocxJRZDDQEJGubrjhBhw6dAiLFi3C6tWr8Y9//AOXXnopBg8ejGnTpuGjjz4CAJSUlOCdd97B66+/jt/+9rcAgA8++AAzZszAiBEjMGTIENx5552wWq2az2HVqlUYM2YMRowYgVmzZuHrr78GAPdmmr/61a+wbds2OBwOPPfcc5gwYQJGjBiBG2+80b0HT1VVFX7xi1/g+eefx4gRI3DBBRfgqaeeAhdXJ4pRgohIZxdddJF4//33xRdffCEGDhwoNm3aJOx2u/j000/FwIEDxa5du4QQQixevFgsW7ZMCCHEoUOHxLnnniu2b98uhBDi4MGDYsyYMeL1119v91h/vvjiCzF69GhRW1srHA6HePLJJ8WMGTPcx+zfv7+or68XQgjx/PPPi8mTJ4sffvhBNDc3ixUrVojLL79cOJ1O92Pnz58vTpw4Ifbv3y/GjBkj/vrXv+r+fhFR6FihIaKw2bBhA6ZMmYILL7wQCQkJGDduHC655BJs2LCh3WOzs7Px7rvvYtiwYbBYLKirq0NmZqZ7t2i1kpKSYLFY8Prrr6OyshK33nor/va3vyk+9vXXX8cNN9yAPn36wGw246677sL333+Pf//73+7H3HPPPUhLS0NeXh6KioqwceNGbW8CEUUEAw0RhU19fT1yc3O9buvduzcOHTrU7rFJSUl48803MWbMGEyfPh0lJSVobm7WPMQzdOhQPP7449i2bRuuvPJKXHTRRXjjjTcUH3vo0CH84Q9/wNChQzF06FBccMEFcDqdqK6uBgCYTCaceeaZ7sf36NFD9ewtIoqsxGifABHFr549e6KqqsrrtoMHD6Jr167tHvv+++/j73//O/72t7+hR48eAICZM2dqfs7Dhw8jNzcXL774Ipqbm/HBBx9g8eLFuOCCCyBJktdju3fvjnvuuQfjx49333bgwAH07t0btbW1cDqdqKmpQffu3QEA1dXV6Nmzp+ZzIqLwY4WGiHSXlJSEEydO4PLLL8e7776LzZs3w+FwYNOmTXj33XfdzbnJyck4ceIEAMBiscBkMiE5ORl2ux1vvPEGvv76a7S2tmp67j179mD+/Pk4cOAAUlJSkJmZieTkZJxxxhlITk4GAPe07RkzZuBPf/oTqqur4XQ68fLLL+Pyyy/3mqX1+OOPw2q1Yt++fXjllVdw2WWXhf4GEZHuWKEhIt3NmDEDDz/8MGbOnInly5dj5cqVqKqqQq9evfDoo49i5MiRAID//u//xh133IEZM2bglVdewfbt2zFhwgSYzWYMGjQI06ZNw759+zQ996RJk1BZWYnZs2ejsbERubm5ePLJJ5GVlQUhBC6++GJceumlWLlyJebMmQO73Y7f/OY3aGhoQN++fVFcXIzu3bu7K0vZ2dkoLCwEAPz2t79loCGKUZLQOkBNRNQBVFVV4Ve/+hW2bt2KrKysaJ8OEQXAISciIiIyPA45EZGh3Hbbbdi8ebPP+z/66CNkZ2dH8IyIKBZwyImIiIgMj0NOREREZHgMNERERGR4DDRERERkeAw0REREZHgMNERERGR4DDRERERkeAw0REREZHgMNERERGR4/x8VxCrEEpKW+AAAAABJRU5ErkJggg==", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
train/loss_rec_steplr-AdamWsteptrain/loss_pred_steptrain/ntrain/auroctrain/l1_losstrain/loss_rec_epochtrain/acctrain/loss_pred_epochtrain/l2_loss
epoch
0.0293.1958310.000161100.692220307.00.50.0299.2658080.6156350.688221299.265808
1.0298.4237370.000162200.685151307.00.50.0298.2809450.6156350.688221298.280945
2.0301.1660160.000165300.689863307.00.50.0297.4486390.6156350.688221297.448639
3.0289.8016660.000169400.687507307.00.50.0296.6733700.6156350.688221296.673370
4.0297.5350650.000174500.685151307.00.50.0296.1014100.6156350.688221296.101410
5.0294.0636900.000180600.682794307.00.50.0295.2809140.6156350.688221295.280914
6.0287.5799260.000187700.699288307.00.50.0294.3201900.6156350.688221294.320190
7.0287.7253110.000195800.694576307.00.50.0293.2725220.6156350.688221293.272522
8.0278.4123840.000205900.704001307.00.50.0292.1415100.6156350.688221292.141510
9.0301.0352480.0002151000.680438307.00.50.0290.8933110.6156350.688221290.893311
10.0284.9781800.0002261100.687507307.00.50.0289.7179570.6156350.688221289.717957
11.0293.8103030.0002391200.685151307.00.50.0288.6334840.6156350.688221288.633484
12.0286.5197140.0002531300.694576307.00.50.0287.7211300.6156350.688221287.721130
13.0292.6480410.0002671400.687507307.00.50.0286.5991820.6156350.688221286.599182
14.0292.1063540.0002831500.682794307.00.50.0285.4570010.6156350.688221285.457001
15.0277.0412600.0003001600.687507307.00.50.0284.4149170.6156350.688221284.414917
16.0271.3198550.0003171700.682794307.00.50.0282.9579160.6156350.688221282.957916
17.0276.4292910.0003361800.692220307.00.50.0281.4170840.6156350.688221281.417084
18.0274.3139650.0003561900.687507307.00.50.0279.7935180.6156350.688221279.793518
19.0281.5897520.0003772000.692220307.00.50.0278.0731510.6156350.688221278.073151
20.0274.9028930.0003982100.689863307.00.50.0276.2457890.6156350.688221276.245789
21.0284.7583920.0004212200.682794307.00.50.0274.3131100.6156350.688221274.313110
22.0267.2964170.0004452300.685151307.00.50.0272.2608340.6156350.688221272.260834
23.0282.1111450.0004692400.689863307.00.50.0270.0726930.6156350.688221270.072693
24.0262.5397340.0004952500.687507307.00.50.0267.7651370.6156350.688221267.765137
25.0245.3038940.0005212600.687507307.00.50.0265.3090520.6156350.688221265.309052
26.0267.8875430.0005492700.689863307.00.50.0262.6967160.6156350.688221262.696716
27.0263.9476320.0005772800.687507307.00.50.0259.9410710.6156350.688221259.941071
28.0247.6013030.0006062900.689863307.00.50.0256.9997560.6156350.688221256.999756
29.0254.5294490.0006363000.689863307.00.50.0253.8906560.6156350.688221253.890656
30.0241.2816620.0006673100.682794307.00.50.0250.5789950.6156350.688221250.578995
31.0251.6149750.0006983200.694576307.00.50.0247.0963440.6156350.688221247.096344
32.0248.6170650.0007313300.685151307.00.50.0243.5082700.6156350.688221243.508270
33.0237.0637050.0007643400.687507307.00.50.0240.8448640.6156350.688221240.844864
34.0241.0744780.0007983500.694576307.00.50.0237.1151730.6156350.688221237.115173
35.0236.7638090.0008323600.689863307.00.50.0232.8804170.6156350.688221232.880417
36.0228.4263460.0008683700.685151307.00.50.0228.4315950.6156350.688221228.431595
37.0222.1100920.0009043800.689863307.00.50.0223.7438660.6156350.688221223.743866
38.0209.4218600.0009403900.685151307.00.50.0218.8326260.6156350.688221218.832626
39.0204.5552670.0009784000.696932307.00.50.0213.6874240.6156350.688221213.687424
40.0207.1764830.0010164100.682794307.00.50.0208.3183590.6156350.688221208.318359
41.0196.7011570.0010544200.701645307.00.50.0202.7299650.6156350.688221202.729965
42.0189.7840580.0010944300.673369307.00.50.0196.9298100.6156350.688221196.929810
43.0182.7792360.0010944310.685003307.00.50.0196.9298100.6156350.688221196.929810
\n", - "
" - ], - "text/plain": [ - " train/loss_rec_step lr-AdamW step train/loss_pred_step train/n \\\n", - "epoch \n", - "0.0 293.195831 0.000161 10 0.692220 307.0 \n", - "1.0 298.423737 0.000162 20 0.685151 307.0 \n", - "2.0 301.166016 0.000165 30 0.689863 307.0 \n", - "3.0 289.801666 0.000169 40 0.687507 307.0 \n", - "4.0 297.535065 0.000174 50 0.685151 307.0 \n", - "5.0 294.063690 0.000180 60 0.682794 307.0 \n", - "6.0 287.579926 0.000187 70 0.699288 307.0 \n", - "7.0 287.725311 0.000195 80 0.694576 307.0 \n", - "8.0 278.412384 0.000205 90 0.704001 307.0 \n", - "9.0 301.035248 0.000215 100 0.680438 307.0 \n", - "10.0 284.978180 0.000226 110 0.687507 307.0 \n", - "11.0 293.810303 0.000239 120 0.685151 307.0 \n", - "12.0 286.519714 0.000253 130 0.694576 307.0 \n", - "13.0 292.648041 0.000267 140 0.687507 307.0 \n", - "14.0 292.106354 0.000283 150 0.682794 307.0 \n", - "15.0 277.041260 0.000300 160 0.687507 307.0 \n", - "16.0 271.319855 0.000317 170 0.682794 307.0 \n", - "17.0 276.429291 0.000336 180 0.692220 307.0 \n", - "18.0 274.313965 0.000356 190 0.687507 307.0 \n", - "19.0 281.589752 0.000377 200 0.692220 307.0 \n", - "20.0 274.902893 0.000398 210 0.689863 307.0 \n", - "21.0 284.758392 0.000421 220 0.682794 307.0 \n", - "22.0 267.296417 0.000445 230 0.685151 307.0 \n", - "23.0 282.111145 0.000469 240 0.689863 307.0 \n", - "24.0 262.539734 0.000495 250 0.687507 307.0 \n", - "25.0 245.303894 0.000521 260 0.687507 307.0 \n", - "26.0 267.887543 0.000549 270 0.689863 307.0 \n", - "27.0 263.947632 0.000577 280 0.687507 307.0 \n", - "28.0 247.601303 0.000606 290 0.689863 307.0 \n", - "29.0 254.529449 0.000636 300 0.689863 307.0 \n", - "30.0 241.281662 0.000667 310 0.682794 307.0 \n", - "31.0 251.614975 0.000698 320 0.694576 307.0 \n", - "32.0 248.617065 0.000731 330 0.685151 307.0 \n", - "33.0 237.063705 0.000764 340 0.687507 307.0 \n", - "34.0 241.074478 0.000798 350 0.694576 307.0 \n", - "35.0 236.763809 0.000832 360 0.689863 307.0 \n", - "36.0 228.426346 0.000868 370 0.685151 307.0 \n", - "37.0 222.110092 0.000904 380 0.689863 307.0 \n", - "38.0 209.421860 0.000940 390 0.685151 307.0 \n", - "39.0 204.555267 0.000978 400 0.696932 307.0 \n", - "40.0 207.176483 0.001016 410 0.682794 307.0 \n", - "41.0 196.701157 0.001054 420 0.701645 307.0 \n", - "42.0 189.784058 0.001094 430 0.673369 307.0 \n", - "43.0 182.779236 0.001094 431 0.685003 307.0 \n", - "\n", - " train/auroc train/l1_loss train/loss_rec_epoch train/acc \\\n", - "epoch \n", - "0.0 0.5 0.0 299.265808 0.615635 \n", - "1.0 0.5 0.0 298.280945 0.615635 \n", - "2.0 0.5 0.0 297.448639 0.615635 \n", - "3.0 0.5 0.0 296.673370 0.615635 \n", - "4.0 0.5 0.0 296.101410 0.615635 \n", - "5.0 0.5 0.0 295.280914 0.615635 \n", - "6.0 0.5 0.0 294.320190 0.615635 \n", - "7.0 0.5 0.0 293.272522 0.615635 \n", - "8.0 0.5 0.0 292.141510 0.615635 \n", - "9.0 0.5 0.0 290.893311 0.615635 \n", - "10.0 0.5 0.0 289.717957 0.615635 \n", - "11.0 0.5 0.0 288.633484 0.615635 \n", - "12.0 0.5 0.0 287.721130 0.615635 \n", - "13.0 0.5 0.0 286.599182 0.615635 \n", - "14.0 0.5 0.0 285.457001 0.615635 \n", - "15.0 0.5 0.0 284.414917 0.615635 \n", - "16.0 0.5 0.0 282.957916 0.615635 \n", - "17.0 0.5 0.0 281.417084 0.615635 \n", - "18.0 0.5 0.0 279.793518 0.615635 \n", - "19.0 0.5 0.0 278.073151 0.615635 \n", - "20.0 0.5 0.0 276.245789 0.615635 \n", - "21.0 0.5 0.0 274.313110 0.615635 \n", - "22.0 0.5 0.0 272.260834 0.615635 \n", - "23.0 0.5 0.0 270.072693 0.615635 \n", - "24.0 0.5 0.0 267.765137 0.615635 \n", - "25.0 0.5 0.0 265.309052 0.615635 \n", - "26.0 0.5 0.0 262.696716 0.615635 \n", - "27.0 0.5 0.0 259.941071 0.615635 \n", - "28.0 0.5 0.0 256.999756 0.615635 \n", - "29.0 0.5 0.0 253.890656 0.615635 \n", - "30.0 0.5 0.0 250.578995 0.615635 \n", - "31.0 0.5 0.0 247.096344 0.615635 \n", - "32.0 0.5 0.0 243.508270 0.615635 \n", - "33.0 0.5 0.0 240.844864 0.615635 \n", - "34.0 0.5 0.0 237.115173 0.615635 \n", - "35.0 0.5 0.0 232.880417 0.615635 \n", - "36.0 0.5 0.0 228.431595 0.615635 \n", - "37.0 0.5 0.0 223.743866 0.615635 \n", - "38.0 0.5 0.0 218.832626 0.615635 \n", - "39.0 0.5 0.0 213.687424 0.615635 \n", - "40.0 0.5 0.0 208.318359 0.615635 \n", - "41.0 0.5 0.0 202.729965 0.615635 \n", - "42.0 0.5 0.0 196.929810 0.615635 \n", - "43.0 0.5 0.0 196.929810 0.615635 \n", - "\n", - " train/loss_pred_epoch train/l2_loss \n", - "epoch \n", - "0.0 0.688221 299.265808 \n", - "1.0 0.688221 298.280945 \n", - "2.0 0.688221 297.448639 \n", - "3.0 0.688221 296.673370 \n", - "4.0 0.688221 296.101410 \n", - "5.0 0.688221 295.280914 \n", - "6.0 0.688221 294.320190 \n", - "7.0 0.688221 293.272522 \n", - "8.0 0.688221 292.141510 \n", - "9.0 0.688221 290.893311 \n", - "10.0 0.688221 289.717957 \n", - "11.0 0.688221 288.633484 \n", - "12.0 0.688221 287.721130 \n", - "13.0 0.688221 286.599182 \n", - "14.0 0.688221 285.457001 \n", - "15.0 0.688221 284.414917 \n", - "16.0 0.688221 282.957916 \n", - "17.0 0.688221 281.417084 \n", - "18.0 0.688221 279.793518 \n", - "19.0 0.688221 278.073151 \n", - "20.0 0.688221 276.245789 \n", - "21.0 0.688221 274.313110 \n", - "22.0 0.688221 272.260834 \n", - "23.0 0.688221 270.072693 \n", - "24.0 0.688221 267.765137 \n", - "25.0 0.688221 265.309052 \n", - "26.0 0.688221 262.696716 \n", - "27.0 0.688221 259.941071 \n", - "28.0 0.688221 256.999756 \n", - "29.0 0.688221 253.890656 \n", - "30.0 0.688221 250.578995 \n", - "31.0 0.688221 247.096344 \n", - "32.0 0.688221 243.508270 \n", - "33.0 0.688221 240.844864 \n", - "34.0 0.688221 237.115173 \n", - "35.0 0.688221 232.880417 \n", - "36.0 0.688221 228.431595 \n", - "37.0 0.688221 223.743866 \n", - "38.0 0.688221 218.832626 \n", - "39.0 0.688221 213.687424 \n", - "40.0 0.688221 208.318359 \n", - "41.0 0.688221 202.729965 \n", - "42.0 0.688221 196.929810 \n", - "43.0 0.688221 196.929810 " - ] - }, - "metadata": {}, - "output_type": "display_data" } ], "source": [ @@ -1488,7 +874,7 @@ }, { "cell_type": "code", - "execution_count": 98, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -1520,7 +906,7 @@ }, { "cell_type": "code", - "execution_count": 100, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -1572,7 +958,7 @@ }, { "cell_type": "code", - "execution_count": 101, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -1660,7 +1046,7 @@ }, { "cell_type": "code", - "execution_count": 103, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1673,7 +1059,7 @@ }, { "cell_type": "code", - "execution_count": 104, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1682,7 +1068,7 @@ }, { "cell_type": "code", - "execution_count": 105, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1694,7 +1080,7 @@ }, { "cell_type": "code", - "execution_count": 106, + "execution_count": null, "metadata": {}, "outputs": [ { diff --git a/research_log.md b/research_log.md index d3ef5ba..72ed8ca 100644 --- a/research_log.md +++ b/research_log.md @@ -737,3 +737,4 @@ https://github.com/ai-safety-foundation/sparse_autoencoder TODO: - try SAE +Perhaps consider diff --git a/src/vae/sae.py b/src/vae/sae.py new file mode 100644 index 0000000..b1c4de9 --- /dev/null +++ b/src/vae/sae.py @@ -0,0 +1,76 @@ +# from https://github.com/callummcdougall/sae-exercises-mats/blob/116ecf3f8f7ffcd66cf628518009f81989e57bac/solutions.py#L692 +import torch as t +from torch import nn, Tensor +from torch.nn import functional as F +from dataclasses import dataclass +import einops + +from jaxtyping import Float +from typing import Optional, Union, Callable + +@dataclass +class AutoEncoderConfig: + # We optimize n_instances models in a single training loop to let us sweep over + # sparsity or importance curves efficiently. You should treat `n_instances` as + # kinda like a batch dimension, but one which is built into our training setup. + n_instances: int + # this is the hidden states, and the latent size + n_input_ae: int + n_hidden_ae: int + l1_coeff: float = 0.5 + tied_weights: bool = False + + +class AutoEncoder(nn.Module): + W_enc: Float[Tensor, "n_instances n_input_ae n_hidden_ae"] + W_dec: Float[Tensor, "n_instances n_hidden_ae n_input_ae"] + b_enc: Float[Tensor, "n_instances n_hidden_ae"] + b_dec: Float[Tensor, "n_instances n_input_ae"] + + def __init__(self, cfg: AutoEncoderConfig, importance_matrix: Float[Tensor, "n_instances n_input_ae"] = None): + super().__init__() + self.cfg = cfg + self.importance_matrix = importance_matrix + + self.W_enc = nn.Parameter(nn.init.xavier_normal_(t.empty((cfg.n_instances, cfg.n_input_ae, cfg.n_hidden_ae)))) + if not(cfg.tied_weights): + self.W_dec = nn.Parameter(nn.init.xavier_normal_(t.empty((cfg.n_instances, cfg.n_hidden_ae, cfg.n_input_ae)))) + self.b_enc = nn.Parameter(t.zeros(cfg.n_instances, cfg.n_hidden_ae)) + self.b_dec = nn.Parameter(t.zeros(cfg.n_instances, cfg.n_input_ae)) + + def forward(self, h: Float[Tensor, "batch_size n_instances n_hidden"]): + + # Compute activations + h_cent = h - self.b_dec + acts = einops.einsum( + h_cent, self.W_enc, + "batch_size n_instances n_input_ae, n_instances n_input_ae n_hidden_ae -> batch_size n_instances n_hidden_ae" + ) + acts = F.relu(acts + self.b_enc) + + # Compute reconstructed input + h_reconstructed = einops.einsum( + acts, (self.W_enc.transpose(-1, -2) if self.cfg.tied_weights else self.W_dec), + "batch_size n_instances n_hidden_ae, n_instances n_hidden_ae n_input_ae -> batch_size n_instances n_input_ae" + ) + self.b_dec + + # Compute loss, return values + h_err = h_reconstructed - h + if self.importance_matrix is not None: + importance_matrix = self.importance_matrix[None, : ].to(h_err.device) + h_err = h_err * importance_matrix + l2_loss = h_err.pow(2).mean(-1) # shape [batch_size n_instances] + l1_loss = acts.abs().sum(-1) # shape [batch_size n_instances] + loss = (self.cfg.l1_coeff * l1_loss + l2_loss).mean(0).sum() # scalar + + return l1_loss, l2_loss, loss, acts, h_reconstructed + + @t.no_grad() + def normalize_decoder(self) -> None: + ''' + Normalizes the decoder weights to have unit norm. If using tied weights, we we assume W_enc is used for both. + ''' + if self.cfg.tied_weights: + self.W_enc.data = self.W_enc.data / self.W_enc.data.norm(dim=1, keepdim=True) + else: + self.W_dec.data = self.W_dec.data / self.W_dec.data.norm(dim=2, keepdim=True)