diff --git a/smartmeters-ANP-RNN-mcdropout.ipynb b/smartmeters-ANP-RNN-mcdropout.ipynb index f843a34..42f8d33 100644 --- a/smartmeters-ANP-RNN-mcdropout.ipynb +++ b/smartmeters-ANP-RNN-mcdropout.ipynb @@ -211,11 +211,11 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 47, "metadata": { "ExecuteTime": { - "end_time": "2020-03-14T01:46:01.901463Z", - "start_time": "2020-03-14T01:46:01.848538Z" + "end_time": "2020-03-14T02:12:56.588211Z", + "start_time": "2020-03-14T02:12:56.517782Z" } }, "outputs": [ @@ -223,7 +223,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "now run `tensorboard --logdir /media/wassname/Storage5/projects2/3ST/attentive-neural-processes/optuna_result/anp-rnn-mcdropout\n" + "now run `tensorboard --logdir /media/wassname/Storage5/projects2/3ST/attentive-neural-processes/lightning_logs\n" ] } ], @@ -231,10 +231,10 @@ "PERCENT_TEST_EXAMPLES = 0.5\n", "# EPOCHS = 5\n", "DIR = Path(os.getcwd())\n", - "MODEL_DIR = DIR/ 'optuna_result'/ 'anp-rnn-mcdropout'\n", + "MODEL_DIR = DIR/ 'lightning_logs'\n", "name = 'anp-rnn-mcdropout' # study name\n", "MODEL_DIR.mkdir(parents=True, exist_ok=True)\n", - "print(f\"now run `tensorboard --logdir {MODEL_DIR}\")" + "print(f\"now run `tensorboard --logdir {MODEL_DIR}`\")" ] }, { @@ -246,11 +246,11 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 48, "metadata": { "ExecuteTime": { - "end_time": "2020-03-14T01:46:02.003319Z", - "start_time": "2020-03-14T01:46:01.903776Z" + "end_time": "2020-03-14T02:12:57.054405Z", + "start_time": "2020-03-14T02:12:56.976397Z" } }, "outputs": [], @@ -351,6 +351,115 @@ " return model.logger.metrics[-1]['val_loss']\n" ] }, + { + "cell_type": "code", + "execution_count": 49, + "metadata": { + "ExecuteTime": { + "end_time": "2020-03-14T02:12:57.151181Z", + "start_time": "2020-03-14T02:12:57.098057Z" + } + }, + "outputs": [], + "source": [ + "from typing import Callable\n", + "\n", + "def convert_layers(model:nn.Module, original:nn.Module, value: bool):\n", + " \"\"\"\n", + " Turn dropout on\n", + " \"\"\"\n", + " for child_name, child in model.named_children():\n", + " if isinstance(child, original):\n", + "# print(child, 'from', child.training, 'to', value)\n", + " child.train(value)\n", + " \n", + " else:\n", + " convert_layers(child, original, value)" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": { + "ExecuteTime": { + "end_time": "2020-03-14T02:12:57.360254Z", + "start_time": "2020-03-14T02:12:57.294090Z" + } + }, + "outputs": [], + "source": [ + "from src.plot import plot_rows\n", + "\n", + "\n", + "def eval_mc(model, loader, i):\n", + " device = next(model.parameters()).device\n", + "\n", + " data = loader.collate_fn([loader.dataset[i]], sample=False)\n", + " data = [d.to(device) for d in data]\n", + " context_x, context_y, target_x_extra, target_y_extra = data\n", + " target_x = target_x_extra\n", + " target_y = target_y_extra\n", + "\n", + " # do MCDropout estimation\n", + " model.eval()\n", + " convert_layers(model, torch.nn.modules.dropout.Dropout2d, True)\n", + " model.model._lstm.train()\n", + "\n", + " y_preds = []\n", + " y_stds = []\n", + " with torch.no_grad():\n", + " for i in range(n_mcdropout):\n", + " y_pred, losses, extra = model(context_x, context_y, target_x, target_y)\n", + " y_std = extra['dist'].scale\n", + " y_preds.append(y_pred)\n", + " y_stds.append(y_std)\n", + " y_stds = torch.stack(y_stds)\n", + " y_preds = torch.stack(y_preds)\n", + "\n", + "\n", + " mean = y_preds.mean(0)\n", + " sigma = y_stds.mean(0)\n", + "\n", + " dist = torch.distributions.Normal(mean, sigma)\n", + " log_p = dist.log_prob(target_y).mean(-1)\n", + " loss_mc = - log_p\n", + "\n", + "\n", + " # Try without MCDropout\n", + " model.eval()\n", + " with torch.no_grad():\n", + "# y_pred, kl, loss_test, loss_mse, y_std \n", + " y_pred, losses, extra = model(context_x, context_y, target_x, target_y)\n", + " y_std = extra['dist'].scale\n", + " \n", + "\n", + " dist = torch.distributions.Normal(y_pred, y_std)\n", + " log_p = dist.log_prob(target_y).mean(-1)\n", + " loss = - log_p \n", + "\n", + " return loss_mc.cpu(), loss.cpu()" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "metadata": { + "ExecuteTime": { + "end_time": "2020-03-14T02:12:57.509900Z", + "start_time": "2020-03-14T02:12:57.456668Z" + } + }, + "outputs": [], + "source": [ + "\n", + "def init_random_seed(seed):\n", + " # https://pytorch.org/docs/stable/notes/randomness.html\n", + " np.random.seed(seed)\n", + " torch.random.manual_seed(seed)\n", + " torch.backends.cudnn.deterministic = True\n", + " torch.backends.cudnn.benchmark = False \n" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -360,11 +469,11 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 52, "metadata": { "ExecuteTime": { - "end_time": "2020-03-14T01:46:02.059875Z", - "start_time": "2020-03-14T01:46:02.006336Z" + "end_time": "2020-03-14T02:12:58.355195Z", + "start_time": "2020-03-14T02:12:58.275135Z" } }, "outputs": [], @@ -450,11 +559,25 @@ "execution_count": null, "metadata": { "ExecuteTime": { - "start_time": "2020-03-14T01:47:05.000Z" + "start_time": "2020-03-14T02:12:59.800Z" }, "scrolled": true }, "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "e3c4c62f46d74d48b4731cb140d1352e", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='seeds', max=4.0, style=ProgressStyle(description_width='i…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, { "name": "stdout", "output_type": "stream", @@ -547,30 +670,18 @@ "metadata": {}, "output_type": "display_data" }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOydd3iUxdbAf5NOEghpQEgghIQOUgygEoqiiOXCvVgAQ1VAQcCuCBYsCNgBQUSKCBEQPyl6KQKXFgQlSFGklxBaIAkhpGez8/3x7saUbUl2s7vJ+3uefdidmXfes++SOTNzzpwjpJSoqKioqNRcXOwtgIqKioqKfVEVgYqKikoNR1UEKioqKjUcVRGoqKio1HBURaCioqJSw1EVgYqKikoNR1UENRghhBRCRNlbDhUVFfuiKgIVh0II8bgQ4lchRLYQYoeZtncIIbYIIdKEENeFEKuFECHF6oUQYqYQIlX3mimEEMXqFwghTgghtEKIEaX6HqSruymEuCaEWCqEqGNClrZCiM1CiBQhhCxV5ymEWCSESBRC3BJCHBJCPGCiL4vbCyHe0in0e030FyCEWCOEyNL1+USp+mAhxHe673pDCBFnoq8PhRBJQogMXV+TS9V3EEIc0P1+B4QQHUz0Ze73sbgvlcqhKgIVRyMN+ByYYUFbf2AB0AQIB24BS4rVjwH+DbQHbgP+BTxdrP4wMA74w0Dfe4BuUko/oCngBrxvQpYC4HvgKQN1bkAS0BPwA94AvhdCNDHSl0XthRCRwGPAFRNyAcwF8oH6QCzwpRCiTbH6H4GrQGOgHvCxib4WAS2llHWAu4BYIcQAnTwewDpgOcpvsxRYpys3hNHfpwJ9qVQGKaX6qqEvQAJRuvd+wLfAdSARZfBx0dVFATuBm0AKsEpXLoDPgGtABvAn0NZKso0CdpTzmk7ArWKffwXGFPv8FLDPwHXxwAgT/frqns0GC2SIUv6szLY7AjxSju9Wpj2wCXgQOA/ca+Q6HxQl0LxY2TJghu59H931rhX4jUJ1v/mrxfq6BIhibS4AfY1cb/T3KW9f6qtyL3VFoKJnDooyaIoyEx0GjNTVvQf8gjIzC9O1BeWPtQfQXHft40AqgBBikhAi3djLRt+hB3C02Oc2KLN+PYd1ZRYhhIgRQtxEWWk8grJSqTRCiPooz+youbbG2gshHgPypJQbzFzeHNBIKU8WKyv+HO4ATgBLddsz+4UQPYvdZ5IQ4udS8kwSQmQCF1EUzXe6qjbAEakbtXUc0d9LCPGEEOJIsTpTv4/JvlSsi6oIVBBCuAKDgNellLeklOeBT4ChuiYFKFsvDaWUuVLK+GLltYGWKDO3Y1LKKwBSyhlSyrrGXjb4DrcBbwGvFCv2RVnF6LkJ+BbfhzaFlDJeKltDYcBHKDPnysrpDsQBS6WUxyvSXghRG/gAeM6CW/qirNaKcxPldwPlu/UBtgMNUH73dUKIICj6HR8ufrGUcobu+k4oqwv9My79vEvcS0r5nZTytlKyGft9TPalYl1URaACEAS4o2wJ6UlEWfoDvIqyDfS7EOKoEOJJACnl/4AvUPagr+mMr0YNqrZCKJ5PG4HnpJS7i1VlAsXlqQNklpplmkVKeQllG2al7n6xQohM3WtjOeR0QRk484Hxxco3Fusv1lx7YCqwTKewzVH6GaD7fEv3Pgc4L6VcJKUskFKuRLFPdDPVqVQ4qLv+HQvvZU624r9PeftSqQSqIlABZd9fP+vX0xhljxYp5VUp5WgpZUMUY9483eCLlHK2lPJ2oDXKNsQrAEKIycUGtzIvawkuhAgHtgLvSSmXlao+imKI1NMeC7djDOAGRAJIKeOklL66l1Hvn1JyChRDa32Uvf4CfZ2U8oFi/cWZaw/0BiYKIa4KIa4CjVCMya8ZuPVJwE0I0axYWfHncATFVlSc8ijKouei6/O2Uiuu2zD+zE39PuXtS6Uy2NtIob7s96KksXg5sAZl6R0OHAdG6eoeA8J079ugzAKbAp2BriirCR+UWfM7lZTJFfACngF26d67G2kbCpwBXjZS/wxwTNeuIcog8kyxeg9d/3uA0br3egN5LNBY9z4cxVj+owm5he761rrn6gV4FqufD+wDfC18DkbbA4Eo2zj6V5LuNzLYN8pKZoXuN+qGssXSRlcXANwAhuue/aMonltBBvpxQZkI+Ou+bxcUj6WJxZ5nIsqWlSfKKiYR8Cjv71PevtRX5V52F0B92fHHL6kI/HXK4LpuYHmr2KD4IcrqIFM38I7RlfdGmVFmoqwq4iwd6EzINEInV/HXN0bavq2rzyz+KlYvdLKn6V4fUtILZYeBe/XS1U1DMYZm6f5dAASakLuJgb7O6+rCdZ9zS8kaa6Sv8rY/jxGvIV19ALBW910uAE+Uqu+O4v2TCSQA3YvVTQY26t67oCj7NF3bk7r64s+0I3AAZbLwB9CxWF0scLQcv4/RvtSXdV9C98BVVFRUVGooqo1ARUVFpYajKgIVFRWVGo6qCFRUVFRqOKoiUFFRUanhqIpARUVFpYbjZm8ByktQUJBs0qRJha69dQsKCsy3U1FRUXE0PDzA17fi1x84cCBFShlsqM7pFEGTJk1ISEio0LU7dsDVq9aVR0VFRaUqCAuDmJiKXy+ESDRWp24NqaioqNRwVEWgoqKiUsNRFYGKiopKDUdVBCoqKio1HFURqKioqNRwbKoIhBB9hRAnhBCnhRCTDNSHCyG2CSGOCCF2CCHCbCmPioqKikpZbKYIdOkP5wIPoMRoHyyEaF2q2cfAt1JJX/cuMN1W8qioqKioGMaWK4IuwGkp5VkpZT5Kcoz+pdq0Bv6ne7/dQL2KioqKQ5GYeJjTp39Dq9XaWxSrYUtFEIqS4ETPRf7JgavnMDBA9/4/QG0hRKANZVJRUVGpFOvWTefNN+9g3ryhZGWl21scq2BvY/HLQE8hxEGgJ0oWrMLSjYQQY4QQCUKIhOvXr1e1jCoqKipF1K+vpGjes+c7Tp3aZ2dprIMtQ0xcQkmqrSdMV1aElPIyuhWBEMIXJUl3GRUrpVyAkiqQ6OhoNaWaioqK3Rg4cBr5+TmcOLEHV1eni9JjEFt+i/1AMyFEBIoCGAQ8UbyBECIISJNSaoHXgcU2lEdFRaUaoNVqcXGx72bG0KGf2vX+1sZmT1NKqQHGA5uBY8D3UsqjQoh3hRD9dM16ASeEECeB+igJw1VUVFSMsnXrl0ya1IH9+9dW+b1TUi6Qm5tZ5fe1NTZVq1LKDVLK5lLKSCnlNF3ZW1LK9br3P0gpm+najJJS5tlSHhUVFecnIWEdiYmHmT9/BBcv/m22/cGDG1i69Hny83Mrfe/p0/swcmRtkpKOAlQbzyF7G4tVVFRUysUrr/wEQHb2Ta5ePWW2/X//+wmbNs1i584llb63u3st3N29yM/PYdy4UCZP7lTpPh2B6mHpUFFRqTG4u3vy0kvrSEu7SKNG7cy2v//+8SQnn6Zlyx6VvveMGQeRUnLrVio3blymoKDyqwxHQFUEKioqToOUEiEE0dH9zDfW0bnzf+jc+T9Wk0EIQe3agcyZk0jduiFW69eeqFtDKioqTsH16+eZODGCVaummG2r1RayYsUkTp7ci5TW9zgXQhAU1Bg3N3er920PVEWgoqLiFBw8uIGUlESuXDnFjRtXOHx4M+fO/WGw7bFju1i/fiZz58YCkJubxdatX/HLL/MqfP///W8hr7zSll9+mVvhPhwVdWtIRUXFKbj33meIiOiEm5snR45sZv78kXTvPoxx45aWaRsU1JiHHnqJgIBQhBBoNHksWvQM3t5+9OkzrkL3v3z5OBcvHiUn5xYAu3cvIyFhHXff/RQdOjxQqe9mb1RFoKKi4hS4uLjQrNkdAOTkZNCu3X2EhbUx2LZ+/UiGDPm46LO3tx933z0KX9+AIjtDeRkw4E1iYoZQu3YQABcu/Mnvv/8fERGdnF4RCFvsn9mS6OhomZCQUKFrd+yAq1etK4+KikrN5Ny5P7h69RQREbfToEGUze8XFgYxMRW/XghxQEoZbahOXRGoqKg4PMnJZ1m37gPCwtrw4IMvmGyblZXOyZO/Uq9eU0JDW9pMpoiITkREVI9zBKqxWEVFxeG5fv0827cv4sCB9SXKCwrKBiNITDzEhx8+xNdfjypRnpGRwqVLx8nNzSr3/TWafBYsGMWaNdUzCo6qCFRUVByekJDmjBq1gL59JwKQnZ3BsGFePP10cJm27u5etGt3H82a3VmifNasx3j55VacPv1bue+flnaJ7dsXsW3bV0Vl2dk3iY+PY8+eFeXuz9FQt4ZUVKo5iYlH+PrrUYwcOZfIyM72FqdCBAaG0bv36KLPXl6+aDT5SCnRagtxcXEtqmvW7A4mT/6lTB/BwU1o0KBZhe7v5eXL8OGz8fT0KSq7dSuVuXOHEBQUTrdugyvUr6OgKgIVlWpOfPxyzpzZz5w5g/n889P2FscquLi48M03Wbi7e1nsAfTMMxWPNVSnTjB9+04oUebnV4877xxIQEBYhft1FFRFYCfi4+NYtWoKKSkXCApqzMCB04iJibW3WCrVkB49hnHjxmU6dnzI3qJUmMTEw6SkXKBx43YEBzcBwMOjlsG2+fk55VIQFcXLy5eJE1fa9B5VhWojsAPx8XF8/fUYUlISAUlKSiJffz2G+Pg4e4umUg1p1Kgt48cvd+rtix07FvPxx/0sykHw7rs9GTnS1+ip44qQnHyGI0d+ITn5rNX6dCRURWAHVq2aQn5+domy/Pxsi2KoqKiUl6tXT5OenuzUsfMbNmxFx44PERLSvKjsxx/fZ/r0vpw+/XuJttnZN8nLy8bXN7BE+b59q3nppZasWPF6ue+/b99qpk+/n61b55coz8vLJiXlAvn5OeXu05FQFYEdSEm5YLA8NdVwuYpKZXj//XsYO7YB+/f/yMmTv9pbnApx333P8OqrP9Ox44NFZYmJBzlyZHOZv5tPPz3BokU3CQxsVKK8oCCPy5dPVOjvLDCwEW3b9iY0tFWJ8hkz+jJhQniFPJEcCdVGYAeCghrrtoVKEhjY2A7SqFR36tQJJjU1ic8/f4xWrXry1ls77C2SVfjXv17l7rtH0aRJ2UNd3t51ypR17PgQH3/8N3Xq1Cv3vWJiYg3a8Pz9Q/H3b4hGU1DuPh0JmyoCIURfYBbgCiyUUs4oVd8YWArU1bWZJKXcYEuZHIGBA6fx9ddjSmwPeXh4M3Bg9TysomJfPvjgABkZKUyd2o3Q0Nb2FqdC5OVl4+FRq4QBOCqqa7n68PX1x9fX36pyTZzo/GcIwIZbQ0IIV2Au8ADQGhgshCj9v/ANlKT2HYFBQMVjxDoRMTGxjB69gDp1/jkMM3r0AoMzjvj4OCZMaMLgwS5MmNBENSirVIg6dYL49NMTPPWUc/6Jvfxya2Jj3bh+/bzJdn/+uZVp0+5l48ZZVr2/Vlto1f4cDVvaCLoAp6WUZ6WU+cBKoH+pNhLQr+H8gMs2lMehuO22+3n66cVFn+fOHVpmoFe9i1RUFLRaDVJqqVXrny2fK1dOsmXLlxw+vLmo7NKlv/nrr21cuXKyTB+5uVl8991rLF/+crnv/+67vRg2rJbT2ljMYUtFEAokFft8UVdWnKnAECHERWADMAEDCCHGCCEShBAJ169ft4WsVc6OHYv56KN/FSspO9Cr3kUqleXAgZ94/vmoIk+ZnJxbTJ3aneeei7RJ5i5bMXfuRZYty8fH55+tndOnf2fx4nHs2vVPPoKuXR9j0qRN9O49pkwfLi4u/PTTh/zyyxcm73X69O8sW/ZSCVfR3NxbFBTk4uHhXaLtoUObmDz5duLiXq3oV3MI7G0sHgx8I6X8RAhxJ7BMCNFWSlnCz01KuQBYAEoYajvIaXXc3DwMlusH+piYWNW7SKXSpKYmkZx8hqysNEA5BHXx4lGysm5w48ZlAgJKz80cl9JpIRs0iKJnzxF06PCPJ5G/fwj+/obzCLu7ezF48Axq1fJDq9Xi4mJ4Hrxx4+f8+usK3Nw8GDx4OgDTpx8kLy8bd3fPEm0LCnI5d+4P/P0bVuar2R1bKoJLQHH/rTBdWXGeAvoCSCn3CiG8gCDgmg3lcggefPB5li17EWV3rCT6gV71LlKpLD16DKdt2964u3sBSq7d11/fTGBgI/z86ttZusrRrNkdRYlqLEEIQb9+r5lt99BDL+Hq6s4994xGoynAzc0dIQReXj5l2rZq1YP33vvN6RWBLbeG9gPNhBARQggPFGPw+lJtLgC9AYQQrQAvoHrs/VhAUJDhAV0/0D/++PsoNvd/UL2LVMqDl5cPDRu2IDg4vKgsMrIzdes2sHkIBmtx5cop3nvvHr791nQegiVLxrN48bNGV9KW0rTp7QC88EIUBw6sM9nW1zeAqKguBAY6d7whmykCKaUGGA9sBo6heAcdFUK8K4Top2v2EjBaCHEYWAGMkM60cVlJBg6cViZeSvGBPjS0JVIq3gpCCIKCwo16F6moVFfS06/w99/bOXu2bGbCvLxszpzZz6VLx9mxYzFbtpj2irpw4U8OHtxARobp+aanp2ILyMy8QW5uJjNmPMD8+U9W/Es4ODa1EejOBGwoVfZWsfd/A91sKYOjMn/+k+zZE0fDhq24ceMSt26llgk+l5+fS1BQOO7unnz66Qk7S6zijKxZM42cnAz69p1YZA/IyEhh7dpp5OdnM2rUV2Z6sD+NGrVj8uQtBoPMrVgxic2b5zBo0HTeemsXZ88mGF1pA3z33ascPryJV175mU6dSgbh++uvbeza9S19+owjNvZjRoyYg6urGykpFzh8eJPBKKNSStasmUZmZipDhnxi1O7g6NjbWFxjSUu7hEaTj4eHFwsWGJ6dtGwZw7PPLmfp0uf48MOHefXVn6tYShVnZ+fOJSQnn6FXr39ms+7unmzc+Dlubh6MGPFFGSOso+Hr60+7dvcarIuI6ERYWBtq1apNZGQ0kZEGU/IWERnZGSkl3t5+Zeq2bVvAvn3f06BBsxKH1Xx9A3nlFcN/e0II/vvfj8nOvsmAAW/i6xtQjm/mOKiKwE489dQ8Llz4q0w8lNJcu3aW8+f/4PJlwyF3VVRM8cgjU0lNTSoxm61VqzbDh8+mXr0IO0pmHXr2HEHPniMsbv/YY+8arRs06AMaNGhGz57DS5R7efmUWT0U59//noIQLri4OO9wKpxtSz46OlomJJTdK7SEHTvg6lXrymMNVqyYzM8/f4RWqyEoKLzIRrB8+UvcvHkNd3cvWrXqweuvb7KzpCoqVc+xY7tITDxEixYxBpPFnzq1j40bZ9G9+9ASQekqQ0rKBb799nlq1arD2LHfWKXPyhIWBjExFb9eCHFASmlwyeScG1rViPj4ODZs+BStVgNASkoi8+eP5KuvnuTmzWRAUlCQw/Hju9UTxSo1koSEdSxd+hx//bXNYP3hw5vZu3clhw5ZFqZMSolGk2+yjRAu7N+/hiNHNnPx4t9s2TK/2p4qBnVryG4sXvwsZ88mkJx8Bo0mr0RdYWHZSIbFD5qpqBneLCE9PZnTp3+jXr0IGjduV6IuM/MGhw5toLBQU2YrxNFo2TIGjSafpk0N7/97eNTC09Obnj1Hmu1r795VzJ07lDvvHMizzy4jOfksQUGNiY+PIy8vi+jofxMQ0BA/v/o8//xq6tYN4e+/d7BkybP07v00zZvfVaZP5dDeWerVizBpqHZkVEVgJ/744+dynxBWTxQr6GMw6cNv6ENzAKoyKMbp07/xySf96dDhQV577b8l6m7cuMzcuUMIDm7i8Iqgc+f/0Lnzf4zWP/jgC/TqNbJEEEdjeHr6UFhYQG7uLY4f38077/SgZ8+RnD9/kMTEQzRp0oGAgIa4ubnTteujgJLH4J57RtOqVQ+Dff788yds2jSLIUM+4aGHXqzYl7QzqiKwEz16DOP06d9JTDxo1qdZT3VIkm0NTMVgUhXBP/j41KVjx4eIiip7+jYkpDldujxCo0ZtkVI6zeEyQ7i5uVukBEAJ9rh0aQ4eHl6cPLkXUDyrRoyYw7lzBwgLa1vmmrZt76Ft23uM9tmwYUtatOhmsQyOiGostjOlZ7emGDlyHn36jK0CqRybwYNdMBSaQwjBd985bzpGFcOcPLmXevUi8POrb1WFJaVky5YvadfuPkJCmpWp/+uvbZw/f5COHR8mNLSl1e5bUWxpLFZXBHZGP4NdsmQ82dnpBtu4uLgyduxSdbarw1gMJh8f5/ThVjFOZmYab799F15evixenGHVvoUQ9Okzzmj9rl1L2b17GRpNAT4+I/H1DXT4MxcVRVUEdmLHjiUUFOTSpcsjxMTEEhHRiXfe6UFmZhrFg696eHirYSVKMXDgNObPH1nGqJ6be4vFi8dx8OAG1YiMEn/f09Pb6Cxaqy0kOfkM+fk5hIe3r2LpLCMrK52mTaPx9PSxympASsmcOYO5dSuF2267n1u3UoiM7ExQUDiNGrUtcXq5ffu++PoG8vPPH7Fq1WReeeUnOnV62GTfzrrFpm4N2Ylhw7woKMhj4sRV3Hnn40Xlem+Y1NQLBAb+M5BptYpycNYj7NZm9OggMjNTDdQIim8b1WRF+t57d3PiRDxTpmwzaOg8dGgjM2c+WK3yGFvCqFEBZGXdICgovMTK8qmn5nPvvU+XaT979iCOHt3OSy+tMeg1dPHi30yd2o2goHBmzDhkM7nVraFqiL9/KBkZ10rEgzfmEqkf9IYPn03fvgZz99Q4MjPTjNSUnNjUZCNyXl4WhYUao2EPmjTphKurO6mpF/joo38RGdmFAQPerGIpq56xY5fi7u5JevpV0tIucuHCnxw9uo3o6NIJFBUmTlxpsj8vL1+ystLx9CwbptpZUBWBnZg160yJz/HxcSxYMJqCghygpEuknuzsm1Umn6OiPKdRGDIWG6Omut2+//7v5OfnGt3Xrlu3PlOn7ubkyb0sW/aCQ+bltcV2y+23/6vE58reIyAglK++uo6vr7/5xg6KqggchFWrphQpAT362ewnnxzHw8MbLy9vI1fXDPQeVgUFuUZalNwW0lOTE/l4eHiZrI+K6kpQUBPq1YsgONjxYg9Nn34/V66c4LnnVhMV1cUm96isonFxcaVOnSArSWMfzCoCIYQL0B5oCOQAf0kpq30GsarGVFpKZ/9PZi0MnR/QExQUTseOD7Jjx2IKCv45qa0m8jFP3br1jW6L2Jvk5DOkpFzAx6eu1fo8eXIvf/21FSkl0dH9HdZQXpUYVQRCiEjgNeBe4BRK5jAvoLkQIhv4ClhaOr+winm0Wi2xsa6AYNmyfNzc3NS0lBZgKvNUUFA4Tz45j+bNuzFv3jCk1OLtXZeRI7+okfaBixf/ZtGisURFdSU29kN7i1NhPvnkOCkpiQQHN7Fan3v3rmLTplkA/PnnFqZO3V3pPn/44R3On/+DJ574kIYNW1S6v6rGlAvK+8ByIFJKeb+UcoiU8lEp5W1AP8APGFoVQlY3srP1/tASNzdFFyvZykpu/ehns0uWTGDs2Ia6HMc1F1NxXI4f3wUo5zIWLrxBXFwhixbdqJFKABSlefy4ErXTEnbvXs769TPJz88x37gKcXNzp0GDKFxdrbeL3apVDyIjO+Pu7mnQC6giHDu2kwMH1lc6Taa9MPp0pZSDTdRdAz4317kQoi8wC3AFFkopZ5Sq/wy4W/fRG6gnpbTeGtBB8fauw/z5ySWMv/oBy5Dr6LZtC0hPv8LZswfsJbJDMHDgNBYsGFXCRuDh4U1AQCiRkZ3RaPJxc/PA27uOHaV0DKKiujJlyjaDWb0MsWrVZFJTk7jzzoFWnX07Il26DKBLlwFW7fPf/57M/fePd9ptJovUrBDiLqBJ8fZSym/NXOMKzAXuAy4C+4UQ63XpKfV9vFCs/QSgY3mEd1ZcXFzw86uHn1+9EuUxMbEGZ7APPvgiUVFdad26VxVJ6JjExMTy559b2LVrKUBR7obiz+yTT/5DQsI62rS5hzfe2GovUe2Or6+/yfg4penZcwT5+Tm4u5s2LlclJ0/uZdOm2bRs2d3kCWBHwFgGNWfBEmPxMiASOATo/cskYFIRAF2A01LKs7p+VgL9gb+NtB8MvG2BzDWOzp3707mzYxrzqpoePUaQnHyGgIAwJk5cUaZeCeAnOXp0G9Om3ceUKVuqXkgnxFTmLntx9eop9u5diaurm1UVQWGhhuvXzyOEoH79SKv168xYsiKIBlrL8h9BDgWSin2+CHQ11FAIEQ5EAP8r5z2ckhMn9jB79mCCghrzzjvx9hbHqWjTphdt2pQ07q1bN5P162eQnX2TwMAwmjW7k1On9nLx4lE7SWl/9u1bTXr6FTp1+pfTpqRs2bI748fHmU3nWl4uXDjC5Mm3A/Dqq/+1Slaz5OSzHD++i6CgcNq0udv8BQ6GJYrgL6ABcMWGcgwCfpBSGjzRIoQYA4wBaNzY+b1orlw5SVpakpEQCWU5fXo/W7d+SXBwEx555C0bS+dcxMfH8f33U4oOQ6WmJpGRcZ0uXQZYlKikurJ9+yKOHNlMvXqRFimC3Nwsbty4jKend4nT7vakXr0Imyix2rX/cce2llvq8eO7mT9/JN26xVYvRSCE+AllC6g28LcQ4negyEFbStnPTN+XgOKqPExXZohBwLPGOpJSLgAWgBJryMx9HZ7mze+iT5/x+PoGWtT+yJHN7Ny5BF/fwBqvCI4d20V6+lWioroSHBzOqlVTypyILSjI5ezZA7zwwv/ZSUr706XLIzRo0IyQkOYWtf/lly9YsWISDz/8MrGxH9lYOvsSFNSYFSsk+fm5VvNGCg1tRffuQ2nRohLBgOyIqafwcSX73g80E0JEoCiAQcATpRsJIVoC/sDeSt7PaWjYsAUjR86xuH2TJh2oXz+S0NA2NpTK8TAUe2nduhlcvPgXXbo8wgsv/GDUXc9Z3fisRe/eo8vVPiAgjHr1muLlVdtGEpWfU6f2ce3aOaKiuthkL9/cqevyEBXVhagoc2ZTx8WU++jOynQspdQIIcYDm1HcRxdLKY8KId4FEqSU63VNBwErK2CDqDF06vSwyfC31RFj6SiDg5vg7u5ZNDAYO4jn6urG++/fy6RJGxql8LIAACAASURBVHBz86hS2Z0RYx5r9mT79kVs376QUaO+Uo26NsYSr6EBwEygHkowFwFIKaVZZ20p5QZgQ6myt0p9nloOeasFR4/+j337fiAysjO9etXcfWxTGEtHeeXKiRJJegYOnFYmw5uLiyuFhQUcPbqN69cTDWafqs5otYUkJh6hTp0gqxtaq5LIyM7k5t6yeHvL3mi1hSQl/YWXl6/TKS5Lgtt/CPSTUvpJKetIKWtbogRUjLNr1zK2bv2StWs/sKi9VqslOfkMZ89WLA+DM2Jsa0erLeTrr8cQHx8HoAvTvaAocJivbwBjxy6lWbM7iYq6AxeXmhdXMTMzjcmTOzFpUgd7i1Ipevcew8SJK53m/MzatR8waVIHfvllnr1FKTeW/JUkSymP2VySGkSjRm0JCgq3OJpiamoSzz8fBcCKFTVjB83Ylg+UzTEQExPLd9+9yo0bl+nT51mH3OaoSgoKcmnc+Da8vf0sviY/P5cpU6LJzc1kzpzzthOuGhMVdQf16kVQq5bzzZONZijTbQkB9ERxH11LSa+hH20unQGqS4ay8rB9+xIWLHgSQGc0/aDaD3SlbQRlEaxY8U+8w4yMFDSafH74YSr16zelf/9JVSNoNUFKyfDh3hQU5LJkSSZeXvZPspKZmYa7uxceHrWcIgWkrVNV2itDWfHsDdlAn2KfJWAXRVDTiI+P45tvxhd9Tkm5UJSwpjorA/13+/LL4QYTpgQGhpX4XKdOEPv3r2P79q9xc/PA3b0Wq1e/SW7uLQB8fQMZPnxWtX5mlUEIwVtv7SQkpLlDKAGAt9++i8uXT/Dxx38TGtrK3uKYxRmUlTFMKYItwGYppWWnnlTMUtwd0t+/IY8//j69eo0weY0xo2lNSL94111K3MPSKwN3d08GDZpepr23tx/BwRFIKVm27PkSdZmZqXz1lbKqqu7PraLYKvFLRXFz88TDoxZeXr72FqXaY8pY3AhYLYTYLYSYKoToKpxZ5dkZ/VaHsu8tuXHjEl99NZLFi03HUDGVsKa6M3ny7Xz55QhatepR5AJau3YQY8YsKjOYnz2bwKxZj+nCKBve7tRo8lm1aoqtxbY7v/wylwkTwvn558oeBbIvM2ceZunSbKfyfFq69DmGDfNi27YF9halXJg6RzATmCmEqI2SnOZJYL4Q4hiwCWW1kFw1Yjo/xrJrbdkyn+bNuxmdpdbkhDWpqUlotRqaNo1m0qSNJtv6+ARy61aK7pPx+UpNUKBpaZdISblQ7twCqakXWbv2A1xcXBg58gsbSVf9KSjIIy/PmG3LMTHrPiqlvCWlXCOlfFpK2RElYU0w5qOPqhTD+ElXaXKWaiphTXVnzpwLvPbaBu69dywAe/as4Pnno5gzp8wBdU6e3FN0KtbFxfh/65qgQP/97ynMmnWWe+99plzXubi4snXrl8THL0c931kxBg+ewdKl2TzwwHP2FqVcWJqPIBQIL9Z+v5TyE5tJVc2Ij4/DxcXFoNETTM9S9SuFb799gVu3UggICGXw4BnVfp/bUHiJ48fjSU4+UyInsb7twoVPF624jD1nNzePGqFAvbx88PIqf7C2unUbMHz4bMLCWtvcA8YcmZlpzJz5IAEBoU4VM8rSRECOhiUni2cCA1HyCBTPR7DLhnJVG/S2AWODE5ifpdY0v3hD4SXmzx9JYWEBADk5N4mPjyuR1c2wm6lAby9wc/Pk6afL2hZU/kEIQd++E+wtBgDZ2Tc5ffo3goLC7S1KjcCSFcG/gRZSyjyzLVXKYHyQUqgp2zzlwdAz0ysBgJycWyVcaI1tuwkB331X87Y4Vq6cTF5eFv37v07dug0q1IdWqzW5xWZr6tYN4d139yKl1nxjB+Kvv7axdetXtG7dy+GzqhXHkl/6LOBua0GqK6aiYAYFhTN69AKLZqnx8XG8804Pli9/xZriOSSWRA7Vu9CC8aT2NcEeYIjdu5exadPsMltolnLkyC9MntyJ9HT7+YJ4eHjRrNkdVksuX1WkpFzgt99Wc+bM7/YWpVyYykcwB2VdnQ0cEkJso+TJ4om2F8/5Meb1ExQUXq6j/EePbuf48d2kpV1iyJDqHy/eWHiJ4uhtK4YCz5VeaeXn51o17LAj88QTM0lPv4qfX/1yXyul5Mcf3yUx8TBbt37Jo49Otb6A1ZjWrXsxceJK6tVzrqBzpraG9HEcDgDrTbRTMYElg5QldOjwAMnJp2nZsru1RXQ4DD0zQ+hn/MVtBampFwgMbFyU1F6j0TB8uBdabSFLltyy++GkjRtnsXz5S7Rv35eJE1faRJ5u3cp6VVmKEILnn/8/du36hocftt/qMynpKAcOrKNx4/Z06vSQ3eQoL7bKqmZrTCmCm8CvUsprVSVMdUQ/SC1dOpHMzDQCAxszaFD5YwV17foIXbs+YgsRHY7SA7uPTwC5ubfQaPKL2pRWpsYM6m5u//wXT0w8TIsW3WwouXmOHduJVlvIwYP/5erV0zRp4ngRQuvWrU+/fq/ZVYZz5w6watUUuncf6lSKwFkxpQiGAHOFENnAr8AeFMXwV5VIVo2oaV4/1iAnJ4Po6P7cccfjtGjRrcidtPSM3xI++OAP6tdvavfVAMDQoZ9x+fIJ6tdviqent/kLysmtW6l8+eVw7r13rFUGUHsZjcPC2tCv3ySHVJSmuHUrlUOHNuDlVZvOnf9tb3EsxtTJ4kcBhBBNgLt0r6eFEI1RzhE8WBUCqihoNPns27eajIzrPPjg8+YvcHJ++ulDrl8/T0FBHi1adKuUMg0Pv83K0lWc4OBwPv74qM3637z5Cw4e/C+ZmWmVVgRffz2G33//P95/fz/16ze1koSW0bTp7TRtenuV3tMapKYmMW/eMMLD21cPRaBHSnleCOEF1NK99O/NIoToC8xCSVW5UEo5w0Cbx4GpKIbpw1LKim9wOijXrytpFgMCQnnmmcUV6kOjyWfu3CEA9OkzvsSWR3UkIuJ2Cgs1NGnSsdJ9GTqcVt1WaDk5t5g7dwgeHrW4557RdO36aKX7zMxMIzMzjZMn91S5InBW6tQJplu3WKezE5jyGpoM3IkSTuIEsA/4AhgjpTR+Ouqf612BucB9wEVgvxBivZTy72JtmgGvA92klDeEEPUq82UclaSkP/nzz19wc/OosCLw8vLFy6s2rq5uZGenU6dOkJWldCxeeOEHq/QTHx9X4jCaPvcxVH0U0j/++C+nTu0lOro/9etHkpV1w2opDdPTr3LgwHrq1Ytg1qyzVunzscfe5YknZtol7eL164nk5t4iMLBRuRLs2JuAgFDGj19ubzHKjalp5TAgC/gJxUbwm5TyZjn67gKcllKeBRBCrAT6o5xQ1jMamCulvAFQXQ3TAQFhREZ2wdc3oFL9LFmSYSWJag6rVk0pcRgN7BfGOyFhDdu3L+LmzWS2b19IaGgrPv74b/MXWoCfX31efPFHq8YICgtrbbW+ysuaNe8XJa7v3XuM3eSoKZiyEbQUQgSg2AZ6AZOEEL7AYRSj8RIzfYcCScU+XwS6lmrTHEAIsQdl+2iqlHJTub6BE9CkSQfef/83q/ZZ3bc7kpPP4OPjj7d33UoZKx0pjHeHDg8SFBRO+/YPsHfvSqumNPT2rkPnzv+xWn/2xs+vPqGhrSt0FsKeSCnJyckgPz+nwqe67YHRVJUlGgnhBtwO9ACeBiKklK5mrnkU6CulHKX7PBToKqUcX6zNz0AB8DgQhhK/qJ2UMr1UX2OAMQCNGze+PTHR/GEjQzhrqsrS7N69vESQNVDcKS09pewMxMa6odUWMnVqfKVcPidMaGKVA33Wxt5B3SxlxYrXuXr1FGPGLMTHp669xXF4tFotsbHK0BgXV2hVjytbpqo0KqUQop8QYoYQYjdwDfgYCAReQslhbI5LKMlt9ITpyopzEVgvpSyQUp4DTgLNSnckpVwgpYyWUkYHBwdbcGvHIjn5DPv3ryMpqXLeIq+80o7BgwWLF48zmrWsuqAfJCs7q3LUMN7WVgInT+5l586lXL58wqr9JiSs4fff/4+0tNJ/uiqGcHFxoU6devj51UejcZ7wbKZsBCNQzg68ChyQUuabaGuI/UAzIUQEigIYBJT2CFoLDAaWCCGCULaKrGPpciDWrJnGzp1LqFevKbNmnalET8rqTZ+HtzTOmnTF0DbX8uUF5i+0AP0KadGisUXGx0GDpttl5fTXX9uoVasOERG3W903f8+e7/jlly8YNuxzGjZsYbV+H3/8fbTaQvz9G1qtz+rOV185X74uo/8bpZQDdDkH6pZWAkIIsxkvpJQaYDywGTgGfC+lPCqEeFcI0U/XbDOQKoT4G9gOvFIdcyTXquWHm5tHpf+YwsM7Yir7VlUHWYuPj2PChCYMHuzChAlNiI+Pq1AfxVN46r16KtKXMWJiYsnLywLgxRfX2kUJFBZqmDbtXt58UzGT/fLLXF54oRmbNs22Sv9RUV2IiRlC48bWPTPRteuj3HnnQHx9/a3arzmmTo3hueciSU6udvNCh8QSZ/Q3hRB5Usr/AQghXgXuBuabu1BKuQHYUKrsrWLvJfCi7lVtGT78M4YP/6xSfcTHx7FnTxzG8vFW9XaHoZwBptwyjRm3DYWctoVXT0REJwoK8nB397Ran+WhoCCPNm3uQatV9o3z8rK5evU016+ft0r/3bsPpXv3oVbpyxG4fPkEt26lOG2iF2fDrLFYt2XzM/AK0BdoCQyuwFaRVYiOjpYJCQnmGxrAmY3Fxoyeep59dnmVznTLY4QtrTTgH+P23LlDMabcgoMjmD27es4I09OTycq6QUBAKLVq1ba3OEZJSbnAiRN78PcPoXXrXlV238zMG9y8eZWQkBZ2zYtQERYsGEVi4mHGjfuW0NBWVuvXLsZiPVLKFKAfyuGwhsCj9lICVYk1tj2siakY/f7+Dat8u6M8bpmmZv3GcgkA3LhxuXJCOjB169YnNLSl1ZTAtWvnyM7OsHqu4ePHd/PFF0+wdavZDQCr4uvrT2hoK6dTAgAXL/7N2bMJZGXdsLcoFmPKa+iWECJDCJEBnEYx5D4G6MuqLdbet37jja7ExrqzdOkLFZbJ1IBZmbDDFaU8yWBMKQ3DXj216NlzBCNHflF5QXUooRL2cuXKKav1WR5smQxeqy3k+ecjGTWqrsmUqBWhYcOWdO36KM2b2zdqqzPx1FNf8t57v9GoUTt7i2IxpozFtaWUdYq9vKSUvvryqhSyqjE1g60IGRnX0Wo1FBTkVFgmQwMmQP36UdxzT9WfvCyPW6YppRETE1siwqSSte1rnnlmCffcM8pq8s6Y8QBvv30XS5aMN9/YBiQkrGP4cG/mzRsOQF5eNitWTOKbbyqf3yk7O4OgoHACAxvh6mrdGFRNm97O88+vrtJcxqdP/8b8+SPZuXNpld3TmoSHtycqqotDb/mVxlSsoSZSyvMm6gUQKqW8aAvB7Im1T6O+/vpmrl8/Xym3PlPJV+xBTEws2dkZfPvtRAoLNQQFhRuVx3BynlpFSkM/i23b9l6mTNliE3kDAkI5d84VNzf7ZF3VnzbV5+B1dXVn/fqZCOHCsGGf4eJi8nymSXx9/Zk9+5y1RLU7iYlH2LnzG4RwoWfP4fYWp0ZgavrwkRDCBViHkqXsOkrk0SgUr6HewNsoh8KqFcZSJVbUPTMkpBkhIWXOyZUbR8trEBgYSmGhBldXd5OndPUyL148jpwcZVfxzjsHFZU///wPnD17gMBA5fzhnj0r2LVrKW3a3EO/fq9aRdYXX/zRKv1UlO7dh9Kly4Aipefm5k5s7Md4e/vpPIkURSClZM2a92nRIoY2be62p8hFFBTkkZ5+lYCAUKuvOAzRunVPRo/+2i7B7qzBoUObOHt2P+3bP0BkpEHbrMNhKtbQY0KI1kAs8CQQgpK/+BiKS+g0KWVulUhZxVgrvaStOXBgPWfP/kH79vfTvPmdVX7/evUiad/+Aa5cOcGGDZ+bzJOgV2JTp3bn3LkD7Nz5DUeP/q9oFfHnn1v57LMBSKnFw6MW+fk5pKVdtJoisDdCiDKJcR5++KUy7U6ciGf1asXDesUK03aFP/74me3bFxEa2ork5DOMGbPQJtsRL7/cimvXzvHZZ6do0CDK6v2XJiSkOSEhzW1+H1tx4MB6tm79El/fAOdXBAC6kNHVJ26BhehnqitXvk5q6kX8/RvyxBMzKzwbf//93mRl3eDppxdbNePS8uUvc/XqKVJSzttFETRq1IYHH3yB6dP7EBf3itmEOfHxcZw79wf5+YqtRG+EP3lyDzt3Li3aNsnPz0EIF4KDnSumuzUICWlB3bohhIW1Mdv2+PHdJCSsJSFhLaBEuR069BOryxQQ0AiNJp+cHMMn2lVK0r59X3x9A4iIcJ7EOtU7u0kliIjohJdXbUDSrNkdZZRAeaJ/Hj++i8JCDTdvWvfoeePG7cjKSicoqIlV+y0PoaGt8PLyxcfHdIjt9es/ZM2a9w0a4bds+bJMeym1JCX9aTU5T5/+jffeuxs3N08WLap6t75Nm2Zz8uRe+vR5lpYtFWfwq1dPc+nSMUJDWxXNtP386vHll5a5zfbu/TSNG9+Gq6sbu3cv4667BttE9rfe2lGlAfISEtbh4uJK69a9HCK9aHmJju5HdHQ/8w0dCFURGGHy5NuLZq76maqe8p6q7dVrFDduXLZollceXnjh/6zaX3nZtGkOGk0+H3101KBn0D/KsmLRYq0ZO8nT05f8/Bzy8+2zm3ns2C5+//3/SoSK3rx5Dps2zWbIkE946KHyH66vX79pUeawO+8caDVZS1PVUVKXLn2OlJREPv/8tFMqAmdEVQQGkFJSWKgBYNassyXSzsXHx/Hll8PL+GubCoswalTZGW914Pvv3yAnJwMhRJmBzNBpYuMIDJ0urlvXeoHOQkKaMXbsUurVs0/KxYcffpnOnf9Ds2Z3FJU1adKRDh0eICAgrKjshx+msnfvKgYMeItu3Wwzw3d0OnR4gOvXz1O3boi9RakQmZk3SE29gLe3H8HBTewtjkWYVQRCiB+BRcBGWXpqXE0RQrB8eT5paZcJCPhnMNIPbsYO7dgj+qdWq0WjycfDw6vK712/fiTJyWeIjOyCRqMBtLi5eQCGz2IYwsPDm549h7Nz59Iy7a25gnJz86BHj2FW66+8NGt2RwklANCz5wh69hyBVlvIvn2rad36bpKTz3D58nG++OIJhBDcddcgg/2dO3eQw4c30qbNPWX6tTb7969l7dppdOz4EI8+OtWm9wLlQJYz89tvq1m48GnuvvspxoxZaG9xLMKS89vzUMJHn9LlJ7BejFsHpHhoibffvqvEaWJzg5sh99L8/FxWr36LTZvmWF3WhQvHEhvryquvVt0Jxn+ej+DChSPk5GTw3nu9GDrUnfXrPyxqZyokhh7l8NgCnnxyHqNHLyizvdSu3b1Wl98R2bdvNbNmPc7KlZMYPnwW7drdB5h+hkeObGbVqin8+utKm8uXl5fF2bMJVs91UF3x86tPo0btnCp0t9kVgZRyK7BVCOGHkjtgqxAiCfgaWC6ltE7geAfA0N7/3LlDWLBgNN9+m23yD9OYe2lqahI//vgeIKx+OtPTU4nMWFUJMEo/H/3KSP/vsWO7itoaO4vxT70SnO7gwQ388MM7tGt3L3PmJPLVV6OoXTuQvn2fK7Eas4bsCxc+TV5eFnXq1GPo0E+r9EzGrl3f4u7uSefO/ylaNYGyDan3zT969H/4+gbwzDPfUFiYb3IgiYi4nQcffJEOHR6wuey33daH9977jYCAUJvfKy8vm4KCPHx86jpFBjdDREf3Jzq6v73FKBeWpqoMBIYAQ4HLQBwQg5JWspctBSyNLaOPmorw+d13WiZOjDBY7+LiytixSw0OLJcvn2Dq1O64uXkwb551z97l5+ei1Wrx8iobesIWmIuAWjzyqCkbgYeHN0OHfsq1a2fZuHEWGk0ebm6ePP30IgCr52I2Ff20qpTBsGFeFBTksXRpdlFo5bS0S0ycGIGXlw/Tpx/E3z/UbiefHYU9e77jiy9iueuuwUyY8J29xXEobBl91BIbwRqgBbAM+JeU8oquapUQomIjsoNifMYvkFIaPWjWs+dwVq2awty5Q8sMXg0btmDBgms2kbeq7QLmtnuK20iKh8RISUnExcUVrbaQoKBw+vWbxKJFJXMbaTR5zJ8/EiEEGk2+7n6mvbEspapyHhhDqy0kJmYoubm3cHf/5zfz9Q3Ex8efBg2iCAoKB2DhwmeoXTuQxx57DxcXF7RaLXv3ruT22/vj5eVjc1ntTX5+DrVq1aFOnXr2FqVGYYnX0Gwp5XZDFca0i7NibDsjKKgxLi4uBuL9NKJFi+5s3764aHvGWoOXI2Juu6e0jcRUSIzFi8eWKSssLLvLaI0B29qxo8qLi4srY8Z8Xabcw8OL227rQ926IQghyM6+ybZtXwGQm5tFQUEOhYUaduxYzCOPTOXRR98GlMHy5Mm9hIQ0KwrLYUs0mgJ++ulDcnMzGTx4uk3vdffdT3H33U+h1TqvX8qFC3/y4YcPERLSwmaxs6yNJcZifyHEgFKv3kKIaqeyLYmoGRMTy5w55/nuOy3PPLOEPXviyuzRF49UqtFobPafOjHxCGPHNuT5521/7B+MR0DVk5KSWCJ3Q2ZmGp999milwm9D5QdsY9FPhXCxe56JZ59dRmzsh2ze/AWzZg2kd+8xPPHEh2zaNIvt2xfRvftQAgJCS4RcuHTpGNOm9WbGDNvbB0BRZKtXv8X69TPQaKrGJOiMeQj0CCFITU1yqnwalqwIngLuRMkpDNALJQhdhBDiXSnlMmMXCiH6ArMAV2ChlHJGqfoRwEcoye0BvpBS2s3fylCEz0aN2rFhw2dIKenefQgAQ4d6UVhYwFtv7TTal37wWrPmXX788T38/Rsyb94lo+0rQmGhhvT0KyixAW1PTEwshYUafvjh7RLbPcUpviIKCgrn99//DxcX1xKpOpXB1/DZAUNUNhezoS09ULZsqmL1lp+fS3b2TXx86hpNlXniRDxHjmxm3Lhv6d59KB4etUhPv0Lz5nfx+ednSlyn1RbSokVMlcXjcXFx4ZFH3sbT01v3e9dsO4Y5QkKaM3v2OTw9necwnCUjiDvQSkr5iJTyEaA1yl9wV+A1YxcJIVxRspo9oLtmsC6IXWlWSSk76F52d7q9fv08Pj7+PPHER8yZc56zZxM4d+4A8+YNLZrtajR5SKnF3z/UbIKWM2f2A0q2LWtnOmvQIIqHH36FJ56YabU+zbF8+Yukp19l/PjviIvTFO1tF0e/IvLy8iUsrC1Nm5bcQVRWS4aVQHGPGrBOsL+YmFhGj15gMNRzZfJMWMrx47sZO7YBM2c+aLTNvfc+w8SJK4vSQXbs+BBr137Aa6+1L/NMIiM7M3Xq7iLjelXwyCNv8fDDL9vcLvX554/z3nvKeQpnxc3Ng+DgJtSpE2RvUSzGkhVBmJSyeJCca0AjKWWaEMLUOrELcFpKeRZACLES6A/8XWFpq4CEhLUkJh6idu0g6tZtQGZmalGdfrY7bNgsmjXrSmBgIzp2fIgtW+ZTfGDTD17x8XElXCqtbT/w9q5DbOyH5htaid27l5OZmQbA8uUvIaXW6P57SkoiFy8e5aOPysYLMmV0fvrpxTbJuRATE6vLj1wWW9sKtFoNtWsHUadOsNE2rVv34sqVUyQl/UVhoYbr188TEBBK48a3AYr32YULR+ja9VGndau0hNOn95GamoSLixr0oCqx5GnvEEL8DKzWfX5EV+YDpJu4LhRIKvb5IsoqojSPCCF6ACeBF6SUSaUbCCHGAGMAGjeu3DaBKeLj47h2TUmWfv78Qc6fP1jGgJmfn82GDZ8yZ8554uPjdFmUis9uBT17DicmJpYJE5rY1VvFmuj98PWkp1/h66/H4OsbUEJZFseY0jNulA+3ac4Fa+eZsJQOHR5gwYLrZtv99ttqVq2aQr9+rzF48Ay++CKpKOLn2293IzMzlS++uICfX/0yqwRbc/16IikpiYSENKdu3QY2u89rr23k5s2r+Ps7Z3gJPT/++D4XL/7FyJFzqV070N7imMWSraFngSVAB93rW+BZKWWWlLKymTN+AppIKW8DtgAGc9NJKRdIKaOllNHBwcZnVZVB72uun/FmZqYaHeD0M0jDJ40lBw9uAKrGW2Xp0uf4/PPHSE83cUCikujjKxlSaoBRA3J+fjbLl79c9Ez1lCfNpTWp6vuuWvUGf/9t3I5UnPz8XE6d2gdAWFhbQDE6envXQQhB+/Z96dTpYfLysnnppZY880wDUlPLzJlsxurVb/Luuz05fHiTTe/TqFEb2rbtXeWKztr8/vsP7N27imvXnCNznMkVgW6ff6tuwC9vqMtLQHHftjD+MQoDIKUsPtIuBKpun6MUlsbGAcXb5L337jGxLaKUV8UM9Jdf5qHVarjvvrE2mamZi6+UlZXGuHHLmDt3iMH6mzev8sEHffjgg3+OnNgr7aa+/2XLXiQj4xp164YQG/uRTe57+vTvrF07jY0bP2fu3Iv4+NQ1c4Xkjz9+wtXVzWCwufHjlwMUhTPPz8/Fz6++1eU2Rmhoa5o3vwtvb3PfQwWgf//JaDR5BAeXtaE5IuYS0xQKIbRCCD8p5c1y9r0faCaEiEBRAINQYhYVIYQIKXZArR9K9jO7YElsHFAMQRpNPmfO/GZioFeiSQ4cOK3MAGntGWhUVBeyszPM5gOoKJbEV4qJiTUZbtrQgGWvtJsxMbGsWDEJgK5dH7WZDFptIYMGTadevQgLlICSw7lHj+H4+PiTn59r9PCYq6sbixffIj39apXOmvv3n0T//pNseo8rV06xe/e3hIe3p2vXR216L1tz552P21uEcmGJjSAT+FMIsQXI0hdKKSeaukhKqRFCjAc2o7iPLpZSHhVCvAskSCnXAxOFEP0A+NhVbgAAIABJREFUDZAGjKjY16gc8fFxulOcZWe9tWrV0e3TSoKCwrnvvme5du0MPj7+NGrU1uBJ40GDlEM3MTGxnDz5K7t3LyM3N9NqIROK8847e6zWlyEsja9k7NR1VYZxsJTQ0NZFMYdsRfPmd5Y7a9zYsd+YrNdqtdy4cYnAwEZWjcPkKCQl/cmaNe8THd3f6RWBs2E21pAQYrihcimlwf18W2PtWEPmYuKYG8j0yVeqcoujKjEWX8hQfKXq/izsSWGhhiFDFP/9JUsy7RZuQkppM6+lpKSj/PbbD4SENKNbtyfMX+DAZGamcezYTlxd3enU6WGr9GnXWENSyqVCiFpAYylltYtDu3TpcwaVgIuLq0Wz2eJbHGvXzmTx4meZO3cIQUHhVTIQpqVdJi0tiYAA28wSyzPTj4mJZe/elbi5eTBp0kbq14+0ujzOQlLSX2RlpRMa2soqXiOurm40a3YHp07t4+bNZLy8qjbBzvHj8Xz8cT+aNo1m8uRfbHKPRo3a0KiRdbP42YsrV07y6acDiIjoZDVFYEvMeg0JIf4FHAI26T53EEKst7VgtiY+Po7Ro4OMegZJqS0x0Gm1WrRaLQcOrGf58pf444+fy/S3evUUcnIUU0pKSiLz54/k1Vfbs3v3cpt9hwkTwnnzzTt48cUWNgmXEBMTy4gRXxAY2AghRFEOAWMK7s8/t3L16ilOnNjD2LEhDB/uXeRF5WjYMm3l99+/yTvvdOfYMcu8hizhuedW8+67e4vSU1YlHh61yMq6QUaGeTdYFahXrykdOz5Eu3Z97C2KRVhiI5iKcjhsB4CU8pAQwj75/qyEJWkUi3v2DB/uQ35+Nm+8sY2NG2dz9Og2Tp3aV0LTr1o1pYyNobCwgKSkI8THLy8KT2Ht76DVKik18/IybRYu4cSJ3aSmJtGwYUs++cS0Pb9796Hk5WURHt6BjIwUtFqNwRO99mTLli9ZvHgc3t51bZbIvmHDFjRvfpdVPXsCA8OKHBGqmsaNb+Orr67h4+Nvs3ucP38IKSUhIc2dPtKqn189Xn31Z/MNHQRLFEGBlPJmqX1B5w0NiGWuoiU9exQ7SmrqRRo1asfly8fLhE0wZVQ9c2Y/8fFxVh2gqzK0cl5eJkCJEMrGGD16QdH7N9/8H8nJZ2nWrHxGU1tTu7ZyFqWgwHYrgsGDZ5hv5ES4ubmbPBltDZYuncjx47t5443/0aZNZY8oqZQHSxTBUSHEE4CrEKIZMBH41bZi2RZzrqK+voElBtOPPz5G7dqBeHn50qPHsBIB1PSYCtGclZVm9dl6VYZWfu6573n2WY3F5yz0tGzZnZYtu1tdnsrSqdPDzJmTWCJpvIr9adCgGdnZN6skE1pVoNVquXkzGU9Pb7y9/ewtjkksOVk8AWgD5AErgAzgeVsKZWuMBYoDxRA6fPisEmXBweF4eZmOJGguRLO1g5uZC3Znbdzc3PD2rmO2XWrqRdauna5Lz+mYeHh4FeWYULGcZcte5JNP/kNGRopN+n/66UXMnHm4yqKq2pqFC8cwblxD9u5dZW9RzGL2L0FKmS2lnCKl7KwL8zBFSmm7NXUVMHDgNIPbHL6+gWY9hdLSLpObm1mmXB/h0tXVeIhea87W7RWmwRzr1k1n1arJrF79FoMHCwYPFlaPuuoMvPZae0aNCnDqKJqlOXRoIwkJa8nIsE3GvepGUFA4tWsHkZdXvpW0PbAkVWVz4GWgSfH2Usp7bCeWbYmJieXChSP89JMS0cKcq2dc3Kvs2RNHdPS/iY9fTk5OBsOGfc4DDzxXpt99+1Zz4MA6g/1Yc7aul3Xx4nHk5GQghIvNDm+9/vrtpKVdZOTIudxxh/GDPvHxcezYsaRMuSNmbZsypQvXr5/j5ZfXl/vglyVkZqaRlXXD5MTA2Rg8eCZarababN3YmgED3mTAgDftLYZFWGIjWA3MR4kFZDjgjBNy//0TkFLi6urOoEGmZ9FJSX9x48Zlzpz5HSWhCkZPpY4f/x379//I/Pkjizx6wDaz9ZiYWFq0iOHEiXgaNmxRxoBtLa5cOU5eXja5ubdMtlu1agoFBTkG6xwt6uqlS3+Tl5fFhQtHbKIIPv30BPn5ORaFl3AWoqP72azvK1dOMnny7YSHd2Dq1N02u4+KYSxRBBop5Zc2l6SKCQwMsziWf58+4wgNbUnr1r24/fZ+JlNPenl50737EIQQVXLKNjg43OaBrUaOnMeVKydo27a3yXblSW5vbwYMeIu8vGyb+Xl7enrj6WncZqRSkqysdHJzM23qyaViHEsUwU9CiHHAGhSDMQBSyjTjl1QvOnV6uMSZAVNGRn2YhZSUCwQFNWbcuGUOMwuuKD17GowyUobyJre3JwEBoaxaNYUff3zXJjGgqiNJSUc5ezaBsLA2REZad/XZtGk0ixbdrFaKICXlArNnD8Lb249JkzbaWxyTWOI2MRx4BcVl9IDuVbFgPw7E/v3rmD//Saue+tUf8lIGQ1m0N25LQ2lubibvvtuLN9+8w2b3sBRTnlOOYMjWY+vfSaMpYNasgXz11VNW6c9RSEhYy/z5I/j99/JGpDePi4sL3t518POzXSDAqsbNzYNTp/Zy7twBe4tiFktiDUVUhSBVzc6dSzhwYB3nz/9h9tRvaupF1vx/e+ceHkV5/fHPyY0AAQTC/RKuAUURkR8il2orXlDU1htiVMQLCoJXROsNsGKp1VpFRJFi1YaLVVutFSylagmKiIpUvCAoQhAQAgFCCCFwfn/MbBjCbpJNdnZmd9/P8+yT2Zl3Z795Z2fOzHvOe87fHiY//0u++SaPQCbSineR0ZzkFSAlJa08jUFZWRkpKZEt8VdYuIWZM68nMzOLa6+dXmlbZ50BZ3H7aOVdqi5uH6fS0n0sW/YK6ekZUa0r7DYdO/Zm0KCr6NChl9dSYoKGDZsxaVKe6xPxIkHIq4aITFDVR+3lS1X1r45tj6jqvdEQ6BYdO/Zm48b/0a1b1ROeCgo2snjxc0esCxYJE81JXgFSUtLo0eMM6tSpT1lZacQNwcaNq/nss3+SlJRSpSEA7+oMhIPbxyk1tQ7jxs2lqsy+sUavXkPo1WuIK/tevvx1li9/nb59L6Jv34tc+Y5ok5SUTLduA7yWUS0qu2pczuGKYb/mcM1igHOAmDYEF1/8IBdf/GC12rZo0bn87tZJxbtIr2ri3n//v13bd/36jenY8eQqJ9TFEm4fp9TUOvTvf3lE9pUofP/9Jyxdmkvr1t3ixhDEEpX5CCTEcrD3cU2jRs1DRgo57yL9OsmrNnTq1JtHHlnBgw++57WUiBGPxykaqCrFxbvYsWNT1Y3DpF+/YYwZ8xInn+xeiKoXLF06h9zcCWze/K3XUiqlsicCDbEc7H3MsXHjatLS0mnatF2VJf8qq2DmvIv0qhbvqlWLyM9fTa9eQ2jdupur3xUPuH2cdu/ezuefL+SYY1pywgmDI7JPP1BQsJFx47Jo0qQt06dvjOi+s7J6kpXVM6L79AMffDCXTz99i27dBtCqVVev5YSkMkNwoojsxrr7r2svY7+vOg0lICLnAE9ilaqcpapBUzKKyMXAq8D/qWpUIpImTx7E3r07ufzy31Zai7Wy4u3B7iK9GCOfPj2H3bu3UVRUwGWXRS7HT15eLrm5d1FYuIXGjVtzxRW/8/34f3Xp128YWVm9KCzcwgknVD4/Ily2bPmWZ565ii5dTokrQ5CR0YT09Iy4GiZ0mwEDcsjOHkDr1t29llIpIQ2BqtYqibyIJAPTgTOBfOBjEXlTVb+s0K4BcCvwUW2+L1wCU/+ryhcfKmV1dSuYRYPmzTtz4EApGRm1r4QVoGLNhp07N/kuTURt2LVrCxMmHA/A3LmRfcCtX78xAwZcEXcV2tLTM3jhhcpnl9eUTz75BwcOlHD88YPJyHCv5kG0iRVfUWRDTI6kL7BWVb8DEJF5wIXAlxXa/Qb4HdZchajx3HNbq9UuVIRJxQpmXvKb33wY8X16EQobTRo3bo1IEklJyXa0VeXDg+HQpk13xo5NrCR7tWX+/HvZuPELpk5dGVeGIFZw0xC0AZwDifnAKc4GItIbaKeq/xSRqBqC6uJVJJDXeBEKG02SkpKYMyduUmfFPCeeOIRWrbpFtKKbHygu3kV+/pekpdX19fwLzxKyi0gS8Afgzmq0HSUiK0RkxbZt0a2ZGksRJpXlQAqXaNc7iCdKS0soLt5NWdkBr6VEnBdfvI177unFt98ui+h+c3Ie5fbbX+WYY1pGdL9e89VX/2XixP688sr9XkupFDcNwSagneN9W3tdgAbA8cB7IrIe6Ae8KSJHJTFR1Zl2LYQ+zZrVfpZecfFurr66Ltdf36TKtoE6A5mZWdUq3u4Fs2aNZvhwKR/zjgQnnXTuUev8agD9xpIlL3PddY2YPXu011Iizvbt6/nhh8/ZufNHr6XEBE2atKVz576+L7bj5tDQx0BXEemIZQAuB64IbFTVXUBm4L2IvAeMj0bUUEHBRg4cKOHAgf1VN8b/s2WTk63DuG/f0QVzakJeXi7vv/9ihbXCaaeN8HU/hMuvf92b/PzV9O17MWvWfFCeKDASoaTp6RnUqRPbBdiDMWzYI1x00cSIOsIPHTpIUdEO6tVrFFFfjR/o2PEkHn44qnEwNcI1Q6CqZSIyFngHK3x0tqquFpGHgBWq+qZb310VzZp1ZNy4eeVF2WOdSy+dzNCh4/nyy/cYN65DrS9owSOllM8+ezsygn3Cnj3bKSsrZdmyv5bXjohEEZ0zzriBM864IWI6/UTbtsdFfJ/bt2/g1ls7kZmZxbRp6yO+f0PVuPlEgKq+DbxdYV3QvA6qerqbWpykp9ejf/9h0fo618nIaMLKlQuYPXtM+QW8Nhe0eHcUBxg9+iWeeuoydu8+0u8UT9FRscD+/cVkZDSlQYPIhT/7jUDeKRF/JmUw1bvjhMrCPcMlURzFPXqcHrIQe7wZvUixceMXvPrqJJYunROxfbZr14Pnn9/OlCkxn90+KBMm9OSqq9LYuXOz11JCkpCG4MMPX2HSpEHMmxfTefPKWbDgqZAFYWpyQYulSKna4obRe/vtJ5g8+TSWL3+9xvvwKwUF+bz22mQWL54Z8X379W65tqSkpKGqvnawJ5whyMvL5bnnruWbb/L4xz9+72rRmGiQl5fL3LmhU2TU5II2cGAOXbueSiC3oB8jpSLBsmWvkpSUctQFqLZG78cfv+brr//Lrl0/1Vai7+jYsTdDh97F2WePo7h4N/fc04uXX77Da1m+Zvz4N3nxxeKIV3WLJK76CPzGv/99ZNqEQ4fKYj5tQmUF42tzQbPKcSpduvRzZeayH/juu4/56ad11K3bEFD27dtjO9kfqdXvYejQu+jffzgtWnSJnFif0KhR8/Ja3ytXLuSHHz7nhx8+JyfnsUpLuFbGkiV/YdGiZxg06GrOPPOmSMr1BU2atPZaQpUklCGYNSv+0iZUVjC+Nnfx11wzjXXrPqZ587gsUAfAiSeezfr1n9GpUx/eeOO3AEycuCTkcFF1admyCy1bxp8RqEivXucwY8ZmDh06VGMjALB16zq+/fbDuErQF2sklCH46af4i4YJlQIjMzOrVsatdetucZ/SukePX9CqVXcWL55JRkZTGjZsxsGD8TcbONLs3r2ddeuW06BBJl269K3xfvbsKWDSpIGcffZYJk5cQuPG/r9zrglFRTuYPXsMJSVFTJjwltdygpJQPoLmzeMvGiZUwfgLL4wPR7jbLF/+Gq+/PpkDB0p4/PGvIjJRaunSObz11uMUFEQ2Z79f+Oijv/Loo+exaNEz5euKi3dX8ongrFu3nB9//JrCwi107z6QFi06RVKmb0hPb8Dy5a+xcuXblJYGH8b1moQyBNdfH3/RMBVTYKSm1qVz577063dZrfb71FPDeeihn/PNN0sjpNR/LFnyF+bMuRuAsrIDEQscWLToGXJzx7Nt2/qI7M9vdOx4Mmlpdfnvf19k/vz7mTp1CDfc0CRsw1ev3jGcfPIF5Snh45WUlFTGjZvHQw996Nv/NaGGhgYPzqGwEGbMuIZDh8po1KgFV175eMz6BwK4kQLj00/fZP/+YjZvXhMzBbjDIS8vl1mzbix3tB88WMrMmdezZ08BQ4bcUqt99+9/BZ0796Vp03ZVN45BunTpy7BhU3j55TvYtWsLqal1UD3Et98uC+t/zs4+lfHj33BRqX845ZSLvZZQKRKY8RYr9OnTR1esqNnEk/fegy1bIqsnXnn22WvZunUtI0c+Q/v2kUtm5xesVBxH+1ZSU+vy0ktHFyIyHMnBg2Vs2LCK1NR0u3JZA9LT67Np09ckJSXTokXncgfy7t3b+fTTfzBo0FXlebEM4dO2LQwcWPPPi8gnqho0htUclThj1apFfPfdCo4/fjBduvxfjfdz002zI6jKf4SKtgoVims4kuTkFDp27H3U+ldfnciyZa9w442zOf30kQD89rdnsX79ZxQXF3LuubeXt923bw9paXUTwjgUFOTz4YfzSU/PYPDgG72WcxQJ5SMAKCsri2jefr8xe/YY5s+/l0WLpnstxdeEChHNzMyq9b7z879k8+Y1QetcxzsZGU045piWZGQ0Lf//Tz3VKtdYsT/++MdLueqqNFat+lfUdUabwsIt5OaOZ+HCp7yWEpSEMwR///sUcnKSufXW+KonG6Bt2x7Ur9+Exo3b1HgfJSXFvP32H/noo9ciqMxfRCKNRl5eLuPGdWD48CTGjetQ7myeNGkAd9zRrUaRNLHK3Ln3cO+9fRg6dDx16zbk8ccvLHeWX3DBBObOVYYOHX/EZ8rKSgGoX7/quiCxTrt2PRg8+CaGDLnNaylBif9nsgoExoWtAmnxx/jxf6/1PvLzV/Pyy7cjEr/lHAPO9fnz76OgYANNm4aXtjsv78hZ6s5sr82bd2bfvl2kpaW7I96HbNjwP77//hPWr/+Mpk3bU1y8i127tlYajvvAA//h4MGyuM0x5CQtrS7XXTfDaxkhSUhncWHhFvbv3xvR4hrxxJo1HzJ16hDq1KnHjBn+TZQVSdau/YgpU84kNTWdmTOrzhEUytmcqDn116z5kOLiQrp1G0hKShqpqXUAePzxX9G4cWsuuWQyderUIyUlLSF8Am5gnMURJt7qogajtLSkxnek2dmnMnt2YYQV+Zu0tPqUlOypdrGiRKnZUF2ys0/lySeHMW3acMaOncNJJ51LUdFOPvnkTVJT67B58xpWr17M5Mkf0LVrP6/lekJJSRHr139GUlIy2dn9vZZzBPE5PpLALFw4jeHDhTFj4nO6vlu0bp3NyJHTueuu6qUASJSaDeFQULCR4uJdNGnSFoCMjMZMnbqSYcOm0LBhM5KSkssnnf300/dMnDiAP/1pjJeSo8oXXyxm8uSf8dprk72WchSuPhGIyDnAk1ilKmep6tQK228CbgYOAkXAKFX90k1N48f3oLS0mLFj55CdfaqbX+UJGRmW46269ZgNFikpaZx1VvCLUl5eLvPn31deAvSkk86lpOToJ4e0tHqcf/4Ebr65Hc2adWDSpCVuy/YVDz30ATt2/Ejdug2YPPk0Cgt/5A9/WEP79iewZ08Bo0f/ubwm8Y4dm1iz5gNibWi6NmRlnUinTn1o375nlW2t+gWbadSoeVSG0lzzEYhIMrAGOBPIxypmP9x5oReRhqq6216+ABijqudUtt/a+Ajuvz+XKVOuBIibWcUVKS0tobi4kIYNm4edEfLwBc8a+27ZsitPPLHGDZm+o+LFPuA4rugUDkVGRlNGjHiSY4/9GQ880I969Y7hscdWR0m9v1BVRo3KpKhoB9Onbwqahrm4eDcbNnzuy2ESP1BcvJvrrmtEWlpd7rzzDXr2PDNmfQR9gbWq+p0tYh5wIVBuCAJGwKY+4NrtQW5uLo89Nqr8/a5dW2O+FkEw0tLSSUsL3wcS7IK3des68vJy46p/ghEsAmj69CuZNetGSkv3oVr1vJP09Izyfpo4cQkbN37hqmY/IyLcffcCGjduRePGrYK2qVevId27D4qysthh796dpKSkcf75E2jTprvr3+emj6AN4MxClW+vOwIRuVlE1gGPArVL8lIJ9913H/v3R6ambzwSrOax6qGE6J9g/zvA/v17q2UEwHIel5aWANCiRSf69LkgohpjjS5drFxLztDQF14Yy+23dw1ZJzpRKC7ezYIFTx4xsVVVmTnzBlauXAhAs2ZZvPzyfi65ZFJUclZ57ixW1emq2hm4G7g/WBsRGSUiK0RkxbZt22r0PRs2JEaUR15eLldemcbw4cLo0a2rnVEzkaNgKivuU32U2bPHUFS0IwL7ik/y81ezZctavvkmj48//jtvvfU4mzZ97bWsqKKqPPzwL3jppdvIy/tL+fovvljMu+/O4umnr6C4eFfUdblpCDYBTlPW1l4XinnAL4NtUNWZqtpHVfs0a9asRmLat4//KI/AEEeguEph4Waef35UtYxBqCiYRJj1WduKZAHef/8FbrmlY8zXwXaLYcMeYcqUFfTuPZSlS+eQmzueDRs+91pWVBERzjnnFrKz+9OmzXHl61u37s7IkdMZOfJp6tVrVL5+z54CFi2awcKF01zV5aYh+BjoKiIdRSQNuBx409lARLo63p4HfOuWmClTpkS8SLnfCDbEUdnwlzNFQklJEZZ//0hKSvbE/YUtVHGfmrBv3+5qG99EIzv7VDp1Opnk5BT69LmQIUNuo23b+MtsWxUDB17JpEl5RxSzb9q0LWedNYYBA644ou2OHZuYPXsMq1a942qElWuGQFXLgLHAO8BXwCuqulpEHrIjhADGishqEVkJ3AGMcEtPTk4OJ598FklJKYCQmZlVq5q+fiSc4Z3A04MVIaQUFRWgenQ6ibKy0rj3EwSK+2RkNI3I/ozvqWoGDszh6qufoF27Hl5LiTpJSUmICKrKokUzeOKJS8rzLlWkffsT6Njx5KP8m5EmIVNMxCuh0h7A4fDGgOGrrG1FRIQ5c+I3Y6uTvLxcpk+/stb7SaQ+C4ePP/4bH3wwj7PPHmuihoC77jqe/PzV9OlzIZdd9jDt2h39hHTwYBkHDx6gU6e6roWPeu4sNkSOyoY4iooKeO65a8uHLMJxkMaTH6UqBg7MCZmKOikp+Yi/mZlZIZ8iEqnPwmHlygUsW/YK3333iddSfMEFF9xDmzbHsmLFGyxY8GTQNsnJKaSl1XVVR0IYgtzcXFq2bMnPfy7cfHO7uB2/DQxxBC5UFXEO81TXQRpvfpTqECpF9ejRLzJ3rpKbW8bcucq0aesZMeLJuKuD7SbnnXcnXbueSqtW2V5L8QWDBl3JiBFPcdpp19C791DPdMT90FBubi6jRo2iuPjwGFtaWr248w84GT48iVBz8wJDFtWZMZuZmRVWauZ4IjDTuDopqsNpazDUFDdnFse9IejQoQM//JBY6YIrG/93/t95ebnMmDEiaCWteO4fgyEWidUUE74gUSaSORk2bArPPjuyfD6Bkz17tnPDDZkUFRWQlJRsGwHB+QRhhjYMhsQi7n0EiTCRrCIDB+Zw000vBHVk7t+/l6KiAsBZQ1axjAFxGVZrMBgqJ+4NwZQpU6hXL/GceQMH5vD889vDKMau5cNBxggYDIlF3BuCnJwcZs6cSVZWFiLxOZGsMsIJE43n4TKDwRCauPcRgGUMcnJy4n5CWTAyM9tXe+JYPA+XGQyG0MT9E0GiU908OokwXGYwGIKTEE8EiUxgCGzOnLvZufPI5K8iSageSuj5AgaDwRiChCBwga84gSw1NT2h/CUGgyE4ZmgoQQg3RbXBYEgcjCFIEBK5ApnBYKgcYwgShFBJ5kykkMFgMIYgQQiVUdNEChkMBuMsThACDmGTJdNgMFTEGIIEYuDAHHPhNxgMR+Hq0JCInCMi34jIWhG5J8j2O0TkSxFZJSKLRaS6iXEMBoPBECFcMwQikgxMB4YAxwHDReS4Cs0+A/qoak/gVeBRt/QYDAaDIThuPhH0Bdaq6neqWgrMAy50NlDVd1U1ENy+DGjroh6DwWAwBMFNQ9AG2Oh4n2+vC8V1wIJgG0RklIisEJEV27Zti6BEg8FgMPjCWSwiVwJ9gNOCbVfVmcBMu+02EaleOk33yAS2e6whGEZX+PhVm9EVHn7VBf7RFtIH66Yh2AS0c7xva687AhEZDNwHnKaq+6vaqao2i5jCGiIiK0LV/vQSoyt8/KrN6AoPv+oCf2sL4ObQ0MdAVxHpKCJpwOXAm84GInIS8Bxwgar+5KIWg8FgMITANUOgqmXAWOAd4CvgFVVdLSIPicgFdrPfAxnAX0VkpYi8GWJ3BoPBYHAJV30Eqvo28HaFdQ86lge7+f0uMtNrASEwusLHr9qMrvDwqy7wtzYARFW91mAwGAwGDzFJ5wwGgyHBMYYgCCLSwmsNwRCRxl5rCIaINPVaQyhMn4WHiHgelRcMv56T4N8+CwdjCByISIaI/BFYICLPichFXmsCEJF6IjIdWCgi4+xoK0TE0+Nn99cTwD9F5GER+bmXepyYPgsPEUkXkRnAu3ZAxy/s9X7oL9+dk+DfPqsJMSfYLUSkDfAyIMC5wPv4J/fRHUBTYASQjhVyi6oe8kqQiHQF/gYcBK4FtgH3eqUnCKbPwuNaoDnWpM7vgdkiku5xf/n5nAQf9llNMYbgMCXALFW9VVW3AK8AK0WkpxdiRCTd/psCpAFzVPVrVf09sM2+q/Ty7mMvMFNVx6vql1jRYZtFxLN8UabPwkNEMpxvgQ9VtUBVXwA+BB6x24kX+vDZOQkx0Wc1ImENgYh0E5FnRaQugKoWAO85mrQDOgHfRFlXtojkAtNEpI89HyMDONXR7CbgKhFpG627D7u/yu/GVPVHjswNVQ/orqr50dBTQZvps/B0dRGRV4A/i8h5IlLH3tTc0ewu4Fci0llVNRoXNr+ek7a/V8/TAAAKYklEQVQ2X/ZZpEhIQyAiA7EeOUdhDSEgIqKqex3N0oD11Ul7EUFddbGGMD4HVgE3i8h1wO+Am0QkE0BVNwJ/AW6Ikq7zgNeB8WLXlRCRFFUtcjRrgjcnqOmz8HQlAX8EvsA6B84HHgReAs4VkeMBbOP0BvbQlbocZ+7Xc9LW4cs+iyQJaQiAAqzxvWxgpIhkBTloJwHrAETkhig9jnYG9qrqo6o6DZgF/AqoC8zgyIkpa7AyukbjMXQrkIPVX3eLSIaqlolIkuO7jwNW23quEJFslzUFMH0WHq2AQmCKqr4B/AY4E+iKZVDvk8MROguBaCV49Os5Cf7ts4iRkIZAVb/CqpWwFlgEPARHjR2fATQVkdeAK7DGK93W9QXQQUR+Zq9aBSwGJmAl5msiIhNF5DLgemCf/TlX7zxUdQXwtd1fC4Fn7U3i+O6BQDMR+RvWBfCAm5oc2kyfVYHT6KnqJqxMv2c63s8AHrEN6V5gsohcj/VUtcMNTUF0+eqcdH6vn/rMNVQ1bl9Yj94NHe+l4jLQAFgLnFHhswuw7tYucUFXJtDCqQVIspfHAX9xbOsF/Mn+TDZwNfAvICcauoL0V0Osu6P/c2yrg3UB/gS4zKVj2QroX2GdH/rsKF1+6DNb13UV1iXbf68B8hzrj8FyxPYCGmMNfcxzsb+uC7HNs3PS3n8b4DEgzU99Fo2X5wJc+8fgAaxkd/OASfa6pAptAgf5NuAte3m4fWE+3SVd9wNfY5XmnOrUYS93AV4DRtjvm2Il7mvpcn8F0xWqvx4A3rWXz7H//tJlfaux7qx72++dRsqTPqtKl1d9Zn/XJ8AdIbYnAf8BbnOsewk43uW+qlRXhf6K2jlpf8dNWD6Ap7ECDSr+9j3ps2i9PBfgwgGtj5XVdD7QAuuOcBfQwd5+1F2uvbzTbvcnIN0FXenAVKy7iGa2tmKgib09ydH2LKyx0N7AZcC7QHuX+qtSXRXaOvurDNgDPAmkung8k+wTcwHWY/ftQH0v+6waukL9xlzvM6w72j3B9l9By8lYse+/BK60L9DHudhfIXVVotHVc9L5GwKmhbqoc/hJJap9Fs2X5wIieECPsf+mAKcDKY5tzwPXhvhcIyzDsQoY4JYue7mVY/l0YC7Qo0L7wI9uNPCU/WPzXJdje6bdn5+5oauiNse6p4Hx9kX0Z173WXV0RavPKhzLY7HqfzfAugm6FuhXoX3g4nchMBn4LzDQa12Otq6ek0G0JQH/w7oJ6mF/9zU4boY4/KTiap959fJcQAQOaFP7ZPwXcAvQxbFNsELO3gV6hfh8EtAzCrq62+tTsO5e1wOPA8uxHn0DPzTnXW6yX3Q5Pp/i4snp1DYOONZe3wX4s718B9Yd+G1A28BxjmKfVVuX230W5FgGdE3Ccjx/AEzEimK52vEbk0hriYQux+ddOSdDHMsT7PWPAE/Y62/Emuz3iOP8SHJDj19eMR01JCL9sB7NtwMPYzl7bnE0ScL64ZUSpEwmWCkHVHVVFHSNsb+vDPgU6KSqd2LdXdxha0Udk51U9aBfdDk0lanq0kjqCqGtrUPbWmC3iCQD3YBbgVPUnoil9plqL7vdZ9XW5dAU8T4LcSxH25ufxIq6Gayqk7EKRI1x6FFcoja6HPoifk6G0NYWuM7eHBhW/KuqPgfcDbTEnjCmMZg2Ihx8Uby+FhQCf1DVeVCeafIMe9bfAVU9KCKd7OVtYiWsSgu090BXOrBfVcsLWavqP0VkPNYJsz5BdVWmLRUr3v4ErKGCdVjjzSoiXVX1W6OrXNdg+1juUtXfBBqq6j9E5A6sC5/bMe5+1RVK25n2tvexZqL3trX9z56MGNcGIEDMGAJ7luERdzKq+rWIbHRsO4A1NOSceXgGUFdEXsK6a7vbQ10lzs+IyLFYdyY/AD8mgq4aaDsAfC4iecASVf2XiHTEqoEd0TjyONDVOXAsHZ89DutYrse731hUdYWprZO9ba2IPA3cI1ZOqk5YT8PfR1qbH4mJoSERSXUe1AoTUfY6tnXAujtzkgkcjxUDfIqqvucDXXVE5CqsyKb/qOo1qloa77pqo01VH1DVf9nL36vqb9VKG2F0BdElIiki8ius8OnFqjrSNl5xras22lR1JdZQVR7wpqoOVWvyWNzje0MgImOBf4uV7/t8sMY4nTP/HMudgI/sdReJSCOstL8dVDWidUNrowsr2dg7WOPJ0xNBV221iUjLSOuJZ11YkTfvA3199htzTVdttYlIa1XdoaqvqeqfIq3Nz/jWEIhIE7GGc84Gfo3l4BlhP36XO29EpKceduQcB2SLyALgUix/wNequs9nupJV9adE0BUhbRfjwlhtHOu6BCvKZUfFIZl41BUhbRdjzfNITNQHoUvBXkAyVibCQIx4J+DP2PHtWB79l4AlQGugPbAbK22ta7Ncja740WZ0xYcuv2uLhZfnAhwHMgVrYk47x7oMx3IS1oSUrvb7M4ExFfYx0ujyVpeftRld8aHL79pi8eW5APuAnIAVw74VmBuizbHA2yG2pRld3uvyszajKz50+V1brL784iPYjpUaoDtWSuGzAEQk2eHxb4VV4xUROUXsot8iIhrhyBajKy61GV3xocvv2mISXxgCVd0MzFfVnVjjeoEKPwex0kQAnAikicjvsaaCBz7r2ixJoyt+tBld8aHL79piFV8YAgA9HKnyElAiIrfY6w/ZVv5nwC+AHaraX1XfNbr8p8vP2oyu+NDld20xiddjU8FeWCFgH9nLPe2/F+BwDBld/tflZ21GV3zo8ru2WHl5LqCSg7sQ2I+VBTDTaz1GV/xpM7riQ5fftcXCyzdDQwHEKu79MJbXf6yqnquOZGhGV2zoAv9qM7riQxf4W1ssEZh84StEZAhWrpv9VTaOIkZX+PhVm9EVHn7VBf7WFiv40hAYDAaDIXr4bmjIYDAYDNHFGAKDwWBIcIwhMBgMhgTHGAKDwWBIcIwhMBgMhgTHGAKDoQpE5KCIrBSR1SLyuYjcKY6KVyE+00FEroiWRoOhNhhDYDBUzT5V7aWqPbDy2g8BJlbxmQ6AMQSGmMDMIzAYqkBEilQ1w/G+E/AxkAlkAS8D9e3NY1X1AxFZhjXb9XvgRay0yVOB04E6wHRVfS5q/4TBUAnGEBgMVVDRENjrCoFuwB7gkKqWiEhXrEIpfUTkdGC8qg61248CmqvqwyJSB1gKXKqq30f1nzEYgpDitQCDIcZJBZ4WkV7AQSA7RLuzgJ4icon9vhHQFeuJwWDwFGMIDIYwsYeGDgI/YfkKtmIVQkkCSkJ9DBinqu9ERaTBEAbGWWwwhIGINAOeBZ5Wa1y1EbBZVQ8BVwHJdtM9QAPHR98BRotIqr2fbBGpj8HgA8wTgcFQNXVFZCXWMFAZlnP4D/a2Z4DXRORqrJz4e+31q4CDIvI5VjnFJ7EiiT61K2htA34ZrX/AYKgM4yw2GAyGBMcMDRkMBkOCYwyBwWAwJDjGEBgMBkOCYwyBwWAwJDjGEBgMBkOCYwyBwWAwJDjGEBgMBkOCYwyBwWAwJDj/DxvKGs1gcQRiAAAAAElFTkSuQmCC\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - }, { "name": "stdout", "output_type": "stream", "text": [ - "step val 0, {'val_loss': '1.0232683420181274', 'val/kl': '0.500285804271698', 'val/mse': '0.22834806144237518', 'val/std': '1.0002851486206055'} {}\n", + "step val 0, {'val_loss': '1.0510647296905518', 'val/kl': '0.500446081161499', 'val/std': '1.011029839515686', 'val/mse': '0.321998655796051'} {}\n", "\r" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "6c20adf0b61c495eb80f5a32bdff8482", + "model_id": "0135c39b2b944755acf806dbf6ba72e8", "version_major": 2, "version_minor": 0 }, @@ -584,7 +695,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "daf96d8ae3b141a9a5eb3efe0c6a15a9", + "model_id": "", "version_major": 2, "version_minor": 0 }, @@ -594,11 +705,593 @@ }, "metadata": {}, "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step val 2194, {'val_loss': '-0.13158796727657318', 'val/kl': '0.00033035798696801066', 'val/std': '0.21403183043003082', 'val/mse': '0.032815106213092804'} {'train_loss': tensor(16.5981, device='cuda:0'), 'train/kl': tensor(0.0179, device='cuda:0'), 'train/std': tensor(0.4463, device='cuda:0'), 'train/mse': tensor(0.1485, device='cuda:0')}\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=234.0, style=Pr…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step val 4389, {'val_loss': '-0.4325900673866272', 'val/kl': '9.704002877697349e-05', 'val/std': '0.09763659536838531', 'val/mse': '0.014293229207396507'} {'train_loss': tensor(-0.2103, device='cuda:0'), 'train/kl': tensor(0.0009, device='cuda:0'), 'train/std': tensor(0.1743, device='cuda:0'), 'train/mse': tensor(0.0344, device='cuda:0')}\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=234.0, style=Pr…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step val 6584, {'val_loss': '-0.43620601296424866', 'val/kl': '4.939149948768318e-05', 'val/std': '0.09694259613752365', 'val/mse': '0.013617178425192833'} {'train_loss': tensor(-0.3606, device='cuda:0'), 'train/kl': tensor(0.0005, device='cuda:0'), 'train/std': tensor(0.1432, device='cuda:0'), 'train/mse': tensor(0.0260, device='cuda:0')}\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=234.0, style=Pr…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step val 8779, {'val_loss': '-0.42683538794517517', 'val/kl': '8.107969915727153e-05', 'val/std': '0.09468298405408859', 'val/mse': '0.0107091274112463'} {'train_loss': tensor(-0.4410, device='cuda:0'), 'train/kl': tensor(0.0004, device='cuda:0'), 'train/std': tensor(0.1285, device='cuda:0'), 'train/mse': tensor(0.0230, device='cuda:0')}\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=234.0, style=Pr…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step val 10974, {'val_loss': '-0.5034099817276001', 'val/kl': '3.636208930402063e-05', 'val/std': '0.0672297552227974', 'val/mse': '0.00938461348414421'} {'train_loss': tensor(-0.5030, device='cuda:0'), 'train/kl': tensor(0.0003, device='cuda:0'), 'train/std': tensor(0.1138, device='cuda:0'), 'train/mse': tensor(0.0200, device='cuda:0')}\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=234.0, style=Pr…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step val 13169, {'val_loss': '-0.24954310059547424', 'val/kl': '3.8109676097519696e-05', 'val/std': '0.060545068234205246', 'val/mse': '0.01272621750831604'} {'train_loss': tensor(-0.5629, device='cuda:0'), 'train/kl': tensor(0.0001, device='cuda:0'), 'train/std': tensor(0.1040, device='cuda:0'), 'train/mse': tensor(0.0180, device='cuda:0')}\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=234.0, style=Pr…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step val 15364, {'val_loss': '-0.11352463811635971', 'val/kl': '1.6889993275981396e-05', 'val/std': '0.05332550033926964', 'val/mse': '0.013464201241731644'} {'train_loss': tensor(-0.6062, device='cuda:0'), 'train/kl': tensor(9.2420e-05, device='cuda:0'), 'train/std': tensor(0.0985, device='cuda:0'), 'train/mse': tensor(0.0169, device='cuda:0')}\n", + "Epoch 6: reducing learning rate of group 0 to 2.0000e-04.\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=234.0, style=Pr…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step val 17559, {'val_loss': '-0.04702010378241539', 'val/kl': '1.287278973904904e-05', 'val/std': '0.04811198264360428', 'val/mse': '0.012275541201233864'} {'train_loss': tensor(-0.6955, device='cuda:0'), 'train/kl': tensor(5.2590e-05, device='cuda:0'), 'train/std': tensor(0.0835, device='cuda:0'), 'train/mse': tensor(0.0139, device='cuda:0')}\n", + "INFO:root:Epoch 00008: early stopping\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b730393947d64b688442724cbe3cfd9d", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='Testing', layout=Layout(flex='2'), max=234.0, style=Progr…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step val 17560, {'val_loss': '-0.04702010378241539', 'val/kl': '1.287278973904904e-05', 'val/std': '0.04811198264360428', 'val/mse': '0.012275541201233864'} {}\n", + "----------------------------------------------------------------------------------------------------\n", + "TEST RESULTS\n", + "{}\n", + "----------------------------------------------------------------------------------------------------\n", + "\n", + "1 None\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "e8b65004d75441cb989598eeca7e3fb4", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='MCDropout eval', max=600.0, style=ProgressStyle(descripti…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Lower is better, validation loss\n", + "MCDropout: -1.32\n", + "Inference: -0.14\n", + "INFO:root:GPU available: True, used: True\n", + "INFO:root:VISIBLE GPUS: 0\n", + "INFO:root:\n", + " | Name | Type | Params\n", + "---------------------------------------------------------------------------------------------------------------\n", + "0 | model | LatentModel | 3 M \n", + "1 | model._lstm | LSTM | 807 K \n", + "2 | model._latent_encoder | LatentEncoder | 328 K \n", + "3 | model._latent_encoder._encoder | BatchMLP | 131 K \n", + "4 | model._latent_encoder._encoder.initial | NPBlockRelu2d | 65 K \n", + "5 | model._latent_encoder._encoder.initial.linear | Linear | 65 K \n", + "6 | model._latent_encoder._encoder.initial.act | ReLU | 0 \n", + "7 | model._latent_encoder._encoder.initial.dropout | Dropout2d | 0 \n", + "8 | model._latent_encoder._encoder.encoder | Sequential | 0 \n", + "9 | model._latent_encoder._encoder.final | Linear | 65 K \n", + "10 | model._latent_encoder._self_attention | Attention | 0 \n", + "11 | model._latent_encoder._penultimate_layer | Linear | 65 K \n", + "12 | model._latent_encoder._mean | Linear | 65 K \n", + "13 | model._latent_encoder._log_var | Linear | 65 K \n", + "14 | model._deterministic_encoder | DeterministicEncoder | 787 K \n", + "15 | model._deterministic_encoder._d_encoder | BatchMLP | 262 K \n", + "16 | model._deterministic_encoder._d_encoder.initial | NPBlockRelu2d | 65 K \n", + "17 | model._deterministic_encoder._d_encoder.initial.linear | Linear | 65 K \n", + "18 | model._deterministic_encoder._d_encoder.initial.act | ReLU | 0 \n", + "19 | model._deterministic_encoder._d_encoder.initial.dropout | Dropout2d | 0 \n", + "20 | model._deterministic_encoder._d_encoder.encoder | Sequential | 131 K \n", + "21 | model._deterministic_encoder._d_encoder.encoder.0 | NPBlockRelu2d | 65 K \n", + "22 | model._deterministic_encoder._d_encoder.encoder.0.linear | Linear | 65 K \n", + "23 | model._deterministic_encoder._d_encoder.encoder.0.act | ReLU | 0 \n", + "24 | model._deterministic_encoder._d_encoder.encoder.0.dropout | Dropout2d | 0 \n", + "25 | model._deterministic_encoder._d_encoder.encoder.1 | NPBlockRelu2d | 65 K \n", + "26 | model._deterministic_encoder._d_encoder.encoder.1.linear | Linear | 65 K \n", + "27 | model._deterministic_encoder._d_encoder.encoder.1.act | ReLU | 0 \n", + "28 | model._deterministic_encoder._d_encoder.encoder.1.dropout | Dropout2d | 0 \n", + "29 | model._deterministic_encoder._d_encoder.final | Linear | 65 K \n", + "30 | model._deterministic_encoder._self_attention | Attention | 0 \n", + "31 | model._deterministic_encoder._cross_attention | Attention | 524 K \n", + "32 | model._deterministic_encoder._cross_attention.batch_mlp_k | BatchMLP | 131 K \n", + "33 | model._deterministic_encoder._cross_attention.batch_mlp_k.initial | NPBlockRelu2d | 65 K \n", + "34 | model._deterministic_encoder._cross_attention.batch_mlp_k.initial.linear | Linear | 65 K \n", + "35 | model._deterministic_encoder._cross_attention.batch_mlp_k.initial.act | ReLU | 0 \n", + "36 | model._deterministic_encoder._cross_attention.batch_mlp_k.initial.dropout | Dropout2d | 0 \n", + "37 | model._deterministic_encoder._cross_attention.batch_mlp_k.encoder | Sequential | 0 \n", + "38 | model._deterministic_encoder._cross_attention.batch_mlp_k.final | Linear | 65 K \n", + "39 | model._deterministic_encoder._cross_attention.batch_mlp_q | BatchMLP | 131 K \n", + "40 | model._deterministic_encoder._cross_attention.batch_mlp_q.initial | NPBlockRelu2d | 65 K \n", + "41 | model._deterministic_encoder._cross_attention.batch_mlp_q.initial.linear | Linear | 65 K \n", + "42 | model._deterministic_encoder._cross_attention.batch_mlp_q.initial.act | ReLU | 0 \n", + "43 | model._deterministic_encoder._cross_attention.batch_mlp_q.initial.dropout | Dropout2d | 0 \n", + "44 | model._deterministic_encoder._cross_attention.batch_mlp_q.encoder | Sequential | 0 \n", + "45 | model._deterministic_encoder._cross_attention.batch_mlp_q.final | Linear | 65 K \n", + "46 | model._deterministic_encoder._cross_attention._W | MultiheadAttention | 262 K \n", + "47 | model._deterministic_encoder._cross_attention._W.out_proj | Linear | 65 K \n", + "48 | model._decoder | Decoder | 1 M \n", + "49 | model._decoder._target_transform | Linear | 65 K \n", + "50 | model._decoder._decoder | BatchMLP | 1 M \n", + "51 | model._decoder._decoder.initial | NPBlockRelu2d | 262 K \n", + "52 | model._decoder._decoder.initial.linear | Linear | 262 K \n", + "53 | model._decoder._decoder.initial.act | ReLU | 0 \n", + "54 | model._decoder._decoder.initial.dropout | Dropout2d | 0 \n", + "55 | model._decoder._decoder.encoder | Sequential | 524 K \n", + "56 | model._decoder._decoder.encoder.0 | NPBlockRelu2d | 262 K \n", + "57 | model._decoder._decoder.encoder.0.linear | Linear | 262 K \n", + "58 | model._decoder._decoder.encoder.0.act | ReLU | 0 \n", + "59 | model._decoder._decoder.encoder.0.dropout | Dropout2d | 0 \n", + "60 | model._decoder._decoder.encoder.1 | NPBlockRelu2d | 262 K \n", + "61 | model._decoder._decoder.encoder.1.linear | Linear | 262 K \n", + "62 | model._decoder._decoder.encoder.1.act | ReLU | 0 \n", + "63 | model._decoder._decoder.encoder.1.dropout | Dropout2d | 0 \n", + "64 | model._decoder._decoder.final | Linear | 262 K \n", + "65 | model._decoder._mean | Linear | 513 \n", + "66 | model._decoder._std | Linear | 513 \n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='Validation sanity check', layout=Layout(flex='2'), max=5.…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step val 0, {'val_loss': '1.0469273328781128', 'val/kl': '0.5012975931167603', 'val/std': '1.0240424871444702', 'val/mse': '0.2834925353527069'} {}\n", + "\r" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "3a08f15a36684e3696aabf65f5451d99", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=234.0, style=Pr…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step val 2194, {'val_loss': '-0.530510663986206', 'val/kl': '0.000497261353302747', 'val/std': '0.10500864684581757', 'val/mse': '0.008976762183010578'} {'train_loss': tensor(-0.1966, device='cuda:0'), 'train/kl': tensor(0.0111, device='cuda:0'), 'train/std': tensor(0.2003, device='cuda:0'), 'train/mse': tensor(0.0401, device='cuda:0')}\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=234.0, style=Pr…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step val 4389, {'val_loss': '-0.5772886276245117', 'val/kl': '0.00011175952386111021', 'val/std': '0.09931007772684097', 'val/mse': '0.007692516781389713'} {'train_loss': tensor(-0.4069, device='cuda:0'), 'train/kl': tensor(0.0007, device='cuda:0'), 'train/std': tensor(0.1353, device='cuda:0'), 'train/mse': tensor(0.0239, device='cuda:0')}\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=234.0, style=Pr…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step val 6584, {'val_loss': '-0.5756369233131409', 'val/kl': '6.505264900624752e-05', 'val/std': '0.06530915945768356', 'val/mse': '0.007986452430486679'} {'train_loss': tensor(-0.4907, device='cuda:0'), 'train/kl': tensor(0.0003, device='cuda:0'), 'train/std': tensor(0.1155, device='cuda:0'), 'train/mse': tensor(0.0198, device='cuda:0')}\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=234.0, style=Pr…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step val 8779, {'val_loss': '-0.5241847634315491', 'val/kl': '4.0910555981099606e-05', 'val/std': '0.06427723169326782', 'val/mse': '0.01057453267276287'} {'train_loss': tensor(-0.5611, device='cuda:0'), 'train/kl': tensor(0.0002, device='cuda:0'), 'train/std': tensor(0.1037, device='cuda:0'), 'train/mse': tensor(0.0176, device='cuda:0')}\n", + "Epoch 3: reducing learning rate of group 0 to 2.0000e-04.\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=234.0, style=Pr…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step val 10974, {'val_loss': '-0.5098873972892761', 'val/kl': '4.259546039975248e-05', 'val/std': '0.054850272834300995', 'val/mse': '0.007577850017696619'} {'train_loss': tensor(-0.6559, device='cuda:0'), 'train/kl': tensor(0.0001, device='cuda:0'), 'train/std': tensor(0.0854, device='cuda:0'), 'train/mse': tensor(0.0144, device='cuda:0')}\n", + "INFO:root:Epoch 00005: early stopping\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "638d8934f25648848dc65929296423b7", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='Testing', layout=Layout(flex='2'), max=234.0, style=Progr…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step val 10975, {'val_loss': '-0.5098873972892761', 'val/kl': '4.259546039975248e-05', 'val/std': '0.054850272834300995', 'val/mse': '0.007577850017696619'} {}\n", + "----------------------------------------------------------------------------------------------------\n", + "TEST RESULTS\n", + "{}\n", + "----------------------------------------------------------------------------------------------------\n", + "\n", + "10 None\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "cc42a5e826104a26aeb00bb1d512f05e", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='MCDropout eval', max=600.0, style=ProgressStyle(descripti…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Lower is better, validation loss\n", + "MCDropout: -1.35\n", + "Inference: -0.98\n", + "INFO:root:GPU available: True, used: True\n", + "INFO:root:VISIBLE GPUS: 0\n", + "INFO:root:\n", + " | Name | Type | Params\n", + "---------------------------------------------------------------------------------------------------------------\n", + "0 | model | LatentModel | 3 M \n", + "1 | model._lstm | LSTM | 807 K \n", + "2 | model._latent_encoder | LatentEncoder | 328 K \n", + "3 | model._latent_encoder._encoder | BatchMLP | 131 K \n", + "4 | model._latent_encoder._encoder.initial | NPBlockRelu2d | 65 K \n", + "5 | model._latent_encoder._encoder.initial.linear | Linear | 65 K \n", + "6 | model._latent_encoder._encoder.initial.act | ReLU | 0 \n", + "7 | model._latent_encoder._encoder.initial.dropout | Dropout2d | 0 \n", + "8 | model._latent_encoder._encoder.encoder | Sequential | 0 \n", + "9 | model._latent_encoder._encoder.final | Linear | 65 K \n", + "10 | model._latent_encoder._self_attention | Attention | 0 \n", + "11 | model._latent_encoder._penultimate_layer | Linear | 65 K \n", + "12 | model._latent_encoder._mean | Linear | 65 K \n", + "13 | model._latent_encoder._log_var | Linear | 65 K \n", + "14 | model._deterministic_encoder | DeterministicEncoder | 787 K \n", + "15 | model._deterministic_encoder._d_encoder | BatchMLP | 262 K \n", + "16 | model._deterministic_encoder._d_encoder.initial | NPBlockRelu2d | 65 K \n", + "17 | model._deterministic_encoder._d_encoder.initial.linear | Linear | 65 K \n", + "18 | model._deterministic_encoder._d_encoder.initial.act | ReLU | 0 \n", + "19 | model._deterministic_encoder._d_encoder.initial.dropout | Dropout2d | 0 \n", + "20 | model._deterministic_encoder._d_encoder.encoder | Sequential | 131 K \n", + "21 | model._deterministic_encoder._d_encoder.encoder.0 | NPBlockRelu2d | 65 K \n", + "22 | model._deterministic_encoder._d_encoder.encoder.0.linear | Linear | 65 K \n", + "23 | model._deterministic_encoder._d_encoder.encoder.0.act | ReLU | 0 \n", + "24 | model._deterministic_encoder._d_encoder.encoder.0.dropout | Dropout2d | 0 \n", + "25 | model._deterministic_encoder._d_encoder.encoder.1 | NPBlockRelu2d | 65 K \n", + "26 | model._deterministic_encoder._d_encoder.encoder.1.linear | Linear | 65 K \n", + "27 | model._deterministic_encoder._d_encoder.encoder.1.act | ReLU | 0 \n", + "28 | model._deterministic_encoder._d_encoder.encoder.1.dropout | Dropout2d | 0 \n", + "29 | model._deterministic_encoder._d_encoder.final | Linear | 65 K \n", + "30 | model._deterministic_encoder._self_attention | Attention | 0 \n", + "31 | model._deterministic_encoder._cross_attention | Attention | 524 K \n", + "32 | model._deterministic_encoder._cross_attention.batch_mlp_k | BatchMLP | 131 K \n", + "33 | model._deterministic_encoder._cross_attention.batch_mlp_k.initial | NPBlockRelu2d | 65 K \n", + "34 | model._deterministic_encoder._cross_attention.batch_mlp_k.initial.linear | Linear | 65 K \n", + "35 | model._deterministic_encoder._cross_attention.batch_mlp_k.initial.act | ReLU | 0 \n", + "36 | model._deterministic_encoder._cross_attention.batch_mlp_k.initial.dropout | Dropout2d | 0 \n", + "37 | model._deterministic_encoder._cross_attention.batch_mlp_k.encoder | Sequential | 0 \n", + "38 | model._deterministic_encoder._cross_attention.batch_mlp_k.final | Linear | 65 K \n", + "39 | model._deterministic_encoder._cross_attention.batch_mlp_q | BatchMLP | 131 K \n", + "40 | model._deterministic_encoder._cross_attention.batch_mlp_q.initial | NPBlockRelu2d | 65 K \n", + "41 | model._deterministic_encoder._cross_attention.batch_mlp_q.initial.linear | Linear | 65 K \n", + "42 | model._deterministic_encoder._cross_attention.batch_mlp_q.initial.act | ReLU | 0 \n", + "43 | model._deterministic_encoder._cross_attention.batch_mlp_q.initial.dropout | Dropout2d | 0 \n", + "44 | model._deterministic_encoder._cross_attention.batch_mlp_q.encoder | Sequential | 0 \n", + "45 | model._deterministic_encoder._cross_attention.batch_mlp_q.final | Linear | 65 K \n", + "46 | model._deterministic_encoder._cross_attention._W | MultiheadAttention | 262 K \n", + "47 | model._deterministic_encoder._cross_attention._W.out_proj | Linear | 65 K \n", + "48 | model._decoder | Decoder | 1 M \n", + "49 | model._decoder._target_transform | Linear | 65 K \n", + "50 | model._decoder._decoder | BatchMLP | 1 M \n", + "51 | model._decoder._decoder.initial | NPBlockRelu2d | 262 K \n", + "52 | model._decoder._decoder.initial.linear | Linear | 262 K \n", + "53 | model._decoder._decoder.initial.act | ReLU | 0 \n", + "54 | model._decoder._decoder.initial.dropout | Dropout2d | 0 \n", + "55 | model._decoder._decoder.encoder | Sequential | 524 K \n", + "56 | model._decoder._decoder.encoder.0 | NPBlockRelu2d | 262 K \n", + "57 | model._decoder._decoder.encoder.0.linear | Linear | 262 K \n", + "58 | model._decoder._decoder.encoder.0.act | ReLU | 0 \n", + "59 | model._decoder._decoder.encoder.0.dropout | Dropout2d | 0 \n", + "60 | model._decoder._decoder.encoder.1 | NPBlockRelu2d | 262 K \n", + "61 | model._decoder._decoder.encoder.1.linear | Linear | 262 K \n", + "62 | model._decoder._decoder.encoder.1.act | ReLU | 0 \n", + "63 | model._decoder._decoder.encoder.1.dropout | Dropout2d | 0 \n", + "64 | model._decoder._decoder.final | Linear | 262 K \n", + "65 | model._decoder._mean | Linear | 513 \n", + "66 | model._decoder._std | Linear | 513 \n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='Validation sanity check', layout=Layout(flex='2'), max=5.…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step val 0, {'val_loss': '1.012698769569397', 'val/kl': '0.4999396800994873', 'val/std': '0.9445148706436157', 'val/mse': '0.2685103118419647'} {}\n", + "\r" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "3b98eac0167e4a049604ff011da70e29", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …" + ] + }, + "metadata": {}, + "output_type": "display_data" } ], "source": [ "name = 'anp-rnn-mcdropout'\n", - "\n", "params =default_params.copy()\n", "params.update({\n", " 'det_enc_cross_attn_type': 'ptmultihead',\n", @@ -609,34 +1302,67 @@ " 'latent_dim': 128*2, \n", " 'use_deterministic_path': False,\n", " 'use_rnn': True,\n", + " 'vis_i': 670\n", "})\n", - "trial = optuna.trial.FixedTrial(params)\n", - "trial = add_sugg(trial)\n", - "trial = add_trial_number(trial, MODEL_DIR/name)\n", - "\n", - "\n", - "checkpoint_callback = pl.callbacks.ModelCheckpoint(\n", - " os.path.join(MODEL_DIR, name, 'version_{}'.format(trial.number), \"chk\"), monitor='val_loss', mode=\"min\")\n", - "\n", - "logger = DictLogger(MODEL_DIR, name=\"anp\", version=trial.number)\n", - "\n", - "trainer = pl.Trainer(\n", - " gradient_clip_val=trial.params[\"grad_clip\"],\n", - " checkpoint_callback=checkpoint_callback,\n", - " max_epochs=trial.params['max_nb_epochs'],\n", - " gpus=-1 if torch.cuda.is_available() else None,\n", - " early_stop_callback=True\n", - ")\n", - "model = LatentModelPL(trial.params)\n", - "\n", - "trainer.fit(model)\n", - "\n", - "# plot, main metric\n", - "loader = model.val_dataloader()[0]\n", + "n_mcdropout=10\n", + "n_steps=600\n", "vis_i=670\n", - "plot_from_loader(loader, model, i=vis_i)\n", + "results = []\n", "\n", - "print(trainer.test(model))" + "for seed in tqdm([1, 10, 20, 100], desc='seeds'):\n", + " init_random_seed(seed)\n", + "\n", + " trial = optuna.trial.FixedTrial(params)\n", + " trial = add_sugg(trial)\n", + " trial = add_trial_number(trial, MODEL_DIR/name)\n", + "\n", + "\n", + " checkpoint_callback = pl.callbacks.ModelCheckpoint(\n", + " os.path.join(MODEL_DIR, name, 'version_{}'.format(trial.number), \"chk\"), monitor='val_loss', mode=\"min\")\n", + "\n", + " logger = DictLogger(MODEL_DIR, name=\"anp\", version=trial.number)\n", + "\n", + " trainer = pl.Trainer(\n", + " gradient_clip_val=trial.params[\"grad_clip\"],\n", + " checkpoint_callback=checkpoint_callback,\n", + " max_epochs=trial.params['max_nb_epochs'],\n", + " gpus=-1 if torch.cuda.is_available() else None,\n", + " early_stop_callback=True\n", + " )\n", + " model = LatentModelPL(trial.params)\n", + "\n", + " trainer.fit(model)\n", + "\n", + " # plot, main metric\n", + "# loader = model.val_dataloader()\n", + " \n", + " plot_from_loader(loader, model, i=vis_i)\n", + "\n", + " \n", + " # Testing will load best checkpoint\n", + " print(seed, trainer.test(model))\n", + "\n", + " \n", + " # MCDropout\n", + "\n", + " loader = model.val_dataloader()\n", + " device = next(model.parameters()).device\n", + " inds = np.random.randint(0, len(loader.dataset), n_steps)\n", + "\n", + " losses_mc = []\n", + " losses = []\n", + " for i in tqdm(inds, desc='MCDropout eval'):\n", + " loss_mc, loss = eval_mc(model, loader, i)\n", + " losses_mc.append(loss_mc)\n", + " losses.append(loss)\n", + "\n", + " loss_mc = torch.stack(losses_mc).mean()\n", + " loss = torch.stack(losses).mean()\n", + " \n", + " results.append(dict(seed=seed, loss=loss, loss_mc=loss_mc, inds=inds))\n", + " print(f\"seed={seed}. Lower is better, validation loss\")\n", + " print(f\"MCDropout: {loss_mc:2.2f}\")\n", + " print(f\"Inference: {loss:2.2f}\")" ] }, { @@ -644,44 +1370,21 @@ "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2020-03-14T01:46:54.331599Z", - "start_time": "2020-03-14T01:46:54.258559Z" - } - }, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2020-03-14T01:46:17.333555Z", - "start_time": "2020-03-14T01:45:49.700Z" + "start_time": "2020-03-14T03:00:15.800Z" } }, "outputs": [], "source": [ - "logger.save_dir, logger.experiment.log_dir" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2020-03-14T01:46:17.335032Z", - "start_time": "2020-03-14T01:45:49.800Z" - }, - "scrolled": true - }, - "outputs": [], - "source": [ - "# # plot lots of metrics\n", - "# loader = model.val_dataloader()[0]\n", - "# for i in range(0, len(loader), 10):\n", - "# plot_from_loader(loader, model, i=i)\n", - "# plt.show()" + "\n", + "\n", + "# Lower is better, validation loss\n", + "# MCDropout: -1.32\n", + "# Inference: -0.14\n", + " \n", + " \n", + "# Lower is better, validation loss\n", + "# MCDropout: -1.35\n", + "# Inference: -0.98" ] }, { @@ -713,21 +1416,7 @@ } }, "outputs": [], - "source": [ - "from typing import Callable\n", - "\n", - "def convert_layers(model:nn.Module, original:nn.Module, value: bool):\n", - " \"\"\"\n", - " Turn dropout on\n", - " \"\"\"\n", - " for child_name, child in model.named_children():\n", - " if isinstance(child, original):\n", - "# print(child, 'from', child.training, 'to', value)\n", - " child.train(value)\n", - " \n", - " else:\n", - " convert_layers(child, original, value)" - ] + "source": [] }, { "cell_type": "code", @@ -754,7 +1443,7 @@ "outputs": [], "source": [ "from src.plot import plot_rows\n", - "loader = model.val_dataloader()[0]\n", + "loader = model.val_dataloader()\n", "device = next(model.parameters()).device\n", "\n", "# Get data\n", @@ -882,94 +1571,6 @@ "## Full test" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2020-03-14T01:46:17.344434Z", - "start_time": "2020-03-14T01:45:49.800Z" - } - }, - "outputs": [], - "source": [ - "from src.plot import plot_rows" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2020-03-14T01:46:17.345856Z", - "start_time": "2020-03-14T01:45:49.800Z" - } - }, - "outputs": [], - "source": [ - "\n", - "\n", - "def eval_mc(model, loader, i):\n", - " loader = model.val_dataloader()[0]\n", - " device = next(model.parameters()).device\n", - "\n", - " data = loader.collate_fn([loader.dataset[i]], sample=False)\n", - " data = [d.to(device) for d in data]\n", - " context_x, context_y, target_x_extra, target_y_extra = data\n", - " target_x = target_x_extra\n", - " target_y = target_y_extra\n", - "\n", - " # do MCDropout estimation\n", - " model.eval()\n", - " convert_layers(model, torch.nn.modules.dropout.Dropout2d, True)\n", - " model.model._lstm.train()\n", - "\n", - " y_preds = []\n", - " y_stds = []\n", - " with torch.no_grad():\n", - " for i in range(n_mcdropout):\n", - " y_pred, kl, loss_test, loss_mse, y_std = model(context_x, context_y, target_x, target_y)\n", - " y_preds.append(y_pred)\n", - " y_stds.append(y_std)\n", - " y_stds = torch.stack(y_stds)\n", - " y_preds = torch.stack(y_preds)\n", - "\n", - "\n", - " mean = y_preds.mean(0)\n", - " sigma = y_stds.mean(0)\n", - "\n", - " dist = torch.distributions.Normal(mean, sigma)\n", - " log_p = dist.log_prob(target_y).mean(-1)\n", - " loss_mc = - log_p\n", - "\n", - "\n", - " # Try without MCDropout\n", - " model.eval()\n", - " with torch.no_grad():\n", - " y_pred, kl, loss_test, loss_mse, y_std = model(context_x, context_y, target_x, target_y)\n", - "\n", - " dist = torch.distributions.Normal(y_pred, y_std)\n", - " log_p = dist.log_prob(target_y).mean(-1)\n", - " loss = - log_p \n", - "\n", - " return loss_mc.cpu(), loss.cpu()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2020-03-14T01:46:17.347199Z", - "start_time": "2020-03-14T01:45:49.800Z" - } - }, - "outputs": [], - "source": [ - "n_mcdropout=10\n", - "n_steps=300" - ] - }, { "cell_type": "code", "execution_count": null, @@ -982,6 +1583,9 @@ "outputs": [], "source": [ "# Get data\n", + "n_mcdropout=10\n", + "n_steps=600\n", + "\n", "inds = np.random.randint(0, len(loader.dataset), n_steps)\n", "\n", "losses_mc = []\n", @@ -990,7 +1594,12 @@ " loss_mc, loss = eval_mc(model, loader, i)\n", " losses_mc.append(loss_mc)\n", " losses.append(loss)\n", - " " + " \n", + "loss_mc = torch.stack(losses_mc).mean()\n", + "loss = torch.stack(losses).mean()\n", + "print(f\"Lower is better, validation loss\")\n", + "print(f\"MCDropout: {loss_mc:2.2f}\")\n", + "print(f\"Inference: {loss:2.2f}\")" ] }, { @@ -1003,13 +1612,7 @@ } }, "outputs": [], - "source": [ - "loss_mc = torch.stack(losses_mc).mean()\n", - "loss = torch.stack(losses).mean()\n", - "print(f\"Lower is better, validation loss\")\n", - "print(f\"MCDropout: {loss_mc:2.2f}\")\n", - "print(f\"Inference: {loss:2.2f}\")" - ] + "source": [] }, { "cell_type": "markdown", diff --git a/src/models/lightning_anp.py b/src/models/lightning_anp.py index edaa347..b4d7da8 100644 --- a/src/models/lightning_anp.py +++ b/src/models/lightning_anp.py @@ -28,7 +28,7 @@ class LatentModelPL(pl.LightningModule): context_x, context_y, target_x, target_y = batch y_pred, losses, extra = self.forward(context_x, context_y, target_x, target_y) y_std = extra['dist'].scale - loss = losses['loss'] + loss = losses['loss'].mean() tensorboard_logs = { "train_loss": loss, @@ -45,13 +45,13 @@ class LatentModelPL(pl.LightningModule): context_x, context_y, target_x, target_y = batch y_pred, losses, extra = self.forward(context_x, context_y, target_x, target_y) y_std = extra['dist'].scale - loss = losses['loss'] + loss = losses['loss'].mean() tensorboard_logs = { "val_loss": loss, # This exact key is needed for metrics "val/kl": losses['loss_kl'].mean(), - "val/mse": losses['loss_mse'].mean(), "val/std": y_std.mean(), + "val/mse": losses['loss_mse'].mean(), } return {"val_loss": loss, "log": tensorboard_logs} @@ -69,7 +69,7 @@ class LatentModelPL(pl.LightningModule): # agg and print self.train_logs HACK https://github.com/PyTorchLightning/pytorch-lightning/issues/100 train_logs = self.agg_logs(self.train_logs) - train_logs_str = {k: f"{v}" for k, v in train_logs.items()} + train_logs_str = {k: f"{v.mean()}" for k, v in train_logs.items()} self.train_logs = [] print(f"step val {self.trainer.global_step}, {tensorboard_logs_str} {train_logs}") return logs