diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..2d1bbac --- /dev/null +++ b/LICENSE @@ -0,0 +1,174 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. diff --git a/docs/lstm_mse1.png b/docs/lstm_mse1.png new file mode 100644 index 0000000..f19cb29 Binary files /dev/null and b/docs/lstm_mse1.png differ diff --git a/readme.md b/readme.md index adf79a0..86f0fdc 100644 --- a/readme.md +++ b/readme.md @@ -20,6 +20,8 @@ This repository has lots of options so you can run it as a ANP-RNN, or ANP or NP I've also made lots of tweaks for flexibility and stability and [replicated the DeepMind ANP results](anp_1d_regression.ipynb) in pytorch. The replication qualitatively seems like a better match than the other pytorch versions of ANP (as of 2019-11-01). You can see other code repositories in the see also section. +It's not heavily documented, because most of my code never gets read or used. If you are using it, and it's confusing, make a github issue are we will add comments or docs together. + - [Neural Processes for sequential data](#neural-processes-for-sequential-data) - [Experiment: Comparing models on real world data](#experiment-comparing-models-on-real-world-data) @@ -37,7 +39,9 @@ I've also made lots of tweaks for flexibility and stability and [replicated the - [Smartmeter Data](#smartmeter-data) - [Code](#code) - [ANP-RNN diagram](#anp-rnn-diagram) + - [Tips](#tips) - [See also:](#see-also) +- [Citing](#citing) ## Experiment: Comparing models on real world data @@ -192,6 +196,19 @@ Changes for stability: ![](docs/anp-rnn-mcdropout.png) +## Tips + +- Make you normalise all data, ideally the output two, this seems to be very important +- Batchnorm, lvar, dropout: it's unclear to me how to make these help reliably +- The deterministic path had unclear value, I found it best to leave it out +- The size and comparitive size of the context and target is important for performance. + - If the context is too long and complex the model cannot summarize it + - If the target is too long and complex hte model cannot fit it well + - If the context is in the target, the model may collapse to just fitting this. To fix + - make it small + - or make the loss on this part downweighted, this seems like the best approach since x_context->y_context may still be a usefull secondary task + - or do not include context in target + ## See also: A list of projects I used as reference or modified to make this one: @@ -206,14 +223,20 @@ I'm very grateful for all these authors for sharing their work. It was a pleasur Neural process papers: -- [2019, Attentive Neural Processes](https://arxiv.org/abs/1910.09323) (using attention to prevent underfitting) -- [2019, Functional Neural Processes](https://arxiv.org/abs/1906.08324) -- [2019, Recurrent Neural Processes](https://arxiv.org/abs/1906.05915) (2d and 3d over time) -- [2019, Spatiotemporal Modeling using Recurrent -Neural Processes](https://www.ri.cmu.edu/wp-content/uploads/2019/08/msr_thesis_document.pdf) (infilling spatial information, using a RNN for time information, no code) -- [2018, Conditional Neural Processes](https://arxiv.org/abs/1807.01613) [code](https://github.com/deepmind/neural-processes) -- [2018, Neural Processes](https://arxiv.org/abs/1807.01622) +- [2019-10-17, "Recurrent Attentive Neural Process for Sequential Data"](https://arxiv.org/abs/1910.09323) - LSTM on X before encoder, no code +- [2019-10-29, "Convolutional Conditional Neural Processes"](https://arxiv.org/abs/1910.13556). [code](https://github.com/cambridge-mlg/convcnp) +- [2019-10-01, "Wasserstein Neural Processes"](https://arxiv.org/abs/1910.00668) would be helpfull if the output dist never converges for your problem +- [2019-08-08, "Spatiotemporal Modeling using Recurrent Neural Processes"](https://www.ri.cmu.edu/wp-content/uploads/2019/08/msr_thesis_document.pdf) (infilling spatial information, using a RNN for time information, no code) +- [2019-06-13, "Recurrent Neural Processes"](https://arxiv.org/abs/1906.05915) (2d and 3d over time, using LSTM in encoder/decoder, no code) +- [2019-06-19, "The Functional Neural Processes"](https://arxiv.org/abs/1906.08324) +- [2019-01-17, "Attentive Neural Processes"](https://arxiv.org/abs/1901.05761) (using attention to prevent underfitting) [code](https://github.com/deepmind/neural-processes) +- [2018-07-04, "Conditional Neural Processes"](https://arxiv.org/abs/1807.01613) [code](https://github.com/deepmind/neural-processes) +- [2018-07-04, "Neural Processes"](https://arxiv.org/abs/1807.01622) Blogposts: -- [2018, Neural Processes as distributions over functions +- [2018-08-10, "Neural Processes as distributions over functions" ](https://kasparmartens.rbind.io/post/np/) + +# Citing + +If you like our work and end up using this code for your reseach give us a shout-out by citing or acknowledging diff --git a/smartmeters-ANP-RNN-mcdropout.ipynb b/smartmeters-ANP-RNN-mcdropout.ipynb index 2c6e7da..302e62b 100644 --- a/smartmeters-ANP-RNN-mcdropout.ipynb +++ b/smartmeters-ANP-RNN-mcdropout.ipynb @@ -1227,9 +1227,26 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 1, + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-16T13:45:00.767510Z", + "start_time": "2020-02-16T13:45:00.758163Z" + } + }, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'loss' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mloss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;31mNameError\u001b[0m: name 'loss' is not defined" + ] + } + ], "source": [] }, { diff --git a/smartmeters-lstm-deterministic.ipynb b/smartmeters-lstm-deterministic.ipynb new file mode 100644 index 0000000..8f6beca --- /dev/null +++ b/smartmeters-lstm-deterministic.ipynb @@ -0,0 +1,857 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# This notebook uses pytorch lightning & optuna & an LSTM\n", + "\n", + "This notebook trains an LSTM on timeseries data from smartmeters.\n", + "\n", + "It uses pytorch lighting for the training loop. And Optuna for the hyperparameter optimisation.\n", + "\n", + "It also pushes results to the tensorboard hyperparameter dashboard for examination.\n", + "\n", + "- https://github.com/optuna/optuna/blob/master/examples/pytorch_lightning_simple.py" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-16T02:36:50.055558Z", + "start_time": "2020-02-16T02:36:49.665610Z" + } + }, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "%reload_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-16T02:36:52.060467Z", + "start_time": "2020-02-16T02:36:50.058998Z" + } + }, + "outputs": [], + "source": [ + "import sys, re, os, itertools, functools, collections\n", + "import pandas as pd\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import collections\n", + "from pathlib import Path\n", + "from tqdm.auto import tqdm\n", + "\n", + "import pytorch_lightning as pl\n", + "import optuna\n", + "from optuna.integration import PyTorchLightningPruningCallback\n", + "\n", + "\n", + "import math\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-16T02:36:52.110420Z", + "start_time": "2020-02-16T02:36:52.063895Z" + } + }, + "outputs": [], + "source": [ + "import torch\n", + "from torch import nn\n", + "import torch.nn.functional as F" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-16T02:36:52.251225Z", + "start_time": "2020-02-16T02:36:52.113801Z" + } + }, + "outputs": [], + "source": [ + "from src.models.model import LatentModel\n", + "from src.data.smart_meter import collate_fns, SmartMeterDataSet, get_smartmeter_df\n", + "# from src.plot import plot_from_loader\n", + "from src.models.lstm import SequenceDfDataSet, LSTM_PL, plot_from_loader\n", + "from src.dict_logger import DictLogger" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-16T02:36:52.301399Z", + "start_time": "2020-02-16T02:36:52.253328Z" + } + }, + "outputs": [], + "source": [ + "import logging\n", + "logging.basicConfig(stream=sys.stdout, level=logging.INFO)\n", + "logger = logging.getLogger(\"smartmeters.ipynb\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-16T02:36:52.350206Z", + "start_time": "2020-02-16T02:36:52.303615Z" + } + }, + "outputs": [], + "source": [ + "# Params\n", + "device='cuda'\n", + "use_logy=False" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Load kaggle smart meter data" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-16T02:37:01.154608Z", + "start_time": "2020-02-16T02:36:52.352543Z" + } + }, + "outputs": [], + "source": [ + "df_train, df_test = get_smartmeter_df()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-16T02:37:01.885138Z", + "start_time": "2020-02-16T02:37:01.158353Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEGCAYAAABrQF4qAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO2dd5gUVdaH30OOIlEF1EFERVBREWVxTRhAVl0/V9fsurrornGDirsqCAZWd13XHDGwiiIGQJCkICpxiJJzGIacGWZgwvn+qOqhZqZnumemuru657zP089033ur6tc11adunXvuuaKqGIZhGKlLtUQLMAzDMGKLGXrDMIwUxwy9YRhGimOG3jAMI8UxQ28YhpHimKE3DMNIcczQG0YpiMhdIvKi+z5NRFREaviw3wtEJKMc7d8XkafKqFcROb6UuitE5NOK6DRSBzP0hhEGEakFPAY8H0XbsSLyiOdzK9f4his7shwalorICeXV7kVVRwIdROTUyuzHSG7M0BspiQ8976uAJaq6IYq2k4HzPJ/PA5aEKVuuqpuiObiItAWqq+qyKPWWxRCgtw/7MZIUM/RG3BGRliLyuYhsFZHVInK/W95PRIaKyIcisldEFopI50jbebYdJiL/E5E9wO9EpK6IfCAiO0VksYg8HHKZiMhDIvJ5MV0vich/3Y89ge/L+A7XiMgaEemIY+i7iUjo9/RL4EWgc7GyycX28VcR2SIiG0Xk9mKH6AWM9nxuLCKj3PMy3b0ReLlYRJaLyC4ReVVExFM3yd2fUUUxQ2/EFdfwjQTmAa2A7sCDInKZ2+RK4BPgcGAE8EqU24HTCx/mbvsR0BdIA44DLgFu9rT9H9BDRA53918DuB740K0/BVhayne4HfgncLGqLgBmALWB09wm5wHjgRXFyryG/kigkftd7gBeFZHGnvrLgVGez9cDTwKN3f0+XUzWr4CzgFOB6wDveVkMpInIYeG+j5H6mKE34s1ZQHNV7a+qB1V1FfA2jiED+FFVR6tqPjCYQ4Yy0nYAU1X1K1UtUNVsHIP3jKruVNUM4KVQQ1XdiGN4r3WLegDbVHWW+/lwYG8Y/Q8CDwEXqOoKd18HgOnAeSLSBGjk6vvBU3YyRZ8QcoH+qpqrqqOBfcCJACJSz/2+kzztv1TVGaqah3MT61RM10BV3aWq64CJxepD3+PwMN/HqAJUOoLAMMrJsUBLEdnlKauOYxTXAl4f9n6gjtvbLmu7EOuLHatlsbLi9R8Af8S5YdyMc2MJsRNoGEb/QzgGunjUTMhPvwb4yS37EbjdLVuvqms97be7RjvEfqCB+747MMW9gYQofl4aUJSy6kPfw3vujCqE9eiNeLMeWK2qh3teDVX1ch+2K56KdSPQ2vP56GL1XwGnun72X+H0lEPMB8JFvFwKPCYi1xQrn4zjhz+PQzefn4BulHTbROJyivrnK0t7YI2q7vFxn0YSYYbeiDczgL0i8og7WFpdRDqKyFkx2G4o8KiINBaRVsC93kpVzcHx6X8MzHDdHiFGA+eH2edCHDfPqyJypad8Ko5r5GZcQ6+qO4Gtbll5DH1PivrnK8v5wDc+7s9IMszQG3HF9b3/CseHvBrYBryDMzDp93b9gQy3/QQco36gWJsPcAZeBxcrHwmcJCItw2iZ52p5W0R6umVZwCygFrDA0/wHoAVRGnr36WJfsZtOZbkBeNPH/RlJhtjCI0ZVQUT+CFyvqud7yo7BiXk/srhrQ0R6Ayer6oNx1Pgw0ExVH/Zpf1cAt6jqdX7sz0hOzNAbKYuIHIUTWjkVaIfjDnlFVUNpDaoBLwCHqervEybUg4hcB/ysqosTrcVIHczQGymLiByLY9zb4EScfAI8qqoHRaQ+sBkn0qeHqhaPyDGMlMEMvWEYRopjg7GGYRgpTiAnTDVr1kzT0tISLcMwDCNpmDVr1jZVbR6uLpCGPi0tjfT09ETLMAzDSBpEZG1pdea6MQzDSHHM0BuGYaQ4EQ29iBwtIhNFZJGbH/yBMG3EzeW9QkTmi8gZnrrb3DzZy0XkNr+/gGEYhlE20fjo84C/qupsEWkIzBKR8aq6yNOmJ86ElHbA2cDrwNlueta+QGechFOzRGSEmwPEMAzDN3Jzc8nIyCAnJyfRUmJKnTp1aN26NTVr1ox6m4iG3s3bvdF9v1dEFuMsluA19FcBH6oTlD9NRA53ZyVeAIxX1R0AIjIeJyHUkKgVGoZhREFGRgYNGzYkLS2NogtspQ6qyvbt28nIyKBNmzZRb1cuH72IpAGn4yyy4KUVRXN9Z7hlpZUbhmH4Sk5ODk2bNk1ZIw8gIjRt2rTcTy1RG3oRaQB8DjwYi7zWItJbRNJFJH3r1q1+7x6A9DU7yC+wmcCGkaqkspEPUZHvGJWhF5GaOEb+I1X9IkyTDRRd1KG1W1ZaeQlU9S1V7ayqnZs3DxvzXymmrdrOb96Yyhvfr/R934ZhGEEmmqgbAd4FFqvqC6U0GwHc6kbfnAPsdn37Y4FL3YUfGuOszjPWJ+3lYuPubACWbw63DKhhGEbl2LVrF6+99lq5t7v88svZtSu2qzxG06PvBtwCXCQic93X5SJyt4jc7bYZDazCWZ3+beBPAO4g7ABgpvvqHxqYNQzDSCVKM/R5eXlhWh9i9OjRHH54bNdtjybq5kegTKeQG21zTyl1g4BBFVJnGIaRJPTp04eVK1fSqVMnatasSZ06dWjcuDFLlixh2bJl/PrXv2b9+vXk5OTwwAMP0Lt3b+BQypd9+/bRs2dPzj33XKZMmUKrVq0YPnw4devWrbS2QOa6MQzDqAxPjlzIokx/Y0ZObnkYfa/oUGr9wIEDWbBgAXPnzmXSpEn06tWLBQsWFIZBDho0iCZNmpCdnc1ZZ53FNddcQ9OmTYvsY/ny5QwZMoS3336b6667js8//5ybb7650toDaejzLDLGMIwkp0uXLkVi3V966SW+/PJLANavX8/y5ctLGPo2bdrQqVMnAM4880zWrFnji5ZAGvrFG32P3jQMowpRVs87XtSvX7/w/aRJk5gwYQJTp06lXr16XHDBBWFj4WvXrl34vnr16mRnZ/uixZKaGYZh+EDDhg3Zuzd8VN/u3btp3Lgx9erVY8mSJUybNi2u2gLZo48FtmKiYRixpGnTpnTr1o2OHTtSt25djjjiiMK6Hj168MYbb9C+fXtOPPFEzjnnnLhqqzKGPkRVmDlnGEZi+Pjjj8OW165dm2+++SZsXcgP36xZMxYsWFBY/re//c03XVXGdfOXofMSLcEwDCMhVBlDbxiGUVVJSUN/5oDxXPqf7xMtwzAMIxCkpI9+e9ZBtmcdTLQMwzCMQJCSPXrDMAzjEGboDcMwUpwqZ+gtuNIwjFhQ0TTFAC+++CL79+/3WdEhqpyhj5alm/bSqf84tuxJ7YWGDcPwhyAb+oiDsSIyCPgVsEVVO4apfwi4ybO/9kBzVd0hImuAvUA+kKeqnf0SXmGi7NK/99Nqdu3P5dslW7ihyzGx1WQYRtLjTVN8ySWX0KJFC4YOHcqBAwe4+uqrefLJJ8nKyuK6664jIyOD/Px8Hn/8cTZv3kxmZiYXXnghzZo1Y+LEib5riybq5n3gFeDDcJWq+jzwPICIXAH8udjiIheq6rZK6jQMw4ieb/rApp/93eeRp0DPgaVWe9MUjxs3jmHDhjFjxgxUlSuvvJLJkyezdetWWrZsyahRowAnB06jRo144YUXmDhxIs2aNfNXs0tE142qTgaiXRXqBmBIpRQZhmEkOePGjWPcuHGcfvrpnHHGGSxZsoTly5dzyimnMH78eB555BF++OEHGjVqFBc9vsXRi0g9oAdwr6dYgXEiosCbqvpWGdv3BnoD1DryeL9kGYZRFSmj5x0PVJVHH32Uu+66q0Td7NmzGT16NI899hjdu3fniSeeiLkePwdjrwB+Kua2OVdVzwB6AveIyHmlbayqb6lq50D48T1Y1kvDMKLBm6b4sssuY9CgQezbtw+ADRs2sGXLFjIzM6lXrx4333wzDz30ELNnzy6xbSzwc2bs9RRz26jqBvfvFhH5EugCTPbxmDHDklwahlEevGmKe/bsyY033kjXrl0BaNCgAf/73/9YsWIFDz30ENWqVaNmzZq8/vrrAPTu3ZsePXrQsmXLhA3GRkREGgHnAzd7yuoD1VR1r/v+UqC/H8czDMMIIsXTFD/wwANFPrdt25bLLrusxHb33Xcf9913X8x0RRNeOQS4AGgmIhlAX6AmgKq+4Ta7GhinqlmeTY8AvnTzv9cAPlbVMf5JNwzDMKIhoqFX1RuiaPM+Thimt2wVcFpFhcUKsbmxhmFUMWxmrGEYKYNWgeiJinxHM/QRUFL/wjGMVKBOnTps3749pY29qrJ9+3bq1KlTru0Cm4/+hrem0b19C+785XGV2s/Q9PX8sNwm5hpGqtO6dWsyMjLYunVroqXElDp16tC6detybRNYQz911XamrtpeKUM/d/0uHh42v1I6zKdvGMlBzZo1adOmTaJlBJKkdt20/fto0vqMYuveA2HrLfOkYRhGkhv6/ALHF/fh1DVRb2MToQzDqGoE1nWTKLbtO8DenLzCzzYYaxhGspPUPXq/2JuTS4H7dHDOM99y4b8mESlx/ZptWSk9um8YRupQ5Qx9cfO9M+sgp/Qbx4vfLgcgryCy8Z61dgcX/GsSH01fFwOFhmEY/pIShr4ybvftWc5A7qj5mVFvs3Krk+lh7vpdlTiyYRhGfEhKH/3a7Vks37wvYrvS+uZ7cnLZuCuHE49sWFi2abdF6BiGkZokpaE///lJldr++jensWjjHtYM7FVYlnUwP2xbc8MbhpHspITrprws2rgnYhsLwzQMI1VIDUNvVtkwDKNUIhp6ERkkIltEZEEp9ReIyG4Rmeu+nvDU9RCRpSKyQkT6+Ck81kTjsjG3jmEYyUA0Pfr3cRb9LosfVLWT++oPICLVgVdx1os9GbhBRE6ujNjYcOhpIFzKhHAPC/b8YBhGMhHR0KvqZGBHpHZh6AKsUNVVqnoQ+AS4qgL7iUh5DG9ZXp5wHXTrtRuGkez45aPvKiLzROQbEenglrUC1nvaZLhlYRGR3iKSLiLpPmkKy4G8gqjamdvfMIxUwQ9DPxs4VlVPA14GvqrITlT1LVXtrKqdfdDk7rNk2fC50U+MMgzDSAUqbehVdY+q7nPfjwZqikgzYANwtKdpa7esSjNr7U7+OnSe5ckxDCNuVNrQi8iRIo6jQ0S6uPvcDswE2olIGxGpBVwPjKjs8cKxMHMPd36QTm5+dG4ZL14XjV/emkWZe3jgkzmFaZS93PrudD6fnVHqBC3DMAy/iTgzVkSGABcAzUQkA+gL1ARQ1TeA3wB/FJE8IBu4Xp3uap6I3AuMBaoDg1R1YSy+xITFmwFYvS2LE45oGKF1+Sir311aCuN7Pp7N6m1ZPNC9Hcc1b+CrHsMwjPIS0dCr6g0R6l8BXimlbjQwumLSEktZvXuxkVrDMJKI1JgZaxiGYZRKUiY1K40rX/mRS04+suI78KmjbgOthmEEiaTo0X8+K4O0PqPIyS17ADMnt4CR87zhk5ENrhR574+lDx21LBeP3QwMw4gXSWHo/zNhGQBb9x5IsJLyEc7Mm3/fMIx4kxSGPpHk5RewYVc2Q9PXl3yqKKVTbp11wzCCREr56GPBkyMX8eTIRdSvVR2AXftzo3bwWOfdMIwgUKV79AfzCrjzw/CpdXZn5xb5vD/C+EC0mG/eMIx4k1SG/oFP5vi6v8Ub97DKXegbivbAv56/sdTtIpnq0iZSGYZhJIKkMPQhAzx73a7YHieKNk8MX8COrANlbhDqtIeL4qnqg7H7D+bx16Hz2Jl1MNFSDKPKUKV99Nuzyh/FM27RZsYt2hxV2ypu08PyyYz1fD47g4Z1atDvyg6RNzAMo9IkRY++okRyh//+/Uqmvo/SQ3PN61O48e1plTtWimBOLcOIP1W6R18ehOiNVPEbzKy1O/2WYxiGETUp3aMvr+ukzJmscTh+WSzfvJehM9dHbhgnsg/ms277/kTLMAwjCgJv6Bdv3ONbaoLK4HdUZHl3d8l/JvPw5/OLlM1et5O0PqOYsboiS/pWjt6D0znv+Ynl3i4UXmrjF4YRPyIaehEZJCJbRGRBKfU3ich8EflZRKaIyGmeujVu+dyKrgXb878/FPm8fPPeiuwmrpQWK787O/rJVtHw4/JtAPywfKuPe42OH9xjG4YRfKLp0b8P9CijfjVwvqqeAgwA3ipWf6GqdvJrLdhL/jPZj93EheKuoFvfnZ4gJYZhVGUiGnpVnQyU6htQ1SmqGhptnIazNmxSUt7edmnul9LKl23eR7ZPM2yLk7krO7Czbm94axpvT15VpCwI7jjDqCr47aO/A/jG81mBcSIyS0R6l7WhiPQWkfSKunjCUV6757ffuPjusnPzyXPXkfXzUHPW7eIXA7/j0wAN1nqZumo7T49enGgZhlFl8c3Qi8iFOIb+EU/xuap6BtATuEdEzitte1V9S1U7h3PxVGTR70QSzQ3Gj7536GaxfIszbjFzjYVxGoZREl8MvYicCrwDXKWq20PlqrrB/bsF+BLoUpH9b9yd44fMiPjt+Yh3ZMnnszPie8BKMOin1YmWYBhVhkobehE5BvgCuEVVl3nK64tIw9B74FIgbOROULjZBksNw0hBogmvHAJMBU4UkQwRuUNE7haRu90mTwBNgdeKhVEeAfwoIvOAGcAoVR0Tg+9QKuUd+FyYuceX44ayV5Y14HjruzPYvKdyTyrJGIse0PFiw0hpIqZAUNUbItTfCdwZpnwVcFrJLeJHrCJcIlGYvbIMQzx3/S5en7TSEnsZhhFzLNdNJYgUzpi5K5v8gth1YZNsjNowjASR0ob+mVGJCenb4i5ifvv7M9m1PzdC64qx/2Be4aLphmEYZRH4XDeVIetgYlw3Ifbm5MVs33uyD+07SH7vL+dkMHlZ6SkZbPUtw4g/KW3oU5GTnxhDfoGyauu+REsJy58/ncetg2YkWoZhGB7M0MeQSP75ikTN7D+YT05uPrPXxW5y1PhFm7ktSmMd1LQLsSS/QNl3IHZPa4bhN2bofeL378/kvOfKn7a3ongTpoXGBADmFLsBZOzcz8LM3eXa9x8+TOf7ZVtp/7j/0bC5+cl/Y+jz+Xw69h1bJW9yRnJiht4nvluyhXU7Er8Qx9WvTSny+dx/TqTXSz9WaF/lDU8d9GPZs1237TvA82OXVkiLX2zanVNpt9dns5JnBrJhQIpH3aQqQZ0o1f/rRWXW3/mBb/nqKsw5z34LwJqBvRKsxDDihxn6JKB4FIsgURn7nARNGAuR1mcUN519TOHnTXHKWRQvVIN70zUML+a6CTjfLdkcNoolmnzuJ8XAx15ePpq+rvD9pkqmfDAMo2KYofeBX738Q+RGFWTzngMlykSsJxkEbCjWSBbM0PvAgg0VS4ZWvFeeuSubXz73HRt2ZVdKz+Nf+ZckNJZhhLPW7mRo+nqLXjGMGGOGvhL4bZ4+nbme9TuyGRrFSlFldegHT1vrm6Zh6WVrKc1Gb91b8kmkOA8Pm8fDw+YzOUkXGrcblAGwYVd24BdHisrQi8ggEdkiImG7iuLwkoisEJH5InKGp+42EVnuvm7zS3gqMOin1aT1GcUTw0vvgYcz6CKwO7vsHDo9XkzsIuqPfD4/YpvQzSAriScffTpzHQ98MifRMowEsTs7l24Dv/P1KToWRNujfx/oUUZ9T6Cd++oNvA4gIk2AvsDZOKtL9RWRxhUVm6p8ONXpgXv7hzuzDrInJ7wxF4TXJq0sc59LNu31RVvxPuvqbVmk9RlVpOzFCcu49+PZRcqiifgJTfoS4MOpa0jrM4qCGGb79IOpKwsXUEOBRz7/meFzMxMnyEgooU7KpKWl53cKAlEZelWdDOwoo8lVwIfqMA04XESOAi4DxqvqDlXdCYyn7BuGgdNjP33AeDoPmMCBvJKPhBUZiL3Fp9WzxizYVKLsxQnL+Xr+xnLvK+T6ePm7FTwxfCEAuQXxeQSes24nCzN3l9v9MnzuhsL35rkxkiUowi8ffSvA68zNcMtKKzfCUcxyHMwvoO+Ihb7s+ocA+8EXbfRnZa/y8NyYpfR66ceEzXL955gl/PnTuQk5tuE/Qc/KGpjBWBHpLSLpnqUIqxyhSyWaGPlUQMJ0hyYu2cq9H8+Oehbtgbx87h48i9XbsiK2XeRZKnLVNicNwrIoXVxDZqzjs2ID05X5cb8+aSVfztkQuaERaEK/1aif7goK4LunISu+HS+/DP0G4GjP59ZuWWnlJVDVt1S1s6p29klT0hFK+LVlb9kTiyp6Gzi139hyJzjzMmXFNv45ZkmFt4+Gu/83i6/nb2TC4s3MWF2Wt9Bh+qodjFm4qcwB7RCXv3RovkO4+Qll8egXP/PQsPlJ86huBJS1P8Lk52DEfXE9rF+GfgRwqxt9cw6wW1U3AmOBS0WksTsIe6lblhL47aOdtHQLUHQ2aTjyK3jgPTl5RRKc7cnJZfu+6Azelr053PhOST//cX8fXSEtEDly6IvZkd0qIcMb6ZTs9mmlrxVbDiVEK37M7fsOsCPrYOHnjbuz6frst6zbXnayu0e/mM+b3zuD69kH89m4u3LzKAx/2JOT638akXz3OsyNbwLEaMMrhwBTgRNFJENE7hCRu0XkbrfJaGAVsAJ4G/gTgKruAAYAM91Xf7fMCEO0kTKdB0zw5Xhdn/mWM58qe18hY/Z9OaMKpniiUypKNPez0KNzQRmN127P4rT+4yqtB2BNKUZ72KwMznxqAmcMGF9Y9sXsDWzcncN5z0/knR9WlbrPITPW8+w3zpPSTe9Mo+uz3/mi1agcp/YbV+QpMByFHY2o93rIQUtBPiwaHpdR/Wijbm5Q1aNUtaaqtlbVd1X1DVV9w61XVb1HVduq6imqmu7ZdpCqHu++3ovVF6lK7K1k3Pn6Hfu5a3B64VKL3y3ZHHGbWK19WxbhjHdBgfLVnA3kuRNUqnl69Ms376Vj37FkFptZXJpx9pO/fTavzPpo0zPPXrer8P3v3pvBs98kZt1jw2HV1tLHfsYu3MRHxScn/vgf2PRz5B2LwE//haG3wqKvYNk4eK4t5LrXboG/TxKBGYyt6sxbvytyI594cuRCxi48ZNx//346GTvDG8MdWQe5a3A6T4+Ov8EJF1L/xZwNPPjpXN4J5b6XUFvlf9PWsu9AHuMWFg0BnbWmcg+R3lmP3hm/5emIeX37eVHMoly+eS+Tlm7lze9XsXlPDr949tuoBpyN+HHX4Fm89N0KwL0WVnwLE/rBG+eWvpH3mtnjDldmbYPxj8P+bbBzDezeAP2bwOwPfdNqhj4gXPXqT3E71oTFW0qUrdiyj4HfLCkRV/7KxBVFbgrxJFyM+44sx9Bucw1uNdeCKocGWItvFfoxlkV+gfLW5JVhfbK3vzcz7DZ5FYz5XxvFAjX7PQvbj5yXSebuHAZP9S+1RTJz/5A5XPSvSYmWUQyFrFLcm6rw1gWOm8brugkx+m9F229f7vz9+TPf1JmhNwD400ezeeP7lSze6M+MWj8oy+8eIvRzUVXGeHryfxk6l5e/XR5x+3d+XM1Xczbw1ZwNPDN6Cf+ZsKxI/ay1O/lxRfhQuMfKMe29MiGzT40y942XEfMyWRXHp5t/j1vK/oOR3KWl/H8n/wv+dw1kzoHP7/Q0F9jqced5r/UY+Oxt4ZFKEO7fEfQp/KUR6kEGaeKH91TmFyjjFxV9spiychs3vu1EAhX/nXwx23ksvq97u4jHedAzcWlfzqEf9MSlW0rtzQN8E2aWsJdoQjG96SS2RYiA2pF1gGWb93LCEQ0j79jwjZe/W0FegfJIj5NKbdO9YAqsDBMl9t2Aop+9F+oaz0DvtpDRF1j6jfP2wF7o1wiufBnOuBUy0qFuY2ja9tB2ezc7YwLtLi7zO5ih95n/TbfHa78YMS+Tbsc3BWDn/lwGfrOEIw6rDTghpq9OPOSS8fb+K3OrUpywz4HfLCataf0y2x4Mk56iNKIx+t6cOeG+w1dzM/lqbibLnupJrRr2MO43I+dl0unowzm6Sb0SdQdyi/6vPyr2O/9nwb8hch4/wrpuijPjTefvRneHI+6DRq1h8NXO5367Yf0MaNwGBl3q+PX7lT0/xq4Wn8nYmdwx0EHL3/LI5z/zyOc/s8E9ryE//Hs/reGnFUUTjIXj3+PKvxj5axNXMGTGel/TPQPMXLODtyevKvUcD4iw5m6If49P7ALrqcp9Q+Zw5Ss/llr/7eLNpPUZxZY9OfzjywhuuymvwL6SY2F86/bwS7vz53nsh3rGi0JGHmDWB/DuJfD2hY6RjwLr0VeCcP+qtyaXHi9tVJxILiXverTeDJMvRzEQ60U4dNPw86YnwLVvTAXgwpNaRGw/tYx5CJm7bElGPxmzYCNz1zs94p2lhBErTlQXwIJis8ubs7PkBuP+AcvCLOW52Q29XF7KvI63LogseOT9zt/dkdetCGE9+kow+ufyZ2w0KkYko7vRY+gnLK54lNBH09cVRt5UNKomEsXHGsJRVqoJW/DEX+7+nxOIUBZl3Xhn1rknfMWBOCbr++z2MqvN0FeCvCQdeC2Ln0qJMEk08TzTofUBypsPJ8QL45eR1mdUkdQH3gRusc4XVFUYGmH1Mz/xa32HmLHwizKrzdAbRQhNxQ8aydSLfckN6/xuSRgfrQ+Ey/pZFXl42Py45gVaX97xt+IZKhN4DZuhN5KCJLLzhWzxzKL1c5H1ZLrpxZrcvMqdi9JmhIfDm9AuKvYUS9RbEP80IiHM0BtJgdm2omQfzOf1SSvJT0H3YTz500ezS5RFs7B9smFRN0ZS8Gkc/bFB5+v5GwuXbmzRsDbXnNk6wYqSg6wDeXw8fR13nNuGam42vHBzIR74ZA5r45AIL56YoTeMJCYnz+d86SlEfoHS9u+jefLKDlxzZms69nWWwji6SV16dDyq1O0ipdieHsWCOEEj2nz0PURkqYisEJE+Yer/IyJz3dcyEdnlqcv31I3wU7xhVHWquktLUdbv2E9an1GMKrZAfSjr6DOjF3PB8xMLyxdv3EvXZ79l8wu4TnMAAB+ASURBVJ6KzUd48/vkmysTsUcvItWBV4FLcBb3nikiI1S1cBqfqv7Z0/4+4HTPLrJVtZN/kg3DMA6x0F0LePjcDfQ6NXxPfdu+Qyt//deNiip+Y0hlounRdwFWqOoqVT0IfAJcVUb7G4AhfogzDKNsHvtqQWEUzsqt+6pcRM7a7fu552NnQLW0b36glJxEizfuIStiVsrUIBpD3wrwjoRluGUlEJFjgTaAdy20OiKSLiLTROTXpR1ERHq77dJLa2MYRkmyc/OZtXYH3f/9feFkr6rC48MXlIg8Gj53A3OjWMjns1kZrN+R3LmposXv8MrrgWGq3mw8HKuqnYEbgRdFpG24DVX1LVXt7LY1DCNKBClc8q7viIUJVhNfvNExoWlkD3wyl1+/+lNU6xlUFaIx9BuAoz2fW7tl4bieYm4bVd3g/l0FTKKo/94wjEoycl5mEaMWTW82FRm3aDNPeTKAnvzE2ASqCRbRGPqZQDsRaSMitXCMeYnoGRE5CWgMTPWUNRaR2u77ZkA3ILpcrIZhRMXDn88v4rIJtxxiVaFwLWGjCBGjblQ1T0TuBcYC1YFBqrpQRPoD6aoaMvrXA59o0dGg9sCbIlKAc1MZ6I3WMQzDH6Kdzfn4VwsY/fNGjmten8/u/kWMVfnPN5YxtkJENWFKVUcDo4uVPVHsc78w200BTqmEPsMwfCS0mMr2rIMRWgaTP4ZJWWBExmbGVpIR8zIjNzKMGLMlBvlZ9h3Io36t6oHIlnnfkDlc3D7ygi1GeCypWSW5f8icREswjCL4YZY37c6hY9+xvP1DMGaBjpyXyQOfzI3c0AiLGXrDqMIcyMsvTBXgZcMuJ2zxmwWb4i3JiAHmujGMKkDXZ7/l2Kb1SpSf+NgYjm5Slx8evqhIeSikIvFOG9iRpOMJQcIMvWGkGL99axoAsx67mKYNagPOmrredXUB+rmTq8LNDg2FzgXBP//kyKo1CSwWmOvGMFKUBZl7KChQhs4Mn8v//SlrIu6jomb+xQnLmL1uZwW3Lkq4nPFG+TBDbxgpysQlWzju76N5+PP55d62tOwBW/ceYMiMdRG3f3HCcv7vtSnlPm44Zq3154ZRlTFDbxgpSjQ99hDFJ1yF5j0W99zcNTidR7/4uVxrrUbDnpxc/jV2KXlhBoZjETpa1TBDbxgGQ4st1Viajz400erViSt58/uVEfc7cckW7oliktNfPp3LKxNX8NSoxdEJNsqFGXrDMFi5dR8/Lt8GwLiFmxi3cDMAM1bv4G+fzStsFzL7Q2as49lvlkTc7+3vz2RUGWkL9h3IY9CPq5mweAvgPIWE69UblcMMvWEYfDF7Aze/O538AqX34FkM+ulQcrBhszJI6zMKKNnDf/SL+dw1OJ2CgoqlBB4wchH9vy6a/uqvn80j60Aed36QzsbdVSNffKyx8ErDMAopaybsmAUbS0ThDJnhuHw69htLn54ncUOXY1i9LSvq4+3KLhkjP3xuJt2Ob8aExZuZtTb5FuIOImboDcMo5JMyImpWbs0qNd5y/8F8nhi+kP0H8xkYxqXz5ZwMWjeux1lpTQDHZTN91Xa+W7Il7P5CTwg79+eW8xsY4TBDbxhGIdll5LIfMmMdtWuU7e3dlxN+DdY/f+r4+dcM7AXAw8PmMfrn0tMr9Pni50hSjXIQlY9eRHqIyFIRWSEifcLU/05EtorIXPd1p6fuNhFZ7r5u81O8YRj+kptfuq89Y2e206svg2rVyp5iFRpoLcvIG/4TsUcvItWBV4FLcBYGnykiI8IsIPKpqt5bbNsmQF+gM07E1ix3W5sBYRgBpLJ5ZV76dnmZ9SPmZbJ5j8XFx5toXDddgBXumq+IyCfAVUS3JOBlwHhV3eFuOx7oQbF1ZQ3DqBr8Zei8yI0M34nGddMK8M6myHDLinONiMwXkWEiElpMPNptEZHeIpIuIulRaDIMwzCixK84+pFAmqqeCowHPijvDlT1LVXtrKqdfdJkGIZhEJ2h3wAc7fnc2i0rRFW3q2rI8fYOcGa02xqGYRixJRpDPxNoJyJtRKQWcD0wwttARI7yfLwSCCWsGAtcKiKNRaQxcKlbZhiGYcSJiIOxqponIvfiGOjqwCBVXSgi/YF0VR0B3C8iVwJ5wA7gd+62O0RkAM7NAqB/aGDWMAzDiA+ipSWeTiC1j2qnR932YqJlGIZhRGRNnRsTLQEAeXLPrNLGOC2pWQpxWB2b6GwYRknM0Cc5L99wOv+4vD0AXdo0TbAawzCCiHUBk5wrTmuJqnLHuW14aFj5l4wzDCP1sR59CiAiVKsmJZZ9Cwq3dj020RIMo0pjhj6F6NnxyIht+vQ8qcjnf117WtT7v797O/57fSeuOK1l1Nt8dOfZ9L+qIycd2TDqbQzD8Bcz9EnIZ3d3pUXD2iXKu7c/gjUDezHi3m4A3Nb1WGb8vTsXt29R2KaGJ7vgm7ecyW/ObF3qcV64ruhN4MHu7biqUysuPfmIsO2bNahF+mMX85/fHtqu2/HNAHjiipOj+GaGYcQCM/RJyFlpTZj6aHeWDOgRtv7U1oezZmAvnryqIy0Oq8M7t51VWHfmsY0BeP/2s7isQ8kngNXPXs7lpzjltTy5xzu2OqwwBa3XRfTRnWcXvv/xkYto1qA2V58e5uYRvChew6gy2GBsknHj2ccAUL2aUL1a9ai3m/vEJeTkFnBkozosfaoHtWsc2nbNwF58v2wrrQ6vg4gg7jJCqrCo/2XUqFatiNE//RjnZvHe786i2/HNeP43p/LM6MXUrH6oTc3qwm1d0wo/H16vVoW+r2EYlccmTCUZ3/71fNo2bxDTY6zZlsXDw+Yz6PazaFDbv77AP8cs4fVJK33bn2EEAZswZSQlac3qM/Turr4aeYBHepwUuZFhGL5jht4wDCPFMUOfZDSrXzLaJpm458K2iZZgGFUOM/RJxJIBPWhUr2aiZVSKhy47iQ9/3yXRMgyjShGVoReRHiKyVERWiEifMPV/EZFF7lKC34rIsZ66fBGZ675GFN/WiJ46NaOPsgky553QnIVPXsYlpcTjG4bhLxENvYhUB14FegInAzeISPHZL3OAzu5SgsOA5zx12arayX1d6ZNuI8mpX7sGb9+auFUjxzz4y4Qd2zDiTTQ9+i7AClVdpaoHgU+Aq7wNVHWiqu53P07DWTLQcFnU/zLWDOzFgF93TLQUw6VR3Yq7wEKTzgwjWYjG0LcC1ns+Z7hlpXEH8I3ncx0RSReRaSLy6wpoTHrq1XLCFI9pUi/BSoLH0qcOze6d3+9SXrvpjLgct6AS00eevrojZ7dpErHdDV0OLZfcJa0JXY8rO410m2b1S5T1veJkftHW0k8blcPXwVgRuRnoDDzvKT7WDeK/EXhRRMKGXYhIb/eGkO6npiBxrpv3JRxXdYo+UVgqUbtGde7v3o7h93TjsDo1ufyUoyJvVEHuPLdN4fum9Ss2U/fPF5/AiUc05L3bzyrMynlx+yOY/fglJdo+eWVHJvzlfL6+71yG3t2VIb3PYcY/ugMw5A/nUKdm0Z/f3911BUK8fMPp3N6tDR//4RzWDOzFhSc2B+AC968RDO45eH/FNmwev3klEWfGikhXoJ+qXuZ+fhRAVZ8t1u5i4GXgfFXdUsq+3ge+VtVhZR0zVWbGXtbhCM45rim3dztkYDo/NYFt+w4Uabfi6Z6MXrCJ+4fMKXN/awb2ionOoFFQoMzL2MXVr03xbZ8f3Xk23Y5vxqLMPbQ4rDbNGjhhqjm5+Zz0+BgAOrQ8jIWZe0rdx6j7z6VDy0aFn/PyC/hyzgauOaM11aoJ63fsZ/W2LF76djn9ruxAx1aNSt0XgKrS5tHRAMz4R3eaN6jN4GlrSWtan7PSmlC3VtHB95zcfDJ27uf4Fg1Zuz2L85+fxFu3nEnvwbMqdE6M8jHzHxdz1tMTipS9euMZjJyXyX9rvkztJV+Wb4fn/AnSB0FejvP57Lth+hvQuA3sXO2UnXIt/PyZ877XCzDjbajdADJmlthdWTNjozH0NYBlQHdgA85C3zeq6kJPm9NxBmF7qOpyT3ljYL+qHhCRZsBU4CpVXVTWMZPd0Pe/qgPXnNGa+mFmlubmF9DuH45n66UbTudgXgG/ObM1o+Zv5J6PZ5e536pi6EOk9RkVtnzArzvy+FcLot7PuD+fxwlHlJ4mecqKbdz4znT69DyJkfMyWZi5hy5pTZixpug69qufvRwJYNL/zF3ZXPvGVDbsyk60lJRmzcBeJa7JwXd04Zft3CesjHSo19TJ+rdvK7x7MbTt7hjwFRPg4n6waDh8/aBj3P+eCbXqQ34e5GZBrQZwcB/UaQQTn4WFX8C9JQ06c4fAV3c77zv8H7S9CEbcW6ahjzjHXVXzROReYCxQHRikqgtFpD+QrqojcFw1DYDP3B/COjfCpj3wpogU4LiJBkYy8snKcc3rc+VpLXnvpzXc6knmVZxQ4q9GdWtypSeve73aTu/tt52PZu76XSzdvLew7pmrT6FV47qxER5gnv/NqSVWzZr80IUc07Qee7JzeX7sUgBuOvsYPpq+rrDNRSe14LslzkNl3ytOLtPIA/zi+GYMu7srZxzTmN93a8OExZvp2fFIRIQflm/llndnMOzuroE08gAtD6/LT30uou/wBXwwdW2i5SQ1t5zjuOMGT3PO4/knNOf7ZVtJa+qMr13cvgUTFh9yWISeDAFo7bGxhx8L3fvC6bdAg+ZwwqVOeacboN2lkLPLMfIA1WtAdffpr47798JHnVc4TrkWVk+GLn+AVmdAQT5kzgZK7xxbUjOfOO3owxl+T7eo2q7cuo/G9WrRxOMnVlXen7KGazsfTX6+clr/cYV1Va0nX5ydWQf5cs4Gbu+WVsLYbtt3gGYNajNx6RaGpWfwh/OO46QjG/Ldki386aPZviSBy8nNT4o5DAszd9PrpR8BZy2CD6auZfAdXbjl3RkA9LviZPqNjL6fdcIRDVi2eR/v/e4sbn8/TM8yAu1aNGDYH3/Bgg27uemd6UXqXrnxdFo3rkfmrmz+9NFsPvh9F45rVp//TFhGvVrVefLKjrT9++gi2zx3zals2pPDC+OXFSnv0qYJObn5zM/YHVbHqzeeEfFpefg93Vi7Y39h5+vf45ZywYktOLpxXbo88y33XXQ8f730RPbm5PLlnA20a9GQWjWEM4+NPCgfL0Sk4q6bRJCMhv6kIxsy5sHzfNtf6BFx6F1d6RJFhIdhgHPdtD/qMEbffy4F6qSzXrBhNz8s38YfL2jLgK8XkbFzP2MXbgacdNIDrurI3pw8FmTuZvjcTACWP92T/ALlYH4Bh9WpyeRlW7l10IyIx/+k9zm0P+qwsOGroWu6R4cjeeOWMwvLc/MLiqS4DvHCuKW89N0Krj2zNX+59ASOalSX/AJlyaY9dGjZqHB/K5+5nAJV0tfs5LVJK/hh+bbCfTx9dUduOvtYvpidwV+GzmPcn8+jeYPanD5gPOCEPucVKIfVKT3cdvu+AzSuV6twPYagYoY+Dpx81GGMfsC/STj3D5nDiHmZfH3fuREH9QwjRG5+AdVEqF5Bo5RfoOTk5ocdX8o+mM/G3dlc9O/vOfPYxtx70fF0a9uMGtWEzXtzaFinZpkZT/fm5PL0qMU80uMkGkcR9bTvQB7Pjl7MP3q1LwxR9tLn8/m0bd6AP5x3XJHycQs30XvwLIbf043Tjj487L7HLNhEbn5BuZbFDDpm6ONAx1aH8fV9/hn6fQfy+G7JliJ+fMMwjNIoy9BbUjOfqObzQF2D2jXMyBuG4Qtm6H0iqBEZhmEYZuh9ombAB2oMw6i6mKH3iRrVzdAbhhFMzND7RI1qdioNwwgmZp18wu+FtA3DMPzCDL1P/KNX+8iNDMMwEoAZep842nLNG4YRUMzQG4ZhpDhm6A3DMFIcM/Q+YGuIGoYRZKIy9CLSQ0SWisgKEekTpr62iHzq1k8XkTRP3aNu+VIRucw/6f5wcfsWzPh7d/57facK7+P1m+OzzqlhGEZFiBgTKCLVgVeBS3AWBp8pIiOKLSByB7BTVY8XkeuBfwK/FZGTgeuBDkBLYIKInKCq+eUROfbB8/jdezPYuDuH535zKhec0JyFG/ewe38uU1dup26t6vS7sgOvT1rJ5j05bNydze9+0Yajm9SlZaO6jFnoLNP3n992YuLSLWzZc4A7ftmGxRv38KcLjgfgqk6tuKpTK3Zn53Lak+OKHP/jP5zNx9PX0bR+LfYfzOezWRmc1roR8zJ288PDF9KiYZ3yfB3DMIy44suasSIy1m0z1V16cBPQHOjjbettV9YxO3furOnpiVsjPL9Ayc7NLzM2Pje/gNz8grDpUw3DMOJNZbNXtgLWez5nuGVh26hqHrAbaBrltoGjejWJOAGqZvVqZuQNw0gKAjMYKyK9RSRdRNK3bt2aaDmGYRgpQzSGfgNwtOdza7csbBvXddMI2B7ltgCo6luq2llVOzdv3jw69YZhGEZEojH0M4F2ItJGRGrhDK6OKNZmBHCb+/43wHfqOP9HANe7UTltgHZA5IUnDcMwDN+I6GRW1TwRuRcYC1QHBqnqQhHpD6Sr6gjgXWCwiKwAduDcDHDbDQUWAXnAPeWNuDEMwzAqRyDXjE101I1hGEayYWvGGoZhVGEC2aMXka3A2iibNwO2xVBORQmiLtMUPUHUZZqipyrqOlZVw0ayBNLQlwcRSS/tcSWRBFGXaYqeIOoyTdFjuopirhvDMIwUxwy9YRhGipMKhv6tRAsohSDqMk3RE0Rdpil6TJeHpPfRG4ZhGGWTCj16wzAMowzM0BuGYaQ4ZugriIhIojUkC3auosfOVfTYuYoeM/QV53AozNYZGETkRhE5zX0flB9C4RJcAdIEgIgE7TfQAApXdgsEInKliLRNtI4wFJ6jIF1XAbymgm3oReTXIjIg0Tq8iEgjd6WsMVC40ErCEZGLReQH4EXgdABN8Ei7iFwqIlOAV0TkpiBogkLD9ZdE6wghDi1EZBLwDkAQkv+519RUnKSFRyVaTwgR6SUiE4AXROQ8SPx1FbRrqjiBM/TuRV9dRO4E/gX0EZFfJlqXh2xgF9BRRK6FxPW+3HNV180Q+hjwFDAMqJdIXe6xmwP9geeAj3DWEH7UrUvIdSciNUTkEeAl4F8i0klVCxLde3aNVI77OlVEekJizpN7TTUQkZE419RjwDTg2ERpKqYvDXgaeBlYDPR2bUWizlcgr6kSqGogX8AFQEPgD8CkROtxNVUHjgD+DPwK2OSpkwTqusrz/mZgaoLPkwAdgTc9ZSfjpLBulsjzBfwax5X0IDA9kefJo6mae34GAlcl+v/navqt5/29wNBEa3K1dAdecd/Xce3EPKCxWxb36yqI11TxV2B69CJyv4i8Hbo7A9+r6l5VfRuoLyJ3uO3iptmj6fciIuo8Tu8Beqnq18B8EXlCRDqqqsbLT+jR9QcAVR3ullcHVgMLReTosvYRA023icglrh4F9gG/EJEmbtkiYChOTyyeuu4XkYEicp1bNEpVc1T1RaCFiNzotquZAE3XAKhqAZAJnAD8BGwUkbtFpF0CNF3ravrULa8G7ATWi0jteOnx6PqNiJztKcoArhGR2u7/cRIwBXgijpoCd01FJNF3GveO+Ducx8MewPfAo0BbT31PYCHuXTtBmv4OtAVaAE+5bX6Ps6BKuvu5ZoJ0HeepPwVnVbCGcTpPjXHcRRuB+UB1T92HwOBibacDbeKgS3CevH7CWfVssXvuWnjaXA1siOM1VZqmJkBnoK/b7m9AFjDS/VwjAZqae9r8AlgSr/PkHrOFe31nAl8B1YpdVy969J/mXoNHVLVrKtpXUHr03YF/quoY4K84j0E3hSpV9RsO+eMahnodcdZUG7gWx0ffU0TGAfcD33EopXI8BmaL66qF464BQFV/xvH1Xh8HLajqTmAc0B6YRdGe1b1ADxE5y/2chfOYfTAOuhS4EHhMVYfh/EBPBS7ztPkSWCYifwNn8DEBmjoBlwCbgF+KyGjgdhxjssrdNGYDs6VoOg2nIxFqMwXIEJErY6UjjK4twHBXx0bgLk/1k8CvRKSDqz8H2IvzFBlLTYG7pqIl0QMroePPwfF5o6rpwFSglYh08zR/BHgWWA4cmSBNxwHnAuOBGaraSVUvBS4QkTbuhRBvXdNwztW5bjvBWfaxTqxdSZ79f6iqu4DXgP8TkWNdfXtwfpSPi8htOAN7HYjxD9JzrtKBX7paxuBcOx1E5ERP8z8Cz4nIJqBVAjQtxTGsp+O4JWaqagecG/UFItIqVtdVGZqW4Zynk9x2hwFLgNxY6ChD18s4y5COA3qJyFGuxpU4kUCvudf9zThPAAVx0BSYa6o8xNXQFzc86vgmwem9VAuFSgELcO7iLd3tjscxIl8BZ6iqb37ecmhaiPNDbAg8oaqPeTY7RlVX+6WpnLoW4Dzehn4EinPRZ/ltIMJoUvdvjvt3JvANTlREqM0rOCGfZ+JEblyrqrt91lUkwsFzrlYADUXkFPfz90AjnP8hItIJeBv4HOe6+iABmia7erYAd6tqX7f9DqCbqm5IgKbQeWrgttsDtMYJRPCd0nSpaq464ctTcG40D3jaPItj7O8ATgTuUNVsHzU18moLwjVVGeJi6EWki4i8DTwiTthdqDz0D16OY0h/KyLVVTUD56JKc+t3A/eq6v+pamaCNK3HufEcq6oHxQkBrQagqll+aKqgrgycJ5w0z27+pqqD4qCpWpjB8VeA40Wkg4gcISLHq+p3wJ9V9Ta//n/u8TuLyGDgCfFM6JFDk9hm4LjTLhWRGuoMCLfC8YcDbAf+pKrX+nhdlVfTQpwb4OmqmuNeVwKgqr48+fhwngCuV9X3/dAThS4p1qnYBowAThCR1uLMOWisqh8Cd6nqdaq6yQc91UTkMBH5GidcEnXnM3h+f3G/pvwgpobevWifxUnN+RNwBtBXRI6AIpNC9gI/4PjB/+WOVjfGOWmo6lZVXR4ATYd7NOV77vKJ1lV4rty2vvjAo9BUoE7McF0RCfX+1gFfAj/j9HYOK6bfD13VROQV4E3gW5ynmX6ujmpuLxBVXYHzqN0W6ONufgB3TEVV17tjGkHQtMatz/frScwvTW6bHD80RalLVVVFpLY40TX5qjoZp4OzAOe6aubq8m28x/0978UZ92olIr919dYIXb/xvKZ8RWM7Sl0T+BNwgvu5Fc5JSvO0eRL4DDgJ5x/+Po7f+U08ERyprCmouqLU1Bf4AjjV/XwDzkX/HDGMQgKuAQ5337fDicSo5akfgPNon+aerxE4g8Vv4ongME3x1xSlrieBwaFrDbgbx731zxhfV+1xJvhd4Z6Lhp66hJwrX75XDE7UOR7DUN3zz6zt/v0K6Oy+PxX4mKKhlNXwOTQwiJqCqssHTecQg/BJr65i5RfjzFQejzOT+mTgPFfX8Z52DULfxTTFV5NPui72fo7BtR5am6Mm8B5O0MB/gftwXGvnxutcxeLl50k7HBiF8+jzGNAgTJuGOOF1LcPU+X5HDKKmoOryQVOsnnSK66rvlod+mJ2By933/YFncAbH43muTFPsdMXiSTWsJreuK/Bf931vYCsw0vt7iNW5iuXLTx99fZywvvvc9+Hy03QBFqpqpjj5NNqBM/iiPvq7A64pqLoqqylWsd7FdRVJYqWq6ao62m07Gsdw7HB1VYvTuTJNsdMVi+sqrCaXdThRNZ8CDwOzgRXqDozH+FzFjEoZehG5VUTOF5HD1AkDewtnmnsOcLaIhMIjQ6P7jXGmUt+OM3uzE/ibeS6ImoKqK4iayqMrDGfihJqGBs78HCw3TUmsqxyaGgPNcSawnY4zNnCiiLT3W1NcqcBjj+AMBE7EGTF/C2fwopmnTTcc/9bNxbYdjDOp4T3cwTs/XkHUFFRdQdRUGV04UT2X4Nx4RhLGD2yaYqspqLrKqekWT5m3vgHQxM9zlYhXuXr04sRtK46vdoOqdseZBbYDz+rmqvoTTmjWSW5cagO3ahRwnarerqrzy3PsZNIUVF1B1FQJXY1EpI46k3kUJ//QFaq6zDTFT1NQdVVA04mupvqqus0NLa6mqvvUmbyW3ER5Z6yOM0jyT+B8nNCjDzz11XAedc4vdid8EedOvRk4ys87VBA1BVVXEDX5qKvEwLBpir2moOqqpKYZsTpXiX5F7NGLyPk4saKNcab/DsDJeXGhiHSBQr9VP/cVohdODPZc4BRV3RjpWNESRE1B1RVETT7r8nOmrWlKYl0+aJrnt6bAEMUd8pcU9V+9hvMI9DtglucueSTO4EaaW3YVcF4s7k5B1BRUXUHUFFRdpim5dQVRU1Be0Zy8ejjT7au7n28CnnXfzwXuc993BobERXQANQVVVxA1BVWXaUpuXUHUFJRXRNeNqu5X1QN6KJ71EpxJBODkzW4vThKgITgxpyWyHPpNEDUFVVcQNQVVl2lKbl1B1BQUakRu4iBO9jbFySo5wi3ei7PCUUdgtbopVdW9bcaaIGoKqq4gagqqLtOU3LqCqCnRlCe8sgAnD8Q2nJXqvwYeBwpU9Uf1MW92kmsKqq4gagqqLtOU3LqCqCmxlMfPg5MEqAD4ESfRf8J9T0HUFFRdQdQUVF2mKbl1BVFTIl+hxEJRISKtgVuAF1T1QHluKLEiiJogmLqCqAmCqcs0RU8QdQVRUyIpl6E3DMMwko+ELg5uGIZhxB4z9IZhGCmOGXrDMIwUxwy9YRhGimOG3jAMI8UxQ28YhpHimKE3DMNIcf4fTopKzmxkRmIAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "# Show split\n", + "df_train['energy(kWh/hh)'].plot(label='train')\n", + "df_test['energy(kWh/hh)'].plot(label='test')\n", + "plt.title('energy(kWh/hh)')\n", + "plt.legend()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-16T00:24:15.905017Z", + "start_time": "2020-02-16T00:24:15.747859Z" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Train helpers" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-16T02:37:01.954099Z", + "start_time": "2020-02-16T02:37:01.888477Z" + } + }, + "outputs": [], + "source": [ + "def main(trial, train=True):\n", + " # PyTorch Lightning will try to restore model parameters from previous trials if checkpoint\n", + " # filenames match. Therefore, the filenames for each trial must be made unique.\n", + " \n", + " checkpoint_callback = pl.callbacks.ModelCheckpoint(\n", + " os.path.join(MODEL_DIR, name, 'version_{}'.format(trial.number), \"chk\"), monitor='val_loss', mode='min')\n", + "\n", + " # The default logger in PyTorch Lightning writes to event files to be consumed by\n", + " # TensorBoard. We create a simple logger instead that holds the log in memory so that the\n", + " # final accuracy can be obtained after optimization. When using the default logger, the\n", + " # final accuracy could be stored in an attribute of the `Trainer` instead.\n", + " logger = DictLogger(MODEL_DIR, name=name, version=trial.number)\n", + "# print(\"log_dir\", logger.experiment.log_dir)\n", + "\n", + " trainer = pl.Trainer(\n", + " logger=logger,\n", + " val_percent_check=PERCENT_TEST_EXAMPLES,\n", + " checkpoint_callback=checkpoint_callback,\n", + " max_epochs=trial.params['max_nb_epochs'],\n", + " gpus=-1 if torch.cuda.is_available() else None,\n", + " early_stop_callback=PyTorchLightningPruningCallback(trial, monitor='val_loss')\n", + " )\n", + " model = LSTM_PL(trial.params)\n", + " if train:\n", + " trainer.fit(model)\n", + " return model, trainer" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-16T02:37:02.018283Z", + "start_time": "2020-02-16T02:37:01.956427Z" + } + }, + "outputs": [], + "source": [ + "def add_suggest(trial):\n", + " trial.suggest_loguniform(\"learning_rate\", 1e-5, 1e-2)\n", + " trial.suggest_uniform(\"lstm_dropout\", 0, 0.75)\n", + " trial.suggest_categorical(\"hidden_size\", [1, 2, 4, 8, 16, 32, 64, 128]) \n", + " trial.suggest_categorical(\"lstm_layers\", [1, 2, 4, 8]) \n", + " trial.suggest_categorical(\"bidirectional\", [False, True]) \n", + " \n", + " # constants\n", + " trial.suggest_int(\"window_length\", 24 * 4, 24 * 4)\n", + " trial.suggest_int(\"target_length\", 24*4, 24*4)\n", + " trial.suggest_int(\"max_nb_epochs\", 20, 20)\n", + " trial.suggest_int(\"num_workers\", 4, 4)\n", + " trial.suggest_int(\"grad_clip\", 40, 40)\n", + " trial.suggest_int(\"vis_i\", 670, 670)\n", + " trial.suggest_int(\"input_size\", 17, 17)\n", + " trial.suggest_int(\"batch_size\", 16, 16) \n", + " return trial" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-16T02:37:02.081996Z", + "start_time": "2020-02-16T02:37:02.020607Z" + } + }, + "outputs": [], + "source": [ + "\n", + "def objective(trial):\n", + " # see https://github.com/optuna/optuna/blob/cf6f02d/examples/pytorch_lightning_simple.py\n", + " trial = add_suggest(trial)\n", + "\n", + " \n", + " print('trial', trial.number, 'params', trial.params)\n", + " \n", + " model, trainer = main(trial)\n", + " \n", + " # also report to tensorboard & print\n", + " print('logger.metrics', model.logger.metrics[-1:])\n", + " model.logger.experiment.add_hparams(trial.params, logger.metrics[-1])\n", + " model.logger.save()\n", + " \n", + " return model.logger.metrics[-1]['val_loss']\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Train" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-16T02:37:02.139269Z", + "start_time": "2020-02-16T02:37:02.084857Z" + } + }, + "outputs": [], + "source": [ + "PERCENT_TEST_EXAMPLES = 0.3\n", + "EPOCHS = 2\n", + "DIR = Path(os.getcwd())\n", + "MODEL_DIR = DIR/ 'optuna_result'/ 'lstm'\n", + "name = \"lstm1\"\n", + "MODEL_DIR.mkdir(parents=True, exist_ok=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-16T02:37:02.198129Z", + "start_time": "2020-02-16T02:37:02.141432Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "now run `tensorboard --logdir /media/wassname/Storage5/projects2/3ST/attentive-neural-processes/optuna_result/lstm\n" + ] + } + ], + "source": [ + "print(f\"now run `tensorboard --logdir {MODEL_DIR}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "[I 2020-01-29 08:42:54,398] Finished trial#8 resulted in value: 0.005635036621242762. Current best value is 0.005635036621242762 with parameters: {'batch_size': 16, 'bidirectional': False, 'grad_clip': 40, 'hidden_size': 128, 'input_size': 17, 'learning_rate': 0.00019134834148401144, 'lstm_dropout': 0.4080689425353674, 'lstm_layers': 8, 'max_nb_epochs': 20, 'num_workers': 4, 'target_length': 24, 'vis_i': 670, 'window_length': 48}." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-01T12:52:22.492191Z", + "start_time": "2020-02-01T12:52:22.416653Z" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-16T01:47:59.598921Z", + "start_time": "2020-02-16T01:47:02.100Z" + } + }, + "source": [ + "Note that the LSTM has access to the y values for the first half of the plot (the context) to match the NP setup." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-16T04:08:58.961223Z", + "start_time": "2020-02-16T04:08:58.864759Z" + }, + "scrolled": true + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/callbacks/pt_callbacks.py:239: UserWarning:\n", + "\n", + "Checkpoint directory /media/wassname/Storage5/projects2/3ST/attentive-neural-processes/optuna_result/lstm/lstm1/version_109/chk exists and is not empty with save_top_k != 0.All files in this directory will be deleted when a checkpoint is saved!\n", + "\n", + "INFO:root:gpu available: True, used: True\n", + "INFO:root:VISIBLE GPUS: 0\n" + ] + } + ], + "source": [ + "trial = optuna.trial.FixedTrial(\n", + " params={\n", + " 'number': 45,\n", + " 'batch_size': 16,\n", + " 'bidirectional': False,\n", + " 'grad_clip': 40,\n", + " 'hidden_size': 128,\n", + " 'input_size': 17,\n", + " 'learning_rate': 2e-5,\n", + " 'lstm_dropout': 0.4,\n", + " 'lstm_layers': 8,\n", + " 'max_nb_epochs': 20,\n", + " 'num_workers': 4,\n", + " 'target_length': 24*4,\n", + " 'vis_i': '670',\n", + " 'window_length': 24*4\n", + " })\n", + "trial = add_suggest(trial)\n", + "trial.number = 109\n", + "model, trainer = main(trial, train=False)\n", + "trainer.fit(model)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-16T02:16:06.159986Z", + "start_time": "2020-02-16T02:16:06.045538Z" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-16T04:10:17.489144Z", + "start_time": "2020-02-16T04:09:03.917103Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:root:\n", + " Name Type Params\n", + "0 _model LSTMNet 1 M\n", + "1 _model.lstm1 LSTM 999 K\n", + "2 _model.linear Linear 129 \n", + "INFO:root:model and trainer restored from checkpoint: /media/wassname/Storage5/projects2/3ST/attentive-neural-processes/optuna_result/lstm/lstm1/version_109/chk/_ckpt_epoch_3.ckpt\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b51a0a6c1a704a6c89d712401a461b7b", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='Testing', layout=Layout(flex='2'), max=234.0, style=Progr…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "304562568c6e4a14bd47be59577e05b5", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEgCAYAAACkfIiyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOy9d3xcZ5X//36maYpGGjWru8TdTlzSCOmkkAChhEC+hLYhQJbvl8BSll3YsFm+wP4WwtK+S7awuxBYCAkJkIQQkkB6cRLbiVNs2bFcJVm9jDS9Pb8/nnunaUYaSTOasX3fr5de0tx755lnRnfOPfdzznOOkFJiYGBgYHD8Yyr3BAwMDAwMioNh0A0MDAxOEAyDbmBgYHCCYBh0AwMDgxMEw6AbGBgYnCAYBt3AwMDgBMEw6CcBQojbhRDf1P6+QAixb57j/LsQ4u+LO7uFI4S4XgjxTLnncTIhhHhCCPGJRXotKYRYtRivdbxjGPSTDCnl01LKtbMdl8tISik/JaX8RrHnJIRYI4S4TwgxLIQYE0I8LISYdY7lRAixRQixUwgR0H5vSdsnhBDfFkKMaj/fFkKI420sg+MPw6AfZwghLOWeQwnwAPcDa4Fm4EXgvrLOaAaEEDbU/H4B1AE/A+7TtgPcCLwH2AxsAt4J/OXxNJbBcYqU0vgp8w9wGPgKsAcYB34K2LV9FwO9wN8CA8D/aNuvAnYBE8BzwKa08bYCLwFTwF3AncA308dLO7YT+C0wDIwCPwLWAyEgDviACe3Y27PHAb4IDAH9wMfSxm0Afg9MAtuBbwLPFPh51AMSaMizvwF1AZhEGf9vpI8N/BDo0fbvBC7QtrcAgfRxgdO1924FVgFPAl5gBLgrz+u/FegDRNq2o8CV2t/PATem7fs48PxxNlYd8ID22Yxrf3ek7X9C+9yf1c6zR4DGtP3naK83AbwCXJy272NAl/a8g8BfZr32l7Tz6Rhwg3YurCr39/R4+DE89MrhQ8AVwEpgDfDVtH0tKCO3DLhRCLEV+AnKu2oA/gO4XwhRpXlj9wL/oz3nbuCaXC8ohDCjvqhHgOVAO3CnlLIL+BSwTUpZLaX05JlzC1CrPe/jwG1CiDpt322AXzvmL7SfQrkQGJBSjubZfxvqgtOK+sLfkLV/O7AF9f7vAO4WQtillAMoQ3Rt2rEfQb3nKMpAPYIyZh3Av+R5/Y3Aq1KzPhqvatv1/a+k7XslbR9CiFeFEB+stLGyMKEci2XAUiCIutin80GUcV4C2IC/1ubUDvwBdRGv17b/RgjRpD1vCOWQ1GjP/74Q4nTtuVdqx18OrAYuyzM/gxwYBr1y+JGUskdKOQb8I3Bd2r4E8A9SyrCUMoi6df4PKeULUsq4lPJnQBjlFZ2D8jZ/IKWMSinvQRm4XJwNtAFfklL6pZQhKeVcgotR4Ova6zyI8ubXaheKa7Q5B6SUe1C3/7MihOhAGewv5Nmvj32LNufXs8eWUv5CSjkqpYxJKb8LVKHkHLRjP5w21nWoi5/+fpYBbbN8FtUoLz4dL+DOs98LVOt6tZRyk5TyjkobKx3t8/uN9v+bQp2TF2Ud9lMp5RvaOflr1EUU1Of7oJTyQSllQkr5J2AH8HZt7D9IKQ9IxZOoi+gF2nOv1cZ9XUrpB76WPTeD/BgGvXLoSfv7CMrQ6gxLKUNpj5cBXxRCTOg/KOmkTfvpy/LSjuR5zU7giJQyNs85j2Y9N4AyGk2Ahcz3lP53TjQP7hHgX6WUv8pzWK6xM96fEOKvhRBdQgiv9tnUAo3a7vuADUKIFSgv0CulfFHb9zeAAF4UQuwWQmR7/jo+lHeZTg1KQsi1vwbwZf1PKnosIYRTCPEfQogjQohJ4CnAo10EdQbS/tb/96DOz/dnnZ/no+6oEEK8TQjxvBYAn0AZev3/08YM/1uDmTEMeuXQmfb3UpR+qJP9hesB/lFK6Un7cWpGsB9oz/K6luZ5zR5gaZ5A60LKcA4DMZRsodOZ51gANKnmEeB+KeU/FjB29uelj3MByjBfC9RpcpEXZajRLoy/RnmRHyHlnSOlHJBSflJK2YaSs/41T7rcbmBT1me8Sduu79+ctm9z2r7jZawvou5q3iSlrEHJYKB9jrPQg4r1pJ+fLinlt4QQVcBvgH8GmrX/z4Np4/aT539rMDuGQa8cPi2E6BBC1AM3o4KZ+fhP4FNCiDdpqWguIcQ7hBBuYBvK4H1WCGEVQrwXJa3k4kXUF+hb2hh2IcR52r5BoCMtQ6JgpJRxVKD1a5qntw74aL7jhRA1wMPAs1LKL89x7A1k6vNu1PsfBixCiFuY7rX+HLgeeBdpBl0I8X5N8gEVCJQouSubJ1AB489qcYubtO2PpY3/BSFEuxCiDWUcb8/zlip1LDdKN5/Qzsl/yHNcLn4BvFMIcYUQwqydVxdrn60NJYENAzEhxNtQwVydXwPXCyE2CCGcc3zdkx7DoFcOd6A81IPAAVRAKSdSyh3AJ1FBqnGgG2WgkFJGgPdqj8eA/4UygLnGiaNS11ahsiF6teNBGYHdwIAQYmQe7+cmlNQxgDKav0Lp/Lm4GjgL+JgQwpf2k887uwl1ez+AMkg/Tdv3MPAQ8Abqdj1EltwjpXwWZahfklKm39KfBbwghPChsmj+Skp5MPvFtc/4PaiL1AQqKPsebTuoIPXvgdeA11EBwv/Qn6/JOR+qtLGy+AHgQGX7PK99pgUhpewB3g38Hcpw96AyV0yaHv9ZlOEeRwVW70977h+1134MdV4/hkHBiNxSnMFiIoQ4DHxCSvnncs+lVAghvg20SCnnku1SMoQQjwF3SCn/q9xzMTAoFoaHblAShBDrhBCbNEnobFRa4+/KPS8AIcRZqPzzmWQtA4PjjhNx1aFBZeBGySxtKD3+u1TA6k8hxM9QssRfabf/BgYnDIbkYmBgYHCCYEguBgYGBicIhkE3MDAwOEEom4be2Ngoly9fXq6XNzAwMDgu2blz54iUsinXvrIZ9OXLl7Njx45yvbyBgYHBcYkQIm85BENyMTAwMDhBMAy6gYGBwQmCYdANDAwMThCMhUUGFUM0GqW3t5dQKDT7wScBdrudjo4OrFZruadicJxgGHSDiqG3txe3283y5cvJ0XPhpEJKyejoKL29vaxYsaLc0zE4TjAkF4OKIRQK0dDQcNIbcwAhBA0NDcbdisGcMAy6QUVhGPMUxmdhMFcKMuhCiCuFEPuEEN1CiGkNCIQQy4QQjwrVsPaJtCYBBgYnHIcPH+aOO+6Y/UCDE4ZAJMZX732NoanKvmOa1aBrPQRvA94GbACu07rEpPPPwM+llJuArwP/VOyJGhhUCjMZ9Fhsvu1ZDSqZh14f4BfPH+WpN+bT62XxKMRDPxvollIe1Dqf3InqRpLOBlKdRR7Psd/AoOK55ZZb+MEPfpB8fPPNN/PDH/5w2nFf/vKXefrpp9myZQvf//73uf3223nXu97FJZdcwqWXXsoTTzzBVVddlTz+pptu4vbbbwdg586dXHTRRZxxxhlcccUV9Pf3l/x9GSych15X/bAHvMEyz2RmCjHo7WS28OrVtqXzCqrtGah2Ym4hRMPCp2dgsHjccMMN/PznPwcgkUhw55138uEPf3jacd/61re44IIL2LVrF5///OcBeOmll7jnnnt48skn844fjUb5zGc+wz333MPOnTu54YYbuPnmm0vzZgyKRjAS56n9wwD0eytbcilW2uJfAz8SQlwPPAX0oZrVZiCEuBG4EWDpUqOZt0F+/u/vd7Pn2GRRx9zQVsM/vHNj3v3Lly+noaGBl19+mcHBQbZu3UpDQ2F+yeWXX059ff2Mx+zbt4/XX3+dyy+/HIB4PE5ra2vhb8CgLDz5xjChaAKLSTA4efwb9D6gM+1xh7YtiZTyGJqHLoSoBq6RUk5kDySl/DHwY4AzzzzT6KxhUHF84hOf4Pbbb2dgYIAbbrih4Oe5XK7k3xaLhUQikXyspx5KKdm4cSPbtm0r3oRPYLzBKD/8834+f/lq3PbyLa56ZPcAtQ4rmzpqTwgPfTuwWgixAmXIP4Dq1J1ECNEIjEkpE8BXgJ8Ue6IGJxczedKl5Oqrr+aWW24hGo3mDXy63W6mpvJ3r1u2bBl79uwhHA4TDAZ59NFHOf/881m7di3Dw8Ns27aNN7/5zUSjUd544w02bizPe6107t7Rw0+ePcT5qxu4ZF1z2ebx5BvDXLJuCXaria7+4t41FptZDbqUMiaEuAl4GDADP5FS7hZCfB3YIaW8H7gY+CchhERJLp8u4ZwNDEqGzWbjLW95Cx6PB7PZnPOYTZs2YTab2bx5M9dffz11dXUZ+zs7O7n22ms59dRTWbFiBVu3bk2Ofc899/DZz34Wr9dLLBbjc5/7nGHQ83DfrmMATIXKlznkD8cY9UdY0+wmEksw4osQjsWpsuQ+N8pNQRq6lPJB4MGsbbek/X0PcE9xp2ZgsPgkEgmef/557r777rzHWK1WHnvssYxt119/fcbjW2+9lVtvvXXac7ds2cJTTz1VlLmeyHQPTfFanxeAyWC04OcNTYaoc9mwmouzZrJfy2pp89gJRePaa4TprHcWZfxiY6wUNTDQ2LNnD6tWreLSSy9l9erV5Z7OSc29Lx/DpC2UnSzQQ3/uwAjnf/txvvnAnqLNo29CaeZtHgcttQ4ABio4MGoU5zIw0NiwYQMHDx5MPn7ttdf4yEc+knFMVVUVL7zwwmJP7aRCSsl9r/Rx3qpGXjw0hrcAD33vwCR/+fOdROIJ7t7Zy19fsbYogdRjE7qH7sCnXVgGKjgwahh0A4M8nHbaaezatavc0zjp2H1skp6xIDe9ZRX7BqYKkly+8tvXcNjMfOf9m/jUL17i3pf7+Mibly94LscmgpgENLurqK5S5rKSDbohuRgYGFQUj+wZxCTgsvXN1DisTIZmNuhHRv28fHSCG85fwRUbWzitvZZfPH8UKReeGX1sIkRLjR2L2USN3YLDaq5oycUw6AYGBhXFn/YMcsayOhqqq6ixW5gMzqyh//4VlQ3zzs1tCCH48DlL2Tc4xau93gXP5dhEkFaP0s6FELTW2g0P3cDAwKAQesYCdPVPcvkGlXc+m4cupeTeXcc4e3k97Zrh3bpUpZH2ji+87soxb5A2bVyA5hq74aEbGBgYFMKf9gwCcPmGFgBq7NYZNfSu/im6h3y8c0tbcluNFgwtJJg6E4mEpH8iRJvHntxmeOgGBgYGBfL0/mFWNrlY0ahKKdQ4LDOmLT6+bwiAt5/aktxW41DBy9m099kY9UeIxBO01aZ56LV2BidDJBKVWbnEMOgGBmXm8OHDnHrqqeWeRkUw7AuzrCFVF0f30PMFOI9NBKlzWmmorkpuc1jNWExiTguS8o0NZEou7ipiCclYILKgsUuFYdANDEpEPD6t4KjBLEwGY7jtqWzqGoeVWEISiOT+LAcnQzTX2DO2CSGodVgXLLmkDHpq/DqXDYCJwMLGLhWGQTcw0Ci0wcUTTzzBhRdeyDve8Q7Wrl3Lpz71qWR1xerqar74xS+yefNmtm3blrehxc6dO9m8eTObN2/mtttuW5w3eBwwFYomNXBI6eH55JN+b4iWWvu07SqYurAaMH2aQW9P89A9TmXQvcHK9NCNhUUGlckfvwwDrxV3zJbT4G3fyrv7hhtu4L3vfS+f+9znkg0uXnzxxZzHvvjii+zZs4dly5Zx5ZVX8tvf/pb3ve99+P1+3vSmN/Hd736XaDTKRRddxH333UdTUxN33XUXN998Mz/5yU/42Mc+xo9+9CMuvPBCvvSlLxX3fR6nSCmZDMWSGjhArUMz6MEYrbXTnzM4GWJTx/QdKt1xYV50vzeEw2pOzgHAo/097q9MD90w6AYGGnNpcHH22WdzyimnAHDdddfxzDPP8L73vQ+z2cw111wD5G9oMTExwcTEBBdeeCEAH/nIR/jjH/84p7l6g1Fq7BaEEPN9uxVHMBonnpAZS/ZnCnDq1Q+zJRf1vIVLLv3eIK0ee8Zn7HGquU0scOxSYRh0g8pkBk+6lBTa4CLbkOqP7XZ7suxuvoYWExPTer/MialQlHP+v0f55ntO5ZozOhY0ViWhLyDKKbnkMKBDUyp9sCWPQdclk/kyEYhSp0ksOh6HrqFXpuRiaOgGBmlcffXVPPTQQ2zfvp0rrrgi73Evvvgihw4dIpFIcNddd3H++edPOya9oQWonqK7d+/G4/Hg8Xh45plnAPjlL385pzn2e0MEo3Ge1vpcnihMaV54dlAUcnvoeju45lwaut066wrT2eejBWgDYxCPJedmEgvPcS8VhkE3MEhDb3Bx7bXX5m1wAXDWWWdx0003sX79elasWMHVV1+dc6x77rmHv/3bv2Xz5s1s2bKF5557DoCf/vSnfPrTn2bLli1zrjkyMhUG4KWjC/P0Kw3daNc40j10TXLJYZz1dnCtOQx6bQE1YGZjKhRlteiDH2yCO66FRByTSWXQjFeoh25ILlkMToZY4q46obRJg8IppMEFQE1NDQ888MC07T6fL+NxvoYWZ5xxBq+88kryca5mGPkY8StjcnQswIgvTGNaDvbxTEpySZkl9wySi75iM7fkYiESSxCKxrFb59ddSAa9/OWxr4KMw4FH4Yl/gku+isdpM9IWjwcmAhEu+Pbj3Lurb/aDDU44jpcGF7qHDvDSkfEyzqS4TCYll5SHbrOYcFjNOSWOwckQVRZTRhaKzkzae6H8n9jPqIv2w4d/A1s/DE99B3q243EuPOBaKgwPPY0xbanvM/tHuXrriRNsMiiMuTS4uPjiixd5dilG/WHMJoFJwMs9E7x1Y8vsTzoO0PPG09MW9ce55JOByTAttfacd9Pp2vuSHB78bIQiMS4ROznQdDlrlp0LLZvg1bthz714HO9m2BeefZAyYBj0NEJRtTjkpaMnjtdjMH8qtcHFyFSEBpeN1lr7CeWh60HRmqxOQ7WO3AHOQW8op9yiPwfmH7wM9O2mSXjZ33QOawCqqmHZubD/T3ia3s/+Id9sQ5SFgiQXIcSVQoh9QohuIcSXc+xfKoR4XAjxshDiVSHE24s/1dITjqnlxYdG/IxW6BX4RKcYTQlOFPJ9FqN+pZtvXVrHq71eYvHEIs+sNEwGY9jMJqosmWapxp47wDkwmXuVqHpO/mBqISQOPgnAVNu5qY2rLoORfSwzj+A9XjV0IYQZuA14G7ABuE4IsSHrsK8Cv5ZSbgU+APxrsSe6GOgeOsDLJ1gGwfGA3W5ndHTUMOooYz46OordPt1gDfsiNFTbePPKBoLROJ/6xU5GTgAHZDIUpcYxfbFUrproUkpl0PN46DOlOxaC9egzHEkswVy3PLVxtVogdlpwB1PhGNEKvJAWIrmcDXRLKQ8CCCHuBN4NpLfWlkCN9nctcKyYk1wsQrFUAaCdR8e5TCuyb7A4dHR00Nvby/DwiZVfPV/sdjsdHdNjOaO+MCsbXbx1QzN/f9UGvv3QXj74n8/zyOcvKsMsi4fK+84V4LTQPZTpaY8HokRiiZyrRGHukks8ITGbtAtJIo6rfxsPJs7glLSMGxrXQG0nq6eeB07DG4xWXIZRIQa9HehJe9wLvCnrmK8BjwghPgO4gMuKMrtFJhxVBt1lM7NzFm1yaDLE4GSY03LUkTCYH1arlRUrVpR7GhWNlJIRX5iGahtCCD5+/gqi8QTf+uNexvwR6l222QepUCa1cgbZ5PLQ+71qFWiuHHRILU4qJMslHItz2fee5KPnLOeTF54C/a9giUyyLbGRzekXGCFg1WW0vXIXVj7GRKDyDHqx0havA26XUnYAbwf+RwgxbWwhxI1CiB1CiB2V6IXpksvZK+p5pWdixluq7zy8jxv/Z8diTc3AAAB/JE4omsgwJOtb1c3xvoGpck2rKEyFonk8dFUTPb2pxNP7RwDY2JbboaqymLFbTQVVXHzglX56xoLs6Z9UG44+D8C2xIaMVasArLgASyzAGtFTkRUXCzHofUBn2uMObVs6Hwd+DSCl3AbYgcbsgaSUP5ZSnimlPLOpqWl+My4hIc1Df/PKBsKxBG8M5v+C7BucqtjFBQYnLnqwPr2hw7oWNwD7BibLMqdikV1pUae9zkFCZvYIvX/XMbZ0elja4Mw7Xq3DOmvwUkrJz7YdBmBYz+8feYOQpZZhPNMybmjdAsBppkMVWXGxEIO+HVgthFghhLChgp73Zx1zFLgUQAixHmXQK88FnwXdoG/q8ADQnSc1SUrJgSEfwWi8YltRGZyY6MHPxuqUtLLEXYXHaWXfDA7I8YCSXKZ76Gu1C9Ze7YLVPeRjT/8k79rcNu3YdPJlx6Szq2eCV3u9WM0iZdDHDjBuVz5sdbaHXn8KCVsNp4lDFVlxcVaDLqWMATcBDwNdqGyW3UKIrwsh3qUd9kXgk0KIV4BfAdfL4zBVIRRTEsv6lhosJpHXQ+/3hvBrHVTSA6kGBqVmxKdu89MlFyEEa5vd7E2XXIITsOsOiFee0clHshhWFmub9TsQ9f7uf+UYJgFXbWqdcbxc2ns2P992hOoqC1dtaktWb2T0IEO2Dlw2cypQqiMEiZZNbDQdqsiKiwUtLJJSPgg8mLXtlrS/9wDnFXdqi4/uoVfbLSxvdLF/MLeHfmA4td0fjuO0GeuzDBaHlIeeGYxb2+LmNzt7kVIi4lG480Nw5Bnw9sFFld9AIxpPEIzGc3rorioLS+ud7B2YQkrJ7185xjmnNMy6ArTWYU153TnwBqM8+Fo/7zujgyVuu8qcCfqxTfYy0HRFTj0fwNy+lfVHXuAx38LK85YCo5ZLGuFYAqtZYDYJVi+pzrsaLF2KCebpdWhgUApGNQ89O5tlbYsbfySudOY/fF4Z8yUb4KlbYXhfOaY6J6ZCqfK0uVjX4mbvwCT7h3wcGvHz9tNm9s5BpTvOlLb4wKvHCMcSXHtmJ0tq1AXSe0x9Vj2iNe9cRNsWqkQU23jlfa6GQU8jFI1jt6jKbKub3RwZ9Se99nTSDXogurCaywYGc2HEF6bWYcWWtZpSD4weObAXXv4FnPsZ+Oh9YHXCA18ox1TnhJ5eWJOj0Bao93d4NMAfXlU9WS9bP/sakdkkl7t39LKmuZpNHbU0aXc8fs2gH6Ilr0GnbSsAdd7ds85hsTEMehqhaIIqrdTmmuZqEhIODvunHZdh0A0P3WARGfVFMgKiOms0ndl3SOuBuvG9UL0ELviC8tbHjyzmNOeM7qHnklwA1rXWEE9Ifr7tMKe11+Zd8p+Onu6YK5y3f3CKXT0TXHtmJ0IImtzKoEeHugHojjXnlVyoW4FfuGj27S3gnS0uhkFPIxyNY7eqj2T1EvUF2T80PTB6YNjHci1dypBcDBaTYV84I2VRx2230u5xYB7YBSYrNG9UO9ZdpX6/8fAiznLuTOboVpSOnukyHohy6folBY1Z67CSkDAVnn4X/cieQQDevaUdIGnQTeMHwdXEUMSW30M3mThqW01n+I2C5rGYGAY9jVAsVQx/RaMrZ6bLRCDCiC/CaVpqo+GhGywmI75wUh7IZm2Lm4bJLmXMLdoxDSuhYTW8Mbcm1IvNbJLL8gZXsmhXIXILpBo658pF7+qfpKPOkTTkepDZPnkI6lfmXeSkM+xcSWfsKFRYMp9h0NMIRRNJD91mMeXMdNEzXE5rV6vzAhFDQzdYPEa1wly5WNtczcrofuLa4pcka66Aw89AuHLz1JOSSx6DbjYJ1jS7aamxs7GtJucx2Xi0Bs+52sXtG5hKxh1Afd/rnFZqAkehYZVa5JTPQwemalbiJATe3oLmslgYBj2N9KAokDPT5e4dvZgEnLGsHjAkF4PFIxJLzFgQ6vSaCWqFn+Hq9Zk71r4N4hE48PgizHJ+zCa5ANzyzg18739tLrg9ZJ3moY9rHvqAN0QoGicci3NoxJ+UcXSWVsdxx0aJ1a0gEkvMOJewZw0AkYGuguayWBgGPY1QNE6VNfWRrG1xc3jUnzTazx0Y4c7tPXzyglNY2eQCDMnFYPEY03qJ5vPQ1ycOALDXtDJzR+c5YK+taB19PBDBbBJUz7Cm46zl9Zy7clpFkbzoHvpEIIKUkqv+5Wm++8g+Dg77iSUka1syPf0N9jEAAtXLAGaUXBKNqkVhqH9P3mPKgWHQ0wjHEhke+vrWGqRUdVui8QR/99vXWNbg5HOXrcFhU8cZkkvhvDE4xfce2WfUO58n+RYV6bT6uwhLKzsCWS3pzBZYdj70PF/qKc6bnrEg7R4HpuyVmQtA19AnAlH8kTgjvggP7R5IrjjVV6DqrKhS2302FXSdyUN3eZoZkTXEBysr08Uw6Glkdwhfr13B9/ZP8krPBIdHA/z1W9fisJmxmU2YTcLw0OfAL58/wv97rJttB0bLPZXjklx1XNIxD7zCIcsK9g6Hpu/sOANGuyEwVsopzpujYwGW1ucvtDUfPA5dcokkG2v3jAV54NVjWM2CU7S7bJ12s2pq47UoOXUmD73OZaNbtmMerazFRYZBT0Ploac+ko46B9VVFrr6J9l+WNVHP3dlA6DqZzitZsOgz4FXer0A/PKFo2WeyfFJrjouGYwdwutanlnTRafjLPW776USzW5h9IwF6CyyQbeYTbjtFiYCUUb9qRIAf+4aYmVTNVZzpvlrFsqgH40oR24mD73OaaM70YZ94kBFZboYBj2NcCzTQzeZBGtb3HQNTLH98Bgrm1wZOcAOm9kIihZINJ5gT/8kVRYTD+8eSBVCMiiYXKVzkyQS4BvAXNtO73iQFw+N8YW7dqUkwbatgIC+yqvh7wvHGPVHiu6hg5JdJgIRhqfUxdCpSaXZAVGAejnOuKyme2z2AG2dy8p+2YEt6gXfUNHnPV8Mg55GKJqpoQOsb3XT1T/JjsNjnLW8PmOfq8pCIEdpAIPp7BuYIhJL8H8uXkUsIbl7R2Wlex0PjPjC2K0mXDbz9J2BUUjEqG5ULes+9F/P89uX++jq17z1KjcsWQ+92xdxxoXRMxYAKIlBr3PaGA9Ek3LVO7QaMLkMem18lCHp4ak3VOXvfKtW9XH3S7UoieHK0dENg55GKG2lqM66lhqmQjEmQ7FpBt1hNRM0gqIF8aomt7xnaxtnLa/jodcHMgQOPI4AACAASURBVPYfmwhyhyHFzMiIL0KDqyp32t6UauPb2KoyNGyanDCa3jy640zo21lREgEo/RxKY9A9ThsTwWiyqNmHzlmGScDpS+umHVsTHWXCXM8Lh8YwCah15jfoVrOJAetS9aCCip8ZBl0jFk8QS8gMyQWUh66TbdCdNkNDL5RXeyfwOK0srXeyrqUm+SXW+d3Lffzd716ryBrTlcKIL0yjO49+PqUukI1tK7j1mk3cfsPZQCrVEYD2MyE4DmMHSz3VOaF76J31jqKP7XFYtdXdqqjZlk4P22++jHNOaZh2rDUwxNmnrecPnz2fX33ynBk9dICYs5mgyQUjlVMCwCjkraE3t6jKqmKn56o211RNO+EcNnNyhZvBzLza6+W09lqEEHTWO/AGo3iD0WR3dt2Qj/kjyfxhg0xGfBHaPXmKUk2pKoS4W7i2oyMZ2xlNN+gdZ6rfvdtVSYAK4ehYALfdkjwXikmd08q4Xxl0PTsoZwxCSvANItytefuUThvbZWMg1saK8cNFnPHCMDx0jbCmhWd76NVVFta1uDl/VdO0W13loRsGfTZC0Tj7BqfYrNW/0W+te9K89Mmg+hxzLdM2UIz6wjS4ZvbQqVZ1Thw2Mw6rOdNDb1oH5ioYfL3EM50bespioStA54LHaWMyFGNoKpw/OwhUOmciCu6W/MdkUeeycUwsAcOgVx66h56toQPceeM5fOM9G6dtd9oshuRSAG8MThFPSE5tV55PR910g643IhirwMa7lUAiIRn1R2h057l7meoHVxOYU15uQ7Ut06CbzNC4BoYqJ4gH6jwohX4OqcVFB4d9Mxt0X+YFsRDqnTYOJ5pg4ojKMqoADIOuEcrjoYO6yudqM2ekLRaGHpBq1rrC6J3ae8bTPHStlse43/DQc+ENRokn5MweepZ32eCyZUouAEvWVVRWRiIh6RkPlsyg1yULdEXzLsgCMiSrQvE4bRyINKg6Ob6B2Z+wCBgGXUM36FWWHClheXAZQdGC0L1vXSOtsVupdVgzAqNJD92QXHKSXCWaLyg6eQzcmW3Z6l02xvxZPTWb1oG3p2IqLw5NhYnEEkVfVKTjSctUmdFDn1L10efkobusdMe02jIVIrsUZNCFEFcKIfYJIbqFEF/Osf/7Qohd2s8bQmhLro4jQtH8kks+HDYLwWicRKKy0sAqDT3gmR7s7Kx30DOWarJreOgzk1wl6sonuUz30OtdVYz5sj10rRLjcGVkZvRqd2kddcXPcIHMcy5nMFRH97DnqKH3SK3ZRoV0hJrVegkhzMBtwNuADcB1QogN6cdIKT8vpdwipdwC/Avw21JMtpTkC4rOhL7qLBQzvPSZmNCbF6StvFta78yQXPQmBEZQNDczeujxKPiHwd2WsbmhWkkuGcXQmtap38OVUfZVv5CXKrOpLsNDn0lyGYSqGrC58h8zbWwbfbIRiTiuPPSzgW4p5UEpZQS4E3j3DMdfB/yqGJNbTHSjnJ22OBPOZMVFw6DPhDcYxV1lwZJWO6OzzknvWJBEQpJIyGSbMCMomhvdoDfk8tB9Q4DM4aHbCMcS+NPPz7rlYLHDUGUYdH9YzS3n6tcikH6hyCtXgfLQ5yC3gDLoEaxEnC3HlUFvB3rSHvdq26YhhFgGrAAeW/jUFpeU5FL4ieXQjjUCozPjDUSnrbrrrHcSiScYmgozFY4lFy8aHnpuRn0RTCIV5MtAT1nMoaEDmbKLyQyNqysmMKqn/TqrSrMkxl1lQa/I25gvoAw5JavZ0D9fn7NdZbpUAMUOin4AuEdKmdPCCSFuFELsEELsGB4eLvJLL4xwbD6SizoJ/UYu+oykLyDS0YNgR8cCyX6SYGjo+Rj1h6l32XLXC8+ToaF786PTAqPrKyZ1sdQeuskkkl563pRPmJdBT3ZEqmo/rjz0PqAz7XGHti0XH2AGuUVK+WMp5ZlSyjObmpoKn+UiMJ+gqLPKkFwKYSIYzcg2AOjUgmA9Y4FkhktzTZWR5ZKHcX80t3cOaQY900PXg4Bj2RfJprUw2QuhyWJPc84kPfQZOhUtFI/DitNmnvk1fENzllz0C8WIuUX9D6LBWZ5RegqxXtuB1UKIFUIIG8po3599kBBiHVAHbCvuFBeHZB76HNIWnYbkUhDeYBSPI9MYtdc5ECLTQ1/W4MIbjBKLV8YijUpiIhiZdlFMMtUPwgyuzPZsKQ8926BrgdGR/cWe5pzxR+LYzCZsc4hdzRWP05q3bR8AET/EgtM+v9mwWUy4qyz0Ce1CMNEz8xMWgVk/RSllDLgJeBjoAn4tpdwthPi6EOJdaYd+ALhTHqf9xeajoetXfMNDn5mJQHRaN/cqi5lmt52+iWAy02FFgwspUznpBikmAtH8mSD+YWWMTJnnblJDzzbodcu1Qcuv+wbCseSdbqnY1OFhS+f06opJ/Jr865ybQQdY0+Jm56RW+6UCZJeC7nOklA8CD2ZtuyXr8deKN63FJ7WwaC556EZf0dmQUuLN41221znoGw8mDfiyRqWrjwciM+cMn4RMBKKc1p7HQw+Og6N+2manzUyVxZTDoC/TBi2/QfdH4rhKKLcAfO1d08t2ZE5Ca4k4Rw8d4PSlHh7eVqUs6WT5a/wbK0U1QrE4NotpTk1q9bRFQ3JJEYrGOTLqTz4ORuNE4zJnJb02j0N56FphrhUNKgd4PGB46NmMByLU5VtUFJwAh2faZiGEWv6fvbioyq0uABWwGCYQiSW/R+WbxIj6PQ8P/fSldRyL1ahc9KnyL/83DLpGOJqYk3cORh56Lm5/7jCXfPdJnutWX5IJzTh7chj0do+Dfm+Q8YBKydOLdk3zKE9yQtE44Vgiv4YeHAdHbkmhvto2PcsFlJc+Uf6GIr5wvGQpiwXj1wz6fDz0ZXXEsBC0NajyC2XGMOgaqlvR3DwFXXIJGm3okuw5Nkk8IbnpVy/TO57KYMknuUTjku4hHzWOVODKSF3MRM/Nzw4sJwlO5DforqrcF0jPsoqQXALhWMlSFgufxPwNenONnXaPgxFRl8o2KiOGQdcIxxJzSlkE1ebLYhL4w4aGrnNg2Me6FjfReIJb7tud9NCzg6IAHR6Vutg1MEmN3ZpMyzNSFzMZ11bP1s3DQ88puQB4lioPvcxlX/2ReElTFgubxLCqE2+rntfTT19Wx5ForSG5VBKhaHxOKYugNEqHUXExSSIhOTjs59yVjVy1qZWXj47jDeb3LtuTuehBah3WZFMGw0PPZEL7DHP2uIyFIeoH+3QNHZQHOTQVIpqdClq3rCLKvgYiMVwlznKZFf+o8s7n2WDj9KUejkRqSXgNyaVimI/kAkpHN4Kiiv7JEMFonJVLXKxe4mY8EOXAsAqQ5jJG7Z5Uhb0ah/LSVMlXIyiajn6Xk3NhUVArbJojKApod0vqQpuBZ7k2eHl1dH+4Ajz0wAg4p/cYLZQtnR4GZR2m0BhEQ0Wc2NwxDLpGKDp3yQXAZbMYS/81Dgz5AFjVVM2aZtVc+8VDY0DuoKirypLU1vUsmDqX1ajnksWMBj2kG/Tcksv6VtUTd0+/N3OHnrpY5kyXQKQCNHT/iOr2NE/qXTYG0NJGy3zHYxh0jVBsfh6622Fl0mgUDUC3ZtBXLqlmTbPSI186Mo7VLPKmpuleut5hvd5VxTPdI7zv357jhYOjizDryicZFM0luQTH1e88Bv2UJhc2s4mu/qyGFrVaNY8yBkYTCUkgUgFZLoGReQVEdRxWM4NS+/wnyxsYPekN+m9f6uWv7nyZ4anwnNMWQXmeXsOjBFRAtNZhpcFlo8ldRa3DylQ4Rq3DmrcBsG7QdQ/9i5ev4QNndfL6MS/37spXMujkwhuMYreacjscSYOeW3Kxmk2sbq6mqz+rbovVDtUtZTXoenZYddk19JF55aDrVFnNDEjNQy9zpkuZL43lJZ6QfOfhffR7le61uTP3l2Im6pxWDo/6Zz/wJODAsI+VTa6k8V7TXM32w+M5FxXp6IFRPQtmc6eHzZ0edvVM0DdRXj2yUhj3R2ZOWYS8Hjoo2eWJfUPTd9QtK6vk4l+EwlyzEglANACu+WvoGR56mQ36Se2hP7V/mH5viK+8bR1XbGzmLWuXzHkMj9NmZGVoHBj2s7Iplfq1WtPRZzTonkyDrtNaa6d/ovzV6yqB8cD0apVJZpFcQBn0EV+EoamsC2SZc9EDeunccnroC1glqmM1C6aEi6ioKvviopPaQ7/rxR7qXTY+dt4K/vKilfMao1bT0OMJiXkOZQNONLzBKMNTYVYuSRn0NdrfM7UX03tJprenA2itdfDM/hGklHnlmpOFfLVwAM2gC6iqzfv89a3qwtrVP8UStz21o7YDdh9TueimxfftKsJDT64SnX9QVAiBw2phytZIfZlz0U9aD33EF+bPXYO8d2v7gkp36l+0yZO8QuDhESU7ndKY6sm4pgAPfUNrLTaziVVLMhd1tHsc+CNxI+CM8tDz1kIPjoO9dkaDvEHLdJmmo7tbIRGDQHmCz/r6jVIX55p5EvMvzJWOw2bGa2k0JJdy8dQbw8QSkvdszdlNr2B0gz5xkht0fYl/et3pQiSXpQ1Our5xJRvbMj3MVo/yJPu9lSm77Dwyxv2vLM7t9Yylc0P5l/3reJw22mrtPL1/mEQirbq1W6vjXaZUO19Ybz9XRslF99AXkIcOqhz0uLn89VxOWoM+pXl+LbX2WY6cGf2LNnGSZ7ro5Q/Sb58bq22874wO3rJu5thELqmqtVZJMf0VGhj9f49287X7d5f8daSUTARmkVxmMegAN5y/gme7R/nun/alNuodjsokEyQ19LJKLlot9CJ46GOmRvVZlrElxEmroeu3ewst3akvmJk4yUu++nPcPgsh+Of3b57XeG2ah95XoYHRrv5JxvwR/OEYrhLmUfsjcWIJOa86Lul8/PwVdA/5uO3xA5y+tI5L1zenWq6VyaCnNPQyB0VNVqiqWdAwdquJkUS96nxUwF1TqThpPfSgdjLNtX5LNkkPPXhye+h6k49iZSwscdsxm0RFSi4jvjBDU6okbc94oKSvpWdQ5U9bLMygCyH4xntOpcFl44FXs5pKl81D18+ZcnroC6vjouOwmhlBkw19w0WY2Pw4aQ16IBLHYTXPqaFFLgwPXeEr8pfTbBK01Ng5VoGSS3pwsWesNBecSCzBDbdv567tqk9lfskld3OLXFjNJs5d1ciz3Sp7CEuVuhiUSUP3F+kueUEEFraoSMduNTMkNYPuz5Hzv0icvAY9Gi/KiVRjGHRA6aEmMbcWfrPRWmvnWAVKLpkGvTQeeu94gMf2DvGjx7sBcncrSiTmfHt//qoGhqbCyTINuFvL56FHYphNoqjnzJzxDaaCwwvAbjUzFFdJAEldvgyctAY9GIkXJbpuNglq7JaTvrGxPxLDZbMUNWe8zeNIruKtJLr6p2ipseO0mUsmuehNKfQ8/SXuHD1Ww5MgE3My6OeuVN7oM1pHKaqby6ehh+O4bObyrjPwDYFr7gsKs7FbzQzGNR2+0iUXIcSVQoh9QohuIcSX8xxzrRBijxBitxDijuJOs/gEIjGc1uLIA3Uu20lfITAQjhddC2312On3BjNT7SqArv5J1re66axzlkxyGdGaUtz2wdO5/6bzWNbgmn5QAatEs+msd7Kswcmz3Vr+tbtVeallQNVCL6N+LqUy6NULN+gOq4nBqBOEqawe+qyfphDCDNwGXA70AtuFEPdLKfekHbMa+ApwnpRyXAix8E+oxAQi8WQLuYXicVhPesnFF4kVPZ+4rVa1qBvxhzNXOJaRcCxO95CPS9YtwSQEvSXy0PU+oM019vyptXrp3DzNLfJx3qpG7t91jFg8gcXdnEq1W2RPWXUrKqN+HhyHRDSV7bMAHFYzgTgqn73CNfSzgW4p5UEpZQS4E3h31jGfBG6TUo4DSCnL944KJFDEk6nWaTvpFxap3pBF9tA1Q1ZJuejdQz5iCcn61ho66530jgdVgLHI6G3j6nNp5zrz8NABzjmlAV84xt6BKVVxMRGFwNh8pzpvAiVO+ZwVn2amiuCh261aoxtXU2qxUhkoxKC3Az1pj3u1bemsAdYIIZ4VQjwvhLiyWBMsFcU06EYJXeVtFbvIUnJxUQXp6G8Mqrri61vddNQ58IVjJbk7G/WFqbFbZi5LMUvp3Hx0arr84GQoLXVx8Zesl91D16WmIhn0cCyBdDWlLhRloFhBUQuwGrgYuA74TyHEtLNMCHGjEGKHEGLH8HD5dCZQeeiOInmUHqf1pPfQ/SXw0OtcegZR5Vwsdc+5qdpOZ70TKE0u+qg/QmN1jkBoOiGtC9EcJZclNerOZ2gqnDLoZUhdDESKf87MiaSHXpwsF4C4s6niJZc+oDPtcYe2LZ1e4H4pZVRKeQh4A2XgM5BS/lhKeaaU8sympvlXNysGgUgc5zw6FOXC47DiDUaJa8G75w6M8F9PHyzK2McLpeg8U5dctFU5F0u9ZES13UJnnWbQSxAYHfVFZpZbYNZ+ovlo1OrtDKcb9DJkuvjDZe5W5C+e5OLQ2lfG7A0VL7lsB1YLIVYIIWzAB4D7s465F+WdI4RoREkwFW3RgsUMijptSAlToSiRWIIv3f0qP/zz/qKMfbzgD8eK3nnGaTNjM5sqKoNoMhSlusqC2STorFfSRWk89HBGobOchLxgtoFlbgHjKosZj9Oq6qNXl9Ogl7mfqG9QfX5zvMPJhe6hR+yNEPGpxhllYFaDLqWMATcBDwNdwK+llLuFEF8XQrxLO+xhYFQIsQd4HPiSlLJiG0JKKYu2sAjSKi4Gotyzs5e+iSCBaLwkwbJKRcUkiuttCSGUnOWvLA/drdVud9uteJzWkiwuGvVFaJhVcplQxmge2SlL3FUMTYZVKzq7pywGvRTnzJzwDSm5pQjZPbpzGLJprejKlLpY0KcppXwQeDBr2y1pf0vgC9pPxROJJ4gnZNEN+rAvzG3ayr54QhKOJebVePp4Q0qpLSwq/nutc1ZWjv9UKJo06ADNbruSLopIPCEZC0RonE1yCXlVLfR50OSuYtinzdvdsugaevKcKWfpXN/gghpbpKN/z4NVaQa9bllRxp4LJ+VK0aBWQ6JYQdFarXDS1+7fTd9EkEu0crF6RccTjYlAhO2HU2luwWgcKSmJHupxVlaOv/LQU3VVGt02RnzFNegTgQhSzpKyCHOq45LNErddeehQllS7iUAUKcnfuGMx0D30IqAb9IBFSyEtk4d+Uhr0VKeU4nrou49N8r8vXsmVp7Zor1M53Xb2D05xcNhXlLFuf+4wH/jx80yFlKH1J3tDFt+gV56HHstol9fgqkqu6iwWo9qy/9kll/l76EvcVQxPhZUs6GpcdAOkV6tcUjPLeywlRVolCmphEYDP2pAauwyc1Aa9WEHR5ho77ioLN154Cn9zxdpkKlYleehfuudV/mEODRlC0ThHRv059/VPhIgnZDInW29uURLJxWVlvII89MlQNNNDr65itMgeuu7xzx4UnZh3QK/JXUUknmAyGCuLh67LVE2zXbRKRSKuKi0WzUNXpnTKpP0/ypS6eFIa9GCybGdxPMrqKgs7//5y/u7t6xFCJLV53dBVAkfHAmohSYH8+KmDXPmDp3PeZejd47v6NYNewma/tQ6bJkFURoA5PSgKSnLxR+LJc6oY6LnuBeWhL0BDB+1/6WxUF4f44l04h32hjHksOv4RVdisyB56QFpUs4wypS4eFwY9kZBF7VwTKEGnlPQVffq4leKhByNxxvyRZAW/Qni9z0swGufVXu+0fXowbe+AKiOrv8/qkkguVmIJmay3Xk6klFpQNM1DdymDVEwdXff4Z9TQpVywhg6a9OHSZIJFbBY9nJRcylSjp4irRCEtKBqJKwnLkFzy87uX+3jLd55Idm9ZKIFocSWXbHQtuVI89GNa158xf6TgyoXdmt7+8tGJafv0YNpezUMvZbPf5OKiCpBdwrEE0bic5qFDcQz60FSI1/u8jPkjCDFLwDDiBxmfv4auadfDU+FUpsci6uhDk2EcVnP58tCLuEoUUgY9FI2rcrxGUDQ/r/V5icQTScO0UIIl7pRSaR5637j63BKysFWX4VicI6Mqt/qlo+MZ++IJmTReewemVE5/CZv96gHnSgiMTmpBYL2pCaRkkWIERm99aB/X/NtzvNbnpd5py9k8O8k8Ky3qTJNcYFFlgmFfmCZ3VflqoRfdQ1emNBRNlCXIrHNcGPQDmrc4WqRsgmSH+iLVQ89G99ArxaCnd/0Z88/uSR4eCRBPSNxVFl4+Op6hX4/5IyQkrGtx4wvH6B0PJjX0UuQU6516KiEwOhlU7zMjy6U6v+TynYf3cv8rxwoe//U+L+FYgsf3DRe2ShTm7aG7qyzYrSZ1t6V76IssuZRNPweY0v4vRfbQg9HyVlw8Pgy61i6rWDplsMSSS8pDrxDJJc2gF3JR1NuTvXNLGyO+CL3jqefrAdEL1ygjsHdgKtXstwQeep2zcgp06Wma7oy0RWV4c2W63PHCUe7flV32KDd6nXX9YlGqOi46QgiWuO0qHuLSPfTF8yqHp8K5uzAtFqMHVdkDW47GIfPAajZhNQtNcmlSF8fE4jt0FW/Q/eEYx7TyqcXy0AMll1x0Db0yPPTeDA999s9w/9AUQsA1p3cAmbKLnj983iplBPb2T6aa/ZbAQ/dUkIauF+ZKD4rarWbcdss0ySUWTzAeiDJQYGbR/kFVZ/0rb19PvctGu8c58xMW6KGDkl2GJsNa+QDzonqVQ+X20McOQMOqog5pt5g1D70RkKnyxotIxRv0QyOpXOhieejJPPQSLcvXG99WkoeuF5IaLcig+1ha72RzRy1Om5mXjqROTD07YUWDi6X1TroGJvGHY1hMApu5+KeTx1E5GnrKoGfeiTRWpy2j19AlogFvYefsHq3x9JtW1PPAZ87n769aP/MTFqihg1bPZSoEJpPWaWdxPPRwLI43GC1fDjrA6AFoOKWoQ9pt5pSGDmXR0SveoOv6uRDFCTyBqoVut5owzRR0WiCuKktSWy43xyZCnNauPLlC7nIODPlY1VSNxWzi1PZaXutLpS4Op63wW9/qpqt/ikBE9RMtRYDLYjbhtlsqxEPXgqJpHjqocrTZkot+JzTqDxONJ2Ydu6t/EofVzLIGF20eR/LOJC9F8ND11aKAMkKLpKHr3+OyeejBCbWoqH5lUYe1W01KcilDkFmn4g1695APs0mwttk9o4f+b08c4Kv3vlbQmIFIvOSF9Z02czL7o5wkEpJ+b5BlDS5q7JZZg6KxeIKDw35WNVcDsLKpmsOjqWqCQ5Mh3HYLdquZU9tqOTTiZ3AyVNL0s0pZ/j+Th57tbOgGXsqUTDUTXf2TrGt1z5zZko6uoS/AoHucNiZDMWLxxc3MGC73sv+xA+p3kSUXh9Wc0tDB8NBzcWBY3f631tqTjXOz2XZglFsf3su9Lx8raEVhMWuh58Nls1RElsuwL0w0LmnzOGiorppRcvn1jh6++YcuIvEEq5e4AVjR6GTMH8GrpTvq6WYAG9trANh+eLykjQrqnJWx/H8yFKVZjON64YcQS32ODdXTC3Slf84Ds7TQk1Ky59gk61trCp9MyKtWJJrmfx7rAefJUEx5lYvkUQ5pcYWm6jItKhrVWjU0FNtDT9fQWdSsIZ3KN+hDflY2uWiormJkarox8gajfPHXu5BSLXDxFpBnXcx+ovlwVpkrQnLRV9i2e+zUu2x5g6KhaJy/uedVfrbtMG67hTOWqapxyxpUFsBhLZYxNJnKTji1TXmHI75wSZv9epy2CslyifGJqj9hevwb8NStye2N1VVMBKI8sW+Iy7/3JFOhaIYEM1vJhWPeEJOh2BwN+vzruOjoss54IKJJLotj0PV4Q9kkl9FuQEDdiqIOm2wU7ahX4xuSSybxhOTQiJ+VTdWqCJI/PM0Df+j1fo55Q3zyAvXPKaQdWCAaL1rp3HxUioeuLypq9zhnNOiT2oXw6+8+lde+dgUrGpUh138f1gp1DU2Fk8vGl9TYk1/K0kou1oqQXCZDUd4iXgYEPP1dOPoCkFpc9A/372b/kI/uIV/G5zybh951TAVEN7S6C5/MAuq46NSmNWbB1aTGjJX+cx6eCiNEAcXHSsXYAajtVM09iojDaiYUS4DZAo46Q3LJpnc8QCSe0Ay6jWhcJhd36OjBsis2qpK1hbQDC0ZiResnmg+HzVz2pf+haJxtB9VtX5vHTmO1LW9gWb+zqXVkBvyW1jsRQmUbSSmnLQg5tU15laXsPONx2iqia5F1qo9V8ghc9LfKIPxB9XPRe3Tqq2t7x4OM+iPUOa3YLKZZPXQ9k2tV0xwM+gLquOjopQW8wYjKcoFFkQmGp8LUO21YS5AVVRAlyHABLSiqO3GupkW740mnog26nuGycokrtcQ6S0f3h2MIAaub1ZehkHZgiyG5uGzmsnroPWMBzv/249zxwlHOX9WI226l3qWCi7nqueQz6HarmbZaB4dH/PjCMYLReMaCkI2a7FLsfqLp1DltTIVjBWWLlJK1k8+oP057P5z1CRh8HaYGkuemXqagbyKYbCHXUmNnYDKz3HA2R8b81DqsSY+5IIrgoSdTQv3RRQ3kzZiDPtlfWqlCSs2gFzcgCrqHrhv0xYtJpFPZBn1IeS6nNFYnb89GsjIGpsIxqqss6gvhsBbooZc+KOqsspQ1D/3RrkFGfGH++y/O5H8+fjYA9a4q4gmZrEmSTj6DDrC80cmh0UAyWyPDQ9cCo6UMiq5comSf9C5J5WCT/3kGLB3QuAqWn6c2HnmWVo/K8f/4eSvwOK30jgcY80docNmUQfeGuHtHD2/9/lM82z39S350LMjS+lkWEmVTFA1dk1yC0bRAXumN0IgvPL008MDr8NO3w/fWwfc2wMM3pzJ5iklgFMLeoqcsQpqGDoZBz8WBYR8NLht1/qNuAAAAIABJREFULlvyBMjO0vCFYrg1Y9JZ7yhMQz8JPPRXe700uau4ZN2SZH54cpl6Dh19RoPe4OLwiJ/9g+qOqaU2pT3qHnopNfTL1jfjrrJwz87ekr3GrET8bIq9Spf7XPW4ZTPY3HD4Wdo9Dn73f87lf1+8ko46hya5hGmottFcqzz0P+1RxaC++Ycu4ll3SEdH/SxtmKNBD04s2EN3260IAd5AZFFzp0d9kaRMBUAiAfd9Gob3wltuhlOvgW23wU+uhMnCa+EURP8r6nfTmuKOizLoIa2siMoaMjT0DA4M+1jZpPKhG/MUQfKFY1RrecGddU56xgMEI3E+f9euZGZGNoFIrOTdxp1aULTQcrXF5pXeCTZ31GYs9tHrg+QKjM5k0Fc0uvAGo9z2eDettXbOWl6f3NdR5+DKjS28eWVDsd9CErvVzFWbW3no9YHyxSVG3sBGjAHPFvXYbIGlb4IjzwKwdWkdFrOJDo+TPk1Dr3fZaKmpot8b4tkDI5zS6KKrf5LfvJS6MMUTkt7xOXro8ShE/QvW0M0mQa3DmumhL4pBD2e219tzL/Tvgrf+I1z0N3D1v8FH7wNvL/z3W2Gku3gvvu+PYHHA0nOLN6aGMuiaLOhqUkv/44t7vhZk0IUQVwoh9gkhuoUQX86x/3ohxLAQYpf284liTO7AsJ+VS5RBr3Nac64W9WmSC0BnvZPe8SB/6hrkdy/38eeuwZzjBqOLkIdelVZ9bZGZCkU5OOJnU0fmF16XrXKtFtUNevaiGVAeOqgyxh998/KMYJYQgn//yBlcsq44Vevycc3pHQQicR58rb+kr5MPqXmKserW1MZl5ymvMs0Ittc56BkPMBGI0uCqornGTiSWIBRN8PdXbWBLp4fbHk8ZqH5vkFhCsmwuBr0Iq0R1PA4tx1+v51JiySUYieOPxFMZLvEoPPp1WLIBNl2bOvCUi+D6ByAahJ++DQYLb5+YFylh34Ow8hKwzfGOqAAcVjOReELdgSXruSyuTDirQRdCmIHbgLcBG4DrhBAbchx6l5Ryi/bzXwudmN5hZ2WTMiYWs4l65/QFHFOhWDIHuqPOQSSW4JfPHwFSWQfpRLQmBaXOckkW6CqDjv5anxcpYVNH5he+waXLVtMXaHmDUVw2c87Mg+Va6qLDaua6sztLMOPZOWNZHcsbnHMqR1tMouOqaqJ0t6U2Lj9f/da8dFDnoO6lNVTbkvKU3WrizSsbuGJjC0dGA3i17Kyj2jk6Jw9dv4A4F35XVKvn+JtM4KwveZaL/v3VOz3R/WcYP6SkluxFUm1b4GN/BJNFGfXuRxf24v27YLIP1r19YePkIVUTPb6odzzpFOKhnw10SykPSikjwJ3Au0s7rfQMl+rktoYcNTN84VSPx8469aV44ZC6Kh7JkfGSbG5RwiAepJXQLcPy/9e0tnHZHnq9y4bVLNiVowuRNxjNKbeAMjZOm5n3n9kxe42REiGEYEunJ+dFutQcGPbxWlcXMWnCUpPWEKFtK1idcDjdoKcMc70WFAU4b2UjdquZ9VqueZfWvu+odo7OSUPXtVk9M2UB1DmtqcV4i7BaVI/fJD30Q0+DuQpWXZb7CU1r4IaHoKYDfvk+eOo78+99uvdBECZYc+X8nj8Luk3xhWNpMYnF1dELMejtQE/a415tWzbXCCFeFULcI4RYsBun10Bf1ZQy6LlqZvgzJBdHcnu9y8bRHF3rA9Hi9xPNhe6hlyMw+mqvl446x7Sa2jaLiQ+fs4zfvNRLl1bdT2cyGM3oxJP9vAc/ewE3v2OWCoAlpt5VVbQ2hIUy7o/wnh89y6FD3YyIOja0p+IHmK3QugWOvZTc1O5JnYMNriqWNjixmARXnKrWSeirQfdqn/+RsQBWs6C1NvW8WSmiQVeSi/aZOhtK7qHrDlkyy+XwU9B59syLfOqWwccfgQ3vgce+Cf9+Puz9w9zqjScS0PV76HxTynsuMq3axbvfG1rUrKF0ihUU/T2wXEq5CfgT8LNcBwkhbhRC7BBC7BgenvnKdWDYR5XFRJsn00hP89BDMaqrlCHSvSOXzcx7t7bTOx5UhYfSGNT6YTbM1kBggegaejlSF1VANHfA7K8uXY3bbuWbf9iTsep2Jg8dlOxSZSlT/0eNepeVqXCMcGzxLpL//cwhpsIxruhM0Ny+gi2dWZ9r21aVcqcFv9rr0gx6tY0lbjtP/s1beP8Zqrb8EncV9S4bXVo/1qNjATrqnIUX5YKU0S2GQXfaUpUsXYth0NM89MCY+uyWXzD7E6uq4f0/hevuglgI7vwg/Mvp8Ng/wsBrSh+fiUf/Lwx3wel/UYR3kRvdVh2bCKbl9S9uPZdCDHofkO5xd2jbkkgpR6WUuqX9L+CMXANJKX8spTxTSnlmU9PMJ+OBYT8rGl0ZJ7on/fYQVUnQF0lluditZjrrHVy2oZnVzdXEEpJjE5mr9I5oXruuC5eKlIa+uB76fzx5gN7xIG86pT7nfo/Txl9duppnu0d5vS/lpc9m0CsBvR3dYpXSnQhEuP25w7zjtFbckSFETev0g9q2QCwII/sAlSWU3XWo3eNIZhsJIVjX4mavLrmMBuicaw66fxgQSvNeILUOK1N6xcVFkFz0hYENrio48hwgYUUBBl1n7ZVw0054/+1qte7T/6w89u+uhd98Al78T+jZrtI6E3G1UOnJW+HZH8AZH4PNHyjJ+4LUxbxvPKiW/iMWXXIpREjeDqwWQqxAGfIPAB9MP0AI0Sql1NMP3gV0LWRSUkr2DUyxZWmmN+Rx2PAGoyQSEpNJEIjGkZJkHjrArz55DjUOK7v79FvazBzfwyPzCELNg6SHvohpdne8cJR/+uNe3rm5jQ+9aVne4y5co24HD474OE0LnB4PBj3V7i1Cc03pK/X9fNsRfOEYn7l0FfykX2VHZNOqpTEe2wXNGwFor3MyNTCZXFqfzfrWGn75whHiCcnRsQCbO+eYreIfVsZ8AZUWddIrLtY7G1SqXSJelLFzMeqL4LKZVZbZ4WfAYof2nP5ffswW2Hi1+vENwRsPw8HHlR7/2t25n7P6Cnj7d1RjhRJRY7dQXWVRBfFM5kVtGqIzq0GXUsaEEDcBDwNm4CdSyt1CiK8DO6SU9wOfFUK8C4gBY8D1C5nUn7uG6JsI8vnLM5P/ax1WEhJ8kRg1dis+rT51dVqqnS67LG9Uv4+MBrhgdWqMI6N+2mrtyaaupcJVBg/9F88fYVNHLd+7dvOMt/D6rWF6r9DJYKziDXpdenXARWDvwCQrm1ys8wCRKXDn8NAbVoGtGo69DFs/BKhMl8HJUN7/wboWN6FogucOjOANRufuXPiHiyK3QGbFxfr01mkl0pkzctAPP600bcsCqi5WL4HTP6J+pISJoyrFcbQbogGwOlQQtGltcd7ADAghaPPYUz18q5dUnkEHkFI+CDyYte2WtL+/AnylGBOSUvLDR99gWYOT92xpy9inGxxvIKoMeljdelfnyFhpdtuxWUzJLAKdw6P+ZEnYUuIoQ6NobzDKOac0zFr0yGmz0OCyJQ16JJYgGI1XvEGfaWFUKQhHE+r/OKXdfNa0TT/IZIKWTSolTuOjb17GeTMstNIDo5/51cv/f3tnHidnWeX776mtt+p9TbrT2fcESEjYJOzKogKCKAwKjjq4oSjOVRA/XsTLXAZnGOeOziguE3QQDKIX8CKbEBaHJQECWSBbJyRpkvSW3ru6tuf+8bxVXd2p6q7ufmvpruf7+dSnu99666m3TiqnTp3nnN8h3+3gg8vqxndhfW22OfRhiouRMsi+ttQ59D6/zp/7+7QWzjm2uA2NiN5ALU/87TTVzCwriEpW462B3vi9MKki6zpFn323hW3N3Xzl3AW4RjimSBVGJI8emSATz6E7HEJjRWE0Zx7hvfb+aPSeSooyUOXS2e+P6nOMhW5R1x920S7R8YhDZYDyNDt0XzCkN4Ij7efxHDroPHrMxui6hdV85gOJtbYX1HhxOoTO/gB3XLoiKlGcNH2ttjncdCsutvYM6vx52259oCazlVN2U19WMBShF9XolFAaySqHHgyF+eGTO2msKORjq46vjIw4q4h2d+/g8SmXWGZXFA6rW+72BWjv86clQs93OxBJXw49EArT508+yq4vH4okRmv7zybKCnS3cDoj9Hy3YyhCj5dyAV3pErMxOhb5bifrFlZx9dpZXLWmYfwXZmfKZZjiYupL7dr7LB2Xtl36QJX9miqZZGZZAcf6A/qbuddy6ElMUbOLrHLoD2w6yLtHerj14iVx0walIyL0iK5HvAgddLPGgY7+aHlepCtvzniFkCaAiFDkcaUthx6xSfIRutYcUUpFH5uoDj1bcDkdlBakb9iFLxgiP5kIPbox+mbSa6//21O468oTxj9YO+jXrf+25dBjFBdjUy4pIBxWWoUy4tDFmRLVw0xSH1u66K3RH/T+3rQ9f9Y49CNdPu55aienzavgohXxc4oRh96ZRMoFtAZJvz8UHUobmbqTjggdrEHRacqhR0r5ko2yG8oLGAyGae0djH7jyfYIHaCi0DPqXFQ78QXC5Lkd2qHnl+kNtnhULoC8Umh+PfUXFUmH2ND2DyMVFyMpl9Toj3QNBAiFlW4qat0JFXPBlaGpRSkiWrrY6QOvpW+UxrRLavvfk2DAH+LL97/O87tacTqE731kecKoZWSEHkm5xBOUgqHxaXtb+6gpyY+mX2anIUIHKMpz0Zum1v/xpk0ikUTzsYEpk3IBvTGarm7RwUiE3nMYSuI1R1s4HFC/Cg5tTv1F2dglCiMUF115evB0ilIuEQ2hSm+ejtCnWboFRjQXVVj/Rr0ttg+kTkTGI/Q9Lb08t7OVj61q4M83rWPZzMSDcgs9TtxOGXLoVoSeaEDxAksHZo+lC7O/rY+a4ryUS+dGKMpL3xi6rgHt5JLVWomUdx6aYg69fJS5qHYzLEKP11QUS/3JulzOn2KtGZsdOsQoLkJKBboish1VBQ49NWgaOvTa4jwcYjUXRSP09FW6ZNyh91jTc65a08CCmtFnKoroaCI2Qs93OxKW6c0ozafQ44zqwrzX3h+Vgk0HxXnu6OtLNZGUS9k4NkVhuEMvyc9+h15RmD6HPhiwqlx6jkDxGKWF9WtAhYYGKKSKSH7bRoceVVyElHaLRtr+68JHIRyYlg7d5XRQV5I/lEOHtNaiZ9yhd/tGT5uMpKTAHZUe7YkR5oqHiDC/2sve1l6UUuxu6Rl/idgkKM53RfP8qWa8UbY3z0VZoZvmzn66BgIUuJ14XBl/O4xJhVfPRVVpqBzwBcPkuUSnIMZyoA1r9M/mFKddohG6fXXiZQWxiouVKU+5VPn26wNpaPbJBNEKssJKre6YixF6cV5yjmhYhO4b3aGDTrvsaellb2svx/oDnDy7fHIXPA6K891pc+iRCH08lSqRcWlToe0/QkWhh0BIRfdPUoVSCn8wTKkMQDg49iaktwZKG1O/MdrXCg63LcMtIpQXxlQOFVWlbFO0rdePCHh7mvSBqoWjP2CKUldawJFun9X+X5XWTdEscOjji9Bjo4m+mPFziVhQ4+Vwl4+NO3Vks3bu5AWNkqWkwBWtIEk1XQMBSvJd41Ltaygr5GBH/5Ry6OlqLhoMapXOEmUJmBUmERE3nAyHUuzQ+60uThs1SYYpLhZW6pRLCr4BdQ8E8Oa5cLTvBm+drR9K2URl7Ma9tzY3HfpYjjlCbIQ+VsoFiE48euC1A1R589JSgx6hON9Nrz+YlrmiXQOBcXd6NpQXsLe1j6d3HE26fj3TVKbLoVtTh0qjDj2JMsH6NdB1QOfcU0UK2vKHKy5WQmgwJbXT3T4t2UHb7mkbnYPuvu32BQmEwuCthr4cKlvs8en87Vj6IxFKC9zRDZxeX3CYXno8IpUue1v7uGRl3fgbOSZBSb4LFSMmlko6+/2UFYyvpvf6M+bgzXchCGcvtm+TLZVEIvRUNxf5LM31kpA1v7MoCYceGUnXtDF1Mq02dolGGKa4GO0WbYe80YsUxkuPz5ou1nkAFiaYUDQNKC8a0sep9tYOyRykgSxw6MGk0y1gRRODOuqNHT+XiNmVRbgcQjCshk2rTweRa+vxpd6hdw0Exh1lz6oo5OsXTK1Kg4rCxIOu7cRnDfcuijj0ZCL0uhN0ambPX1Ln0LsOQfUSW5ccprgYHZ3WBuVzbH2e7oEAlfkKjhyBsswJaKWaWFXQ6qLqofb/NASTGU+5JOOUYykpcKOUdpK9SaRc3E5HVA89/Q5dO9h0lC52jjJCbjpR4U1PhB7JoReFrPmryeTQHQ6tmb73WT3yzG58XbpiwuZ0xTDFxeikHftL7Xp8QWa7rA3Xskbb188WhqmCemt1CsvXlZbnzrhD7/YFoo4vGcqi6nABen3BhE1FsSys8VKc54rKlqaLyAdV90DqK126+gNJ16BPZYo8TjxOR8rb/yMRekGgUw8x9iRZ7rrgfL1xeeRt+y+qbY/+WWmvQy+LdmD7h2qnU7CR1+0L0ChWSWTppMcOZy3RCL3Pn/Za9CmZcgFo7fXhD4WTeuzff2gxR05LPHAgVZSkKUKPCGxNlY3NySAilBYO9SKkikiEXhA4ZtUTJ/nemXeu/rn3L1pW107arVyszQ05Qw4oJkJPgUPv8QWZieXYciFC7/dDdeQD0v5vVvHIeITe4wtMyKFHhjOMlXIBWFhbzLqF6d/0i82hp5I+f4hgWE2Z0sPJElvplCoiEXqe/1hyG6IRimuhdqXOo9tNRKHQ5tz2MMVFd74WGrO5MiMcVvT4AtSEj4LDlViKeBoQseexSMoF0tZclAUOPZh0UxEMOfS9rVo5MRmHninSlUOPVP2Mt8plqpIOhx4pW/T4O8evbLjkEj0A+dh+ey+qbXdKFAqHKS6CLrWzOULv8wcJK6gMHtVCZ87s/X87WfLdToo8Tjr6AkMfXBEJ5hSTHQ59HBF65NPvNy/vx+0UTh9l1FemiebQUxyhT5WJQ3aRlgjdKlt0+dqT2xCNZfV1OkXz+np7L6ptt+35cxihuAh60o7NOd/It9Qy/+FpnW6JUF6kJSrIL9UzZ7sOpeV5M+rQAyE9y3I8m6KRCP1Yf4ArVjWMWYeeSfLdegMv1SmXrnFqoU910pNy0RG6y3ds/BF6aQMsuhje+A0EB+25oHAIOppSlocdpriYggi92/qW6h3IDYdeEVEFFdHvh2xy6CJykYjsFJE9InLLKOddKSJKRNYks27fGHrm8ci3RKQcAl86J/unnRTnu6Jv5lQRiaxyYVMU0pRyCYZwE8Th757YMIm1n9XVLjseteeCOg/o8rcUOfRhiotFNbbn0Ht8QTwEyPe15IRDLy/0DJXWltRDd3NanndMhy4iTuAnwMXAMuAaEVkW57xi4Cbg1WSffLw6LhEaKwq5fFU9c9KonDhRSgpSL9AVHT+XIzn0iE1DKZRU8AXClNGj/xjPpmiEeefpSUYb/8EejfS21FS4RBimuOit0XXTdn27QDcVzZB2BJUTDr0iVre/tB66ssShA6cAe5RSTUopP/AgcFmc834A/CPgS/bJI5HreFIuAH/48hncdcUJ43pMptASuqneFM29lAukdrN5MBiiQiyHPpEI3eGAj/xIp0me/cHkLyhSspiCHDqMUFxMQe10jy9Ig0z/ksUI5YUxAl0lDfobj40fkIlIxqHXAwdj/j5kHYsiIquBWUqp/zeeJ59ohF6S754S2t2QHk30I10D5LsdekJ9DjByFGEq8AXCMQ59gmJYc9fB2r+DV/4Ddj05uQs6/Jb+YJnIt4UkGKa4WGR/c1G3L0BDpKkoBxx6RZGbPn9Il7+WWu4yDWmXSXsAEXEA9wDfTOLcG0Rks4hsbm1tnbBDn0qkemqRUopnd7Zw+rzKtAqPZZJ0OPTBYIhap6U4OJmBzBfcDnUr4cG/ga2/n9gaAR/s/LPeaE0RwxQXUxSh10srSpxQPNO2dbOViIhcZ39Ab4pCWtIuyTj0ZiC2T7fBOhahGFgBbBSR/cBpwKPxNkaVUvcqpdYopdZUV1cPDbeYAqPPJkpxviulrf87DndzsGOAi1aMMSJtGpEWhx4IU2OHQ8/zwmf+BLNOhYc/B3/6Bvi6x7fGnmdgsBtWXDHx6xiDWMXFVHSLdg8EaHS0IyUzp3UNeoSIiFxHn1+nXCBrIvRNwEIRmSsiHuBqILp1r5TqUkpVKaXmKKXmAK8AlyqlxpzFlRMRen5qI/Qntx3BIXDB0tqUPUe2kZ6US4gqR8ShT1LULb8UPvUwnH6jrk3/ySnw5v3JC3hte1h/qMw9e3LXMQqxiotDei72dTd2+4LMcrRPaw2XWIbJPJdY30i6Do7yCHsY06ErpYLAjcCTwDvABqXUdhG5Q0QuncyTD0Xo09ehlxS46POHUlaR8cT2I6ydU0GlNy8l62cj6Um5hKl09Ghn7LThG6S7AC68Ez73jP4P/siX4d6zxpYI8PfBridg2eUpjWyHKS66C8BTbGvKpdsXoF7aoCw3HPowxUVPIRRUZE3KBaXU40qpRUqp+UqpO61j31NKHVdkq5Q6J5noHPTEIY/LoSerT1Mi6aTeFGyMNrX2sutob06lWyB9EXoFPZNLt8Sj4WTt1K/8pU69/NcVcN9H4eCm+Oe/8E8Q6IcVV9p7HSMYprgItjcX9Q0MUq3ahvLJ05xYTXRAv+4sSbmkDD34YfpG5xDb/m+/89l1VFdhpFvnPdPkux14nA66BgK09gzyu00HbH+OwWCYSjqHKj7sxOGAlR+HGzfBRXfB0R3wywvgVxfD2w9pR9rbCs/fDS/dA6s+DbPPsP86YhimuAi2t/+7+o7gJJwzKZdIk99QLXpDWiL0jHpTreMyfTdEgegH1oGOfh59632+cNY8XEmO2xuLyGZrrnSIRhARSgrcdA8EeOC1A9zz9C7OW1JLdbF9aSdfIES56gTvHNvWPA5XHpz2Je2wX18Pr/4M/vD54ecsvwI++q8pn3YTVQiMrUVv3Wnb+t4BS5wqR1IubqeDknzX0GStknrY/9eUP2+GHXogq9US7SDygfX9x7az62gv6xZWcUJDmS1rR6L+XJhUNJLSApcecjKoRbSO9fttdeiDwTBlKkUR+kjyvHDGjdq5v78FDvw3iANql8OcdeBIfUqytMBNaYGbpjatYoq3Bva9YNv6xX5rg7V0+tegR2isLOS9DqtLuLQeBrt0mi0/dYN2siBCn+4OXb++XUd1xUSnjYMZugcCiIDXM71tGI9II0zEnh02TzAK+AcpDvcMVXykA4dT59gbTk7fc1qICItri9l5RKfxDgbLmOXr1LIFnsJJr18ROKITvDmSQweYX+1l8/5j+o/yufrnsX0w48SUPWeGc+jjG24xFRk5HNrOjbxuX5DiPBeONE9iygZKLXXApjb9QXnMZode4LdmXxalfzBKplhcV8yuIz0opfjL+/p9e7S5adLrDgZD1IZbGXCX2fLhMFVYUO2luXOAfn9wSIOndVdKnzPjm6LTPYdeXuTB5RCuWK3bfzvtdOg5Mhg6HqUFbva29EZlbjtsHhpdFLQcujd36vsX1RXTMxjkcJePV9q1433n3e2TXld3ibbRVzD9O0RjmV/jBaCptQ8q5+s0Wts0duid/YFpLyhVWuDm8ZvWceflK4GYqTA20O0LHPcNIFcoLXDjDw015tiZygLwBqyvyulMuWSYxbXFAGzc2cq2Pp3nPbRv96TWfPdINwc6+qmXNgaLcsyhV2uHvre1V2+Al8+BNvs2muORsXxHWCkGAiFqS6Z/Q8wi6z9Kocdpcw49SEnB9E5ZJSL2m4nTIbbn0EvClkPPpZSL9T594LUDHFHlhBF6WvYTCqsJDVgfDIa47Md/RSnFFlc7rcW5kz8HmFNViEOGxmVStXhIBjlFZCxCD4R052RNcX6mLiHtlMWO+bKBXI/QASqLPNSV5NueQy8N5V6EXlropq4kn63NXbjcHgbzq6kMtrC1uWtC67X3+hkMhpmVN0ChDJJXOdvmK85u8lxOGisKdYQOejhJ+x4IpU7bKYMOXX9drsmBCD1CaaxEqQ3keg4d9NfaiqKY6TA2EA4rylUnfkcBeLJ/iIqdLKrTUfoJDWW4yxuZ6WjjhV0TazCK1GD/r/NKAahrTI2WezYzv9rL3hbLoVcvhpAfOt9L2fNlzKEHrQi9tiS3IvRoa7UNdPuCOR+hz6/xUlbopsPGD8rBYJgq6WLAk70DyFPF4lqd913dWI6rYhZzXcd4pakdgLbeQW7esCVpsbm2Pj3QoSZk1aDngA76SObXeGlq69NaTlWL9cEUboxmPkK3sRkk2ykrdNsWoQdDYXoHczeHHnHoC2qsCN3GlMtgMEQVXQzm5ZakAgzt96xuLIPSBmpUGzve70IpxVPbj/KHN5p5palj2GNef+9YXCffYUXoFT5LZbBiXmovPguZX12EPxim+djA0DzY6ejQg2FFocc57TtFYykrtC+H3msN2M7VCH1+dRFzKgs5Y37l8IG8NuALhKmSbgbzJzipaApz4Yo6vnb+Qs5aVA2ls3ArP46BDo50+6K59GhOGGjp9nHVT/+b//zr/uPWarcidG/fAd1xm1eclteQTSywShf3tPZAQZkug01hLXpGI/TakvycmbIDUFrgoas/gFKTl9KN6Ljkag690pvHxv9xLktnlFBe6KHHF4x+65ssvkCIKukikIMOvSTfzc0fXES+2xnt6pwpbex4v5vt72uHvqdlyKFv3NVKWOnyxJG09/rJczlwde3Tddg5yIIa/SEW6RSnalFKSxczmkO3U3tjKlBWqGunBwKhSa8V1XGZ5p22yVBRNEJYapIM+v1U0ENoorNEpwuWQ6+Xdt461MW7h7UsQKxDf35n63HHIrT1+qny5iEd+3Iy3QI6NTizND8qqUDdSjiyDYL2VmVFyHiEnktENKftyKN3D+SuMNcoSgHiAAAXGklEQVRIhs1vtIFgTysOUYQKcqcGPS6W1O1ybzePbmnGHwpT5fWwt6UXpRTBUJgXd2uHvq+t77hvSO19g8wsCkPPYaiYm/bLzxYW1xXzzmHrG0zjaRAc0EO/U0BGc+i1ORihg00OPRqhG4c+bH7jOHnrYCfPvjt81Fo4Mtghh2rQ41JQDq4ClhZ0s79dqwZ+9MSZ9AwGae0ZZMvBTrp9QS5YWkMgpHjPOidCe6+fJR5dIZOrETrA4roS9rb26g+8xtP1wQMvp+S5MubQw0rlVA066Bw6QKcNpYtDOXSTconOw5yAQ7/9se3c8vDW4QcjDj2HukTjIgKlDcx1aadcnOfi/CVa22ZPSy8bd7bidAifOWNu9Fgs7b2DzHdZH5Y57NCXzigmEFJa08VbAxXzJ+zQt/3wwlHvz6iWS86lXKwIvcvOCN2kXIbmN44zh36sz8+Wg5209AwOmygl1qQeR3HuCHMlpHI+tUE9aWd5fQkLrTr1nUd7eHzrYVY3lrGqUev772npiT5MKUVbn59GjugDOezQF1vNWtGN48bT4cAryQ8Jt/D1dbGk97VRz8moQ8+ltn+ISbnYULqYy1roI5loKuvFPW1ECo72xkSXfc5SXgitxFliHDrVS/D2NOEiyIqZpdQU51Gc5+LnLzTR1NbH335gLkV5LurLCoZF6L2DQfzBMDNC70NhlR62naPMq/LicsjQxujs02GgY9z16O+99TwuGf1DICmHLiIXichOEdkjIrfEuf+LIrJVRLaIyEsisiyZdXMt5VJWYN/mXS5roY8k3+2k0OMcdw59484WXJb9Yp3RgcoPcF3gVjyFueuEotQsRcIB7jrXy6dPn42IML/Gy/tdPpbUFXPRcj2gfEGNl90xNoz8W1QONud0dA7gcTmYX+3l3YhDHyOP3jsYpGvg+PLm3l0vElKj/38f06GLiBP4CXAxsAy4Jo7D/q1SaqVS6iTgbuCesdaF3Eu55LsdeFwOm3LouavjEo/ywvF1i4bDihd2tXLh8jrcTokq4h3r8/PLl/ZR6HFGUzk5TfUSAD7e0MPsSq1rE2mW+foFi6IBxYIaL3tbewmHtRNqs7pESwYO5rxDB1gyY2gaFBXzwFsHe54Zds7Le9u5+t6XOfH7T3Hi959ixf98Miq7AFB0dBN7HaNXCyUToZ8C7FFKNSml/MCDwGWxJyilYrsKioAxO2ccIjnVJQp6zFdZgdu2HLqpcBliNIGuZ3Yc5cHXDgw7tv39btp6/Zy/tIY5lUXsaenFFwjx6V+9SlNbH/d+eg1FOfb+jEvVIkCg5Z3ooStXN/CZM+Zw4fKhlNTCGi++QJjmzgFAb4gW4iO//3DONhXFsriumObOAY52+/Rm84orYdeT0K9lFFp6fHzxv17nYMcAN5w1j+9+eCkOER5+/RAAKuhn9sAODpedNOrzJOPQ64GDMX8fso4NQ0S+IiJ70RH618Za1OXMzVTBZPVcnth2hOd3tWphLlPhEqWs0M3+9n58I5q2dh3t4cu/fYPv/HErTTEt6/e+2ITbKZy1qDoaXT77bgvbmrv556tO5MyFOd5UFMFTqGvIW4cc+unzK7n90uXDuryXzNADMd4+pLtJ2/v8LBVLVbBuZfquN0u5ZMUMXA7hR89YefOT/gbCAdj6e5RSfPeP2xgIhLjvs6fw7YuW8Pl18zh3SQ3PvttCKKxo3bOJAgYJzzpt1OexbVNUKfUTpdR84NvAd+OdIyI3iMhmEdlcgL0TZqYKZQWeSaVcbn90O9/c8BZtvYMmQo/hitX17G/v45M/e5mWHh+gRbZuenALxXku8lxO/u3ZPQA8uf0Ij731Pl89byFV3jzmV3s50NHPH95opsqbxyUrZ2TypWQf1Uuh5d1RT1k+s4R8t4NN+3XE2d47yErHPn3njNGjylxgTlURnz59Nr/bdFCnXupW6A+6t37Lxl2tPLXjKDd/cFE0nQXwwWW1tPf5efPAMVq3bQSgdsU5oz5PMg69GZgV83eDdSwRDwKXx7tDKXWvUmqNUmpNY23uKdmBHiIw0Qj9cNcAR7p9tPUO0tTaZ3LoMXxsVQM/+9TJ7Dray1fuf4NwWPHPT+3incPd3P3xE7jujNk8sqWZHz2zi9v+uJVlM0r40jk6FbCgxksorHjmnaNcsrJuQtN5pjU1S/RghuBgwlPcTgerZpVHHXpbr59V7v1ajKrEfEAC3HT+Qrx5Lv7u15v57v/dytF5V8D7b7LjzZcp9Dj53JnD8+NnL67G7RSe2nGUvL1/pknNZOH80TXlk3Hom4CFIjJXRDzA1cCjsSeISOyzfBhI7ZylKUxZwcQd+psHOgEotvRbTIQ+nA8tr+MHl69g0/5jfGPDFn7+YhPXntrI+Utr+cJZ8ylwO/nRM7up8ubxo6tPwu3Ub//YqOgjJ+TW3MukqF4KKqSd+iisnVvBO4e76fEFaO/zs9LxHsw4MU0Xmf2UFXq45xMnUVuSx4bNh7jzwArweDll7//hpIbS6PsxQkm+m9PmVfLqS8+wYGAr++Z84rhzRjKmQ1dKBYEbgSeBd4ANSqntInKHiFxqnXajiGwXkS3AzcD1E3i9OcGCGi9Hun3D8rnJ8uaBY3hcDr51oRbKNzn047lydT3nL6nhkS3vM7eqiO9+WBdkVRR5+NPX1vHit87lia+fFdX9BphXras36kryWTO7PCPXndXU6EqX2I3ReJwyp4Kw0vroLR0dzA4fNOmWEVywrJaHvngG16ydxVPvhej7wLdZG9jM1cXxtV0+vHIGf+t8HL+ziPOuuXnM9ZPKoSulHldKLVJKzVdK3Wkd+55S6lHr95uUUsuVUicppc5VSm1P/iXmFh9bVY/TIWzYfCjpx7T16q+6bx7oZGV9KVetmcVZi6o5ZW5upq1GQ0T431es5OIVdfzbNaso8Dij982tKmJWReFxjyn0uDhlbgXXntpo6vrjUbUInHnQ/Maop61qLMPpEO55ehf+Q1txEjYRegLOXlyNLxDmx33nsSM8mwsP3APdh48775OLHVzmfg3P2uuRJJqzMtopmovUlORz7uJqHn7jEMEk9Ltf3N3K2juf4c9bD7O1uYtVs8rIdzv59WdP4Yz5phIjHjUl+fzHp05m+czkG4M2fOF0vnp+7s28TApXHjSeCk0bRz2tKM/FipklvH2oiw9VWM5pponQ43HavEo8LgfrXz7ELcG/wxPqhV9dCO17h07qa0MeuBoRB5z6haTWNQ49A3xizSxaewb59cvv8ey7RxnwJ9ZHf2FXK0rBNzZsYTAYZlWjSQkYMsC8c6FlO/QcHfW00+dX4XYK1zQc0y3/JcdVOBvQ3wpPnVvBQCCEv+ZE5PrHYLAH/v10ePjz8Odb4BcXQNtuuPoBKJ+T1LrGoWeAc5fUUFOcxx1/2sFn12/mly81JTx30/5jzKooiA7VjgghGQxpZf65+ucYUfrXzl/AU18/i7LWTVC/WjfRGOJy9iKt5nny7HKoPxlueA5Ovh52PwVv3Kfli6/9PSy8IOk1jUPPAG6ng/s/fyo/v24Ny2eW8PSO+FFPvz/ItuYuPnrCTL5zyVLWLaxiRmluySUYsoS6E7WDGcOhF3pczA2/Bx1NsPji9FzbFOX8pbW4HMK6hZZMc/kcuOSH8K19cGuzdvBz141rTVMmkSEW1hazsLaYnUe6+aendtHS7aNmhLbNlgOdBMOKtXMrOHdxDZ89M3envhgyjMMBc8+GpudAqdEj73ceAwQWfzhtlzcVmVtVxMu3nk+Vd4RmkMMZ/wFJYCL0DHP+Uq2H8dzOluPu27T/GCKw2uTNDdnA/HP1OLkjW0c/753H9Kg1oyc/JtXFecMkFCaLcegZZkldMTNL83nmnXgOvYMldSWUmo5QQzaw5KPgKoBXf5r4nPa9cHQbLL008TmGlGEceoYREc5bWsNLu9uiwlL+YJhHtjTz+nvHWDvHROeGLKGoElZfB2//DroS9FFsfUj/XPqR9F2XIYpx6FnA+UtqGQiEeHWf1sG44TebuenBLdSU5HHtqbMzfHUGQwynf0Xn0F/+9+Pv6zoEf/1XnTsva0z/tRmMQ88GTp1Xgcfp4KXdrbT1DvL8rlY+d+ZcnvvmOdF5hAZDVlA+G1Z+HDb9Ava/NPy+J27Rzv7iuzJzbQbj0LOBQo+Lk2eX8+LuNp7fqRuJLj+p3rShG7KTC/9Bl9j99pOw4xHo2AeP3aQ3Q8/+lonOM4gpW8wS1i2q4u4ndrJh80Gqi/NYPrMk05dkMMSnqAquewTWXwIbrtPHxAmnfQXO+Gpmry3HMQ49SzhrYTV3P7GTV/d1cNXJDSY6N2Q3JTPgi3+FQ6/B0e0w7xyoXZ7pq8p5jEPPEpbNKKGiyENHn5/zltRk+nIMhrHxFGpHPu+czF6HIYrJoWcJDodw5oIqXA7hA2aepcFgmAAmQs8ivn3xEq5a02AmERkMhglhHHoWUV9WQH1ZQaYvw2AwTFFMysVgMBimCcahGwwGwzTBOHSDwWCYJiTl0EXkIhHZKSJ7ROSWOPffLCI7RORtEfmLiBgBEoPBYEgzYzp0EXECPwEuBpYB14jIshGnvQmsUUqdAPweuNvuCzUYDAbD6CQToZ8C7FFKNSml/MCDwGWxJyilnlNK9Vt/vgI02HuZBoPBYBiLZBx6PXAw5u9D1rFEfA7482QuymAwGAzjx9Y6dBH5FLAGODvB/TcANwA0NhpFNoPBYLCTZBx6MzAr5u8G69gwROQC4DbgbKXUYLyFlFL3Avda5/eIyM5xX/HxlAJdNqyTaqqAtkxfRBIYe9qLsae9GHvC4kR3JOPQNwELRWQu2pFfDfxN7Akisgr4GXCRUur44Zjx2amUWpPkuQkRkXuVUjdMdp1UIyKb7Xi9qcbY016MPe3F2FOvnei+MXPoSqkgcCPwJPAOsEEptV1E7hCRyCTYHwJe4CER2SIij9pw3cnyWBqfKxcw9rQXY097MfYcBVFKZeaJp0hEYBe59npTjbGnvRh72kuqI/REa2eyU/TeDD53Jsi115tqjD3txdjTXlJpz4RrZ8yhWxukUUTkVyLSIiLbYo79UETetTpQ/ygiZfHWStTJKiJzReRV6/jvRMSTulc0OiNfbzqYzjY19rQXY097SaU9R1s7m7Rc1gMXjTj2NLDC6kDdBdw68kFjdLL+I/AvSqkFwDF0jXwusR5jUztZj7GnnazH2NNWssahK6VeADpGHHvK2pSFxB2ocTtZRUSA89BSBAD3AZfbca3xooNkIwMRudU6Z6eIXDjampNlqtjU2NPYM+ZY1tkTpo5NUUplzQ2YA2xLcN9jwKes32cCj1u/fxz4Rcx5nwZ+jK4D3RNzfFaitcd5jU5gLzAP8ABvoaOEDcDV1jk/Bb4U57HLrPPzgLnWOs5Ea+aCTY09jT2z2Z5TzaZZE6GPhojcBgSB+wGUUu8rpS7J0OUk0rZJJjK4DHhQKTWolNoH7LHWG1Mvx26yyKbGnvZi7Gk/U8amWe/QReQzwEeAa5X1kTeCRJ2s7UCZiLhGHJ8sibRtOtXQV8Wo3o2IXCoid4zx2PHq5UyKLLOpsaex5zCyzJ4whWya1Q5dRC4CvgVcqobUHEcS7WS1clhXA49ab4Tn0F/PAK4HHkn1NY9EKfWoUup76X7eREx1mxp72ouxp/1k0qZZ49BF5AHgZWCxiBwSkc+h82LFwNOiO1B/ap07U0Qeh8SdrNay3wZuFpE9QCXwSxsuNVF0kExkkOixSenljJcpYlNjT2PPbLYnTCGbTnpTI9duaP2bJvQGR2QzYznwEMM3SL4c57HLGb5B0oTeHIm7ZqZfq7Hn1LsZe+a2TTNurKl4Ay5B18juBW6zjs0DXkNvejwE5FnHLwXuiHnsbdbjdgIXj7ZmrtyMPY09s/02VWyaMS0Xg8FgMNhL1uTQDQaDwTA5jEM3GAyGaYJx6EkgIrNE5DkR2SEi20XkJuv4D0SLCG0RkadEZGaCx2+0WnzfFi089GNJIDqUCySyZ8z93xQRJSJVCR5v7BnDKO/P20Wk2Xp/bhGRuI05IrJeRPaJyFsisktEfi0iOTvofbT3p4h81XrPbReRuxM8PnP2zPRmw1S4ATOA1dbvxeiNjGVAScw5XwN+muDxG4E11u8e4J+B5zP9urLNntbfs9DlaO8BVcaeE7cncDvw90k8fj3wcet3Ab5hreHJ9GvLMnueCzzD0OZnTbbZ00ToSaCUOqyUesP6vQdd+1qvlOqOOa0IGHOHWek2328BjSJyIujh2iLymhVF/Uy0mlxEvOcN65P+L3a/rkyRyJ7W3f+Ctk9Su/XGnmPac7xrKaXUvwBH0GqGiMiHRORly3YPiYjXOr5WRP7bsudrIlJszyvKLKPY80vAXcqamaySGLeZbnsahz5ORGQOsAp41fr7ThE5CFwLJNUdppQKoetOl4jIUuCTwAeUUicBIeBaEakGfg5cqZQ6EbjK5peSFcTaU0QuA5qVUm+NZw1jzyFGvj+BG63U1K9EpHwcS72BtmcV8F3gAqXUamAzunHHA/wOuMmy5wXAgE0vI2sYYc9FwDrRCovPi8jacSyVFnsmMyTaYGF9kj4MfD0SnSulbgNuE5Fb0d1r/zPZ5ayf5wMnA5tEBKAAaAFOA15QWtAHpVRHvEWmMrH2RAsxfQf40ESXs34ae1rvTxH5D+AH6G87P0Cnpj6b7HLWz9PQ6Ya/Wvb0YHV3AoeVUpsARnxbnRbEsacLqEDbZC2wQUTmKSu3MtZy1s+U2tNE6EkiIm70P+79Sqk/xDnlfuBK69wnra/7v0iwlhNYif4qJ8B9SqmTrNtipdTtKXkRWUQce85Hd829JSL70a3Qb4hInbHn2MR7fyqljiqlQkqpMPrbySnWuf9p2fPxUZZcxZA9n46x5zKl1LQfGpHg//sh4A9WGuU1IAxUZZU9U52knw436x/h18CPRhxfGPP7V4HfJ3j8RoY28dzA3VibeOhP691YGyzoCGA2UI1WY5sbOZ5pO6TaniPO2U9ym6LGnonfnzNifv8GWsY13uPXM3wT72uWDT2W3Q4AC6z7i9CpBw+6dX2tdbwYcGXaFim25xexOkAtGxwE3ZyZLfbMuPGmwg04E/219W1gi3W7BP0Jvs06/hh6ozTe4zei237ftn7+BCiLuf+T1ppvA68Dp1nHLwbeROeHn860HVJtzxHnjOXQjT3Hfn/+BthqHX+UGAc/4vHrgX2WXXZbj2uIuf88tMLh29btUuv4WvRUobesn95M2yLF9vQA/2X9n38DOC/b7Gla/w0Gg2GaYHLoBoPBME0wDt1gMBimCcahGwwGwzTBOPQJMop+RoWIPC0iu62f5dbxJVZ32KCI/H3MOvlWV9hb1jrfz9RrMhgMUxuzKTpBRGQGumrgDatF93X01O/PAB1KqbtE5BagXCn1bRGpQZfPXQ4cU0r9k7WOAEVKqV6r9vUldLfYKxl4WQaDYQpjIvQJohLrPVwG3Geddh/agaOUalG6CywwYh2llOq1/nRbN/MpazAYxo1x6DYwQu+hVil12LrrCFCbxOOdIrIF3aL+tFLq1bEeYzAYDCMxDn2SxNN3iaB0PisZBcaQ0kJSDcApIrIiJRdrMBimNcahT4IEeg9Hrfx6JM8+psRmBKVUJ/AccJHd12owGKY/xqFPEGsz85fAO0qpe2LuehS43vr9euCRMdapFmvajogUAB8E3rX/ig0Gw3THVLlMEBE5E3gRrZURtg5/B51H3wA0oqfufEIp1SEidWjt4xLr/F60kNQc9OapE/0Bu0EpdUf6XonBYJguGIduMBgM0wSTcjEYDIZpgnHoBoPBME0wDt1gMBimCcahGwwGwzTBOHSDwWCYJhiHbjAYDNME49ANBoNhmmAcusFgMEwT/j/uwuEMKoJamwAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step 10973, {'val_loss': '0.015320336446166039'}\n", + "\n" + ] + } + ], + "source": [ + "# test\n", + "trainer.test(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-16T04:10:18.042094Z", + "start_time": "2020-02-16T04:10:17.491821Z" + } + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "0ba1a3bfe5284a898e91b6ca883f4dc5", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEgCAYAAACkfIiyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOy9eXhjZ3n3/3m0y7JseRvvM+PMPpPMTFayJwRCEghrIC9pSd8QltISeKFAC02b8gP6lqVsb0kX2kKgNCQkQJJCSMKSPZNtklmS2Xd7xvsiW/v2/P54ztFmyZJtWZLH53NdvmwdHR09kqX73Od7b0JKiYGBgYHB4sdU6QUYGBgYGJQGw6AbGBgYnCYYBt3AwMDgNMEw6AYGBganCYZBNzAwMDhNMAy6gYGBwWmCYdCXAEKIu4QQX9H+vkwIsX+Ox/lXIcTflnZ180cIcYsQ4plKr2MpIYR4Qgjx4TI9lxRCrC7Hcy12DIO+xJBSPi2lXFdov1xGUkr5MSnll0u9JiHEWiHEg0KIYSHEmBDiUSFEwTVWEiHEViHEdiFEQPu9Ne0+IYT4mhBiVPv5mhBCLLZjGSw+DIO+yBBCWCq9hgXAAzwErANagReBByu6ohkQQthQ6/sJ0AD8CHhQ2w7wUeBdwBZgM/B24E8X07EMFilSSuOnwj/AMeALwB5gHPgh4NDuuxLoA/4KGAD+S9t+PbADmACeAzanHe9s4BVgCrgXuAf4Svrx0vbtBn4BDAOjwPeADUAIiAM+YELb967s4wCfAYaAfuCDacdtAv4HmAReAr4CPFPk+9EISKApz/1NqBPAJMr4fzn92MB3gV7t/u3AZdr2NiCQflzgHO21W4HVwJOAFxgB7s3z/G8BTgIibdsJ4Frt7+eAj6bd9yHg+UV2rAbgV9p7M6793ZV2/xPa+/6s9jl7DGhOu/9C7fkmgJ3AlWn3fRDYqz3uCPCnWc/9Oe3zdAq4VfssrK7093Qx/BgeevXwx8A1wCpgLfA3afe1oYzcCuCjQoizgR+gvKsm4N+Ah4QQds0bewD4L+0x9wE35HpCIYQZ9UU9DqwEOoF7pJR7gY8B26SUtVJKT541twH12uM+BNwphGjQ7rsT8Gv7/G/tp1guBwaklKN57r8TdcJpR33hb826/yVgK+r13w3cJ4RwSCkHUIboxrR9b0a95ijKQD2GMmZdwD/lef5NwC6pWR+NXdp2/f6dafftTLsPIcQuIcQfVduxsjChHIsVwHIgiDrZp/NHKOO8DLABn9XW1An8GnUSb9S2/1wI0aI9bgjlkNRpj/+2EOIc7bHXavtfDawB3pxnfQY5MAx69fA9KWWvlHIM+HvgprT7EsDfSSnDUsog6tL536SUL0gp41LKHwFhlFd0Icrb/I6UMiqlvB9l4HJxAdABfE5K6ZdShqSUswkuRoEvac/zMMqbX6edKG7Q1hyQUu5BXf4XRAjRhTLYf5Hnfv3Yd2hrfi372FLKn0gpR6WUMSnlNwE7Ss5B2/cDace6CXXy01/PCqCjwHtRi/Li0/EC7jz3e4FaXa+WUm6WUt5dbcdKR3v/fq79/6ZQn8krsnb7oZTygPaZ/BnqJArq/X1YSvmwlDIhpfwt8DLwVu3Yv5ZSHpaKJ1En0cu0x96oHfc1KaUf+GL22gzyYxj06qE37e/jKEOrMyylDKXdXgF8Rggxof+gpJMO7edklpd2PM9zdgPHpZSxOa55NOuxAZTRaAEsZL6m9L9zonlwjwH/LKX8aZ7dch074/UJIT4rhNgrhPBq70090Kzd/SCwUQjRg/ICvVLKF7X7/hIQwItCiNeFENmev44P5V2mU4eSEHLdXwf4sv4nVX0sIUSNEOLfhBDHhRCTwFOARzsJ6gyk/a3/70F9Pt+X9fm8FHVFhRDiOiHE81oAfAJl6PX/Twcz/G8NZsYw6NVDd9rfy1H6oU72F64X+HsppSftp0Yzgv1AZ5bXtTzPc/YCy/MEWufThnMYiKFkC53uPPsCoEk1jwEPSSn/vohjZ79f+nEuQxnmG4EGTS7yogw12onxZygv8mZS3jlSygEp5UeklB0oOeuf86TLvQ5sznqPN2vb9fu3pN23Je2+xXKsz6Cuat4gpaxDyWCgvY8F6EXFetI/ny4p5VeFEHbg58A/Aq3a/+fhtOP2k+d/a1AYw6BXDx8XQnQJIRqB21HBzHz8O/AxIcQbtFQ0lxDibUIIN7ANZfA+KYSwCiHeg5JWcvEi6gv0Ve0YDiHEJdp9g0BXWoZE0Ugp46hA6xc1T2898Cf59hdC1AGPAs9KKT8/y2NvJFOfd6Ne/zBgEULcwXSv9cfALcA7SDPoQoj3aZIPqECgRMld2TyBChh/Uotb3KZt/0Pa8f9CCNEphOhAGce78rykaj2WG6WbT2ifyb/Ls18ufgK8XQhxjRDCrH2urtTeWxtKAhsGYkKI61DBXJ2fAbcIITYKIWpm+bxLHsOgVw93ozzUI8BhVEApJ1LKl4GPoIJU48AhlIFCShkB3qPdHgP+F8oA5jpOHJW6thqVDdGn7Q/KCLwODAghRubwem5DSR0DKKP5U5TOn4t3A+cDHxRC+NJ+8nlnt6Eu7wdQBumHafc9CjwCHEBdrofIknuklM+iDPUrUsr0S/rzgReEED5UFs3/kVIeyX5y7T1+F+okNYEKyr5L2w4qSP0/wG7gNVSA8N/0x2tyzh9X27Gy+A7gRGX7PK+9p0UhpewF3gn8Ncpw96IyV0yaHv9JlOEeRwVWH0p77G+05/4D6nP9BwyKRuSW4gzKiRDiGPBhKeXvKr2WhUII8TWgTUo5m2yXBUMI8Qfgbinlf1R6LQYGpcLw0A0WBCHEeiHEZk0SugCV1vjLSq8LQAhxPir/fCZZy8Bg0XE6Vh0aVAdulMzSgdLjv0kVVH8KIX6EkiX+j3b5b2Bw2mBILgYGBganCYbkYmBgYHCaYBh0AwMDg9OEimnozc3NcuXKlZV6egMDA4NFyfbt20eklC257quYQV+5ciUvv/xypZ7ewMDAYFEihMjbDsGQXAwMDAxOEwyDbmBgYHCaYBh0AwMDg9OEqiosikaj9PX1EQqFCu+8RHA4HHR1dWG1Wiu9FAMDgyqnqgx6X18fbreblStXkqPn/pJDSsno6Ch9fX309PRUejkGBgZVTlVJLqFQiKamJsOYawghaGpqMq5YDAwMiqKqDDpgGPMsjPfDwMCgWIoy6EKIa4UQ+4UQh4QQ0wYQCCFWCCF+L9TA2ifShgSclhw7doy777678I4GBganBYFIjL95YDdDU9V9tVzQoGszBO8ErgM2AjdpU2LS+Ufgx1LKzcCXgH8o9UKriZkMeiw21/GcBgYG1cojrw3wk+dP8NSBucx6KR/FeOgXAIeklEe0ySf3oKaRpLOR1GSRx3Pcvyi44447+M53vpO8ffvtt/Pd73532n6f//znefrpp9m6dSvf/va3ueuuu3jHO97BVVddxZve9CaeeOIJrr/++uT+t912G3fddRcA27dv54orruDcc8/lmmuuob+/f8Ffl4GBwfx45DU1D3vAG6zwSmamGIPeSeYIrz5tWzo7UWPPQI0Tcwshmua/vPJy66238uMf/xiARCLBPffcwwc+8IFp+331q1/lsssuY8eOHXz6058G4JVXXuH+++/nySefzHv8aDTKJz7xCe6//362b9/Orbfeyu23374wL8bAwKAkBCNxnjo4DEC/t7oll1KlLX4W+J4Q4hbgKeAkalhtBkKIjwIfBVi+fOZh3v/f/7zOnlOTJVqeYmNHHX/39k1571+5ciVNTU28+uqrDA4OcvbZZ9PUVNx56eqrr6axsXHGffbv389rr73G1VdfDUA8Hqe9vb34F2BgYFB2njwwTCiawGISDE4ufoN+EuhOu92lbUsipTyF5qELIWqBG6SUE9kHklJ+H/g+wHnnnVeVkzU+/OEPc9dddzEwMMCtt95a9ONcLlfyb4vFQiKRGhavpx1KKdm0aRPbtm0r3YINDE5TvMEo3/3dQT599RrcjsoV1j32+gD1Tiubu+pPCw/9JWCNEKIHZcjfj5rUnUQI0QyMSSkTwBeAH8x3YTN50gvJu9/9bu644w6i0WjewKfb7WZqKv/0shUrVrBnzx7C4TDBYJDf//73XHrppaxbt47h4WG2bdvGRRddRDQa5cCBA2zaVJnXamBQzdz3ci8/ePYol65p4qr1rRVbx5MHhrlq/TIcVhN7+0urGpSaggZdShkTQtwGPAqYgR9IKV8XQnwJeFlK+RBwJfAPQgiJklw+voBrXlBsNhtvfOMb8Xg8mM3mnPts3rwZs9nMli1buOWWW2hoaMi4v7u7mxtvvJEzzzyTnp4ezj777OSx77//fj75yU/i9XqJxWJ86lOfMgy6gUEOHtxxCoCpUOUyx/zhGKP+CGtb3URiCUZ8EcKxOHZLbttQaYrS0KWUDwMPZ227I+3v+4H7S7u0ypBIJHj++ee577778u5jtVr5wx/+kLHtlltuybj99a9/na9//evTHrt161aeeuqpkqzVwOB05dDQFLtPegGYDEaLftzQZIgGlw2ruTQ1k/1aVkuHx0EoGteeI0x3Y01Jjl9qqq5StJLs2bOH1atX86Y3vYk1a9ZUejkGBkuWB149hUkrkp4s0kN/7vAIl37tcb7yqz0lW8fJCaWZd3ictNU7ARio4sBoVTXnqjQbN27kyJEjydu7d+/m5ptvztjHbrfzwgsvlHtpBgZLBiklD+48ySWrm3nx6BjeIjz0fQOT/OmPtxOJJ7hvex+fvWZdSQKppyZ0D92JTzuxDFRxYNQw6DNw1llnsWPHjkovw8BgSfH6qUl6x4Lc9sbV7B+YKkpy+cIvduO0mfnG+zbzsZ+8wgOvnuTmi1bOey2nJoKYBLS67dTalbmsZoNuSC4GBgZVxWN7BjEJePOGVuqcViZDMxv046N+Xj0xwa2X9nDNpjbO6qznJ8+fQMr5Z0afmgjRVufAYjZR57DgtJqrWnIxDLqBgUFV8ds9g5y7ooGmWjt1DguTwZk19P/ZqbJh3r6lAyEEH7hwOfsHp9jV5533Wk5NBGn3KO1cCEF7vcPw0A0MDAyKoXcswN7+Sa7eqPLOC3noUkoe2HGKC1Y20qkZ3rOXqzTivvH591055Q3SoR0XoLXOYXjoBgYGBsXw2z2DAFy9sQ2AOod1Rg19b/8Uh4Z8vH1rR3JbnRYMLSaYOhOJhKR/IkSHx5HcZnjoBgYGBkXy9MFhVrW46GlWrTTqnJYZ0xYf3z8EwFvPbEtuq3Oq4GUh7b0Qo/4IkXiCjvo0D73eweBkiESiKjuXGAa9Gjh27BhnnnlmpZdhYFBxhn1hVjSl+iLpHnq+AOepiSANNVaaau3JbU6rGYtJzKogKd+xgUzJxW0nlpCMBSLzOvZCYRj0BSQen9Zw0sDAYAYmgzHcjlQ2dZ3TSiwhCURyf5cGJ0O01jkytgkhqHda5y25pAx66vgNLhsAE4H5HXuhMAx6GsUOuHjiiSe4/PLLedvb3sa6dev42Mc+luyuWFtby2c+8xm2bNnCtm3b8g602L59O1u2bGHLli3ceeed5XmBBgZVzlQomtTAIaWH55NP+r0h2uod07arYOr8esCc1Ax6Z5qH7qlRBt0brE4PvXoLi37zeRjYXdpjtp0F130179233nor73nPe/jUpz6VHHDx4osv5tz3xRdfZM+ePaxYsYJrr72WX/ziF7z3ve/F7/fzhje8gW9+85tEo1GuuOIKHnzwQVpaWrj33nu5/fbb+cEPfsAHP/hBvve973H55Zfzuc99rrSv08BgESKlZDIUS2rgAPVOzaAHY7TXT3/M4GSIzV3T71DpjvPzovu9IZxWc3INAB7t73F/dXro1WvQK8BsBlxccMEFnHHGGQDcdNNNPPPMM7z3ve/FbDZzww03APkHWkxMTDAxMcHll18OwM0338xvfvObMrxCg9MFbzBKncOCEKLSSykZwWiceEJmlOzPFODUux9mSy7qcfOXXPq9Qdo9joz32FOj1jYxz2MvFNVr0GfwpBeSYgdcZH+R9NsOhyPZdjffQIuJiWmzPwwMimYqFOXC//t7vvKuM7nh3K5KL6dk6AVEOSWXHAZ0aEqlD7blMei6ZDJXJgJRGjSJRcfj1DX06pRcDA09i3e/+9088sgjvPTSS1xzzTV593vxxRc5evQoiUSCe++9l0svvXTaPukDLUDNFH399dfxeDx4PB6eeeYZAP77v/97YV7MYiAWgePPQQnKtJcK/d4QwWicp7U5l6cLU5oXnh0Uhdweuj4OrjWXhu6wFqwwLbweLUAbGIN4LLk2k5h/jvtCYRj0LPQBFzfeeGPeARcA559/PrfddhsbNmygp6eHd7/73TmPdf/99/NXf/VXbNmyha1bt/Lcc88B8MMf/pCPf/zjbN26tSQ9JxYlrz8A/3QO/PA6eP0XlV7NomFkKgzAKydOrys93WjXOdM9dE1yyWGc9XFw7TkMen0RPWAKMRWKskachO9shrtvhEQck0ll0IxXqYdevZJLhQhHYwUHXADU1dXxq1/9atp2n8+XcTvfQItzzz2XnTt3Jm/nGoZxWiMl/PJj0LASnI2w79dw5g2VXtWiYMSvjMmJsQAjvjDNaTnYi5mU5JIyS+4ZJBe9YjO35GIhEksQisZxWOc2XUgGvfzpqb8BGYfDv4cn/gGu+hs8NTYjbXExsGv3a6xavZpLLr/SGHCx0ER8EAvC1ptg/Vvh4O8gXp1fkmpD99ABXjk+XsGVlJbJpOSS8tBtFhNOqzmnxDE4GcJuMWVkoejMpL0Xy5/HfkRDtB8+8HM4+wPw1Deg9yU8NfMPuC4Uhoeexrr1G3j42R3JQMhMAy6uvPLKCqzwNCKoGSJnA6y9Dl79idLSz7iisutaBIz6w5hNApOAV3sneMumtsIPWgToeePpaYv67VzyycBkmLZ6R85Mn3TtfVkOD74QoUiMq8R2DrdczdoVF0PbZth1H+x5AI/znQz7woUPUgEMg56G3p5Br0ozBlwsIOkGfdUbwWyHA48YBr0IRqYiNLlstNc7TisPXQ+K1mVNGqp35g5wDnpDOeUW/TEw9+Bl4OTrtAgvB1suZC2AvRZWXAwHf4un5X0cHPIVOkRFKEpyEUJcK4TYL4Q4JIT4fI77lwshHhdCvCqE2CWEeOtcF1TJAKH+3OFYnFg8UbF1pHPaBkzTDbrNpQz5/oeNbJciGPUr3fzs5Q3s6vNWzWd1vkwGY9jMJuyWTLNU58gd4ByYzF0lqh6TP5haDIkjTwIw1XFxauPqN8PIflaYR/AuVg1dCGEG7gSuAzYCNwkhNmbt9jfAz6SUZwPvB/55LotxOByMjo5WzIilN1DL1zuinEgpGR0dxeGY/SVj1RPUMjScqnc1666D8WMwvL9iS1osDPsiNNXauGhVE8FonI/9ZDsjVSoBzIbJUJQ65/RiqVw90aWUyqDn8dBnSncsBuuJZzieWIa5YWVq4xpVIHhW8GWmwjGiVXgiLUZyuQA4JKU8AiCEuAd4J5A+WlsCddrf9cCpuSymq6uLvr4+hocrk18bisYZ8akMgsCQJWewpdw4HA66uk6f4pEk6R46wNprgU8rL33Z+ootazEw6guzqtnFWza28rfXb+Rrj+zjj/79eR779OKWq1Ted64Ap4VDQ5me9nggSiSWyFklCrOXXOIJidmknUgScVz923g4cS5npGXc0LwW6rtZM/U8cBbeYLTqMoyKMeidQG/a7T7gDVn7fBF4TAjxCcAFvHkui7FarfT09MzloSXhkdf6+dhDr+CymdnUWc/P/vSivPsOTYYYnAxzVo4+EgZFkG3Q6zqgfYvS0S/7i8qtq8qRUjLiC9NUa0MIwYcu7SEaT/DV3+xjzB+h0WUrfJAqZVJrZ5BNLg+936uqQHPloEOqOKmYLJdwLM6bv/Ukf3LhSj5y+RnQvxNLZJJtiU1sST/BCAGr30zHznux8kEmAtVn0EuVtngTcJeUsgt4K/BfQohpxxZCfFQI8bIQ4uVKeeEzEYqqS6gLehrZ2Tsx4yXVNx7dz0f/6+VyLe30IzgOFgdYU53sWPdW6H0RfNX32agW/JE4oWgiw5BsaFcXx/sHpiq1rJIwFYrm8dBVT/T0oRJPHxwBYFNHbofKbjHjsJqK6rj4q5399I4F2dM/qTaceB6AbYmNGVWrAPRchiUWYK3orcqOi8UY9JNAd9rtLm1bOh8CfgYgpdwGOIDm7ANJKb8vpTxPSnleS0vL3Fa8gISiSje/aFUT4ViCA4P5vyD7B6eqtrhgURAcT3nnOmuvBSQcfKwiS1oMjGpaefpAh/VtbgD2D0xWZE2lIrvTok5ng5OEzJwR+tCOU2zt9rC8qSbv8eqd1oLBSyklP9p2DIBhPb9/5AAhSz3DeKZl3NC+FYCzTEersuNiMQb9JWCNEKJHCGFDBT0fytrnBPAmACHEBpRBX3Rulm7QN3d5ADiUJzVJSsnhIR/BaLxqR1FVPcFxcHgyt7VvAXeH0tENcqIHP5trU9LKMrcdT42V/TM4IIsBJblM99DXaSesfdoJ69CQjz39k7xjS8e0fdPJlx2Tzo7eCXb1ebGaRcqgjx1m3KF82NpsD73xDBK2Os4SR6uy42JBgy6ljAG3AY8Ce1HZLK8LIb4khHiHtttngI8IIXYCPwVukYsw3y4UUxLLhrY6LCaR10Pv94bwa1kwoVjls2EWJcGJ6R66ELDuWjj8OESrdxBvJdGD9umSixCCda1u9qVLLsEJ2HH3oqq+TTbDymJdq34Fol7fQztPYRJw/eb2GY+XS3vP5sfbjlNrt3D95o5k90ZGjzBk68JlM6cCpTpCkGjbzCbT0arsuFiUhi6lfFhKuVZKuUpK+ffatjuklA9pf++RUl4ipdwipdwqpVyU18y6h17rsLCy2cXBwdwe+uHh1HZ/2DDocyKX5AJKR4/64djT5V/TIiDloWcG49a1uTkwMKVSfmMRuOeP4YE/g2e+k+swVUc0niAYjef00F12C8sba9invb7/2XmKC89oKlgBmq8gSccbjPLw7n7eubWDlU0ulTkT9MNkHwPmzpx6PoC582w2iF4mffNrz7sQGL1c0gjHEljNArNJsGZZbd5qsHQpJlgF+eqLknwGfeVlYHXBfmPgRy5GNQ89O5tlXZsbfySudOZffxqOPwPLNsJTX18Uuf1ToVR72lysb3Ozb2CSg0M+jo74eetZM3vnoNIdZ0pb/NWuU4RjCW48r5tldeoE6T2l3qte0Z53LaJjK3YRxTZefe+rYdDTCEXjOCyqM9uaVjfHR/1Jrz2ddIMeiM6v5/KSJTQBTs/07VaHagVw4BGjajQHI74w9U4rtqxqSj0wevzwPtUX5+JPwJ88CNYa+FX1p4Hq6YV1eWo/1re5OTYa4Ne71EzeN29oLXjMQpLLfS/3sba1ls1d9bRoVzx+zaAfpS2vQafjbAAavK8XXEO5MQx6GqFoArvWanNtay0JCUeG/dP2yzDohoc+e6IhiAZye+igqkYnT8LArvKuaxEw6otkBER11mo6s++oNgN303ugdpnK6T/+DIwfL+cyZ43uoeeSXADWt9cRT0h+vO0YZ3XW5y35T0dPd8wVzjs4OMWO3gluPK8bIQQtbmXQo0OHADgUa80rudDQg1+4aPXtK+KVlRfDoKcRjsZxWNVbsmaZ+oIcHJoeGD087GOlli5lSC5zIJRV9p/NmmsAAfsfKduSFgvDvnBGyqKO22Gl0+PEPLADTFZo3aTuWH+9+n3g0TKucvZM5phWlI6e6TIeiPKmDcuKOma900pCwlR4+lX0Y3sGAXjn1k6ApEE3jR8BVwtDEVt+D91k4oRtDd3hA0Wto5wYBj2NUCzVDL+n2ZUz02UiEGHEF+EsLbXR8NDnQHaVaDa1LdCxFY48UbYlLRZGfOGkPJDNujY3TZN7lTG3aPs0rYKmNXCgumMShSSXlU2uZNOuYuQWSA10zpWLvrd/kq4GZ9KQ60Fmx+RRaFyVt8hJZ7hmFd2xE1UnCxoGPY1QNJH00G0WU85MFz3D5axOVZ0XiBga+qwpZNABei6HvpcgEijPmhYJo1pjrlysa61lVfQgca34Jcnaa+DYMxCu3jz1pOSSx6CbTYK1rW7a6hxs6qjLuU82Hm2uQa5xcfsHppJxB1Df94YaK3WBE9C0WhU55fPQgam6VdQQAm9fUWspF4ZBTyM9KArkzHS57+U+TALOXdEIGJLLnEga9BxBUZ2Vl0MiCr3Pl2dNi4BILDFjQ6hz6iaoF36Gazdk3rHuOohHVH5/lVJIcgG44+0b+db/2pJzoEUuGjQPfVzz0Ae8IULROOFYnKMj/qSMo7O8No47NkqsoYdILDHjWsKetQBEBvYWtZZyYRj0NELROHZr6i1Z1+bm2Kg/abSfOzzCPS/18pHLzmBViwswJJc5UYyHvvxCMFng6PR5rEuVMW2WaD4PfUPiMAD7TKsy7+i+EBz1Va2jjwcimE2CWlt+I3r+ykYuXjWto0hedA99IhBBSsn1//Q033xsP0eG/cQSknVtmZ7+RscYAIHaFQAzSi6JZjWiMtS/J+8+lcAw6GmEY4kMD31Dex1Sqr4t0XiCv/7FblY01fCpN6/FaVP7GZJL8RwYnOJbj+1HBtQXZ0aDbq+FzvMMg55GvqIinXb/XsLSysuBrJF0ZgusuLSqr3Z6x4J0epyYsisz54GuoU8EovgjqjX2I68PJCtO9QpUnR672u6zqaDrTB66y9PKiKwjPlhdmS6GQU8je0L4Bu0Mvq9/kp29ExwbDfDZt6zDaTNjM5swm4Thoc+C/37+OP/vD4c42X8KhBnsBbTQnsvh1KsQ8pZngVVOrj4u6ZgHdnLU0sO+4RxtE7rOhdFDoJ9Mq4wTYwGWN+ZvtDUXPE5dcokkB2v3jgX51a5TWM2CM7SrbJ1Os8q+8lqUnDqTh97gsnFIdmIera7iIsOgp6Hy0FNvSVeDk1q7hb39k7x0TMkEF69qAlT/jBqr2TDos2BnnzLMR3tPKv28kBbacxnIBBzfVobVVT+5+rhkMHYUr2tlZk8Xna7z1e+TryzQ6uZH71iA7hIbdLQye7cAACAASURBVIvZhNthYSIQZdSfmuj0u71DrGqpxWrONH+tQhn0ExHlaMzkoTfU2DiU6MAxcbiqMl0Mg55GOJbpoZtMgnVtbvYOTPHSsTFWtbgycoCdNrMRFC2SaDzBnv5J7BYT3rFhYvYZAqI6neeCMMHJ7Qu/wEVArta5SRIJ8A1gru+kbzzIi0fH+It7d6QkwY6zAQEnq6+Hvy8cY9QfKbmHDkp2mQhEGJ5SJ8MaTSrNDogCNMpxxmUth8YKB2gbXFYOyi5sUS/4hkq+7rliGPQ0QtFMDR1gQ7ubvf2TvHxsjPNXNmbc57JbCORoDWAwnf0DU0RiCf78ytW48TMaK2JOqs2l+pEYBh1QkovDasJlM0+/MzAKiRi1zWpc4R//x/P84tWT7O3XvHW7G5ZtUKmgVUbvmEpNXQiD3lBjYzwQTcpVb9N6wOQy6PXxUYakh6cOqM7f+apW9eMelKooieHq0dENg55GKK1SVGd9Wx1ToRiTodg0g+60mgkaQdGi2KXJLe86u4MWR4KhYOb7fGoiyN0vnJj+wM5zlEGvosvaSjHii9DksudO25tSY3yb21WGhk2TE0bTh0d3nVeV7+WJBTTonhobE8FosqnZH1+4ApOAc5ZPD8jXRUeZMDfywtExTALqa/IbdKvZxIB1ubpRRc3PDIOuEYsniCVkhuQCykPXyTboNTZDQy+WXX0TeGqsLG+soc4axxvLfJ9/+epJ/vqXu6f3mO48V7UKGDtSxtVWJyO+MM3uPPr51AAAzR09fP2Gzdx16wVAKtURUFlDwfGqey91D7270Vlgz9njcVq16m7V1Gxrt4eXbn8zF57RNG1fa2CIC87awK8/eSk//ciFM3roALGaVoImF4xUTwsAw6Br6MMt7Fld7PRc1dY6+7QPnNMw6EWzq8/LWZ31CCFwiij+eGZrU92QZxggUAYdqjaYV05GfBFa8mS4MKW6EOJu48bzuzlTm7U5mv5+dp2nfleZ7HJiLIDbYaE+T5XofGiosTLuVwZdzw7KGYOQEnyDCHc7mzrqeUMOgz/t2C4bA+YOGD9W4lXPHcOga4Q1LTzbQ6+1W1jf5ubS1S3TLnWVh25ILoUIRePsH5xii9b/xiGihLEmPTMgOYhgWpl2ywawOA0dHSWfNLlm9tCpVX1OnDYzTqs58wTZsh7Mdhh8bYFXOjv0lMViK0Bng6fGxmQoxtBUOH92EKh0zkQU3G3598miwWXjlFhmGPRqRPfQszV0gHs+eiFfftemadtrbBbDQy+CA4NTxBOSMzuV12iTEULSlmHQdW99LHvwrtmiZo0ucYOeSEhG/RGa3TN46K4WMKe83KZaW6ZBN5mheS0MVU8QD5TkshD6OaSKi44M+2Y26L7ME2IxNNbYOJZogYnjKsuoCjAMukYoj4cO6ixfk6Mk2UhbLA49INWqTYUxJyLKQx9P89C1Xh7j2ZILKNmlf+eimo9ZarzBKPGEnNlDz/Ium1y2TMkFYNn6qsrKSCQkvePBBTPoDckGXdG8BVlAhmRVLJ4aG4cjTapPjn5CqDCGQdfQDbrdkiMlLA8uQ0MvCt371jVSUzwMFnsyuyF9n7Fcg3fbN0M8DKOHF36xVUqySjRfUHTyFLgzx7I1umyMpRXUAEp28fZWTefFoakwkVii5EVFOp60TJUZPfQp1R99Vh66y8qhmNZbpkpkl6IMuhDiWiHEfiHEISHE53Pc/20hxA7t54AQWsnVIiIUzS+55MNpsxCMxkkkqisNrNrQA556sySiQWxOF71jqSG7M3roLevV7+Hq6mxXTpJVoq58kst0D73RZWfMl+2ha50Yh6sjM6NPu0rraih9hgukfebIEwzV0T3sWWrovVIbtlElE6EKWi8hhBm4E7gO2AjcJITYmL6PlPLTUsqtUsqtwD8Bv1iIxS4k+YKiM6FXnYVihpc+ExP68AKHBeIxkHFqnK4MyUUfQpCrdzUt6wBRddpvOZnRQ49HwT8M7o6MzU21SnLJGMFWZSdH/USebnhLSUOGhz6T5DKoegvZXPn3mXZsGydlMxKxqDz0C4BDUsojUsoIcA/wzhn2vwn4aSkWV050o5ydtjgTNcmOi4ZBnwlvMIrbbsFiNkFMNY5yuVz0jQVJJCSJhEyOCZsWFAWwOqGxB4aqq1VpOdENelMuD903BMgcHrqNcCyBP/3z2bASLA4Yqg6D7g+rteWsfi0B6SeKvHIVKA99FnILKIMewUqkpm1RGfROoDftdp+2bRpCiBVAD/CH+S+tvKQkl+I/WE5tXyMwOjPeQDRVdacZ9NpaN5F4gqGpMFPhWLJ4MaeHDip9sYqCeeVm1BfBJFJBvgz0lMUcGjqQKbuYzNC8pmreSz3tt8aev2/KfHDbLegdeZvzBZQhp2RVCP399dV0qkyXKqDUQdH3A/dLKXNaOCHER4UQLwshXh4eHi7xU8+PcGwukov6EPqNXPQZ8QajqaIRzaDXu2sBlYM8mVZglFNDB5WdMXoYYuHc95/mjPrDNLpsufuF58nQ0L350WmB0Q1VI18ttIduMomkl5435RPmZNCTE5HsnYvKQz8JdKfd7tK25eL9zCC3SCm/L6U8T0p5XktLS/GrLANzCYrW2A3JpRgmgtFUtoFmkBvqVEuF3rFAMsOltc6eO8sFlBGScdXTewky7o/m9s4hzaBneuh6EHBa9W3LOpjsg9BkqZc5a5Ie+gyTiuaLx2mlxmae+Tl8Q7OWXPQTxYi5Tf0PosECj1h4irFeLwFrhBA9Qggbymg/lL2TEGI90AAsyubVyTz0WaQt1hiSS1F4g1E8zlSGC0BDfR1CZHroK5pceINRYvEcRRp6dkaVaL/lZiIYyUjBy2CqXw0McWWOZ0t56NkGXQuMjhws9TJnjT8Sx2Y2YZtF7Gq2eGqsecf2ARDxQyw47f0rhM1iwm23cFJoJ4KJ3pkfUAYKvotSyhhwG/AosBf4mZTydSHEl4QQ70jb9f3APVJWWSu3IpmLhq6f8Q0PfWYmAtHUNHfNQ7faa2h1Ozg5EUxmOvQ0uZCSjB4vSZrXKKO1VA16IJo/E8Q/rIyRKfOzm9TQsw16w0rtoJXXfQPhWPJKd6HY3OVha/cM4w79mvxbMzuDDrC2zc32SVUBXQ2yS1HXOVLKh4GHs7bdkXX7i6VbVvlJFRbNJg/dmCtaCCkl3nTvUtPQsdjpbLBxcjyYNOArmlVxyXggMj1n2GKHxjOqJphXbiYCUc7qzOOhB8fB2Thtc43NjN1iymHQV2gHrbxB90fiuBZQbgH44jumt+3IXMSo+j1LDx3gnOUeHt1mV5Z0sm/2iysxRqWoRigWx2YxzWpIrZ62aEguKULROMdH/cnbwWicaFymBUU1ndHipMPjVB661pirp0nlAI8H8pT4t6yrqlal5WQ8EKEhX1FRcEKN9MtCCKHK/7OLi+xudQKogmKYQCSW/B5VbhEj6vccPPRzljdwKlanctGnKl/+bxh0jXA0MSvvHIw89Fzc9dwxrvrmkzx3SH1JJjTj7MmSXLDY6fQ46fcGGQ+olLyuBuWhT/ModRp7lBGqkkZI5SIUjROOJfJr6MFxcOaWFBprbdOzXEB56RM5BoqUGV84vmApi0Xj1wz6XDz0FQ3EsBC0Nan2CxXGMOgaalrR7DwFXXIJGmPokuw5NUk8Ibntp6/SN57KYJkuuTjobHASjUsODfmoc6YCV3lTFxt6VE8XPatjiaDn5icDy9kEJ/IbdJc99wnSs6IqJJdAOLZgKYvFL2LuBr21zkGnx8mIaKiKz6Vh0DXCscSsUhZBjfmymAT+sKGh6xwe9rG+zU00nuCOB19PeujZQVGsDro8qn/H3oFJ6hzWZFpe3tRFPZg3fnShll+VjGvVsw1z8NBzSi4AnuXKQ6/w1Y4/El/QlMXiFjGs+sTbauf08HNWNHA8Wm9ILtVEKBqfVcoiKI3SmFqUIpGQHBn2c/GqZq7f3M6rJ8bxBrO8Sz1XV/PQAXrHgtQ7rcmhDHk99MYe9bsKsgnKyYT2HuaccRkLQ9QPjukaOigPcmgqRDQ7FbRhRVW0fQ1EYrgWOMulIP5R5Z3PccDGOcs9HI/Uk/AakkvVMBfJBZSObgRFFf2TIYLROKuWuVizzM14IMrhYRUgra/JraHr1DmVl6ZavuYJitZ3q9TFpWbQA7qHnkNyCWqNTXMERQHtakmdaDPwrNQOXlkd3R+uAg89MAI1hUfO5WNrt4dB2YApNAbRUAkXNnsMg64Ris5ecgFw2SxG6b/G4SEfAKtbalnbqipBXzw6BqQHRXUN3YnLbklq63oWTIPLmr+fi9kK9V0wtrQklxkNekg36Lkllw3taibunn5v5h166mKFM10CkSrQ0P0jatrTHGl02RhASxut8BWPYdA1QrG5eehup5XJkGHQAQ5pBn3VslrWtio98pXj41jNIpWalpaHDiS9dH3CeqPLzjOHRnjvvzzHC0dGpz9Jw8ol56Eng6K5JJfguPqdx6Cf0eLCZjaxtz9roEW91s2jgoHRREISiFRBlktgZE4BUR2n1cyg1N7/ycoGRpe8Qf/FK338n3teZXgqPOu0RVCepzefR7nEODzso95ppcllo8Vtp95pZSoco95pTQ0AjoVUAEq7rRt03UP/zNVref/53bx2yssDO3K0DGrsWXJBUW8wisNqyu1wJA16bsnFajaxprWWvf1ZfVusDqhtq6hB17PDaiuuoY/MKQddx241MyA1D73CmS5L2qDHE5JvPLqfB3ecom88iL0YDz04Ac98OxnRbqixJgc4LHUOD/tY1eJCCIEQIumlJ4uKQGnoVkfyph4Y1bNgtnR7+NI7z2Rtq5uTEzn0yIaVEBitisZS5WLcH5k5ZRHyeuigZJdpBh2U7FJBycVfhsZcBYkEIBoA19w19AwP3TDoleOpg8P0e0N84br1XLOplTeuW1b4Qa/+BH73Rfinc2H7XXhqbPmzMpYYh4f9rGpJpX6t0XT0DIMeDaoBCxpJycWZKSe01zvon8jRva5h6WW6jAeiMxcVQUGDPuKLMDSVdYKscC56QG+dW0kPfR5VojpWs2BKuIgKe8WLi5a0Qb/3xV4aXTY+eEkP/3bzebz33K7CDzqxDeo6oW0z/PqzNFsjTIZixJf4XFFvMMrwVJhVy1IGfa32d0ZTqVg4qZ9DapZknSPTS2uvd3JqIsi0Xm/JXPRjJVt7teOdqdNicBwQYK/P+/gN7erEOl1H71IGqEK56FXhoSerROceFBVC4LRamLI1VzwXfcka9BFfmN/tHeQ9Z3cW37pTSmXQe66AN/41JKJsCL0CkDGkYSlybESlxZ3RnJrJuDaXhx4LgSWVrrixvR6b2cTqZZlFHZ0eJ/5IfHrAOZmLvnR09PHADL3Qg+PgqAdT/s/wRi3TZZrs4m6HRExJWBVAr99Y6OZcMy9i7o250nHazHgtzYbkUimeOjBMLCF519k5p+nlZuSg+gCsuAiWXwj2elaNPwuw5HV0vcQ/ve90TsklFsrw0Jc31bD3y9eyqSPTw2z3KFmm35sluzjqVRFNhdPtth8f46Gd5bm8nrF1bih/2b+Op8ZGR72Dpw8Ok0i/knRrfbwrlGrnC+vj5yoouege+jzy0AHsFjPj5sr3c1myBn1K8/za6h0F9kzjhDa7Y/lFKid69VW0Dz2FIMHEEs900dsfpF8+N9faeO+5XbxxfVpsIhbK0NABzDk6XLbXKy++P1dgVJcKKsj/+/0hvvjQ6wv+PFJKJgIFJJcCBh3g1kt7ePbQKN/87f7URn3CUYVkgqSGXlHJReuFXgIPfcykSS4VHAmxZA26frk3q9adJ7Ypra1ptbq95hrsoWE2iWPJ4o+lij/H5bMQgn983xauWJumT2ZlueSjQ/PQT+YKjNZ1wGS+KYjlYW//JGP+yIL38fFH4sQSck59XNL50KU9vP/8bu58/DC/3zuoNuoj1ypk0FMaeoWDoiYr2OvmdRiH1cSIaFTtofVirwqwZA16UPswzap/y4ltSmrRc6rXXI1E8CbTq8l+G0sVfchHwYyFrCyXfCxzOzCbxHTJBVRQuoIe+ogvzNCUamHQOx5Y0OfSM6jypy0WZ9CFEHz5XWfS5LLxq11ZQ6Ur5qHrn5lKeujz6+Oi47SaGUGTDX3DJVjY3FiyBj0QieO0mosfaOEfUZkV3W9IbXM1E1+2iXNNB5a8h+4r9suZleWSD7NJ0Fbn4FQuyaWuU3lWFeqbkR5c7B1bmMHAkViCW+96iXtfUnMq80suuYdb5MJqNnHx6maePTSisocsdnUyqJCG7p/LVXKpCcyvqEjHYTUzJDWD7jcMetkJROOz+yDpl/h62pyGqXUjZ5j6l7xBD4TjmEQRI/yyslxmor3ewal8kgtULKMg06AvjIfeNx7gD/uG+N7jhwByTytKJIoKiqZz6eomhqbCyTYNuNsr56FHYphNYk4V2iXDN5gKDs8Dh9XMUFwlAeAfmvfx5sqSNejBSHx20XX9MsqVWXxkal5Llxgh6PPmeNDSwR+J4bJZUiX++cjKcpmJDo+Tfm8uD10z6BWSXfb2T9FW56DGZl4wyUUfSqHn6S9z53jPwpMgE7My6BevUt7oM9pEKWpbK6ehh+O4bObCn5mFxDc07Ts9FxxWM4NxTYfXM2cqQFEGXQhxrRBivxDikBDi83n2uVEIsUcI8boQ4u7SLrP0BCIxaqyz0O58eiAp65/fshYAq/dIiVa2OAmE48VpoTmyXPLR7nHQ7w1mptqBklygYoHRvf2TbGh3091Qs2CSy4g2lOLOPzqHh267hBVNruk7FVElmk13Yw0rmmp49pCWf+1uT322y4zqhV5B/VxKZdCzv9NzwGk1MRitAWFSx6wQBd9NIYQZuBO4GugDXhJCPCSl3JO2zxrgC8AlUspxIcT836EFJhCJJ0fIFYV+GZX9z29eB4B7amkbdF8kVtwVT5FZLgAd9WpE3Yg/zDJ32mOSHnr5DXo4FufQkI+r1i/DJAR9C+Sh63NAW+sc+VNr9WyKPMMt8nHJ6mYe2nGKWDyBxd2aSrUrs6esphVVUD8PjkMimsr2mQdOq5lAHJXPXuUa+gXAISnlESllBLgHeGfWPh8B7pRSjgNIKSt3iiqSwGw/TL4hNaLKluUpNZ5BHBONwaVTuZgLNRuygH8gZdFZLqA0dMiRi26vVQVGFZBcDg35iCUkG9rr6G6soW88R3uCEqCPjWvMpZ3rzMFDB7jwjCZ84Rj7BqZUx8VEFAJjc13qnAmEK+yh+/I4aXPAYdUG3bhaqt6gdwK9abf7tG3prAXWCiGeFUI8L4S4tlQLXCjmZNBz9Xuw2BixdtIarvwE9Urij8QLpyzGo4AsWkNPFhfl1NErk7p4YFD1Q9nQ7qarwYkvHFuQgPioL0ydwzJzW4oCrXPz0a3p8oOTobTUxfIHmCvuoeeTUeeAw2omHEsgF4FBLwYLsAa4ErgJ+HchxLRPmRDio0KIl4UQLw8PV+5Fg8pDd86mQs03mPcfP1azkq54b877lgr+Yjz0tGlFxdDgUql6OatwK1RcpHvOLbUOuhtrgIXJRR/1R2iuLXDiC2mB+FlKLsvq1JXP0FQ4ZdArkLoYiBTxmVlIkh56abJcAOLO5opq6MUY9JNAd9rtLm1bOn3AQ1LKqJTyKHAAZeAzkFJ+X0p5npTyvJaWuXc3KwWBSJya2Uwo8g/nNehTtT10y37iMeWpPXd4hP94emlp6kVNnsmaVlQIvSFVzj45dR0V8dD1lhG1DgvdDZpBX4DA6KgvMrPcAgXnieajWeu3M5xu0CuQ6eIPV3haUb642BxwauMrY87mqs9yeQlYI4ToEULYgPcDD2Xt8wDKO0cI0YySYKraogVnGxT1DeZNbwrVr8Ym4vgHDhGJJfjcfbv47u8OlmiliwN/OFZ48kzSoBenodfYzNjMptwzRus6lScUK2+F7mQoSq3dgtkk6G5UVxoL46GHMxqd5STkBbOt6PdTx24x46mxqv7otZU06BWeJ+obVO/fLK9wcqF76BF7E0SmVKyoAhQ06FLKGHAb8CiwF/iZlPJ1IcSXhBDv0HZ7FBgVQuwBHgc+J6WsTE/OIpBSzq6wKBZRemWeS7N4k7oYCZ7aw/3b+zg5ESQQjS9IsKxaUTGJIqpEAazFSS5CCDw1Vib8eTx0ZNm136lQDLfWu93tsOKpsS5IcdGoL0JTQcllQhmjOWSnLHPbGZrUMo4cnooY9KI+MwuJb0h9p0uQ3aM7hyG7NoquQjp6Ue+mlPJh4OGsbXek/S2Bv9B+qp5IPEE8IYs36Po/pza3TGRuXgVAePgwd+5WGQfxhCQcS8xp8PRiQ0qpFRYV0ccFipZcQMkueT10ULKLPsG+DEyFokmDDtDqdijpooTEE5KxQITmQpJLyKuyfeZAi9vOsE9bt7ut7CfG5Gemkq1zfYPzGmyRjv49D9o0g+4bBs/ykhx7NizJStGg1kOi6KCof+bgiau+hbC08uLuvZycCHKV1i5W7+h4ujERiPDSsVSaWzAaR0qK0NA1AzILicBTY82dRZIs/y+vjq489FRflWa3jRFfaQ36RCCClAVSFmFWfVyyWeZ2KA8dtFS78uq+E4EoUpJ/cEc50D30EqAb9IBVSyGtkIe+JA16alJKkd6BHrXOo6F7XDaGqcfkH+LPrlzFtWe2ac+zsK1VZ8PBwSmODPtKcqy7njvG+7//PFMhZWj9ydmQxQZFizfoeT10/YtY5oyCqVAsY1xek8uerOosFaNa2X9hyWXuHvoyt53hqbCSBV0tqdmaZULvVrmsrvirtZJToipRUIVFAH6LNiijQv1clrRBLzooWqAAobXOwSgNnN0Q5i+vWZdMxaomD/1z9+/i72YxkCEUjXN81J/zvv6JEPGETOZk6z3BC54g52LQXVbGc3noDo/qY13msvXJUDTTQ6+1M1piD133+AsHRSfmHNBrcduJxBNMBmMVKYbRZaqWQiethSIRVyexknnoypROmrT/h+Ghl49gsm1nkZJLgQKEWruFs9avo8fuQwiR1OYXevjBbDgxFlCFJEXy/aeOcO13ns55laFPj9eHDhc97HeWaYsA9U6bJkFkBZhNJvX/qICHnq6hN7tt+CPx5GeqFOi57kXloc9DQwftf+lqVscqY8bQsC+UsY6y4x9Rjc1K7KEHsILNXbGe6IvCoCcSMvfkmjkSmO2kFP+wmmgyQ3aGyd2aNPz6cavFQw9G4oz5I8kOfsXw2kkvwWicXX3Tu0jqwbR9A6qNrP46a4vV0IvMcgFoqLESS8hkv/UMapeV1UOXUmpB0TQP3aUMUil1dN3jn1FDl3LeGjpo0oc+fq2Mw6KHk5LL7FIuS0YJq0QhLSgaiav30/DQ8/PLV0/yxm88kZzeMl8C0TlILoWi4e42CI5BLJLUkqvFQz+lTf0Z80emdy7MwyFNb3/1xPRxWnowbZ/moRc97HeOWS5A7sBobWtZDXo4liAal9M8dCiNQR+aCvHaSS9j/ghCFAgYRvwg43PX0DXtengqnPpsl9EIDU2GcVrNlctDL2GVKKQMeigaVycJQ0PPz+6TXiLxRNIwzZfgbCelFBMNTwbpBqvOQz85rt63hMxTdZlFOBbn+KjKrX7lxHjGffGETBqvfQNTKqe/2GG/c9DQ9Uk9uQOj5ZVcJrUgcJ0zU0MHShIY/foj+7nhX55j90kvjTW2nMOzk8yx06JOhuSiT+wpo0Ef9oVpcdsr1wu95B66MqWhaKIiWUM6i8KgH9a8xdESZRMkJ9QX2w/dP5Q3Bz1JWtaF7qFXi0FPn/oz5i/sSR4bCRBPSNx2C6+eGM/Qr8f8ERIS1re58YVj9I0Hkxp6wZziOaQt6pN6cgZGa1uVEUqU532eDKrXmZHlUptfcvnGo/t4aGfxaZWvnfQSjiV4fP9wcVWiMGcP3W234LCa1NWW7qGXWXKpmH4OqXTXEnvowajWcbFC/VwWh0HXxmWVSqcMzlZyCYyBs3HmffQxVr6BNA+9SiSXNINezElRH0/29q0djPgi9I2nHq8HRC9fq4zAvoGp1LDfBQiK6tPuczboqm1Vga0yGSI9TdOdkbaoDG+uTJe7XzjBQzuKayCm91nXTxYL1cdFRwjBMrdDxUNcFfDQp8K5pzCVi9Ejqu1BdjvsOWI1m7CahZJcXC3qM1kmRyOdqjfo/nCMU1r71FJ56IHZSC5SanMbC3xxMiQXXUOvDg+9L8NDL/weHhyaQgi44ZwuIFN20fOHL1mtjMC+/snUsN+iPHSh+mcUiWdGDV27XC6Tjq435koPijqsZtwOyzTJJRZPMB6IMlBkZtHBQdVn/Qtv3UCjy0anp2bmB8zTQwcluwxNhtUxTNbyauiV9tDHDkPT6pIe0mExK2exdhkgy3rFo1P1Bv3oSCoXulQeejIPvZiy/IgfErHCQwRcywABU4PJwbfV5KHrjaRGizLoPpY31rClq54am5lXjqcMup6d0NPkYnljDXsHJvGHY1hMApu5mAHR9ln1zvA4Z9LQUyfRcpAy6JlXIs21aWX0GrpENOAt7jO7Rxs8/YaeRn71iUv52+s3zPyAeWrooPVzmQqp/4erfF0Cw7E43mC0cjnoAKOHoemMkh7SYTNrGnr5r3h0qt6g6/q5EKUJPIHqhe6wmjDNFHRK7qwZs0JfHLNF/SO1vtIuuyWpLVeaUxMhzupUnlwxVzmHh3ysbqnFYjZxZmc9u0+mUheH0yr8NrS72ds/RSCi5okWHhAdnpXcAmAxm3A7LAU89PLolbrkUpfmoYNqR5stuehXQqP+MNF4ouCx9/ZP4rSaWdHkosPjTF6Z5KUEHrpeLQqU1aDr3+OKeejBCVVU1LiqpId1WE2a5KJ9Lg2DPp1DQz7MJsG6VveMHvq/PHGYv3lgd1HHDETixTfWD81Cq6xtTRqXGps5mf1RSRIJSb83yIom46bOuwAAIABJREFUF3UOS8GgaCye4Miwn9WttQCsaqnl2Giqm+DQZAi3w4LDaubMjnqOjvgZnAwVl342iwHR6eQt/3dVSnKZ7qFnOxu6gZcyJVPNxN7+Sda3u2fObElH19DnYdA9NTYmQzFi8YTKdCmTARqudNn/2GH1u8SSi9NqTmnoUJHioqo36IeH1eV/e70jOTg3m22HR/n6o/t44NVTRbWsnVUv9GTwqYi5jbWtyTakLpulKrJchn1honFJh8dJU619RsnlZy/38pVf7yUST7BmmRuAnuYaxvwRvFq6o55uBrCpsw6Al46NFzeoIB6ZtYcOKjCaM8vFXqvmvJbJQ58MRWkV47he+G5GVWVT7fQGXenv80CuEXppSCnZc2qSDe11xS8m5FXFbqa553HrAefJUKys/VyGtLhCS22FiopGtVENTaX20HUNvfx5/TrVb9CH/KxqcdFUa2dkarox8gajfOZnO5BSFbh4i8izntU80WIlF1DFRbqHbjdXheSiV9h2ehw0umx5g6KhaJy/vH8XP9p2DLfDwrkr1AlsRZPKAjimxTKGJlPZCWd2KO9wxBcubtjvHD10T40td5YLlLVadCoU48P232J6/Mvw1NeT25tr7UwEojyxf4irv/UkU6FohgRTqOXCKW+IyVBslgZ97n1cdHRZZzwQKavkoscbKia5jB4CBDT0lPSwyUHRep+hChQXVbVBjyckR0f8rGqpVU2Q/OFpHvgjr/VzyhviI5epf04x48AC0XjxrXNnJbloxkXKqvHQ9aKiTk/NjAZ9UjsRfumdZ7L7i9fQ06wMuf77mNaoa2gqnCwbX1bnSH4pi5NcwmCeq4eez6C3ltVDf6N4FRDw9DfhxAtAqrjo7x56nYNDPg4N+TLe50Ie+t5TKiC6sd1d/GLm0cdFpz6ZEhpVBj3iK8ukneGpMEIU0XxsoRg7DPXdarhHCXFazYRiCS3IXJlh0VVt0PvGA0TiCc2g24jGZbK4Q0cPll2zSbWsLWYcWDASK36e6KwklzZIRCEwhtNmrnjpfygaZ9sRlTrV4XHQXGvLG1jWr2zqnZkBv+WNNQihso2klNMKQs7sUF5lUY3O9CyXWeKpseWeWgRl9dCtUydZLY/DFX+lDMKv1TwXfUanXl3bNx5k1B+hocaKzWIq6KHrmVyrW2Zh0OfRx0VHby3gDUbSyv8X3ksfngrTWGPDWigraqFYgAwX0IKiuhPnajY09Gz0DJdVy1ypEussHd0fjiEErGlVX4ZixoHNWnIRZqXVFkLPuvAP4bKZK+qh944FuPRrj3P3Cye4dHUzboeVRpcKLubq55LPoDusZjrqnRwb8eMLxwhG4xkFIZs02aXgPFFQuvMcg6JT4VjubJEy9nNZN/mM+uOs98H5H4bB12BqIPnZ1NsUnJwIJkfItdU5GJjMbDeczfExP/VOa9JjLooSeOjJlFB/tKzl/zPmoE/2L+xJRUrNoJc2IAq6h65952uXGR56NoeHlOdyRnNt8vJsJCtjYCoco9ZuUV8Ip7VID30WQVG9qKiY3GndYwpOUGO3VDQP/fd7BxnxhfnP/30e//WhCwBodNmJJ2SyJ0k6+Qw6wMrmGo6OBpLZGhkeuhYYLSooOkcPfdUyJfukT0lKUrtMGbdo8a2B58pm//MMWLqgeTWsvERtPP4s7R6V4/+hS3rw1FjpGw8w5o/Q5LIpg+4Ncd/Lvbzl20/x7KHpxurEWJDljQUKibIpiYauSS7BaFk99BFfeHpr4IHX4IdvhW+th29thEdvT10dl5LAKIS9JU9ZhDQNHQzJJReHh300uWw0uGzJD0B2loYvFMOtGZPuRmdxGvqsPPSJ4uQWSH3BQhMV99B39Xlpcdu5av2yZH54skw9h44+o0FvcnFsxM/BQXXF1Faf8rJ1D71oDX0OBv3NG1px2y3cv71v+p16cdFCB6AifjbHdrHXfbG63bZF9b0+9iydHie//POL+bMrV9HV4NQklzBNtTZa65WH/ts96iriK7/eSzzrCunEqJ/lTbM06MGJeXvobocVIcCrB0WhLJkuo75IUqYCIJGABz8Ow/vgjbfDWe+F5/8Z/vMtMNFb2ifv36l+t6wt7XFRBj0UTZNc/MPqiqCMVL1BX9WipI7mPE2QfOEYtVpecHdDDb3jAYKROJ++d0cyMyObQCRW/HCL4HjxnlC6h64FRYttV1tqdvZNsKWrPqPYR+8PkiswOpNB72l24Q1GufPxQ7TXOzh/ZaqvTVeDk2s3tXHRqqbCi5qjh+6wmrl+SzuPvDYwPS5RrlF0IwewEWPAs1XdNltg+Rvg+LMAnL28AYvZRJenhpOaht7ostFWZ6ffG+LZwyOc0exib/8kP38ldWKKJyR947P00ONRiPrnraGbTYJ6p1Xz0MsnuYz6wpnj9fY8AP074C1/D1f8Jbzrn+FPHlIpwP95tfLeS8X+34DFCcsvLt0xNZRB12RB1zL1eQ/nltkWiqIMuhDiWiHEfiHEISHE53Pcf4sQYlgIsUP7+XApFnd42M+qZcqgN9RYc1aL+jTJBaC7sYa+8SC/3TvIL189ye/25tZWg9E5SC7FoBv+4Hiy86DeCKycTIWiHBnxs7krc926bJWrWlQ36NlFM6A8dFBtjP/kopUZwSwhBP9687lctb6IrnXx8Jw0dFB9ZQKROA/vzppOX6Z+LnJSdeeL1banNq64RHmVaTJFZ4OT3vEAE4EoTS47rXUOIrEEoWiCv71+I1u7Pdz5+KHk/v3eILGEZMVsDHoJqkR1PE4tx99Wq/43C2zQg5E4/kg8leESj8LvvwTLNsLmG1M79lwGtz4CCPjhdXDkyfk/uZSw/2FYdRXYZnlFVAROq5lIPKGuwCrQYx6KMOhCCDNwJ3AdsBG4SQixMceu90opt2o//zHfhekTdla1KGNiMZtorJlewDEViiVzoLsanERiCf77+eNAKusgnYg2pGBWWS5FSy7aFyw0kWrQVQEdffdJL1LC5q7ML3yTS5etphdoeYNRXDZzzsyDlVrqotNq5qYLuue+sDlKLgDnrmhgZVPN9Ha0ZernEh1XXROluyO1ceWl6rfmpYP6DOpeWlOtLSlPOawmLlrVxDWb2jg+GsCrZWed0D6js/LQ9RNITRFXRQWo13P8k6l2C9tQSv/+6pOeOPQ7GD+qpJbsIqnWjfDh30JdJ/zkBth9//yevH8HTJ6E9W+d33HykOqJXrniomI89AuAQ1LKI1LKCHAP8M6FXVZ6hksqu6QpR88MXzg147G7QX0pXjiqgmfHc2S8JIdbFBPEg9lJLiYz2Os1yUVroVuB8v/d2ti4bA+90WXDahbsyDGFyBuM5pRbQBmbGpuZ953XVbjHyEzMsbAI1JXA1m7P9JN0ssx64SSXw8M+du/dS0yasNSlDUToOBusNXAs3aCnDHOjFhQFuGRVMw6rmQ1arvlebXzfCe0zOisNXTcShaZoFUFDjTVVjFfTtOAGSI/fJD30o0+r2oTVb879gPou5al3vwF+/iF44mvKq58L+x4GYYK1187t8QXQbYovHKteDx3oBNIjE33atmxuEELsEkLcL4SYhxun0Hugr25JGfRcPTP8GZJLalZlo8vGiRxT6wPRWcwTTSTU5e1stEpnfYaHXonA6K4+L10Nzmk9tW0WEx+4cAU/f6WPvVp3P53JYDRjEk/24x7+5GXc/rYCHQALEQvPqnVuNo0u+/QxhGarMkQL5KGP+yO863vPcvToIUZEAxs70/rim63QvhVOvZLc1OlJfQabXHaWN9VgMQmuOVPVSejVoPu09//4WACrWdBeX/yc1VIadCW5RFLHW2iDrnvouoZ+7CnovmDmIh+nB27+BZx1Izzxf+FfL4V9v55dv/FEAv7/9t47PpKryvv+3o5qqZXjSJqgmdHksT3jGScY4wQOgDH2AiaasLCwGEzYF/DChyW8sDywr2Ef4FlgwWuijUkP9q7BAQds4zBje+wJnuwZzWiCcmhJne/7x63qIHWrU1W3Wqrv59Oflqqrq6pvV586de45v/PyverCoM8XGMwi7eJ9atSfoDNU3GpRoyZF7wWWSSnPAh4EfppqJSHEh4QQO4QQO/r7Zz9xDvf7cDtstNclG+kZHro/jNetDJHuHVW57Fy/qYMTw1NKeCiBM1o/zMZMDQQAAmOAzD7kAmrdqZFYDL0UqYtqQjT1ReiWy7uprnDy//7P3qSq29k8dFBhF7ejgP6PUmohl/yr8xqqnIwHwgTC037IJlaL/uSJVxgPhLlycZTWji7OWTxtXNs3qUm7iPqeO+oTDLrXRUt1BY995lLecq7Slm+pdtNQ5eJlrR9rz9AknfWV2YtyQVxn2wiDXumKK1nqjRlMRJ+/afS6VOOY07th2bbMb3S44Yb/hLf/Wt3p3fUO+O5mePBfoOfpzMb9L1+G/pdh800GfIrU6Lbq5MhUwiRzcVvRZWPQe4FEj7tTWxZDSjkopdQt7Y+Bc1NtSEr5IynlFinllubm2U/Gw/0TdDVVJZ3odYm3hyglQV8wnuVS4bSzuMHDFeta6W71Eo5KTo4k5ycf07x2PS48K7nouOhU1E2LoRfXQ//hY4c5MTzF+ctTd1iqq3Rxy+XdPHlokN29cS89k0EvmEgIkHnH0CHejm6GlK5J1aIjk0Hu+NtRXr9xEdXBPkTNopkrtZ8D4SkY2A+oLKHpXYc66jyxbCMhBGvaqtmnh1wGJ1mcaw76RD8goDJDF60sqPU4GdcVF6saTU+10wsDG6vccOxvgFQToNmy+iq4+Tl4yx1QtxSe+h7cfiX8Wzf85r3w5L+rCdTJIWXkx07BY9+EJ78D574Pzr7RjI8FxC/mvcNT6u7NU190PZdsAsnbgW4hRBfKkN8IvCNxBSHEIimlnn5wLfByIQclpWT/6XHOWZJsSOs8LkanQkSjEptNMBmKICWxPHSAOz94ATUeJ3t69Vva5BzfowM5TELlouOi46mDvn1xD72I5f+/eqaHf/3TPt54djvvPH9p2vUuXqW8hyMDPjZqE6emG/Q8GkRPJ97uLUhrTcJ2vK3Q81QhR5eSnz11DF8gzMcuXwm3n1LZEdNZpKUxntwJresB6KivZPz0WKy0fjprF9Xwy2eOEYlKeoYmOXtxjtkqE/3KmBegtKiTqLjYUNWsvqegD9w5yBDkwKAvSJXLrrLMjj6hzoeOlP5feuwOWP9m9ZgagcN/UemIPc/Anj+kfk/3lXDNt3JqrpIrNRUOvG5HTBCvFMVFGQ26lDIshLgZuB+wA7dLKfcIIb4C7JBS3gN8XAhxLRAGhoD3FnJQD73cR+/IFJ98bXLyf63HSVSCLximpsKJT9On9iak2ulhl2VN6vnY4CTbuuPbODY4QXttRayp66zkouOio3noVSXw0H/x9DHO6qzltreePestvH5rmNgrdGwqbLJB1xtEF+ChJ6oDJqKHXKQ09Ae77/QYK5qrWFMHBMehOoWH3rhSpfydfAE2vRNQmS5nxvxpv4M1bdX4Q1H+dniA0alQ7lWiE/2GhFsgWXGxIbFa1DSDnpCDfvRxFdMu4JzAUwcbblAPUFk6p16AM3shNAlOj5oEbV5d+MFnQAhBe11FvIdvVUvR9VyySvWQUt4H3Ddt2RcT/r4VuNWIA5JS8u9/OcDSxkquO6c96TXd4IxOhpRBD6hbb2+KjJXW6gpcDlssi0Dn6OBETBI2I/mEXDx1MDUSy3MvZgx9dCrEBcsbM4oeVbocNFa5YgY9GI4yFYoUyUPP/8ebtjDK26oVcYwZkputEwhF1fc4rt181rTPXMlmg7azVEqcxnsuXMqrZim00idGP3bnC1Q4bbx2XVtuBzYxYJhBT1JcrEyI+zYYKy2rMzgRVPHz4ITSwrnEELMRp6pRZcyky5oxmfY6T4KH3gRn9hR1/3OuUvThfX3s7h3jo5euxDHNMOlZGHocXe8gk8qg22yCJQ2VsZi5zrHByZj3npF8Qi4VdRAJUCXUMRYzy2VkMhjT58iEKlFXF7tYlWgu4lC5EtGMcAEhl/rZDDoYPjHqD0fURLBWVJTSoIOKoydMjG7rbua9r0pvEFe2eLHbBCOTIb5y7YaYRHHWTPQblqmRrLhofrVo/3hAxc8HDqoFLQVmTs0xOuo8cQ/d21L0GPqcMujhSJRv3b+fJQ2VvHnTzMxI3Vjp2t2+wMyQSyJLGyqT8pbH/CEGJ4I5eOh5hFw0418RGUOI4sXQQ5EoE8HsveyO+rgnMVvZv2EY4KHXeVS18EyDbk61aCAUVcUiuoeeKuQCKtMlYWI0ExVOO9u6m7hx62LesqUz9wMzMuSSqLiob9NEPZfBCU3HZeCAWtBkvKZKKWmv8zA8GVJ35lXNKu05nFqy2gzmlEG/c/tx9p0e59ar16QMG9RO89B1XY9UHjqoYo2eoclYep5elbcs2yKOqWFV9ODMIUdYC88I/yhVLkfRYuj6mGTvoSvNESll7L3p8tANwYBJUYfdRq0nRbMLk6pF/eEIFdl46LGJ0Rey3vYd7zuPb9xwVubG2tMJB5WRMCyGnqi4aK6HHo1KpUKpG3RhN0X1sJR0JKUuFr+4aM4Y9NOjfm57YD8XLG/gqg2pY4q6QR/JIuQCSoNkMhiJNaXVu+5k7aHnouOikyTQZS9aDF1P5cvWy+6s9xAIR+n3BWJ3PEWZFC2gsAigodI1Uy3Sa04Rhz8Uxe20KYNeUZf+wt64UlUI9z5n6P5ToueJG1D2D9MUF50eNcFrUu706FSISFSqoqL+/SpO7yhR1yKTiKUujvhLYtCzrH83j6lghH/85XM8dqAfu03wxTesj3stUqr+f40rQYgZHroeckklKAXx9mmH+ydoqamIhV+WZu2h56DjopMooev24CtS6X+uYRPdk+gdnipSyEXPcims7VdDlWtmtainXvVwNDrkonvo46eUnkg6bDbo2AQndhi6/5QYWCUK0xQXwdTeorqGUKPXrTz0eRZugWnFRYv0hjfFKy4quYd+qM/HI/v7efOmTv50yzbWaS3NmByC374PvrcFnv8ZoMr1nXYRN+iah56uQfFKTQfmkKYLc3RggpZqtznSuToJHnqVu3ht6EanlJHLVmtFT+88UXSDXlhj4PpUfVGFMKVaNMlDT1VUlEjHuSqjIZi5wUpBGGzQIUFxEVSmi0kepS7b0eSxqa5B89Cgt1a7sQmtuCgWwirexGjJDfq41j3nLVs6WdmSkPv6s2uV9kJVMzz7I5ASoXnpiR56hdOWNk1vUW0FlS57TBfm2OBkTAo2K/IJuSR46NVuZ+zzmY0ecqnLYVIUkg16TcXcjqGDCrmkbHRtQrVoIKRluYyfhuoMqYUdW0BG4g0UzEL39gw06DHFRX27ZnnomkFvi55RvXfnoUF32G201Wi56Lqey0KKoY/5U4RNRo7D6V1wxZeVrOaZ3XD8WUBN3OnSo+MJwlypEEKwotnL4X4fUkoO9o3nliI2NZpHyEXLg54aobrCEYvzm02uXrbX7aCu0knvyCSjUyE8Tjsuh4mng0EeeoNX9UWV08vTTegt6g9HcTuEyvrIZEA7t6jnXpPDLjEP3TiBqboEJ4mqJtOyXPSQS5P/qFpQhGKfUhDLIHNVqWYaRRToKrlB1z3YaneCITr+jHru2qYa8rprYMdPAJI9dP/sBh1U2OVQn4/D/T6GJ0OcuzQHA51PyEWX0PWPUF3hLJpB1z30XDJV9HZpppf9g2puAYUb9EoXoYiMzZ/E8LYY+sORUhIMR6kVUxANZ56E9LZA7RLzJ0Yn+tV8gYEFVPWViYqL5rVOG/AFVXRs/Iha0NQ9+xvKlLZaD6fH/Foo0Lw7nlTMAYOewkPveUr1a2xZD26vEtTZ8wfwjyZ5ExMJ7efSsbLFy6lRP4/uV57N1q4sBY0iYVXunU+bL4/SRK/xOGIZJGYzOhWipsKRk2pfZ10lx4cmi2PQDZoUnbW4aKI/N0nVWQiElUpnjdQEzCqz8Ig7z4UTJhv0yQFldA2UOJihuBgNx4vqDGRsKoTX7cA2eBC8bYZelOYSjYkT91XNCy2GnqI4qOdpdQtr15atukpVGp58IclDzxRyAWIdj+58tocmrzv7HHS9zVeuIReI6blUVzjxBcNF6Ss6OhXKudKzs97D4f4JHtx7Juv89bwxoLAI4gJdKYuLZNQw+deA1nWoNmbQs0gT7NgCoz0q5m4WEwOG63knKy7qqXbGy+iO+ZVkB4OH5q13Dqr6dswfJqSP50KKoY/7Vfw2NrHpH1XZAksujK/Uvkk9n9ypUqy028NELfR06Jkuh/snOK+rPvtCjnx0XHQ0PZeaCgdSExMzm5HJIHWe3HJ6b7poGZ+4optPXrGKW68xuQTboElR3UM3u7jIr2mu10S0C3tVFgZdb0l35FFDjiElBlaJ6iQqLsYuXCYYoXG/1l1spEdJ385T6qsS9HGqmosq0DUHDHo4OdxyfDsgYckF8WWVDeoE0Dz08YDyehPbz6VjaWMVDi0MkditPiP56LjoVNTB1HDs2IoRRx+dCuXsZS9uqOQTV6ziliu6ZzZuMJpwABBgK6z0oaEyTaNr3aCPG2TQtebeVbpBz8ZDbztLhWYO/cWQY0jJ6In0EgR5kqi4aGYxzNhUiIYKqfL66wpuajZnqZ8+npMDqmNSESi5QZ9hlHueUiXBetaATvs5cPIFajxOpFRG0pdFyMVpt8X00HMy6PnouOh44iEXoCipiyOztJCbE+jdigqM/TZ403noxuq56DH0qoh2HmQTQ7fZlGb64YfN+QH7R9XnMzhckaS4aKKey7g/zDKHdudbt8Tw7c8VklRBvS2mzUmkouQGfcwfihk+QKUrtqxVKT+JtG+CkWM0O+IKgT5/OG1RUSLdLV6q3Y6YbGlWFBJyqagF/1jsQjU2VQQPfTKUdQ56SQgHCo6fg2ov6LLbZin/N9ZD94RGlJ7P9PMxHSsvV8bw9EuGHEcSA4fUc6OxBr0uVoEdTAi5GG/Qx/whFtu02HztAvDQJ8y940lFyUv/Z4RcRo9Dw/KZK2px9M6pA4Cbfp+fYCSaMeQC8E+vW83pC9I3HEhJISEXdw2Ep6jRQtpme+i6wJbpE5uFEPYXHD8HVVtQWxmvRYjhqlKZUQalLuoeuic0rIxctncWyy9Vz4f/ou4qjWRQk5w1uCAnboBCSlulota0GHo72vczj0MuMQ99Mgh6q01fX1Hy7kvuoY/7Q3GjLKWaMEl19V50NgAtPtXdTm/OkCnkAtDdWs227hwnkvSQSz4eulvdCdTa1DGaHUOfCEYIR6X5qYeFYJCHDsm1CEkYWC2qe+ju4HB2E6I61a3QutGcOLquUFi/zNDNJikuginVotGoZNwfoiXaD8I2uzZOmaOP57AecoGipS7OAYMejhcVTQ2rfoaprt6eeqjvom5kN6CyViA7g54XU8PgrMpPDa5CGfRqdINuroeuZ/3kmuVSVCLFMOjG6bnoaYuu4EjuyoZrrlENkIePGnIsMQYOmqJQmKS4CKbouUwEw0QlNIbPQHW7aqI8T6lw2qly2RmaCMUnsHUJZpOZGwZd99BHj6vndPG1RWfjGVQtnX7+1FGcdsGFs7T6Koh8dFx0tH6MXlS8f8xkD70oHYcKpdw8dC1t0eEfzG5CNJHN71EhmufuMORYYgwcNDx+DsVRXNTvUuuC8zvDRae+SklUUFGrJIlHTxRlvyU16KGI6mUZmxQd0Qx6ui+8eTX2seO4CDE8GeL6TZ0xuUrDyUc6V0cLubgjE7jsNtNDLqM5aqGXBINi6FAcD92veegO/3DuHnptJ6y6Gp7/ebxCtlCiERg6YlpBTpLiop5qZyBj2l2q139qXk+I6jToqqBCqPNhLhl0IcRVQoj9QohDQojPzbLeDUIIKYTYkm6dRCam65nHPPQ0KU2N3QgZZYWjH5uAj1xiYreTfHRcdLSQi57pMmZ2yCXHbkUlQU9bNIBZPfTAKISmCt5HIBzBSRhbcCy/ZhJb36+M4t57Cj4WQM0tRQKmGfRkxcUmVXFrkIwCKA/dTgTP1Jl5nbKoU1/piqfW1nTAWG9R9pvRoAsh7MD3gauBdcDbhRDrUqxXDdwCPJPtzmfouIz0qLh1ZZp88UZlwM+rHuS6TR0sy7W5bi4UFHLRDHpgjBqP+QJdsfZzczmGbmDIRR/TyHRJBQObRftDUeoYV//kMimqs/wy1Zjl0a8bo5E+YE6Gi06y4mKzklHQU3cNYGwqRCvDCBlZECGXhkTd/toOGJ0jBh04DzgkpTwipQwCdwFvSrHeV4H/Bfiz3bnuucZDLj3qy06XIta4EoBbz3fyjevPynY3+TFlhEEf1yR0zZ4ULYeQS0DlcxuA/jlnjKuBBj0QjtAgNIOej4dus8EbvqPCJA9/teDjiaUsmhBDhxSKi2BoHH3cH6ZTaBOtCyDkUl+ZINBV06myXIwKv81CNga9Azie8P8JbVkMIcRmYLGU8n9y2fkMD330+OxfdkUNeFupGDlirnY3GBRyGS2KJvrp0SkqnDbVoX6uEvYbOikKzAy7GFhc5A9FEwx6nmJYXdtg6wfh6f+AA/cXdkCnXlQXlnzuFrIgSXGx0vhm0WP+EB1Cu0DMYx0XnYYqJxPBiEp/rdXMZRHCLgVbACGEDbgN+HQW635ICLFDCLGjv78/RcjleObbscZupdZmJuEAhKfy99AdbuWNBsZM71okpeTh/X1cuLwx9w7yxcTgGDqkMujGCXQFwhFa7arTVUENma/4ErRthLveAbt+m982Qn7Y/yc10WoSqRUXjTPo4/5w3KDXdhq23bmKLiI3MhmKf94ihF2yMei9QKKV7dSW6VQDG4BHhRBHgQuAe1JNjEopfySl3CKl3NLc3BxvblHhhIAPpoYy3441rojffppFITouOhU18UlRE0v/954a4/jQFFdtyNAirdQYnIcOKQx6VRMgjAncc/guAAAgAElEQVS5hKK0GGHQ3V5473/D4vPhdx+A//4k+Mdy28ahhyAwBhuuz/84MpCkuBjTczFOQndsKsRS+4Bqy+Y05sI+l9FF5IYmgirkAnPGQ98OdAshuoQQLuBGIDZ1L6UclVI2SSmXSSmXAU8D10opM/biSvLQ9QyXTDPgTd3qRJscyuLQ86QQHRcdd7UWQzfXQ79/92lsAq5Y22raPgyhGCEXu1MZ9fFTBe/DH4rQZNMNeg6ibqmoqIV3/Q4uvFnlpn9vq3qOZHmh3/07dVHpek1hxzELSYqLlQ2AMDjkEmaJfXBBTIjCNJnnmna1cPT4LO8whowGXUoZBm4G7gdeBu6WUu4RQnxFCHFtITuPe+iOeA56Rg9dTYwyeLiQXc9OITouOu4aLcvFwUQwMjMjwyD+vOc0W5c10Og1xliahsGFRZDCoINhKWKBcJRG27gyxkZUNTo9cOXX4AMPKafl3lvgPy6Cff8ze7u34AQc+DOsuy7e8MUEkhQXbXZl1A2OobczsCBSFmGa4qKrEjwNcybkgpTyPinlKinlCinl17RlX5RSzkiylVJeko13DqrjkMthU53VRzMUFenos/xmhl0MDbmoH4rPhInRI/0+Dpzxzf1wi5SGFxZBGoNe22nID8cfitDAeGHhllR0ngsfeADe9guVGnjXO+C/roaeNNm+f/03CE3ChhuMPY5pJCkuguF6Lr6pIG2yb0FkuMA0TXRQ5+UcCbmYxrg/TI0+IaqfPJm6sdQvVU0SBsw06Fo4p6CQi/LQYxK6JoRdDpxRWRg56byXgoh2UhvkoVc4bbjsNkanQvSPB/j19p74iwZV5QXCURoZUTFfoxEC1r4R/vFpeMO3VWrj7a+D26+Gl36j5gB8/fDYN+GJ22DTu2HpRcYfRwJJioug6bkYZ9BtE/04CS8YD10v8ovnohvjaGSipPK5SsdFu52dHMzu9tbuVGlPQ0fMOzD9VtNbwI/ZXQOB8dgFq2dokntePMk/XLwch92Y66g+2TqnK0TBsAbROkIIajxOxqZC3PlsD7c9eIDL1rTSXO1WIZfguGoGUUATYn8oQr0cAe8yQ445JXYHbHk/bHyriqk/80P4/d8nr7P+enjjvxvaFDoVMYXAxFz0M3sM236VXxOnWiAG3Wm3UVPhiHfWqumAo0+avt8SG/RQXC1xciD729uGLhh+xbwD8/WBw6NEdfJlWsjly/fu4cAZH9u6mzir05h2b7rXP6c7FUHcoNuNq2St9ThUk5OAKk8fngwqg67n/I6eKMigB8JR6qRJHvp03F646Ga44CNwcif0/E1JzLauh2XbVEzbZGo9Tmo9To4MKBVTo5sb1wS0ieoFEnIBWNJYybEhrUq4tkPJUvjH4nUqJjAHPHTdoOegalffpXqPSmmO5zLRD97mwrath1zcyhs/cEZlTIxMb8xQAGNTIYQAr6vkfUpmx6AG0YnohTD6eMZvbTWDMdqrDGKehIIBqqPjhd2l5YrNrmLsnecWb58aQghWt1az/7QK4x0P17LYP6JkC1yVBW+/IXRaBXgXSJYLwIpmLzuOahlz9V3qefiVWG8HMyhxDD2hucXEYG4eemDUvNRFX1/hnpm7GpDU2pMNeMqJvDwZ84epdjuw5dKJqRQYHHIB5VEOT4Y4MqAulPEya91DLyxFzBPUzq1MczrziNVt1Rw4PY6Ukr+cVHd9Z3oLD20GwhFao/34HTUxaemFwMpmL70jU0wGw3ENnv4Dpu6z5JOiSTH0bA164tXODCb6C/8ha7dV9XY/Dpvg+s3K0IwYadDnemNonYhu0I1Lraz1ODnc54vJ3A7psd/qNtXVp8CMgqqwZtCL6aGXmFVt1YwHwpwa9fP0oPLKX95XeBxdrxKd8LQXvK1yYkWLCtke6Z9QBZHCprpOmUhJDfrIZEiloEmpDHq2OhV6z9Ehkwy6r0+FXApBE+iqEZPcd8s2vnbdRiChK4wBjPlD1FSUgUGPhVyMNejBSDT2fyyUZbOrQo4CM128Ie1WuRgx9DnC6lblPT+6v5/dE+r8PfFKYdlk+06P0TM0SafoJ1A1f9vOpWJFszLoh/t96tyvXwYD+03dZ8mCr1EpmQpFaK1xq7ZzkUAOHrom7mOGhx6NqAnaQn/ICZroq5aoH0qly25wDD1MjWeOx88hrk9uYMgl8c7EbhPxGDoYkiJWE9UMeqEX9jJCN+h3PtvDaVlPFMF431EiUZlbg3WNQDjCm773JFJKXnAMMFg9/zVcElnWVIlNxNtl0rTa3HRrSuihhyKqOq6luiKuGZHtpKjTo/oSmpG6ODmkCj4KvdVOkNDVqUts82UAZeOhB7UT2l1A1tA09OKixioXbTUV8Rg6aNWihXnotZGF56HXVjppq6lgV+8oDqeLQEUzjeE+dvWO5rW9QV+QQDhKp9tPlQjgalpm7AHPcdwOO0saKpWHDkq2ZPBQ9pIPeVBCg65ul1tq3AkGPYeqvIYuc0IuenfuQmPoMYMe/zHUJkqUGkDZxNCD2gldSBroNHSDvqLZS0NVQncYiHvo0Wiad89ONCqplyOEbBWGXoTKgVVtyks/q7MOZ/0S2m0D/PVAfumLeg721y5Tv4W2JeZouc9lVjR7Odynnf/Nq1WR3cgx0/ZXMoMe1jz01poKleECuRn0epNy0XWlvkI99ISQi47qCmNkDD1cJh66lovrLDz9TSdm0Fu81FU6GUq8UNZ2QjSUdx51IBylSYwy6cpTB72MWd2qLmCbl9TjaFhMl2OYp4+o3+eAL8Cn7t6ZtdjcwISaDG+JaHLGC6SoKJEVLV6ODEwoLaem1WqhiROjpffQqxM89FzE+xu6lO61fjtvFLoRMCRtESV7qlFX6TTMQw9HovgCZRJD178jl3EtA3WDvrJF89Cnh1wg74nRQDhCE6ME3HNcUsEEVmlx9M1L6qC2kxY5wN6To0gpeWDPGX7/fC9PH0lOF37u2HBKIz+keegNfi2FVE9mWECsaK4iGI7SOzwV7wc7Hw16OCqpdNlVpajeYTzXkAvA8FFjDyzmoRcYcnF5VZpSYgy90rgYuk9rsF0eHrrxIZcVzVUsa6zkohWNyQ15IaGhQE/qN2fAH4rSJMYIVCw8D/3KDW18/PJuLl7VDLWLccogtqkhTo/5Y7H0WEwY6Bvz85Yf/I3/evLojG0Nah66d6JHOUgLKAddZ6WWuniof1ypt3pbTc1FL6mH3lpTobrsTA6CzRmPO2eDnotu9MToRJ8qUS9EmAtUlam7OinkUutxMToZQs4ml5oluo5LecTQJ9T36zCu9L/R6+bR/+dS1i6qob7Sxbg/HLvri3mCeUos+0MRmsQooQVo0GsqnHzqtauocNpjF8Z2McDek2PsOakM+qG+uEF/9EA/UanSE6cz6AvidthwjL4Sa/C+0FjZoi5ieqU4TatMTV0saQy9uVrLS9aLinIptdc9dKMnRn1aUZERkgLu2hkhl2AkylQoUvCmYzouFWUScjEw3DKdhqppwlJuL1QvyrtVYSAYpIFxIvn2Ep0vaAa9Qwzy4olR9p1Sd5uJBv2x/f0zlukM+II0ed2IoVcWZLgFVGiwvbYiJqlA20Y4vRvCxs2lJVJyDx3Irexfx1OvvGijJ0Yn+owr99YEunR0zWkj4uhjU2UizAUQMtegJ/Vv1GlcmbdBD4/3YxOSiGfh5KCnRNPFWe8d456dvQQjUZq8Lg73+ZBSEo5EefygMuivDEzE75A0BicCtFdFVQcp3QFbgKxuq+blU5odWHKB6ld86kVT9lXSGHprooeeTzfzhuUmeOh9xpV7V9QqGVeNukoDDXrMQy8Dg262h57Yv1EnS4P+4vERHt6X3FQ6alSmU7njqQeHh7WeMY4OqkylN57dznggTP94gJ3HRxjzh7libQuhiOSYto7OoC/IGpeW8LBAPXSA1W01HO73qQvekgvVwp6nTNlXyQx6VEqVgw65SecmYoaM7kS/ccUkFXVJBr3Wo3mSBqQuxmPoVsgl1g8z0aA3davesHpKbBq+dO8ePve7XckLfQbVIpQ7QkBtJ10ONYbVbgeXr1G9aw/1+Xh0fz92m+C9F3XFliUy6AuwwqFdLBewQV+7qJpQRCpNF28LNKzI26Dv/taVs75eUi2XWMglF+ncROq7VC/SiEHFOtFoXDrXCCpq4/1JiXvoo0Z66OUQcglOGJrhMp1Y/8bJaR46zOqlD08E2Xl8hL7xQFJHKaGlrtqq53jj7WLQuILWsJJRWN9RQ7eWp77/zDj37TrF5iV1bFqiEggO9cUzuqSUDEwEWcJptWABG/TVWrFWbOJ4yYXQ83TOhW/+iVHW+J6ddZ2SGvSW6gpVBjs1kr+HLiMwkl962gz8IxANG+ihpwm5GJC6WDZa6KAMuoFFRdNJGcqKGfT02hmPHxqI9Wc+nOBdOnzKgNlrFxl7oOVI8xq840dwEGZDey0t1W6q3Q7+869HODIwwfte1UWV20FHnSfJQ/cFwgTDURZFTipnrYBmI+XO8iYvDpuIT4wuvVC1ucwxH/3Yi4/hELNfBLIy6EKIq4QQ+4UQh4QQn0vx+oeFELuEEDuFEE8IIdZls92WGre6LUbmZ9CNltHVJVerDWq67KlTWS5RldVS50kxeZcnZaOFDqaHXCqcdipd9uQYet1SlSo5i4f+6P4+HNr4JRoj1/hx+mUtLs/Cy5ueQctaRDTENy718u4LlyKEYEWLl5Ojfta0VXPVevVbWdni5WDCGOrfRWOgd0F75wAuh40VzV726QY9QxzdFwgzOjUzvdl34HEicvbfe0aDLoSwA98HrgbWAW9PYbB/JaXcKKU8B/gmcFum7YIWconpuORRlWe0jK7u6etqjoWieyWal17htOFy2AyKoZeJjguYbtBBNTlOiqHbHeoOLo1Bj0Ylfz3Qz5Xr23DaRUwRb3giyNipQ5ygNRbKWdA0rwHg7zrHWdqovkO9WOYTV6yKORQrW7wc7vcRjSojNKBVidZMHV/wBh1gzaJ4NygaloO3DQ49lLTOU4cHufFHT3H2lx/g7C8/wIZ/uT8muwBQdWY7h22zZwtl46GfBxySUh6RUgaBu4A3Ja4gpUysKqgCMlbO2IRQVaJ6SCKfQp7qNtX706hqUd2g1y0zZnvTDLoQQum5GBRDL4sMFzA9hg7MFOgCFXYZOMRDe89w17PJYbk9J8cY8AW5fG0LyxqrONTnwx+K8O7bn6ExdIqOrjVUucsgnGU2TasAAX0vxxbdsLmT9160jCvXx+cYulu8+ENRekeUVPKgL0AlfiomTy3YoqJEVrdV0zsyxZkxv5ps3nADHLg/1nWtb9zPh3/xHMeHpvjQxcv5wuvXYhOC3z2n5CtkOMjSqb2cqjtn1v1kY9A7gMR+Xie0ZUkIIT4qhDiM8tA/nmmjDrt266AX3uTTOFUIJRpvVLXoSA84q/K7W0jFNIMOheu5/Hn3aR470K+Eucohw0VKVfpvQF/K2airdHJ0cBJ/YtFW40qiQ0e4+Vc7+Oc/7OJIQsn6jx4/gtMuuHhVc8y7fHhfH/t6h+gUg7QsWW3q8ZYNrkp1p9MfN+gXrmjkS9euV1XeGmsWqd/vSyfUuT44EWSt0FQF2zYW73jnKNdsWITDJvjOQ1rc/Jx3KAG5Xb9FSskX/rCbqVCEn77/PD571Rr+fttyLl3TwsP7+ohEJf2HtuMhQHTxBbPux7BJUSnl96WUK4DPAl9ItY4Q4kNCiB1CiB0eNKMW89DznDQxUkZ3+JhShDOq8bR+15GY6eJxFRRy+dI9e/j03S8y4AuUh4ceCaqJa5NDLtdv7uDo4ARv++FT9I2rDkmhhm5skQDrXP24HXa++7AKv9y/5zT3vniSj13WTZPXzYpmLz1Dk/z++V7WV40jiKoYvIWieS307Zt1lfXtNVQ4bWw/qjzOQV+AjTbtd7lodq9yIbCsqYp3X7iUX28/rkIvbRvUhe7FX/HogX4e2HuGT712VSycBfDada0MTgR5oWeY/t2PAtC64ZJZ95ONQe8FElt1d2rL0nEXcF2qF6SUP5JSbpFSblnSqnnBuoeei45LIvVdKuRigD4KIz3GSnym8NBrC/DQT41OcXrMz4AvwJH+ifKIoceUFs0Nubx5Uyc/fNe5HDjj46O/fJ5oVHLHUXWOfXVrkPdctJQ/7uzlOw8d4PN/2MW6RTV85BIVCljZ4iUSlTz08hnevEz7buqXmXq8ZUXLGjUXoTf7ToHTbmPT4vqYQR/wBdnkPKrEqGqsbCGAWy7vxut28MGf7eAL/3cXZ5ZfDydfYO8LT1HpsvOBVyfHx1+zuhmnXfDA3jO4D/+JI7Kd7hWza8pnY9C3A91CiC4hhAu4EbgncQUhROJeXg9k32fJX0DIBZSHHp6CsZP5vT+RkR7jJkQhdcjFk79Bf6FHefrVmn5LWXjoMaVFcz10gNetb+Or121g+9FhPnn3Tr7xnCRg87BeHuIfLl6Bx2nnOw8dpMnr5js3noPTrk7/RK/oNS1ataOR50G507xW3WVlqLzd2tXAy6fGGPeHGJwIstF2DBadXaSDnPvUVbq47a3n0Frj5u4dJ/hazwZweTnv8P/mnM7a2PmoU1Ph5ILljTzzxEOsnNrFK8veOmOd6WQ06FLKMHAzcD/wMnC3lHKPEOIrQohrtdVuFkLsEULsBD4F3JT1pwyMqS7t+eYpN61Sz4VqDE8Nq+5CRnroHi3kMhUPuaxs8XJ6zJ8Uz82WF3qGcTlsfOZKFd8tixi67qGbmIeeyA2bO7h8TQt/3HmSpU3VODo2wcnnaahy8d8f38bjn7mUP3/i4pjuN8DyZnWxaaupYJmtH2yOuKa6hfLQIWliNBXnLWsgKpU+et/QEEujx61wyzSuWNfKbz58EW/fupgHjkWYeNVn2RrawY3VqbVdXr9xEe+z30fQXsVlb/9Uxu1nFUOXUt4npVwlpVwhpfyatuyLUsp7tL9vkVKul1KeI6W8VEq5J+tP6B9T3nm+cetmg7qAxDJcDDTouiZ6gof+5k0d2G2Cu3dk33xhwKdudV/oGWFjRy1v2bKYi1c1c15XGTRg0LsVmRxy0RFC8K/Xb+TqDW189+2bsHduhlMvQThIV1MVixtmXlgqXQ7O62rgnecvQYwcVaJUNntRjrcsaFoFdjf0Pj/rapuW1GG3CW578ADBE7uwE7U89DS8ZnUz/lCU701cxt7oUq7suQ3GTs1Y722rbbzJ+SyurTchsphnLGmlKKA89Hzj56BidO5a6C9QY9gMgy7EjGrRlpoKLl3dzO+eP0E4krn09/GD/Wz92kP8adcpdvWOsmlxHRVOOz97/3lctKIM5F2LGHLRaamp4D/edS7r22uhYzNEAtC3d9b33P0PF/Kxy7vVxLgVP0/G4YYl58ORR2ddrcrtYEN7DS+dGOV1DZpxarc89FRcsLwRl8PGHU+d4HPhD+KK+OD2K5M1/CcGEHfeiBA2OP8fstpu6Q26fzT/+Dkoo9m8ykAP3eDYaUVdUpYLwFu3LKZ/PMDPnjrGw/vOMBVMr4/+1wP9SAmfvHsngXCUTUvqjT0+szGh/VxOdJyrnk/O7l3GGD5qxc9TsfxS6NsD42dmXe3CFU047YK3dw6rkn8rdJWSSpeD87samApFCLacjbjpXtXd7P9cCL/7e/jT5+DHV8DAQbjxzqydjDlg0MeUh10ITasL99CHj4GrWkmGGsk0Dx3g0jUttFS7+cp/7+X9d+zgJ0+kz6PffnSYxQ2eWFNtXQipbCi1Qa9bCp4G6H0u87r+MaWxYXnoM1lxqXrO4KV//PKVPPCJi6nr367ujoxKAZ6HvGaVEgE8d2m9cjw+9AicexMcfACe/6myRe/8LXRfkfU2S2/QA2OFeeigPPSJPk0XJk/0lEWjT8AUBt1pt/HLvz+f/3zPFta31/Dg3tRez2QwzO7eUd54Vjv/fM1atnU3sai2wtjjM5sShFySEEL9WI5vz7yufpfXYFU2zqDtbGVgMhj0SpeDrugxVey3+uriHFuZcvnaVhw2wbZuTd21fhlc8y34zCtwa68y8F3bctpm6Q26v8AYOigPHQprvjpyzJxbbU9dUpaLTndrNa9d18rVG9p48cQofWP+Gevs7BkhHJVs7Wrg/a/u4ucfOD+pOq8sCOmToiUy6KC8y4H9mSUijj+jnju3mn5IZYfNBl2vgSOPZK75ePleQMDq1xfl0MqVrqYqnrr18iQJBUBNyNvyM82lN+iBAmPoEM906Z+9mi0tkZDKsTVDcyKFh57I5WvVl/nI/r4Zr20/OowQsLnc4uaJxNIWS2jQdU9x/59nX+/4M+ouzSqESc2KS1U7udO7Zl/v5XtVqzVLTz4jzdVuQ5200hp0KdVEQKEeet0ScFTkPzE6eEiVqLeaoDmRwaCvaaumvbaCh15OZdCHWNNWQ205VISmI+hT3429hDnzDcvVXdz++9KvIyUcfxYWn1+84yo31rxRieE984P06wwehjO7Ye216dexMI3SGvSgD2S0cA/dZofG7vwnRk/vVs9tGwo7jlRU1KlK1jRl00IILlvbwhMHB2LCUsFwlD/u7OW5Y8NsXVbG3jmY3twia1ZfDceeTH9xHelR3qdl0NNT1Qib3wMv/RpG09RR7PqNel77huIdl0WM0hp0f4E6Lok0r84/5HJml2qGoFedGkmK8v/pXL6mlalQhGdeUToYH/r5Dm65ayctNW7eeX6Zp9AFJ4tWVDQrq69W3aimaVDHOK619rIM+uxc+FF1N/PU/5n52ugJePLfVezcyHoOi6wprUEvRDp3OovOhtHjMDGQ+3vP7FFC/nYTQhsxxcX0Bv385Q247DaeONjPgC/AYwf6+cCru3jk05fE+hGWLUFfaSdEdTq3qqbPL/469evHn1YXntb1xT2ucqN+KWz8O9j+Yzj6RPJrf/6cMvZXf6M0x2YxVzx0A/oNtm9Szyd35v7e07vNCbdASj2X6VS6HJy7tJ7HDw7w2H5VSHTdOR3l0V4uE0XoVpQVNjuc9yE4eH88xJbIsaegc4tV8p8NV35dpdj96m2w949KvvreW9Rk6Gs+Y3nnJWR+eeiI7CsCdSYGwHfaPM8si5ALwLZVTew7Pc7dO47TXO1mfbsBYzIXCE6Y3twia877oPLCn/h28vJX/qqqIFdfU5rjKjeqmuA9fwRvC9z9Hvjf58DzP4cLPgoXfazUR7egKbGHrhk5I2LoFTXQ1A0nX8jtfWc0b63VJA89ZtDTe+gAF2vFBc+8MsQlq5rnh3cOEDK//VzWeOphy/thz+/jDRukhEe+DtWLYHP2IqELnppF8OEnlWG/8uvw4cfhqq+bE7a0yJr546GDCrvkbNA1YUiz2mTpMfQMVazrFtXEmhJftqbFnGMpBXMl5KJz4c3qO/nF9SrFbs8fVPf1bZ8GZ5lV4ZYaVyUsv0RNlFpzD3OCORJDN9Cgj59KKUOZlpM7VQfuKpOUC/X+pFoz2HTYbIJXr2zCYRO8qrsMVBSzZa4Z9OpWuOkeVcH63XPht+9T8eDN7yn1kVlYFExpOyT4R1VzC6N+8PrE6Kmd2VX7Sanip8teZcz+U2F3qlv9iZmFQ9P57NVreMuWzvLoRJQtwYnSVommom0jvPd/4IVfqr+7X6ckYi0sypzSGvRAgc0tptO2UTWU6H0+O2GggQNqQrTrNcbsPx1VzeDLbNA76jx01HnMPZZiEgmptEWjQmpG0rpexXwtLOYRpQ+5GBVuAeXpt22cmR+bDl05bvklxh1DKqpa8suPL3cm+tWzdx7NCVhYzGFKPylqtPe28rVKZCkbKd0jj6r4qdkNDbzNWYVc5h36XUmVZdAtLIrBHPDQDSgqSqT7dapD+eFHZl8vElae/PJLjN1/Kqqa497qQsLy0C0sikpWBl0IcZUQYr8Q4pAQ4nMpXv+UEGKvEOIlIcRfhBDZubxmeOidW9QkZDrNDp2TL6j9mx0/B+Wh+kfTCnTNW2IeenNpj8PCYoGQ0aALIezA94GrgXXA24UQ66at9gKwRUp5FvBb4JtZ7d3oGDqo0u0Vl8HBByE6SxPmPb9XglzLLzF2/6nQUyIXmpfu0zoxWR66hUVRyMZDPw84JKU8IqUMAncBb0pcQUr5iJRSa03D00BnVnsPjIHbBPGp7tepmPWpNLouoSnY+StY+8Z4nriZ6AZtoRn0iX6VsjiX8tAtLOYx2Rj0DuB4wv8ntGXp+ADwp6z2Hhg3x6CvfC3Y3arRair2/F9Vir/lfcbvOxV6yMG3wAy6r09NCFtYWBQFQydFhRDvArYA30rz+oeEEDuEEDsG+vvU5KXbBJ2PqkbY9C7lhY+dnPn6c/8FjSthWW4NWPM/Hs2oLbRMl4k+K8PFwqKICJmh4asQ4kLgS1LKK7X/bwWQUv7rtPWuAL4LvEZKmdFyCSHGgTxbDCVRC8wuZTg3aALKIRndGk9jscbTWKzxhNVSypShjWwqRbcD3UKILqAXuBF4R+IKQohNwA+Bq7Ix5hr7pZRbslw3LUKIH0kpP1TodsxGCLHDiM9rNtZ4Gos1nsZijafadrrXMoZcpJRh4GbgfuBl4G4p5R4hxFeEEHon2G8BXuA3QoidQoh7DDjubLm3iPtaCFjjaSzWeBqLNZ6zkDHkYtqOy8QjMIqF9nnNxhpPY7HG01jM9tDTbbuUlaI/KuG+S8FC+7xmY42nsVjjaSxmjmfabZfMoEspkw5KCHG7EKJPCLE7Ydm3hBD7tArUPwgh6lJtK10lqxCiSwjxjLb810IIl3mfaHamf95iMJ/H1BpPY7HG01jMHM/Ztl1aLZdk7gCumrbsQWCDVoF6ALh1+psyVLL+L+DbUsqVwDAqR34hcQfWmBrJHVjjaSR3YI2nocwZgy6l/CswNG3ZA9qkLKSvQE1ZySqEEMBlKCkCgJ8C1xlxrKm8g2w9AyHErdo6+4UQV862zUIplyI4FgQAAAfgSURBVDG1xtMaz4Rlc248oXzGFCnlnHkAy4DdaV67F3iX9nc7cJ/2998BP05Y793A91B5oIcSli9Ot+0cj9EOHAaWAy7gRZSXcDdwo7bOD4CPpHjvOm19N9ClbceebpsLYUyt8bTGcy6PZ7mN6Zzx0GdDCPF5IAz8EkBKeVJKeU2JDiedtk02nsGbgLuklAEp5SvAIW17GfVyjGYOjak1nsZijafxlM2YznmDLoR4L/AG4J1Su+RNoxd1Jdbp1JYNAnVCCMe05YWSTttmRMZvFWN6N0KIa4UQX8nw3lz1cgpijo2pNZ7WeCYxx8YTymhM57RBF0JcBXwGuFbG1RynE6tk1WJYNwL3aCfCI6jbM4CbgD+afczTkVLeI6X8YrH3m45yH1NrPI3FGk/jKeWYzhmDLoS4E3gKWC2EOCGE+AAqLlYNPChUBeoPtHXbhRD3QfpKVm2znwU+JYQ4BDQCPzHgUNN5B9l4Bunem255QZTJmFrjaY3nXB5PKKMxLXhSY6E9UPo3R1ATHPpkxnrgNyRPkPxjiveuJ3mC5AhqciTlNkv9Wa3xLL+HNZ4Le0xLPljl+ACuQeXIHgY+ry1bDjyLmvT4DeDWll8LfCXhvZ/X3rcfuHq2bS6UhzWe1njO9Ue5jGnJtFwsLCwsLIxlzsTQLSwsLCwKwzLoFhYWFvMEy6BngRBisRDiESHEXiHEHiHELdryrwolIrRTCPGAEKI9zfsf1Up8XxJKeOh7Io3o0EIg3XgmvP5pIYQUQjSleb81ngnMcn5+SQjRq52fO4UQKQtzhBB3CCFeEUK8KIQ4IIT4mRAiu0bv85DZzk8hxMe0c26PEOKbad5fuvEs9WRDOTyARcBm7e9q1ETGOqAmYZ2PAz9I8/5HgS3a3y7g/wMeK/Xnmmvjqf2/GJWOdgxossYz//EEvgT8UxbvvwP4O+1vAXxS24ar1J9tjo3npcBDxCc/W+baeFoeehZIKU9JKZ/X/h5H5b52SCnHElarAjLOMEtV5vsZYIkQ4mxQzbWFEM9qXtQPhVKT08V7nteu9H8x+nOVinTjqb38bdT4ZDVbb41nxvHMdVtSSvlt4DRKzRAhxOuEEE9pY/cbIYRXW75VCPE3bTyfFUKk7HNZbswynh8BviGlDGivZWy3WezxtAx6jgghlgGbgGe0/78mhDgOvBPIqjpMShlB5Z2uEUKsBd4GvEpKeQ4QAd4phGgG/hO4QUp5NvAWgz/KnCBxPIUQbwJ6pZQv5rINazzjTD8/gZu10NTtQoj6HDb1PGo8m4AvAFdIKTcDO1CFOy7g18At2nheAUwZ9DHmDNPGcxWwTSiFxceEEFtz2FRRxjObJtEWGtqV9HfAJ3TvXEr5eeDzQohbUdVr/5Lt5rTny4Fzge1CCAAP0AdcAPxVKkEfpJRDqTZSziSOJ0qI6Z+B1+W7Oe3ZGk/t/BRC/AfwVdTdzldRoan3Z7s57fkCVLjhSW08XWjVncApKeV2gGl3q/OCFOPpABpQY7IVuFsIsVxqsZVMm9OeTR1Py0PPEiGEE/Xl/lJK+fsUq/wSuEFb937tdv/HabZlBzaibuUE8FMp5TnaY7WU8kumfIg5RIrxXIGqmntRCHEUVQr9vBCizRrPzKQ6P6WUZ6SUESllFHV3cp627n9p43nfLJvcRHw8H0wYz3VSynnfNCLN7/0E8HstjPIsEAWa5tR4mh2knw8P7Uv4GfCdacu7E/7+GPDbNO9/lPgknhP4JtokHupqfRBtggXlASwFmlFqbF368lKPg9njOW2do2Q3KWqNZ/rzc1HC359Eybimev8dJE/ifVwbQ5c2bj3ASu31KlTowYUqXd+qLa8GHKUeC5PH88NoFaDaGBwHVZw5V8az5INXDg/g1ajb1peAndrjGtQVfLe2/F7URGmq9z+KKvt9SXv+PlCX8PrbtG2+BDwHXKAtvxp4ARUffrDU42D2eE5bJ5NBt8Yz8/n5c2CXtvweEgz8tPffAbyijctB7X2dCa9fhlI4fEl7XKst34rqKvSi9uwt9ViYPJ4u4Bfab/554LK5Np5W6b+FhYXFPMGKoVtYWFjMEyyDbmFhYTFPsAy6hYWFxTzBMuh5Mot+RoMQ4kEhxEHtuV5bvkarDgsIIf4pYTsVWlXYi9p2vlyqz2RhYVHeWJOieSKEWITKGnheK9F9DtX1+73AkJTyG0KIzwH1UsrPCiFaUOlz1wHDUsp/07YjgCoppU/LfX0CVS32dAk+loWFRRljeeh5ItPrPbwJ+Km22k9RBhwpZZ9UVWChaduRUkqf9q9Te1hXWQsLi5yxDLoBTNN7aJVSntJeOg20ZvF+uxBiJ6pE/UEp5TOZ3mNhYWExHcugF0gqfRcdqeJZ2SgwRqQSkuoEzhNCbDDlYC0sLOY1lkEvgDR6D2e0+LoeZ88osakjpRwBHgGuMvpYLSws5j+WQc8TbTLzJ8DLUsrbEl66B7hJ+/sm4I8ZttMstG47QggP8Fpgn/FHbGFhMd+xslzyRAjxauBxlFZGVFv8z6g4+t3AElTXnbdKKYeEEG0o7eMabX0fSkhqGWry1I66wN4tpfxK8T6JhYXFfMEy6BYWFhbzBCvkYmFhYTFPsAy6hYWFxTzBMugWFhYW8wTLoFtYWFjMEyyDbmFhYTFPsAy6hYWFxTzBMugWFhYW8wTLoFtYWFjME/5/Z539YlPq9ZMAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "loader = model.val_dataloader()[0]\n", + "dset_test = loader.dataset\n", + "label_names = dset_test.label_names\n", + "plot_from_loader(loader, model, vis_i=670)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "ExecuteTime": { + "end_time": "2020-01-27T09:03:28.792465Z", + "start_time": "2020-01-27T09:03:28.346446Z" + } + }, + "source": [ + "# Hyperparam opt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-16T04:08:45.711767Z", + "start_time": "2020-02-16T02:36:49.800Z" + } + }, + "outputs": [], + "source": [ + "import argparse \n", + "\n", + "parser = argparse.ArgumentParser(description='PyTorch Lightning example.')\n", + "parser.add_argument('--pruning', '-p', action='store_true',\n", + " help='Activate the pruning feature. `MedianPruner` stops unpromising '\n", + " 'trials at the early stages of training.')\n", + "args = parser.parse_args(['-p'])\n", + "\n", + "pruner = optuna.pruners.MedianPruner() if args.pruning else optuna.pruners.NopPruner()\n", + "\n", + "study = optuna.create_study(direction='minimize', pruner=pruner, storage=f'sqlite:///optuna_result/{name}.db', study_name='no-name-b60e37fc-4ab6-4793-8a0a-c87a1b40c5c0', load_if_exists=True)\n", + "\n", + "# shutil.rmtree(MODEL_DIR)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-01-27T23:38:16.813496Z", + "start_time": "2020-01-27T23:38:16.766674Z" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-16T04:08:45.713036Z", + "start_time": "2020-02-16T02:36:49.800Z" + } + }, + "outputs": [], + "source": [ + "study.optimize(objective, n_trials=200, timeout=6000)\n", + "\n", + "print('Number of finished trials: {}'.format(len(study.trials)))\n", + "\n", + "print('Best trial:')\n", + "trial = study.best_trial\n", + "\n", + "print(' Value: {}'.format(trial.value))\n", + "\n", + "print(' Params: ')\n", + "for key, value in trial.params.items():\n", + " print(' {}: {}'.format(key, value))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-16T04:08:45.714257Z", + "start_time": "2020-02-16T02:36:49.800Z" + } + }, + "outputs": [], + "source": [ + "study.optimize(objective, n_trials=200, timeout=6000)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-16T04:08:45.715466Z", + "start_time": "2020-02-16T02:36:49.800Z" + } + }, + "outputs": [], + "source": [ + "df = study.trials_dataframe(attrs=('number', 'value', 'params', 'state'))\n", + "df.sort_values('value')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Scratch" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-16T04:08:45.716663Z", + "start_time": "2020-02-16T02:36:49.800Z" + } + }, + "outputs": [], + "source": [ + "model, trainer = main(trial, train=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-16T04:08:45.717955Z", + "start_time": "2020-02-16T02:36:49.800Z" + } + }, + "outputs": [], + "source": [ + "loader = model.val_dataloader()[0]\n", + "dset_test = loader.dataset\n", + "label_names = dset_test.label_names" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-16T04:08:45.719361Z", + "start_time": "2020-02-16T02:36:49.800Z" + } + }, + "outputs": [], + "source": [ + "x_rows, y_rows = dset_test.iloc(10)\n", + "x_rows.loc[x_rows.index[model.hparams.window_length:], dset_test.label_names] = 0.\n", + "x_rows" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-16T04:08:45.720669Z", + "start_time": "2020-02-16T02:36:49.800Z" + } + }, + "outputs": [], + "source": [ + "x_rows.loc[x_rows.index[model.hparams.window_length:]]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-16T04:08:45.721941Z", + "start_time": "2020-02-16T02:36:49.800Z" + } + }, + "outputs": [], + "source": [ + "self=dset_test\n", + "idx=10\n", + "k = idx + self.hparams.window_length + self.hparams.target_length\n", + "j = k - self.hparams.target_length\n", + "i = j - self.hparams.window_length\n", + "idx, k, j, i, self.hparams.window_length + self.hparams.target_length" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-16T04:08:45.723150Z", + "start_time": "2020-02-16T02:36:49.800Z" + } + }, + "outputs": [], + "source": [ + "j-i" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-16T04:08:45.724301Z", + "start_time": "2020-02-16T02:36:49.800Z" + } + }, + "outputs": [], + "source": [ + "self.hparams.window_length + self.hparams.target_length" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "file_extension": ".py", + "kernelspec": { + "display_name": "jup3.7.3", + "language": "python", + "name": "jup3.7.3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3" + }, + "mimetype": "text/x-python", + "name": "python", + "npconvert_exporter": "python", + "pygments_lexer": "ipython3", + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": { + "height": "calc(100% - 180px)", + "left": "10px", + "top": "150px", + "width": "384px" + }, + "toc_section_display": true, + "toc_window_display": true + }, + "version": 3 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/smartmeters-lstm-seq2seq.ipynb b/smartmeters-lstm-seq2seq.ipynb new file mode 100644 index 0000000..70ac855 --- /dev/null +++ b/smartmeters-lstm-seq2seq.ipynb @@ -0,0 +1,776 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# This notebook uses pytorch lightning & optuna & an LSTM\n", + "\n", + "This notebook trains an LSTM on timeseries data from smartmeters.\n", + "\n", + "It uses pytorch lighting for the training loop. And Optuna for the hyperparameter optimisation.\n", + "\n", + "It also pushes results to the tensorboard hyperparameter dashboard for examination.\n", + "\n", + "- https://github.com/optuna/optuna/blob/master/examples/pytorch_lightning_simple.py" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "ExecuteTime": { + "end_time": "2020-03-01T03:41:29.841115Z", + "start_time": "2020-03-01T03:41:29.484738Z" + } + }, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "%reload_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "ExecuteTime": { + "end_time": "2020-03-01T03:41:31.805982Z", + "start_time": "2020-03-01T03:41:29.843485Z" + } + }, + "outputs": [], + "source": [ + "import sys, re, os, itertools, functools, collections\n", + "import pandas as pd\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import collections\n", + "from pathlib import Path\n", + "from tqdm.auto import tqdm\n", + "\n", + "import pytorch_lightning as pl\n", + "import optuna\n", + "from optuna.integration import PyTorchLightningPruningCallback\n", + "\n", + "\n", + "import math\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "ExecuteTime": { + "end_time": "2020-03-01T03:41:31.857758Z", + "start_time": "2020-03-01T03:41:31.809871Z" + } + }, + "outputs": [], + "source": [ + "import torch\n", + "from torch import nn\n", + "import torch.nn.functional as F" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "ExecuteTime": { + "end_time": "2020-03-01T03:41:31.973473Z", + "start_time": "2020-03-01T03:41:31.860963Z" + } + }, + "outputs": [], + "source": [ + "from src.models.model import LatentModel\n", + "from src.data.smart_meter import collate_fns, SmartMeterDataSet, get_smartmeter_df\n", + "from src.plot import plot_from_loader\n", + "from src.models.lstm_seqseq import LSTMSeq2Seq_PL\n", + "from src.dict_logger import DictLogger" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "ExecuteTime": { + "end_time": "2020-03-01T03:41:32.025844Z", + "start_time": "2020-03-01T03:41:31.977741Z" + } + }, + "outputs": [], + "source": [ + "import logging\n", + "logging.basicConfig(stream=sys.stdout, level=logging.INFO)\n", + "logger = logging.getLogger(\"smartmeters.ipynb\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "ExecuteTime": { + "end_time": "2020-03-01T03:41:32.075520Z", + "start_time": "2020-03-01T03:41:32.028741Z" + } + }, + "outputs": [], + "source": [ + "# Params\n", + "device='cuda'\n", + "use_logy=False" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Load kaggle smart meter data" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "ExecuteTime": { + "end_time": "2020-03-01T03:41:40.783929Z", + "start_time": "2020-03-01T03:41:32.077822Z" + } + }, + "outputs": [], + "source": [ + "df_train, df_test = get_smartmeter_df()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "ExecuteTime": { + "end_time": "2020-03-01T03:41:41.497026Z", + "start_time": "2020-03-01T03:41:40.786905Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEGCAYAAABrQF4qAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO2dd5gUVdaH30OOIlEF1EFERVBREWVxTRhAVl0/V9fsurrornGDirsqCAZWd13XHDGwiiIGQJCkICpxiJJzGIacGWZgwvn+qOqhZqZnumemuru657zP089033ur6tc11adunXvuuaKqGIZhGKlLtUQLMAzDMGKLGXrDMIwUxwy9YRhGimOG3jAMI8UxQ28YhpHimKE3DMNIcczQG0YpiMhdIvKi+z5NRFREaviw3wtEJKMc7d8XkafKqFcROb6UuitE5NOK6DRSBzP0hhEGEakFPAY8H0XbsSLyiOdzK9f4his7shwalorICeXV7kVVRwIdROTUyuzHSG7M0BspiQ8976uAJaq6IYq2k4HzPJ/PA5aEKVuuqpuiObiItAWqq+qyKPWWxRCgtw/7MZIUM/RG3BGRliLyuYhsFZHVInK/W95PRIaKyIcisldEFopI50jbebYdJiL/E5E9wO9EpK6IfCAiO0VksYg8HHKZiMhDIvJ5MV0vich/3Y89ge/L+A7XiMgaEemIY+i7iUjo9/RL4EWgc7GyycX28VcR2SIiG0Xk9mKH6AWM9nxuLCKj3PMy3b0ReLlYRJaLyC4ReVVExFM3yd2fUUUxQ2/EFdfwjQTmAa2A7sCDInKZ2+RK4BPgcGAE8EqU24HTCx/mbvsR0BdIA44DLgFu9rT9H9BDRA53918DuB740K0/BVhayne4HfgncLGqLgBmALWB09wm5wHjgRXFyryG/kigkftd7gBeFZHGnvrLgVGez9cDTwKN3f0+XUzWr4CzgFOB6wDveVkMpInIYeG+j5H6mKE34s1ZQHNV7a+qB1V1FfA2jiED+FFVR6tqPjCYQ4Yy0nYAU1X1K1UtUNVsHIP3jKruVNUM4KVQQ1XdiGN4r3WLegDbVHWW+/lwYG8Y/Q8CDwEXqOoKd18HgOnAeSLSBGjk6vvBU3YyRZ8QcoH+qpqrqqOBfcCJACJSz/2+kzztv1TVGaqah3MT61RM10BV3aWq64CJxepD3+PwMN/HqAJUOoLAMMrJsUBLEdnlKauOYxTXAl4f9n6gjtvbLmu7EOuLHatlsbLi9R8Af8S5YdyMc2MJsRNoGEb/QzgGunjUTMhPvwb4yS37EbjdLVuvqms97be7RjvEfqCB+747MMW9gYQofl4aUJSy6kPfw3vujCqE9eiNeLMeWK2qh3teDVX1ch+2K56KdSPQ2vP56GL1XwGnun72X+H0lEPMB8JFvFwKPCYi1xQrn4zjhz+PQzefn4BulHTbROJyivrnK0t7YI2q7vFxn0YSYYbeiDczgL0i8og7WFpdRDqKyFkx2G4o8KiINBaRVsC93kpVzcHx6X8MzHDdHiFGA+eH2edCHDfPqyJypad8Ko5r5GZcQ6+qO4Gtbll5DH1PivrnK8v5wDc+7s9IMszQG3HF9b3/CseHvBrYBryDMzDp93b9gQy3/QQco36gWJsPcAZeBxcrHwmcJCItw2iZ52p5W0R6umVZwCygFrDA0/wHoAVRGnr36WJfsZtOZbkBeNPH/RlJhtjCI0ZVQUT+CFyvqud7yo7BiXk/srhrQ0R6Ayer6oNx1Pgw0ExVH/Zpf1cAt6jqdX7sz0hOzNAbKYuIHIUTWjkVaIfjDnlFVUNpDaoBLwCHqervEybUg4hcB/ysqosTrcVIHczQGymLiByLY9zb4EScfAI8qqoHRaQ+sBkn0qeHqhaPyDGMlMEMvWEYRopjg7GGYRgpTiAnTDVr1kzT0tISLcMwDCNpmDVr1jZVbR6uLpCGPi0tjfT09ETLMAzDSBpEZG1pdea6MQzDSHHM0BuGYaQ4EQ29iBwtIhNFZJGbH/yBMG3EzeW9QkTmi8gZnrrb3DzZy0XkNr+/gGEYhlE20fjo84C/qupsEWkIzBKR8aq6yNOmJ86ElHbA2cDrwNlueta+QGechFOzRGSEmwPEMAzDN3Jzc8nIyCAnJyfRUmJKnTp1aN26NTVr1ox6m4iG3s3bvdF9v1dEFuMsluA19FcBH6oTlD9NRA53ZyVeAIxX1R0AIjIeJyHUkKgVGoZhREFGRgYNGzYkLS2NogtspQ6qyvbt28nIyKBNmzZRb1cuH72IpAGn4yyy4KUVRXN9Z7hlpZUbhmH4Sk5ODk2bNk1ZIw8gIjRt2rTcTy1RG3oRaQB8DjwYi7zWItJbRNJFJH3r1q1+7x6A9DU7yC+wmcCGkaqkspEPUZHvGJWhF5GaOEb+I1X9IkyTDRRd1KG1W1ZaeQlU9S1V7ayqnZs3DxvzXymmrdrOb96Yyhvfr/R934ZhGEEmmqgbAd4FFqvqC6U0GwHc6kbfnAPsdn37Y4FL3YUfGuOszjPWJ+3lYuPubACWbw63DKhhGEbl2LVrF6+99lq5t7v88svZtSu2qzxG06PvBtwCXCQic93X5SJyt4jc7bYZDazCWZ3+beBPAO4g7ABgpvvqHxqYNQzDSCVKM/R5eXlhWh9i9OjRHH54bNdtjybq5kegTKeQG21zTyl1g4BBFVJnGIaRJPTp04eVK1fSqVMnatasSZ06dWjcuDFLlixh2bJl/PrXv2b9+vXk5OTwwAMP0Lt3b+BQypd9+/bRs2dPzj33XKZMmUKrVq0YPnw4devWrbS2QOa6MQzDqAxPjlzIokx/Y0ZObnkYfa/oUGr9wIEDWbBgAXPnzmXSpEn06tWLBQsWFIZBDho0iCZNmpCdnc1ZZ53FNddcQ9OmTYvsY/ny5QwZMoS3336b6667js8//5ybb7650toDaejzLDLGMIwkp0uXLkVi3V966SW+/PJLANavX8/y5ctLGPo2bdrQqVMnAM4880zWrFnji5ZAGvrFG32P3jQMowpRVs87XtSvX7/w/aRJk5gwYQJTp06lXr16XHDBBWFj4WvXrl34vnr16mRnZ/uixZKaGYZh+EDDhg3Zuzd8VN/u3btp3Lgx9erVY8mSJUybNi2u2gLZo48FtmKiYRixpGnTpnTr1o2OHTtSt25djjjiiMK6Hj168MYbb9C+fXtOPPFEzjnnnLhqqzKGPkRVmDlnGEZi+Pjjj8OW165dm2+++SZsXcgP36xZMxYsWFBY/re//c03XVXGdfOXofMSLcEwDCMhVBlDbxiGUVVJSUN/5oDxXPqf7xMtwzAMIxCkpI9+e9ZBtmcdTLQMwzCMQJCSPXrDMAzjEGboDcMwUpwqZ+gtuNIwjFhQ0TTFAC+++CL79+/3WdEhqpyhj5alm/bSqf84tuxJ7YWGDcPwhyAb+oiDsSIyCPgVsEVVO4apfwi4ybO/9kBzVd0hImuAvUA+kKeqnf0SXmGi7NK/99Nqdu3P5dslW7ihyzGx1WQYRtLjTVN8ySWX0KJFC4YOHcqBAwe4+uqrefLJJ8nKyuK6664jIyOD/Px8Hn/8cTZv3kxmZiYXXnghzZo1Y+LEib5riybq5n3gFeDDcJWq+jzwPICIXAH8udjiIheq6rZK6jQMw4ieb/rApp/93eeRp0DPgaVWe9MUjxs3jmHDhjFjxgxUlSuvvJLJkyezdetWWrZsyahRowAnB06jRo144YUXmDhxIs2aNfNXs0tE142qTgaiXRXqBmBIpRQZhmEkOePGjWPcuHGcfvrpnHHGGSxZsoTly5dzyimnMH78eB555BF++OEHGjVqFBc9vsXRi0g9oAdwr6dYgXEiosCbqvpWGdv3BnoD1DryeL9kGYZRFSmj5x0PVJVHH32Uu+66q0Td7NmzGT16NI899hjdu3fniSeeiLkePwdjrwB+Kua2OVdVzwB6AveIyHmlbayqb6lq50D48T1Y1kvDMKLBm6b4sssuY9CgQezbtw+ADRs2sGXLFjIzM6lXrx4333wzDz30ELNnzy6xbSzwc2bs9RRz26jqBvfvFhH5EugCTPbxmDHDklwahlEevGmKe/bsyY033kjXrl0BaNCgAf/73/9YsWIFDz30ENWqVaNmzZq8/vrrAPTu3ZsePXrQsmXLhA3GRkREGgHnAzd7yuoD1VR1r/v+UqC/H8czDMMIIsXTFD/wwANFPrdt25bLLrusxHb33Xcf9913X8x0RRNeOQS4AGgmIhlAX6AmgKq+4Ta7GhinqlmeTY8AvnTzv9cAPlbVMf5JNwzDMKIhoqFX1RuiaPM+Thimt2wVcFpFhcUKsbmxhmFUMWxmrGEYKYNWgeiJinxHM/QRUFL/wjGMVKBOnTps3749pY29qrJ9+3bq1KlTru0Cm4/+hrem0b19C+785XGV2s/Q9PX8sNwm5hpGqtO6dWsyMjLYunVroqXElDp16tC6detybRNYQz911XamrtpeKUM/d/0uHh42v1I6zKdvGMlBzZo1adOmTaJlBJKkdt20/fto0vqMYuveA2HrLfOkYRhGkhv6/ALHF/fh1DVRb2MToQzDqGoE1nWTKLbtO8DenLzCzzYYaxhGspPUPXq/2JuTS4H7dHDOM99y4b8mESlx/ZptWSk9um8YRupQ5Qx9cfO9M+sgp/Qbx4vfLgcgryCy8Z61dgcX/GsSH01fFwOFhmEY/pIShr4ybvftWc5A7qj5mVFvs3Krk+lh7vpdlTiyYRhGfEhKH/3a7Vks37wvYrvS+uZ7cnLZuCuHE49sWFi2abdF6BiGkZokpaE///lJldr++jensWjjHtYM7FVYlnUwP2xbc8MbhpHspITrprws2rgnYhsLwzQMI1VIDUNvVtkwDKNUIhp6ERkkIltEZEEp9ReIyG4Rmeu+nvDU9RCRpSKyQkT6+Ck81kTjsjG3jmEYyUA0Pfr3cRb9LosfVLWT++oPICLVgVdx1os9GbhBRE6ujNjYcOhpIFzKhHAPC/b8YBhGMhHR0KvqZGBHpHZh6AKsUNVVqnoQ+AS4qgL7iUh5DG9ZXp5wHXTrtRuGkez45aPvKiLzROQbEenglrUC1nvaZLhlYRGR3iKSLiLpPmkKy4G8gqjamdvfMIxUwQ9DPxs4VlVPA14GvqrITlT1LVXtrKqdfdDk7rNk2fC50U+MMgzDSAUqbehVdY+q7nPfjwZqikgzYANwtKdpa7esSjNr7U7+OnSe5ckxDCNuVNrQi8iRIo6jQ0S6uPvcDswE2olIGxGpBVwPjKjs8cKxMHMPd36QTm5+dG4ZL14XjV/emkWZe3jgkzmFaZS93PrudD6fnVHqBC3DMAy/iTgzVkSGABcAzUQkA+gL1ARQ1TeA3wB/FJE8IBu4Xp3uap6I3AuMBaoDg1R1YSy+xITFmwFYvS2LE45oGKF1+Sir311aCuN7Pp7N6m1ZPNC9Hcc1b+CrHsMwjPIS0dCr6g0R6l8BXimlbjQwumLSEktZvXuxkVrDMJKI1JgZaxiGYZRKUiY1K40rX/mRS04+suI78KmjbgOthmEEiaTo0X8+K4O0PqPIyS17ADMnt4CR87zhk5ENrhR574+lDx21LBeP3QwMw4gXSWHo/zNhGQBb9x5IsJLyEc7Mm3/fMIx4kxSGPpHk5RewYVc2Q9PXl3yqKKVTbp11wzCCREr56GPBkyMX8eTIRdSvVR2AXftzo3bwWOfdMIwgUKV79AfzCrjzw/CpdXZn5xb5vD/C+EC0mG/eMIx4k1SG/oFP5vi6v8Ub97DKXegbivbAv56/sdTtIpnq0iZSGYZhJIKkMPQhAzx73a7YHieKNk8MX8COrANlbhDqtIeL4qnqg7H7D+bx16Hz2Jl1MNFSDKPKUKV99Nuzyh/FM27RZsYt2hxV2ypu08PyyYz1fD47g4Z1atDvyg6RNzAMo9IkRY++okRyh//+/Uqmvo/SQ3PN61O48e1plTtWimBOLcOIP1W6R18ehOiNVPEbzKy1O/2WYxiGETUp3aMvr+ukzJmscTh+WSzfvJehM9dHbhgnsg/ms277/kTLMAwjCgJv6Bdv3ONbaoLK4HdUZHl3d8l/JvPw5/OLlM1et5O0PqOYsboiS/pWjt6D0znv+Ynl3i4UXmrjF4YRPyIaehEZJCJbRGRBKfU3ich8EflZRKaIyGmeujVu+dyKrgXb878/FPm8fPPeiuwmrpQWK787O/rJVtHw4/JtAPywfKuPe42OH9xjG4YRfKLp0b8P9CijfjVwvqqeAgwA3ipWf6GqdvJrLdhL/jPZj93EheKuoFvfnZ4gJYZhVGUiGnpVnQyU6htQ1SmqGhptnIazNmxSUt7edmnul9LKl23eR7ZPM2yLk7krO7Czbm94axpvT15VpCwI7jjDqCr47aO/A/jG81mBcSIyS0R6l7WhiPQWkfSKunjCUV6757ffuPjusnPzyXPXkfXzUHPW7eIXA7/j0wAN1nqZumo7T49enGgZhlFl8c3Qi8iFOIb+EU/xuap6BtATuEdEzitte1V9S1U7h3PxVGTR70QSzQ3Gj7536GaxfIszbjFzjYVxGoZREl8MvYicCrwDXKWq20PlqrrB/bsF+BLoUpH9b9yd44fMiPjt+Yh3ZMnnszPie8BKMOin1YmWYBhVhkobehE5BvgCuEVVl3nK64tIw9B74FIgbOROULjZBksNw0hBogmvHAJMBU4UkQwRuUNE7haRu90mTwBNgdeKhVEeAfwoIvOAGcAoVR0Tg+9QKuUd+FyYuceX44ayV5Y14HjruzPYvKdyTyrJGIse0PFiw0hpIqZAUNUbItTfCdwZpnwVcFrJLeJHrCJcIlGYvbIMQzx3/S5en7TSEnsZhhFzLNdNJYgUzpi5K5v8gth1YZNsjNowjASR0ob+mVGJCenb4i5ifvv7M9m1PzdC64qx/2Be4aLphmEYZRH4XDeVIetgYlw3Ifbm5MVs33uyD+07SH7vL+dkMHlZ6SkZbPUtw4g/KW3oU5GTnxhDfoGyauu+REsJy58/ncetg2YkWoZhGB7M0MeQSP75ikTN7D+YT05uPrPXxW5y1PhFm7ktSmMd1LQLsSS/QNl3IHZPa4bhN2bofeL378/kvOfKn7a3ongTpoXGBADmFLsBZOzcz8LM3eXa9x8+TOf7ZVtp/7j/0bC5+cl/Y+jz+Xw69h1bJW9yRnJiht4nvluyhXU7Er8Qx9WvTSny+dx/TqTXSz9WaF/lDU8d9GPZs1237TvA82OXVkiLX2zanVNpt9dns5JnBrJhQIpH3aQqQZ0o1f/rRWXW3/mBb/nqKsw5z34LwJqBvRKsxDDihxn6JKB4FIsgURn7nARNGAuR1mcUN519TOHnTXHKWRQvVIN70zUML+a6CTjfLdkcNoolmnzuJ8XAx15ePpq+rvD9pkqmfDAMo2KYofeBX738Q+RGFWTzngMlykSsJxkEbCjWSBbM0PvAgg0VS4ZWvFeeuSubXz73HRt2ZVdKz+Nf+ZckNJZhhLPW7mRo+nqLXjGMGGOGvhL4bZ4+nbme9TuyGRrFSlFldegHT1vrm6Zh6WVrKc1Gb91b8kmkOA8Pm8fDw+YzOUkXGrcblAGwYVd24BdHisrQi8ggEdkiImG7iuLwkoisEJH5InKGp+42EVnuvm7zS3gqMOin1aT1GcUTw0vvgYcz6CKwO7vsHDo9XkzsIuqPfD4/YpvQzSAriScffTpzHQ98MifRMowEsTs7l24Dv/P1KToWRNujfx/oUUZ9T6Cd++oNvA4gIk2AvsDZOKtL9RWRxhUVm6p8ONXpgXv7hzuzDrInJ7wxF4TXJq0sc59LNu31RVvxPuvqbVmk9RlVpOzFCcu49+PZRcqiifgJTfoS4MOpa0jrM4qCGGb79IOpKwsXUEOBRz7/meFzMxMnyEgooU7KpKWl53cKAlEZelWdDOwoo8lVwIfqMA04XESOAi4DxqvqDlXdCYyn7BuGgdNjP33AeDoPmMCBvJKPhBUZiL3Fp9WzxizYVKLsxQnL+Xr+xnLvK+T6ePm7FTwxfCEAuQXxeQSes24nCzN3l9v9MnzuhsL35rkxkiUowi8ffSvA68zNcMtKKzfCUcxyHMwvoO+Ihb7s+ocA+8EXbfRnZa/y8NyYpfR66ceEzXL955gl/PnTuQk5tuE/Qc/KGpjBWBHpLSLpnqUIqxyhSyWaGPlUQMJ0hyYu2cq9H8+Oehbtgbx87h48i9XbsiK2XeRZKnLVNicNwrIoXVxDZqzjs2ID05X5cb8+aSVfztkQuaERaEK/1aif7goK4LunISu+HS+/DP0G4GjP59ZuWWnlJVDVt1S1s6p29klT0hFK+LVlb9kTiyp6Gzi139hyJzjzMmXFNv45ZkmFt4+Gu/83i6/nb2TC4s3MWF2Wt9Bh+qodjFm4qcwB7RCXv3RovkO4+Qll8egXP/PQsPlJ86huBJS1P8Lk52DEfXE9rF+GfgRwqxt9cw6wW1U3AmOBS0WksTsIe6lblhL47aOdtHQLUHQ2aTjyK3jgPTl5RRKc7cnJZfu+6Azelr053PhOST//cX8fXSEtEDly6IvZkd0qIcMb6ZTs9mmlrxVbDiVEK37M7fsOsCPrYOHnjbuz6frst6zbXnayu0e/mM+b3zuD69kH89m4u3LzKAx/2JOT638akXz3OsyNbwLEaMMrhwBTgRNFJENE7hCRu0XkbrfJaGAVsAJ4G/gTgKruAAYAM91Xf7fMCEO0kTKdB0zw5Xhdn/mWM58qe18hY/Z9OaMKpniiUypKNPez0KNzQRmN127P4rT+4yqtB2BNKUZ72KwMznxqAmcMGF9Y9sXsDWzcncN5z0/knR9WlbrPITPW8+w3zpPSTe9Mo+uz3/mi1agcp/YbV+QpMByFHY2o93rIQUtBPiwaHpdR/Wijbm5Q1aNUtaaqtlbVd1X1DVV9w61XVb1HVduq6imqmu7ZdpCqHu++3ovVF6lK7K1k3Pn6Hfu5a3B64VKL3y3ZHHGbWK19WxbhjHdBgfLVnA3kuRNUqnl69Ms376Vj37FkFptZXJpx9pO/fTavzPpo0zPPXrer8P3v3pvBs98kZt1jw2HV1tLHfsYu3MRHxScn/vgf2PRz5B2LwE//haG3wqKvYNk4eK4t5LrXboG/TxKBGYyt6sxbvytyI594cuRCxi48ZNx//346GTvDG8MdWQe5a3A6T4+Ov8EJF1L/xZwNPPjpXN4J5b6XUFvlf9PWsu9AHuMWFg0BnbWmcg+R3lmP3hm/5emIeX37eVHMoly+eS+Tlm7lze9XsXlPDr949tuoBpyN+HHX4Fm89N0KwL0WVnwLE/rBG+eWvpH3mtnjDldmbYPxj8P+bbBzDezeAP2bwOwPfdNqhj4gXPXqT3E71oTFW0qUrdiyj4HfLCkRV/7KxBVFbgrxJFyM+44sx9Bucw1uNdeCKocGWItvFfoxlkV+gfLW5JVhfbK3vzcz7DZ5FYz5XxvFAjX7PQvbj5yXSebuHAZP9S+1RTJz/5A5XPSvSYmWUQyFrFLcm6rw1gWOm8brugkx+m9F229f7vz9+TPf1JmhNwD400ezeeP7lSze6M+MWj8oy+8eIvRzUVXGeHryfxk6l5e/XR5x+3d+XM1Xczbw1ZwNPDN6Cf+ZsKxI/ay1O/lxRfhQuMfKMe29MiGzT40y942XEfMyWRXHp5t/j1vK/oOR3KWl/H8n/wv+dw1kzoHP7/Q0F9jqced5r/UY+Oxt4ZFKEO7fEfQp/KUR6kEGaeKH91TmFyjjFxV9spiychs3vu1EAhX/nXwx23ksvq97u4jHedAzcWlfzqEf9MSlW0rtzQN8E2aWsJdoQjG96SS2RYiA2pF1gGWb93LCEQ0j79jwjZe/W0FegfJIj5NKbdO9YAqsDBMl9t2Aop+9F+oaz0DvtpDRF1j6jfP2wF7o1wiufBnOuBUy0qFuY2ja9tB2ezc7YwLtLi7zO5ih95n/TbfHa78YMS+Tbsc3BWDn/lwGfrOEIw6rDTghpq9OPOSS8fb+K3OrUpywz4HfLCataf0y2x4Mk56iNKIx+t6cOeG+w1dzM/lqbibLnupJrRr2MO43I+dl0unowzm6Sb0SdQdyi/6vPyr2O/9nwb8hch4/wrpuijPjTefvRneHI+6DRq1h8NXO5367Yf0MaNwGBl3q+PX7lT0/xq4Wn8nYmdwx0EHL3/LI5z/zyOc/s8E9ryE//Hs/reGnFUUTjIXj3+PKvxj5axNXMGTGel/TPQPMXLODtyevKvUcD4iw5m6If49P7ALrqcp9Q+Zw5Ss/llr/7eLNpPUZxZY9OfzjywhuuymvwL6SY2F86/bwS7vz53nsh3rGi0JGHmDWB/DuJfD2hY6RjwLr0VeCcP+qtyaXHi9tVJxILiXverTeDJMvRzEQ60U4dNPw86YnwLVvTAXgwpNaRGw/tYx5CJm7bElGPxmzYCNz1zs94p2lhBErTlQXwIJis8ubs7PkBuP+AcvCLOW52Q29XF7KvI63LogseOT9zt/dkdetCGE9+kow+ufyZ2w0KkYko7vRY+gnLK54lNBH09cVRt5UNKomEsXHGsJRVqoJW/DEX+7+nxOIUBZl3Xhn1rknfMWBOCbr++z2MqvN0FeCvCQdeC2Ln0qJMEk08TzTofUBypsPJ8QL45eR1mdUkdQH3gRusc4XVFUYGmH1Mz/xa32HmLHwizKrzdAbRQhNxQ8aydSLfckN6/xuSRgfrQ+Ey/pZFXl42Py45gVaX97xt+IZKhN4DZuhN5KCJLLzhWzxzKL1c5H1ZLrpxZrcvMqdi9JmhIfDm9AuKvYUS9RbEP80IiHM0BtJgdm2omQfzOf1SSvJT0H3YTz500ezS5RFs7B9smFRN0ZS8Gkc/bFB5+v5GwuXbmzRsDbXnNk6wYqSg6wDeXw8fR13nNuGam42vHBzIR74ZA5r45AIL56YoTeMJCYnz+d86SlEfoHS9u+jefLKDlxzZms69nWWwji6SV16dDyq1O0ipdieHsWCOEEj2nz0PURkqYisEJE+Yer/IyJz3dcyEdnlqcv31I3wU7xhVHWquktLUdbv2E9an1GMKrZAfSjr6DOjF3PB8xMLyxdv3EvXZ79l8wu4TnMAAB+ASURBVJ6KzUd48/vkmysTsUcvItWBV4FLcBb3nikiI1S1cBqfqv7Z0/4+4HTPLrJVtZN/kg3DMA6x0F0LePjcDfQ6NXxPfdu+Qyt//deNiip+Y0hlounRdwFWqOoqVT0IfAJcVUb7G4AhfogzDKNsHvtqQWEUzsqt+6pcRM7a7fu552NnQLW0b36glJxEizfuIStiVsrUIBpD3wrwjoRluGUlEJFjgTaAdy20OiKSLiLTROTXpR1ERHq77dJLa2MYRkmyc/OZtXYH3f/9feFkr6rC48MXlIg8Gj53A3OjWMjns1kZrN+R3LmposXv8MrrgWGq3mw8HKuqnYEbgRdFpG24DVX1LVXt7LY1DCNKBClc8q7viIUJVhNfvNExoWlkD3wyl1+/+lNU6xlUFaIx9BuAoz2fW7tl4bieYm4bVd3g/l0FTKKo/94wjEoycl5mEaMWTW82FRm3aDNPeTKAnvzE2ASqCRbRGPqZQDsRaSMitXCMeYnoGRE5CWgMTPWUNRaR2u77ZkA3ILpcrIZhRMXDn88v4rIJtxxiVaFwLWGjCBGjblQ1T0TuBcYC1YFBqrpQRPoD6aoaMvrXA59o0dGg9sCbIlKAc1MZ6I3WMQzDH6Kdzfn4VwsY/fNGjmten8/u/kWMVfnPN5YxtkJENWFKVUcDo4uVPVHsc78w200BTqmEPsMwfCS0mMr2rIMRWgaTP4ZJWWBExmbGVpIR8zIjNzKMGLMlBvlZ9h3Io36t6oHIlnnfkDlc3D7ygi1GeCypWSW5f8icREswjCL4YZY37c6hY9+xvP1DMGaBjpyXyQOfzI3c0AiLGXrDqMIcyMsvTBXgZcMuJ2zxmwWb4i3JiAHmujGMKkDXZ7/l2Kb1SpSf+NgYjm5Slx8evqhIeSikIvFOG9iRpOMJQcIMvWGkGL99axoAsx67mKYNagPOmrredXUB+rmTq8LNDg2FzgXBP//kyKo1CSwWmOvGMFKUBZl7KChQhs4Mn8v//SlrIu6jomb+xQnLmL1uZwW3Lkq4nPFG+TBDbxgpysQlWzju76N5+PP55d62tOwBW/ceYMiMdRG3f3HCcv7vtSnlPm44Zq3154ZRlTFDbxgpSjQ99hDFJ1yF5j0W99zcNTidR7/4uVxrrUbDnpxc/jV2KXlhBoZjETpa1TBDbxgGQ4st1Viajz400erViSt58/uVEfc7cckW7oliktNfPp3LKxNX8NSoxdEJNsqFGXrDMFi5dR8/Lt8GwLiFmxi3cDMAM1bv4G+fzStsFzL7Q2as49lvlkTc7+3vz2RUGWkL9h3IY9CPq5mweAvgPIWE69UblcMMvWEYfDF7Aze/O538AqX34FkM+ulQcrBhszJI6zMKKNnDf/SL+dw1OJ2CgoqlBB4wchH9vy6a/uqvn80j60Aed36QzsbdVSNffKyx8ErDMAopaybsmAUbS0ThDJnhuHw69htLn54ncUOXY1i9LSvq4+3KLhkjP3xuJt2Ob8aExZuZtTb5FuIOImboDcMo5JMyImpWbs0qNd5y/8F8nhi+kP0H8xkYxqXz5ZwMWjeux1lpTQDHZTN91Xa+W7Il7P5CTwg79+eW8xsY4TBDbxhGIdll5LIfMmMdtWuU7e3dlxN+DdY/f+r4+dcM7AXAw8PmMfrn0tMr9Pni50hSjXIQlY9eRHqIyFIRWSEifcLU/05EtorIXPd1p6fuNhFZ7r5u81O8YRj+kptfuq89Y2e206svg2rVyp5iFRpoLcvIG/4TsUcvItWBV4FLcBYGnykiI8IsIPKpqt5bbNsmQF+gM07E1ix3W5sBYRgBpLJ5ZV76dnmZ9SPmZbJ5j8XFx5toXDddgBXumq+IyCfAVUS3JOBlwHhV3eFuOx7oQbF1ZQ3DqBr8Zei8yI0M34nGddMK8M6myHDLinONiMwXkWEiElpMPNptEZHeIpIuIulRaDIMwzCixK84+pFAmqqeCowHPijvDlT1LVXtrKqdfdJkGIZhEJ2h3wAc7fnc2i0rRFW3q2rI8fYOcGa02xqGYRixJRpDPxNoJyJtRKQWcD0wwttARI7yfLwSCCWsGAtcKiKNRaQxcKlbZhiGYcSJiIOxqponIvfiGOjqwCBVXSgi/YF0VR0B3C8iVwJ5wA7gd+62O0RkAM7NAqB/aGDWMAzDiA+ipSWeTiC1j2qnR932YqJlGIZhRGRNnRsTLQEAeXLPrNLGOC2pWQpxWB2b6GwYRknM0Cc5L99wOv+4vD0AXdo0TbAawzCCiHUBk5wrTmuJqnLHuW14aFj5l4wzDCP1sR59CiAiVKsmJZZ9Cwq3dj020RIMo0pjhj6F6NnxyIht+vQ8qcjnf117WtT7v797O/57fSeuOK1l1Nt8dOfZ9L+qIycd2TDqbQzD8Bcz9EnIZ3d3pUXD2iXKu7c/gjUDezHi3m4A3Nb1WGb8vTsXt29R2KaGJ7vgm7ecyW/ObF3qcV64ruhN4MHu7biqUysuPfmIsO2bNahF+mMX85/fHtqu2/HNAHjiipOj+GaGYcQCM/RJyFlpTZj6aHeWDOgRtv7U1oezZmAvnryqIy0Oq8M7t51VWHfmsY0BeP/2s7isQ8kngNXPXs7lpzjltTy5xzu2OqwwBa3XRfTRnWcXvv/xkYto1qA2V58e5uYRvChew6gy2GBsknHj2ccAUL2aUL1a9ai3m/vEJeTkFnBkozosfaoHtWsc2nbNwF58v2wrrQ6vg4gg7jJCqrCo/2XUqFatiNE//RjnZvHe786i2/HNeP43p/LM6MXUrH6oTc3qwm1d0wo/H16vVoW+r2EYlccmTCUZ3/71fNo2bxDTY6zZlsXDw+Yz6PazaFDbv77AP8cs4fVJK33bn2EEAZswZSQlac3qM/Turr4aeYBHepwUuZFhGL5jht4wDCPFMUOfZDSrXzLaJpm458K2iZZgGFUOM/RJxJIBPWhUr2aiZVSKhy47iQ9/3yXRMgyjShGVoReRHiKyVERWiEifMPV/EZFF7lKC34rIsZ66fBGZ675GFN/WiJ46NaOPsgky553QnIVPXsYlpcTjG4bhLxENvYhUB14FegInAzeISPHZL3OAzu5SgsOA5zx12arayX1d6ZNuI8mpX7sGb9+auFUjxzz4y4Qd2zDiTTQ9+i7AClVdpaoHgU+Aq7wNVHWiqu53P07DWTLQcFnU/zLWDOzFgF93TLQUw6VR3Yq7wEKTzgwjWYjG0LcC1ns+Z7hlpXEH8I3ncx0RSReRaSLy6wpoTHrq1XLCFI9pUi/BSoLH0qcOze6d3+9SXrvpjLgct6AS00eevrojZ7dpErHdDV0OLZfcJa0JXY8rO410m2b1S5T1veJkftHW0k8blcPXwVgRuRnoDDzvKT7WDeK/EXhRRMKGXYhIb/eGkO6npiBxrpv3JRxXdYo+UVgqUbtGde7v3o7h93TjsDo1ufyUoyJvVEHuPLdN4fum9Ss2U/fPF5/AiUc05L3bzyrMynlx+yOY/fglJdo+eWVHJvzlfL6+71yG3t2VIb3PYcY/ugMw5A/nUKdm0Z/f3911BUK8fMPp3N6tDR//4RzWDOzFhSc2B+AC968RDO45eH/FNmwev3klEWfGikhXoJ+qXuZ+fhRAVZ8t1u5i4GXgfFXdUsq+3ge+VtVhZR0zVWbGXtbhCM45rim3dztkYDo/NYFt+w4Uabfi6Z6MXrCJ+4fMKXN/awb2ionOoFFQoMzL2MXVr03xbZ8f3Xk23Y5vxqLMPbQ4rDbNGjhhqjm5+Zz0+BgAOrQ8jIWZe0rdx6j7z6VDy0aFn/PyC/hyzgauOaM11aoJ63fsZ/W2LF76djn9ruxAx1aNSt0XgKrS5tHRAMz4R3eaN6jN4GlrSWtan7PSmlC3VtHB95zcfDJ27uf4Fg1Zuz2L85+fxFu3nEnvwbMqdE6M8jHzHxdz1tMTipS9euMZjJyXyX9rvkztJV+Wb4fn/AnSB0FejvP57Lth+hvQuA3sXO2UnXIt/PyZ877XCzDjbajdADJmlthdWTNjozH0NYBlQHdgA85C3zeq6kJPm9NxBmF7qOpyT3ljYL+qHhCRZsBU4CpVXVTWMZPd0Pe/qgPXnNGa+mFmlubmF9DuH45n66UbTudgXgG/ObM1o+Zv5J6PZ5e536pi6EOk9RkVtnzArzvy+FcLot7PuD+fxwlHlJ4mecqKbdz4znT69DyJkfMyWZi5hy5pTZixpug69qufvRwJYNL/zF3ZXPvGVDbsyk60lJRmzcBeJa7JwXd04Zft3CesjHSo19TJ+rdvK7x7MbTt7hjwFRPg4n6waDh8/aBj3P+eCbXqQ34e5GZBrQZwcB/UaQQTn4WFX8C9JQ06c4fAV3c77zv8H7S9CEbcW6ahjzjHXVXzROReYCxQHRikqgtFpD+QrqojcFw1DYDP3B/COjfCpj3wpogU4LiJBkYy8snKcc3rc+VpLXnvpzXc6knmVZxQ4q9GdWtypSeve73aTu/tt52PZu76XSzdvLew7pmrT6FV47qxER5gnv/NqSVWzZr80IUc07Qee7JzeX7sUgBuOvsYPpq+rrDNRSe14LslzkNl3ytOLtPIA/zi+GYMu7srZxzTmN93a8OExZvp2fFIRIQflm/llndnMOzuroE08gAtD6/LT30uou/wBXwwdW2i5SQ1t5zjuOMGT3PO4/knNOf7ZVtJa+qMr13cvgUTFh9yWISeDAFo7bGxhx8L3fvC6bdAg+ZwwqVOeacboN2lkLPLMfIA1WtAdffpr47798JHnVc4TrkWVk+GLn+AVmdAQT5kzgZK7xxbUjOfOO3owxl+T7eo2q7cuo/G9WrRxOMnVlXen7KGazsfTX6+clr/cYV1Va0nX5ydWQf5cs4Gbu+WVsLYbtt3gGYNajNx6RaGpWfwh/OO46QjG/Ldki386aPZviSBy8nNT4o5DAszd9PrpR8BZy2CD6auZfAdXbjl3RkA9LviZPqNjL6fdcIRDVi2eR/v/e4sbn8/TM8yAu1aNGDYH3/Bgg27uemd6UXqXrnxdFo3rkfmrmz+9NFsPvh9F45rVp//TFhGvVrVefLKjrT9++gi2zx3zals2pPDC+OXFSnv0qYJObn5zM/YHVbHqzeeEfFpefg93Vi7Y39h5+vf45ZywYktOLpxXbo88y33XXQ8f730RPbm5PLlnA20a9GQWjWEM4+NPCgfL0Sk4q6bRJCMhv6kIxsy5sHzfNtf6BFx6F1d6RJFhIdhgHPdtD/qMEbffy4F6qSzXrBhNz8s38YfL2jLgK8XkbFzP2MXbgacdNIDrurI3pw8FmTuZvjcTACWP92T/ALlYH4Bh9WpyeRlW7l10IyIx/+k9zm0P+qwsOGroWu6R4cjeeOWMwvLc/MLiqS4DvHCuKW89N0Krj2zNX+59ASOalSX/AJlyaY9dGjZqHB/K5+5nAJV0tfs5LVJK/hh+bbCfTx9dUduOvtYvpidwV+GzmPcn8+jeYPanD5gPOCEPucVKIfVKT3cdvu+AzSuV6twPYagYoY+Dpx81GGMfsC/STj3D5nDiHmZfH3fuREH9QwjRG5+AdVEqF5Bo5RfoOTk5ocdX8o+mM/G3dlc9O/vOfPYxtx70fF0a9uMGtWEzXtzaFinZpkZT/fm5PL0qMU80uMkGkcR9bTvQB7Pjl7MP3q1LwxR9tLn8/m0bd6AP5x3XJHycQs30XvwLIbf043Tjj487L7HLNhEbn5BuZbFDDpm6ONAx1aH8fV9/hn6fQfy+G7JliJ+fMMwjNIoy9BbUjOfqObzQF2D2jXMyBuG4Qtm6H0iqBEZhmEYZuh9ombAB2oMw6i6mKH3iRrVzdAbhhFMzND7RI1qdioNwwgmZp18wu+FtA3DMPzCDL1P/KNX+8iNDMMwEoAZep842nLNG4YRUMzQG4ZhpDhm6A3DMFIcM/Q+YGuIGoYRZKIy9CLSQ0SWisgKEekTpr62iHzq1k8XkTRP3aNu+VIRucw/6f5wcfsWzPh7d/57facK7+P1m+OzzqlhGEZFiBgTKCLVgVeBS3AWBp8pIiOKLSByB7BTVY8XkeuBfwK/FZGTgeuBDkBLYIKInKCq+eUROfbB8/jdezPYuDuH535zKhec0JyFG/ewe38uU1dup26t6vS7sgOvT1rJ5j05bNydze9+0Yajm9SlZaO6jFnoLNP3n992YuLSLWzZc4A7ftmGxRv38KcLjgfgqk6tuKpTK3Zn53Lak+OKHP/jP5zNx9PX0bR+LfYfzOezWRmc1roR8zJ288PDF9KiYZ3yfB3DMIy44suasSIy1m0z1V16cBPQHOjjbettV9YxO3furOnpiVsjPL9Ayc7NLzM2Pje/gNz8grDpUw3DMOJNZbNXtgLWez5nuGVh26hqHrAbaBrltoGjejWJOAGqZvVqZuQNw0gKAjMYKyK9RSRdRNK3bt2aaDmGYRgpQzSGfgNwtOdza7csbBvXddMI2B7ltgCo6luq2llVOzdv3jw69YZhGEZEojH0M4F2ItJGRGrhDK6OKNZmBHCb+/43wHfqOP9HANe7UTltgHZA5IUnDcMwDN+I6GRW1TwRuRcYC1QHBqnqQhHpD6Sr6gjgXWCwiKwAduDcDHDbDQUWAXnAPeWNuDEMwzAqRyDXjE101I1hGEayYWvGGoZhVGEC2aMXka3A2iibNwO2xVBORQmiLtMUPUHUZZqipyrqOlZVw0ayBNLQlwcRSS/tcSWRBFGXaYqeIOoyTdFjuopirhvDMIwUxwy9YRhGipMKhv6tRAsohSDqMk3RE0Rdpil6TJeHpPfRG4ZhGGWTCj16wzAMowzM0BuGYaQ4ZugriIhIojUkC3auosfOVfTYuYoeM/QV53AozNYZGETkRhE5zX0flB9C4RJcAdIEgIgE7TfQAApXdgsEInKliLRNtI4wFJ6jIF1XAbymgm3oReTXIjIg0Tq8iEgjd6WsMVC40ErCEZGLReQH4EXgdABN8Ei7iFwqIlOAV0TkpiBogkLD9ZdE6wghDi1EZBLwDkAQkv+519RUnKSFRyVaTwgR6SUiE4AXROQ8SPx1FbRrqjiBM/TuRV9dRO4E/gX0EZFfJlqXh2xgF9BRRK6FxPW+3HNV180Q+hjwFDAMqJdIXe6xmwP9geeAj3DWEH7UrUvIdSciNUTkEeAl4F8i0klVCxLde3aNVI77OlVEekJizpN7TTUQkZE419RjwDTg2ERpKqYvDXgaeBlYDPR2bUWizlcgr6kSqGogX8AFQEPgD8CkROtxNVUHjgD+DPwK2OSpkwTqusrz/mZgaoLPkwAdgTc9ZSfjpLBulsjzBfwax5X0IDA9kefJo6mae34GAlcl+v/navqt5/29wNBEa3K1dAdecd/Xce3EPKCxWxb36yqI11TxV2B69CJyv4i8Hbo7A9+r6l5VfRuoLyJ3uO3iptmj6fciIuo8Tu8Beqnq18B8EXlCRDqqqsbLT+jR9QcAVR3ullcHVgMLReTosvYRA023icglrh4F9gG/EJEmbtkiYChOTyyeuu4XkYEicp1bNEpVc1T1RaCFiNzotquZAE3XAKhqAZAJnAD8BGwUkbtFpF0CNF3ravrULa8G7ATWi0jteOnx6PqNiJztKcoArhGR2u7/cRIwBXgijpoCd01FJNF3GveO+Ducx8MewPfAo0BbT31PYCHuXTtBmv4OtAVaAE+5bX6Ps6BKuvu5ZoJ0HeepPwVnVbCGcTpPjXHcRRuB+UB1T92HwOBibacDbeKgS3CevH7CWfVssXvuWnjaXA1siOM1VZqmJkBnoK/b7m9AFjDS/VwjAZqae9r8AlgSr/PkHrOFe31nAl8B1YpdVy969J/mXoNHVLVrKtpXUHr03YF/quoY4K84j0E3hSpV9RsO+eMahnodcdZUG7gWx0ffU0TGAfcD33EopXI8BmaL66qF464BQFV/xvH1Xh8HLajqTmAc0B6YRdGe1b1ADxE5y/2chfOYfTAOuhS4EHhMVYfh/EBPBS7ztPkSWCYifwNn8DEBmjoBlwCbgF+KyGjgdhxjssrdNGYDs6VoOg2nIxFqMwXIEJErY6UjjK4twHBXx0bgLk/1k8CvRKSDqz8H2IvzFBlLTYG7pqIl0QMroePPwfF5o6rpwFSglYh08zR/BHgWWA4cmSBNxwHnAuOBGaraSVUvBS4QkTbuhRBvXdNwztW5bjvBWfaxTqxdSZ79f6iqu4DXgP8TkWNdfXtwfpSPi8htOAN7HYjxD9JzrtKBX7paxuBcOx1E5ERP8z8Cz4nIJqBVAjQtxTGsp+O4JWaqagecG/UFItIqVtdVGZqW4Zynk9x2hwFLgNxY6ChD18s4y5COA3qJyFGuxpU4kUCvudf9zThPAAVx0BSYa6o8xNXQFzc86vgmwem9VAuFSgELcO7iLd3tjscxIl8BZ6iqb37ecmhaiPNDbAg8oaqPeTY7RlVX+6WpnLoW4Dzehn4EinPRZ/ltIMJoUvdvjvt3JvANTlREqM0rOCGfZ+JEblyrqrt91lUkwsFzrlYADUXkFPfz90AjnP8hItIJeBv4HOe6+iABmia7erYAd6tqX7f9DqCbqm5IgKbQeWrgttsDtMYJRPCd0nSpaq464ctTcG40D3jaPItj7O8ATgTuUNVsHzU18moLwjVVGeJi6EWki4i8DTwiTthdqDz0D16OY0h/KyLVVTUD56JKc+t3A/eq6v+pamaCNK3HufEcq6oHxQkBrQagqll+aKqgrgycJ5w0z27+pqqD4qCpWpjB8VeA40Wkg4gcISLHq+p3wJ9V9Ta//n/u8TuLyGDgCfFM6JFDk9hm4LjTLhWRGuoMCLfC8YcDbAf+pKrX+nhdlVfTQpwb4OmqmuNeVwKgqr48+fhwngCuV9X3/dAThS4p1qnYBowAThCR1uLMOWisqh8Cd6nqdaq6yQc91UTkMBH5GidcEnXnM3h+f3G/pvwgpobevWifxUnN+RNwBtBXRI6AIpNC9gI/4PjB/+WOVjfGOWmo6lZVXR4ATYd7NOV77vKJ1lV4rty2vvjAo9BUoE7McF0RCfX+1gFfAj/j9HYOK6bfD13VROQV4E3gW5ynmX6ujmpuLxBVXYHzqN0W6ONufgB3TEVV17tjGkHQtMatz/frScwvTW6bHD80RalLVVVFpLY40TX5qjoZp4OzAOe6aubq8m28x/0978UZ92olIr919dYIXb/xvKZ8RWM7Sl0T+BNwgvu5Fc5JSvO0eRL4DDgJ5x/+Po7f+U08ERyprCmouqLU1Bf4AjjV/XwDzkX/HDGMQgKuAQ5337fDicSo5akfgPNon+aerxE4g8Vv4ongME3x1xSlrieBwaFrDbgbx731zxhfV+1xJvhd4Z6Lhp66hJwrX75XDE7UOR7DUN3zz6zt/v0K6Oy+PxX4mKKhlNXwOTQwiJqCqssHTecQg/BJr65i5RfjzFQejzOT+mTgPFfX8Z52DULfxTTFV5NPui72fo7BtR5am6Mm8B5O0MB/gftwXGvnxutcxeLl50k7HBiF8+jzGNAgTJuGOOF1LcPU+X5HDKKmoOryQVOsnnSK66rvlod+mJ2By933/YFncAbH43muTFPsdMXiSTWsJreuK/Bf931vYCsw0vt7iNW5iuXLTx99fZywvvvc9+Hy03QBFqpqpjj5NNqBM/iiPvq7A64pqLoqqylWsd7FdRVJYqWq6ao62m07Gsdw7HB1VYvTuTJNsdMVi+sqrCaXdThRNZ8CDwOzgRXqDozH+FzFjEoZehG5VUTOF5HD1AkDewtnmnsOcLaIhMIjQ6P7jXGmUt+OM3uzE/ibeS6ImoKqK4iayqMrDGfihJqGBs78HCw3TUmsqxyaGgPNcSawnY4zNnCiiLT3W1NcqcBjj+AMBE7EGTF/C2fwopmnTTcc/9bNxbYdjDOp4T3cwTs/XkHUFFRdQdRUGV04UT2X4Nx4RhLGD2yaYqspqLrKqekWT5m3vgHQxM9zlYhXuXr04sRtK46vdoOqdseZBbYDz+rmqvoTTmjWSW5cagO3ahRwnarerqrzy3PsZNIUVF1B1FQJXY1EpI46k3kUJ//QFaq6zDTFT1NQdVVA04mupvqqus0NLa6mqvvUmbyW3ER5Z6yOM0jyT+B8nNCjDzz11XAedc4vdid8EedOvRk4ys87VBA1BVVXEDX5qKvEwLBpir2moOqqpKYZsTpXiX5F7NGLyPk4saKNcab/DsDJeXGhiHSBQr9VP/cVohdODPZc4BRV3RjpWNESRE1B1RVETT7r8nOmrWlKYl0+aJrnt6bAEMUd8pcU9V+9hvMI9DtglucueSTO4EaaW3YVcF4s7k5B1BRUXUHUFFRdpim5dQVRU1Be0Zy8ejjT7au7n28CnnXfzwXuc993BobERXQANQVVVxA1BVWXaUpuXUHUFJRXRNeNqu5X1QN6KJ71EpxJBODkzW4vThKgITgxpyWyHPpNEDUFVVcQNQVVl2lKbl1B1BQUakRu4iBO9jbFySo5wi3ei7PCUUdgtbopVdW9bcaaIGoKqq4gagqqLtOU3LqCqCnRlCe8sgAnD8Q2nJXqvwYeBwpU9Uf1MW92kmsKqq4gagqqLtOU3LqCqCmxlMfPg5MEqAD4ESfRf8J9T0HUFFRdQdQUVF2mKbl1BVFTIl+hxEJRISKtgVuAF1T1QHluKLEiiJogmLqCqAmCqcs0RU8QdQVRUyIpl6E3DMMwko+ELg5uGIZhxB4z9IZhGCmOGXrDMIwUxwy9YRhGimOG3jAMI8UxQ28YhpHimKE3DMNIcf4fTopKzmxkRmIAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "# Show split\n", + "df_train['energy(kWh/hh)'].plot(label='train')\n", + "df_test['energy(kWh/hh)'].plot(label='test')\n", + "plt.title('energy(kWh/hh)')\n", + "plt.legend()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-16T00:24:15.905017Z", + "start_time": "2020-02-16T00:24:15.747859Z" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Train helpers" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "ExecuteTime": { + "end_time": "2020-03-01T03:41:41.584788Z", + "start_time": "2020-03-01T03:41:41.503778Z" + } + }, + "outputs": [], + "source": [ + "def main(trial, train=True):\n", + " # PyTorch Lightning will try to restore model parameters from previous trials if checkpoint\n", + " # filenames match. Therefore, the filenames for each trial must be made unique.\n", + " \n", + " checkpoint_callback = pl.callbacks.ModelCheckpoint(\n", + " os.path.join(MODEL_DIR, name, 'version_{}'.format(trial.number), \"chk\"), monitor='val_loss', mode='min')\n", + "\n", + " # The default logger in PyTorch Lightning writes to event files to be consumed by\n", + " # TensorBoard. We create a simple logger instead that holds the log in memory so that the\n", + " # final accuracy can be obtained after optimization. When using the default logger, the\n", + " # final accuracy could be stored in an attribute of the `Trainer` instead.\n", + " logger = DictLogger(MODEL_DIR, name=name, version=trial.number)\n", + "# print(\"log_dir\", logger.experiment.log_dir)\n", + " hparams = dict(**trial.params, **trial.user_attrs)\n", + "\n", + " trainer = pl.Trainer(\n", + " logger=logger,\n", + " val_percent_check=PERCENT_TEST_EXAMPLES,\n", + " checkpoint_callback=checkpoint_callback,\n", + " max_epochs=hparams['max_nb_epochs'],\n", + " gpus=-1 if torch.cuda.is_available() else None,\n", + " early_stop_callback=PyTorchLightningPruningCallback(trial, monitor='val_loss')\n", + " )\n", + " \n", + " model = LSTMSeq2Seq_PL(hparams)\n", + " if train:\n", + " trainer.fit(model)\n", + " return model, trainer" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "ExecuteTime": { + "end_time": "2020-03-01T03:41:41.652356Z", + "start_time": "2020-03-01T03:41:41.588888Z" + } + }, + "outputs": [], + "source": [ + "def add_suggest(trial):\n", + " trial.suggest_loguniform(\"learning_rate\", 1e-5, 1e-2)\n", + " trial.suggest_uniform(\"lstm_dropout\", 0, 0.75)\n", + " trial.suggest_categorical(\"hidden_size\", [1, 2, 4, 8, 16, 32, 64, 128]) \n", + " trial.suggest_categorical(\"lstm_layers\", [1, 2, 4, 8]) \n", + " trial.suggest_categorical(\"bidirectional\", [False, True]) \n", + " \n", + "\n", + " trial._user_attrs = {\n", + " 'batch_size': 16,\n", + " 'grad_clip': 40,\n", + " 'max_nb_epochs': 200,\n", + " 'num_workers': 4,\n", + " 'num_extra_target': 24*4,\n", + " 'vis_i': '670',\n", + " 'num_context': 24*4,\n", + " 'input_size': 18,\n", + " 'input_size_decoder': 17,\n", + " 'context_in_target': True,\n", + " 'output_size': 1\n", + " }\n", + " \n", + " # For manual experiment we will start at -1 and deincr by 1\n", + " versions = [int(s.stem.split('_')[-1]) for s in (MODEL_DIR / name).glob('version_*')] + [-1]\n", + " trial.number = min(versions)-1\n", + " print('trial.number', trial.number)\n", + " return trial" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "ExecuteTime": { + "end_time": "2020-03-01T03:41:41.723534Z", + "start_time": "2020-03-01T03:41:41.655316Z" + } + }, + "outputs": [], + "source": [ + "\n", + "def objective(trial):\n", + " # see https://github.com/optuna/optuna/blob/cf6f02d/examples/pytorch_lightning_simple.py\n", + " trial = add_suggest(trial)\n", + "\n", + " \n", + " print('trial', trial.number, 'params', trial.params)\n", + " \n", + " model, trainer = main(trial)\n", + " \n", + " # also report to tensorboard & print\n", + " print('logger.metrics', model.logger.metrics[-1:])\n", + " model.logger.experiment.add_hparams(trial.params, logger.metrics[-1])\n", + " model.logger.save()\n", + " \n", + " return model.logger.metrics[-1]['val_loss']\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Train" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "ExecuteTime": { + "end_time": "2020-03-01T03:41:41.777701Z", + "start_time": "2020-03-01T03:41:41.726164Z" + } + }, + "outputs": [], + "source": [ + "PERCENT_TEST_EXAMPLES = 0.3\n", + "EPOCHS = 2\n", + "DIR = Path(os.getcwd())\n", + "MODEL_DIR = DIR/ 'optuna_result'/ 'lstm'\n", + "name = \"lstm1\"\n", + "MODEL_DIR.mkdir(parents=True, exist_ok=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "ExecuteTime": { + "end_time": "2020-03-01T03:41:41.829950Z", + "start_time": "2020-03-01T03:41:41.779861Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "now run `tensorboard --logdir /media/wassname/Storage5/projects2/3ST/attentive-neural-processes/optuna_result/lstm\n" + ] + } + ], + "source": [ + "print(f\"now run `tensorboard --logdir {MODEL_DIR}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "[I 2020-01-29 08:42:54,398] Finished trial#8 resulted in value: 0.005635036621242762. Current best value is 0.005635036621242762 with parameters: {'batch_size': 16, 'bidirectional': False, 'grad_clip': 40, 'hidden_size': 128, 'input_size': 17, 'learning_rate': 0.00019134834148401144, 'lstm_dropout': 0.4080689425353674, 'lstm_layers': 8, 'max_nb_epochs': 20, 'num_workers': 4, 'target_length': 24, 'vis_i': 670, 'window_length': 48}." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-01T12:52:22.492191Z", + "start_time": "2020-02-01T12:52:22.416653Z" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-16T01:47:59.598921Z", + "start_time": "2020-02-16T01:47:02.100Z" + } + }, + "source": [ + "Note that the LSTM has access to the y values for the first half of the plot (the context) to match the NP setup." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "start_time": "2020-03-01T03:47:34.500Z" + }, + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "trial.number -21\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:root:gpu available: True, used: True\n", + "INFO:root:VISIBLE GPUS: 0\n" + ] + } + ], + "source": [ + "trial = optuna.trial.FixedTrial(\n", + " params={\n", + " 'bidirectional': False,\n", + " 'hidden_size': 128,\n", + " 'learning_rate': 2e-5,\n", + " 'lstm_dropout': 0.0,\n", + " 'lstm_layers': 2,\n", + " })\n", + "trial._user_attrs = {\n", + " 'batch_size': 16,\n", + " 'grad_clip': 40,\n", + " 'max_nb_epochs': 200,\n", + " 'num_workers': 4,\n", + " 'num_extra_target': 24*4,\n", + " 'vis_i': '670',\n", + " 'num_context': 24*4,\n", + " 'input_size': 18,\n", + " 'input_size_decoder': 17,\n", + " 'context_in_target': True,\n", + " 'output_size': 1\n", + "}\n", + "trial = add_suggest(trial)\n", + "model, trainer = main(trial, train=False)\n", + "trainer.fit(model)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-03-01T03:31:43.522361Z", + "start_time": "2020-03-01T03:31:15.800Z" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-03-01T03:47:33.307396Z", + "start_time": "2020-03-01T03:41:28.300Z" + } + }, + "outputs": [], + "source": [ + "# test\n", + "trainer.test(model)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-03-01T03:47:33.308984Z", + "start_time": "2020-03-01T03:41:28.300Z" + } + }, + "outputs": [], + "source": [ + "loader = model.val_dataloader()[0]\n", + "dset_test = loader.dataset\n", + "label_names = dset_test.label_names\n", + "plot_from_loader(loader, model, i=670)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-03-01T03:47:33.310698Z", + "start_time": "2020-03-01T03:41:28.300Z" + } + }, + "outputs": [], + "source": [ + "# Look at input data, low temp, and holidays imply \n", + "rows_x, rows_y = dset_test.get_rows(670)\n", + "rows_x[['apparentTemperature', 'holiday', 'dayofweek']].plot()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "ExecuteTime": { + "end_time": "2020-01-27T09:03:28.792465Z", + "start_time": "2020-01-27T09:03:28.346446Z" + } + }, + "source": [ + "# Hyperparam opt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-03-01T03:47:33.312313Z", + "start_time": "2020-03-01T03:41:28.300Z" + } + }, + "outputs": [], + "source": [ + "import argparse \n", + "\n", + "parser = argparse.ArgumentParser(description='PyTorch Lightning example.')\n", + "parser.add_argument('--pruning', '-p', action='store_true',\n", + " help='Activate the pruning feature. `MedianPruner` stops unpromising '\n", + " 'trials at the early stages of training.')\n", + "args = parser.parse_args(['-p'])\n", + "\n", + "pruner = optuna.pruners.MedianPruner() if args.pruning else optuna.pruners.NopPruner()\n", + "\n", + "study = optuna.create_study(direction='minimize', pruner=pruner, storage=f'sqlite:///optuna_result/{name}.db', study_name='no-name-b60e37fc-4ab6-4793-8a0a-c87a1b40c5c0', load_if_exists=True)\n", + "\n", + "# shutil.rmtree(MODEL_DIR)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-01-27T23:38:16.813496Z", + "start_time": "2020-01-27T23:38:16.766674Z" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-03-01T03:47:33.313923Z", + "start_time": "2020-03-01T03:41:28.300Z" + } + }, + "outputs": [], + "source": [ + "study.optimize(objective, n_trials=200, timeout=6000)\n", + "\n", + "print('Number of finished trials: {}'.format(len(study.trials)))\n", + "\n", + "print('Best trial:')\n", + "trial = study.best_trial\n", + "\n", + "print(' Value: {}'.format(trial.value))\n", + "\n", + "print(' Params: ')\n", + "for key, value in trial.params.items():\n", + " print(' {}: {}'.format(key, value))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-03-01T03:47:33.315413Z", + "start_time": "2020-03-01T03:41:28.300Z" + } + }, + "outputs": [], + "source": [ + "study.optimize(objective, n_trials=200, timeout=6000)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-03-01T03:47:33.317152Z", + "start_time": "2020-03-01T03:41:28.300Z" + } + }, + "outputs": [], + "source": [ + "df = study.trials_dataframe(attrs=('number', 'value', 'params', 'state'))\n", + "df.sort_values('value')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Scratch" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-03-01T03:47:33.318533Z", + "start_time": "2020-03-01T03:41:28.300Z" + } + }, + "outputs": [], + "source": [ + "model, trainer = main(trial, train=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-03-01T03:47:33.319880Z", + "start_time": "2020-03-01T03:41:28.300Z" + } + }, + "outputs": [], + "source": [ + "loader = model.val_dataloader()[0]\n", + "dset_test = loader.dataset\n", + "label_names = dset_test.label_names" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-03-01T03:47:33.321408Z", + "start_time": "2020-03-01T03:41:28.300Z" + } + }, + "outputs": [], + "source": [ + "x_rows, y_rows = dset_test.iloc(10)\n", + "x_rows.loc[x_rows.index[model.hparams.window_length:], dset_test.label_names] = 0.\n", + "x_rows" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-03-01T03:47:33.322849Z", + "start_time": "2020-03-01T03:41:28.300Z" + } + }, + "outputs": [], + "source": [ + "x_rows.loc[x_rows.index[model.hparams.window_length:]]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-03-01T03:47:33.324307Z", + "start_time": "2020-03-01T03:41:28.300Z" + } + }, + "outputs": [], + "source": [ + "self=dset_test\n", + "idx=10\n", + "k = idx + self.hparams.window_length + self.hparams.target_length\n", + "j = k - self.hparams.target_length\n", + "i = j - self.hparams.window_length\n", + "idx, k, j, i, self.hparams.window_length + self.hparams.target_length" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-03-01T03:47:33.325661Z", + "start_time": "2020-03-01T03:41:28.300Z" + } + }, + "outputs": [], + "source": [ + "j-i" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-03-01T03:47:33.327059Z", + "start_time": "2020-03-01T03:41:28.300Z" + } + }, + "outputs": [], + "source": [ + "self.hparams.window_length + self.hparams.target_length" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "file_extension": ".py", + "kernelspec": { + "display_name": "jup3.7.3", + "language": "python", + "name": "jup3.7.3" + }, + "mimetype": "text/x-python", + "name": "python", + "npconvert_exporter": "python", + "pygments_lexer": "ipython3", + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": { + "height": "calc(100% - 180px)", + "left": "10px", + "top": "150px", + "width": "384px" + }, + "toc_section_display": true, + "toc_window_display": true + }, + "version": 3 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/smartmeters-lstm_std.ipynb b/smartmeters-lstm_std.ipynb index 76d6d42..fcec987 100644 --- a/smartmeters-lstm_std.ipynb +++ b/smartmeters-lstm_std.ipynb @@ -2187,14 +2187,48 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "metadata": { "ExecuteTime": { - "end_time": "2020-02-16T10:16:31.859241Z", - "start_time": "2020-02-16T08:59:28.600Z" + "end_time": "2020-02-16T13:43:24.006382Z", + "start_time": "2020-02-16T13:43:23.406865Z" } }, - "outputs": [], + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "955c80382b0f41d79488afa4bb98e49b", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEgCAYAAACkfIiyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOy9eZxcZZn3/b1r33tPutPdWSB7gA6rIvsog/gC7iAOOooO6iM6zOP+MA+DuG+Pzii+jr4q7qDoA9EBcRAQhCAkkEAWAoEs3Ul3eq99OafO/f5xn1q7qtOdVHdXp8/v82lInXPqrrtOVf3O7/yu674uIaXEggULFizMf9jmegIWLFiwYKE2sAjdggULFk4QWIRuwYIFCycILEK3YMGChRMEFqFbsGDBwgkCi9AtWLBg4QSBRegLAEKIO4QQnzf/fYEQYs8xjvM9IcT/ru3sjh9CiPcIIf461/NYSBBCPCKEeP8svZYUQqycjdea77AIfYFBSvmYlHLN0Y6rRJJSyg9KKT9X6zkJIVYLIe4VQgwJIUaFEA8IIY46x7mEEGKjEGKrECJh/n9j0T4hhPiKEGLE/PuKEELMt7EszD9YhD7PIIRwzPUcZgCNwCZgDbAYeAq4d05nNAmEEC7U/H4ONAE/Ae41twPcALwJ6AFOA64EPjCfxrIwTyGltP7m+A/YD3wG2AWMAT8GPOa+i4E+4FPAAPAzc/sVwDZgHHgCOK1ovNOBZ4AocBdwJ/D54vGKju0GfgcMASPAd4B1QArIAjFg3Dz2jvJxgI8Bg0A/8N6icVuA3wMR4Gng88Bfp3g+mgEJtFTZ34K6AERQ5P+54rGBfwd6zf1bgQvM7e1Aonhc4AzzvTuBlcBfgDAwDNxV5fX/HjgEiKJtB4HXm/9+ArihaN/7gCfn2VhNwB/MczNm/ruraP8j5nl/3Pye/QloLdr/avP1xoHtwMVF+94L7Daf9wrwgbLX/oT5fToMXG9+F1bO9e90PvxZCr1+8A/AZcDJwGrgX4v2taNIbhlwgxDidOBHKHXVAvwnsEkI4TbV2D3Az8zn/AZ4a6UXFELYUT/UA8ByoBO4U0q5G/ggsFlKGZBSNlaZczvQYD7vfcDtQogmc9/tQNw85h/Nv6niQmBASjlSZf/tqAtOB+oHf33Z/qeBjaj3/0vgN0IIj5RyAEVEVxcd+y7Ue9ZQBPUnFJl1Ad+u8vobgOekyT4mnjO35/ZvL9q3vWgfQojnhBDvrLexymBDCYtlwFIgibrYF+OdKHJeBLiAj5tz6gT+C3URbza3/1YI0WY+bxAlSELm878phDjDfO7rzeMvBVYBr6syPwsVYBF6/eA7UspeKeUo8AXg2qJ9BvBvUsq0lDKJunX+Tynl36SUWSnlT4A0ShW9GqU2vyWl1KSUd6MIrhLOAZYAn5BSxqWUKSnldIKLGnCb+Tr3odT8GvNC8VZzzgkp5S7U7f9RIYToQhH2/6yyPzf2Leacd5SPLaX8uZRyREqpSym/AbhRdg7msdcVjXUt6uKXez/LgCVHORcBlIovRhgIVtkfBgI5v1pKeZqU8pf1NlYxzPP3W/Pzi6K+kxeVHfZjKeWL5nfy16iLKKjze5+U8j4ppSGl/G9gC/AGc+z/klK+LBX+grqIXmA+92pz3B1Syjhwa/ncLFSHRej1g96ifx9AEW0OQ1LKVNHjZcDHhBDjuT+UdbLE/DtUptIOVHnNbuCAlFI/xjmPlD03gSKNNsBB6Xsq/ndFmAruT8B3pZS/qnJYpbFL3p8Q4uNCiN1CiLB5bhqAVnP3vcB6IcQKlAoMSymfMvd9EhDAU0KInUKIcuWfQwylLosRQlkIlfaHgFjZZ1LXYwkhfEKI/xRCHBBCRIBHgUbzIpjDQNG/c589qO/n28u+n+ej7qgQQlwuhHjSDICPo4g+9/ksYZLP1sLksAi9ftBd9O+lKP8wh/IfXC/wBSllY9GfzyTBfqCzTHUtrfKavcDSKoHW4ynDOQToKNsih+4qxwJgWjV/AjZJKb8whbHLz1dunAtQxHw10GTaRWEUUWNeGH+NUpHvoqDOkVIOSCn/SUq5BGVnfbdKutxO4LSyc3yauT23v6doX0/Rvvky1sdQdzWvklKGUDYYmOfxKOhFxXqKv59+KeWXhRBu4LfA14HF5udzX9G4/VT5bC0cHRah1w8+LIToEkI0AzejgpnV8APgg0KIV5mpaH4hxP8jhAgCm1GE91EhhFMI8RaUtVIJT6F+QF82x/AIIc4z9x0BuooyJKYMKWUWFWi91VR6a4F3VzteCBECHgAel1J+eppjr6fUnw+i3v8Q4BBC3MJE1fpT4D3AVRQRuhDi7ablAyoQKFF2VzkeQQWMP2rGLW40tz9UNP7/FEJ0CiGWoMjxjipvqV7HCqJ883HzO/lvVY6rhJ8DVwohLhNC2M3v1cXmuXWhLLAhQBdCXI4K5ubwa+A9Qoj1QgjfNF93wcMi9PrBL1EK9RXgZVRAqSKklFuAf0IFqcaAvSiCQkqZAd5iPh4FrkERYKVxsqjUtZWobIg+83hQJLATGBBCDB/D+7kRZXUMoEjzVyifvxLeDJwNvFcIESv6q6bObkTd3g+gCOnHRfseAP4IvIi6XU9RZvdIKR9HEfUzUsriW/qzgb8JIWKoLJp/llK+Uv7i5jl+E+oiNY4Kyr7J3A4qSP174HlgBypA+J+555t2zj/U21hl+BbgRWX7PGme0ylBStkLvBH4Xyji7kVlrthMP/6jKOIeQwVWNxU9937ztR9Cfa8fwsKUISpbcRZmE0KI/cD7pZQPzvVcZgpCiK8A7VLK6WS7zBiEEA8Bv5RS/n9zPRcLFmoFS6FbmBEIIdYKIU4zLaFzUGmN/3eu5wUghDgblX8+ma1lwcK8w4m46tBCfSCIslmWoPz4b1AHqz+FED9B2RL/bN7+W7BwwsCyXCxYsGDhBIFluViwYMHCCQKL0C1YsGDhBMGceeitra1y+fLlc/XyACSTIDNJfPEXSHpPQnc0AOBO9+HUxogFTp32mEJAIHD04yxYsGDhWLB169ZhKWVbpX1zRujLly9ny5Ytc/XyADz/POgvP8Hp2y5nW88PGG+6GIBl+7/Miv1f4i8XbkbanNMa026HCy44+nEWLFiwcCwQQlQth7CgLRcpwZ6NA5C1F2S15mwBwKmNTnvMbLY2c7NgwYKF6cIi9GwMgKzdn9+ecXUAyno5FhiVFotbsGDBwgxjQRM6FCv0AqEnfCcD4E1OWPU9JViEbsGChbnAnOWhn3XWWbLcQ9c0jb6+PlKpVJVn1RaZDIhMFKc+RsrdBSJ3fZN4Ur3ojoZ8oHQ6cLtVcHQ+wOPx0NXVhdM5vViBBQsW5gZCiK1SyrMq7aurlaJ9fX0Eg0GWL19OhZr7NUciAfbEAO6Mk2hgXRGhgz8mydr9pLzLpz2u3w+2eXDvI6VkZGSEvr4+VqxYMdfTsWDBwnGirmgnlUrR0tIyK2QOykMHA4mgvMyzYXNjM6oVB5zKuPUPIQQtLS2zdkdkwYKFmUVdETowa2Sefz1pILFN8EjyhD5f2PkYMdvn24IFCzOHKRG6EOL1Qog9Qoi9QogJDQiEEMuEEH8WqmHtI0VNAuofMltiteRg2NwIsogK3dkOHNjPr3/9ywnb80Oe2NcACxYWHBIZnX+953kGo/V9N3tUQjd7CN4OXA6sB641u8QU4+vAT6WUpwG3AV+q9URnCgJJpa5ahs2j9suJtsvBg/v5zW8qE7quH2t7TgsWLNQr/rhjgJ8/eZBHXzyWXi+zh6ko9HOAvVLKV8zOJ3eiupEUYz2FziIPV9hfx5Cmhw6f//wt3H77twAwbC5u/vJ3+H+/+x8TnvFv//ZpNm9+jPPO28h3vvNNfvGLO7jmmqu44oq/48orX8sjjzzCFVdckT/+xhtv5I477gBg69atXHTRRZx55plcdtll9Pf3z/QbtGDBwnHijztUP+yBcHKOZzI5pkLonZS28OoztxVjO6rtGah2YkEhRMvxT282IPP++bvedT133vlTALLSyZ2b/sQ733bVhGd89rNf5txzL+Dxx7dx443/AsD27c/w05/ezf33/6XqK2maxkc+8hHuvvtutm7dyvXXX8/NN988A+/JggULtUIyk+XRl4YA6A/Xt+VSq7TFjwPfEUK8B3gUOIRqVlsCIcQNwA0AS5dO3sz7s7/fya7DkRpNT2H9khD/duWG0jnJguWybNlymptb2L79WQYHj7DxlHW0NfqYykd4ySWX0tzcPOkxe/bsYceOHVx66aUAZLNZOjo6juWtWLBgYZbwlxeHSGkGDpvgSGT+E/ohoLvocZe5LQ8p5WFMhS6ECABvlVKOlw8kpfw+8H1QC4uOcc41gwpelnro7373+/nFL+5gcHCAf7z2mimnLvp8hZWmdrsDo2i5aC4tUErJhg0b2Lx58/FP3oKFExzhpMa/P/gS/3LpKoKeuVv49qedAzR4nZzW1XBCKPSngVVCiBUoIn8HqlN3HkKIVmBUSmkAnwF+dLwTK1fSMweJLErdu/LKN/OFL9yCrmv89LvfxJ49gl2PkXUUincFAkFiserdy5YuXcauXbtIp9Mkk0n+/Oc/c/7557NmzRqGhobYvHkz5557Lpqm8eKLL7Jhw2y9VwsW5g9+s6WXHz2+j/NXtfB3axfP2Tz+8uIQf7d2ER6njd39tXUNao2jErqUUhdC3Ag8ANiBH0kpdwohbgO2SCk3ARcDXxJCSJTl8uEZnHONUarQXS4XF154CQ0NjRjexRjxEVzpwyTtq/Je+ymnnIbdbuc1r+nhne98D01NTSUjdnV1c/XVV3PKKaewYsUKTj/99PzYd999Nx/96EcJh8Pous5NN91kEboFCxVw77bDAERTc5c5Fk/rjMQzrF4cJKMbDMcypPUsbod9zuY0GabkoUsp7wPuK9t2S9G/7wburu3UZgdCSmRRHrphGDz99JP85Ce/AWEn42rHk+7DoY+hO5pACJxOJ3/4w0Ml4/zDP7yn5PFXv/pVvvrVr054vY0bN/Loo4/OyHuxYOFEwd7BKM8fCgMQSWpTft5gJEWT34XTXps1k/1mVsuSRg8pLWu+RpruZl9Nxq816m6l6OyjkLb4wgu72LhxJRdd9FpWrlwFqNroWZsHb+oAvsSL2LJHT1uyFhZZsHB8uOfZw9jMG+fIFBX6Ey8Pc/5XHubzf9hVs3kcGlee+ZJGL+0NXgAG6jgwWlfFueYGhbTFtWvX89xzhZK5O3c+zw03vAsAIXWEoeN2u3jwkWdAWKfOwtwhk4HeXnA4YNmyuZ5NbSGl5N7thzhvZStP7RslPAWF/sJAhA/8dCuZrMFvtvbx8cvW1CSQeng8p9C9xMwLy0AdB0YXNCtJmftP5XomGzacyuOPb8s/tmXj+BIvkU0eIOk9qWqNXEuhW5hp7NoF4+Pg8Zx4hL7zcITe0SQ3XrKSPQPRKVkun/nd83hddr729tP44M+f4Z5nD/Guc5cf91wOjyexCVgcdBNwK7qsZ0Jf8JZLtaX/lWDY/aTdS3BkI9iM+MxOzIKFKjhyRJE5QCoF0eoJV/MSf9p1BJuA161bTMjrJJKanNAPjMR59uA415+/gss2tHNqZwM/f/Igtej1cHg8RXvIg8NuI+Rx4HXa69pyWfCEXuyhTwW6Qy0ecuixmZqQBQtVkc3Cyy+Xbhsampu5zBT+e9cRzlzWREvATcjjIJKc3EP//XaVDXNlzxKEEFz36qXsORLlub7wcc/l8HiSjkblnQsh6GjwWAq9viGrWicVj7Y5yNq82LPVZZFluViYKQwPK/+8GCcSofeOJtjdH+HS9Srv/GgKXUrJPdsOc87yZjpN4j19qUoj7hs7/rorh8NJlpjjAiwOeSyFXteYxEOvhqw9oHqRSqt5qIXZRSXyTiYhfoI4gP+96wgAl65vByDkcU7qoe/uj7J3MMaVG5fkt4XMYOhUgqmVkEwqW8swJP3jKZY0evL7LIVe55iOh56Dbg8ikPkG0+WwFLqFmUA2C6OjlfdF6nsB45Tx2EtDnNzmZ0WrKqUR8jomTVt8eM8gAG84pT2/LeRVwcujee+VoGnw3HMwMAAj8QyZrMGShiKF3uDhSCSFYdTnj3zBE/p0PXSArMOPBOzZ2vjo+/fv55RTTqnJWPWMVArGxuZ6FvMXIyNgVLkpPFEIfSiWZllLoS5STqFXC3AeHk/S5HPSEnDnt3mddhw2Ma0FSTkcOKAUejgMfaOFlMUcFgfd6IZkNJGpNsScYsESeuH7MT0PHQDhwLD5qvroubGz2QkFJxcshofh6adhx44Txx6YbUzmlZ8ohB5J6gQ9hWzqkNeJbkgSmcq/pSORFItDnpJtQggavM5jslxyd0CGAS/2FVaJ5tDkdwEwnjg2O2emsWAJHQCZ0+YTG1wA3HbbzXz3u/8+4WmPPfYIF7/53bzxne/jjDPWcNNNH8xXV+zoCPCZz3yMnp4eNm/eXLWhxdatW+np6aGnp4fbb799pt/pnOPgQWUZZLOK1KspTQvVEZ4kaSORgBOhWVY0peU9cCj44dXsk/5wivYGz4TtKpg6vROSSqnzmMPLA4rQO4sUeqNPEXo4WZ8KvX4XFt3/aRh4vrZjtp8Kl3+5aENOphcaXFx33Vv48IdvwjAM7r77Th5++KmKQz397HZ2PfwbWtb8PW9+2xVs2vQ73vSmtxGPxznrrFfx7W9/A03TuOiii7j33ntpa2vjrrvu4uabb+ZHP/oR733ve/nOd77DhRdeyCc+8Ynavs86QypVqiCTSaU2F89dAb15B8OYmN1SDClVPnpZnbh5BSklkZSe98ABGrwmoSd1OhomPudIJMVpXRN3qHTH6ano8vhE70gKr9OenwNAo/nvsXh9KvT6JfRZgTT/W7nBRU/P6bS0VG68dNYZZ3DSsi7iNp23ve1aNm/+K29609uw2+288Y1vBao3tBgfH2d8fJwLL7wQgHe9613cf//9M/1m5wyVrIL+fovQp4N0WVn+uKbhczgQRXZhJDK/CT2pZckasmTJ/mQBzlz1w3LLRT1v+pZLeXxnKJ5kUdBTco4bfWpu48eYQTPTqF9CL1HSMwVToYvKDS6uu+766k8VqnymzciYQ6gxPB4PdrsdKas3tBgfn9D744RGJUIfH1dK3euduM/CRBQTelLX+JdH/sw/rj+F8zq78tvnu4+eW0BU0XKpQKCDUZU+2F6F0A+NTz0PXcqJhB7XNDzCVbKt0Zvz0OvTclmwHrqUufZzUJy2eOWVb+bBB//IM888zeted1nV5299Zgv7Dh5C6il++9u7OPfc8yccU9zQAlRP0Z07d9LY2EhjYyN//etfAfjFL35RuzdWZ0inqxPNwMDszmU+I1WU+jyaTJHJZtkxPFSSIzvfCT1qqvDyoChUVui5dnCLK3noHudRV5gWY3BwYgwiqeu4hYOREWV3HTyo5mYTx57jPtNYsISuMJHQcw0u3vzmq7HbqxexP+OMs/nwzV9l42vOZ/nyFVx55ZtLR5aFhhaf+tSn6OnpYePGjTzxxBMA/PjHP+bDH/4wGzdurEnNiXrFZCRz5MjszWO+o5jQw5k0r7Vt5WNjN3LBY0vwJPcBKod6PmcQ5Ug75C1W6KblUoGcc+3gOioQesMUasDkXzcCe/ZM3J7UNbwOJy+/DFu2wP79IKXKoBmrU4Vev5bLrMD00Issl4yuFxpcTIJgMMTv7/g6UjhI+lbmt/f3l+amV2toceaZZ7J9+/b840rNME4ETEYwqZTa7/dXP8aCQrHl0jT2CD90fYM+2YrdSLBo8HccXPYxQFlZ8/V8FiyXAi0FJ7Fccis2K1suDjK6QUrL4nFWF2apVPWsq4Su43U6SjJfRkZUpouVtliHKLdcdu7aQc/Glbzm/IvzDS4mg2FzIYz6vFLXC4p/DJUwMjI785jvyCl0YaR53cBt7DMW89r01+n39NA2tCl/3HwOz0TylktBobscNrxOe0WL40gkhdthK8lCyWEy7z2HXAptteyhpKbjc5Rq3qEhFRitV8vFUuhAjtBXr1nHfY9vI+BUgY/iBhc5uFxuHn74b1xwwcUYqUM4ZMQ05EsXJ53ALsq0MBVCX7p0duYyn5FT6N29/0GbfoCP659GF26edF7Am6PfwZPcT8q7fNJc9XpHLm+8OG0x97iSfTIQSdPeUJqFUnhOwXtfVEHBA+zdC7Eqi70z2Sy6NPA6Si8Wo6PQ4HEyHE9XfuIcwyJ0CmmLubuutLnCs7zBxYRn21wIJELqSHH83VFONEh5dEKPRJT367RO36RIp0EYGt29t/OM63y2y7NZ6vFwT+ZM3gy0Dv+evu6PkMmoc+6rz5aXkyIXFA2VdRpq8FYOcB4JpyraLbnnwOTBy8nuDpOaku3eMoWezYLX4WI8UZ/ls6dkuQghXi+E2COE2CuE+HSF/UuFEA8LIZ4VQjwnhHhD7adaWygFnUtbtJnb1GPNyJKdwlJGw6aUvJD1ebWea6RSR18RKqVluxwNmqaIpHH8UZz6GH+0XUrI5WZlYxObYz4igdNOCNslktRx2W24HaW0FPJUDnAORCqvElXPqR5MBfXdrGa1LN/3RV73zJkESeBzTtS8fruT8Hz10IUQduB24HJgPXCtEGJ92WH/CvxaSnk68A7gu7We6ExAlPkixY/SU6jDIoUqCGSr4KNblsvR1XkO1SoIWlDI+edtQ/eg2wP8xegh5HKxtrmFTDbL/ZnTCEW2YDebrszXAmiRlEbI65hgoVSqiS6lVIReRaFPlu4I1bOvOg7/iOUHvoI/c4gLbM9NsFwAvHYX0bSOlq2/+hVTUejnAHullK9IKTPAncAby46RQMj8dwNwuHZTnEmUeujF6YOp7NFzWA2b+rArEbqFsgwXKVk8cBerXvw4tmxpPen5qihnCzm7pW3oD4y0vJ4RTRJyuzlj0WKuXbueB+KdCAyC0WcBRejzUVBEU3rFxs6VuhaNJTQyulFxlSgc3XIpJ3RDSryJvax+8WOMNF9Kyt7Ia+3PTgiKArjF8dVbn0lMhdA7gd6ix33mtmLcClwnhOgD7gM+UpPZzThyaYum5WI+tglBWp9coeuGQdqQGMJxXJku99xzD7t27Trqcd/73vf46U9/Oukx27Zt47777jvmudQaOYVuyyY49fm3se6FG+g8/ANW7v1kyXGZTPXglAWl0BvH/4pTH2Ww7Y1E0mlCLhdCCC5bvoKly9UCOPfYk4BaIDMfg6ORpFaSsphDJYXeH1arQCvloENhcVK1LJdiQteMLJ969GGGXvo5AoMXV3+LV/wXcJFtOz77RIp0YS7/r0PbpVZB0WuBO6SU3xBCnAv8TAhxipSlLX2EEDcANwAsnUJqwyOP1Gh2Ji6+uGxDueViPnTbHaR0HSllxQg6wFgqRSqr02RzYpMTP9ipKCRd17nnnnu44oorWL++3MUqxQc/+MGjjrdt2za2bNnCG95QHyGMHKEvGvwdLaMPsvfkz+PURlh28JtEg2fSv+Qf88eOjUEgMEcTrXOk09Ay8gBZm4/DoUvIGI8Schfqf7c0LuWV3nbc44VCcqOj0Ng4F7M9dkRTWhWFrmqiG4bEZlO/x8deGgZgw5IKFbsAt8OOx2mrWHFRylIB8VR/P0PJJEvtjxH3rSXt6WK39wLWR37PkvQO4IKS53tt9VtxcSoK/RDQXfS4y9xWjPcBvwaQUm4GPEBr+UBSyu9LKc+SUp7V1tZ2bDOuIUS55YLkUO9BrrjkHCSSjJHlP/7j63zxi7fyhjdczC23fIqLLz6H009fzebNj5GVEt0QfPLWL/CqV53Cueeexve+920Annmmctnciy++mJtuuomzzjqLr3zlK2zatIlPfOITbNy4kZdffpkf/OAHnH322fT09PDWt76VhMmKt956K1//+tfzY3zqU5/inHPOYfXq1Tz22GNkMhluueUW7rrrLjZu3Mhdd93FqlWrGDILqRiGwcqVK/OPZwM5Qm8bupekZyl9XTeyb8X/ZrzhfFbsuw1hFC6E89X3nQ1kMuBJ7SPhO5lwVv1kQ64CoXcFgzwrV9IWfyavJOZjoLm80mIOnU1eDFnaI3TTtsNs7G5kaUv1dJ4Gb+XgZSxWCNZLKXnwwH48pFmrb2e0+XUA7HSdQ1YKlkYenvB8v6t+Ky5OhdCfBlYJIVYIIVyooOemsmMOAq8FEEKsQxH6PGhdW+6h5x6px+VBD13XeeSRp/jyl7/Jt7/xJaSUfP/nv+FAbx+PP76NzZuf4+qr/wFN07jppo9w9913s3XrVq6//npuvvnm/DiZTIYtW7Zw8803c9VVV/G1r32Nbdu2cfLJJ/OWt7yFp59+mu3bt7Nu3Tp++MMfVpy5rus89dRTfOtb3+Kzn/0sLpeL2267jWuuuYZt27ZxzTXXcN111+XrxDz44IP09PQwWxdSXVd/Dm2cprGHGWp7o8rVF3YOLv0oLm2YlpE/5o8Ph60a6dWQzYIn1Uva3U3YTEgPuQpFoxrdbnaLNQSzw7jTyh2Nx0vLBcwHKMtlokJf0x4E4IUB5ZPsHYyxqz/CVT1LJhxbjGrZMcV2yyvhcfZFwrzG/gIuNEab/w6AEcPPFrmGJSP3giy1XwPO+q24eFRCl1LqwI3AA8BuVDbLTiHEbUKIq8zDPgb8kxBiO/Ar4D2yzguUFKct5pb+5zx0IRSpa0bpB3nVVW8B4NSeMzjcexCAhx59gg9c9xYcZt2X5uZmXnppDzt3qrK5Gzdu5POf/zx9fX35ca655pqq89qxYwcXXHABp556Kr/4xS/YuXNnxePe8hY1lzPPPJP9+/dXPOb666/P++65GuyzhdxCmJaR+7FJjaG2Qq2bsabXkna109H/s/y2bHb+F5eaKeg6eFJ9pDxdRMxcu2KFLoSg37tRbY9syW+fbypdBUUnKvQ1ixWh7xlQHcI2bT+MTcAVp3VMOl4l7x1Ulc8cHjx4AI/dwdsDe0lKF+GG89Qxus6dxmX4k3tpG7qn5Pl+Z/1WXJyShy6lvA8V7CzedkvRv3cB59V2arOAsqX/hgSHw440DBw2G5msQapI5rjMH5EBZM0sGIkwrRsDsJvDStav38CTT5aWzciEvNUAACAASURBVM3BP0mxjfe85z3cc8899PT0cMcdd/BIlUCC2/RQ7XY7epVWNd3d3SxevJiHHnqIp556alarOuZyfNuG7iHl7iYaPCO/T9ocDLS/k6UHv4Ur3U/GrX6Y4fD8831nAzIZxpENk3Z3ETGvlA1FHjqAFjqN1KCTUPgphhapi/3ICHSWpy/UKbSsQVLLVlTofreDpc0+XhiIIqXk99sP8+qTWqquAM2hwetkKDpxjUjuJx3XNLYM9HPxkkWcPfYkfzPWoQsXDlRhrq3284n7HmD5/q8ipMHy/V9k97ofYgRPxybqMyi6sGu5VPDQW1oXMzQ0SDw8RjyZ5I9//MOE5+lF3sAlF1/Cf/7st2Q19S0ZHR1l1arKZXMrIRgMEo0WepNGo1E6OjrQNG3aBFw+FsD73/9+rrvuOt7+9rdPWj2y1sil2jWPPsxw6xUTSiMMtF+HwGDxkTvz2+ZjZsZswBZTIatihR50ldbp7gg2sV2ejH+8ICLGx9Wdz3xA1AxeVlLoAGvbg7wwEOGlwRj7huO84dTJ1TmodMdKqYU5hf5U/2E0I8snje/Smj3Ez7OvI5JRF4CEruN2ujmw7BP4Ey+wfvf78SVfoaP/J9iEwO90Mj5Pg6InMHKEnlspCi6Xk0996hbeevnFXP+Oq1i1eu2EZ2WKrJh3vesfWdrZzqvPO5PXvKaH3/zml7hcLu68s3LZ3HK84x3v4Gtf+xqnn346L7/8Mp/73Od41atexXnnncfatRNfezJccskl7Nq1Kx8UBbjqqquIxWKzareAUuje5D5sMk002DNhf9J3MpHg6bQO/1d+WyQyP/OnZxqOuPLF0+6lRDJp/E4nTruN1lZYtEgd0xUM8qSxjsb4c9h15V0ZxvwJNufSC0MVCm2BIvT9Iwn+6zmVXPC6dUdvd1XNcskp9McO9fFJ359YNXo3m9s+woPGmYybd0BJXXWEGlz0ZoZar+Bg90cZar2K1uHfIwwdv7M+Ky7WdS2XCWmGNUfOQ889UgbKhz70Ud79vg8ymEywJBDEXaZsg43N3L/5OQCEw8P/ufVjfN5zEllnIYXqtNMql80tt1DOO++8kjz0D33oQ3zoQx+a8Lxbb7214hitra15D725uZmnn3665Hnbt2+np6dn2heH40U6Db6EKjKd8K2peMxIy+Us3/8lnJkhNFcbuq6CeVb6YimcCUXoSqH3E3K5WLMG2ttVxsbgoCL0nxnr+Gf+Lw3hvzHaotoejoxA64R8s/pDTqFXslwA1naEyBqSn27ez6mdDVWX/Bcjl+5YnH6cK6NwKBZlVfS/+R+un3Bk0dvYveRj0PtEPuic1HQCLhcIOztPUXfKrUP30ja8iYbwX/E7nIzG6o/QF7ZClxOzXHLOgNMkca3CPatmGDht6tRlTd9cyPpruf7lL3+Zt771rXzpS1+a9ddWhP4CAAnf6orHjLRchkDSPPrf+W2W7VIKXQd3qg9DOMm4FhNJp2nyuWlvV/sDAdXGz+twcsB1GjoOGscfyz9/vgRGIxW6FRUjl+kyltB47bpFUxqzwevEkBBNF36bObvlcO/jfNP5XUYDZ7JnzXdocKsLRI7QE7o+oTDXaPOlZG0+2obuxe9yMha3LJe6gZSVPfRcyqLTZquY6ZI1DLLSwGVXH3Y211u0wuKiucanP/1pDhw4wPnnT2yPN9PIZMAff5GUu5uso7LkjgV6SLs6aB0uNMi2CL0UuQyXtHsJCBuRTJru1tKAaE6BtwZb2clqGsf/mt83X1bhHs1yWd7izxftmordAoWGzsW56Dm7Ze343diFwQun/grD7s0HmcMllkvpXAy7j5GWy2gb+j1Bh52xOrRcFiyhK1RX6EKIfKZLMTQzIOo2lwQbUiCx1aVCn0vkLJdq6hwAIRhpuYymsYfy5RMsQi+FIvReUh61sjqqZVjcWBoQzS0t6AoEeUxfQyC6DbteCI7Ph1o5eculCqHbbYLVi4O0hzxsWBKqeEw5Gn3qPBW3i8sR+tL0Dl6xrSbjVhcHh81GwOkknFYH5LoVlWO0+XW4tCFOsg1ZQdGpYHbT15Umz7G4oag5v9dlt+UJPIeYWSfZbSp0iUQKxwRCny/BvZk635m0gS/xInH/5N79SMvlOLIxGscfB9SFIG1VI85D18Gd7iPt7kI3DGIZjdZAqUIPhcDtVj76ZmMtNrI0hP+W3z8fLpJHs1wAbrlyPf/nmp6q5TjK0WQq9JySHginCEez6HqKNXIv/d7TSo5vcLsJZ9IYZNENY4LlAgX7cLk4TErPktLqK42orgjd4/EwMjIya6SuyucWvhzljYecNjuakcUw55PUNaKZDA1ud95DNyQVCX0+QErJyMgIHs/RA0zTQSYD7mQvdiM5uUIHxhvPxxBOmsYeyW8ry7xc0NAzOu70YVKeLqKmmGgJuCYc19CgCH2rsRpduGgaeyi/bz4Q+lgig90mCLiqE/rZy5t5zclTj/DmFPp4IoOUkiu+/Rg/fHoP6ZGteIRGJHRWyfENbjfhdJqlJ6vfcrnlApAw+wd3Gb3m2PVlu9RVlktXVxd9fX2zUm8kmwWZGMNuxEm7dwMwnk5hE4KYuYAok80SyWRIu904bIKxlJKOusfNCDCSTBF1Ohg3xhFkybgKH67dPj+68Hg8Hrq6umo6ZiZTyHApVuiH41G2DB7myuWr8yor6wgQCZ1D09jDwGcBRejzITNjNiAjhxEYpN1d4Fbfv3KFDkqld/gDaMLDHteZrBh5gJdXfhFQn0cyqYKn9Yre0SSdjd588a1aIOehjyc04pksw7EMTxwc4PWtqsywbHlNyfENbg/7oqPYvSahV7BcdGczGWcL7dle4CxG45kpZdzMFuqK0J1OJytWrJiV1xoYgOymf6FtaBNPnPcyAP/rsUfoCob48MZ1AAwmEnzy0Yd574ZTWRII8IW/beZDPafzqo4lSCn5wJ/u5w0rTuJm4y6aRx9i82t258dvb4dZzhSsG5SmLCqFHgjAjpED/G7PAdY2t7C6ocDYY00Xs3z/F3FmRtBcLZZCL8a4KhmR8nQh84Q+UaGHQsoHbvf7+StnsSH5bbyJvSRNRRkO1zehHxxNsLS5tn3zGr05yyXDsLlidDCRxDW6mWEZIthUmk7b4HIzlkznW+G1NlRWZEnvStr0/QCM1lmmS11ZLrMJKcFmaCW9QDNZA5etcEpavV48dgcHoxFeNFdorGtpAVTQ1G23k85m0ZytOLWhEuN8IReaUhkue8g4F6E7mwE46STYNaDu/Z8ePVhibY01XYJA0jiu8vYtQi9CVPWKSbs7SQtFHpUUeiAANht0BoL8IX0KoEru5lDvtkvvaILuGhO6w24j6HEwntAYKWrqvCyzg922dfn6Szk0e92kdYPD4yq3sb2lst5N+FbRlN4PwJFxi9DrBuXNnTUjm88/B9XooisYpC8a5cWxUTr8/pKiSDlCz7jasEkNh1741cyXoOhMIJ0Gb/KVvN/ocEAgZLCrP4LbYePhlwbAU6iREw2ejm4P5X10TSstoLSgkVQpKrqzOZ9S11KB0G02RepdwSDPp4KMuleTOnA3abPOTz0TeiytMxLP1Fyhg7JdxhMZhqKKeNvsSU629XPIfeqEY5c0q/P68pBqtdW5qBqhr8SrDxEkwVDEIvS6gJQgZCbfRg5yCr30qr00GORgNMJLY6Osamou2ee2O0jrSqEDOLXCKo6FTuiuTL/KnUb54S8eiZLRDf7HxSvRDcnfhgrVJ6XNwXjjBaaPrmCpdBMpxcR2X4iRWBqP04bfVbkmTygE3UGV0vfr+FrWaNsZGFcKP5ms37vG3lFVOH8mCL3J52IsoTEcUxfDN7Yoso5UKEexdJEi9EdfVDG8RY1OgsGJYya9qwA4SRxmJFpfQdEFTejllotmZHGVtZzqCoZI6joJXWfNBEK3k87qZFw5Qi8Ec+v1xzMbyKQl7vQAGZdaztjWBs/1KWJ60+lLOHt5E4/uGygJGvcFzsWbOoA7pcoSW4RuIh3BEC5cPg/DsQwtfnfVtL1QSOWiAzzKmThFlmYz20XK+r3rOTiDhN7oczGe1BiJKSV9YaMi4EDzhgnHnrkyyOKQm7/tG8UmoMHnZNmyiWMmfIrQ1zoG8+PWCxY0oRdbLmoFqMRZptC7iy7RExV6zkNXKztcmeGS8RcqsokwdiNB2t2BwwFNTfBc3ziNPidLm32sbQ/RO5pgcdGCv4dSKhjuHlH56BahK4h0GN0Rwu0RDMfStAYn2i05hELQ5vNx/SmncckZ72Rc+umO/Dm/v14JPafQu5trH7Vt9CrLZTimipotFUNIBJ2LSy2XUAg6W9xs/vRr+a+Pns+v/unVhDxOWluVlSUENJs//6R3BRI7q+1H6m75/4IldAAhtbzlkjEltbOCQgfVFaatLE2gOCgKlkLPQcQGAMi42mloUP7uc31hTu1sQAhBd7OXcFLD21C4Xd3LUmLSQzCsyr/Oh+XqswFbJozuaMDjgeFYhrYKGS45eDxqgdGFXd0sbWjhEaOHlfHHwGztW6+EfnA0QdDjoKHKKtHjQZNP1VwZiqrG2p7UftLuTqStcB5dLthgCnabTbBhSQOvOqklv//kk2HjRli3ThG7tLlIepdxsu0wY3XWtWjBErqyXDJ5hZ6r2VLuoXsdDroCQTa0tE241XU7lOWiOZsAcOiFNdYLWaHb4qrEadrdgc8HKS3LniNRerpU94rcrfVYJkGuT0MsC88Yq2iOqBWOuj7/WqjNBGyZCLqjAbcbRmJpWvzVFToUGoS47Xb+Is8iYIwRjD4D1DehL232TXkF6HTQ6HMRSekMjKcJudx4UgdIeQo+it0Op54K7klOa1OTWrjldBbOb9K7imX0112j6AVN6MWWS65/aLlCB/j0Oa/m3esnem4uu4N0Noth82IIJ06tQOgLVaEbBjiSSqGn3R34/SogmjUkp3Sq8sJdTYrQe0cTmFmgxDWNrcZqmlN7cJjn0VLpYNOU5eJyS0biGVqD1RU6lHZ82u44CwNbvndrrml3vaF3BnLQc8gtLto3EiPkduNNlhL62rVUDHxWQ65uTtK7nMXGANF0uq5+6wuc0DUMm0pNymQrK3SAgMuFu0Jdh5zlghDojkYrbRGVcuhOq8yKjKsdn4984GhxSMmgXKf23rECoSc0jaflGgQy3xfTInSwa0qhpw2NrCGnrNABpLuFXbYNeUKvR4VuGJLeseSMEXqTufw/nNJodqrsq6R3OQDLlhUIeqpoa1O2S8qzFB8J7NpYXd1JLmhCV5aL+sBzXYhc02jT5skROqA7GixCRy0qcmUG0BwNGHYfPh/5NmA5jzTkcdLgdXJwNEFTk/LYE7rGNmMlWew0WD46YN7t6GGy9hBR3VwlOklQFNRq0Jx9EHS52Cw2Eow9j0MbJZ2uvzvHwWiajG7UfFFRDjmFDrDcPopA5hV6rqb8dOB0KvslN0ab0U84Wj8FuqZE6EKI1wsh9ggh9gohPl1h/zeFENvMvxeFEPOgYGcVy8U29Wuc2+4gk1XFu5RCtywXpdD7ybg6cLvVoqJcd/RcsSRQGQ29o0lsNuVRJjSNBB76nKsImT76Qid0XTcJ3dWQL9Xa6p/ccoGCSg863fwtq0ovNISfAupPpfeNKR+oq2lm6hIUf+e6hEpaSHmW4XQeeymEQIB8OeMuMcyR8foJjB6VvYQQduB24HJgPXCtEGJ98TFSyn+RUm6UUm4Evg38biYmW0tMsFyOQaG7i7oaWQpdQdOUQk+7ld0CMJ5rXlBUGnVps49e88fc0qI8dICXbGsJRp8FmSWVUqS2UKGnM9iNJHga8wtjjqbQoYjQXS6ezCzDEM78XU+9+ei5srnFxFtLNBUp9A6pgvVJ7zJCUyupXhE+XzGhDzEYrp/A6FTk6DnAXinlK1LKDHAn8MZJjr8W+FUtJjeTyBF63nIxrZPpKPQc+afzhG4p9GKFniP0cFIj6HbgKAo4dzf56BtNYhiShkZJ0mTuXWIVjmwMX2IvsLBVejaumj0LbyhP6C1TUOi5fOmQy0XMcBAO9NAQfhKoP4UeT6vfXbXVr8eLRm/hfC0yBjCEm4yr/bgJXXc2kbYF6RaDDNfR8v+psFcn0Fv0uM/cNgFCiGXACuChSvvrCeUrRXONLI5FoatcdCsoCqBlDFOhqwwXUC3AGnylOcbdzT4yWYPBaBpd6PneUduNkwHyqXYLmtAT6vtk8zUwEstgE4Ug32Rwu8HvVwod4Ij/bILRZ7BlU3VH6ImMWarWPTOFX1MxR77jQZN2iJSnG4RtWpkt5cgJlbi7my4xzEgdNYuudVD0HcDdUsqKUQIhxA1CiC1CiC2zUfP8aCi2XHJ56OUrRSdDLvMlldULCt1k8oWq0LPREWxSn6DQyxeN5IJgB0cT+X6SALu1RWRtfmW7sLAJ3Ujk6rg0MBJP0+x3TbleeEtLgdB7vadjkxmC0WfrKiMDZl6hD/QLAk51HkJabz7D5XgUusul8tdTnmV0i8G6KqE7FUI/BHQXPe4yt1XCO5jEbpFSfl9KeZaU8qy26eYL1RgFy8VcKZrNKfSpX+M8JZZLIzapYzMS+fEXJKK5RUWlHnpjuUI3g2C9o4l8Fkyj201YzxIN9uQVejw+S/OuQ8hkoTDXWFybkjrPoaWFfGXQV5yq1VpDeDNa/YhJoEihT9Kp6FiRSsHYGPidTtx2Gz5zUZHPp4L1xwOfD3TfcrrEMOOJNNk6SXSZCns9DawSQqwQQrhQpL2p/CAhxFqgCdhc2ynODAqWi/qRaJPkoVdDznLJmB46kLddpFyYpC5MQte9HZgCkXBSK/EyATqbvAhRqtAX+/wkNI1w4HQCsecRhkY8vjDPIwAp00P3qSyX8oviZAiFoMn02weNAAnvSkKRLWTqR0wCEM+ognguR+0zqAcH1f8DLicrnVGc+jgJ39rjUuc5+HyQ9i7DJ9IY6SN1c+dz1LMopdSBG4EHgN3Ar6WUO4UQtwkhrio69B3AnXJ2uzwfMyZmuSiF7pnGrV+uUXS6iNCLV4vOjzNRW+SW/ctAIcl3PKFN6ObudthZHPRwaDyZz3ToavAjgWHfqdiNFP74bgyj/gJ5s4ai0rnjCW1amSBCwLLF6vhoJkPCtwZv8pX6U+hpHZ97huwWtWCZ5aFGLvQrizca7MnfOR4PvN5CLroe2Tt/CB1ASnmflHK1lPJkKeUXzG23SCk3FR1zq5RyQo56vWKi5aIUenvbMWS56Dq6Q+WKLfTAqC1xRP0/pEopSikJV1GXnU1eDo0l85bL8jb1SzvkVmUWFrqPLk1Ct/kbFKFPs3jVkkV2nDYbkUyGpHcFntQ+pGHUFanHM1n8M2C3RKOFFM3r1m/g6rYYEkHcv6EmrfiKUxftiX1E4/URNFuwK0WREpvUC5aLkcVhs7F40dQLBLnL0haBBZ26KCWI5BAZewNOjzqvSS2LlpUVK+ktafQqhZ5UPurqdpUWM2DrRHM0EIhtAxaujy7SESQChy/IWCKTt1CmitZWQdDlImoSut1IqVW8dUToiYyObwYCouU5F4HYcyS9K8k6Anhq0NO5mNA75CA7DtVHS6gFS+gyq77VhVouBk6bjaamqQdM3GVBUVjYCl3TYDR8kMOal13Dqjb8eMIMeFYg9M5GL/3hJGMJlZK3qlMp9KimEfefQiC2A1i4Cl2k1bJ/HUlaN6bloYP6Hjd6XUQzaZJeVW/em9xXVz56LJ2dkZTF0dHSx8HodqJBFRyuBaF7vZB1hMg4mugWgzx3qD4Wxy9YQsdQRJO3XIwsbrsdh2PqH7jbMTmhLzSFrmlgzwwxQpDPPfQsfWNFGSxVLBctK9k7GCPkddLeqBRoLJMhFjiFQGwnSGPhKnSzdG7UZODywPJU0Bp05xU6UHc+eiKt1zxlMZMpFQEObRRPupdY4DQcDko6ZR0r7HaV75/2LGWFY5QdR8aOf9AaYOESeraU0DXDwO1Up2OqHptD2LALQTqroztU6Hwh10TXNPBmR0nam8hKg1vu3ZlX6OVBUYCuRnWidw9ECHmc+bS8qKYI3W7E8Sb3LdgSALZ0GN0Zyp/DpmkqdIDFDS4imQxp91IM4ag7hR7PZGuesjhWxq2B2HMAxAKn1USd5+DxqMDoctswe4YtQp9TCCNnuRTSFr1OpRSmSuhCCFxmxUVpc6LbAwtaoafSkqARBvci/n5tB88eHMs3AKikLjvzuehJGrxOvC47HoedWCZD3H8KAP64sl0Wokq3aWEMZ6EwV/lq26mgs9nDeDqFho20uxtvcl99KfSMjr/GWS7lhB6M5gi9ZwYIfSmL5CAjyST94blPx1qwhF6wXArFubyu6RE6FNVEhwn1XBaaQj80mqSJCLjbWNMeZCyh8fKQYuJKZNTZWDjRIa/6HJr8LqIZjbh/HRJb3kevt6JSswG7FlGVFvMKffqWy9r2IFkpGYjHSXpX4E2+Ul8KPT3zCj0YfZaUuxPN1VJzQk96l+OUadoI5xuhzyUWLqGXWy5ZA597epYLgMfuIGX6AeVNLhaaQj84MIBLZLH72lnXqYplPLVPRacqBUX9bkfeW89lwbQEnMT1DIbdS8K3akETui0TRpZYLtMn9HUdygo8GA2T9J6EJ1VflksiU1sPPZGAdLrw2KGN0jJyP6PNrwWOvWRuJeQUOkC3GGQ8Nve+4AImdHXy85aLzOYXOEznQ/c5nSTyhN6woBcWDR7pA8Ab7GBtewCAZw6M4bSLqqlpOZUe8ihCb/a72Tk8zBeefIJ+Z4HQF+LiIocexnA3MJavJz99y+WkNj9Ou42DkShJ7wqcehgjPnr0J84CDEOSyNQ2y6VcnXf0/xy7keRQ5weA2mS45FBM6F1imFhq7tf/L1xCN9SP5IWxCN/b/izhdBq3ufzY7VZddKYCv9NJXFNjLfSa6OEx1XrOG1pMW9BNg9dJNK3T4HVWbQCcI/ScQv/Ypat5w9puDkTCPKN14kkfxKGNLzxCNwzsegTpChFOanicNjzO6StZp93G6sUBDsUj+UwX2/i+Ws/2mJDUFAEGauihlxC6zNJ56AeMN5xPPKBiMjNH6IPELUKfO+SCoo8PHOHJ/sMMJZK4i34wU/3gA05nvjnDQrdckhG11lr42xBCsHqxUumVFhXlkAuM5rJgerob+dfLT6EzEGSXoWrC+eM7SSYX1gXSSEURSPA0MBbPHFPKYg7rOkL0RiMkvScB4IjWB6HHa1yYS0oYL0oHbxn5I570Qfq6PpDfVmtClw4/GWcrXWKIRHruf/ALltBzQdGxjMHVa9byqs7FXLJmUX73VG0Xv9NFLFOs0Beu5ZJNqmpI9qDqsLBqsfLRJyX0xlJCB1VYqtnr4RlNld0PxHZgGKXe6ImObDKq/uFWweVjsVtyWNcRYjyV4YhQ329X/GBdiI1ErnRujRR6NFqa3trR/3PSrnZGWt4AFMre1gpCqLt5VUZ3OF8KeC4xM1Xl5wGEGRR12D38/bIVtDafzMaNhf0+H4yMHH0cv+mh5/uKZiMgsyDsdfGjmS2EkxoebRScYAu2ArB6kVLokxWVyvWSLG5P53ZDq8/LX4a9ZLzNBGLPA8pHr6XCqmcYSbUyRngCVWvhTBXrOtSFdX/cIGvz4dRGVGepo3ezm1HUWqEX2y3OzAjNo3+ir+tDSHM1eC0DojnkbJdu2+b8BWousWAVeiyl0ibWtLbjsNkmeOa5bjtHQ8BcdpbQtIoldBcK9g/HaRYxdOHE5VdEvnoKCn19RwMuu42VJvnn0NnkJZU1iPg34I/vBBZWpks2T+hKoR9LhksO681Ml95YhIyrFac2VBeZLolMrrlF7Qm9beh32KTOkcXvyG8LBCo86TiRI/ROhkml5z7Bf84Ifa7J7pURVWtkfZsq81pO6FMtsel3KbKKlRC6sl0WkkIfi2s0EyHlaMHlVgHQqVguS1t87P7c69mwpKFke3ezkuKD7jX447tBZusqMLr1wCibth+esfGNlCJ0mycw7dK55Wj0uVjS4GHXyBAZZxuuzFBdLC6KpXPt547fB8lmIRIpPG4fuJOY/5R8MBRmktCX4RQ6rtRg7V9gmliwCj2jKUPW71Kfcrm3NnWFrn5ocS1TtPxf+Z9zfdGaTYTjOs0iQsbRkm9s0Rpw8bYzu7hk7aJJn2uv0FZt+SJ1f9znXIXdSOJLvFxXhP4ff97LrZt2ztj40iR04fYxnjg+ywXg+vNXsGN4hN6MB6c2XB8KPT1RoWvasaWojo0VBJQn+Qqh6BaOLL6m5JiZJHSAhvTMXeCnigVL6IauCN3hUEZiuUK326fm1/qdBYWetZsXh6xaHbmQCD2SzNIsomScBUIXQvD1t/dw0erptxs8uUOd/L2oVDt//Pm6IvTd/RFG4xni6ZlZTCLT6juUcfjQDXlMdVyK8b7zV/D6Vd3sijsgdaQuFHrBQy+oqf374fnnp1+7pzje1TLyIABDbVfmtwkxdZE2HShCV9lYIa2/9i8wTSxYQs9mFaHbqxA6TO0L4C9S6AVCV+pqIVkusaROM1F0V2tNqtl1tniwCcGLRgeGcBCI7aib1MXhWJrBqPr+9I7NjLEv0+ouL2bW6z+etEVQF9dPXHIKUXsTXn0ELTP3JzJhXgz95sKiZBIOH1axkj17pj6OlKWE3jT2CEnPclJm3j2o3/JU15ZMBx4PZFzqDtSvj87593PBeuiGbt5zmitFj5XQc0HReEZDd+QIfeFZLtGUTrOIYrhaj7sBLygbpsXr4UjaIOFbQyC2AympC6tgd3/BrO0dnZnbhqxpudyzW8VjjtdyAQj4bTi9S3Cgk03Mfd2RuBkUzSn0ffsKv5mRkakLomi08L0Qhk7j+GOMNV1ccsxM2C2gMoWyzkY0nISM0Tn/fi5YhS6z6szninNVyk+dCqH7KlguDn3hKfRkIkVIJJDe1pqNuTjoYTSZNGuj3MUCPQAAIABJREFUqxIAc/2DgXJCnxmFHo+OYUjBtx9X5RSm262oErxeCIa6ADjcf+C4xzteJDI6dpvA7bAhZWlTCsMoDXJOBrOXCgDB6DM4spFZI3QhwO0RhEUjjcbInH8/FyyhG3lCr67Qp5LpYhMCn8NBfIF76EZS3fNKb0vNxlzS6GU0lSLu34A7cxiHNloXi4t290dpD3nwuewzZrlkEhESuOlsUqpiUfD4k8Y9HmhpXA7AvsMvH/d4x4t4OovfZUcIQSw20Tcfn2IToHK7RSIYa7qo5Jhg8DgnOwm8XgjbmmiQ43Mem5gSoQshXi+E2COE2CuEqNgIWghxtRBilxBipxDil0cbc67Jzigrn1vNcqlSgqQEAZeL2AL30B0moeOrnULvavYwmkoS964EqJvSr7v7I6zrCNLd5Jsxy0VPR4nj4fZ3nsGmG89jWcvxR/RcLgg0KoU+OHTwuMc7Xqha6Or3V4m8p0LoyWRprfymsUeIBXrQnc35bTbbzCl0UBfKiK2FJjlW/4QuhLADtwOXA+uBa4UQ68uOWQV8BjhPSrkBuGkG5lpT5CwXw6Ysk0qEbrNNTaX7zXou0ubEEO4F6aE7Mub9cg0JfWmrl6yUDDpUFoEv8fKcE3paz7J3MMa6jhBdTV76ZjAoGpceFoc8nNbVWLNxbQH1+cQih9Czc6s4VLci5XWGK1j6kcjRRVGx3WLXY4QiT02wWxYtqu2S/3J4PBCzt9DM+Jx31pqKQj8H2CulfEVKmQHuBN5Ydsw/AbdLKccApJRzn2F/FMgyhV7tA+/sPPpYfqerqEBXYEEqdJdJ6MJfS8tFpS72sRiJDW9y7gl972AM3ZCs6wjR3eyjbyyJnIkrtxYjjofmGnjnxXA2KEIPGWO8MBCt6djTRSJdUOiVCN0wVMBzMhQTekP4CWxSY7T5kpJjpvIbPh54PBC3t9BCGF2b2x/9VAi9E+gtetxnbivGamC1EOJxIcSTQojXH23QOVevZU2iq6U0tbeTz6uuhkBRCd2sPbAgPXSvptZdi0DtFHpHg1pcNJKRpDxddUHoLx5RDLOuI0hXk5dYWs83oKgl7HqMpPDhctQ2zOX2u0nbgrSICAPhVE3Hni5yCj0Wo6pVMZntksmUBk6bxh4ha/MQCb06v62hYWb9c1CEnnC04BAGemxua83X6tviAFYBFwPXAj8QQky4TxRC3CCE2CKE2DIyMlSjlz5GGLmg6OSEbrMd/QrvdzqJmd/IrD2Qz3JZSITu0xWh2wNNNRuzyW9mEGUyJL0r8SX2zjmhj8TUBNoCHrqblR83E4FRRzaOZpti/YlpwOMBzdlKi4jQPz63EWbVrchRUZ3nMBmhj4yU/saaxh4m3PBqDHthReBMq3PIEbpaPKdHB2b+BSfBVAj9ENBd9LjL3FaMPmCTlFKTUu4DXkQRfAmklN+XUp4lpTyrpWX6qwdrirxCV17LZIsOOjuhbZLp+p1OEpqGISVZe4BUapQ/7ntlwVguUkLAGCcqQrg8tSvgmStIFdc0kt6T8SZfIZ2a26tkNKVM0oDHQXeTSegzEBh1ZRNo9tovbfR4QHe30UKEwcjcEno8rboVTdYAPByubl0W2y2u9BEC8V0l/rnDAa21u2GsCrcb0i71QtnIkZl/wUkwFUJ/GlglhFghhHAB7wA2lR1zD0qdI4RoRVkwr0w26FyrVyF1dBz5NJbJgiYOB2zYoP4qIeB0IVEVFzV7gPH4EPe+/NKcv8fZgq5DUIaJ2hprsko0B5/LjtNuI6ZlSPhOVqWJE8NHf+IMIpLSCLgd2G2C7mZlCc2EQnfLBNJZe0J3uUBzLaLNFuHIXFsuadVPNDXJNKr56NlsaXXFxvG/ADDWVPDP29pmZnVoJehetVrUiNc5oUspdeBG4AFgN/BrKeVOIcRtQoirzMMeAEaEELuAh4FPSCmnUE18biClBEPHEAUWn8oH39YGHR0Tt+fqucS1/5+9Nw+TJL/LOz+/iIzIjDwqj7r6rL57ZnpGo7l0zWiEBgkkIRCDtCwShjVeL3jBWrTGNoYHFmNAXhs9Dza2hVjZ2NwCIUAI6xhAaNAx0mjuo6e7p+/uqq67Ku8jrt/+8YvIzKrKqsqezqM1xfs880x3ZmRGVXTEG2+8v+/3/TrM2xoxv0rD8/C8ncHojgNpWaDaY0IXQpCNG9Q8pdABopXzQy0NK9VdUsFTSCpmkIkbfWkusmQNafTe/DWM0HIpNeMLhoWq7RE3I1sSOnS2XVZW1ir33MqXcCJZysk7m6/t2tWjH7QbJNQjvKgM13Lp6vlYSvk54HPrXvuFtj9L4KeC/2562J6v2p/bfv1u7+RHjqiTqb3BJST0gt3gXMXjdaKOLyW25wN9rJe6SdBoSDIUqeiHe0rooGyXmm83CV356G/s+X66RanuNAkdVDfrYo+J0fM8EqKOFu09oZumIvRJiiyXhpd2JqWkYrskomsV+uTcH6J7Na7t/cfN11ZX4cCBtZ9vt1uQHrmVv2Il9+0g1IUci6kF0UHBTOawpY425AjdHdkpWrM9Inj4osUK3RJ6JLLxzh9G6P7BqZOsuAYjQl3gYYD/qw35qs0Tl1qr+aUgabGu53pOtJm4QcV1qMem8EVk6JUuSqG3fsmxlMlSubeEvhpIUj3W+24YXQcnOo6Oj10Z3kN0vuogJYxEzbbY20vccuZDHD/7U+ya/YPmtuvr0deHcY0Un8B0lpqj5kDVng8SqUSEJdJE6sMt9tiR4VxV2yOCiy9aSut6Gg/Wd52FCv1yscju9G7i1ABJzRlyl0Ebzs6XuLBY7sl3/fZjl3j/x79Bqa68j0LZJUsJ28x11Vl7PcjGVReu1AzqsQM3BaG3j8sbTURZKvf2B1pZUeawYfVHYvpB3o5WXexPDX0XCO2edNscvEMXfwUpdPLpN3H85Q8xUngCUGTebrssL6+NCRhb+iy+MFge/Y7ma7lWo+hAkLR0lmSaqL1DCX2YqNoeBt4aQr+exZP1da2ZWAwrEuGdBw9zbOIAGj4x7JtKof/LTz3Pv76OgQx1x+Pycufyg9l8Hc+XzZrscn4FQ3i40d5fRdmEQTG4cdSsw1i1i0PNcynWnbUKPRlludcKPVjtsxIjPf3eJuKq+SslCxRrwxEdoU01YihCT5aeZXLhT5je9xO8eMcn8PQke6799+b2Z8+qhVDXVX9ux9jS58hn3owXTAzTdTVofJBIxjRF6M7iUMXqjiT0mu0REV6zBh2uj9BjMdZExFqRCP/527+D9996G15EsX2C+k2l0K+sVJkvdl/V8PEvX+Cd//ErVO2Nv8NCSX3PqVlF6I2iUiV+rPc1YmnLpFCzEULSiO4m2pgbukJv99DHUiYV26PWw5t3MZCj8VTvWv7XIK5uvFlRbv5bDhqLZbXfZDCPYPfs7+NpCa5MfQjXyJLPPEgm/9Xm9rUanDsHFy6sXb+KV14mXjvH0ti7m69lMoOrbgmRiiuFHneXhtr+/y1hufi+ZCbfuwWciu1i4CGDHBchugvhasd62yUSnEFeUDucEHVqzs2h0Gu2x0rFZqXSPRO+OFOg5ng8P72x62MxUKSn51SbnlMKHjPjve8tyMYNXF+iRV1sc5cacFwfzhUjpQwWRdsUekIRUi999EpJEXpypD+ELgJCz4jy0CpdQoWeCJqA0oWvUUi/oamy85kHiTWuEKu1Yn5nZ9UAjHaMLn8WgOXRdzVfy/aut61rJOM6i6SJuytDbf//llDof/7MDA995FFWr4OQtkK14aHTUuivJLhns/S2MHExSe2mUejXCupmuFKx8f3u7qTnAr/9mSsba8bChpTTgUL3K4rQIz1s+w8RNhe5moNtTiCQ+OXh1KI3XB/HkxsUOvSG0BdKdV6cKVAvqxtlqk+ErqeU5ZKj1PMKnW6xUGxgGTrC1Yk4KyQrL5HPPNB8P595M8Aald4JY0ufo5R8LY3YvuZrwyD0VOCh67i45S5zf/uAbwlCf2GmgO35TWK6UVSDKhe2iM7dDtsReoI6NffmUOgzq+q4+RLyte2LuBuux+VlVVv99JXVNe95vmyS1+m5ElJKRBCda6R6X1oQTuqxhY1tBuVFQ2qvDr38EWuthw70ZGH0V79whvd97DEWV9QNy4j3J4QkYsXxtBgZURqi5dJgPBWl0RBk8o8BUEi3CL2SuBUnkiNd+Nqm32HYC4wUn1hjt0Sj/Zkduh0sU6Mg1Y5ldQcS+vVYLucDtbjco2qC9ZZLbwk98NBFnfpNYrlca7OrVirbK7JLS1U8X5KKRnjmyuqaSoiVio0v4dZdKcoNl+nVGnpdEVAs3QfLJUgbrEunSeh6dTjdeOEC4poql+TmlstHHjnNZ57rfhL8izMFGq7P9JJ64tGs/jCTYYATyTEqKkNr/18sKUKv11VKogrVuqe1gdDIZx4gk//Kpt8xtvR5BJKlsVa54jDUOUDM0CmhoiC8apejlvqAbwmFfn5BEXqvfMpqQ5UtcgOEvtnQ2aaHTp3asMORA7QTejc3xXPB8f6eu/awVLaZXm19PlR0bzmuyPvkTAnTXaYmTZJ9KC0Ip93Xfbs5jFcbEqGHZZqpNWWL6obTqdLlDx+/wmeeXR971BlhzvpILEISdbx1qz9TGUwTXCPHuF4dmuWyWGowGo8ipbJViiOvQ2prpzLlMw9i1df66O0YXf4c9egUlcQdzdcyfVpH3g6GrlEVgUKvDW9e601P6JWGy7Ugc6JXCr1qexjCQ2wxfm47CKFGT61HOCg6IWo3jUKfXqPQtz+GZxdKCAHvu0f5ku22S7iI9sBR5Ze/NFMk5qywQoqReO+7YjNhQJfrYJuTgApi8oZwaMNgrvZF0Zihk4pFNlguruezWnWY67Ky6Oy8yln/2e+6jVFD3Ti0aH8IXbX/ZxkVZeaHpNAXSg2yVhTdLZAsv7DGbgkR+uidbBfNq5BdfVSp87aKhmEpdIC6pv69/NrfK/RNcXGpVQvdM4UeeujaK18Uhc456aGHnhaNm0qhh0FSy10RepmpXJzX7ksTN3Wevtwi9FDRHRpNMJWL89JsEctbZVWOEI/1/nTKBH51sW4TsWI4kQymPT+UPJcWoa9NzBhLRpuVPyFWg4z0uUJ35+xLweDpNxzK8T2HR/G0GOi9S65shyL0UbJD8tAbrkeh5pCJRUmVnkHgU0i/ccN2lcRtOJFcx4XR3MqX0P36GrslHlce+rDQCOxWdiKhd+uhh/65EL1ZeAKo2S7mDXroQMc295DQRzSb+k1D6HVes1eVg3XzlHN+oczR8SQRXeOOvWlemGk9QoaEPjES5bbdKc7Ml4h7q+TFCKbZ4zZRIKJrpGIR8lUHywLbnMS054ZS6xtaLiOxtf/wY0lzg+USPgktVxo4XYx6OzVbxDJ0DowmiMs6fqR/QzBVnkuOEVnacCMaBMLrOBONYtUuAVCNH9244RY++ujy53AiaQrp+5uvDVOdAzhG8G/W2IGE3i3OLZTRNcEtk6ktFfrHHj3Pz3/6ha6+s2p7RIS/7XCL7dBJoUvNxBcmI1qDuusNPULX9yWzhRoHRhOMxCLbLoq6ns+FxQpHJ9XJeWQ8yaXlVprgQrFOKhYhZujcsSfN1XyFuL9KUaT7FpiVjZusVu02Qr/5FPp6sRESvJR0Vet9arbIrbtT6JpAOJXmWkw/oOvgGTkSski5bg98tuhiW9t/rH4ZX0RoRPd03Lajjy49Rpe/wEruHU1RBsMndC9Mx6z/vYe+Kc4vqsf/3ekYy5uQ0dfPL/Orj5zm089c6yqbomZ7mMJrzhPtJaGDUukjok7DGz6hL5YbOJ5kT8ZiNBnd0nL55JNX+ZXPnsL2fI5NqJPz0FiclYpNISh3DMvNAG7fqxZBk16eosj0bRBvNm6wWnWIx8E2d2Ha80NR6MW6gxCQMNcS+mhyY0BX+3HebtSblJKXrhW5bbc6nsIp4/chOrcdfiyHhk+KKsUBN2otBOsKqUiMWP0Kjeg+EJ1Pnk4+errwOKazvMZuEWJ4C6IhTNOkirUzFXrXlstChSPjCUaTUZZKG8moUHP45598Fimh3HCbxLMVqo6Hgb/tgOjtsJkidSNJkqJOw3WHPrUo7LDdm1EDhzdbFK07Hj/9qef5na9fIhWLcO8BJXcOjCqleClYy1goNpgICP2OPWmi2CRFjYreP3mUiZvkmwp9Qil0e/B3ylLdJRmNoGlrraWxZJR81eHRMwt8x6/9HaW6s8aC2S5y4VqhTrHuriF02UfLBUBaQXORKLFaHWyWQmjzJCNKoddjrWxcTVt7PSofPUsm3yL0MIxrJfe25mvp9No4jmEgbupUSCDsv1foHeH5kotLFY6MJ1UIUqWxQYF/4cVZrhXq/OiDh4DuxoE143Nv0EPfXKGnSIgGdc8bPqGvhoQe35LQi8GN8Je+9w5e+MV3cGhMEXn4/0tBUNdCqcFESrVrT4zEOBJT31+OjPbtd1AKXRF6I7oL3a/jVgZ/0RTrzgb/HFrNRf/6Myc5u1Dm3EJ5zXHeTqGfuqYU3YndSpVrbhm/D9OK1iDMc6HclyHXW2Gx1EAIiOsmVu0yNatF6Hv3wokTbYUrTR/9y0oFSo+xpc+Sz7wFL9Iqkx3t3+nXNSxTpyLiaPYOVOjdYHq1iu35AaGbOJ7ckA4XnozvuF01nXQzDqxquxjC7YuHDqoWPUGdhucO1XKpOx6PnVNdnHsyMcaS5qYLy+GTTdpaS1hTuThCqGojKWWzISTEa5Lqc1Wjf1dUJm6Sr7QWRQEoD36QwPpgrhBjSXUihN2106s1lis22biBGdG2VehhJdfR8RS+D7pbRhr9Vei05bkUagNW6KUGubgJjTqms0A9NgUohX3ggCLnw4db2y+NvRurfoXds/+DfdO/iVW/yOzuH17znYOYHbod4lGNkoyjDVGhD+0hpRuiCytcjkwkmsp7qdIgHW+RTqXhIgQcm1TqpptxYDVHKXTnBj30zSwXT0+SYJa6OzyFfnWlyvf9xmMslRvcs2eMVMwgl1CLi74vN9gGmxF6zNDZk7a4tFSh3HCpOV7TcgE4GrOhDI1o/66obNyk1HDx8fHjAaGX5lCjaweHUqDQbXvtzTxU6Jm4Qb7qMJOvsVy2GU1GSbk+c0UVN3x2vszBbArTXHvOXV6pkLYM0nH13RG3iBvtb/6rllAWWZYSq5XBKvSFQBRoxasA1GMHAUXmoW2ye7dKVpQS5iffz+T8H3P03M8BkqXRd7I4/nDz++Lxzj0hg4YVdIvqzt8r9I44v6CUy+GxJKOBClpaVzFQaihfM20ZpC2jK4UeWi43Es4FWy+KWtSGqtC/eGqepXKDf3bvffzsm14PQC4RxfNlM5OkHZsROsDBsTgXl6vNao1QoUsJByLq38iN9b7tP8SRCWU/PHFpBZLqSUwMYRhvseaSjEU4c2bt67szik3+8QOHyMQNplerrFRsRhMmu0ZizBXq/NaXrvLOX/8y//2zSxs+f2WlxlROtY27LkTcPDLa3xU+LameqLKi3FW+Ty+xVG6QS0SJBpUroYc+OdnaJhJpy2QRGmdu+Q18zUAKg5eP/4c1zUQ3g90CSvwUsdD+ntA74/ximdGESTZhNlXQ+iqNct0lFVW39f05qysPveqotMUb9dDXL+CEcCMpLFmjMUQP/fnpAjkryp1jE1QqAsdpa1Pv4KNvSeijCS4tVTg7r56YdqWVh27bsEdXr/nx/in0t982SSoa4VNPTSOCALBhEHqp4RAVEZaXVT53iL0Ziz//ifv58bceYV/WCiyXBqNJk8l0jOnlOp99Vv28nzh9itk5yVxbvtiV5QpTo4rQnVoD3a8hY/0ldD2exkcnJ0oUBrwouly2yVkmsXqL0C1ro0Bqr1ppxPby7F2f59m7/if2uhLHm8FuAUXoBd8i4haGdt3f9IR+ZFx5iWObhCCVG0o1AezPxrm6WqVme/yzP362WZmxHjXHRZfuDZctwmbdogliskrD83C94Uj056bzHM2lEYGSKRQgFxB6p4XRrQj90FiCQs3ho186x+50jNcdVP5rrQYZuUpJjHD3gf6NWI8ZOt/92t184cU5bDOJL6LotcEnLpbqLhGpjs/s7Nr37p7KEtE19mXizAQeei5hMmpFWSjXObW8xK54gqulIl+bmebcOfWE4/mS6dWWQvcqQVJfrL8Tjg1T4BpZxrXqwBX6crlBOqaaijwthm1OdCw5XP9aJXk75dRr17xmGIOfTrQZYoZOXsaVZTaknsKuqEwI8U4hxBkhxDkhxM90eP9HhBCLQohng//+j+2+szsPvcKRCUXo2bjRsVu0HFguAPtzcaZXa/z1qXn+/JkZ/uZUZxVXC2aK9ovQfc3CkI3mvgaNUt3hwlKFw21XxOoqTduqU7doSOidFv0OBqWLL8wU+N/edBBDVwesXoeoPY9m7eGtxyc3fK6XeN89+6jaHl+fmcMxRtHqq9t/qIdQwy1cNFcdn/n5zufw3qzF1dUq+arDaCKKqMVwfB/b9/nAbSc4nM7wlxfO4bpQLsNsoYbrSw4EhO5XgwU1q79dMqr9P8eoVmnGFAwCNdujYnukoyax+hW1ICpER0JPd3FPGxu7/uE0/YIVKHRN2rj14cQSb0tlQggd+CjwLuAE8AEhxIkOm/6xlPKu4L//1s3OtyL1cMLOkXFFJhFdIxff2MBRqrskAkLfl7WwXZ8/+IZ6lLu8vNFPt10f1/dveMBFiM7t/3EMbDR8ykOYrvPCTAEpYSrZuiLyeTXQGOjYoFWoOSRMvUnW7TgYlC5ahs4HXr+/+XqtBqa9iG2O961LNMS9B7IcHI3zpQvXcIwcuj1YQq85Hp4viQY2XaMBKysbt9uXtagHE2t018RC2VOmpnFbbpR7J3exUK1ScRzyebgSnKOhQg+ztIXVX8slzHPJiXLPBsd0g/D6HTHX1qB3Im/D2DymOsTNYrcAxAyNYhihWxmOj96NNn09cE5KeUFKaQN/BHxvL3a+lc/UqnBp/YuOdsjMKDdapWT7s+pgPn5RXWmXO1S8hAuiwA2XLcLmCh0ghk2lMXiF/kIwNm5/okUKlQqkYyaGLni2wxSiQs1ZY7fU63DpkvrzVC5O3NT5/vv2NdMPw21Mex7bnOw7oQshuGt/hmvFKo6RJeKsDixx8fximd9+7BKg5seGmOvg+uwLzkEAu2SSjSlCv210DFPX2R9MGL9aKipCD87R0EP3m4TeZ8vFADeSIytKA61DD9dvkobZJPRYTM3p7YStuj91ffjt/u2IRyOUpLr2vepwShe7obK9wNW2v08Hr63H+4QQzwshPiWE2N/h/Q3YSqGHGehHx1uE3ikzo7LGcmnVLuUSJlc6TK2vOm6T0H2tXx66ujgtGlSHYLk8P11gb8Yite6HcxoaP/TGA/zp09Ocml2rIIo1pzmJx3Hg+ecVoZ89C2ZE43M/+SA/9+7b1nymXkfN+DQn+k7ooKp0CnUbN5LBcFYH4lOuVmwe/i9f41e/cIakGeHASItol5bYkCmzN9M6BxN6lPF4HF0I7ptUawz7U8rwvVoqUiiop0hDF+xOq8/JmiJ0LT4IhZ5lRA62UzQUZBndw3DzNKJ7tyTtrWyXXG7ww6C3wu6RGEXU06w/pCEXvTocfwkclFLeCfw18DudNhJC/JgQ4kkhxJP5/OK2Cj0a0diTWUvSGxR63SUZVWwSqqOEqfPeu/cyvVrbEDw0X2xg9FChdyKyUKFbwqbSGLzl8tx0nhOTG6+SSgU+9LZjpGIGv/LZl9Z03bYr9AsXoBo83MzMKK/34FiCaGStN9UoVYh45QESukG54dKIZIi4qwMJ6Pqtr16k1HD59D99gE98/3euWZeQcqNK35ttna8jpkkmGuNX3/IQb96rsuUz0Sgpw+RqqYTrwoXFKvuycfSwLyAIduo3oQsBXjRHShYH2lgUrt9kXfV7NqK7trRVtiL7m8luAdiTsZoKfVhDLrqhshmgXXHvC15rQkq5LKUMmfa/Afd2+iIp5cellPdJKe/LZMa3VuiLFQ6NJVonOqpxoz2rxfclZbtV5RIzdPbnLN5+YpJjk0lcX3Itv3Zx4vJyBX0dod+Ih95ZoYeWS2Pglsv/93fnmV6tcefu3Ib3ymXVdfmhtx3ja+eWeXGmpSJCQpcSFhfXfm55eeN+fB+oqG5NLzYxkIWpcBxdVaQxnBVcp78VRPmqzW8/dol3v2Y3d+3PYNsbf8nZWeWnA3geLEwbxANbJnxCGrWsZrWREIJ9qRRXS+rYX1qqsj/XsmlEPVDoif5aLgB+bBQDG69RGVji4lKwfpMK1kAcY2JTuwU299GFuHnqz0PszVrNMXT+TWy5PAEcE0IcEmrEz/uBz7RvIITY3fbX9wCnutn5ZgpdSsmZudIa/xwgY5kUak5zcn3VUWmGYR06wCd+9I388sN3MJVTjz6XV9baLpeWqm0K/cYtl84KPbRcbKr24BT6Hz5+hf/386f5ntfu4buOH9jwfiU4FG85rqTNhaVy872Q0FdW2GBlLC1t3JdaEFWE7sd7Pxy6E8I6+oqeQpM2Tm37JrIbwe9+/TLlhsv/9TaV1d3oEPZZrcLjj8OZM+r/MzMwasURQHKTzrOp1AgzpRK+lMzkq0y1WYWinsfT4uibda31EH6slecyqMTF5bJNwtQx60ol2ObWhA6dVXomM/wwrvUYiUVwDWWpyfpNarlIKV3gg8AjKKL+pJTypBDil4QQ7wk2+0khxEkhxHPATwI/0s3ON1Pof3NqgZl8jYduWUsUacvAl1AOSDKsIEm2ldrty8YZiRkcHFOkur7S5fJyhV1Jtf2NNhbB1grdokFlgB7673/jMnfuS/Nr/+trcTqoyZDQQxurfVZoseaStowN6hygVFJNRO1QC6KK0GViMISeDRZky8FkGL/S30qX03NFjownuHXXCFJuPAYhfF8p9fD9McsiaZhomzxgBuEoAAAgAElEQVS27EulsH2fl5aXKNtOs8IFQLPzuJH+RRGvgRUQ+gATF5fLDUaTUbSaKil+pYR+s9ktoJ6+EiPBk/GQCL2re5yU8nPA59a99gttf/5Z4Gevd+edFLqUkl//4sscGI3z8F1rO8JCj7dQVZka5YayX5LRjb/GZCqGGdGaVQQhLi1X2D9iwko/FXpQ5icGq9ALNYc3Hh7F0DU6lcE6jiKduBlhNGE2Cd12fWqOx0jM6KjGQan0PW3/HGGFC4BIDobQw8aoUnB8/fIqygHsDxqOj2UqZu2kzjfD26YOcGILPyBcGP3Yc89gahpvPqQWTH0fqOdxjAzRQRB6mLgoBpe4qELLTMzGIhKBjI9tq7QzGWWxtAvAm81uCZHNZPHLYmiZ6ENdI+6k0P/29AIvzhT5pw8dJbKuJjqswgh99HCCTCdC1zTBVC7O5XWVLpeXq+xLqe/pxaJop5Ox3UOvDXBQdL5qk4m36qQB9sz8V+5++h0g1d2zHLgsqkVd3ezC42lgbFo5st5HD2vQJQItNRi5FHro+eZ09f4q9LrrNReCr4fQ7xgb5zsOHNr0/T3JJJoQVByHHzpxB5lgOtHKCuhOAc8YzKQGkQjyXCgNbGF0sdQgE4uqTHtjlFh8e00Ziaz10UdGNi9zHDb25hKUpYVo3Lweet+wXqG7ns9HHjnDVC7O9929sTIyJKswu7vc2Gi5tONALr7GcinWHZYrNnuSIaHfuEIXYuOialjlEh+gh+54vurAs1qEbjauceT8vyZd/AaJilrWCG2XvVmrOfwiJHTN3bxUZXV1rbceKnTHyGF2yAjvBzKW6hZekUETTp8tl4bjEzPUyRESeqx2icPn/x+Ef50EKCXxystkV77EiH2ZO0bHeMu+/Ty4dx+F4NpfWgLDyeMZ/V8QBdASrQjdQSUuLldsRkwzKHed7JqYjxxR15oQcLTD+NGbBXuzFkXiyMbGXo9BYKjLCusV+ieeuMrpuRIf+wf3dOxYTK9T6GFJYCeFDqpZ4+sXlpFSIoRoduXtSqjte6HQQSmI9iaXsA49pbkDq0MPj0kmiGD1fTh84RcRMhjIm/8KleTtTULfl43zxVMLSCmbn5UNAza5wHxfEc6uXYrY83mYDC7KAazfAapbOG0ZLMng36/aoVWzh6i7a2+QAPumP8a+md+kOHIfS+Pd9dfpboFbT3+Q8SVVS+BpFj9/76NUE7cC6lhKqZ6CDrp56uYdvf9lOv1cSdWVk6M0kDwX35esVGyShhk0pI13TeiZDOwL3LWbJbulE/ZmLEoyTro2HEK/aRT6XKHOr/3VGd54OMc77+gc9BReXPkuLBdQGSRV22sOpQ2n7kwm1Pf4mtG8698INlPoKd2h5gxGoYceaNoyqNchUT7Jrvk/5ur+n6QWO9CcnF4qqe33ZS0ars9iudF84gmDpzbDQjBTYnZWkXq0MaNmfA6I0AFycZMFV7GA6HOeS93xiRqt3Bqkz9jSXwKwe/b3uvqOiLPCvU89xNjSZ7l48Od47s4/xdOTnHjpH6F56gmp0VD17I6DSurrc3RuCCMawdFHyIryQBIXCzUHz5ckI1FMe+G6FDrAoUPqv5sZw1boQ/fQa7bHP/of3+T+f/dFyg2XX/ju25s1u+uxXqGHlkunQClojU87v6iIPLRfJpsKPdKT+un1Pnqo0BOaM7Aql/a0RNuGZPl5AOZ2/SD5zINk8l8F6VOpqIXRsKNxZrXW/GwisjWhr66qz05Pq79bNdW6PYimohC5hMmSo+Fpsb4TesP1iLV56KnS08QaM1StY+RWvki0PrPNN8CR8z+PVbvEc6/9Cy4f/GlWc2/n1G2/SbLyEgcu//vmdhcvAtIj4hWR0cFYLpEIuMYoYwMK6AozhCxhYtoLOEb3Ch3Uk/TN1BnaCXsyFkUZR3N2qId+bqHMl84s8n137+PzH3qQE3s2f56KmzqGLlqEHij0xCYK/WhQx34uyIW5tFRhIhXFRD0aSGH05ARZT+hSGEh0kmJwCj1c1MrETRwHYnXFuo3oXvKZt2C4eZLlFwD1iB921U63E/o2zCwlnDqlyE13ixjuCjXr4EAVejZhkq/ZuJEsWmNwCr3RgPHFv8AXEV468V8R+EzOf2LLz2dWH2X33B9wZeonyWcfbL6+mns7K9m3Mbb0+eZralKRIoF+Z6GHaE9cXBmAhx7GdoxIG92vYZuTN8WkoV5iMhWlRBzTKwwlQnfoCr0UTM/5/vv2cXQiteX2QgjSlrFGoccMraPfDrA7HSNu6s1cmMvLVRUF6weecZ8IHSHw9DhJzWmWVvYboeWSsYzADrmKbYzj6xarAZmEtks+32pRn16tNRfE4l1I7dWAQ9uHEwzaclmt2ThGFt3ub0BXw2mrcqlLxhc/w2r2rZRTd7OaeZDds7/frB5aD82rccuZ/5uqdZjLB/7VhvfzmTeTqJ7GsFt1ooajDm6/kxab+wsIPTugxMWw7X8cVdLXTQ36txoiuoatjxDzSzuP0H2fZofaZrbJeoxYBoWAvEptwVydIITgyHiS84tlpJScXShxaCyBDI60FJH+EDrKR08Im8qAqlzaLZdQoddjahXJju6hah0hk/8aoEg5GY2QiRvM5KvMLjuYuk6k7WDsv/IfuO+JN2NVz3fcn1W7pL47cXAwTTABckmTfDUM6Mr3Nc+l7iqFLiXoxQtY9Ussj34XAHO7fhirfpF04bGOnz146d9h1S/y8vH/iK9vlKH5zAMApAtfb74WKnQGTegMJnExtFxGUf6yHx+/6S2UVwLfzGDJCq4z+LFFN41CT0W7M2LXKPT61oQOynY5t1Dm/GKZ1arDvQeyTYXua/0jdE+3iAuHyoAXRUcCQo82pmlEW003pdTdTculVlMWQjgubX7VWeOf75r9PY5c+EUSlZPc9ey7sarnNuwvVOjeyMaIgX4iFzdxPBkEdK30jdCllNiuTyyi02hAtK4CR6txNZh6cfx7cPWRjoujifIL7L/6n5nd9UPks9/W8ftLqbvxtNiaG0LEHUx0bghNA88cJSWL5AewKLpUthFAWqonES3V36EoQ0Msi4YcSib60BV66ToVeqaN0Ctt4+c2w9GJJLOFOo+eUT3trzuUA6+3lksnheprceKiQcUejOVSqDmMxCLomsB1ZKDQp5rvVxIniDWuogcqMJ+HfZk4lxerarhFYLckyie55cyHWMl+O0/d+yWEtLn19E9s2J9Vu4Srp9ETgw2kDpuL6pERDGd103b8G0XDVeoqaqiu22jjmno9mGfp63EWJt7H+OJfNI8pQKx2mde88IM4Ro7zR3550++Xmklx5HVqsTpASOj9Tlpshx/LEadKtb79LN4bRbHmEDcjxILICDEymA7jgSNYA/Gqg690uQkU+tbNQevRrtC3s1yA5sSjT3zzCmPJKAdH40gvtFyMvlS5gFLoFjY1x22GifUThZpDOmi88qur6H5ljUKvJG4HaDYYnTkDcSwurlR4ZmG+SeiT83+EFBovnfgtyqm7mNn744wUv4nZWDvOL1a/TM2awowOdv5XK6Ar3dcI3UbwuByL6NRqEAsqWhptA4pnd/8wul9j78zHQUpSxae569l3o3tFnr/zU7jGxsTLduQzbyZZfqF5QxgOoasbsmHn+564WKwr4WA4C0g0tMRN2r9/g9CD8YFOaYcRulLoDpbRefRZJ6Qto/l42J6FvhnCSpfzixVefyirSiL9QXjocWI0kLTCxPqJfNUmYymyE0VV4RJ66ADlpJoamKi8pH4+H940epCHjx7j+44e5wduuQ2kZGLh06xmv71JRktj70Igya08smZ/sfol6rGDAy1ZhJZCL2tJdL+OU+2Psqy7arU1ZihCjzZmsI3RNX54KXUPq5kHOXzxV3j9N+/j3qcfQvPrPPfav6CcumvbfRTS9yOQpAuPAy0PfRDRuSFkrBXQ1e/ExVLdJaZHgg7jMczYABdfBohIXBF6Ob9JMFIfcVMo9G7tFlCEXmoo1ds+fm4zHBhNEAky1cNp9aHl4mt9qnJBKfQoahGoNIBo0kLNaUYjRCphyWKL0BvR/bh6imT5peZr4/E4Dx89zvcePcbhTIZU6SlijSssTHxfc5tK4nbq0ak1JXZIPxjwO9gKF1AeOkBRqBu116f2/3qQwRONtCyXdnUOgBA8f+enOXP813EjKS5P/RSPv+GprsgcoDhyH74wmraL4eTxhYFhxbf5ZO8g4q0I3X4nLhZrDpZuNJuKBn3uDAqxlDqm1VKHQQJ9xlBb/32frki5HSPBEIZS3aXcheVi6BpTo3EuLFaahC577KFvVuViypDQHaC/Bbf5msPuoFkoUlYLePVY21wSIagkbmsq9E6YWPhzfGGyPPquNZ9bGnsXu2d/F82r4utxTHse3a9Tsw6SHDShBzvM0564uGeLT7wyhB56S6Ffox7bmC8ktQize36E2T0/ct378PU4pdS9zYVR057HjWSJGAO0sQLbIzOA2aKFmosViRFtzNCIDrbDeJCIp1VYXaM8eEIfukIv1h1S1xHuFA4pLtQcynV306aidhybSJKKRrhtd9C01GPLpdOiqKfHMaXKsC3WBqDQqw4Zy8DzwKxfxRdRHGNtCmIlcYJE5WTnmEspGV/8NCu5t+GuS/tbHn0Xul8ju/p3QHsN+qGBX5QJU8fUtVZAV5/yXNoVuiL0aRrRTqN0bwz5zP2kSs+guyVyK18kn3nTQAc3hAFdWVHue+JiseYQj0SIVy9Qs468agl9JKtukm61v41vnXATeOjXb7kALJbr2J7f1Wf/xXfewm/80D3NcXaiqdD7W4duBIQelmb2C2HAViaumoqaNehCIATNbrxK4nYMN49pz274jkTlRWKNaZbGvnvDe/nMA7h6mvFFFS4V1qDXrMFbLkII0nGDZV8pdPpE6KFCNzQdv1HFcFc3Wi49QCH9AJp02Tf9MUxngeXx7xloXb+WbFku/e4WLdVdJrUiul+hZh1+1RJ6LpvFkwK/z9EUnTB0hV6qO6+I0MPhDNtZLgDHJlM8eGy8td+2TtH+VbnEMXz1M/bbQ6/YHq4vm01F7TXou3fDZFDuGy6MtvvoIUaX/xqAldzbN7wnNZOlsXcztvRZhN8gVXoWX0RoRPcP5aJMWwbzAaGLWn8WnkKFjq8RbagbYHhMe9kMU0i/AYnG1JX/iC8MChPf2bsv7wKGZeFpFllRYqXUP0L3fUnFdpgSaqr2q1mhj6aiFEkghhDQdXMo9C6biqBF6GHgVjeEvh6ix4uinevQLXS/Bsi+K/Sw6idjmW1dovuJRFQ6XTjCq5IIK11e2PAduZUvUkq+BjvaOelyYeK9RLwCEwufZtfc77M4/jC+Hhsaoc95ynLR+kToYdkink60ES4yK4V++PD1f188DnfdBfffD1Ot9gC8SIpS6rXofoV85sGBNRWFUN2io2QpsVLun+VSsV18CfukKv+0U4delV2iAAlLp0QC3R58QNdNoNCvz3IJKzl+7+uXMHTBm45cfy2r9MMh0b0hdCE6JS5aaHgYeH0vB2u2/ccN3IaDac/RiO5j3z51wY6MKFXpGjmq1jEy+bXt6rpbZKT4jY7qPMRq9q04kSzHzv4LIl6Jq/s/iGEMJ/0ubRnkbXAiaSL2Ul8yM8KyReFpbU1FexkZUaP4rqdcM5eDe+9VN1bTVDfZdBtvF9IqBmBp7LsHXgYatv+P6/0N6AqfUnd51/BFBH/AHcaDhK5DWSQx3B3WKRrOsryeRdFQoa9WHd57977mwOPrgfAdJBoIrWeEtDETPRxyYffdcim0ZaF75RUEEtscIx5Uv2lai0BWsw+SLnwd4bd+puzq36FJd0tCl5rB4vj3EPGK5NNvppy6e2hJeWnLoFR3cIxxTHuxL81F9UCh+65OtNFqKhofV8dzssuudcOAW29de34IoV4LsTj+MJX4LSyOv2fgk+wVoY8zJop9nVoUio5x7yr12BTGK3iy/lZCTSSJecW+dTJvhq7oTAjxTiHEGSHEOSHEz2yx3fuEEFIIcV833xuOZ7sehR4zdMyIhibgx996pOvPrYHv9GxaUYjNMtGzEY9ivy2XtmlFflCX7Rg5otHWNqHtks88SMQrkSw/23wvt/I3uHqK4sgbttzP/OQHkAiuTP0zYHhzHdOWQbHuYJvjGM5SXwi9ESh031EK3Ynk8HWL8WApZvfu7r7n6FE62lKWRfOGW0y/jide/00cc3wohG6bE4yTJ9/HKpflorrWc86VV/WCaIi6PoLll6j1P1FhDbalMyGEDnwUeBdwAviAEOJEh+1SwIeAx7vdeWmbARWbYSoX5+G793IwGGBx3fBcfG1AhK77lPpcttgcP2eZyIqq+nAjuTWE207oQLMEUfOqjC/+BSu5tyO1rZ+UCpn7eez+s6yMKiU/LIU+EjSX2cYopr3UFxUUKnTP1oO66b2kUq2bWCIBqa3Tnhkd3VrJpzvY5YMmdE0DJzZBljzFPjYWLRUcQJK1L7+qF0RDOJEREpRvPkIHXg+ck1JekFLawB8BnYYp/jLw74F6tzsPg6uux3IB+LOfuJ9/9947r+sza+A7SKGegftF6OEYuozu9l+ht1kuYV22a2bXXDShj+6Y45QTJ5rZ6JPzf4LhrjKz90fXfKeuK0JaD8dsVQsNU6ED1COjGM5iXy6aUKFLVyPamKEe27uBgLdS6ZoGx45tvY9Mh8iWQRM6gBebwMTBq/evKmOl7DJGEeNVXrIYwjUzpOTNSeh7gattf58OXmtCCHEPsF9K+dnr2fl2I+Q2w0jMwIy8ciYWbZZLL8oWofOiKMBIxO27hz5XqBEzNDWhvhYMSUjk1vxuQrQe8fOZB0kXvoHwG+yd+TjlxB0U0vev2fbECbjjDjUUejMMm9CrkSyGs0K92vspF6FCNzSNWH2aRnTPGgsLYGJic0Fw8OD2x+dmIXQZV6mHpt2/7JHVssOBHVCyGEKaaWLCoVSsDHS/N6xPhRAa8GvAP+9i2x8TQjwphHgyn19sDn+4XkK/YQzAQw8XRUd0r69li1JK/vbMAm86PIoQAlFTCj2S3BhrmwgcqtXst6P7NV73xBtJVl5ket8/WXNnO3BAqXMh4JZb2EBkIYa5KApQMTIIfOxi7xs4Gq6HqWsYbgHDXaUeO7ThOEQiND31dkxMwP79G19fj2h0I+kPusoFQCYUoSfd5b4lLuYrLoe0kNBf/QpdBCmWq0uDbf/vhs5mgPbTc1/wWogUcAfwqBDiEvBG4DOdFkallB+XUt4npbwvkxmn/AotlxuFkF7PCX19lUtToWtOX8sWX5otcnWlxjvvUFJa1FdVwFMiuWHbkNCXR9/B6Vv+C54+Qi12gIWJ71+zXbsqFwLG1iYINF/fjOj7jZDQS7q6aLzCYs/30XB8orpOrH4RgJq1kdBBlTC2PwlNTsJtt3X/5LdepQ9DoRMQ+rgokK/251wtVB0OawtIdOqxqVc9oRvBnIDSgBMXuzl9ngCOCSEOoYj8/cAPhm9KKQtA85IXQjwK/Asp5ZPbffErqXLpCXwHX1P77LdCT2puXxX6Iy/OoQl4+21q9U1rrOAYOWLWRkYJCR0hmNv9w8zt/uEN24yMbFSNo6Mws27AfSzWO7vqehESelFX2Tx+eQkpe/vz1B2PiKZh1bYm9HQa7rsPpqeVWs9tHYG+AZkMzM21/j4UQk8pQh8TBeZWbMZSvWXbsN/ksDZPPTaF1IxXPaGbQehZo6L6JAb177otnUkpXeCDwCPAKeCTUsqTQohfEkK850Z23loUHexZLAZStqgUelKzqdoeXp+GXHzh5ByvO5hjNKnYRrdXcCPZjuST3CjaN6CThZDJbHwCGeZw35DQV4QidMNepN71Unx3aLg+hqY3Cb1uHdz0iSSRUNbU9ZI5bKx0GQah68kcPjrjIs9ioffio9GAiuNwUMxTs1Sb7TCspUEiPqJOBr+xMtCF0a7oTEr5OSnlcSnlESnlh4PXfkFK+ZkO2761G3UOUHFczIjWnKw+KAjfHZiHnhCt+ae9xoXFMi/Pl5t2i+tCxMkrhd6BcKPR7QljosNUME3bSFY3E6Gb9lLPL5pqQyn0WP0StjGObiX78kRiWS3rSojhHFfD1KhFRhmjwGKx96WL1SpUHYf9XKMaP0I0OpwO40EimVHKSNorPRcbW2Goh7Vqu4wM2m6BYFG0t5bLZh56QqgLpB+liy/Pl4DW4A7HAcNZ2dBU1I7EFqX7IyOb++LrffRhLYgCxAwNU9dY8uNIBNeWz/ec0Ct1HzNQ6DXrUF+JNlTpIyPDUa6RCNjmOGOiwFKx9+dprQams0SCGjXr8FDPnUEhjNAVzurNp9D7hYrjDnxBFELLRRF6v8oWwzp0KyD0KytVPvqlcz2tIghz1sN8m5DQ3UhmUwLaitA71Z23v9d+8xumQhdCMGIZlF2fipbm6uJZri03erqPasPD0DSs2qVN/fNeIVwY7WR3DQKGAY45ybgosFTqvUKv1WDMvaL+bB3ZEYQeTalFUd3JU60Obr/DVeiO84rSEm8UQjp97xRFaHhaDAt1gfybvzzJRx45w0uzvQvsCVX/iNUi9Ii7ihfNbZqpvRWhb+UBRyKQDSohhejc5ThIpK0IFcdhlQyjoshCj62CuuMT1zyijemOJYu9RHgsO1UTDQKGAV50kglR4Mpq7+umazWY8FRiZc063OyHeDUjEotSJ4q0VykMMHRxyIR+fUmLPYPv9d1yAaXSY8Fc0ZfnywA9HfNVrDkIAUlT/S5OtYru18HaWIMeYrN2ddPcvpU99NfHx4dXshgiEzcpNRyWZJKcKPXc+63ZHnvFIgLZd4WeSKib5bCeepRCn2BUFLm0WkRK+ObFFX7tr8705PtrNdjtz+CjUY9N7QiFHolAScsQcxao1RhYSNdQCb3mDk+h93tRFFSeS0joIcLclV6gWHdJRSNowSQmrxRM74lvLrVTqc43n+zm94AmxsbU8eqmaabfSFsGhbrDrJtkjALLPbQKpFRli3uZB6C2RYVLr3DkFebM9QIqoGsSE4dSZY5aTfLbj13kP/3tOa7lb8wAlhJKFY8pZlnRdyM1c0cQumFA3tjPLv8aDdcdmEofMqFf33CLXkBK0PpQtri5Qq+ja4L33qPSEvK9JPSa07RbgGbSItbmhL6ZXdJNyZ2uK+LZTskPAmnL4MJSmSWZIidKFBt2z6oJqlVwfJ+9Uk0q6rflAt2VlPYLmgaeFXSLestcWqjz9GWV6/KVszfWtNVoqOEWB8Q8q6aa7LETCF0I1bswJeaZq1Z2BqGXnesbEN0L+D4I2fu0RSE6VbrEMWWD3/sHD/Lhh18DQKGHiXbFusNI2/ELg7nEFpYLbFTjQnRfQ72393OSXxHSloHj+SzLEbKiTNWuUir15rvLZbA9n93+LJ6WwDYnhm4x9RsyrlZkx0WBR88sMldUd8cvv3xjnY7PXymyUK1wUMxTMA8Si736SxabGDnCuCiyXJx/9RO6LyW25zGeHOyVImVoufS2ygU6NRcl0L0Kh3IpLFMnbuo99tBdRqzWTpuEntiande3m4+Ofus1eoRPJvOou1Okfo1yuTffXS6D43vs8a9Ssw4jNPGqJ3SSqtN4jAJ/9ryqSLlt9whfPbf0ipviGq7HD/3O1/j4N/+aEVGlHDu0I9R5iGjuOABO8WXKZfB6nyG3AUMjdE+qk2QsOdiVoPWE3ku10Km5SPcq+EGlYsYyemu5rFPoIkhaDCe5b4Zkcu3PerOo7utB2Fy0FFE/fKJxuWcKvVIBx/PZ556jnLwdwxhezMGgIIL2/wNGmXPLBWKGxo8+eIhCzeGFmVcmL5fLNrbnc9xQto2fOr6jCF2m1cJIpHwWKaE4gIl0QyN0N2C5sfhgpY/vgxZ0igrR2wu1k+Wie9UmoafjZs+rXNo9dFFXCl1LbG25CNFS6WGFxbcaQkJ3k6qVPGv3jtCLJUlCFsn6i1QSJ4Zacz8o6MkcnpbguKm889t3ZXjrLRMIAV9++ZX56GEp6T/Yq9RDInfHjiJ0L30IgHj9EgCrvQ8F3YDhK/TEMBS6ixSRnnt5Gy2XJJpfIfhVyVgGhR6O+SrW3TUKXauv4mlxDGv7Y3r4sEoKvBkqVl4JQkKP5w7iYJBzp3EctQh3I7Bt1SV6q1AjAMrJ23cECZlRQc06xCFtAYBjuSy5hMnte0b4xgUVAbtUbvBTn3y267C5qwvqH2PSn0GiUY8d2BE16CE0K01ZpMk4V/GlJN+/+SGtffZ/F53hBbI1NwSFHjYW9Z/Q1yr0TNzomUJ3PZ9yo+WhSwmavYpj5Lryw+NxOH586wEWNzNCQj8wmmZB28WEp+Igb1SlK7vF41ZN+ciVxM5QlYahyjP3+NcAmEqoR7jX7E3z0mwRKSV/dXKeP3t6hm9cWFnz2acur3Yk+elFJV7GnMvUY/t3TMliCMOAvHmA/cyxVKtRKqm8pX5iiIQuieo61oDj5Voeeu8JvbPl0qbQ473z0MNpT6FCb7X9Z4cTwTpgHBlPcHA0zuunRlmO7GOPVER0o4Sez6uSxVvFFSpaFtuc3BGWi2mqMrusO8PDhw9zNDFOvQ4ndo+QrzrMFetNL/38Ymv1eaFY5/t/8zH+x9cubfjO2VWl0Efsy2pxeUjhY8NCJAJ16xAHxALXyiWk5IarXbbz4YfnoUufTDSGlINdbWpZLv1X6L6WQPdr+EF+S9oyKVQdpLzxKN0wx6W97V8Fc2W+5SpWXglGk1Ee/ZcPcevukUAFzeN63g1XuszPg+153KpdZTF6DITYEapSEfphdGnzA1NJTF3n1Cm4ZZdKtHzpWpGT1xQbnVtoHeRHX17El3B6bi3TVCqwVLIxNEGidoGatTNSFtsRiYBMHmWPWGK+pAz0G/XR188lWI+hKvRMNNq0IwYFtSiqqlx6XbnQyXIBkLZK58nEDWzPp+PbpBsAACAASURBVObceP1SM8cliE5oEro5tmmOy6sRpgml6BRJUcepztxQJUE+D/U6OJ7LLeIqS7FbgZ3RCGOaqoEKaGbAFwogCorQn5sucHpWPf60E/rfnVnc8JrnwcmTUGjYTJk2hlfcMSmL7YhEwE0eRReSevEcwA356JXK9gp/6Ap90IQeWi798NA3Wi4qCUs4AaEHaroXPnqxtjGYy3CW8KJbRCa+CmEYUIupShetfB7HUSf+K0E4OShav0hcNFiJ3Yqu86qfrgMthQ4tQgeo5CNMxON86okZbM9nLGlyfqGMlBLX85udpBeXKjiej23D6dOq27ZkN7glohqTdkrKYjsikdZNUi+/DARNa6+wLmI7dQ7DVuixwSt06fsI/AFVuQTRhrZSL2HMbU8IvanQA0JvuETcPH5s5xG6nVD1vkb1+lXQc1fz/O3peTwPFoPqvHTlJQAK8dt2jOcrBHipffjCwKpdWPPe/tQI14pKlLxuYg+lhsvTLzV45Mk8xbrL/QcmcDzJF75a5etfbx3Hom1zNBLm4exMhV5JqKe83Y1TzVLt9pGD3cLzlB24HYZG6JIhWS7BMvNgCD2o0QoUetpSUi/fg9LFloeuduqW8wgkfmxIGaxDgmmCHz+EJwWxqiKi6yH0X/zLk/zMn77A2bOtTr507RS+FJTit+woEjKjaoBzOBg7xFRK2S5WJMIdOdVR+vjpMl94bhFNCB4YVyr05bky7ctDRbvBQbGARKNmHdhRxxICy8XIsmge5T5xmrng0fHaNbjeZbS5OXV+/qcnntlyu6EuUQzFcvGUsh1ElYuvBQrdaXnooCag3yjWZ6HLcpC5sUXS4qsRug7JeIpZRkk2LgHdE/pqxebZq3kWSg0uXG39m+Rqp7kod6FFkjtGoUNYunh4g0KfGlGEfmBkhD1BithMqcQTc7MczWQ4EnSpzZRbJUZSSoq2zX55jXpsH1KL7jhCD8+dlZE3co92lpmiWhGt12FlZYsPdsC1a2qx/vFrs1tut+MInT4S+uaWi7ozNy2XHpQurs9ClxXV/EF8Zyl0gFzC4JI/Sdq+DNC1j/6Vc0tNpTRbaS3qjdZPc1rux9S1HUVCYemiVbu0RkKGCv3ASJpMNIoVifD5SxeYq1b4jgOHiEUijMYsZttKjOqei+v77PJnWt78DjqWoPggEgFn9EFSooaff7b53rVr3X9PPq/O58vFQrMhczN0RWlCiHcKIc4IIc4JIX6mw/v/pxDiBSHEs0KIrwohTnTzvcO0XAazKBpUuTQCQg8tl54o9LVZ6DQJfWd56ACJmM55pthlnwepfJNuVPoXX1pAD0qdrgVkpHkVss5VzvhTGJq+o0goXBiNeCUMp5WymIvF+MCtJ3jb1AGEEOxOJFmp19mfSnHvpOpM25NMcq3tplgKVv7G3Ks7smQxhGVBKfsAAKPFbzRfX16GCxc6f6bccCnUWuXNV1SPG+e6OKm3bUERQujAR4HvAKaBJ4QQn5FSvtS22R9KKX8z2P49wK8B79zuuzOxYSr0QZQthgpdWS4xQ8OMaD3y0NfmuFBVF6CW3HmEbppwXj9GlM8Tr56lmriV6Wk1YWmzmvzlFcmjpxe5Z3IXT8/PMRtIei3/AgLJOXGQE+bO6mw0TchbxwBIll9kNfcQoGa4vuPgoeZ2e5JJLhTyPHz0OFpwEe1JJDmzsowvJZoQFBs2+8QCll+kkrhtRx3HdlgWlGL7WRITHGw8u+a9K1fUNCfLUh2kj51b5k9OvcyppVV8KUmYOh9+5+vI2OqaPp9fZSJucXmL/XVzz3w9cE5KeUFKaQN/BHxv+wZSyvbq3wSwreUvEFiRyOA9dHeQlku4KKrIQgih8lx65KGHFS5Sgqwqha6ldh6hR6NwJaKqCVIlddHUavDCCyrb5ff+dp5/+0dXeOwx+OY34amn4C+/UqTQsLlrfILJRILZShnb83jspf8JwJtOvJu4GdlRHrppQiH9Rnxhklv5m023e2DPPt4+dZB7Jiabr+1JJrF9n+VgxH3RbnC/dhKAfObBHU3oAFesu3mtPMlqbe0EqMVFRewvXazzka8/xVypxrsOHuYHb7sNKQV/+pSaxSql5Fx+lWO5rZP0umkS3wtcbfv7NPCG9RsJIf4p8FOACXz7dl8aCcemDSAjuB3SVerY16IYPSZ0TVP/hTep9XXocON5Ll94cQ7L1FUwV1DhUquB0VjG1VNEXvXB3RthWbBiHqLumsSLT8Ou9wOqTfpTf13i3zz2NJ7vcziVY1dCLep9/uIFdCF4zdg4zyzMc7VU4rnFBcYbL9Mw4uzffRfx+Ks/NrcdpgleJEk+8wC5lb/iPB/uuN1to6PcNrpWOOwPfPaLxQLj8ThF2+Z+7SR1Y5xq/BZ27VBCDwWBP/5WJquP8PWX/5bveu2712wjpeR3T75Iw/P4uTfc31x4vpAv8OziAr6UrNbr5BsNjmUz63exBj2jNCnlR6WUR4B/Bfx8p22EED8mhHhSCPGkEVgfA7dcXHWH9LVYXzy9dh89rHLR3PZMdPOGLJdf/MxJ/vknn2Op3Ggq9GoVDGcZxxjdEW3/62FZ8MZ9U5z0D5Cf/Sr5hpq24/gev/ncs1iRCIau8xfnVZ36U/NzPD53jfccOcZINMruRJLFWpXHrs1whz5NPXk7CI1EYpi/1eARqsnl0XeQqL5MrHZx6w+0YWpkBFPTOLuqyjeKjTr3ay+xmnnLjolP6ITw9/Yn3g5AZOGLTK8LHHphaZGnF+Z579HjTTIHuHtikpJtcz6/yrm8qpDZTqF3Q2kzQHvI6r7gtc3wR8DDnd6QUn5cSnmflPK+yYwqrxs4oTvqYvc1qy+E3m67SC2CL0x0r9JMWUvfgEKfLdSYK9ZZKje4sFhpeuh/T+hw/559aKNv4Ih/no898yS+lPzpyy9ztVTkH99xJ2+fOsA3rs3w6XMv8zsnX2AqNcK7D6uGpN3JJL6UPLMwx23aVSrJ2wF2HKHHYuqJZCX3nQCMLv9V15+NaBpHMlleDgg9UTvLhMhTzH4bsPMqXEKEv3c1foy6uYs366f49aef5HdPvsDVknKqn1tcIKrrfGfbOgXAa8bG0YXgqfl5npqfw9Q0TmhbOejdEfoTwDEhxCEhhAm8H/hM+wZCiGNtf303cLaL7wWGodBDQo/2XaGD8tE1v9q0ljLWKyf0Z66oVe5UkN+yVqEv4RijOyJpcT3Ci8aaeJPKdCmc4uPPP8sjly7w0P4p7pqY5F2HjhDVdT597izpaJR/cuddRIITYE9gw+xhmaQsUkkoQh/m4OZhIExDrMWPULWOMLr8yHV9/ng2x5VikZrrcLD6OACr2bcAO5fQm9U9QpDPfhtvNU6Tixp8eWaaP3n5NADn8qscTmea52OIuGFwa26URy5d4Jtzs7xjaj8nnn7/lvvbltKklC7wQeAR4BTwSSnlSSHELwUVLQAfFEKcFEI8i/LR/2G3v/CwCN3Trb74o52GXOhetanQj04kmSvWubB4/bGAz1xZxYxo/PQ7bgFaXaKK0FdwjLEdqdA1TV04pdRdALw7vcw3Zq8xmUjw/ltVBW3KNPnF+x/kI295iF9+4C3sTaWan98dSPH/JfoUAKvZtwI7T6FDy/NdGvsusquPklvefHF0PY5nc0jg7OoqdzS+xhwT1GMHMc2NQmcnIbyZ5TPfRtxb4VfuyPJt+/ZzenmZkm1zpVjk2CZjw16/azcSeO+x4/zY6AWija0L2LvSqFLKz0kpj0spj0gpPxy89gtSys8Ef/6QlPJ2KeVdUsqHpJQnu/1lB2+59NdD7zzkotwk9O+7ey+6Jvjkk9Ndf+dSWeVKP3Mlz2v2pvn++/bzluPjvP6Qsq1Cy8WN5nZkrS+oi6YavwVPi/G9mRXum9zFj7/2bqJtTLIrkWC8w8icaCTCLZksHzC/SjF1H9XEcSIRXv2DoTsgJJ/LB36acvIEJ176h4wUvrH1hwIcyWTQhKB6+qO8wX+S59Lv29H+eYjwJhk+rWTyX+Y1Y+PYvs/nL15AAscynTu837JvPx95y0O858gx9l77LRqxfVvua+iX/6AJXTQtl/4viqr9qKlFoeUyMRLjoVvG+dOnp3G97X/5r5xd5HUf/hs+/8IsL8wUuHt/hpih87v/++u5/8gYjQb4jSq6X91xwVztsCyQmkEpdR+T5cf54N33cmAk3fXnP3x7nN3OeeZ2/SCwM9U5tAjdi4zwwmv+BDeS5Z5n3sE9T72VPTP/lYizec+6pcEPJE7zIfc3eFrciX7nL6/5zp2K8Pf//9s79yC5qjqPf37T0/N+ZTKT9EwmCXm/MZEEcI2ioAi4Rdz1AS6uuMSl1i0fi+uKmC1FLEsXdtUqtRYfYBBBCeqWuIulicsrCwIhJCEkTBJCAkwyeb9mJjM90/3bP87pmds9/ZqZfs30+VTdmu5zzz1973du//p3f+ec3+mrmE5P5WwmnXqCRY2TKS0pYdPrBxAYTJ8Qi4jQXFVFZc9eGk89zpHpf5f0s/Iecc1fDD07Bj025BHyVeML9dDrWXrqIyuns2n3UX72zEEuaKribbObqCyL/0z65J5jqMItG7bRNxBmxYzoR7OIdw4Qriy+af8RIl7Q6YbVzDx4J6X9pxnwJx/i5SXQ+QvCUs7RKX8NOIMOECxvYcvKJ5l65CECnQ8wf+8XmLf3VrqrF9FTNRcVPyrmS1Ted4jq7l1cNnCCQ0zm1WU/pdnnH9ZmMeK9/pON76Xl8H1U61kWTGrk5RPHaauppSpFrHRaxz2Exc+xto8D/5KwXtF56EMGvTIrcb14Br0k3B21luC7F05hSm05d/z3Lm5av4V7NieYAww8f+AU0xsrGQiZuVorZkQbKa9Bp7K4PXQwBl0IU59mmACgsmcfgc4HOd50DQN+84PpDLphwN9IR9uneGHlZrZc9BQHZ36evvIANV07qT33AvVnnqX+zDOUhHs52XglLy5cz+aVz9PcODdhm8WGzW0GwOGWj+ML9xLofJBlTc0ACePnEUpC3QQ6H+RY8xr6y6ckrVt0HroMjkMvz4pBj10MIeSroqI32qD7fSU88MlLOHCih+9u2sPGXUf49OXziKUnOMDOjjPc/M7ZNNWU81j7UVrqo6cudnczlHejung99IjROFu3irCU03D6KU40pcw+QWn/aZa9dD3hEj/7Z39tsLzYRrhESDYztqv2QrpqL0zZRqzJKXaDXlNjQrGhEHTXLOVM3SW0HrqX5Utu5OE9r7DUGvZETD3yK0pDZzjU+smUn1WEHrrpYAz5cuehe2PoEeZNreW9i6dy9dIA2988w9GzvcPa2vb6aQbCyqpZjdy0ehb3r70EiRmac+6cGeECINXOQw/7Kjhbt4qG05tTH6QhFu1eS0Xva7y85H56K2cO7ipWD93ny3xncLEbdJFoL/1Q61qqzr/KouAWvv2uK6JSKAxDldZDP6aregln6i9N+VlFZ9AldJ6w+EF8WYmhx3roplM02kP3csUi8898rP3osH3PHziFCLx1RvxHMtWIh27zuNQUVy50L6WlQ9qfblhNTdcOSvuTZ6ebvf+rTD65ib3z7uJMw9sHyysqho9WKiYymb/G7y9uLSN4+zyPNa8h6G9i/p5bmBruHHTSJDyAhPuijqs/8zS1XS9xqHVtWnko8m7QIbdGXQZ6CZcYlyE3HnpV1Dj0WBYGammtr2DT7ngG/SQLA3XUV8bvMOnuNtqVBY+j+PBVp98JOBGJhElOTXoHQphA54OD+yTcT0moGzSMb6CLOfvWMeON79HR+kkOt94Ut51iJZMedZxRokVJvWfAVdhXwUvLHqJ04BQrXrySxS/fyPIX38/qzTNYvXkGy3Z8hMnHH8UfPMqi3X9Pb/l0jky9Lq3PKYjfznA4d7mSJXSecIl5psxNDL2GEu0j1D9APLlFhMsXTeHXL3TQ2x+iwu8jOBDm9zsP88LBU3x4ZeJxp5GUEJXn99NbMR1/WUH8PueN2lqzEsyZ+r/gROP7mPPqOkT7aT72CHXntgAQllJUyvGFuznUciP75n5rWDvFbtDr6ka37mWithxGB5GhdUPO1a1k2/L/YUH7Z6nu3k3IV8vhlhsAoen4oyzb+VEGfPWIBnlxxR8JlaZ3UxaEQQ+FcvdYJqG+rHroPt9QBwgMpdAN93UD8cdFX7FwKj//8+s8+9pJLpvfzM33b+Hx9mPMnFzFDZfMjHsMDBn0qp699FTNL4rV6ZMxaIilhF2L72X5tvczZ/9X6C2fxoGZXyTkq6J04Az+/lN0Bm7gbP3FydspUhozGLlLMLy66CgpMQ7HWU+i8e6apWy96H+H1X11zjdoPfRTpnXczf7ZX02rIzpCQRj03IZczhPymSBhtqYj+/3DDbr29ZDIoF8yu5EyXwmb9x5jSWsdT+w5xtrVs1h3zaKhFYni0NUFaIjK83s5NekyJhfhtH8vntn8hEpr2HHhb2g8uYljzdcS9qUfRyjWDtEIFRVGg3SW8UuGiDPoXhoaog16IrTET0fbzXS03TzizyiIZ/ScGvRQL+GSckSyl+va6ykPLhQd7IlfGagqK+WimZN4au9xnmg3E4k+sHxaUmOuagx6Re/r+MK9dFcvKMo8Ll4qKqL7MPrLJnMkcN2IjLnP50ZlAEzOwICp2trizuESSy5+3IrUoGdnyGIEr1GJLHKhweTJuN4xv4lXOs+xYcsbNNeWs6Q1efAx0iFa1bMHgPNV891oAsYeLin2cEuETIRdUsyXKTrq67PfV1h0Br0k1Eu4pCKrBt3roUdCLhLsIdmC3e+cZyYXPPvaSd41vzmpdw5w1A6KiRj0YP38olpdJxHesEs+jp8o1NePvV/LhVui8fnGfn+lenosOoNuRrnk0qDbVYvCiceiAyxuqaOx2hx4+cLk03v7+uBNm6yxurudoL8ZqSreMehexuphT00yx6OYEIE5c0Z/vN8fPVTPYRjrU0tra/L9RWjQ+7Ju0KNDLsZDjzdb1EtJibB6bhOlJcLb5yWfwn/w4JBmVT176KmaX/Tx8whjGSZXXe08dC8tLTAz8SCrpCxalLuhyOOJsTy1lJVBU4rsHgURdc1pyCV8npAvO8vPRYjnoSebLRrh1qsX8uGVbYMrEcXj/Hk4fNi+UaWqp51jzX/lDLqlosJsvcMzKaTEeefDmTXLeJWdnWaF+nQWdb/ggswOfZxI1NVFD2seCa2tqQdyFJ9Bt6NcchVyiYxy8S5ykYhpDZVMa0geJDtwYGhygr//OP6B026ESwwNDSOfGCMCgUB2zme809Bgtnnz4MQJM/Sup2foeyti8r9UVUFzsxsllIySEqPRaO7PlpbU9YrQoJ/P6SiXfpuO1d9/clS/yl56eoY6QwGqetpNedU8ap1BH2Q0Br2tbfgsX0c0Ph9MmWI2x+iZNm3k92dTU3pJ0woiypXTGHo4+zF0r2HQkjKC/ibK+w6n9NBT4fXOAWq6XgKgu3qp89A9jDROOXkyzJ6dnXNxOGKprR15X02qztAIxWXQVfGFsz9s0e+PjnUFy1ooC3YSDI6+zf5+E8P0UntuG31lAYLlAWfQPUTi6OlQVWU68NyQT0cuSddAg7lH0x0dk5ZBF5GrRKRdRPaJyJfi7P+8iOwSkR0i8icRGVHfeM4MepaXn/PiNbB95QHK+w7T0TH6az16lGHj2GvPbRtc6d6FC6JJx0svLYWlS116V0fuCQRMLD0d2pKvCx1FSrMmIj7gB8DVwGLgoyKyOKbai8BKVb0Q+BVwZ/qnkHuDnq3FLbx4413BsqmUBY/Q2+sZoTJCYmNuJaFuqnr20FVjDLrz0KMJBJJ73eXlsGSJS+/qyA8isHgxTJ9uQn6BgOl0nj/fvI/cu83NI/Pm0/FNLgb2qep+cyLyS2ANsCtSQVUf89T/M/Cx9E9hdEN4RkV/xEPP7igXMD39kWyIfeUtlAWPgIY4eNBHIDCyHBc9PUNtRajpegkhPOihO8MUTUOD+XK0t5svRXOz+ZKEQmYb6f/A4cg0iSZvtbaaPE0dHTB37vD9yUjHoE8D3vC8fxO4JEn9tcDvR3ISufLQtf88QvYWiPbiNbDBsgBCmLLgMYIS4OmnjcFpa0svNnbkyPCy2nPbADhXu5zycmec4tHSYvTN5Ao8DkcuqKmBBQtGflxGI8ki8jFgJXBXgv03i8gWEdly+vRQD19/fybPIjHhoFneKdudohBt0PvKzQDSsqCJt4RCZjzv9u3wyiup2zo6fDEjas9tp69sKsHyFuedJ8EZc0cxIZosYxQgIm8DblfV99n3twGo6jdj6r0H+B5wmarGMUHD2j0HtI/yvL3UA2cy0E62aQKO5/sk0sDpmVmcnpnF6QkLVDXuwMd0Qi7PA/NEZBbQAVwP/I23goisAH4IXJWOMbe0q+rKNOsmRER+pKojzwSfY0RkSyauN9s4PTOL0zOzOD1N24n2pQy5qOoA8GngD8BuYIOqviwid4jItbbaXUAN8LCIbBORRzJw3unyuxx+VjHg9MwsTs/M4vRMQsqQS9Y+eJx4BJmi2K432zg9M4vTM7Nk20NP1HY+Z4r+KI+fnQ+K7XqzjdMzszg9M0s29UzYdt4MuqpGnZSI3CsiR0Vkp6fsLhF5xc5A/S8RiTv/L9FMVhGZJSLP2vKHRCRv8yljrzcXTGRNnZ6ZxemZWbKpZ7K2CyKXi2U9cFVM2UZgqZ2Buge4LfagFDNZ/w34jqrOBU5hxsgXE+txmmaS9Tg9M8l6nJ4ZpWAMuqo+CZyMKfuj7ZQFMwM1XlaDwZmsqhoEfgmsEREBLsekIgC4D/hAJs41nneQrmcgIrfZOu0i8r5kbY6V8aKp09Pp6SkrOD1h/GiKqhbMBlwA7Eyw73fAx+zrVuBR+/pDwE889f4W+D5mHOg+T/n0RG2P8Bx9wKvAbKAM2I7xEjYA19s6dwOfinPsYlu/HJhl2/ElarMYNHV6Oj0LWc/xpmnBeOjJEJF1wADwAICqHlLVa/J0OnG9A9LzDNYAv1TVPlV9Ddhn20vUZtYoIE2dnpnF6Zl5xo2mBW/QReQTwF8CN6j9yYuhA/NLHKHNlp0AGkSkNKZ8rMTLbTMNOK1Dj4qRMkTkWhG5I8WxicqzQoFp6vR0ekZRYHrCONK0oA26iFwFfBG4VlV7ElQbnMlqY1jXA4/YG+ExzOMZwI3Ab7N9zrGo6iOq+pVcf24ixrumTs/M4vTMPPnUtGAMuoj8AngGWCAib4rIWkxcrBbYKGYG6t22bquIPAqJZ7LaZm8FPi8i+4DJwD0ZONVE3kE6nkGiYxOVj4lxoqnT0+lZyHrCONJ0zJ0axbZh8t/sx3RwRDozlgAPE91B8o9xjl1CdAfJfkznSNw2832tTs/xtzk9i1vTvIs1HjfgGswY2VeBdbZsNvAcptPjYaDcll8L3OE5dp09rh24OlmbxbI5PZ2ehb6NF03zlsvF4XA4HJmlYGLoDofD4RgbzqA7HA7HBMEZ9DQQkeki8piI7BKRl0Xkc7b862KSCG0TkT+KSNz1uUXkcTvFd4eYxEPflwRJh4qBRHp69v+ziKiINCU43unpIcn9ebuIdNj7c5uIxJ2YIyLrReQ1EdkuIntE5GciEm/KfVGQ7P4Ukc/Ye+5lEbkzwfH50zPfnQ3jYQNagLfa17WYjozFQJ2nzmeBuxMc/ziw0r4uA/4DeCLf11Voetr30zHD0Q4CTU7P0esJ3A58IY3j1wMfsq8FuMW2UZbvayswPd8NbGKo83NKoenpPPQ0UNXDqrrVvj6HGfs6TVXPeqpVAyl7mNVM8/0iMENE3gJmcW0Rec56UT8Uk00ukrxnq/2l/1OmrytfJNLT7v4ORp+0euudnin1HGlbqqrfATox2QwRkStF5Bmr3cMiUmPLV4nI01bP50Qk7jqX440ken4K+Jaq9tl9KZfbzLWezqCPEBG5AFgBPGvff0NE3gBuANKaHaaqIcy404Uisgi4Dni7qi4HQsANItIM/Bj4oKq+Bfhwhi+lIPDqKSJrgA5V3T6SNpyeQ8Ten8CnbWjqXhGZNIKmtmL0bAL+FXiPqr4V2IKZuFMGPAR8zur5HuB8hi6jYIjRcz7wDjEZFp8QkVUjaConeqazSLTDYn9Jfw38U8Q7V9V1wDoRuQ0ze+2r6TZn/14BXAQ8LyIAlcBR4FLgSTUJfVDVk/EaGc949cQkYvoycOVom7N/nZ72/hSR/wS+jnna+TomNHVTus3Zv5diwg3/Z/Usw87uBA6r6vMAMU+rE4I4epYCjRhNVgEbRGS22thKqubs36zq6Tz0NBERP+af+4Cq/iZOlQeAD9q6f7CP+z9J0JYPWIZ5lBPgPlVdbrcFqnp7Vi6igIij5xzMrLntInIAMxV6q4gEnJ6piXd/quoRVQ2pahjzdHKxrftTq+ejSZpcwZCeGz16LlbVCb9oRILv+5vAb2wY5TkgDDQVlJ7ZDtJPhM3+E34GfDemfJ7n9WeAXyU4/nGGOvH8wJ3YTjzMr/VebAcLxgOYCTRjsrHNipTnW4ds6xlT5wDpdYo6PRPfny2e17dg0rjGO3490Z14n7UallndXgfm2v3VmNBDGWbq+ipbXguU5luLLOv5D9gZoFaDN8BMziwUPfMu3njYgNWYx9YdwDa7XYP5Bd9py3+H6SiNd/zjmGm/O+zfHwANnv3X2TZ3AC8Al9ryq4EXMfHhjfnWIdt6xtRJZdCdnqnvz/uBl2z5I3gMfMzx64HXrC577XFtnv2XYzIc7rDbtbZ8FWZVoe32b02+tciynmXAz+13fitweaHp6ab+OxwOxwTBxdAdDodjguAMusPhcEwQnEF3OByOCYIz6KMkSf6MRhHZKCJ77d9JtnyhnR3WJyJf8LRTYWeFbbftfC1f1+RwOMY3rlN0lIhIC2bUwFY7RfcFzKrfnwBOquq3RORLwCRVvVVEpmCGz30AqhCcrQAAAVhJREFUOKWq/27bEaBaVbvs2NfNmNlif87DZTkcjnGM89BHiSbO97AGuM9Wuw9jwFHVo2pmgfXHtKOq2mXf+u3mfmUdDseIcQY9A8Tke5iqqoftrk5gahrH+0RkG2aK+kZVfTbVMQ6HwxGLM+hjJF5+lwhq4lnpZGAMqUkk1QZcLCJLs3KyDodjQuMM+hhIkO/hiI2vR+LsKVNsRlDV08BjwFWZPleHwzHxcQZ9lNjOzHuA3ar6bc+uR4Ab7esbgd+maKdZ7Go7IlIJvBd4JfNn7HA4JjpulMsoEZHVwFOYXBlhW/xlTBx9AzADs+rOR1T1pIgEMLmP62z9LkwiqQswnac+zA/sBlW9I3dX4nA4JgrOoDscDscEwYVcHA6HY4LgDLrD4XBMEJxBdzgcjgmCM+gOh8MxQXAG3eFwOCYIzqA7HA7HBMEZdIfD4ZggOIPucDgcE4T/B/FqDmCNZ0CMAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], "source": [ "loader = model.val_dataloader()[0]\n", "dset_test = loader.dataset\n", @@ -2207,11 +2241,38 @@ "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2020-02-16T10:16:31.860515Z", - "start_time": "2020-02-16T08:59:28.800Z" + "start_time": "2020-02-16T13:43:24.400Z" } }, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:root:\n", + " Name Type Params\n", + "0 _model LSTMNet 471 K\n", + "1 _model.lstm1 LSTM 471 K\n", + "2 _model.mean Linear 129 \n", + "3 _model.std Linear 129 \n", + "INFO:root:model and trainer restored from checkpoint: /media/wassname/Storage5/projects2/3ST/attentive-neural-processes/optuna_result/lstm_std/lstm_std1/version_128/chk/_ckpt_epoch_4.ckpt\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "04ec2606596e421dbc39cda143372bd8", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='Testing', layout=Layout(flex='2'), max=234.0, style=Progr…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# test\n", "trainer.test(model)" @@ -2250,14 +2311,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": { "ExecuteTime": { - "end_time": "2020-02-16T05:01:00.333887Z", - "start_time": "2020-02-16T04:41:35.100Z" + "end_time": "2020-02-16T13:42:53.932617Z", + "start_time": "2020-02-16T13:42:52.434558Z" } }, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[I 2020-02-16 21:42:53,848] A new study created with name: no-name-b60e37fc-4ab6-4793-8a0a-c87a1b40c5c0\n" + ] + } + ], "source": [ "import argparse \n", "\n", @@ -2288,17 +2357,109 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "metadata": { "ExecuteTime": { - "end_time": "2020-02-16T05:01:00.335900Z", - "start_time": "2020-02-16T04:41:35.200Z" + "end_time": "2020-02-16T13:42:59.435563Z", + "start_time": "2020-02-16T13:42:55.499781Z" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "trial 0 params {'batch_size': 16, 'bidirectional': False, 'grad_clip': 40, 'hidden_size': 64, 'input_size': 17, 'learning_rate': 4.3424980235857914e-05, 'lstm_dropout': 0.15334701471662443, 'lstm_layers': 8, 'max_nb_epochs': 20, 'num_workers': 4, 'target_length': 96, 'vis_i': 670, 'window_length': 96}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:root:gpu available: True, used: True\n", + "INFO:root:VISIBLE GPUS: 0\n" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/core/decorators.py\u001b[0m in \u001b[0;36m_get_data_loader\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 16\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattr_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 18\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m 584\u001b[0m raise AttributeError(\"'{}' object has no attribute '{}'\".format(\n\u001b[0;32m--> 585\u001b[0;31m type(self).__name__, name))\n\u001b[0m\u001b[1;32m 586\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mAttributeError\u001b[0m: 'LSTM_PL' object has no attribute '_lazy_train_dataloader'", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mstudy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobjective\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_trials\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m200\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m6000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Number of finished trials: {}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstudy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrials\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Best trial:'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/optuna/study.py\u001b[0m in \u001b[0;36moptimize\u001b[0;34m(self, func, n_trials, timeout, n_jobs, catch, callbacks, gc_after_trial)\u001b[0m\n\u001b[1;32m 300\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mn_jobs\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 301\u001b[0m self._optimize_sequential(func, n_trials, timeout, catch, callbacks,\n\u001b[0;32m--> 302\u001b[0;31m gc_after_trial, None)\n\u001b[0m\u001b[1;32m 303\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 304\u001b[0m \u001b[0mtime_start\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdatetime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdatetime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/optuna/study.py\u001b[0m in \u001b[0;36m_optimize_sequential\u001b[0;34m(self, func, n_trials, timeout, catch, callbacks, gc_after_trial, time_start)\u001b[0m\n\u001b[1;32m 536\u001b[0m \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 537\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 538\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_trial_and_callbacks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgc_after_trial\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 539\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_storage\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mremove_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 540\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/optuna/study.py\u001b[0m in \u001b[0;36m_run_trial_and_callbacks\u001b[0;34m(self, func, catch, callbacks, gc_after_trial)\u001b[0m\n\u001b[1;32m 548\u001b[0m \u001b[0;31m# type: (...) -> None\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 549\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 550\u001b[0;31m \u001b[0mtrial\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_trial\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgc_after_trial\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 551\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcallbacks\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 552\u001b[0m \u001b[0mfrozen_trial\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_storage\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_trial\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_trial_id\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/optuna/study.py\u001b[0m in \u001b[0;36m_run_trial\u001b[0;34m(self, func, catch, gc_after_trial)\u001b[0m\n\u001b[1;32m 567\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 568\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 569\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 570\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mexceptions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTrialPruned\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 571\u001b[0m message = 'Setting status of trial#{} as {}. {}'.format(trial_number,\n", + "\u001b[0;32m\u001b[0m in \u001b[0;36mobjective\u001b[0;34m(trial)\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'trial'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrial\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumber\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'params'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrial\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 10\u001b[0m \u001b[0;31m# also report to tensorboard & print\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m\u001b[0m in \u001b[0;36mmain\u001b[0;34m(trial, train)\u001b[0m\n\u001b[1;32m 23\u001b[0m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mLSTM_PL\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 24\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 26\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, model)\u001b[0m\n\u001b[1;32m 700\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 701\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msingle_gpu\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 702\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msingle_gpu_train\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 703\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 704\u001b[0m \u001b[0;31m# ON CPU\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/trainer/distrib_parts.py\u001b[0m in \u001b[0;36msingle_gpu_train\u001b[0;34m(self, model)\u001b[0m\n\u001b[1;32m 439\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizers\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moptimizers\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 440\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 441\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_pretrain_routine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 442\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 443\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mdp_train\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36mrun_pretrain_routine\u001b[0;34m(self, model)\u001b[0m\n\u001b[1;32m 776\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 777\u001b[0m \u001b[0;31m# transfer data loaders from model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 778\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_dataloaders\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mref_model\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 779\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 780\u001b[0m \u001b[0;31m# print model summary\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/trainer/data_loading.py\u001b[0m in \u001b[0;36mget_dataloaders\u001b[0;34m(self, model)\u001b[0m\n\u001b[1;32m 198\u001b[0m \"\"\"\n\u001b[1;32m 199\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 200\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minit_train_dataloader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 201\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minit_test_dataloader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 202\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minit_val_dataloader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/trainer/data_loading.py\u001b[0m in \u001b[0;36minit_train_dataloader\u001b[0;34m(self, model)\u001b[0m\n\u001b[1;32m 56\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 57\u001b[0m \u001b[0;31m# determine number of training batches\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 58\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0mEXIST_ITER_DATASET\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_train_dataloader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mIterableDataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 59\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_training_batches\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'inf'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 60\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/core/decorators.py\u001b[0m in \u001b[0;36m_get_data_loader\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 18\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 19\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# Lazy evaluation, done only once.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 21\u001b[0m if (\n\u001b[1;32m 22\u001b[0m \u001b[0mvalue\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/attentive-neural-processes/src/models/lstm_std.py\u001b[0m in \u001b[0;36mtrain_dataloader\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 210\u001b[0m \u001b[0;34m@\u001b[0m\u001b[0mpl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata_loader\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 211\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mtrain_dataloader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 212\u001b[0;31m \u001b[0mdf_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_cache_dfs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"df_train\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 213\u001b[0m dset_train = SequenceDfDataSet(\n\u001b[1;32m 214\u001b[0m \u001b[0mdf_train\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/attentive-neural-processes/src/models/lstm_std.py\u001b[0m in \u001b[0;36m_get_cache_dfs\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 203\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_get_cache_dfs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 204\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dfs\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 205\u001b[0;31m \u001b[0mdf_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdf_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_smartmeter_df\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 206\u001b[0m \u001b[0;31m# self._dfs = dict(df_train=df_train[:600], df_test=df_test[:600])\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 207\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dfs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf_train\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdf_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdf_test\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdf_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/attentive-neural-processes/src/data/smart_meter.py\u001b[0m in \u001b[0;36mget_smartmeter_df\u001b[0;34m(indir, use_logy)\u001b[0m\n\u001b[1;32m 143\u001b[0m \u001b[0mholidays\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf_hols\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'Bank holidays'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mround\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'D'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 144\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 145\u001b[0;31m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'holiday'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtstp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mdt\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mdt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'D'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mholidays\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 146\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 147\u001b[0m \u001b[0;31m# Add time features\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/pandas/core/series.py\u001b[0m in \u001b[0;36mapply\u001b[0;34m(self, func, convert_dtype, args, **kwds)\u001b[0m\n\u001b[1;32m 4036\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4037\u001b[0m \u001b[0mvalues\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobject\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 4038\u001b[0;31m \u001b[0mmapped\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap_infer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconvert\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconvert_dtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 4039\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4040\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmapped\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmapped\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mSeries\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32mpandas/_libs/lib.pyx\u001b[0m in \u001b[0;36mpandas._libs.lib.map_infer\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/attentive-neural-processes/src/data/smart_meter.py\u001b[0m in \u001b[0;36m\u001b[0;34m(dt)\u001b[0m\n\u001b[1;32m 143\u001b[0m \u001b[0mholidays\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf_hols\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'Bank holidays'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mround\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'D'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 144\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 145\u001b[0;31m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'holiday'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtstp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mdt\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mdt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'D'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mholidays\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 146\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 147\u001b[0m \u001b[0;31m# Add time features\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32mpandas/_libs/tslibs/timestamps.pyx\u001b[0m in \u001b[0;36mpandas._libs.tslibs.timestamps.Timestamp.floor\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32mpandas/_libs/tslibs/timestamps.pyx\u001b[0m in \u001b[0;36mpandas._libs.tslibs.timestamps.Timestamp._round\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32mpandas/_libs/tslibs/timestamps.pyx\u001b[0m in \u001b[0;36mpandas._libs.tslibs.timestamps.round_nsint64\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32mpandas/_libs/tslibs/offsets.pyx\u001b[0m in \u001b[0;36mpandas._libs.tslibs.offsets.to_offset\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m~/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/pandas/tseries/frequencies.py\u001b[0m in \u001b[0;36mto_offset\u001b[0;34m(freq)\u001b[0m\n\u001b[1;32m 147\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 148\u001b[0m \u001b[0msplitted\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mre\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlibfreqs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopattern\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfreq\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 149\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0msplitted\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m\"\"\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0msplitted\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misspace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 150\u001b[0m \u001b[0;31m# the last element must be blank\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 151\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"last element must be blank\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], "source": [ "study.optimize(objective, n_trials=200, timeout=6000)\n", "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "ExecuteTime": { + "end_time": "2020-02-16T13:43:09.961035Z", + "start_time": "2020-02-16T13:43:09.870899Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of finished trials: 1\n", + "Best trial:\n" + ] + }, + { + "ename": "ValueError", + "evalue": "Record does not exist.", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Best trial:'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mtrial\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstudy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbest_trial\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m' Value: {}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/optuna/study.py\u001b[0m in \u001b[0;36mbest_trial\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 87\u001b[0m \"\"\"\n\u001b[1;32m 88\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 89\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_storage\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_best_trial\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_study_id\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 90\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 91\u001b[0m \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/optuna/storages/rdb/storage.py\u001b[0m in \u001b[0;36mget_best_trial\u001b[0;34m(self, study_id)\u001b[0m\n\u001b[1;32m 712\u001b[0m \u001b[0mtrial\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTrialModel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind_max_value_trial\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstudy_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 713\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 714\u001b[0;31m \u001b[0mtrial\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTrialModel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind_min_value_trial\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstudy_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 715\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 716\u001b[0m \u001b[0;31m# Terminate transaction explicitly to avoid connection timeout during transaction.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/optuna/storages/rdb/models.py\u001b[0m in \u001b[0;36mfind_min_value_trial\u001b[0;34m(cls, study_id, session)\u001b[0m\n\u001b[1;32m 186\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0morder_by\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0masc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlimit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mone_or_none\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 187\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mtrial\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 188\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mNOT_FOUND_MSG\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 189\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mtrial\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 190\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mValueError\u001b[0m: Record does not exist." + ] + } + ], + "source": [ "print('Number of finished trials: {}'.format(len(study.trials)))\n", "\n", "print('Best trial:')\n", @@ -2308,7 +2469,7 @@ "\n", "print(' Params: ')\n", "for key, value in trial.params.items():\n", - " print(' {}: {}'.format(key, value))\n" + " print(' {}: {}'.format(key, value))" ] }, { @@ -2339,127 +2500,6 @@ "df = study.trials_dataframe(attrs=('number', 'value', 'params', 'state'))\n", "df.sort_values('value')" ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Scratch" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2020-02-16T05:01:00.340348Z", - "start_time": "2020-02-16T04:41:35.200Z" - } - }, - "outputs": [], - "source": [ - "model, trainer = main(trial, train=False)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2020-02-16T05:01:00.341841Z", - "start_time": "2020-02-16T04:41:35.200Z" - } - }, - "outputs": [], - "source": [ - "loader = model.val_dataloader()[0]\n", - "dset_test = loader.dataset\n", - "label_names = dset_test.label_names" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2020-02-16T05:01:00.343323Z", - "start_time": "2020-02-16T04:41:35.200Z" - } - }, - "outputs": [], - "source": [ - "x_rows, y_rows = dset_test.iloc(10)\n", - "x_rows.loc[x_rows.index[model.hparams.window_length:], dset_test.label_names] = 0.\n", - "x_rows" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2020-02-16T05:01:00.344989Z", - "start_time": "2020-02-16T04:41:35.200Z" - } - }, - "outputs": [], - "source": [ - "x_rows.loc[x_rows.index[model.hparams.window_length:]]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2020-02-16T05:01:00.346671Z", - "start_time": "2020-02-16T04:41:35.200Z" - } - }, - "outputs": [], - "source": [ - "self=dset_test\n", - "idx=10\n", - "k = idx + self.hparams.window_length + self.hparams.target_length\n", - "j = k - self.hparams.target_length\n", - "i = j - self.hparams.window_length\n", - "idx, k, j, i, self.hparams.window_length + self.hparams.target_length" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2020-02-16T05:01:00.349116Z", - "start_time": "2020-02-16T04:41:35.200Z" - } - }, - "outputs": [], - "source": [ - "j-i" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2020-02-16T05:01:00.351704Z", - "start_time": "2020-02-16T04:41:35.200Z" - } - }, - "outputs": [], - "source": [ - "self.hparams.window_length + self.hparams.target_length" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/src/data/smart_meter.py b/src/data/smart_meter.py index 01ef881..1bb9949 100644 --- a/src/data/smart_meter.py +++ b/src/data/smart_meter.py @@ -23,16 +23,18 @@ def collate_fns(max_num_context, max_num_extra_target, sample, sort=True, contex x = torch.from_numpy(x).float() y = torch.from_numpy(y).float() - x[:, :max_num_context, -1] = 0 # Feature to let the model know this is past data - n=x[:, max_num_context:, -1].shape[1] - x[:, max_num_context:, -1] = torch.arange(1, n + 1) / 1.0 / n # Feature to let the model know this is past data + # Last feature will show how far in time a point is from out last context + assert (np.diff(x[:, :, 0], 1)>=0).all(), 'first features should be ordered e.g. seconds' + assert (x[:, max_num_context, -1]==0.).all(), 'last features should be empty' + time = x[:, :, 0] + t0 = x[:, max_num_context, 0][:, None] + x[:, :, -1] = time - t0 # Feature to let the model know this is past data x_context = x[:, :max_num_context] y_context = y[:, :max_num_context] x_target_extra = x[:, max_num_context:] y_target_extra = y[:, max_num_context:] - if sample: @@ -53,9 +55,8 @@ def collate_fns(max_num_context, max_num_extra_target, sample, sort=True, contex x_target = x_target_extra y_target = y_target_extra - assert (x_context[:, :, -1]==0).all() assert (x[:, -1, -1] > 0).all() - # assert (x[:, 0, -1] == 0).all() + assert (x[:, 0, -1] < 0).all() return x_context, y_context, x_target, y_target @@ -75,19 +76,19 @@ class SmartMeterDataSet(torch.utils.data.Dataset): rows = rows.sort_values('tstp') # make sure tstp, which is our x axis, is the first value - columns = ['tstp'] + list(set(rows.columns) - set(['tstp'])) + columns = ['tstp'] + list(set(rows.columns) - set(['tstp'])) + ['future'] + rows['future'] = 0. rows = rows[columns] # This will be the last row, and will change it upon sample to let the model know some points are in the future - rows['future']=1 - x = rows.drop(columns=self.label_names) - y = rows[self.label_names] + x = rows.drop(columns=self.label_names).copy() + y = rows[self.label_names].copy() return x, y def __getitem__(self, i): - x,y = self.get_rows(i) + x, y = self.get_rows(i) return x.values, y.values def __len__(self): @@ -140,7 +141,7 @@ def get_smartmeter_df(indir=Path('./data/smart-meters-in-london'), use_logy=Fals # Also find bank holidays df_hols = pd.read_csv(indir/'uk_bank_holidays.csv', parse_dates=[0]) - holidays = set(df_hols['Bank holidays'].dt.round('D')) + holidays = set(df_hols['Bank holidays'].dt.round('D')) df['holiday'] = df.tstp.apply(lambda dt:dt.floor('D') in holidays).astype(int) @@ -158,7 +159,7 @@ def get_smartmeter_df(indir=Path('./data/smart-meters-in-london'), use_logy=Fals df = df.dropna() if use_logy: - df['energy(kWh/hh)'] = np.log(df['energy(kWh/hh)']+eps) + df['energy(kWh/hh)'] = np.log(df['energy(kWh/hh)']+1e-4) df = df.sort_values('tstp') # split data diff --git a/src/models/lightning_anp.py b/src/models/lightning_anp.py index 0ddc6dc..059f78a 100644 --- a/src/models/lightning_anp.py +++ b/src/models/lightning_anp.py @@ -25,13 +25,14 @@ class LatentModelPL(pl.LightningModule): def training_step(self, batch, batch_idx): assert all(torch.isfinite(d).all() for d in batch) context_x, context_y, target_x, target_y = batch - y_pred, kl, loss, loss_mse, y_std = self.forward(context_x, context_y, target_x, target_y) + y_pred, losses, extra = = self.forward(context_x, context_y, target_x, target_y) + y_std = extra['dist'].scale + tensorboard_logs = { - "train/loss": loss, - "train/kl": kl.mean(), - "train/std": y_std.mean(), - "train/mse": loss_mse.mean(), - "train/mse": F.mse_loss(y_pred, target_y).mean(), + "train_loss": losses['loss'], + "train/kl": losses['loss_kl'].mean(), + "train/std": losses['y_std'].mean(), + "train/mse": losses['loss_mse'].mean(), } assert torch.isfinite(loss) # print('device', next(self.model.parameters()).device) @@ -40,45 +41,65 @@ class LatentModelPL(pl.LightningModule): def validation_step(self, batch, batch_idx): assert all(torch.isfinite(d).all() for d in batch) context_x, context_y, target_x, target_y = batch - y_pred, kl, loss, loss_mse, y_std = self.forward(context_x, context_y, target_x, target_y) + y_pred, losses, extra = self.forward(context_x, context_y, target_x, target_y) + y_std = extra['dist'].scale tensorboard_logs = { - "val_loss": loss, - "val/kl": kl.mean(), - "val/mse": loss_mse.mean(), - "val/std": y_std.mean(), - "val/mse": F.mse_loss(y_pred, target_y).mean(), + "val_loss": losses['loss'], # This exact key is needed for metrics + "val/kl": losses['loss_kl'].mean(), + "val/mse": losses['loss_mse'].mean(), + "val/std": losses['y_std'].mean(), } return {"val_loss": loss, "log": tensorboard_logs} + # def training_end(self, outputs): + # logs = self.agg_logs(outputs) + # tensorboard_logs_str = {k: f'{v}' for k, v in logs["log"].items()} + # print(f"step train {self.trainer.global_step}, {tensorboard_logs_str}") + # return logs + def validation_end(self, outputs): if int(self.hparams["vis_i"]) > 0: - # https://github.com/PytorchLightning/pytorch-lightning/blob/f8d9f8f/pytorch_lightning/core/lightning.py#L293 - loader = self.val_dataloader()[0] - vis_i = min(int(self.hparams["vis_i"]), len(loader.dataset)) - # print('vis_i', vis_i) - if isinstance(self.hparams["vis_i"], str): - image = plot_from_loader(loader, self, i=int(vis_i)) - plt.show() + self.show_image() + logs = self.agg_logs(outputs) + tensorboard_logs_str = {k: f'{v}' for k, v in logs["log"].items()} + print(f"step val {self.trainer.global_step}, {tensorboard_logs_str}") + return logs + + def show_image(self): + # https://github.com/PytorchLightning/pytorch-lightning/blob/f8d9f8f/pytorch_lightning/core/lightning.py#L293 + loader = self.val_dataloader()[0] + vis_i = min(int(self.hparams["vis_i"]), len(loader.dataset)) + # print('vis_i', vis_i) + if isinstance(self.hparams["vis_i"], str): + image = plot_from_loader(loader, self, i=int(vis_i)) + plt.show() + else: + image = plot_from_loader_to_tensor(loader, self, i=vis_i) + self.logger.experiment.add_image('val/image', image, self.trainer.global_step) + + def agg_logs(self, outputs): + if isinstance(outputs, dict): + outputs = [outputs] + aggs = {} + for j in outputs[0]: + if isinstance(outputs[0][j], dict): + # Take mean of sub dicts + keys = outputs[0][j].keys() + aggs[j] = {k: torch.stack([x[j][k] for x in outputs if k in x[j]]).mean() for k in keys} else: - image = plot_from_loader_to_tensor(loader, self, i=vis_i) - self.logger.experiment.add_image('val/image', image, self.trainer.global_step) - - keys = outputs[0]["log"].keys() - # tensorboard_logs = {} - # for k in keys: - # tensorboard_logs[k] = torch.stack([x["log"][k] for x in outputs if k in x["log"]]).mean() - tensorboard_logs = {k: torch.stack([x["log"][k] for x in outputs if k in x["log"]]).mean() for k in keys} + # Take mean of numbers + aggs[j] = torch.stack([x[j] for x in outputs if j in x]).mean() + return aggs - avg_loss = torch.stack([x["val_loss"] for x in outputs]).mean() - assert torch.isfinite(avg_loss) - tensorboard_logs_str = {k: f'{v}' for k, v in tensorboard_logs.items()} - print(f"step {self.trainer.global_step}, {tensorboard_logs_str}") - - # Log hparams with metric, doesn't work - # self.logger.experiment.add_hparams(self.hparams.__dict__, {"avg_val_loss": avg_loss}) - - return {"avg_val_loss": avg_loss, "log": tensorboard_logs} + # # Log hparams with metric, doesn't work + # # self.logger.experiment.add_hparams(self.hparams.__dict__, {"avg_val_loss": avg_loss}) + # if f"{name}_loss" in outputs[0].keys(): + # avg_loss = torch.stack([x[f"{name}_loss"] for x in outputs]).mean() + # assert torch.isfinite(avg_loss) + # else: + # avg_loss = 0 + # return {f"avg_{name}_loss": avg_loss, "log": tensorboard_logs, "progress_bar": {}} def test_step(self, *args, **kwargs): return self.validation_step(*args, **kwargs) @@ -87,8 +108,8 @@ class LatentModelPL(pl.LightningModule): return self.validation_end(*args, **kwargs) def configure_optimizers(self): - optim = torch.optim.AdamW(self.parameters(), lr=self.hparams["learning_rate"]) - scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optim, patience=2, verbose=True, min_lr=1e-5) # note early stopping has patient 3 + optim = torch.optim.Adam(self.parameters(), lr=self.hparams["learning_rate"], weight_decay=0) + scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optim, patience=1, verbose=True, min_lr=1e-7) # note early stopping has patience 3 return [optim], [scheduler] def _get_cache_dfs(self): diff --git a/src/models/lstm_seqseq.py b/src/models/lstm_seqseq.py new file mode 100644 index 0000000..f12cd64 --- /dev/null +++ b/src/models/lstm_seqseq.py @@ -0,0 +1,256 @@ +import os +import numpy as np +import pandas as pd +import torch +from tqdm.auto import tqdm +from torch import nn +from torch.nn import functional as F +from torch.utils.data import DataLoader +from torchvision.datasets import MNIST +from test_tube import Experiment, HyperOptArgumentParser +from src.data.smart_meter import collate_fns, SmartMeterDataSet, get_smartmeter_df +import torchvision.transforms as transforms +from src.plot import plot_from_loader_to_tensor, plot_from_loader +from argparse import ArgumentParser +import json +import pytorch_lightning as pl +import math +from matplotlib import pyplot as plt +import torch +import io +import PIL +from torchvision.transforms import ToTensor + +from src.data.smart_meter import get_smartmeter_df + +from src.utils import ObjectDict + +def log_prob_sigma(value, loc, log_scale): + """A slightly more stable (not confirmed yet) log prob taking in log_var instead of scale. + modified from https://github.com/pytorch/pytorch/blob/2431eac7c011afe42d4c22b8b3f46dedae65e7c0/torch/distributions/normal.py#L65 + """ + var = torch.exp(log_scale * 2) + return ( + -((value - loc) ** 2) / (2 * var) - log_scale - math.log(math.sqrt(2 * math.pi)) + ) + + +class Seq2SeqNet(nn.Module): + def __init__(self, hparams, _min_std = 0.05): + super().__init__() + self.hparams = hparams + self._min_std = _min_std + + self.encoder = nn.LSTM( + input_size=self.hparams.input_size, + hidden_size=self.hparams.hidden_size, + batch_first=True, + num_layers=self.hparams.lstm_layers, + bidirectional=self.hparams.bidirectional, + dropout=self.hparams.lstm_dropout, + ) + self.decoder = nn.LSTM( + input_size=self.hparams.input_size_decoder, + hidden_size=self.hparams.hidden_size, + batch_first=True, + num_layers=self.hparams.lstm_layers, + bidirectional=self.hparams.bidirectional, + dropout=self.hparams.lstm_dropout, + ) + self.hidden_out_size = ( + self.hparams.hidden_size + * (self.hparams.bidirectional + 1) + ) + self.mean = nn.Linear(self.hidden_out_size, self.hparams.output_size) + self.std = nn.Linear(self.hidden_out_size, self.hparams.output_size) + + def forward(self, context_x, context_y, target_x, target_y=None): + x = torch.cat([context_x, context_y], -1) + _, (h_out, cell) = self.encoder(x) + # hidden = [batch size, n layers * n directions, hid dim] + # cell = [batch size, n layers * n directions, hid dim] + outputs, (_, _) = self.decoder(target_x, (h_out, cell)) + # output = [batch size, seq len, hid dim * n directions] + + # outputs: [B, T, num_direction * H] + mean = self.mean(outputs) + log_sigma = self.std(outputs) + log_sigma = torch.clamp(log_sigma, math.log(self._min_std), -math.log(self._min_std)) + + sigma = torch.exp(log_sigma) + y_dist=torch.distributions.Normal(mean, sigma) + + # Loss + loss_mse =loss_p = None + if target_y is not None: + loss_mse = F.mse_loss(mean, target_y, reduction='none') + loss_p = -log_prob_sigma(target_y, mean, log_sigma) + + if self.hparams["context_in_target"]: + loss_p[:context_x.size(1)] /= 100 + loss_mse[:context_x.size(1)] /= 100 + # # Don't catch loss on context window + # mean = mean[:, self.hparams.num_context:] + # log_sigma = log_sigma[:, self.hparams.num_context:] + + y_pred = y_dist.rsample if self.training else y_dist.loc + return y_pred, dict(loss_p=loss_p.mean(), loss_mse=loss_mse.mean()), dict(log_sigma=log_sigma, dist=y_dist) + + +class LSTMSeq2Seq_PL(pl.LightningModule): + def __init__(self, hparams): + # TODO make label name configurable + # TODO make data source configurable + super().__init__() + self.hparams = ObjectDict() + self.hparams.update( + hparams.__dict__ if hasattr(hparams, "__dict__") else hparams + ) + self.model = Seq2SeqNet(self.hparams) + self._dfs = None + + def forward(self, context_x, context_y, target_x, target_y): + return self.model(context_x, context_y, target_x, target_y) + + def training_step(self, batch, batch_idx): + # REQUIRED + assert all(torch.isfinite(d).all() for d in batch) + context_x, context_y, target_x, target_y = batch + y_dist, losses, extra = self.forward(context_x, context_y, target_x, target_y) + loss = losses['loss_p'] # + loss_mse + tensorboard_logs = { + "train/loss": loss, + 'train/loss_mse': losses['loss_mse'], + "train/loss_p": losses['loss_p'], + "train/sigma": torch.exp(extra['log_sigma']).mean()} + return {"loss": loss, "log": tensorboard_logs} + + def validation_step(self, batch, batch_idx): + context_x, context_y, target_x, target_y = batch + assert all(torch.isfinite(d).all() for d in batch) + y_dist, losses, extra = self.forward(context_x, context_y, target_x, target_y) + loss = losses['loss_p'] # + loss_mse + tensorboard_logs = { + "val_loss": loss, + 'val/loss_mse': losses['loss_mse'], + "val/loss_p": losses['loss_p'], + "val/sigma": torch.exp(extra['log_sigma']).mean()} + return {"val_loss": loss, "log": tensorboard_logs} + + def validation_end(self, outputs): + if int(self.hparams["vis_i"]) > 0: + self.show_image() + + avg_loss = torch.stack([x["val_loss"] for x in outputs]).mean() + keys = outputs[0]["log"].keys() + tensorboard_logs = { + k: torch.stack([x["log"][k] for x in outputs if k in x["log"]]).mean() + for k in keys + } + tensorboard_logs_str = {k: f"{v}" for k, v in tensorboard_logs.items()} + print(f"step {self.trainer.global_step}, {tensorboard_logs_str}") + assert torch.isfinite(avg_loss) + return {"avg_val_loss": avg_loss, "log": tensorboard_logs} + + + def show_image(self): + # https://github.com/PytorchLightning/pytorch-lightning/blob/f8d9f8f/pytorch_lightning/core/lightning.py#L293 + loader = self.val_dataloader()[0] + vis_i = min(int(self.hparams["vis_i"]), len(loader.dataset)) + # print('vis_i', vis_i) + if isinstance(self.hparams["vis_i"], str): + image = plot_from_loader(loader, self, i=int(vis_i)) + plt.show() + else: + image = plot_from_loader_to_tensor(loader, self, i=vis_i) + self.logger.experiment.add_image('val/image', image, self.trainer.global_step) + + def test_step(self, *args, **kwargs): + return self.validation_step(*args, **kwargs) + + def test_end(self, *args, **kwargs): + return self.validation_end(*args, **kwargs) + + def configure_optimizers(self): + optim = torch.optim.Adam(self.parameters(), lr=self.hparams["learning_rate"]) + scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau( + optim, patience=2, verbose=True, min_lr=1e-5 + ) # note early stopping has patient 3 + return [optim], [scheduler] + + def _get_cache_dfs(self): + if self._dfs is None: + df_train, df_test = get_smartmeter_df() + # self._dfs = dict(df_train=df_train[:600], df_test=df_test[:600]) + self._dfs = dict(df_train=df_train, df_test=df_test) + return self._dfs + + @pl.data_loader + def train_dataloader(self): + df_train = self._get_cache_dfs()['df_train'] + data_train = SmartMeterDataSet( + df_train, self.hparams["num_context"], self.hparams["num_extra_target"] + ) + return torch.utils.data.DataLoader( + data_train, + batch_size=self.hparams["batch_size"], + shuffle=True, + collate_fn=collate_fns( + self.hparams["num_context"], self.hparams["num_extra_target"], sample=True, context_in_target=self.hparams["context_in_target"] + ), + num_workers=self.hparams["num_workers"], + ) + + @pl.data_loader + def val_dataloader(self): + df_test = self._get_cache_dfs()['df_test'] + data_test = SmartMeterDataSet( + df_test, self.hparams["num_context"], self.hparams["num_extra_target"] + ) + return torch.utils.data.DataLoader( + data_test, + batch_size=self.hparams["batch_size"], + shuffle=False, + collate_fn=collate_fns( + self.hparams["num_context"], self.hparams["num_extra_target"], sample=False, context_in_target=self.hparams["context_in_target"] + ), + ) + + @pl.data_loader + def test_dataloader(self): + df_test = self._get_cache_dfs()['df_test'] + data_test = SmartMeterDataSet( + df_test, self.hparams["num_context"], self.hparams["num_extra_target"] + ) + return torch.utils.data.DataLoader( + data_test, + batch_size=self.hparams["batch_size"], + shuffle=False, + collate_fn=collate_fns( + self.hparams["num_context"], self.hparams["num_extra_target"], sample=False, context_in_target=self.hparams["context_in_target"] + ), + ) + + @staticmethod + def add_model_specific_args(parent_parser): + """ + Specify the hyperparams for this LightningModule + """ + # MODEL specific + parser = HyperOptArgumentParser(parents=[parent_parser]) + parser.add_argument("--learning_rate", default=0.002, type=float) + parser.add_argument("--batch_size", default=16, type=int) + parser.add_argument("--lstm_dropout", default=0.5, type=float) + parser.add_argument("--hidden_size", default=16, type=int) + parser.add_argument("--input_size", default=8, type=int) + parser.add_argument("--input_size_decoder", default=8, type=int) + parser.add_argument("--lstm_layers", default=8, type=int) + parser.add_argument("--bidirectional", default=False, type=bool) + + # training specific (for this model) + parser.add_argument("--num_context", type=int, default=12) + parser.add_argument("--num_extra_target", type=int, default=2) + parser.add_argument("--max_nb_epochs", default=10, type=int) + parser.add_argument("--num_workers", default=4, type=int) + + return parser diff --git a/src/models/model.py b/src/models/model.py index 65d3372..c0cfe63 100644 --- a/src/models/model.py +++ b/src/models/model.py @@ -35,25 +35,28 @@ def kl_loss_var(prior_mu, log_var_prior, post_mu, log_var_post): class LatentModel(nn.Module): def __init__(self, - x_dim, - y_dim, - hidden_dim=32, - latent_dim=32, - latent_enc_self_attn_type="dot", - det_enc_self_attn_type="dot", - det_enc_cross_attn_type="dot", - n_latent_encoder_layers=3, - n_det_encoder_layers=3, - n_decoder_layers=3, - use_deterministic_path=True, - min_std=0.01, + x_dim, # features in input + y_dim, # number of features in output + hidden_dim=32, # size of hidden space + latent_dim=32, # size of latent space + latent_enc_self_attn_type="ptmultihead", # type of attention: "uniform", "dot", "multihead" "ptmultihead": see attentive neural processes paper + det_enc_self_attn_type="ptmultihead", + det_enc_cross_attn_type="ptmultihead", + n_latent_encoder_layers=2, + n_det_encoder_layers=2, # number of deterministic encoder layers + n_decoder_layers=2, + use_deterministic_path=False, + min_std=0.01, # To avoid collapse use a minimum standard deviation, should be much smaller than variation in labels dropout=0, use_self_attn=False, attention_dropout=0, batchnorm=False, - use_lvar=False, - attention_layers=2, - use_rnn=False, + use_lvar=False, # Alternative loss calculation, may be more stable + attention_layers=2, + use_rnn=True, # use RNN/LSTM? + use_lstm_le=False, # use another LSTM in latent encoder instead of MLP + use_lstm_de=False, # use another LSTM in determinstic encoder instead of MLP + use_lstm_d=False, # use another lstm in decoder instead of MLP **kwargs, ): @@ -84,6 +87,7 @@ class LatentModel(nn.Module): batchnorm=batchnorm, min_std=min_std, use_lvar=use_lvar, + use_lstm=use_lstm_le, ) self._deterministic_encoder = DeterministicEncoder( @@ -98,6 +102,7 @@ class LatentModel(nn.Module): dropout=dropout, batchnorm=batchnorm, attention_dropout=attention_dropout, + use_lstm=use_lstm_de, ) self._decoder = Decoder( @@ -111,37 +116,34 @@ class LatentModel(nn.Module): use_lvar=use_lvar, n_decoder_layers=n_decoder_layers, use_deterministic_path=use_deterministic_path, + use_lstm=use_lstm_d, ) self._use_deterministic_path = use_deterministic_path self._use_lvar = use_lvar def forward(self, context_x, context_y, target_x, target_y=None): - num_targets = target_x.size(1) if self._use_rnn: # see https://arxiv.org/abs/1910.09323 where x is substituted with h = RNN(x) # x need to be provided as [B, T, H] - x = torch.cat([context_x, target_x], dim=1) - # h: [B, T, num_direction * H] - h, _ = self._lstm(x) - context_x = h[:, :context_x.shape[1], :] - target_x = h[:, context_x.shape[1]:, :] + target_x, _ = self._lstm(target_x) + context_x, _ = self._lstm(context_x) dist_prior, log_var_prior = self._latent_encoder(context_x, context_y) if target_y is not None: - dist_post, log_var_post = self._latent_encoder(target_x, - target_y) + dist_post, log_var_post = self._latent_encoder(target_x, target_y) z = dist_post.loc else: z = dist_prior.loc + num_targets = target_x.size(1) z = z.unsqueeze(1).repeat(1, num_targets, 1) # [B, T_target, H] if self._use_deterministic_path: r = self._deterministic_encoder(context_x, context_y, - target_x) # [B, T_target, H] + target_x) # [B, T_target, H] else: r = None @@ -150,14 +152,18 @@ class LatentModel(nn.Module): if self._use_lvar: log_p = log_prob_sigma(target_y, dist.loc, log_sigma).mean(-1) # [B, T_target, Y].mean(-1) + if self.hparams["context_in_target"]: + log_p[:, :context_x.size(1)] /= 100 kl_loss = kl_loss_var(dist_prior.loc, log_var_prior, dist_post.loc, log_var_post).mean(-1) # [B, R].mean(-1) else: log_p = dist.log_prob(target_y).mean(-1) + if self.hparams["context_in_target"]: + log_p[:, :context_x.size(1)] /= 100 # There's the temptation for it to fit only on context, where it knows the answer, and learn very low uncertainty. kl_loss = torch.distributions.kl_divergence( - dist_post, dist_prior).mean(-1) + dist_post, dist_prior).mean(-1) # [B, R].mean(-1) kl_loss = kl_loss[:, None].expand(log_p.shape) - mse_loss = F.mse_loss(dist.loc, target_y) + mse_loss = F.mse_loss(dist.loc, target_y, reduce=None)[:, :context_x.size(1)].mean() loss = (kl_loss - log_p).mean() else: @@ -167,4 +173,4 @@ class LatentModel(nn.Module): loss = None y_pred = dist.rsample() if self.training else dist.loc - return y_pred, kl_loss, loss, mse_loss, dist.scale + return y_pred, dict(loss=loss, loss_p=loss_p.mean(), loss_kl=loss_kl, loss_mse=mse_loss.mean()), dict(log_sigma=log_sigma, dist=dist) diff --git a/src/models/modules.py b/src/models/modules.py index d6743e9..f3c6692 100644 --- a/src/models/modules.py +++ b/src/models/modules.py @@ -6,6 +6,24 @@ import numpy as np # from .attention import Attention as PtAttention +class LSTMBlock(nn.Module): + def __init__( + self, in_channels, out_channels, dropout=0, batchnorm=False, bias=False, num_layers=1 + ): + super().__init__() + self._lstm = nn.LSTM( + input_size=in_channels, + hidden_size=out_channels, + num_layers=num_layers, + dropout=dropout, + batch_first=True, + bias=bias + ) + + def forward(self, x): + return self._lstm(x)[0] + + class NPBlockRelu2d(nn.Module): """Block for Neural Processes.""" @@ -208,17 +226,14 @@ class LatentEncoder(nn.Module): use_lvar=False, use_self_attn=False, attention_layers=2, + use_lstm=False ): super().__init__() - self._input_layer = nn.Linear(input_dim, hidden_dim) - self._encoder = nn.ModuleList( - [ - NPBlockRelu2d( - hidden_dim, hidden_dim, batchnorm=batchnorm, dropout=dropout - ) - for _ in range(n_encoder_layers) - ] - ) + # self._input_layer = nn.Linear(input_dim, hidden_dim) + if use_lstm: + self._encoder = LSTMBlock(input_dim, hidden_dim, batchnorm=batchnorm, dropout=dropout, num_layers=n_encoder_layers) + else: + self._encoder = BatchMLP(input_dim, hidden_dim, batchnorm=batchnorm, dropout=dropout, num_layers=n_encoder_layers) if use_self_attn: self._self_attention = Attention( hidden_dim, @@ -232,15 +247,14 @@ class LatentEncoder(nn.Module): self._log_var = nn.Linear(hidden_dim, latent_dim) self._min_std = min_std self._use_lvar = use_lvar + self._use_lstm = use_lstm self._use_self_attn = use_self_attn def forward(self, x, y): encoder_input = torch.cat([x, y], dim=-1) # Pass final axis through MLP - encoded = self._input_layer(encoder_input) - for layer in self._encoder: - encoded = torch.relu(layer(encoded)) + encoded = self._encoder(encoder_input) # Aggregator: take the mean over all points if self._use_self_attn: @@ -282,21 +296,15 @@ class DeterministicEncoder(nn.Module): batchnorm=False, dropout=0, attention_dropout=0, + use_lstm=False, ): super().__init__() self._use_self_attn = use_self_attn - self._input_layer = nn.Linear(input_dim, hidden_dim) - self._d_encoder = nn.ModuleList( - [ - NPBlockRelu2d( - hidden_dim, - hidden_dim, - batchnorm=batchnorm, - dropout=attention_dropout, - ) - for _ in range(n_d_encoder_layers) - ] - ) + # self._input_layer = nn.Linear(input_dim, hidden_dim) + if use_lstm: + self._d_encoder = LSTMBlock(input_dim, hidden_dim, batchnorm=batchnorm, dropout=dropout, num_layers=n_d_encoder_layers) + else: + self._d_encoder = BatchMLP(input_dim, hidden_dim, batchnorm=batchnorm, dropout=dropout, num_layers=n_d_encoder_layers) if use_self_attn: self._self_attention = Attention( hidden_dim, @@ -317,14 +325,12 @@ class DeterministicEncoder(nn.Module): d_encoder_input = torch.cat([context_x, context_y], dim=-1) # Pass final axis through MLP - d_encoded = self._input_layer(d_encoder_input) - for layer in self._d_encoder: - d_encoded = torch.relu(layer(d_encoded)) + d_encoded = self._d_encoder(d_encoder_input) if self._use_self_attn: d_encoded = self._self_attention(d_encoded, d_encoded, d_encoded) - # Apply attention + # Apply attention as mean aggregation h = self._cross_attention(context_x, d_encoded, target_x) return h @@ -343,6 +349,7 @@ class Decoder(nn.Module): use_lvar=False, batchnorm=False, dropout=0, + use_lstm=False, ): super(Decoder, self).__init__() self._target_transform = nn.Linear(x_dim, hidden_dim) @@ -350,14 +357,11 @@ class Decoder(nn.Module): hidden_dim_2 = 2 * hidden_dim + latent_dim else: hidden_dim_2 = hidden_dim + latent_dim - self._decoder = nn.ModuleList( - [ - NPBlockRelu2d( - hidden_dim_2, hidden_dim_2, batchnorm=batchnorm, dropout=dropout - ) - for _ in range(n_decoder_layers) - ] - ) + + if use_lstm: + self._decoder = LSTMBlock(hidden_dim_2, hidden_dim_2, batchnorm=batchnorm, dropout=dropout, num_layers=n_decoder_layers) + else: + self._decoder = BatchMLP(hidden_dim_2, hidden_dim_2, batchnorm=batchnorm, dropout=dropout, num_layers=n_decoder_layers) self._mean = nn.Linear(hidden_dim_2, y_dim) self._std = nn.Linear(hidden_dim_2, y_dim) self._use_deterministic_path = use_deterministic_path @@ -371,19 +375,17 @@ class Decoder(nn.Module): if self._use_deterministic_path: z = torch.cat([r, z], dim=-1) - representation = torch.cat([z, x], dim=-1) + r = torch.cat([z, x], dim=-1) - # Pass final axis through MLP - for layer in self._decoder: - representation = torch.relu(layer(representation)) + r = self._decoder(r) # Get the mean and the variance - mean = self._mean(representation) - log_sigma = self._std(representation) + mean = self._mean(r) + log_sigma = self._std(r) # Bound or clamp the variance if self._use_lvar: - log_sigma = torch.clamp(log_sigma, math.log(self._min_std), -math.log(1e-5)) + log_sigma = torch.clamp(log_sigma, math.log(self._min_std), -math.log(self._min_std)) sigma = torch.exp(log_sigma) else: sigma = self._min_std + (1 - self._min_std) * F.softplus(log_sigma) diff --git a/src/plot.py b/src/plot.py index 9ce5481..e0ade0e 100644 --- a/src/plot.py +++ b/src/plot.py @@ -13,8 +13,10 @@ eps = 1e-5 def plot_rows( - target_y_rows: pd.DataFrame, + x_context_rows: pd.DataFrame, + x_target_rows: pd.DataFrame, context_y_rows: pd.DataFrame, + target_y_rows: pd.DataFrame, pred_y: np.array, std: np.array, undo_log=False, @@ -23,8 +25,10 @@ def plot_rows( """Plots the predicted mean and variance and the context points. Args: - target_y_rows + x_context_rows + x_target_rows context_y_rows: dataframe with datetime index, and labels + target_y_rows: pred_y: An array of shape [B,num_targets,1] that contains the predicted means of the y values at the target points in target_x. std: An array of shape [B,num_targets,1] that contains the @@ -38,6 +42,8 @@ def plot_rows( j = 0 label = "energy(kWh/hh)" + # Plot input data + # Start with true data and use it to get ylimits (that way they are constant) plt.plot(target_y_rows.index, target_y_rows.values, "k:", linewidth=2, label="true") plt.plot(context_y_rows.index, context_y_rows.values, "k:", linewidth=2, label="true") @@ -97,6 +103,8 @@ def plot_from_loader( max_num_context = context_x.shape[1] y_context_rows = y_rows[:max_num_context] y_target_extra_rows = y_rows[max_num_context:] + x_context_rows = x_rows[:max_num_context] + x_target_extra_rows = x_rows[max_num_context:] dt = y_target_extra_rows.index[0] # # for the plotting we are doing to run prediction on the context points too @@ -104,17 +112,23 @@ def plot_from_loader( target_x = torch.cat([context_x, target_x_extra], 1) target_y = torch.cat([context_y, target_y_extra], 1) y_target_rows = y_rows + x_target_rows = x_rows model.eval() with torch.no_grad(): - y_pred, kl, loss_test, loss_mse, y_std = model(context_x, context_y, target_x, target_y) + y_pred, losses, extra = model(context_x, context_y, target_x, target_y) + loss_test = losses["loss"] if "loss" in losses else 0. + + y_std = extra["dist"].scale if plot: plt.figure() plt.title(title + f" loss={loss_test: 2.2g} {dt}") plot_rows( - y_target_rows, + x_context_rows, + x_target_rows, y_context_rows, + y_target_rows, y_pred.detach().cpu().numpy(), y_std.detach().cpu().numpy(), undo_log=False,