diff --git a/neural_processes/lightning.py b/neural_processes/lightning.py index 299a431..b21179d 100644 --- a/neural_processes/lightning.py +++ b/neural_processes/lightning.py @@ -1,15 +1,13 @@ import pytorch_lightning as pl import torch -import optuna -import torch from torch import nn -from pathlib import Path -from pytorch_lightning.callbacks import EarlyStopping -from optuna.integration.pytorch_lightning import _check_pytorch_lightning_availability import torch.nn.functional as F +from matplotlib import pyplot as plt + from .utils import ObjectDict -from .data.smart_meter import get_smartmeter_df +from .data.smart_meter import get_smartmeter_df, SmartMeterDataSet, collate_fns from .logger import logger +from .plot import plot_from_loader class PL_Seq2Seq(pl.LightningModule): diff --git a/neural_processes/models/lstm_seqseq.py b/neural_processes/models/lstm_seqseq.py index 6a72dc3..d6ad0e8 100644 --- a/neural_processes/models/lstm_seqseq.py +++ b/neural_processes/models/lstm_seqseq.py @@ -119,6 +119,8 @@ class LSTMSeq2Seq_PL(PL_Seq2Seq): MODEL_CLS=Seq2SeqNet, **kwargs): super().__init__(hparams, MODEL_CLS=MODEL_CLS, **kwargs) + + DEFAULT_ARGS = {'agg': 'mean', 'lstm_dropout': 0.12013231612195126, 'hidden_out_size_power': 4.0, 'hidden_size_power': 7.0, 'learning_rate': 0.0022924639229335475, 'nhead_power': 2.0, 'nlayers_power': 4.0} @staticmethod def add_suggest(trial): diff --git a/neural_processes/models/lstm_std.py b/neural_processes/models/lstm_std.py index 590f406..9fe764e 100644 --- a/neural_processes/models/lstm_std.py +++ b/neural_processes/models/lstm_std.py @@ -22,67 +22,15 @@ from torchvision.transforms import ToTensor from neural_processes.data.smart_meter import get_smartmeter_df from neural_processes.utils import ObjectDict +from ..lightning import PL_Seq2Seq from torch.utils.data._utils.collate import default_collate def collate_fn(batch, sample=None): return default_collate(batch) -def log_prob_sigma(value, loc, log_scale): - """A slightly more stable (not confirmed yet) log prob taking in log_var instead of scale. - modified from https://github.com/pytorch/pytorch/blob/2431eac7c011afe42d4c22b8b3f46dedae65e7c0/torch/distributions/normal.py#L65 - """ - var = torch.exp(log_scale * 2) - return ( - -((value - loc) ** 2) / (2 * var) - log_scale - math.log(math.sqrt(2 * math.pi)) - ) - -class SequenceDfDataSet(torch.utils.data.Dataset): - def __init__(self, df, hparams, label_names=None, train=True, transforms=None): - super().__init__() - self.data = df - self.hparams = hparams - self.label_names = label_names - self.train = train - self.transforms = transforms - - def __len__(self): - return len(self.data) - self.hparams.window_length - self.hparams.target_length - 1 - - def iloc(self, idx): - k = idx + self.hparams.window_length + self.hparams.target_length - j = k - self.hparams.target_length - i = j - self.hparams.window_length - assert i >= 0 - assert idx <= len(self.data) - - x_rows = self.data.iloc[i:k].copy() - # x_rows = x_rows.drop(columns=self.label_names) - # Note the NP models do have access to the previous labels for the context, we will allow the LSTM to do the same. Although it will likely just return an autoregressive solution for the first half... - x_rows.loc[x_rows.index[self.hparams.window_length:], self.label_names] = 0 - assert len(x_rows.loc[x_rows.index[self.hparams.window_length:], self.label_names])>0 - assert (x_rows.loc[x_rows.index[self.hparams.window_length:], self.label_names]==0).all().all() - - y_rows = self.data[self.label_names].iloc[i+1:k+1].copy() - # print(i,j,k) - - # add seconds since start of window index - x_rows["tstp"] = ( - x_rows["tstp"] - x_rows["tstp"].iloc[0] - ).dt.total_seconds() / 86400.0 - return x_rows, y_rows - - def __getitem__(self, idx): - x_rows, y_rows = self.iloc(idx) - - x = x_rows.astype(np.float32).values - y = y_rows[self.label_names].astype(np.float32).values - return ( - self.transforms(x).squeeze(0).float(), - self.transforms(y).squeeze(0).squeeze(-1).float(), - ) - class LSTMNet(nn.Module): + def __init__(self, hparams, _min_std = 0.05): super().__init__() self.hparams = hparams @@ -103,164 +51,31 @@ class LSTMNet(nn.Module): self.mean = nn.Linear(self.hidden_out_size, 1) self.std = nn.Linear(self.hidden_out_size, 1) - def forward(self, x): + def forward(self, context_x, context_y, target_x, target_y=None): + device = next(self.parameters()).device + x = torch.cat([context_x, context_y], -1).detach() + outputs, (h_out, _) = self.lstm1(x) # outputs: [B, T, num_direction * H] - mean = self.mean(outputs).squeeze(2) + y_pred = self.mean(outputs).squeeze(2) log_sigma = self.std(outputs).squeeze(2) - # log_sigma = torch.clamp(log_sigma, math.log(self._min_std), -math.log(self._min_std)) - return mean, log_sigma + + loss = None + if target_y is not None: + loss = F.mse_loss(y_pred * loss_scale, y[:, -steps:, :] * loss_scale, reduction='none') / loss_scale + + assert torch.isfinite(loss) + + return y_pred, dict(loss=loss), dict() -class LSTM_PL_STD(pl.LightningModule): - def __init__(self, hparams): - # TODO make label name configurable - # TODO make data source configurable - super().__init__() - self.hparams = ObjectDict() - self.hparams.update( - hparams.__dict__ if hasattr(hparams, "__dict__") else hparams - ) - self._model = LSTMNet(self.hparams) - self._dfs = None - self._use_lvar = False - self._min_std = 0.005 +class LSTM_PL_STD(PL_Seq2Seq): + def __init__(self, hparams, + MODEL_CLS=LSTMNet, **kwargs): + super().__init__(hparams, + MODEL_CLS=MODEL_CLS, **kwargs) - self.default_args = {'bidirectional': False, 'hidden_size_power': 4, 'learning_rate': 0.0010825329363784934, 'lstm_dropout': 0.3905792111699782, 'lstm_layers': 4} - - def forward(self, x): - return self._model(x) - - def training_step(self, batch, batch_idx): - # REQUIRED - x, y = batch - mean, log_sigma = self.forward(x) - - # Don't catch loss on context window - mean = mean[:, self.hparams.window_length:] - log_sigma = log_sigma[:, self.hparams.window_length:] - if self._use_lvar: - log_sigma = torch.clamp(log_sigma, math.log(self._min_std), -math.log(self._min_std)) - sigma = torch.exp(log_sigma) - else: - sigma = self._min_std + (1 - self._min_std) * F.softplus(log_sigma) - y_dist = torch.distributions.Normal(mean, sigma) - - y = y[:, self.hparams.window_length:] - - loss_mse = F.mse_loss(mean, y).mean() - if self._use_lvar: - loss_p = -log_prob_sigma(y, mean, log_sigma).mean() - else: - loss_p = -y_dist.log_prob(y).mean() - loss = loss_p # + loss_mse - tensorboard_logs = {"train/loss": loss, 'train/loss_mse': loss_mse, "train/loss_p": loss_p, "train/sigma": sigma.mean()} - return {"loss": loss, "log": tensorboard_logs} - - def validation_step(self, batch, batch_idx): - x, y = batch - mean, log_sigma = self.forward(x) - - # Don't catch loss on context window - mean = mean[:, self.hparams.window_length:] - log_sigma = log_sigma[:, self.hparams.window_length:] - - sigma = torch.exp(log_sigma) - y_dist = torch.distributions.Normal(mean, sigma) - - y = y[:, self.hparams.window_length:] - - loss_mse = F.mse_loss(mean, y).mean() - loss_p = -log_prob_sigma(y, mean, log_sigma).mean() - loss = loss_p # + loss_mse - tensorboard_logs = {"val_loss": loss, 'val/loss':loss, 'val/loss_mse': loss_mse, "val/loss_p": loss_p, "val/sigma": sigma.mean()} - return {"val_loss": loss, "log": tensorboard_logs} - - def validation_end(self, outputs): - # TODO send an image to tensroboard, like in the lighting_anp.py file - if int(self.hparams["vis_i"]) > 0: - loader = self.val_dataloader() - vis_i = min(int(self.hparams["vis_i"]), len(loader.dataset)) - if isinstance(self.hparams["vis_i"], str): - image = plot_from_loader(loader, self, i=vis_i) - plt.show() - else: - image = plot_from_loader_to_tensor(loader, self, i=vis_i) - self.logger.experiment.add_image( - "val/image", image, self.trainer.global_step - ) - - avg_loss = torch.stack([x["val_loss"] for x in outputs]).mean() - keys = outputs[0]["log"].keys() - tensorboard_logs = { - k: torch.stack([x["log"][k] for x in outputs if k in x["log"]]).mean() - for k in keys - } - tensorboard_logs_str = {k: f"{v}" for k, v in tensorboard_logs.items()} - print(f"step {self.trainer.global_step}, {tensorboard_logs_str}") - assert torch.isfinite(avg_loss) - return {"avg_val_loss": avg_loss, "log": tensorboard_logs} - - def test_step(self, *args, **kwargs): - return self.validation_step(*args, **kwargs) - - def test_end(self, *args, **kwargs): - return self.validation_end(*args, **kwargs) - - def configure_optimizers(self): - optim = torch.optim.Adam(self.parameters(), lr=self.hparams["learning_rate"]) - scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau( - optim, patience=self.hparams["patience"], verbose=True, min_lr=1e-5 - ) # note early stopping has patient 3 - return [optim], [scheduler] - - def _get_cache_dfs(self): - if self._dfs is None: - df_train, df_val, df_test = get_smartmeter_df() - self._dfs = dict(df_train=df_train, df_val=df_val, df_test=df_test) - return self._dfs - - def train_dataloader(self): - df_train = self._get_cache_dfs()["df_train"] - dset_train = SequenceDfDataSet( - df_train, - self.hparams, - label_names=["energy(kWh/hh)"], - transforms=transforms.ToTensor(), - train=True, - ) - return DataLoader( - dset_train, - batch_size=self.hparams.batch_size, - shuffle=True, - collate_fn=collate_fn, - num_workers=self.hparams.num_workers, - ) - - def val_dataloader(self): - df_test = self._get_cache_dfs()["df_val"] - dset_test = SequenceDfDataSet( - df_test, - self.hparams, - label_names=["energy(kWh/hh)"], - train=False, - - transforms=transforms.ToTensor(), - ) - return DataLoader(dset_test, batch_size=self.hparams.batch_size, shuffle=False,collate_fn=collate_fn,) - - @pl.data_loader - def test_dataloader(self): - df_test = self._get_cache_dfs()["df_test"] - dset_test = SequenceDfDataSet( - df_test, - self.hparams, - label_names=["energy(kWh/hh)"], - train=False, - - transforms=transforms.ToTensor(), - ) - return DataLoader(dset_test, batch_size=self.hparams.batch_size, shuffle=False, collate_fn=collate_fn,) + DEFAULT_ARGS = {'bidirectional': False, 'hidden_size_power': 4, 'learning_rate': 0.0010825329363784934, 'lstm_dropout': 0.3905792111699782, 'lstm_layers': 4} @staticmethod def add_suggest(trial): @@ -287,75 +102,3 @@ class LSTM_PL_STD(pl.LightningModule): } return trial - -def plot_from_loader(loader, model, title='', i=670, n=1, window_len=0): - dset_test = loader.dataset - label_names = dset_test.label_names - y_trues = [] - y_preds = [] - vis_i = min(i, len(dset_test)) - for i in tqdm(range(vis_i, vis_i + n)): - x_rows, y_rows = dset_test.iloc(i) - x, y = dset_test[i] - device = next(model.parameters()).device - x = x[None, :].to(device) - model.eval() - with torch.no_grad(): - y_hat, log_sigma = model.forward(x) - y_hat = y_hat.cpu().squeeze(0).numpy() - sigma = log_sigma.exp().cpu().squeeze(0).numpy() - - dt = y_rows.iloc[0].name - - y_hat_rows = y_rows.copy() - y_hat_rows[label_names[0]] = y_hat - y_hat_rows['sigma'] = sigma - y_trues.append(y_rows) - y_preds.append(y_hat_rows) - - df_trues = pd.concat(y_trues) - df_preds = pd.concat(y_preds) - - plt.figure() - df_trues[label_names[0]].plot(label="y_true", style="k:") - ylims = plt.ylim() - df_preds[label_names[0]][window_len:].plot(label="y_pred", style="b", linewidth=2,) - - std = df_preds['sigma'][window_len:] - mean = df_preds[label_names[0]][window_len:] - plt.fill_between( - df_preds.index[window_len:], - mean - std, - mean + std, - alpha=0.25, - facecolor="blue", - interpolate=True, - label="uncertainty", - ) - plt.fill_between( - df_preds.index[window_len:], - mean - std * 2, - mean + std * 2, - alpha=0.125, - facecolor="blue", - interpolate=True, - label="uncertainty", - ) - plt.legend() - t_ahead = pd.Timedelta("30T") * model.hparams.target_length - plt.title(f"predicting {t_ahead} ahead. {title}") - plt.ylim(*ylims) - # plt.show() - - -def plot_from_loader_to_tensor(*args, **kwargs): - plot_from_loader(*args, **kwargs) - - # Send fig to tensorboard - buf = io.BytesIO() - plt.savefig(buf, format="jpeg") - plt.close() - buf.seek(0) - image = PIL.Image.open(buf) - image = ToTensor()(image) # .unsqueeze(0) - return image diff --git a/neural_processes/models/neural_process/lightning.py b/neural_processes/models/neural_process/lightning.py index c96466b..569802f 100644 --- a/neural_processes/models/neural_process/lightning.py +++ b/neural_processes/models/neural_process/lightning.py @@ -5,12 +5,12 @@ from argparse import ArgumentParser from test_tube import Experiment, HyperOptArgumentParser from .model import NeuralProcess from neural_processes.lightning import PL_Seq2Seq - +from neural_processes.utils import ObjectDict class PL_NeuralProcess(PL_Seq2Seq): def __init__(self, hparams, - MODEL_CLS=NeuralProcess, **kwargs): + MODEL_CLS=NeuralProcess.FROM_HPARAMS, **kwargs): super().__init__(hparams, MODEL_CLS=MODEL_CLS, **kwargs) diff --git a/neural_processes/models/neural_process/model.py b/neural_processes/models/neural_process/model.py index 052d4b5..54be582 100644 --- a/neural_processes/models/neural_process/model.py +++ b/neural_processes/models/neural_process/model.py @@ -4,7 +4,7 @@ import torch.nn.functional as F from torch.utils.data import TensorDataset, DataLoader import math -from neural_processes.modules import BatchNormSequence +from neural_processes.modules import BatchNormSequence, BatchMLP, Attention, LSTMBlock from neural_processes.utils import kl_loss_var, log_prob_sigma @@ -192,6 +192,11 @@ class Decoder(nn.Module): class NeuralProcess(nn.Module): + + @staticmethod + def FROM_HPARAMS(hparams): + return NeuralProcess(**hparams) + def __init__(self, x_dim, # features in input y_dim, # number of features in output @@ -350,6 +355,8 @@ class NeuralProcess(nn.Module): mse_loss = F.mse_loss(dist.loc, target_y, reduction='none')[:,:context_x.size(1)].mean() loss_p = -log_p.mean() loss = (loss_kl - log_p).mean() + loss_kl = loss_kl.mean() + log_p = log_p.mean() else: loss_p = None diff --git a/neural_processes/models/transformer.py b/neural_processes/models/transformer.py index 053fd46..92917ac 100644 --- a/neural_processes/models/transformer.py +++ b/neural_processes/models/transformer.py @@ -122,7 +122,7 @@ class PL_Transformer(PL_Seq2Seq): super().__init__(hparams, MODEL_CLS=MODEL_CLS, **kwargs) - self.default_args = {'attention_dropout': 0.4151003234623061, 'hidden_out_size_power': 2.0, 'hidden_size_power': 2.0, 'learning_rate': 0.0026738884132767185, 'nhead_power': 1.0, 'nlayers_power': 1.0} + DEFAULT_ARGS = {'attention_dropout': 0.4151003234623061, 'hidden_out_size_power': 2.0, 'hidden_size_power': 2.0, 'learning_rate': 0.0026738884132767185, 'nhead_power': 1.0, 'nlayers_power': 1.0} @staticmethod def add_suggest(trial: optuna.Trial, user_attrs={}): diff --git a/neural_processes/models/transformer_seq2seq.py b/neural_processes/models/transformer_seq2seq.py index 03a21b1..6a15179 100644 --- a/neural_processes/models/transformer_seq2seq.py +++ b/neural_processes/models/transformer_seq2seq.py @@ -28,16 +28,6 @@ from neural_processes.modules import BatchNormSequence from neural_processes.utils import ObjectDict from neural_processes.lightning import PL_Seq2Seq -def log_prob_sigma(value, loc, log_scale): - """A slightly more stable (not confirmed yet) log prob taking in log_var instead of scale. - modified from https://github.com/pytorch/pytorch/blob/2431eac7c011afe42d4c22b8b3f46dedae65e7c0/torch/distributions/normal.py#L65 - """ - var = torch.exp(log_scale * 2) - return ( - -((value - loc) ** 2) / (2 * var) - log_scale - math.log(math.sqrt(2 * math.pi)) - ) - - class TransformerSeq2SeqNet(nn.Module): def __init__(self, hparams, _min_std = 0.05): super().__init__() @@ -143,7 +133,8 @@ class TransformerSeq2Seq_PL(PL_Seq2Seq): MODEL_CLS=TransformerSeq2SeqNet, **kwargs): super().__init__(hparams, MODEL_CLS=MODEL_CLS, **kwargs) - self.default_args = {'agg': 'mean', 'attention_dropout': 0.12013231612195126, 'hidden_out_size_power': 4.0, 'hidden_size_power': 7.0, 'learning_rate': 0.0022924639229335475, 'nhead_power': 2.0, 'nlayers_power': 4.0} + + DEFAULT_ARGS = {'agg': 'mean', 'attention_dropout': 0.12013231612195126, 'hidden_out_size_power': 4.0, 'hidden_size_power': 7.0, 'learning_rate': 0.0022924639229335475, 'nhead_power': 2.0, 'nlayers_power': 4.0} @staticmethod def add_suggest(trial: optuna.Trial): diff --git a/neural_processes/modules/attention.py b/neural_processes/modules/attention.py index 2f51a68..2e199b4 100644 --- a/neural_processes/modules/attention.py +++ b/neural_processes/modules/attention.py @@ -2,6 +2,7 @@ import torch from torch import nn import torch.nn.functional as F +from .modules import BatchMLP class AttnLinear(nn.Module): def __init__(self, in_channels, out_channels): diff --git a/neural_processes/train.py b/neural_processes/train.py index d817ff9..b7d4345 100644 --- a/neural_processes/train.py +++ b/neural_processes/train.py @@ -91,9 +91,9 @@ def run_trial( print(f"now run `tensorboard --logdir {MODEL_DIR}`") (MODEL_DIR / name).mkdir(parents=True, exist_ok=True) - if getattr(PL_MODEL_CLS, 'default_args', None): + if getattr(PL_MODEL_CLS, 'DEFAULT_ARGS', None): # add default args - params = {**PL_MODEL_CLS.default_args, **params} + params = {**PL_MODEL_CLS.DEFAULT_ARGS, **params} else: logger.warning(f"No default args on {PL_MODEL_CLS}") @@ -106,7 +106,7 @@ def run_trial( # Add user attributes trial._user_attrs.update(user_attrs) - print(trial) + print('trial', trial) model, trainer = main( trial, PL_MODEL_CLS, name=name, MODEL_DIR=MODEL_DIR, train=False, prune=False diff --git a/smartmeters-ANP-RNN.ipynb b/smartmeters-ANP-RNN.ipynb index e3cdcd0..d60197e 100644 --- a/smartmeters-ANP-RNN.ipynb +++ b/smartmeters-ANP-RNN.ipynb @@ -39,11 +39,11 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 1, "metadata": { "ExecuteTime": { - "end_time": "2020-04-11T04:49:21.604845Z", - "start_time": "2020-04-11T04:49:21.489780Z" + "end_time": "2020-04-11T05:31:36.604028Z", + "start_time": "2020-04-11T05:31:34.179270Z" } }, "outputs": [], @@ -69,27 +69,11 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 2, "metadata": { "ExecuteTime": { - "end_time": "2020-04-11T04:49:21.753642Z", - "start_time": "2020-04-11T04:49:21.657550Z" - } - }, - "outputs": [], - "source": [ - "import logging\n", - "logging.basicConfig(stream=sys.stdout, level=logging.INFO)\n", - "logger = logging.getLogger(\"RANP.ipynb\")" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": { - "ExecuteTime": { - "end_time": "2020-04-11T04:49:22.002185Z", - "start_time": "2020-04-11T04:49:21.874963Z" + "end_time": "2020-04-11T05:31:36.672837Z", + "start_time": "2020-04-11T05:31:36.609199Z" } }, "outputs": [], @@ -101,14 +85,25 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 3, "metadata": { "ExecuteTime": { - "end_time": "2020-04-11T04:49:22.150671Z", - "start_time": "2020-04-11T04:49:22.036140Z" + "end_time": "2020-04-11T05:31:36.811071Z", + "start_time": "2020-04-11T05:31:36.675979Z" } }, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/wassname/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/pytorch_lightning/core/decorators.py:13: UserWarning:\n", + "\n", + "data_loader decorator deprecated in 0.7.0. Will remove 0.9.0\n", + "\n" + ] + } + ], "source": [ "\n", "from neural_processes.data.smart_meter import collate_fns, SmartMeterDataSet, get_smartmeter_df\n", @@ -129,11 +124,11 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 4, "metadata": { "ExecuteTime": { - "end_time": "2020-04-11T04:49:22.284717Z", - "start_time": "2020-04-11T04:49:22.177605Z" + "end_time": "2020-04-11T05:31:36.865423Z", + "start_time": "2020-04-11T05:31:36.814579Z" } }, "outputs": [], @@ -143,6 +138,22 @@ "use_logy=False" ] }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "ExecuteTime": { + "end_time": "2020-04-11T05:31:36.918980Z", + "start_time": "2020-04-11T05:31:36.868490Z" + } + }, + "outputs": [], + "source": [ + "import logging\n", + "logging.basicConfig(stream=sys.stdout, level=logging.INFO)\n", + "logger = logging.getLogger(\"RANP.ipynb\")" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -152,11 +163,11 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 6, "metadata": { "ExecuteTime": { - "end_time": "2020-04-11T04:49:31.807971Z", - "start_time": "2020-04-11T04:49:22.904020Z" + "end_time": "2020-04-11T05:31:45.902849Z", + "start_time": "2020-04-11T05:31:36.921263Z" } }, "outputs": [], @@ -166,21 +177,21 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 7, "metadata": { "ExecuteTime": { - "end_time": "2020-04-11T04:49:32.477716Z", - "start_time": "2020-04-11T04:49:31.811381Z" + "end_time": "2020-04-11T05:31:46.602152Z", + "start_time": "2020-04-11T05:31:45.906531Z" } }, "outputs": [ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 16, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" }, @@ -215,11 +226,11 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 8, "metadata": { "ExecuteTime": { - "end_time": "2020-04-11T04:49:32.576672Z", - "start_time": "2020-04-11T04:49:32.481350Z" + "end_time": "2020-04-11T05:31:46.658638Z", + "start_time": "2020-04-11T05:31:46.606018Z" } }, "outputs": [], @@ -242,10 +253,17 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, + "execution_count": 9, + "metadata": { + "ExecuteTime": { + "end_time": "2020-04-11T05:31:46.715140Z", + "start_time": "2020-04-11T05:31:46.661863Z" + } + }, "outputs": [], - "source": [] + "source": [ + "results = {}" + ] }, { "cell_type": "markdown", @@ -268,11 +286,10 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2020-04-11T04:49:32.718757Z", - "start_time": "2020-04-11T04:49:32.579382Z" + "start_time": "2020-04-11T05:31:34.200Z" } }, "outputs": [ @@ -281,23 +298,185 @@ "output_type": "stream", "text": [ "now run `tensorboard --logdir lightning_logs`\n", - "WARNING:predict_heading2:No default args on \n" + "trial.number -31\n", + "trial \n", + "INFO:root:GPU available: True, used: True\n", + "INFO:root:VISIBLE GPUS: 0\n", + "INFO:root:\n", + " | Name | Type | Params\n", + "----------------------------------------------------------------------------------------------------------------\n", + "0 | _model | NeuralProcess | 1 M \n", + "1 | _model.norm_x | BatchNormSequence | 34 \n", + "2 | _model.norm_x.norm | BatchNorm1d | 34 \n", + "3 | _model.norm_y | BatchNormSequence | 2 \n", + "4 | _model.norm_y.norm | BatchNorm1d | 2 \n", + "5 | _model._lstm_x | LSTM | 207 K \n", + "6 | _model._lstm_y | LSTM | 199 K \n", + "7 | _model._latent_encoder | LatentEncoder | 98 K \n", + "8 | _model._latent_encoder._encoder | BatchMLP | 49 K \n", + "9 | _model._latent_encoder._encoder.initial | NPBlockRelu2d | 32 K \n", + "10 | _model._latent_encoder._encoder.initial.linear | Linear | 32 K \n", + "11 | _model._latent_encoder._encoder.initial.act | ReLU | 0 \n", + "12 | _model._latent_encoder._encoder.initial.dropout | Dropout2d | 0 \n", + "13 | _model._latent_encoder._encoder.encoder | Sequential | 0 \n", + "14 | _model._latent_encoder._encoder.final | Linear | 16 K \n", + "15 | _model._latent_encoder._self_attention | Attention | 0 \n", + "16 | _model._latent_encoder._penultimate_layer | Linear | 16 K \n", + "17 | _model._latent_encoder._mean | Linear | 16 K \n", + "18 | _model._latent_encoder._log_var | Linear | 16 K \n", + "19 | _model._deterministic_encoder | DeterministicEncoder | 672 K \n", + "20 | _model._deterministic_encoder._d_encoder | BatchMLP | 82 K \n", + "21 | _model._deterministic_encoder._d_encoder.initial | NPBlockRelu2d | 32 K \n", + "22 | _model._deterministic_encoder._d_encoder.initial.linear | Linear | 32 K \n", + "23 | _model._deterministic_encoder._d_encoder.initial.act | ReLU | 0 \n", + "24 | _model._deterministic_encoder._d_encoder.initial.dropout | Dropout2d | 0 \n", + "25 | _model._deterministic_encoder._d_encoder.encoder | Sequential | 32 K \n", + "26 | _model._deterministic_encoder._d_encoder.encoder.0 | NPBlockRelu2d | 16 K \n", + "27 | _model._deterministic_encoder._d_encoder.encoder.0.linear | Linear | 16 K \n", + "28 | _model._deterministic_encoder._d_encoder.encoder.0.act | ReLU | 0 \n", + "29 | _model._deterministic_encoder._d_encoder.encoder.0.dropout | Dropout2d | 0 \n", + "30 | _model._deterministic_encoder._d_encoder.encoder.1 | NPBlockRelu2d | 16 K \n", + "31 | _model._deterministic_encoder._d_encoder.encoder.1.linear | Linear | 16 K \n", + "32 | _model._deterministic_encoder._d_encoder.encoder.1.act | ReLU | 0 \n", + "33 | _model._deterministic_encoder._d_encoder.encoder.1.dropout | Dropout2d | 0 \n", + "34 | _model._deterministic_encoder._d_encoder.final | Linear | 16 K \n", + "35 | _model._deterministic_encoder._self_attention | Attention | 0 \n", + "36 | _model._deterministic_encoder._cross_attention | Attention | 590 K \n", + "37 | _model._deterministic_encoder._cross_attention.batch_mlp_k | BatchMLP | 32 K \n", + "38 | _model._deterministic_encoder._cross_attention.batch_mlp_k.initial | NPBlockRelu2d | 16 K \n", + "39 | _model._deterministic_encoder._cross_attention.batch_mlp_k.initial.linear | Linear | 16 K \n", + "40 | _model._deterministic_encoder._cross_attention.batch_mlp_k.initial.act | ReLU | 0 \n", + "41 | _model._deterministic_encoder._cross_attention.batch_mlp_k.initial.dropout | Dropout2d | 0 \n", + "42 | _model._deterministic_encoder._cross_attention.batch_mlp_k.encoder | Sequential | 0 \n", + "43 | _model._deterministic_encoder._cross_attention.batch_mlp_k.final | Linear | 16 K \n", + "44 | _model._deterministic_encoder._cross_attention.batch_mlp_q | BatchMLP | 32 K \n", + "45 | _model._deterministic_encoder._cross_attention.batch_mlp_q.initial | NPBlockRelu2d | 16 K \n", + "46 | _model._deterministic_encoder._cross_attention.batch_mlp_q.initial.linear | Linear | 16 K \n", + "47 | _model._deterministic_encoder._cross_attention.batch_mlp_q.initial.act | ReLU | 0 \n", + "48 | _model._deterministic_encoder._cross_attention.batch_mlp_q.initial.dropout | Dropout2d | 0 \n", + "49 | _model._deterministic_encoder._cross_attention.batch_mlp_q.encoder | Sequential | 0 \n", + "50 | _model._deterministic_encoder._cross_attention.batch_mlp_q.final | Linear | 16 K \n", + "51 | _model._deterministic_encoder._cross_attention._W_k | ModuleList | 131 K \n", + "52 | _model._deterministic_encoder._cross_attention._W_k.0 | AttnLinear | 16 K \n", + "53 | _model._deterministic_encoder._cross_attention._W_k.0.linear | Linear | 16 K \n", + "54 | _model._deterministic_encoder._cross_attention._W_k.1 | AttnLinear | 16 K \n", + "55 | _model._deterministic_encoder._cross_attention._W_k.1.linear | Linear | 16 K \n", + "56 | _model._deterministic_encoder._cross_attention._W_k.2 | AttnLinear | 16 K \n", + "57 | _model._deterministic_encoder._cross_attention._W_k.2.linear | Linear | 16 K \n", + "58 | _model._deterministic_encoder._cross_attention._W_k.3 | AttnLinear | 16 K \n", + "59 | _model._deterministic_encoder._cross_attention._W_k.3.linear | Linear | 16 K \n", + "60 | _model._deterministic_encoder._cross_attention._W_k.4 | AttnLinear | 16 K \n", + "61 | _model._deterministic_encoder._cross_attention._W_k.4.linear | Linear | 16 K \n", + "62 | _model._deterministic_encoder._cross_attention._W_k.5 | AttnLinear | 16 K \n", + "63 | _model._deterministic_encoder._cross_attention._W_k.5.linear | Linear | 16 K \n", + "64 | _model._deterministic_encoder._cross_attention._W_k.6 | AttnLinear | 16 K \n", + "65 | _model._deterministic_encoder._cross_attention._W_k.6.linear | Linear | 16 K \n", + "66 | _model._deterministic_encoder._cross_attention._W_k.7 | AttnLinear | 16 K \n", + "67 | _model._deterministic_encoder._cross_attention._W_k.7.linear | Linear | 16 K \n", + "68 | _model._deterministic_encoder._cross_attention._W_v | ModuleList | 131 K \n", + "69 | _model._deterministic_encoder._cross_attention._W_v.0 | AttnLinear | 16 K \n", + "70 | _model._deterministic_encoder._cross_attention._W_v.0.linear | Linear | 16 K \n", + "71 | _model._deterministic_encoder._cross_attention._W_v.1 | AttnLinear | 16 K \n", + "72 | _model._deterministic_encoder._cross_attention._W_v.1.linear | Linear | 16 K \n", + "73 | _model._deterministic_encoder._cross_attention._W_v.2 | AttnLinear | 16 K \n", + "74 | _model._deterministic_encoder._cross_attention._W_v.2.linear | Linear | 16 K \n", + "75 | _model._deterministic_encoder._cross_attention._W_v.3 | AttnLinear | 16 K \n", + "76 | _model._deterministic_encoder._cross_attention._W_v.3.linear | Linear | 16 K \n", + "77 | _model._deterministic_encoder._cross_attention._W_v.4 | AttnLinear | 16 K \n", + "78 | _model._deterministic_encoder._cross_attention._W_v.4.linear | Linear | 16 K \n", + "79 | _model._deterministic_encoder._cross_attention._W_v.5 | AttnLinear | 16 K \n", + "80 | _model._deterministic_encoder._cross_attention._W_v.5.linear | Linear | 16 K \n", + "81 | _model._deterministic_encoder._cross_attention._W_v.6 | AttnLinear | 16 K \n", + "82 | _model._deterministic_encoder._cross_attention._W_v.6.linear | Linear | 16 K \n", + "83 | _model._deterministic_encoder._cross_attention._W_v.7 | AttnLinear | 16 K \n", + "84 | _model._deterministic_encoder._cross_attention._W_v.7.linear | Linear | 16 K \n", + "85 | _model._deterministic_encoder._cross_attention._W_q | ModuleList | 131 K \n", + "86 | _model._deterministic_encoder._cross_attention._W_q.0 | AttnLinear | 16 K \n", + "87 | _model._deterministic_encoder._cross_attention._W_q.0.linear | Linear | 16 K \n", + "88 | _model._deterministic_encoder._cross_attention._W_q.1 | AttnLinear | 16 K \n", + "89 | _model._deterministic_encoder._cross_attention._W_q.1.linear | Linear | 16 K \n", + "90 | _model._deterministic_encoder._cross_attention._W_q.2 | AttnLinear | 16 K \n", + "91 | _model._deterministic_encoder._cross_attention._W_q.2.linear | Linear | 16 K \n", + "92 | _model._deterministic_encoder._cross_attention._W_q.3 | AttnLinear | 16 K \n", + "93 | _model._deterministic_encoder._cross_attention._W_q.3.linear | Linear | 16 K \n", + "94 | _model._deterministic_encoder._cross_attention._W_q.4 | AttnLinear | 16 K \n", + "95 | _model._deterministic_encoder._cross_attention._W_q.4.linear | Linear | 16 K \n", + "96 | _model._deterministic_encoder._cross_attention._W_q.5 | AttnLinear | 16 K \n", + "97 | _model._deterministic_encoder._cross_attention._W_q.5.linear | Linear | 16 K \n", + "98 | _model._deterministic_encoder._cross_attention._W_q.6 | AttnLinear | 16 K \n", + "99 | _model._deterministic_encoder._cross_attention._W_q.6.linear | Linear | 16 K \n", + "100 | _model._deterministic_encoder._cross_attention._W_q.7 | AttnLinear | 16 K \n", + "101 | _model._deterministic_encoder._cross_attention._W_q.7.linear | Linear | 16 K \n", + "102 | _model._deterministic_encoder._cross_attention._W | AttnLinear | 131 K \n", + "103 | _model._deterministic_encoder._cross_attention._W.linear | Linear | 131 K \n", + "104 | _model._decoder | Decoder | 607 K \n", + "105 | _model._decoder._target_transform | Linear | 16 K \n", + "106 | _model._decoder._decoder | BatchMLP | 590 K \n", + "107 | _model._decoder._decoder.initial | NPBlockRelu2d | 147 K \n", + "108 | _model._decoder._decoder.initial.linear | Linear | 147 K \n", + "109 | _model._decoder._decoder.initial.act | ReLU | 0 \n", + "110 | _model._decoder._decoder.initial.dropout | Dropout2d | 0 \n", + "111 | _model._decoder._decoder.encoder | Sequential | 294 K \n", + "112 | _model._decoder._decoder.encoder.0 | NPBlockRelu2d | 147 K \n", + "113 | _model._decoder._decoder.encoder.0.linear | Linear | 147 K \n", + "114 | _model._decoder._decoder.encoder.0.act | ReLU | 0 \n", + "115 | _model._decoder._decoder.encoder.0.dropout | Dropout2d | 0 \n", + "116 | _model._decoder._decoder.encoder.1 | NPBlockRelu2d | 147 K \n", + "117 | _model._decoder._decoder.encoder.1.linear | Linear | 147 K \n", + "118 | _model._decoder._decoder.encoder.1.act | ReLU | 0 \n", + "119 | _model._decoder._decoder.encoder.1.dropout | Dropout2d | 0 \n", + "120 | _model._decoder._decoder.final | Linear | 147 K \n", + "121 | _model._decoder._mean | Linear | 385 \n", + "122 | _model._decoder._std | Linear | 385 \n" ] }, { - "ename": "ValueError", - "evalue": "The value of the parameter 'learning_rate' is not found. Please set it at the construction of the FixedTrial object.", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 10\u001b[0m },\n\u001b[1;32m 11\u001b[0m \u001b[0muser_attrs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdefault_user_attrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m \u001b[0mPL_MODEL_CLS\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mPL_NeuralProcess\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 13\u001b[0m )\n", - "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/attentive-neural-processes/neural_processes/train.py\u001b[0m in \u001b[0;36mrun_trial\u001b[0;34m(name, PL_MODEL_CLS, params, user_attrs, MODEL_DIR, plot_from_loader)\u001b[0m\n\u001b[1;32m 100\u001b[0m \u001b[0;31m# Make trial\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 101\u001b[0m \u001b[0mtrial\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moptuna\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mFixedTrial\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 102\u001b[0;31m \u001b[0mtrial\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mPL_MODEL_CLS\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_suggest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 103\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 104\u001b[0m \u001b[0;31m# Add auto number\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/attentive-neural-processes/neural_processes/models/neural_process/lightning.py\u001b[0m in \u001b[0;36madd_suggest\u001b[0;34m(trial)\u001b[0m\n\u001b[1;32m 38\u001b[0m \u001b[0;34m@\u001b[0m\u001b[0mstaticmethod\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 39\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0madd_suggest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 40\u001b[0;31m \u001b[0mtrial\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msuggest_loguniform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"learning_rate\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1e-5\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1e-2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 41\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 42\u001b[0m \u001b[0mtrial\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msuggest_categorical\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"hidden_dim\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m8\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mi\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m8\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/optuna/trial.py\u001b[0m in \u001b[0;36msuggest_loguniform\u001b[0;34m(self, name, low, high)\u001b[0m\n\u001b[1;32m 674\u001b[0m \u001b[0;31m# type: (str, float, float) -> float\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 675\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 676\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_suggest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdistributions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLogUniformDistribution\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlow\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlow\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhigh\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mhigh\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 677\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 678\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0msuggest_discrete_uniform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlow\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhigh\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mq\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/optuna/trial.py\u001b[0m in \u001b[0;36m_suggest\u001b[0;34m(self, name, distribution)\u001b[0m\n\u001b[1;32m 699\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_params\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 700\u001b[0m raise ValueError('The value of the parameter \\'{}\\' is not found. Please set it at '\n\u001b[0;32m--> 701\u001b[0;31m 'the construction of the FixedTrial object.'.format(name))\n\u001b[0m\u001b[1;32m 702\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 703\u001b[0m \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_params\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mValueError\u001b[0m: The value of the parameter 'learning_rate' is not found. Please set it at the construction of the FixedTrial object." + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='Validation sanity check', layout=Layout(flex='2'), max=5.…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOydeViUVRfAfxcQEFARcEFQcDc19y3DNDWz8tMyzQXXUstKbTEzLTVNU7PFXDOXTEhNS7QyTctUzNyX3HdFcQNEZF/mfn+8Aw4wGzDDMPD+nmcemfve994z7zj33HvPuecIKSUqKioqKiUXB1sLoKKioqJiW1RFoKKiolLCURWBioqKSglHVQQqKioqJRxVEaioqKiUcFRFoKKiolLCURVBMUQIIYUQtWwth4qKin2gKgKVQkEI4SKEWC6EiBNC3BJCvGPmfX9qFZuTTllbIcR+IcQDIcRxIUSQkfsHCyEOafu9LoSYnaMtLyHEBiFEghDiqhCiv841XyHEJiFEpFaGwBxtzxZCRGjbviqEmGDis7wkhPhHCJEohPg7x7U6QoiNQoi7QogYIcRWIURdI22ZXV/fM9RTp4n2OSVq/22S43ozIcQuIUS8EOK2EGKMkbZ2aOWKE0IcE0L0yHG9v/Z5JQghwoQQXkbaMvj95LUtFcOoikClsJgC1AYCgCeBcUKIrsZuEEIEA6VylHkBvwCfAZ7AbOAXIUR5A824AW8BPkBroBMwVuf6AiAVqAQEA4uEEA201zTAFuBFA20vA+pJKcsCbYFgIURPIx8pBvgKmKnnmiewCairlWU/sNFIW2bV1/cM9dRx1t4bApQHVgIbteUIIXxQnsM3gDdQC/jDSJNjAF/tcxkBhAghfLVtNdC2M1ArdyKw0EhbBr+ffLSlYggppfoqZi9AArW0f5cDvgfuAleBDwEH7bVawE7gPhAFrNWWC+BL4A4QB/wHNCygTJFAF53304A1RuqXA84BbbSfx0lb3g04maPuOeAVM+V4B/hF+7c7yiBTR+f6KmBmjnuctDIEGmnXT/ucxpkhwzDgbxN1vLR9epv5uXLVN/QM9dzbBbgBCJ2ya0BX7d8zgFX5/N5bAclAK522ftC5XlP7HZTRc6/R7ycvbakv4y91RVD8mYcyINQA2gODgKHaa9NQZnblAX9tXVAGhieAOtp7XwKiAYQQ44UQsYZe+gTQztZ9gWM6xceABvrqa5kBLAJu6WtSz/uGRtrS5QngpPbvOkC6lPJcHuTK3rHyPOKB6ygD1w/m3muGnLeklNEFqG/sGerSADgutaOpluM8fA5tgBjtttYdIcQvQohqmRWFEL8KIcbrNqgtSwb2AX8DB3X6yvp/IKW8iHaw1963UAiROas39f0YbUvFfFRFUIwRQjgCfYEPpJQPpJRXgM9RltIAaShbNVWklMlSynCd8jJAPZRZ4mkp5U0AKeVMKaWnoZcBUTy0/97XKbuv7UOf3C2Ax3momHTZC1QRQvQTQpQSQgxGmQm6mfE8XgZaAHN05IrLUc2gXPqQUs7U1m+GMlu9b/wO0wgh/FG2RMy1o+Sqb+IZ5sSD3HLrPgd/YDDKlk814DKwOrOilLKb9jmgW6a9/1ngDymlxpy+pJSvSylf16lr7PsxJbeKmaiKoHjjg7I/fFWn7CrKNgbAOJTZ9H4hxEntQImU8i9gPsrgckcIsUQIUbYAcsRr/9VtoyzwIGdFIYQDyj7vGClles7r2hlvD5RB7zbQFdiOMiM3iBDieeBT4BkpZZSOXDk/l165jCEVjgBJwMfa/hZrDavxpozIOeSsgLJKWyilXK1THq/zqmasvqlnqAdTzyEJ2CClPCClTNZ+xrZCiHLGGpVSpkkpfwe6CCG6m9lXXuSyyPenoiqC4k4UD2f9mVRD2Q9GSnlLSjlcSlkFeBVYKLRup1LKr6WUzYH6KEvt9wCEEBNyDErZXvqEkFLeA24CjXWKG/Nwi0aXsiiz9rVCiFvAAW35dSFEO217O6WULaWUXiirm3ooxlK9aI3S3wL/k1L+p3PpHOAkhKhthlzm4ISyOkFK+ZqU0kP7mmHOzdottD+ATVLK6brXdNrykFJeM1Hf5DPMwUmgkRBCd8utEQ+fw3EUG0OWOOZ8Hh2ynou2zaz/B0KIGoALyneRE1PfT17aUjGGrY0U6svyL7Ibi0OADSjL5QDgDDBMe6034K/9uwHKzK8G0BLFw6YUyr73FuDjAso0E8UwXR5l4L6J1hiZo54AKuu8Wmo/jx/grK3TVCtbWRQvnD1G+u2IYt94wsD1NSjbHO4oWyn3gQY611211ySKh46rttwBRXmW18rcSvuZRhuRxVHb3mvALu3fpbTXyqIos/lmPk+D9c15hjnqO6OsFMegDKRvat9nPu+OwD2gifa5fwnsNiBXPeAZoLS27gCUfftmOv/P4oB22ucagnGnAYPfT17bUl9G/j/ZWgD1ZYUvNbsiKK/9gdwFIoBJPPQamo2yOogHLgIjtOWdUGaB8SirilDAo4AyuQDLtT/c28A7Oteqafuqpue+QHJ4vGgHhvva11qgopF+dwDp2vYzX7/rXPcCwoAEFE+Z/nqeZbaXttwBRUHGaNs8B0xAx/NGjyxD9LT3nfbaYO37hByy5nomea2v7xnqqdMUOIQyGTgMNM1xfaT2/8o9FPfdqjrXfgcmaP9+BMVA/ACIRVmNvJCjrf7aZ52A4rbqpXNtMbA4D9+PwbbUl/kvoX2YKioqKiolFNVGoKKiolLCURWBioqKSglHVQQqKioqJRxVEaioqKiUcFRFoKKiolLCMRiWtqji4+MjAwMD83VvWhqoTlIqKir2iIMDOBVgxD506FCUlLKCvmt2pwgCAwM5ePCg6Yp6uH0bkpIsLJCKiopKIeDuDhX0DuPmIYS4auiaujWkoqKiUsJRFYGKiopKCUdVBCoqKiolHFURqKioqJRwVEWgoqKiUsKxqiIQQnQVQpwVQlzImcpOez1ACPGnEOK4EOJvbaYlFRUVFZVCxGqKQJsmcQFKbPL6QD8hRP0c1eYA30spGwFTUTJIqaioqKgUItZcEbQCLkgpL0kpU1ESTPTIUac+8Jf27x16rquoqKgUKU6dOsaRI/vQaDSmK9sJ1lQEfiiJUDK5zsNcuZkcA3pq/34BKCOE8LaiTCoqKioFYtGiT+nZsw3vvDOQuLhYW4tjEWxtLB4LtBdCHAHao2RAyshZSQgxQghxUAhx8O7du4Uto4qKikoW1aop6Zc3bvyBI0f+tbE0lsGaISZuAFV13vtry7KQUkaiXREIITyAF6WUuVSslHIJsASgRYsWarQgFRUVm/Hee9NJSUni4ME9ODraXZQevVjzUxwAagshqqMogL4o+UWzEEL4ADFSSg3wAUpOWxUVFRWDaDQaHBxsu5nx4Ydf2LR/S2O1pymlTAfeBLYCp4EfpZQnhRBThRDdtdU6AGeFEOeASsB0a8mjoqJSPAgJWcSzzzbhjz/CCr3vGzeukZAQX+j9WhurqlUp5WYpZR0pZU0p5XRt2SQp5Sbt3+ullLW1dYZJKVOsKY+Kior9s23bRk6fPsbYsUM4f/6Uyfo7dmxm6tS3SElJLnDfgwZ1oWHDMpw7dxKg2HgO2dpYrKKiopInli79BYAHD+5z+fJ5M+p/zooVc1m3bkWB+3Z1LY2LiyvJyUm0aeNHt27NCtxmUaB4WDpUVFRKDC4uLixZspFbt65Tr96jJusPGvQmV65coFWrJwrc92+/HUFKyb170dy+HWmRVUZRQEg7S9nVokULqSamUVEpmUgpEULYWgyklERGRlCxoi+lSpUqlD4tkJjmkJSyhb5r6taQioqKXXD9+hXatavOZ59NNFk3IyODWbPGc/jwXqwx2RVC4OdXrdCUgLVRFYGKiopdsGPHZm7cuMqVK+e5c+cmO3du5cSJw3rr7t+/i8WLZ/HWW8EAJCYm8MMP37Bq1cJ8979mzVKefroh33+/IN9tFFVUG4GKiopdEBz8Gg0bNsPZ2YVdu7by3ntD6dlzEJ9/vjJX3SpVqjFs2LtUruyHEILU1BQmTnyNMmXKMXDg6/nq/+LFM5w7d5KEhAcA/PzzKrZt20ifPq/QocMzBfpstkZVBCoqKnaBg4MDTZu2AeDBgziCgp6idu0GeusGBNRk4sQ5We/LlClHnz7D8PT0yredYdSoj3jhhQGUL+8DwNmz/7Fly080bNjM7hWBaixWUVFRyQcnThzmypXzNGzYnMDAWlbvz5rGYnVFoKKiUuS5du0SCxbMoE6dBrzyyttG68bFxXLo0D9Uq1aDmjXrWU2mhg2b0bBh8ThHoBqLVVRUijzXr1/hxx+XsX37pmzlKSm5gxGcOnWUl19+jvHjh2Urj4mJ4uLFMyQmJuS5/9TUVN5/fxjz5xfPKDiqIlBRUSnyVK9ehxkzljBkyGhAsRHUretKixa590pcXFwJCnqKpk0fy1b+xhu96dz5EY4e3Zfn/m/fvsGPPy5j9epvssri4u4TFhbKpk2r89xeUUPdGlJRKeacPn2c8eOHMXXqAho3bmlrcfKFr68//foNz3rv7u5BWloqIMnIyMDR0THrWtOmbVi16o9cbfj7BxIYWDtf/bu5eTB58te4ublnlcXGRvP22wPw8wuge/d++Wq3qKAqAhWVYk5YWAjHjx9gzJh+/P33BVuLYxEcHBw4dSoBFxdXsz2APvss/7GGvL0rMGTIqBxlFenWrQ+VK/vnu92igqoIbERYWChz5kwkMvIaVapUY+zY6Tz/fLCtxVIphvTsOYjbtyPp2PE5W4uSb06dOkZk5DXq1XsUf/9AQAkAp4/k5KQ8KYj84u7uwbx5a6zaR2Gh2ghsQFhYKBMmjODGjatIKblx4yoTJowgLCzU1qKpFEPq1m3IV1+F2PX2xbp1yxk+vLtZOQj69GlPgwYeBk8d54erVy+ya9cfXLt2yWJtFiVURWAD5syZSFJSYraypKRE5swxHUNFRSWvXLlygbt3b9t17PxatR6hY8fnqF69TlbZvHmfMHhwV44e3Z+t7oMH90lKSsTT0ztb+W+/raNTp3rMnv1BnvvfvHkdgwc/TWjo4mzlSUmJ3LhxjeRk+z6gpCoCGxAZeS1P5SoqBSE4uCOtWlVmy5afOXToH1uLky+Cg19j2bJfefLJZ7PKTp06wq5dW3P9bv766yzHj9+nSpWq2cpTU1O4dOlsvn5nvr5VefzxTtSq9Ui28sGDuxIUFJAvT6SihGojsAFVqlTjxo2restVVCyNl1cFIiMjeOON3rRu3Z41a/62tUgWYcSIcfTpM4wGDXIf6ipTpmyuso4dn2PbtlN4e1fMc1/PPx+s14ZXubIflSpVIS0tLc9tFiWsqgiEEF2BuYAjsFRKOTPH9WrASsBTW2e8lHKzNWUqCowdO50JE0Zk2x4qXdqNsWOL52EVFdvyyy+HiImJolevx6ldu76txckXSUmJuLqWzmYAbtq0dZ7aKFeuPOXKlbeoXF9/bf9nCMCKW0NCCEdgAfAMUB/oJ4TI+b/wQ5Sk9k2BvkD+Y8TaEc8/H8yMGUvw8np4GGbGjCV6ZxxhYaEEBQVSo4YDQUGBqkFZJV94efnw119nmTbNPn9iTz1Vn1q1nLh+/YrReuHh2xkwoDMrVsy1aP8ZGRkWba+oYU0bQSvggpTykpQyFVgD9MhRRwKZa7hyQKQV5SlSPPHE08yevTzr/TvvDMw10KveRSoqChkZ6Wg0Gjw8Hm75XLp0jpCQRezcuTWr7Pz5U+zZ8yeXLp3L1UZiYgIzZ77P9Olj89x/374dqFevtN3aWExhTUXgB0TovL+uLdNlCjBACHEd2AyMQg9CiBFCiINCiIN37961hqyFzo8/LmfYsP9lvdc30KveRSoFZfv2X+jQoVaWp0x8/AN6925H+/Y1rZK5y1rs3Xudc+dSs23tHDu2n48+ep2ff36Yj+C553rz3Xdb6NdvRK42HBwc+Oab2Xz//XyjfR09up9PPnk3m6toQsIDUlKSKV3aLVvdnTu38L//NefTT8fl96MVCWxtLO4HfCel/FwI8RiwSgjRUEqZzc9NSrkEWAJKGGobyGlxnJ2d9ZZnDvTPPx+sehepFJibNyO4evUisbExgHII6vz5k9y/f4/btyOpXDnn3KzokjMtZGBgLXr1GkKHDg89iSpW9KViRV+997u4uPL++zMpU6YcGo0GBwf98+AVK75i06bVODs7M27cp4CStD4pKRFnZ5dsdVNSkjlx4jAVK1YpyEezOVbLR6Ad2KdIKZ/Wvv8AQEr5qU6dk0BXKWWE9v0loI2U8o6hdotTPoIaNRz0zsqEEFy6pCEoKFCvd5GfXwDh4VcKQUIVeycxMYGbN6/j4uKKv38AAMeOHaBKlar4+FQqEongixr//XeI7777mjFjJuPrW9VoXuLY2BiuXLlApUpV8PW1bqgJe01efwCoLYSoLoRwRjEGb8pR5xrQSSvkI4ArUDz2fszAkLtoZvm7736Cg4Njtmuqd5FKXnBzc6dmzbpZSgCgceOWVKhQ2W6UwOXL5+nfvyPTphnPQzBp0pt89NEb3LhRsBXzo482B6BDh1ps27bRaF1PTy+aNGlldSVgbaymCKSU6cCbwFbgNIp30EkhxFQhRHdttXeB4UKIY8BqYIi0p43LAjJ27PRc8VJ0B/qaNeuh0SjeCkII/PwCDHoXqagUV+7cucnevTs4fjz3TkBSUiLHjh3g4sUzrFu3nJAQ415RZ878x44dm4mONj7fzLQF3L9/j4SEeIYMeYZx417O/4co4ljVRqA9E7A5R9kknb9PAY9bU4aiyrhxLxMWFkqtWo9w69YNYmOjcwWfS0lJxt8/gFKlXPjrr7M2lljFHpk/fzrx8XEMGTI6yx4QExPFggXTSUpKZMaMb0y0YHvq1XuUVau26Q0yN2vWeFaunMe4cZ+ydu0u/vvvIH5+hg9mzpw5jp07t7Bs2a+5gvDt2fMnP//8PQMGvM6ECXOYMmUeTk5O3LhxjZ07t+id9UspmT9/OrGx0Uyc+LlBu0NRx9bG4hLLrVs3SEtLxcXFlcOH9c9OWrYM4osvQvj44zG88ko3li37tZClVLF31q9fwdWrF+nd++Fs1tnZheXLv8LZ2ZmPP55vdA+8KFCuXHmCgjrrvdawYTPq1GmAu3sZGjVqQaNGerfAs2jUqCVSSsqUKZfr2urVS/jttx8JDKyd7bBa+fLeBn97Qgi+/XYODx7cZ9Soj/D09MrDJys6qIrARkybtpCzZ0/kioeSk2vXLnHy5GEuXtQfcldFxRhjxkzh5s2IbLNZD48yTJ78NVWrVrehZJahV68h9Oo1xOz677wz1eC1996bQWBgbV58cXC2cjc3d6MhvN94YyIODg44OtrvcGo1ryFrUZy8hjKZPXsC3377Genp6fj5BWTZCKZPf5fo6Du4uLjSsuUTfP/9FhtLqqJS+Ozbt4tTp47SsmWQ3mTxR478y4oVc3nhhYHZgtIVhBs3rjFt2lt4eJRlzpzvLNJmQbFXryEVMwgLC2XZsi9IT08H4MaNq4wbN5Rx414mKuo2UkqSk5M4eHC3eqJYpUSybdtGpk4dw549f+q9vmvXVn75ZQ07dpgXpkxKSWpqqtE6Dg4ObN26gV27tnL+/ClCQxcX21PFoCoCmzFp0hs8/3xrpk4dQ2pqSrZraWlp2nysD1FPFGdHjcFkmrt3b7Nt2ybOnPkv17X79+8RFhbKTz+t1HNn0aJlyyAGDXrT4P6/i0tpSpd2o3fvoSbb+vXXtdSt68L7778CKFuv6enp/PTTSlatWsjt20qUGx+fSixYsI6FC9fz779/8+GHI/n55+/1thkZGcG//+4ssNuqLbHfTS07588/f83zCWH1RLFCZgymzPAbmaE5ANW1VoejR/cxYkQPnnzyWZYv/y3btdu3I3n77QH4+wfm2hMvajz99As8/fQLBq+/8srb9O49FG9v0/smpUu7k5aWRkLCA/bv302fPk/Qu/dQTp48wqlTR6lfvwmVKlWhVKlSPPtsL0DJY9C373BatXpCb5tLl37OihVzmTjxc4YNeyd/H9LGqIrARvTsOYhjx/Zz8uQRYmLMO0NXHJJkWwJjMZhURfCQsmU96djxOZo0aZPrWvXqdeja9UXq1m2IlNJuDpfpo1SpUmYpAVCCPZ45k6T11tsLwLp1K5gyZR4nThyiTp2Gue5p27Yjbdt2NNhmzZr1aNHi8WzRhO0N1VhsY3LObo0xdepCBg4cWQhSFW1MheZQKV4cPryXqlWrWzwkhpSSkJBFBAU9RfXqtXNd37PnT06ePEKnTt2oWbOexfrNL9Y0FqsrAhuTOYOdPPlN4uJi9dZxdHRkzpyV6mxXi6EMb+XK2acPt4phYmNjePHFtri7e/Dff3EWbVsIwcCBrxu8/tNPK9mwYRXp6Wn07j0UT0/vIn/mIr+oxmIb8eOPKwgJWURU1B2efz6Yn376h/LlfRAi+1dSurSbqgRyMHbsdL0/yISEB3z44euqEVlLYmKC0VDTGRkZXLp0jlOnjhWiVHkjLi6WRo1a0LBhc4usBqSUjBrVlwEDOvPNN58xc+b7bN68nmPHDuRKQN++fVdefvktliz5jFatfNm1a6uBVh+2ba+oW0M2om5dV1JTU5g3by3dur2UVR4WFsqcOROJjLyWLeSERqNsedjrEXZL06yZD/fuRecqF0Jk+0GWLu1WYuMz9ev3JAcPhhMS8ietW+c2dP799+8MHfpsscpjbA5Nmnhx//49/PwCsq0sp09fTP/+r+aqP2pUX/bu3cE332ygefO2ua6fP3+KXr0ex88vgM2bj1pNbnVrqBhSqZIfMTF3ssWDN6QEmjb1ITY2msmTv2bIEL25e0ocmfH1c5JzYlOSjciJiQmkp6cbDHvQoEEzSpUqRWTkNV555X80btyK0aM/KmQpC585c1bi7OzC3bu3uHXrOmfP/sc///zJU0/lTKCoMG/eGqPtubl5EBcXi5ubuzXELRRURWAjdu26mO19WFgoH3wwPGt5qusSmbkifvDgfqHKWBQJCwtl/PhheVqGl1S3240b95OSkoyTk/597QoVKrF27W6OHNnLtGlvZ0W6LUpYw6Opc+f/ZXtf0D4qV/bj0KG72bKn2RuqIigizJkzMdceZeZsdvv2M7i6uuHm5mbg7pJBpodVSkqy3us5t4UyMZT3oSTg4uJq9HrTpq3x9w/E3796kYw9NGjQ01y6dJYFC9bRpEkrq/RRUEXj6OiIl5ePhaSxDSYVgVCsl42BKkAScMJYBjGV/GEsLaW9/yezFPrOD2Ti5xdAhw7Psm7d8mwntdVEPqapUKESXbro3xaxNdeuXSQy8hply3parM3Dh/cSHr4dkHTu3IP69RtbrG17xaDlUQhRUwixBLgAzETJL/w6sF0I8a8QYqjI6eKiYhYajYbq1QU1ajhkxRgyla1MxfgWj59fAJ98spBZs5ZlGdTLlvUssYbi8+dP0adPe7tPqr59+xl27DhPtWo1LNbmr7+u5csvJ/Hll5OZPPlNi7Q5d+7HDB/eg4sX7TNviLGB/BMgBKgppXxaSjlAStlLStkI6A6UAwYWhpDFjfh4xR9aSomTk7IoGzt2elZWpEwyZ7OTJ4+idesqTJtmn8fXLYUxpbh//y5AOZdx9Og9Ll7M4NixeyVSCYCiNPfvV6J2msOGDSEsXjwr1/akrSlVqhSBgbWyfieWoFWrJ2jUqCXOzi56vYDyw759O9m+fZPd2qOs6j4qhOgKzAUcgaVSypk5rn8JPKl96wZUlFIaXQMWB/dRjUZDTEwUDx7cz3ai0ZDXUJ8+7dm/fxetWj3B2rU7bSi5bck0FOvaCEqXdqNiRT+aNGnJ7NkrcHZ2tqGERYf79+9x8uQRXF1L06zZYybrP/54NSIjI9i9+zL+/oHWF7CYER6+nfj4OFq0CMLHp6JV+rCm+6hZikAI0RYIRMemIKXUH4rv4T2OwDngKeA6SjL7ftr0lPrqjwKaSimNJgYtDoogr/zxx0YOHdpDmzYdLBZv3V4ZO3ZIVsTMzNwNurP+V199gW3bNvLYYx0JDd1uKzHtji++mERychLDh79LhQqVbS0OoOzlf/fd17Rs2c7oCeCSgk3PEQghVgE1gaNApn+ZBIwqAqAVcEFKeUnbzhqgB6BXEaDYICabkqck0qVLjyJrzCtsevUawtWrF/H19efrr1fnuh4dfRcpJf/88ycDBjxFSMg2G0hpfxjL3GUrLl8+zy+/rMHR0cmiiiA9PZ3r168ghCAgoKbF2rVnzNl4awHUl3nfQ/IDInTeXwda66sohAgAqgN/5bEPu+TgwT2MHt2PKlWqsX59uK3FsSvatOnAunW7s5UtXjyLRYtmEhd3H19ff5o1e4zDh/dy/vxJG0lpe377bR137tykc+f/FUm3UHNo1aodX30Viq+v8XSueeXMmeP873/NAVi+/DeLrLKvXbvE/v278PML4LHHnjR9QxHDHEVwAqgM3LSiHH2B9VJKvSdahBAjgBEA1arl34smPl55KW3qtp+9nrkqL+d95rZz6tQ5bt6M4N69aOLijN8jJZw4cYD16xfh6xvIq69OyrNceSGzf31tGGvX3D4N1cuLzLrP6NdfQ/nss4lZh6Fu3owgOvounTv3pEePoTx4YPp7MtZ3Zl9SPnzpXjP0Pq/TJnOet6E29ZWHhi5j796tlC9fk9Klq5uUKSkpgbt3I3F1VWwuptDXVn5/N4ZwcanOY48pst/Jp8O6PpkyMh66Y6ene3L7dt7a1Cf/n3/uZurUoXTtGkzNmk8arWsuOWX38irY1pAxDCoCIcQvKFtAZYBTQoj9QJaDtpSyu4m2bwC6qtxfW6aPvsAbhhqSUi4BloBiIzDRr0EiIuDWrfzebTk8PNrSpcubeHh4c8rQRpkOmzZtZePGFXh4eNO+vWlFUJw5fXoXsbG3qFWrNRUqBPD55xNznYhNTU3m6NFDvPLKT5wsoYuCBg1epEyZ2khZhwsXTNfftGk+q1ePp1u3sQQHf2Z9AW1KNVavlqSmJuPo6MTlywVvsVSpR2jXbiD+/kFculTw9vSRmgq1almnbWMrgl6tvN4AACAASURBVDkFbPsAUFsIUR1FAfQF+uesJISoB5QH9hawP7uhSpW6DB06z+z6gYFNqFSpJn5+DawoVdEjPDyUtWsnEhV1DR+favTpM52NG2dy/foJWrV6kbffXk9UlH53PUPlJYVOnYbnqb6Xlz8VK9bA1bWMlSTKO+fP/8udO5epVasVlSpZfi/f2dn4qeu8UKtWK2rVMmU2LboYVARSygL5KUop04UQbwJbUdxHl0spTwohpgIHpZSbtFX7AmvyYYMoMTRr1o1mzbrZWoxCJTw8lG+/HUFqqnKSOCrqKt9+O4IKFQIpVcola2Dw8alGVFTu3ASOjk588klnxo/fjJOT6lJqiqCgYIKCitaZix07lrFjx1KGDfvGKopA5SHmeA31BGYBFQGhfUkpZVlT90opNwObc5RNyvF+Sh7kLRacPPkX//67npo1W9Khg+mE2yWRtWsnZimBTFJTE7l58ywjR67MGrT69JmeTWEAODg4kpGRxsmTf3L37lV8fXNnnyrOaDQZXL16nLJlffD2tqyhtTCpWbMlyckP8PWtY2tRzEKjySAi4gSurh52p7jMCRExG+gupSwnpSwrpSxjjhJQMcyuXavYvn0RYWEzzKqv0Wi4ffsily7l7/yEPWJoa0ejyeDbb0cQHq4knAkKCmb48CVZgcM8PLwYOXIltWs/Rq1abXBwKHlxFePjY5gwoRnjxzextSgFolOnEYwevYb69TvYWhSzCAubwfjxTfjjj4W2FiXPmPMruS2lPG11SUoQVas2xMcngFq1zIumGB0dwVtvKVai1atLxg6aoS0fUFYGa9dOzFoVBAUF88MP47h3L5IuXd4oktschUlaWjLVqjXCza2c2fekpiYzcWILkpPjmTfvivWEK8bUqtWGihWrU7q0/c2TDZ4s1m4JAbRHcR8NI7vX0M9Wl04PBTlZ/PffRcNrKK/s2LGCJUuUA9eK0XRGsR/octoIciNYvfphovq4uCjS01NZv34KlSrVoEeP8YUjaDFBSsngwW6kpSWzYkU8rq62T7ISHx9DqVKuODuXtnhOAmtgjdwJuvj7Q1BQ/u/P78li3ewNiUAXnfcSsIkiKGmEh4fy3XcPIyRGRV3j22+VhDXFWRlkfrZFiwbrTZji7e2f7X3Zsj4cOLCRHTu+xcnJmVKlSrNu3UckJz8AwMPDm8GD5xbrZ1YQhBBMmrQTX986RUIJAEye3JbIyLPMmXMKP79HbC2OSexBWRnCmCLYBmyVUuZODKuSL3TdIcuXr8JLL31Chw5DjN5jyGiquzVSXGnbth9ArpVBqVIu9O37aa76bm7lqFChOlJKVq16K9u1+PhovvlGWVUV9+eWX8zdqiwsnJxccHYujaurh61FKfYYMxZXBdYJIXYLIaYIIVoLe1Z5NiZzq0PZ95bcu3eDb74ZyvLlxmOoGDKaRkcXfz/5CROas2jREB555IksF9AyZXwYMWJZrsH80qWDzJ3bm9TUJJQFa27S01NZu3aitcW2OX/8sYBRowL49deCHgWyLbNmHWPlykS78nxauXIMgwa58uefS2wtSp4wdo5gFjBLCFEG6Ay8DCwWQpwGtqCsFvJ4OLvkom9mD7Bt22Lq1Hnc4CzVkNHU27v4J6yJjo5Ao0mnRo0WjB//u9G67u7ePHgQpX1neL5SEhRoTMwNoqKuaZWi+URHXycsbAYODg4MHTrfStIVf9LSUkhJMWTbKpqYdB+VUj6QUm6QUr4qpWyKkrCmAqajj6roYPikqzQ6S+3TZzrOztkT1jg7u9GnT/FPvzhv3jXef38znTuPBGDPntW89VYt5s3LdUCdc+f2ZJ2KzcxQpo+SoECff34ic+deonPn1/J0n4ODI9u3LyI8PERv7mcV0/TrN5OVKxN55pkxthYlT5jlZC2E8AMCdOofkFJ+bjWpihnh4aE4ODjoNXqC8Vlq5krh++/f5sGDKLy8/OjXb2ax3+fWF17izJlwbt++SFpaSq66S5e+mrXiMvScnZycS4QCdXV1x9U17xFHPT0rM3jw1/j717e6B4wp4uNjmDXrWby8/Hj77Z9sJkdecXYubWsR8oU5J4tnAX1Q8gjo5iPYZUW5ig2ZtgFDgxOYnqWWNL94feElFi8eSkZGGgBJSfcJDw/NeiaGtt20h+ABxfD46qu5bQsqDxFC0LXrKFuLAUBi4n0uXNiHj0+ArUUpEZizIngeqCulTDFZUyUXhgcphZKyzZMX9D2zTCUAkJT0IJsLraFtNyHghx9K3hbHmjUTSElJoEePD/D0zF+2MY1GY3SLzdp4evoydepepNSYrlyEOHHiT7Zv/4b69TvQpYv9ZFUz55u+BJSytiDFFWNRMH18Ahg+fIlZs9Tw8FA+/vgJQkLes6R4RRJzIodmutCCYlDXR0mwB+hj9+5VbNnyda4tNHM5fvwPJkxoRmys7XxBnJ1dqV27DXXqWCa5fGERFXWNffvWcfHifluLkieM5SOYh7KuTgSOCiH+JPvJ4tHWF8/+MeT14+MTkKej/CdP7uDMmd3ExNxgwIDiHS/eWHgJXTJtK/oCz+VcaaWmJls07HBRpn//WcTG3qJcuUp5vldKyc8/T+Xq1WNs376IXr2mWF7AYkz9+h0YPXoNFSvaV9A5Y1tDmXEcDgGbjNRTMYI5g5Q5NGnyDLdvX6BevXaWFrHIoe+Z6SNzxq9rK4iOvoa3t2JcDgoKJj09ncGDXdFoMlix4oHNDyf9/vtcQkLepXHjrowevcYq8jz+eG6vKnMRQvDWWz+xa9d3dOtmu9VnRMRJDh3aSLVqjWnW7DmbyZFXKlasTsWK9pca1JgiuA/8I6XMZ5I4FXg4SK1cOZr4+Bi8vavRt2/eYwW1bv0irVu/aA0Rixw5B3Z3dy+Skx+Qnp6aVSenMjVkUHdyevhf/OrVY9St+7gVJTfN6dM70WgyOHLkN27dukBgYNGLEOrpWYnu3d+3qQyXLx9i7dqJtGs30K4Ugb1iTBEMABYIIRKBf4A9KIrhRKFIVowoaV4/liApKY4WLXrQps1L1K37eJY7ac4ZvznMmHGYSpVq2Hw1ADBw4JdERp6lUqUauLi4mb4hjzx4EM2iRYPp3HmkRQZQWxmN/f0b0L37+CKpKI3x4EE0R49uxtW1DC1bPm9rcczG2MniXgBCiECgrfb1qhCiGso5gmcLQ0AVhfT0VP79dx1xcXd59tm3TN9g5/zyy2zu3r1CWloKdes+XiBlGhDQyMLS5Z8KFQKYM8d6iZS3bp3PkSO/ER8fU2BF8O23I9i//yc++eQAlSrVsJCE5lGjRnNq1GheqH1agujoCBYuHERAQOPioQgykVJeEUK4AqW1r8y/TSKE6ArMRUlVuVRKOVNPnZeAKSiG6WNSyvxvcBZR7t5V0ix6efnx2mvL89VGenoqCxYMAKBLlzezbXkUR6pXb05GRjqBgU0L3Ja+w2nFbYWWlPSABQsG4Oxcmo4dh9O6da8CtxkfH0N8fAznzu0pdEVgr5QtW4HHHw+2OzuBMa+hCcBjKOEkzgL/AvOBEVJKw6ejHt7vCCwAngKuAweEEJuklKd06tQGPgAel1LeE0JULMiHKapERPzHf//9gZOTc74VgaurB66uZXB0dCIxMZayZX0sLGXR4u2311uknfDw0GyH0TJzH0PhRyE9fPg3zp/fS4sWPahUqSYJCfcsltIwNvYWhw5tomLF6syde8kibfbuPZX+/WfZJO3i3btXSU5+gLd31Twl2LE1Xl5+vPlmiK3FyDPGppWDgATgFxQbwT4p5f08tN0KuCClvAQghFgD9EA5oZzJcGCBlPIeQHE1THt5+VOzZis8PLwK1M6KFXEWkqjksHbtxGyH0cB2YbwPHtzAjh3LuH//Njt2LMXP7xHmzDll+kYzKFeuEu+887NFYwT5+9e3WFt5ZcOGT7IS13fqNMJmcpQUjNkI6gkhvFBsAx2A8UIID+AYitF4hYm2/YAInffXgdY56tQBEELsQdk+miKl3JKnT2AHBAY24ZNP9lm0zeK+3XH79kXc3cvj5uZZIGNlUQrj3aTJs/j4BNC48TPs3bvGoikN3dzK0rLlCxZrz9aUK1cJP7/6+ToLYUuklCQlxZGampTvU922wGCqymyVhHACmgNPAK8C1aWUjibu6QV0lVIO074fCLSWUr6pU+dXIA14CfBHiV/0qJQyNkdbI4ARANWqVWt+9arpw0b6sNdUlTnZvTskW5A1UNwpzT2lbA8EBzuh0WQwZUp4gVw+R40KtMiBPktj66Bu5rJ69QfcunWeESOW4u7uaWtxijwajYbgYGVoDA3NsKjHlTVTVRqUUgjRXQgxUwixG7gDzAG8gXdRchib4gZKcptM/LVlulwHNkkp06SUl4FzQO2cDUkpl0gpW0gpW1SoUMGMrosWt29f5MCBjUREFMxb5L33HqVfP8Hy5a8bzFpWXMgcJAs6qyqqYbwtrQTOndvLzp0riYw8a9F2Dx7cwP79PxETk/Onq6IPBwcHypatSLlylUhPt5/wbMZsBENQzg6MAw5JKVON1NXHAaC2EKI6igLoC+T0CAoD+gErhBA+KFtFlrF0FSE2bJjOzp0rqFixBnPnXixAS8rqLTMPb07sNemKvm2ukJA00zeaQeYKadmykVnGx759P7XJyunEiT8pXbos1as3t7hv/p49P/DHH/MZNOgrqlSpa7F2X3rpEzSaDMqXr2KxNos733xjf/m6DP5vlFL21OYc8MypBIQQJjNeSCnTgTeBrcBp4Ecp5UkhxFQhRHdtta1AtBDiFLADeK845kguXbocTk7OBf4xBQQ0xVj2rcIOshYeHsqoUYH06+fAqFGBhIeH5qsN3RSemV49+WnLEEFBwaSkJADwzjthNlECGRnpTJ/emY8+Usxkf/yxgLffrs2WLV9bpP1atVoRFDSAatUse2aidetePPZYHzw8ylu0XVNMmRLEmDE1uX272M0LiyTmOKN/JIRIkVL+BSCEGAc8CSw2daOUcjOwOUfZJJ2/JfCO9lVsGTz4SwYP/rJAbYSHh7JnTyiG8vEW9naHvpwBxtwyDRm39YWctoZXT/XqzUhLS6FUKReLtZkX0tJSaNCgIxqNsm+ckpLIrVsXuHv3ikXab9duIO3aDbRIW0WByMizPHgQZbeJXuwNk8Zi7ZbNr8B7QFegHtAvH1tFFqFFixby4MGDpivqwZ6NxYaMnpm88UZIoc5082KEzak04KFxe8GCgRhSbhUqVOfrr4vnjDA29jYJCffw8vKjdOkythbHIFFR1zh7dg/ly/tSv36HQus3Pv4e9+/fwte3rk3zIuSHJUuGcfXqMV5//Xv8/B6xWLs2MRZnIqWMArqjHA6rAvSylRIoTCyx7WFJjMXoL1++SqFvd+TFLdPYrN9QLgGAe/ciCyZkEcbTsxJ+fvUspgTu3LlMYmKcxXMNnzmzm/nz+7N9u8kNAIvi4VEeP79H7E4JAFy/fopLlw6SkHDP1qKYjTGvoQdCiDghRBxwAcWQ2xvILCu2WHrf+sMPWxMcXIqVK9/Ot0zGBsyChB3OL3lJBmNMaej36ilN+/ZDGDp0fsEF1aKEStjLzZvnLdZmXrBmMniNJoO33qrJsGGeRlOi5ocqVerRunUv6tSxbdRWe+KVVxYxbdo+qlZ91NaimI0xY3EZKWVZnZerlNIjs7wwhSxsjM1g80Nc3F00mnTS0pLyLZO+AROgUqVadOxY+Ccv8+KWaUxpBAUFZ4swqWRt+5bXXltBx47DLCbvzJnPMHlyW1aseNN0ZStw8OBGBg92Y+HCwQCkpCSyevV4vvuu4PmdEhPj8PEJwNu7Ko6Olo1BVaNGc956a12h5jK+cGEfixcPZefOlYXWpyUJCGhMrVqtivSWX06MxRoKlFJeMXJdAH5SyuvWEMyWWPo06gcfbOXu3SsFcuszlnzFFgQFBZOYGMf3348mIyMdH58Ag/LoT85TOktpZM5iGzbszMSJ26wir5eXH5cvO+LkZJusq5mnTTNz8Do6lmLTplkI4cCgQV/i4GD0fKZRPDzK8/XXly0lqs25evU4O3d+hxAOtG8/2NbilAiMTR8+E0I4ABtRspTdRYk8WgvFa6gTMBnlUFixwlCqxPy6Z/r61sbXN9c5uTxT1PIaeHv7kZGRjqNjKaOndDNlXr78dZKSlF3Fxx7rm1X+1lvruXTpEN7eyvnDPXtWs2vXSho06Ej37uMsIus77/xskXbyS7t2A2nVqmeW0nNyKkVw8Bzc3MppPYkURSClZMOGT6hbN4gGDZ60pchZpKWlEBt7Cy8vP4uvOPRRv357hg//1ibB7izB0aNbuHTpAI0bP0PNmnpts0UOY7GGegsh6gPBwMuAL0r+4tMoLqHTpZTJhSJlIWOp9JLW5tChTVy6dJjGjZ+mTp3HCr3/ihVr0rjxM9y8eZbNm78ymichU4lNmdKOy5cPsXPnd5w8+VfWKuK//7bz5Zc9kVKDs3NpUlOTiIm5bjFFYGuEELkS43Tr9m6uemfPhrNuneJhvXq1cbvC4cO/smPHMvz8HuH27YuMGLHUKtsRY8c+wp07l/nyy/NUrlzL4u3nxNe3Dr6+dazej7U4dGgT27cvwsPDy/4VAYA2ZHTxiVtgJpkz1TVrPiA6+jrly1ehf/9Z+Z6Nf/JJJxIS7vHqq8stmnEpJGQst26dJyrqik0UQdWqDXj22bf59NMuhIa+ZzJhTnh4KJcvHyY1VbGVZBrhz53bw86dK7O2TVJTkxDCgQoV7CumuyXw9a2Lp6cv/v4NTNY9c2Y3Bw+GcfBgGKBEuR048HOLy+TlVZX09FSSkvSfaFfJTuPGXfHw8KJ6dftJrFO8s5sUgOrVm+HqWgaQ1K7dJpcSyEv0zzNndpGRkc79+5Y9el6t2qMkJMTi4xNo0Xbzgp/fI7i6euDubjzE9qZNs9mw4RO9Rvht2xblqi+lhoiI/ywm54UL+5g27UmcnFxYtqzw3fq2bPmac+f20qXLG9SrpziD37p1gRs3TuPn90jWTLtcuYosWmSe22ynTq9SrVojHB2d2L17FW3b9rOK7JMm/V2oAfIOHtyIg4Mj9et3KBLpRfNKixbdadGiu+mKRQhVERhgwoTmWTPXzJlqJnk9VduhwzDu3Ys0a5aXF95++yeLtpdXtmyZR3p6Kp99dlKvZ9BDZZm/aLGWjJ3k4uJBamoSqam22c08fXoX+/f/lC1U9Nat89iy5WsGDPic557L++H6SpVqZGUOe+yxPhaTNSeFHSV15coxREVd5auvLtilIrBHVEWgByklGRnpAMydeylb2rnw8FAWLRqcy1/bWFiEYcNyz3iLAz/++CFJSXEIIXINZPpOExtGoO90saen5QKd+frWZuTIlVSsaJuUi926jaVlyxeoXbtNVllgYFOaNHkGLy//rLL166ewd+9aevacxOOPW2eGX9Rp0uQZ7t69gqenr61FyRfx8feIjr6Gm1s5KlQItLU4ZmFSEQghfgaWAb/LnFPjYooQgpCQVGJiIvHyejgYZQ5uhg7t2CL6p0ajIT09FWdn10Lvu1Klmty+fZGaNVuRnp4OaHBycgb0n8XQh7OzG+3bD2bnzpW56ltyBeXk5MwTTwyyWHt5pXbtNtmUAED79kNo334IGk0G//67jvr1n+T27YtERp5h/vz+CCFo27av3vYuXz7CsWO/06BBx1ztWpoDB8IIC5tO06bP0avXFKv2BcqBLHtm3751LF36Kk8++QojRiy1tThmYc757YUo4aPPa/MTWC7GbRFEN7TE5Mlts50mNjW46XMvTU1NZt26SWzZMs/isi5dOpLgYEfGjSu8E4wPn4/g2rXjJCXFMW1aBwYOLMWmTbOz6hkLiZGJcnhsCS+/vJDhw5fk2l569NHOFpe/KPLvv+uYO/cl1qwZz+DBc3n00acA48/w+PGtrF07kX/+WWN1+VJSErh06aDFcx0UV8qVq0TVqo/aVehukysCKeV2YLsQohxK7oDtQogI4FsgREppmcDxRQB9e/8LFgxgyZLhfP99otEfpiH30ujoCH7+eRogLH4608VFicxYWAkwcj6fzJVR5r+nT+/KqmvoLMbD60pwuiNHNrN+/cc8+mhn5s27yjffDKNMGW+6dh2TbTVmCdmXLn2VlJQEypatyMCBXxTqmYxdu76nVCkXWrZ8IWvVBMo2ZKZv/smTf+Hh4cVrr31HRkaq0YGkevXmPPvsOzRp8ozVZW/UqAvTpu3Dy8vP6n2lpCSSlpaCu7unXWRw00eLFj1o0aKHrcXIE+amqvQGBgADgUggFAhCSSvZwZoC5sSa0UeNRfj84QcNo0dX13vdwcGRkSNX6h1YIiPPMmVKO5ycnFm40LJn71JTk9FoNLi65g49YQ1MRUDVjTxqzEbg7OzGwIFfcOfOJX7/fS7p6Sk4Obnw6qvLACyei9lY9NPCUgaDBrmSlpbCypWJWaGVY2JuMHp0dVxd3fn00yOUL+9ns5PPRYU9e35g/vxg2rbtx6hRP9hanCKFNaOPmmMj2ADUBVYB/5NS3tReWiuEyN+IXEQxPOMXSCkNHjRr334wa9dOZMGCgbkGrypV6rJkyR2ryFvYdgFT2z26NhLdkBhRUVdxcHBEo8nAxyeA7t3Hs2xZ9txG6ekpLF48FCEE6emp2v6Me2OZS2HlPDCERpNBUNBAkpMfUKrUw+/Mw8Mbd/fyVK5cCx+fAACWLn2NMmW86d17Gg4ODmg0GvbuXUPz5j1wdXW3uqy2JjU1idKly1K2bEVbi1KiMMdr6Gsp5Q59FwxpF3vF0HaGj081HBwc9MT7qUrduu3YsWN51vaMpQavooip7Z6cNhJjITGWLx+ZqywjI/cuoyUGbEvHjsorDg6OjBjxba5yZ2dXGjXqgqenL0IIEhPv8+ef3wCQnJxAWloSGRnp/P33cl58cQq9ek0GlMHy3Lm9+PrWzgrLYU3S09P45ZfZJCfH06/fp1bt68knX+HJJ19Bo7Ffv5Rr1/5j9uzn8PWta7XYWZbGHGNxeSFEzxyvTkKIYqeyzYmoGRQUzLx5V/jhBw2vvbaCPXtCc+3R60YqTU9Pt9p/6qtXjzNyZBXeesv6x/7BcATUTKKirmbL3RAfH8OXX/YqUPhtKPiAbSj6qRAONs8z8cYbqwgOns3WrfOZO7cPnTqNoH//2WzZMpcdO5bRrt1AvLz8soVcuHHjNNOnd2LmTOvbB0BRZOvWTWLTppmkpxeOSdAe8xBkIoQgOjrCrvJpmLMieAV4DCWnMEAHlCB01YUQU6WUqwzdKIToCswFHIGlUsqZOa4PAT5DSW4PMF9KaTN/K30RPqtWfZTNm79ESkm7dgMAGDjQlYyMNCZN2mmwrczBa8OGqfz88zTKl6/CwoU3DNbPDxkZ6cTG3kSJDWh9goKCychIZ/36ydm2e3TRXRH5+ASwf/9PODg4ZkvVqQy++s8O6KOguZj1bemBsmVTGKu31NRkEhPv4+7uaTBV5tmz4Rw/vpXXX/+edu0G4uxcmtjYm9Sp05avvrqY7T6NJoO6dYMKLR6Pg4MDL744GRcXN+33XbLtGKbw9a3D119fxsXFfg7DmTOClAIekVK+KKV8EaiP8gtuDbxv6CYhhCNKVrNntPf00waxy8laKWUT7cvmTrd3717B3b08/ft/xrx5V7h06SCXLx9i4cKBWbPd9PQUpNRQvryfyQQtFy8eAJRsW5bOdFa5ci26dXuP/v1nWaxNU4SEvENs7C3efPMHQkPTs/a2dclcEbm6euDv35AaNbLvICqrJf1KQNejBiwT7C8oKJjhw5foDfVckDwT5nLmzG5GjqzMrFnPGqzTufNrjB69JisdZNOmzxEWNoP332+c65nUrNmSKVN2ZxnXC4MXX5xEt25jrW6X+uqrl5g2TTlPYa84OTlToUIgZcv62FoUszFnReAvpdQNknMHqCqljBFCGFsntgIuSCkvAQgh1gA9gFP5lrYQOHgwjKtXj1KmjA+enpWJj4/OupY52x00aC61a7fG27sqTZs+x7Zti9Ed2DIHr/Dw0GwulZa2H7i5lSU4eLbpihZi9+4Q4uNjAAgJeRcpNQb336OirnL9+kk++yx3vCBjRudXX11ulZwLQUHB2vzIubG2rUCjSadMGR/Klq1gsE79+h24efM8EREnyMhI5+7dK3h5+VGtWiNA8T67du04rVv3slu3SnO4cOFfoqMjcHBQgx4UJuY87b+FEL8C67TvX9SWuQOxRu7zAyJ03l9HWUXk5EUhxBPAOeBtKWVEzgpCiBHACIBq1Qq2TWCM8PBQ7txRkqVfuXKEK1eO5DJgpqYmsnnzF8ybd4Xw8FBtFiXd2a2gffvBBAUFM2pUoE29VSxJph9+JrGxN/n22xF4eHhlU5a6GFJ6ho3yAVbNuWDpPBPm0qTJMyxZctdkvX371rF27US6d3+ffv1mMn9+RFbEz8mTHyc+Ppr5869RrlylXKsEa3P37lWioq7i61sHT8/KVuvn/fd/5/79W5Qvb5/hJTL5+edPuH79BEOHLqBMGW9bi2MSc7aG3gBWAE20r++BN6SUCVLKgmbO+AUIlFI2ArYBenPTSSmXSClbSClbVKhgeFZVEDJ9zTNnvPHx0QYHuMwZpP6TxpIjRzYDheOtsnLlGL76qjexsUYOSBSQzPhK+pQaYNCAnJqaSEjI2Kxnmkle0lxaksLud+3aDzl1yrAdSZfU1GTOn/8XAH//hoBidHRzK4sQgsaNu9KsWTdSUhJ59916vPZaZaKjc82ZrMa6dR8xdWp7jh3bYtV+qlZtQMOGnQpd0Vma/fvXs3fvWu7csY/McUZXBNp9/u3aAT+voS5vALq+bf48NAoDIKXUHWmXAoW3z5EDc2PjgOJtMm1aRyPbIkp5YcxA//hjIRpNOk89NdIqMzVT8ZUSEmJ4/fVVLFgwQO/1+/dvPa8aBwAAIABJREFUMWNGF2bMeHjkxFZpNzPbX7XqHeLi7uDp6Utw8GdW6ffChf2EhU3n99+/YsGC67i7e5q4Q3L48C84OjrpDTb35pshAFnhzFNTkylXrpLF5TaEn1996tRpi5ubqc+hAtCjxwTS01OoUCG3Da0oYioxTYYQQiOEKCelvJ/Htg8AtYUQ1VEUQF+UmEVZCCF8dQ6odUfJfmYTzImNA4ohKD09lYsX9xkZ6JVokn36TM81QFp6BlqrVisSE+NM5gPIL+bEVwoKCjYablrfgGWrtJtBQcGsXj0egNate1lNBo0mg759P6VixepmKAElh/MTTwzG3b08qanJBg+POTo6sXz5A2JjbxXqrLlHj/H06DHeqn3cvHme3bu/JyCgMa1b97JqX9bmscdesrUIecIcG0E88J8QYhuQkFkopRxt7CYpZboQ4k1gK4r76HIp5UkhxFTgoJRyEzBaCNEdSAdigCH5+xgFIzw8VHuKM/est3Tpstp9WomPTwBPPfUGd+5cxN29PFWrNtR70rhvX+XQTVBQMOfO/cPu3atITo63WMgEXT7+eI/F2tKHufGVDJ26LswwDubi51c/K+aQtahT57E8Z40bOfI7o9c1Gg337t3A27uqReMwFRUiIv5jw4ZPaNGih90rAnvDZKwhIcRgfeVSSr37+dbG0rGGTMXEMTWQZSZfKcwtjsLEUHwhffGVivuzsCUZGekMGKD4769YEW+zcBNSSqt5LUVEnGTfvvX4+tbm8cf7m76hCBMfH8Pp0ztxdCxFs2bdLNKmTWMNSSlXCiFKA9WklMUuDu3KlWP0KgEHB0ezZrO6WxxhYbNYvvwNFiwYgI9PQKEMhDExkcTERODlZZ1ZYl5m+kFBwezduwYnJ2fGj/+dSpVqWlweeyEi4gQJCbH4+T1iEa8RR0cnatduw/nz/3L//m1cXQs3wc6ZM+HMmdOdGjVaMGHCH1bpo2rVBlStatksfrbi5s1zfPFFT6pXb2YxRWBNTHoNCSH+BxwFtmjfNxFCbLK2YNYmPDyU4cN9DHoGSanJNtBpNBo0Gg2HDm0iJORdDh/+NVd769ZNJClJMaVERV1l8eKhjBvXmN27Q6z2GUaNCuCjj9rwzjt1rRIuISgomCFD5uPtXRUhRFYOAUMK7r//tnPr1nnOnt3DyJG+DB7sluVFVdSwZtrKH3/8iI8/bsfp0+Z5DZnDmDHrmDp1b1Z6ysLE2bk0CQn3iIsz7QarAhUr1qBp0+d49NEuthbFLMyxEUxBORz2N4CU8qgQwjb5/iyEOWkUdT17Bg92JzU1kQ8//JPff/+akyf/5Pz5f7Np+rVrJ+ayMWRkpBERcZzw8JCs8BSW/gwajZJSMyUl3mrhEs6e3U10dARVqtTj88+N2/PbtRtISkoCAQFNiIuLQqNJ13ui15Zs27aI5ctfx83N02qJ7KtUqUudOm0t6tnj7e2f5YhQ2FSr1ohvvrmDu3t5q/Vx5cpRpJT4+tax+0ir5cpVZNy4X01XLCKYowjSpJT3c+wL2m9oQMxzFc3u2aPYUaKjr1O16qNERp7JFTbBmFH14sUDhIeHWnSALszQyikp8QDZQigbYvjwJVl/f/TRX9y+fYnatfNmNLU2ZcooZ1HS0qy3IujXb6bpSnaEk1MpoyejLcHKlaM5c2Y3H374Fw0aFPSIkkpeMEcRnBRC9AcchRC1gdHAP9YVy7qYchX18PDONpjOmXOaMmW8cXX14IknBmULoJaJsRDNCQkxFp+tF2Zo5TFjfuSNN9LNPmeRSb167ahXr53F5SkozZp1Y968q9mSxqvYnsqVa5OYeL9QMqEVBhqNhvv3b+Pi4oabWzlbi2MUc04WjwIaACnAaiAOeMuaQlkbQ4HiQDGEDh48N1tZhQoBuLoajyRoKkSzpYObmQp2Z2mcnJxwcytrsl509HXCwj7Vpucsmjg7u2blmFAxn1Wr3uHzz18gLi7KKu2/+uoyZs06VmhRVa3N0qUjeP31Kuzdu9bWopjE5C9BSpkopZwopWypDfMwUUppvTV1IdCnz3S92xweHt4mPYViYiJJTo7PVZ4Z4dLR0XCIXkvO1m0VpsEUGzd+ytq1E1i3bhL9+gn69RMWj7pqD7z/fmOGDfOy6yiaOTl69HcOHgwjLs46GfeKGz4+AZQp40NKSt5W0rbAnFSVdYCxQKBufSllR+uJZV2CgoK5du04v/yiRLQw5eoZGjqOPXtCadHiecLDQ0hKimPQoK945pkxudr99991HDq0UW87lpytZ8q6fPnrJCXFIYSD1Q5vffBBc2JirjN06ALatDF80Cc8PJS//16Rq7woZm2bOLEVd+9eZuzYTXk++GUO8fExJCTcMzoxsDf69ZuFRpNebLZurE3Pnh/Rs+dHthbDLMyxEawDFqPEAtIfcMYOefrpUUgpcXQsRd++xmfREREnuHcvkosX96MkVMHgqdQ33/yBAwd+ZvHioVkePWCd2XpQUDB16wZx9mw4VarUzWXAthQ3b54hJSWR5OQHRuutXTuRtLQkvdeKWtTVGzdOkZKSwLVrx62iCL744iypqUlmhZewF1q06G61tm/ePMeECc0JCGjClCm7rdaPin7MUQTpUspFVpekkPH29jc7ln+XLq/j51eP+vU70Lx5d6OpJ11d3WjXbgBCiEI5ZVuhQoDVA1sNHbqQmzfP0rBhJ6P18pLc3tb07DmJlJREq/l5u7i44eJi2Gakkp2EhNj/t3fm4VFUWR9+TzYCBBAI+74FFEVEPkTA0RlxQVFn3BCjIqIoCK6IjhuL4jCOjoOICCKOMmFxXMZlBIdh1CEIsigy4sIiCEFACAQIIYTA+f6o6tiE7iSddHVVd9/3efpJddXt6l9uddWpuufccygszHc0kssQnIoYgvdFZDjwDpbDGABV3RP8I7FFt279j5szUJaT0ZdmYffuLaSnt2T48FmeuQuuLOeeGzDLyAmEWtzeTerVa8a8eY/w9tvjHckBFYts3bqWH35YSfPmnWnXLrxPn23bdueVV/bFlCHYvXsLzz9/HTVq1OGhh+a7LadMKhI2MQh4ACtkdJX9qlyyHw+xYsW7vPTSLWGd9eub5GVdDLVkbNxJR2lhYT7jx5/HY4/1dOw7KkpZkVNecGT7cPo4FRcfYdKkAUybNiQs+/MKK1f+g5deupnly0PNSF8+CQkJ1KhRmzp1nEsEGGmSklJYv34pmzatcltKuVQk11CbSAiJNJ9++iqrVr3L5s1flDvrNzc3h3feeZKcnG/4/vtsfJlIS99FRnKSl4+kpJSSNAbFxcUkJYW3xF9e3g6mT7+V9PRW3HLLlDLb+tcZ8C9uH6m8SxXF6eNUVHSIZcveIDU1LaJ1hZ2mTZtunHPOjbRu3dVtKVFB7doNGDs22/GJeOEg6FVDREar6tP28jWq+ne/bU+p6sOREOgUbdp0Y+vW/9GxY/kTnnJzt7Jo0bTj1gWKhInkJC8fSUkpdO58PtWq1aS4uCjshmDr1rV8+eU/SUhIKtcQgHt1BkLB6eOUnFyNkSPnUF5m32ija9d+dO3az5F9L1/+NsuXv02PHlfSo8eVjnxHpElISKRjx95uy6gQZV01ruOXimG/55eaxQAXA1FtCK666nGuuurxCrVt1Khdyd2tP6XvIt2qifvoo/92bN81a9alTZszy51QF004fZySk6vRq9d1YdlXvLBp0yqWLMmiadOOMWMIoomyfAQSZDnQ+5imTp2GQSOF/O8ivTrJqyq0bduNp55ayeOPf+K2lLARi8cpEqgqBQX72LNnW/mNQ6RnzwEMH/46Z57pXIiqGyxZMpusrNFs377ebSllUtYTgQZZDvQ+6ti6dS0pKanUr9+i3JJ/ZVUw87+LdKsW75o1C8nJWUvXrv1o2rSjo98VCzh9nPbv381XXy3gpJMac9ppfcOyTy+Qm7uVkSNbUa9ec6ZM2RrWfbdq1YVWrbqEdZ9e4LPP5vDFFx/QsWNvmjTp4LacoJRlCE4Xkf1Yd//V7WXs9+WnoQRE5GJgElapyhmqGjAlo4hcBbwJ/J+qRiQiady4czh4cC/XXfeHMmuxllW8PdBdpBtj5FOmZLJ//y7y83O59trw5fjJzs4iK+sB8vJ2ULduU66//o+eH/+vKD17DqBVq67k5e3gtNPKnh8RKjt2rOfFF2+kffuzYsoQpKXVIzU1LaaGCZ2md+9MMjJ607RpJ7ellElQQ6CqVUoiLyKJwBTgAiAHWCEi76nqN6Xa1QLuBj6vyveFim/qf3n54oOlrK5oBbNI0LBhO44cKSItreqVsHyUrtmwd+82z6WJqAr79u1g9OhTAZgzJ7wPuDVr1qV37+tjrkJbamoar75a9uzyyrJq1fscOVLIqaf2JS3NuZoHkSZafEXhDTE5nh7ABlX9AUBE5gJXAN+UavcE8EesuQoRY9q0nRVqFyzCpHQFMzd54omlYd+nG6GwkaRu3aaIJJCQkGhHW5U9PBgKzZp1YsSI+EqyV1XmzXuYrVu/ZuLE1TFlCKIFJw1BM8B/IDEHOMu/gYh0A1qo6j9FJKKGoKK4FQnkNm6EwkaShIQEZs+OmdRZUc/pp/ejSZOOYa3o5gUKCvaRk/MNKSnVPT3/wrWE7CKSAPwZuL8CbYeKyEoRWblrV2RrpkZThElZOZBCJdL1DmKJoqJCCgr2U1x8xG0pYee11+7hoYe6sn79srDuNzPzae69901OOqlxWPfrNt9++1/GjOnFG2886raUMnHSEGwDWvi9b26v81ELOBX4REQ2Az2B90TkhCQmqjrdroXQvUGDqs/SKyjYz003VefWW+uV29ZXZyA9vVWFire7wYwZwxg4UErGvMPBGWdccsI6rxpAr7F48SyGDKnDzJnD3JYSdnbv3syPP37F3r0/uS0lKqhXrznt2vXwfLEdJ4eGVgAdRKQNlgG4Drjet1FV9wHpvvci8gkwKhJRQ7m5WzlypJAjRw6X3xjvz5ZNTLQO46FDJxbMqQzZ2Vl8+ulrpdYK5547yNP9ECq//303cnLW0qPHVaxb91lJosBwhJKmpqZRrVp0F2APxIABT3HllWPC6gg/duwo+fl7qFGjTlh9NV6gTZszePLJiMbBVArHDIGqFovICOAjrPDRmaq6VkTGAytV9T2nvrs8GjRow8iRc0uKskc711wzjv79R/HNN58wcmTrKl/QAkdKKV9++WF4BHuEAwd2U1xcxLJlfy+pHRGOIjrnn38b559/W9h0eonmzU8J+z53797C3Xe3JT29FZMnbw77/g3l4+QTAar6IfBhqXUB8zqo6nlOavEnNbUGvXoNiNTXOU5aWj1Wr57PzJnDSy7gVbmgxbqj2MewYa/z/PPXsn//8X6nWIqOigYOHy4gLa0+tWqFL/zZa/jyTol4MymDqd4dI5QV7hkq8eIo7tz5vKCF2GPN6IWLrVu/5s03x7Jkyeyw7bNFi868/PJuJkyI+uz2ARk9ugs33pjC3r3b3ZYSlLg0BEuXvsHYsecwd25U580rYf7854MWhKnMBS2aIqWqihNG78MPn2PcuHNZvvztSu/Dq+Tm5vDWW+NYtGh62Pft1bvlqpKUlIKqetrBHneGIDs7i2nTbuH777N5//0/OVo0JhJkZ2cxZ07wFBmVuaD16ZNJhw5n48st6MVIqXCwbNmbJCQknXABqqrR++mn7/juu/+yb9/PVZXoOdq06Ub//g9w0UUjKSjYz0MPdWXWrPvcluVpRo16j9deKwh7Vbdw4qiPwGv8+9/Hp004dqw46tMmlFUwvioXNKscp9K+fU9HZi57gR9+WMHPP2+kevXagHLo0AHbyf5UlX4P/fs/QK9eA2nUqH34xHqEOnUaltT6Xr16AT/++BU//vgVmZnPlFnCtSwWL/4bCxe+yDnn3MQFF9wRTrmeoF69pm5LKJe4MgQzZsRe2oSyCsZX5S7+5psns3HjCho2jMkCdQCcfvpFbN78JW3bdufdd/8AwJgxi4MOF1WUxo3b07hx7BmB0nTtejFTp27n2LFjlTYCADt3bmT9+qUxlaAv2ogrQ/Dzz7EXDRMsBUZ6eqsqGbemTTvGfErrzp1/Q5MmnVi0aDppafWpXbsBR4/G3mzgcLN//242blxOrVrptG/fo9L7OXAgl7Fj+3DRRSMYM2Yxdet6/865MuTn72HmzOEUFuYzevQHbssJSFz5CBo2jL1omGAF46+4IjYc4U6zfPlbvP32OI4cKeTZZ78Ny0SpJUtm88EHz5KbG96c/V7h88//ztNPX8rChS+WrCso2F/GJwKzceNyfvrpO/LydtCpUx8aNWobTpmeITW1FsuXv8Xq1R9SVBR4GNdt4soQ3Hpr7EXDlE6BkZxcnXbtetCz57VV2u/zzw9k/Phf8/33S8Kk1HssXvw3Zs9+EIDi4iNhCxxYuPBFsrJGsWvX5rDsz2u0aXMmKSnV+e9/X2PevEeZOLEft91WL2TDV6PGSZx55uUlKeFjlaSkZEaOnMv48Us9+7/G1dBQ376Z5OXB1Kk3c+xYMXXqNOKGG56NWv+ADydSYHzxxXscPlzA9u3roqYAdyhkZ2cxY8btJY72o0eLmD79Vg4cyKVfv7uqtO9eva6nXbse1K/fovzGUUj79j0YMGACs2bdx759O0hOrobqMdavXxbS/5yRcTajRr3roFLvcNZZV7ktoUzEN+MtWujevbuuXFm5iSeffAI7doRXT6zy0ku3sHPnBgYPfpGWLcOXzM4rWKk4TvStJCdX5/XXTyxEZDieo0eL2bJlDcnJqXblslqkptZk27bvSEhIpFGjdiUO5P37d/PFF+9zzjk3luTFMoRO8+bQp0/lPy8iq1Q1YAyrOSoxxpo1C/nhh5Wcempf2rf/v0rv5447ZoZRlfcIFm0VLBTXcDyJiUm0adPthPVvvjmGZcve4PbbZ3LeeYMB+MMfLmTz5i8pKMjjkkvuLWl76NABUlKqx4VxyM3NYenSeaSmptG37+1uyzmBuPIRABQXF4c1b7/XmDlzOPPmPczChVPcluJpgoWIpqe3qvK+c3K+Yfv2dQHrXMc6aWn1OOmkxqSl1S/5/88+2yrXWLo//vKXa7jxxhTWrPlXxHVGmry8HWRljWLBgufdlhKQuDME//jHBDIzE7n77tiqJ+ujefPO1KxZj7p1m1V6H4WFBXz44V/4/PO3wqjMW4QjjUZ2dhYjR7Zm4MAERo5sXeJsHju2N/fd17FSkTTRypw5D/Hww93p338U1avX5tlnryhxll9++WjmzFH69x913GeKi4sAqFmz/Log0U6LFp3p2/cO+vW7x20pAYn9Z7JS+MaFrQJpsceoUf+o8j5yctYya9a9iMRuOUefc33evEfIzd1C/fqhpe3Ozj5+lrp/tteGDdtx6NA+UlJSnRHvQbZs+R+bNq1i8+YvqV+/JQUF+9i3b2eZ4biPPfYfjh4tjtkcQ/6kpFRnyJCpbssISlw6i/PydnD48MGwFteIJdatW8rEif2oVq0GU6d6N1FWONmw4XMmTLiA5ORUpk8vP0dQMGdzvObUX7duKQUFeXTs2IekpBSSk6sB8Oyzv6Nu3aZcffU4qlWrQVJSSlz4BJzAOIvDTKzVRQ1EUVFhpe9IMzLOZubMvDAr8jYpKTUpLDxQ4WJF8VKzoaJkZJzNpEkDmDx5ICNGzOaMMy4hP38vq1a9R3JyNbZvX8fatYsYN+4zOnTo6bZcVygszGfz5i9JSEgkI6OX23KOIzbHR+KYBQsmM3CgMHx4bE7Xd4qmTTMYPHgKDzxQsRQA8VKzIRRyc7dSULCPevWaA5CWVpeJE1czYMAEatduQEJCYsmks59/3sSYMb155ZXhbkqOKF9/vYhx437FW2+Nc1vKCTj6RCAiFwOTsEpVzlDViaW23wHcCRwF8oGhqvqNk5pGjepMUVEBI0bMJiPjbCe/yhXS0izHW0XrMRsskpJSuPDCwBel7Ows5s17pKQE6BlnXEJh4YlPDikpNbjsstHceWcLGjRozdixi52W7SnGj/+MPXt+onr1Wowbdy55eT/x5z+vo2XL0zhwIJdhw/5aUpN4z55trFv3GdE2NF0VWrU6nbZtu9OyZZdy21r1C7ZTp07DiAylOeYjEJFEYB1wAZCDVcx+oP+FXkRqq+p+e/lyYLiqXlzWfqviI3j00SwmTLgBIGZmFZemqKiQgoI8atduGHJGyF8ueNbYd+PGHXjuuXVOyPQcpS/2PsdxaadwMNLS6jNo0CROPvlXPPZYT2rUOIlnnlkbIfXeQlUZOjSd/Pw9TJmyLWAa5oKC/WzZ8pUnh0m8QEHBfoYMqUNKSnXuv/9dunS5IGp9BD2ADar6gy1iLnAFUGIIfEbApibg2O1BVlYWzzwztOT9vn07o74WQSBSUlJJSQndBxLogrdz50ays7Niqn8CESgCaMqUG5gx43aKig6hWv68k9TUtJJ+GjNmMVu3fu2oZi8jIjz44Hzq1m1C3bpNArapUaM2nTqdE2Fl0cPBg3tJSkrhsstG06xZJ8e/z0kfQTPAPwtVjr3uOETkThHZCDwNVC3JSxk88sgjHD4cnpq+sUigmseqx+KifwL97wCHDx+skBEAy3lcVFQIQKNGbene/fKwaow22re3ci35h4a++uoI7r23Q9A60fFCQcF+5s+fdNzEVlVl+vTbWL16AQANGrRi1qzDXH312IjkrHLdWayqU1S1HfAg8GigNiIyVERWisjKXbt2Vep7tmyJjyiP7OwsbrghhYEDhWHDmlY4o2Y8R8GUVdyn4igzZw4nP39PGPYVm+TkrGXHjg18/302K1b8gw8+eJZt275zW1ZEUVWefPI3vP76PWRn/61k/ddfL+Ljj2fwwgvXU1CwL+K6nDQE2wB/U9bcXheMucBvA21Q1emq2l1Vuzdo0KBSYlq2jP0oD98Qh6+4Sl7edl5+eWiFjEGwKJh4mPVZ1YpkPj799FXuuqtN1NfBdooBA55iwoSVdOvWnyVLZpOVNYotW75yW1ZEEREuvvguMjJ60azZKSXrmzbtxODBUxg8+AVq1KhTsv7AgVwWLpzKggWTHdXlpCFYAXQQkTYikgJcB7zn30BEOvi9vRRY75SYCRMmhL1IudcINMRR1vCXf4qEwsJ8LP/+8RQWHoj5C1uw4j6V4dCh/RU2vvFGRsbZtG17JomJSXTvfgX9+t1D8+axl9m2PPr0uYGxY7OPK2Zfv35zLrxwOL17X39c2z17tjFz5nDWrPnI0QgrxwyBqhYDI4CPgG+BN1R1rYiMtyOEAEaIyFoRWQ3cBwxySk9mZiZnnnkhCQlJgJCe3qpKNX29SCjDO76nBytCSMnPz0X1xHQSxcVFMe8n8BX3SUurH5b9Gd9T+fTpk8lNNz1Hixad3ZYScRISEhARVJWFC6fy3HNXl+RdKk3LlqfRps2ZJ/g3w01cppiIVYKlPYBfwht9hq+stqUREWbPjt2Mrf5kZ2cxZcoNVd5PPPVZKKxY8Q6ffTaXiy4aYaKGgAceOJWcnLV0734F1177JC1anPiEdPRoMUePHqFt2+qOhY+67iw2hI+yhjjy83OZNu2WkiGLUBykseRHKY8+fTKDpqJOSEg87m96equgTxHx1GehsHr1fJYte4MffljlthRPcPnlD9Gs2cmsXPku8+dPCtgmMTGJlJTqjuqIC0OQlZVF48aN+fWvhTvvbBGz47e+IQ7fhao0/sM8FXWQxpofpSIES1E9bNhrzJmjZGUVM2eOMnnyZgYNmhRzdbCd5NJL76dDh7Np0iTDbSme4JxzbmDQoOc599yb6datv2s6Yn5oKCsri6FDh1JQ8MsYW0pKjZjzD/gzcGACwebm+YYsKjJjNj29VUipmWMJ30zjiqSoDqWtwVBZnJxZHPOGoHXr1vz4Y3ylCy5r/N///87OzmLq1EEBK2nFcv8YDNFItKaY8ATxMpHMnwEDJvDSS4NL5hP4c+DAbm67LZ38/FwSEhJtIyD4P0GYoQ2DIb6IeR9BPEwkK02fPpncccerAR2Zhw8fJD8/F/CvIatYxoCYDKs1GAxlE/OGYMKECdSoEX/OvD59Mnn55d0hFGPXkuEgYwQMhvgi5g1BZmYm06dPp1WrVojE5kSysgglTDSWh8sMBkNwYt5HAJYxyMzMjPkJZYFIT29Z4YljsTxcZjAYghPzTwTxTkXz6MTDcJnBYAhMXDwRxDO+IbDZsx9k797jk7+KJKB6LK7nCxgMBmMI4gLfBb70BLLk5NS48pcYDIbAmKGhOCHUFNUGgyF+MIYgTojnCmQGg6FsjCGIE4IlmTORQgaDwRiCOCFYRk0TKWQwGIyzOE7wOYRNlkyDwVAaYwjiiD59Ms2F32AwnICjQ0MicrGIfC8iG0TkoQDb7xORb0RkjYgsEpGKJsYxGAwGQ5hwzBCISCIwBegHnAIMFJFTSjX7Euiuql2AN4GnndJjMBgMhsA4+UTQA9igqj+oahEwF7jCv4GqfqyqvuD2ZUBzB/UYDAaDIQBOGoJmwFa/9zn2umAMAeYH2iAiQ0VkpYis3LVrVxglGgwGg8ETzmIRuQHoDpwbaLuqTgem2213iUjF0mk6Rzqw22UNgTC6Qser2oyu0PCqLvCOtqA+WCcNwTaghd/75va64xCRvsAjwLmqeri8napqg7AprCQisjJY7U83MbpCx6vajK7Q8Kou8LY2H04ODa0AOohIGxFJAa4D3vNvICJnANOAy1X1Zwe1GAwGgyEIjhkCVS0GRgAfAd8Cb6jqWhEZLyKX283+BKQBfxeR1SLyXpDdGQwGg8EhHPURqOqHwIel1j3ut9zXye93kOluCwiC0RU6XtVmdIWGV3WBt7UBIKrqtgaDwWAwuIhJOmcwGAxxjjEEARCRRm5rCISI1HVbQyBEpL7bGoJh+iw0RMT1qLxAePWcBO/2WSgYQ+CHiKSJyF+A+SLuvtWjAAALwElEQVQyTUSudFsTgIjUEJEpwAIRGWlHWyEirh4/u7+eA/4pIk+KyK/d1OOP6bPQEJFUEZkKfGwHdPzGXu+F/vLcOQne7bPKEHWCnUJEmgGzAAEuAT7FO7mP7gPqA4OAVKyQW1T1mFuCRKQD8A5wFLgF2AU87JaeAJg+C41bgIZYkzo3ATNFJNXl/vLyOQke7LPKYgzBLxQCM1T1blXdAbwBrBaRLm6IEZFU+28SkALMVtXvVPVPwC77rtLNu4+DwHRVHaWq32BFh20XEdfyRZk+Cw0RSfN/CyxV1VxVfRVYCjxltxM39OGxcxKios8qRdwaAhHpKCIviUh1AFXNBT7xa9ICaAt8H2FdGSKSBUwWke72fIw04Gy/ZncAN4pI80jdfdj9VXI3pqo/cXxuqBpAJ1XNiYSeUtpMn4Wmq72IvAH8VUQuFZFq9qaGfs0eAH4nIu1UVSNxYfPqOWlr82SfhYu4NAQi0gfrkXMo1hACIiKqetCvWQqwuSJpL8KoqzrWEMZXwBrgThEZAvwRuENE0gFUdSvwN+C2COm6FHgbGCV2XQkRSVLVfL9m9XDnBDV9FpquBOAvwNdY58BlwOPA68AlInIqgG2c3sUeulKH48y9ek7aOjzZZ+EkLg0BkIs1vpcBDBaRVgEO2hnARgARuS1Cj6PtgIOq+rSqTgZmAL8DqgNTOX5iyjqsjK6ReAzdCWRi9deDIpKmqsUikuD33acAa20914tIhsOafJg+C40mQB4wQVXfBZ4ALgA6YBnUR+SXCJ0FQKQSPHr1nATv9lnYiEtDoKrfYtVK2AAsBMbDCWPH5wP1ReQt4Hqs8UqndX0NtBaRX9mr1gCLgNFYifnqicgYEbkWuBU4ZH/O0TsPVV0JfGf31wLgJXuT+H13H6CBiLyDdQE84qQmP22mz8rB3+ip6jasTL8X+L2fCjxlG9KDwDgRuRXrqWqPE5oC6PLUOen/vV7qM8dQ1Zh9YT161/Z7L6WXgVrABuD8Up+dj3W3drUDutKBRv5agAR7eSTwN79tXYFX7M9kADcB/wIyI6ErQH/Vxro7+j+/bdWwLsCrgGsdOpZNgF6l1nmhz07Q5YU+s3UNKbUu0f57M5Dtt/4kLEdsV6Au1tDHXAf7a0iQba6dk/b+mwHPACle6rNIvFwX4Ng/Bo9hJbubC4y11yWUauM7yPcAH9jLA+0L83kO6XoU+A6rNOdEfx32cnvgLWCQ/b4+VuK+xg73VyBdwfrrMeBje/li++9vHda3FuvOupv93t9IudJn5elyq8/s71oF3BdkewLwH+Aev3WvA6c63Fdl6irVXxE7J+3vuAPLB/ACVqBB6d++K30WqZfrAhw4oDWxsprOAxph3RHuA1rb20+4y7WX99rtXgFSHdCVCkzEuotoYGsrAOrZ2xP82l6INRbaDbgW+Bho6VB/lamrVFv//ioGDgCTgGQHj2eCfWLOx3rsvheo6WafVUBXsN+Y432GdUd7IND+S2k5Eyv2/bfADfYF+hQH+yuorjI0OnpO+v+GgMnBLur88qQS0T6L5Mt1AWE8oCfZf5OA84Akv20vA7cE+VwdLMOxBujtlC57uYnf8nnAHKBzqfa+H90w4Hn7x+a6Lr/t6XZ/fumErtLa/Na9AIyyL6K/crvPKqIrUn1W6liejFX/uxbWTdAtQM9S7X0XvyuAccB/gT5u6/Jr6+g5GUBbAvA/rJugzvZ334zfzRC/PKk42mduvVwXEIYDWt8+Gf8F3AW099smWCFnHwNdg3w+AegSAV2d7PVJWHevm4FngeVYj76+H5r/XW6iV3T5fT7JwZPTX9tI4GR7fXvgr/byfVh34PcAzX3HOYJ9VmFdTvdZgGPp0zUWy/H8GTAGK4rlJr/fmIRbSzh0+X3ekXMyyLE8zV7/FPCcvf52rMl+T/mdHwlO6PHKK6qjhkSkJ9aj+W7gSSxnz11+TRKwfnhFBCiTCVbKAVVdEwFdw+3vKwa+ANqq6v1Ydxf32VpRv8lOqnrUK7r8NBWr6pJw6gqirbmftg3AfhFJBDoCdwNnqT0RS+0z1V52us8qrMtPU9j7LMixHGZvnoQVddNXVcdhFYga7qdHcYiq6PLTF/ZzMoi25sAQe7NvWPHvqjoNeBBojD1hTKMwbUQoeKJ4fRXIA/6sqnOhJNPk+fasvyOqelRE2trLu8RKWJXia++CrlTgsKqWFLJW1X+KyCisE2ZznOoqS1syVrz9aVhDBRuxxptVRDqo6nqjq0RXX/tY7lPVJ3wNVfV9EbkP68LndIy7V3UF03aBve1TrJno3Wxt/7MnI8a0AfARNYbAnmV43J2Mqn4nIlv9th3BGhryn3l4PlBdRF7Humt70EVdhf6fEZGTse5MfgR+igddldB2BPhKRLKBxar6LxFpg1UDO6xx5DGgq53vWPp99hSsY7kZ935jEdUVora29rYNIvIC8JBYOanaYj0Nbwq3Ni8SFUNDIpLsf1BLTUQ56LetNdbdmT/pwKlYMcBnqeonHtBVTURuxIps+o+q3qyqRbGuqyraVPUxVf2XvbxJVf+gVtoIoyuALhFJEpHfYYVPL1LVwbbximldVdGmqquxhqqygfdUtb9ak8diHs8bAhEZAfxbrHzfl4E1xuk/889vuS3wub3uShGpg5X2t7WqhrVuaFV0YSUb+whrPHlKPOiqqjYRaRxuPbGsCyvy5lOgh8d+Y47pqqo2EWmqqntU9S1VfSXc2ryMZw2BiNQTazjnIuD3WA6eQfbjd4nzRkS66C+OnFOADBGZD1yD5Q/4TlUPeUxXoqr+HA+6wqTtKhwYq41hXVdjRbnsKT0kE4u6wqTtKqx5HvGJeiB0KdALSMTKROiLEW8L/BU7vh3Lo/86sBhoCrQE9mOlrXVslqvRFTvajK7Y0OV1bdHwcl2A34FMwpqY08JvXZrfcgLWhJQO9vsLgOGl9jHY6HJXl5e1GV2xocvr2qLx5boA+4CchhXDvhOYE6TNycCHQbalGF3u6/KyNqMrNnR5XVu0vrziI9iNlRqgE1ZK4QsBRCTRz+PfBKvGKyJylthFv0VENMyRLUZXTGozumJDl9e1RSWeMASquh2Yp6p7scb1fBV+jmKliQA4HUgRkT9hTQX3fdaxWZJGV+xoM7piQ5fXtUUrnjAEAPpLpMrrQKGI3GWvP2Zb+V8BvwH2qGovVf3Y6PKeLi9rM7piQ5fXtUUlbo9NBXphhYB9bi93sf9ejp9jyOjyvi4vazO6YkOX17VFy8t1AWUc3AXAYawsgOlu6zG6Yk+b0RUburyuLRpenhka8iFWce8nsbz+I1T1EvVLhmZ0RYcu8K42oys2dIG3tUUTvskXnkJE+mHlujlcbuMIYnSFjle1GV2h4VVd4G1t0YInDYHBYDAYIofnhoYMBoPBEFmMITAYDIY4xxgCg8FgiHOMITAYDIY4xxgCg8FgiHOMITAYykFEjorIahFZKyJficj94lfxKshnWovI9ZHSaDBUBWMIDIbyOaSqXVW1M1Ze+37AmHI+0xowhsAQFZh5BAZDOYhIvqqm+b1vC6wA0oFWwCygpr15hKp+JiLLsGa7bgJew0qbPBE4D6gGTFHVaRH7JwyGMjCGwGAoh9KGwF6XB3QEDgDHVLVQRDpgFUrpLiLnAaNUtb/dfijQUFWfFJFqwBLgGlXdFNF/xmAIQJLbAgyGKCcZeEFEugJHgYwg7S4EuojI1fb7OkAHrCcGg8FVjCEwGELEHho6CvyM5SvYiVUIJQEoDPYxYKSqfhQRkQZDCBhnscEQAiLSAHgJeEGtcdU6wHZVPQbcCCTaTQ8Atfw++hEwTESS7f1kiEhNDAYPYJ4IDIbyqS4iq7GGgYqxnMN/tre9CLwlIjdh5cQ/aK9fAxwVka+wyilOwook+sKuoLUL+G2k/gGDoSyMs9hgMBjiHDM0ZDAYDHGOMQQGg8EQ5xhDYDAYDHGOMQQGg8EQ5xhDYDAYDHGOMQQGg8EQ5xhDYDAYDHGOMQQGg8EQ5/w/eE+AQa5WNs8AAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step 0, {'val_loss': '0.44150611758232117', 'val/loss_mse': '0.2931194007396698', 'val/loss_p': '0.44150611758232117', 'val/sigma': '1.0362932682037354'}\n", + "\r" ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c1091128cecf44d5a9a0973151c2f7e7", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …" + ] + }, + "metadata": {}, + "output_type": "display_data" } ], "source": [ @@ -313,8 +492,7 @@ " },\n", " user_attrs = default_user_attrs,\n", " PL_MODEL_CLS=PL_NeuralProcess\n", - ")\n", - "trainer.logger.metrics" + ")" ] }, { @@ -322,12 +500,39 @@ "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2020-04-11T03:48:24.104448Z", - "start_time": "2020-04-11T03:48:24.027269Z" + "start_time": "2020-04-11T05:31:34.200Z" } }, "outputs": [], - "source": [] + "source": [ + "%debug" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "start_time": "2020-04-11T05:31:34.200Z" + } + }, + "outputs": [], + "source": [ + "results" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "start_time": "2020-04-11T05:31:34.200Z" + } + }, + "outputs": [], + "source": [ + "trainer.logger.metrics" + ] }, { "cell_type": "markdown", @@ -346,8 +551,7 @@ "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2020-04-11T04:38:49.863878Z", - "start_time": "2020-04-11T04:38:46.400Z" + "start_time": "2020-04-11T05:31:34.300Z" } }, "outputs": [], @@ -395,8 +599,7 @@ "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2020-04-11T04:38:49.865814Z", - "start_time": "2020-04-11T04:38:46.400Z" + "start_time": "2020-04-11T05:31:34.300Z" } }, "outputs": [], @@ -425,8 +628,7 @@ "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2020-04-11T04:38:49.867893Z", - "start_time": "2020-04-11T04:38:46.400Z" + "start_time": "2020-04-11T05:31:34.300Z" } }, "outputs": [], @@ -460,8 +662,7 @@ "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2020-04-11T04:38:49.869900Z", - "start_time": "2020-04-11T04:38:46.400Z" + "start_time": "2020-04-11T05:31:34.300Z" } }, "outputs": [], @@ -490,8 +691,7 @@ "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2020-04-11T04:38:49.871823Z", - "start_time": "2020-04-11T04:38:46.400Z" + "start_time": "2020-04-11T05:31:34.300Z" } }, "outputs": [], @@ -520,8 +720,7 @@ "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2020-04-11T04:38:49.874413Z", - "start_time": "2020-04-11T04:38:46.400Z" + "start_time": "2020-04-11T05:31:34.300Z" } }, "outputs": [], @@ -550,8 +749,7 @@ "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2020-04-11T04:38:49.876588Z", - "start_time": "2020-04-11T04:38:46.400Z" + "start_time": "2020-04-11T05:31:34.300Z" } }, "outputs": [], @@ -585,8 +783,7 @@ "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2020-04-11T04:38:49.878400Z", - "start_time": "2020-04-11T04:38:46.400Z" + "start_time": "2020-04-11T05:31:34.300Z" }, "scrolled": true }, @@ -611,8 +808,7 @@ "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2020-04-11T04:38:49.880310Z", - "start_time": "2020-04-11T04:38:46.400Z" + "start_time": "2020-04-11T05:31:34.300Z" }, "scrolled": true }, @@ -638,8 +834,7 @@ "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2020-04-11T04:38:49.882517Z", - "start_time": "2020-04-11T04:38:46.400Z" + "start_time": "2020-04-11T05:31:34.300Z" }, "scrolled": true }, @@ -691,8 +886,7 @@ "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2020-04-11T04:38:49.884711Z", - "start_time": "2020-04-11T04:38:46.400Z" + "start_time": "2020-04-11T05:31:34.300Z" } }, "outputs": [], @@ -706,8 +900,7 @@ "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2020-04-11T04:38:49.886818Z", - "start_time": "2020-04-11T04:38:46.400Z" + "start_time": "2020-04-11T05:31:34.300Z" } }, "outputs": [],