{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-10T01:25:12.788851Z",
     "start_time": "2020-10-10T01:25:12.783398Z"
    }
   },
   "source": [
    "# Sequence to Sequence Models for Timeseries Regression\n",
    "\n",
    "\n",
    "In this notebook we are going to find the optimal hidden_size for a model vs a dataset. We will use pytorch lightning and optuna."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-08T02:52:04.993589Z",
     "start_time": "2020-11-08T02:52:04.569061Z"
    }
   },
   "outputs": [],
   "source": [
    "# OPTIONAL: Load the \"autoreload\" extension so that code can change. But blacklist large modules\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%aimport -pandas\n",
    "%aimport -torch\n",
    "%aimport -numpy\n",
    "%aimport -matplotlib\n",
    "%aimport -dask\n",
    "%aimport -tqdm\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-08T02:52:06.671206Z",
     "start_time": "2020-11-08T02:52:04.998087Z"
    },
    "lines_to_next_cell": 0
   },
   "outputs": [],
   "source": [
    "# Imports\n",
    "import torch\n",
    "from torch import nn, optim\n",
    "from torch.nn import functional as F\n",
    "from torch.autograd import Variable\n",
    "import torch\n",
    "import torch.utils.data\n",
    "\n",
    "import xarray as xr\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from pathlib import Path\n",
    "from tqdm.auto import tqdm\n",
    "\n",
    "import pytorch_lightning as pl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-08T02:52:06.707927Z",
     "start_time": "2020-11-08T02:52:06.674890Z"
    }
   },
   "outputs": [],
   "source": [
    "from seq2seq_time.data.dataset import Seq2SeqDataSet, Seq2SeqDataSets\n",
    "from seq2seq_time.predict import predict, predict_multi\n",
    "from seq2seq_time.util import dset_to_nc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-08T02:52:06.745323Z",
     "start_time": "2020-11-08T02:52:06.711604Z"
    }
   },
   "outputs": [],
   "source": [
    "import logging\n",
    "import warnings\n",
    "import seq2seq_time.silence \n",
    "warnings.simplefilter('once')\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "warnings.simplefilter(action='ignore', category=DeprecationWarning)\n",
    "warnings.filterwarnings('ignore', 'Consider increasing the value of the `num_workers` argument', UserWarning)\n",
    "warnings.filterwarnings('ignore', 'Your val_dataloader has `shuffle=True`', UserWarning)\n",
    "\n",
    "from pytorch_lightning import _logger as log\n",
    "log.setLevel(logging.WARN)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-10T01:28:32.492160Z",
     "start_time": "2020-10-10T01:28:32.488140Z"
    }
   },
   "source": [
    "## Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-08T02:52:06.843841Z",
     "start_time": "2020-11-08T02:52:06.751591Z"
    },
    "lines_to_next_cell": 0
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "using cuda\n",
      "20201108-095004\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "96"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    "print(f'using {device}')\n",
    "\n",
    "timestamp = '20201108-095004'\n",
    "print(timestamp)\n",
    "window_past = 48*2\n",
    "window_future = 48\n",
    "batch_size = 64\n",
    "num_workers = 5\n",
    "datasets_root = Path('../data/processed/')\n",
    "window_past"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-01T23:28:09.504323Z",
     "start_time": "2020-11-01T23:28:09.453546Z"
    },
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Datasets\n",
    "\n",
    "From easy to hard, these dataset show different challenges, all of them with more than 20k datapoints and with a regression output. See the 00.01 notebook for more details, and the code for more information.\n",
    "\n",
    "Some such as MetroInterstateTraffic are easier, some are periodic such as BejingPM25, some are conditional on inputs such as GasSensor, and some are noisy and periodic like IMOSCurrentsVel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-08T02:52:07.298057Z",
     "start_time": "2020-11-08T02:52:06.850596Z"
    },
    "lines_to_next_cell": 0
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[seq2seq_time.data.data.GasSensor,\n",
       " seq2seq_time.data.data.IMOSCurrentsVel,\n",
       " seq2seq_time.data.data.AppliancesEnergyPrediction,\n",
       " seq2seq_time.data.data.BejingPM25,\n",
       " seq2seq_time.data.data.MetroInterstateTraffic]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from seq2seq_time.data.data import IMOSCurrentsVel, AppliancesEnergyPrediction, BejingPM25, GasSensor, MetroInterstateTraffic\n",
    "datasets = [GasSensor, IMOSCurrentsVel, AppliancesEnergyPrediction, BejingPM25, MetroInterstateTraffic]\n",
    "datasets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lightning\n",
    "\n",
    "We will use pytorch lightning to handle all the training scaffolding. We have a common pytorch lightning class that takes in the model and defines training steps and logging."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-08T02:52:07.347557Z",
     "start_time": "2020-11-08T02:52:07.301918Z"
    }
   },
   "outputs": [],
   "source": [
    "import pytorch_lightning as pl\n",
    "\n",
    "class PL_MODEL(pl.LightningModule):\n",
    "    def __init__(self, model, lr=3e-4, patience=None, weight_decay=0):\n",
    "        super().__init__()\n",
    "        self._model = model\n",
    "        self.lr = lr\n",
    "        self.patience = patience\n",
    "        self.weight_decay = weight_decay\n",
    "\n",
    "    def forward(self, x_past, y_past, x_future, y_future=None):\n",
    "        \"\"\"Eval/Predict\"\"\"\n",
    "        y_dist, extra = self._model(x_past, y_past, x_future, y_future)\n",
    "        return y_dist, extra\n",
    "\n",
    "    def training_step(self, batch, batch_idx, phase='train'):\n",
    "        x_past, y_past, x_future, y_future = batch\n",
    "        y_dist, extra = self.forward(*batch)\n",
    "        loss = -y_dist.log_prob(y_future).mean()\n",
    "        self.log_dict({f'loss/{phase}':loss})\n",
    "        if ('loss' in extra) and (phase=='train'):\n",
    "            # some models have a special loss\n",
    "            loss = extra['loss']\n",
    "            self.log_dict({f'model_loss/{phase}':loss})\n",
    "        return loss\n",
    "\n",
    "    def validation_step(self, batch, batch_idx):\n",
    "        return self.training_step(batch, batch_idx, phase='val')\n",
    "    \n",
    "    def test_step(self, batch, batch_idx):\n",
    "        return self.training_step(batch, batch_idx, phase='test')\n",
    "    \n",
    "    def configure_optimizers(self):\n",
    "        optim = torch.optim.AdamW(self.parameters(), lr=self.lr,  weight_decay=self.weight_decay)\n",
    "        scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n",
    "            optim,\n",
    "            patience=self.patience,\n",
    "            verbose=False,\n",
    "            min_lr=1e-7,\n",
    "        ) if self.patience else None\n",
    "        return {'optimizer': optim, 'lr_scheduler': scheduler, 'monitor': 'loss/val'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-11-08T02:52:04.592Z"
    },
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "from pytorch_lightning.loggers import CSVLogger, WandbLogger, TensorBoardLogger, TestTubeLogger\n",
    "from pytorch_lightning.callbacks.early_stopping import EarlyStopping\n",
    "from pytorch_lightning.callbacks import LearningRateMonitor"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-11-08T02:52:04.595Z"
    },
    "lines_to_end_of_cell_marker": 2,
    "lines_to_next_cell": 0
   },
   "outputs": [],
   "source": [
    "from seq2seq_time.models.baseline import BaselineLast, BaselineMean\n",
    "from seq2seq_time.models.lstm_seq2seq import LSTMSeq2Seq\n",
    "from seq2seq_time.models.lstm import LSTM\n",
    "from seq2seq_time.models.transformer import Transformer\n",
    "from seq2seq_time.models.transformer_seq2seq import TransformerSeq2Seq\n",
    "from seq2seq_time.models.neural_process import RANP\n",
    "from seq2seq_time.models.transformer_process import TransformerProcess\n",
    "from seq2seq_time.models.tcn import TCNSeq\n",
    "from seq2seq_time.models.inceptiontime import InceptionTimeSeq\n",
    "from seq2seq_time.models.xattention import CrossAttention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-11-08T02:52:04.599Z"
    },
    "lines_to_next_cell": 0
   },
   "outputs": [],
   "source": [
    "import gc\n",
    "\n",
    "def free_mem():\n",
    "    gc.collect()\n",
    "    torch.cuda.empty_cache()\n",
    "    gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-02T06:10:41.904480Z",
     "start_time": "2020-11-02T06:10:41.848613Z"
    },
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-11-08T02:52:04.605Z"
    },
    "lines_to_next_cell": 0
   },
   "outputs": [],
   "source": [
    "# PARAMS: model\n",
    "dropout=0.0\n",
    "layers=6\n",
    "nhead=4\n",
    "\n",
    "models = [\n",
    "#     lambda xs, ys: BaselineLast(),\n",
    "#     lambda xs, ys, hidden_size: BaselineMean(),\n",
    "    lambda xs, ys, hidden_size, layers: Transformer(xs,\n",
    "                ys,\n",
    "                attention_dropout=dropout,\n",
    "                nhead=nhead,\n",
    "                nlayers=layers,\n",
    "                hidden_size=hidden_size),\n",
    "\n",
    "    lambda xs, ys, hidden_size, layers:TransformerProcess(xs,\n",
    "                ys, hidden_size=hidden_size, nhead=nhead,\n",
    "        latent_dim=hidden_size//2, dropout=dropout,\n",
    "        nlayers=layers),\n",
    "    lambda xs, ys, hidden_size, layers:TCNSeq(xs, ys, hidden_size=hidden_size, nlayers=layers, dropout=dropout, kernel_size=2),\n",
    "    lambda xs, ys, hidden_size, layers: RANP(xs,\n",
    "        ys, hidden_dim=hidden_size, dropout=dropout, \n",
    "         latent_dim=hidden_size//2, n_decoder_layers=layers, n_latent_encoder_layers=layers, n_det_encoder_layers=layers),\n",
    "    lambda xs, ys, hidden_size, layers: TransformerSeq2Seq(xs,\n",
    "                       ys,\n",
    "                       hidden_size=hidden_size,\n",
    "                       nhead=nhead,\n",
    "                       nlayers=layers,\n",
    "                       attention_dropout=dropout\n",
    "                                     ),\n",
    "    lambda xs, ys, hidden_size, layers: LSTM(xs,\n",
    "         ys,\n",
    "         hidden_size=hidden_size,\n",
    "         lstm_layers=layers//2,\n",
    "         lstm_dropout=dropout),\n",
    "    lambda xs, ys, hidden_size, layers: LSTMSeq2Seq(xs,\n",
    "                ys,\n",
    "                hidden_size=hidden_size,\n",
    "                lstm_layers=layers//2,\n",
    "                lstm_dropout=dropout),\n",
    "    lambda xs, ys, hidden_size, layers: CrossAttention(xs,\n",
    "                ys,\n",
    "                nlayers=layers,\n",
    "                hidden_size=hidden_size,),\n",
    "    lambda xs, ys, hidden_size, layers: InceptionTimeSeq(xs,\n",
    "                ys,\n",
    "                kernel_size=96,\n",
    "                layers=layers//2,\n",
    "                hidden_size=hidden_size,\n",
    "                bottleneck=hidden_size//4)\n",
    "\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-11-08T02:52:04.608Z"
    }
   },
   "outputs": [],
   "source": [
    "# DEBUG: sanity check\n",
    "\n",
    "for Dataset in datasets:\n",
    "    dataset_name = Dataset.__name__\n",
    "    dataset = Dataset(datasets_root)\n",
    "    ds_train, ds_val, ds_test = dataset.to_datasets(window_past=window_past,\n",
    "                                            window_future=window_future)\n",
    "\n",
    "    # Init data\n",
    "    x_past, y_past, x_future, y_future = ds_train.get_rows(10)\n",
    "    xs = x_past.shape[-1]\n",
    "    ys = y_future.shape[-1]\n",
    "\n",
    "    # Loaders\n",
    "    dl_train = DataLoader(ds_train,\n",
    "                          batch_size=batch_size,\n",
    "                          shuffle=True,\n",
    "                          pin_memory=num_workers == 0,\n",
    "                          num_workers=num_workers)\n",
    "    dl_val = DataLoader(ds_val,\n",
    "                         shuffle=True,\n",
    "                         batch_size=batch_size,\n",
    "                         num_workers=num_workers)\n",
    "\n",
    "    for m_fn in models:\n",
    "        free_mem()\n",
    "        pt_model = m_fn(xs, ys, 8, 4)\n",
    "        model_name = type(pt_model).__name__\n",
    "        print(timestamp, dataset_name, model_name)\n",
    "\n",
    "        # Wrap in lightning\n",
    "        model = PL_MODEL(pt_model,\n",
    "                         lr=3e-4\n",
    "                        ).to(device)\n",
    "        trainer = pl.Trainer(\n",
    "            fast_dev_run=True,\n",
    "            # GPU\n",
    "            gpus=1,\n",
    "            amp_level='O1',\n",
    "            precision=16,\n",
    "        )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-01T07:30:40.569795Z",
     "start_time": "2020-11-01T07:29:12.500374Z"
    }
   },
   "source": [
    "Lets summarize all models, and make sure they have a similar number of parameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-23T23:36:11.052891Z",
     "start_time": "2020-10-23T23:36:11.048874Z"
    }
   },
   "source": [
    "## Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-11-08T02:52:04.612Z"
    },
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "from seq2seq_time.metrics import rmse, smape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-11-08T02:52:04.617Z"
    },
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "max_iters=20000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-11-08T02:52:04.620Z"
    }
   },
   "outputs": [],
   "source": [
    "tensorboard_dir = Path(f\"../outputs/{timestamp}\").resolve()\n",
    "print(f'For tensorboard run:\\ntensorboard --logdir=\"{tensorboard_dir}\"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-08T01:38:37.371764Z",
     "start_time": "2020-11-08T01:38:37.315240Z"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-11-08T02:52:04.626Z"
    }
   },
   "outputs": [],
   "source": [
    "class MetricsCallback(pl.Callback):\n",
    "    \"\"\"PyTorch Lightning metric callback.\"\"\"\n",
    "\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.metrics = []\n",
    "\n",
    "    def on_validation_end(self, trainer, pl_module):\n",
    "        self.metrics.append(trainer.callback_metrics)\n",
    "\n",
    "def objective(trial):\n",
    "    # sample\n",
    "    hidden_size_exp = trial.suggest_int(\"hidden_size_exp\", 2, 8)\n",
    "    hidden_size = 2**hidden_size_exp\n",
    "    \n",
    "    layers = trial.suggest_int(\"layers\", 2, 12)\n",
    "    \n",
    "    # Load model\n",
    "    pt_model = m_fn(xs, ys, hidden_size, layers)\n",
    "    model_name = type(pt_model).__name__\n",
    "    \n",
    "    # Wrap in lightning\n",
    "    patience = 2\n",
    "    model = PL_MODEL(pt_model,\n",
    "                     lr=3e-4, patience=patience,\n",
    "                    weight_decay=4e-5\n",
    "                    ).to(device)\n",
    "\n",
    "    \n",
    "    # The default logger in PyTorch Lightning writes to event files to be consumed by\n",
    "    # TensorBoard. We don't use any logger here as it requires us to implement several abstract\n",
    "    # methods. Instead we setup a simple callback, that saves metrics from each validation step.\n",
    "#     metrics_callback = MetricsCallback()\n",
    "    \n",
    "    save_dir = f\"../outputs/{timestamp}/{dataset_name}_{model_name}/{trial.number}\"\n",
    "    Path(save_dir).mkdir(exist_ok=True, parents=True)\n",
    "    trainer = pl.Trainer(\n",
    "        # Training length\n",
    "        min_epochs=2,\n",
    "        max_epochs=100,\n",
    "        limit_train_batches=max_iters//batch_size,\n",
    "        limit_val_batches=max_iters//batch_size//5,\n",
    "        # Misc\n",
    "        gradient_clip_val=20,\n",
    "        terminate_on_nan=True,\n",
    "        # GPU\n",
    "        gpus=1,\n",
    "        amp_level='O1',\n",
    "        precision=16,\n",
    "        # Callbacks\n",
    "        default_root_dir=save_dir,\n",
    "        logger=False,\n",
    "        callbacks=[\n",
    "#             metrics_callback, \n",
    "                   EarlyStopping(monitor='loss/val', patience=patience * 2),\n",
    "                   PyTorchLightningPruningCallback(trial, monitor=\"loss/val\")],\n",
    "    )\n",
    "    trainer.fit(model, dl_train, dl_val)\n",
    "    \n",
    "    # Run on all val data, using test mode\n",
    "    r = trainer.test(model, test_dataloader=dl_val, verbose=False)\n",
    "    return r[0]['loss/test']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-08T02:45:44.106583Z",
     "start_time": "2020-11-08T02:45:44.050637Z"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-11-08T02:52:04.631Z"
    }
   },
   "outputs": [],
   "source": [
    "import optuna\n",
    "from optuna.integration import PyTorchLightningPruningCallback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-11-08T02:52:04.634Z"
    },
    "lines_to_next_cell": 0,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "Path(f\"../outputs/{timestamp}\").mkdir(exist_ok=True)\n",
    "results = defaultdict(dict)\n",
    "for Dataset in tqdm(datasets, desc='datasets'):\n",
    "    dataset_name = Dataset.__name__\n",
    "    dataset = Dataset(datasets_root)\n",
    "    ds_train, ds_val, ds_test = dataset.to_datasets(window_past=window_past,\n",
    "                                            window_future=window_future)\n",
    "\n",
    "    # Init data\n",
    "    x_past, y_past, x_future, y_future = ds_train.get_rows(10)\n",
    "    xs = x_past.shape[-1]\n",
    "    ys = y_future.shape[-1]\n",
    "\n",
    "    # Loaders\n",
    "    dl_train = DataLoader(ds_train,\n",
    "                          batch_size=batch_size,\n",
    "                          shuffle=True,\n",
    "                          pin_memory=num_workers == 0,\n",
    "                          num_workers=num_workers)\n",
    "    dl_val = DataLoader(ds_val,\n",
    "                         shuffle=False,\n",
    "                         batch_size=batch_size,\n",
    "                         num_workers=num_workers)\n",
    "\n",
    "    for i, m_fn in enumerate(tqdm(models, desc=f'models ({dataset_name})')):\n",
    "        try:\n",
    "            model_name = type(m_fn(8, 8, 8, 2)).__name__\n",
    "            free_mem()\n",
    "            study_name = f'{timestamp}_{dataset_name}-{model_name}'\n",
    "            \n",
    "            storage = f\"sqlite:///../outputs/{timestamp}/optuna.db\"\n",
    "            pruner = optuna.pruners.MedianPruner()\n",
    "            study = optuna.create_study(storage=storage, \n",
    "                                        study_name=study_name, \n",
    "                                        pruner=pruner,\n",
    "                                        load_if_exists=True)\n",
    "            study.optimize(objective, n_trials=100, timeout=60*60)\n",
    "            print(\"Number of finished trials: {}\".format(len(study.trials)))\n",
    "\n",
    "            print(\"Best trial:\")\n",
    "            trial = study.best_trial\n",
    "\n",
    "            print(\"  Value: {}\".format(trial.value))\n",
    "\n",
    "            print(\"  Params: \")\n",
    "            for key, value in trial.params.items():\n",
    "                print(\"    {}: {}\".format(key, value))\n",
    "            \n",
    "        except Exception as e:\n",
    "            logging.exception('failed to run model')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "@webio": {
   "lastCommId": null,
   "lastKernelId": null
  },
  "jupytext": {
   "encoding": "# -*- coding: utf-8 -*-",
   "formats": "ipynb,py:light"
  },
  "kernelspec": {
   "display_name": "seq2seq-time",
   "language": "python",
   "name": "seq2seq-time"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "209.162px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
