diff --git a/smartmeters-anp-optuna.ipynb b/smartmeters-anp-optuna.ipynb
index 001b7ca..9621f8f 100644
--- a/smartmeters-anp-optuna.ipynb
+++ b/smartmeters-anp-optuna.ipynb
@@ -14,8 +14,8 @@
    "execution_count": 1,
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2020-01-27T07:40:58.047293Z",
-     "start_time": "2020-01-27T07:40:56.027282Z"
+     "end_time": "2020-01-27T07:43:05.804282Z",
+     "start_time": "2020-01-27T07:43:03.768266Z"
     }
    },
    "outputs": [],
@@ -44,8 +44,8 @@
    "execution_count": 2,
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2020-01-27T07:40:58.087871Z",
-     "start_time": "2020-01-27T07:40:58.050443Z"
+     "end_time": "2020-01-27T07:43:05.846794Z",
+     "start_time": "2020-01-27T07:43:05.807302Z"
     }
    },
    "outputs": [],
@@ -60,8 +60,8 @@
    "execution_count": 3,
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2020-01-27T07:40:58.128831Z",
-     "start_time": "2020-01-27T07:40:58.090749Z"
+     "end_time": "2020-01-27T07:43:05.888146Z",
+     "start_time": "2020-01-27T07:43:05.849843Z"
     }
    },
    "outputs": [],
@@ -76,8 +76,8 @@
    "execution_count": 4,
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2020-01-27T07:40:58.253104Z",
-     "start_time": "2020-01-27T07:40:58.132274Z"
+     "end_time": "2020-01-27T07:43:05.996053Z",
+     "start_time": "2020-01-27T07:43:05.890871Z"
     }
    },
    "outputs": [],
@@ -93,8 +93,8 @@
    "execution_count": 5,
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2020-01-27T07:40:58.316872Z",
-     "start_time": "2020-01-27T07:40:58.256059Z"
+     "end_time": "2020-01-27T07:43:06.038453Z",
+     "start_time": "2020-01-27T07:43:05.998384Z"
     }
    },
    "outputs": [],
@@ -116,8 +116,8 @@
    "execution_count": 6,
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2020-01-27T07:41:07.045353Z",
-     "start_time": "2020-01-27T07:40:58.320118Z"
+     "end_time": "2020-01-27T07:43:15.000291Z",
+     "start_time": "2020-01-27T07:43:06.040818Z"
     }
    },
    "outputs": [],
@@ -130,15 +130,15 @@
    "execution_count": 7,
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2020-01-27T07:41:07.716136Z",
-     "start_time": "2020-01-27T07:41:07.047849Z"
+     "end_time": "2020-01-27T07:43:15.711323Z",
+     "start_time": "2020-01-27T07:43:15.002653Z"
     }
    },
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "<matplotlib.legend.Legend at 0x7f5a5d62b160>"
+       "<matplotlib.legend.Legend at 0x7f86eb027160>"
       ]
      },
      "execution_count": 7,
@@ -245,8 +245,8 @@
    "execution_count": 8,
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2020-01-27T07:41:07.761712Z",
-     "start_time": "2020-01-27T07:41:07.719505Z"
+     "end_time": "2020-01-27T07:43:15.772207Z",
+     "start_time": "2020-01-27T07:43:15.715579Z"
     }
    },
    "outputs": [],
@@ -288,8 +288,8 @@
    "execution_count": 9,
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2020-01-27T07:41:07.803600Z",
-     "start_time": "2020-01-27T07:41:07.764011Z"
+     "end_time": "2020-01-27T07:43:15.819019Z",
+     "start_time": "2020-01-27T07:43:15.775073Z"
     }
    },
    "outputs": [],
@@ -306,8 +306,8 @@
    "execution_count": 10,
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2020-01-27T07:41:07.887159Z",
-     "start_time": "2020-01-27T07:41:07.806060Z"
+     "end_time": "2020-01-27T07:43:15.885829Z",
+     "start_time": "2020-01-27T07:43:15.824304Z"
     }
    },
    "outputs": [],
@@ -416,7 +416,7 @@
    "execution_count": null,
    "metadata": {
     "ExecuteTime": {
-     "start_time": "2020-01-27T07:40:56.200Z"
+     "start_time": "2020-01-27T07:43:03.900Z"
     },
     "scrolled": false
    },
@@ -425,7 +425,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "{'number': 0, 'params': {'learning_rate': 2.7156306033353145e-05, 'hidden_dim': 64, 'latent_dim': 1024, 'attention_layers': 2, 'n_latent_encoder_layers': 8, 'n_det_encoder_layers': 1, 'n_decoder_layers': 1, 'dropout': 0.10823088635411837, 'attention_dropout': 0.019358765571233644, 'latent_enc_self_attn_type': 'ptmultihead', 'det_enc_self_attn_type': 'uniform', 'det_enc_cross_attn_type': 'uniform', 'use_lvar': True, 'use_deterministic_path': True, 'min_std': 0.005, 'grad_clip': 40, 'num_context': 48, 'num_extra_target': 24, 'max_nb_epochs': 2, 'num_workers': 4, 'batch_size': 16, 'num_heads': 8, 'x_dim': 16, 'y_dim': 1, 'vis_i': 670}}\n"
+      "{'number': 0, 'params': {'learning_rate': 1.1362108211828296e-05, 'hidden_dim': 512, 'latent_dim': 64, 'attention_layers': 4, 'n_latent_encoder_layers': 2, 'n_det_encoder_layers': 8, 'n_decoder_layers': 4, 'dropout': 0.29651941930572856, 'attention_dropout': 0.13474149440470473, 'latent_enc_self_attn_type': 'ptmultihead', 'det_enc_self_attn_type': 'dot', 'det_enc_cross_attn_type': 'uniform', 'use_lvar': True, 'use_deterministic_path': False, 'min_std': 0.005, 'grad_clip': 40, 'num_context': 48, 'num_extra_target': 24, 'max_nb_epochs': 2, 'num_workers': 4, 'batch_size': 16, 'num_heads': 8, 'x_dim': 16, 'y_dim': 1, 'vis_i': 670}}\n"
      ]
     },
     {
@@ -436,19 +436,19 @@
       "INFO:root:VISIBLE GPUS: 0\n",
       "INFO:root:\n",
       "                                          Name           Type Params\n",
-      "0                                        model    LatentModel    1 M\n",
-      "1                        model._latent_encoder  LatentEncoder  214 K\n",
-      "2           model._latent_encoder._input_layer  NPBlockRelu2d    1 K\n",
-      "3    model._latent_encoder._input_layer.linear         Linear    1 K\n",
+      "0                                        model    LatentModel    9 M\n",
+      "1                        model._latent_encoder  LatentEncoder    6 M\n",
+      "2           model._latent_encoder._input_layer  NPBlockRelu2d   10 K\n",
+      "3    model._latent_encoder._input_layer.linear         Linear    9 K\n",
       "4       model._latent_encoder._input_layer.act           ReLU    0  \n",
       "..                                         ...            ...    ...\n",
-      "108              model._decoder._decoder.0.act           ReLU    0  \n",
-      "109          model._decoder._decoder.0.dropout      Dropout2d    0  \n",
-      "110             model._decoder._decoder.0.norm    BatchNorm2d    2 K\n",
-      "111                       model._decoder._mean         Linear    1 K\n",
-      "112                        model._decoder._std         Linear    1 K\n",
+      "154              model._decoder._decoder.3.act           ReLU    0  \n",
+      "155          model._decoder._decoder.3.dropout      Dropout2d    0  \n",
+      "156             model._decoder._decoder.3.norm    BatchNorm2d    1 K\n",
+      "157                       model._decoder._mean         Linear  577  \n",
+      "158                        model._decoder._std         Linear  577  \n",
       "\n",
-      "[113 rows x 3 columns]\n"
+      "[159 rows x 3 columns]\n"
      ]
     },
     {
@@ -469,248 +469,14 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "step 0, {'val_loss': '5501.48583984375', 'val/kl': '101.23616027832031', 'val/std': '0.00523672578856349', 'val/mse': '0.29447421431541443'}\n",
+      "step 0, {'val_loss': '5209.85791015625', 'val/kl': '103.47234344482422', 'val/std': '0.00495976721867919', 'val/mse': '0.25139135122299194'}\n",
       "\r"
      ]
     },
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "79b9ccc2d83b445c8be3a14417926d86",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=9.0, style=Prog…"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "step 32, {'val_loss': '1791.584228515625', 'val/kl': '201.39590454101562', 'val/std': '0.009039193391799927', 'val/mse': '0.10792849212884903'}\n"
-     ]
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=9.0, style=Prog…"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "step 65, {'val_loss': '1684.0211181640625', 'val/kl': '259.9749450683594', 'val/std': '0.01560392789542675', 'val/mse': '0.4208507239818573'}\n",
-      "\n",
-      "logger.metrics [{'val_loss': 1684.0211181640625, 'val/kl': 259.9749450683594, 'val/std': 0.01560392789542675, 'val/mse': 0.4208507239818573, 'epoch': 1}]\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "[I 2020-01-27 15:41:34,744] Finished trial#0 resulted in value: 1684.0211181640625. Current best value is 1684.0211181640625 with parameters: {'learning_rate': 2.7156306033353145e-05, 'hidden_dim': 64, 'latent_dim': 1024, 'attention_layers': 2, 'n_latent_encoder_layers': 8, 'n_det_encoder_layers': 1, 'n_decoder_layers': 1, 'dropout': 0.10823088635411837, 'attention_dropout': 0.019358765571233644, 'latent_enc_self_attn_type': 'ptmultihead', 'det_enc_self_attn_type': 'uniform', 'det_enc_cross_attn_type': 'uniform', 'use_lvar': True, 'use_deterministic_path': True, 'min_std': 0.005, 'grad_clip': 40, 'num_context': 48, 'num_extra_target': 24, 'max_nb_epochs': 2, 'num_workers': 4, 'batch_size': 16, 'num_heads': 8, 'x_dim': 16, 'y_dim': 1, 'vis_i': 670}.\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "{'number': 1, 'params': {'learning_rate': 0.009181076644010914, 'hidden_dim': 64, 'latent_dim': 256, 'attention_layers': 4, 'n_latent_encoder_layers': 4, 'n_det_encoder_layers': 16, 'n_decoder_layers': 2, 'dropout': 0.1668729202222619, 'attention_dropout': 0.44446097264782214, 'latent_enc_self_attn_type': 'dot', 'det_enc_self_attn_type': 'multihead', 'det_enc_cross_attn_type': 'multihead', 'use_lvar': True, 'use_deterministic_path': False, 'min_std': 0.005, 'grad_clip': 40, 'num_context': 48, 'num_extra_target': 24, 'max_nb_epochs': 2, 'num_workers': 4, 'batch_size': 16, 'num_heads': 8, 'x_dim': 16, 'y_dim': 1, 'vis_i': 670}}\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "INFO:root:gpu available: True, used: True\n",
-      "INFO:root:VISIBLE GPUS: 0\n",
-      "INFO:root:\n",
-      "                                          Name           Type Params\n",
-      "0                                        model    LatentModel  467 K\n",
-      "1                        model._latent_encoder  LatentEncoder   56 K\n",
-      "2           model._latent_encoder._input_layer  NPBlockRelu2d    1 K\n",
-      "3    model._latent_encoder._input_layer.linear         Linear    1 K\n",
-      "4       model._latent_encoder._input_layer.act           ReLU    0  \n",
-      "..                                         ...            ...    ...\n",
-      "159              model._decoder._decoder.1.act           ReLU    0  \n",
-      "160          model._decoder._decoder.1.dropout      Dropout2d    0  \n",
-      "161             model._decoder._decoder.1.norm    BatchNorm2d  640  \n",
-      "162                       model._decoder._mean         Linear  321  \n",
-      "163                        model._decoder._std         Linear  321  \n",
-      "\n",
-      "[164 rows x 3 columns]\n"
-     ]
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "HBox(children=(FloatProgress(value=0.0, description='Validation sanity check', layout=Layout(flex='2'), max=5.…"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "step 0, {'val_loss': '6021.97802734375', 'val/kl': '100.1371841430664', 'val/std': '0.004819726105779409', 'val/mse': '0.27490323781967163'}\n",
-      "\r"
-     ]
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "6b9b640d4d384559924c5f6b80a14e53",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=9.0, style=Prog…"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "step 32, {'val_loss': '170.01492309570312', 'val/kl': '165.5171661376953', 'val/std': '57.63844680786133', 'val/mse': '27.167020797729492'}\n"
-     ]
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=9.0, style=Prog…"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "step 65, {'val_loss': '6.0421600341796875', 'val/kl': '4.68866491317749', 'val/std': '1.93753981590271', 'val/mse': '0.7113662958145142'}\n",
-      "\n",
-      "logger.metrics [{'val_loss': 6.0421600341796875, 'val/kl': 4.68866491317749, 'val/std': 1.93753981590271, 'val/mse': 0.7113662958145142, 'epoch': 1}]\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "[I 2020-01-27 15:41:53,354] Finished trial#1 resulted in value: 6.0421600341796875. Current best value is 6.0421600341796875 with parameters: {'learning_rate': 0.009181076644010914, 'hidden_dim': 64, 'latent_dim': 256, 'attention_layers': 4, 'n_latent_encoder_layers': 4, 'n_det_encoder_layers': 16, 'n_decoder_layers': 2, 'dropout': 0.1668729202222619, 'attention_dropout': 0.44446097264782214, 'latent_enc_self_attn_type': 'dot', 'det_enc_self_attn_type': 'multihead', 'det_enc_cross_attn_type': 'multihead', 'use_lvar': True, 'use_deterministic_path': False, 'min_std': 0.005, 'grad_clip': 40, 'num_context': 48, 'num_extra_target': 24, 'max_nb_epochs': 2, 'num_workers': 4, 'batch_size': 16, 'num_heads': 8, 'x_dim': 16, 'y_dim': 1, 'vis_i': 670}.\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "{'number': 2, 'params': {'learning_rate': 0.0010775352720169707, 'hidden_dim': 32, 'latent_dim': 256, 'attention_layers': 3, 'n_latent_encoder_layers': 1, 'n_det_encoder_layers': 8, 'n_decoder_layers': 8, 'dropout': 0.4030815943535827, 'attention_dropout': 0.6302759155214426, 'latent_enc_self_attn_type': 'dot', 'det_enc_self_attn_type': 'uniform', 'det_enc_cross_attn_type': 'uniform', 'use_lvar': False, 'use_deterministic_path': True, 'min_std': 0.005, 'grad_clip': 40, 'num_context': 48, 'num_extra_target': 24, 'max_nb_epochs': 2, 'num_workers': 4, 'batch_size': 16, 'num_heads': 8, 'x_dim': 16, 'y_dim': 1, 'vis_i': 670}}\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "INFO:root:gpu available: True, used: True\n",
-      "INFO:root:VISIBLE GPUS: 0\n",
-      "INFO:root:\n",
-      "                                          Name           Type Params\n",
-      "0                                        model    LatentModel  858 K\n",
-      "1                        model._latent_encoder  LatentEncoder   19 K\n",
-      "2           model._latent_encoder._input_layer  NPBlockRelu2d  640  \n",
-      "3    model._latent_encoder._input_layer.linear         Linear  576  \n",
-      "4       model._latent_encoder._input_layer.act           ReLU    0  \n",
-      "..                                         ...            ...    ...\n",
-      "116              model._decoder._decoder.7.act           ReLU    0  \n",
-      "117          model._decoder._decoder.7.dropout      Dropout2d    0  \n",
-      "118             model._decoder._decoder.7.norm    BatchNorm2d  640  \n",
-      "119                       model._decoder._mean         Linear  321  \n",
-      "120                        model._decoder._std         Linear  321  \n",
-      "\n",
-      "[121 rows x 3 columns]\n"
-     ]
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "HBox(children=(FloatProgress(value=0.0, description='Validation sanity check', layout=Layout(flex='2'), max=5.…"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "step 0, {'val_loss': '0.8895980715751648', 'val/kl': '1.8936854758067057e-05', 'val/std': '0.722773551940918', 'val/mse': '0.3085307776927948'}\n",
-      "\r"
-     ]
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "5052c910117f4739831abaad8c1ee1c1",
+       "model_id": "9f35966d29ee48b184640c1292bf5221",
        "version_major": 2,
        "version_minor": 0
       },
@@ -810,7 +576,7 @@
    "execution_count": null,
    "metadata": {
     "ExecuteTime": {
-     "start_time": "2020-01-27T07:40:56.200Z"
+     "start_time": "2020-01-27T07:43:03.900Z"
     }
    },
    "outputs": [],
@@ -827,7 +593,7 @@
    "execution_count": null,
    "metadata": {
     "ExecuteTime": {
-     "start_time": "2020-01-27T07:40:56.200Z"
+     "start_time": "2020-01-27T07:43:03.900Z"
     },
     "scrolled": true
    },
@@ -846,7 +612,7 @@
    "execution_count": null,
    "metadata": {
     "ExecuteTime": {
-     "start_time": "2020-01-27T07:40:56.200Z"
+     "start_time": "2020-01-27T07:43:04.000Z"
     }
    },
    "outputs": [],
@@ -859,7 +625,7 @@
    "execution_count": null,
    "metadata": {
     "ExecuteTime": {
-     "start_time": "2020-01-27T07:40:56.200Z"
+     "start_time": "2020-01-27T07:43:04.000Z"
     }
    },
    "outputs": [],
diff --git a/smartmeters-lstm-optuna.ipynb b/smartmeters-lstm-optuna.ipynb
index 3200093..5294fb1 100644
--- a/smartmeters-lstm-optuna.ipynb
+++ b/smartmeters-lstm-optuna.ipynb
@@ -14,8 +14,8 @@
    "execution_count": 1,
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2020-01-27T07:40:38.121411Z",
-     "start_time": "2020-01-27T07:40:36.074885Z"
+     "end_time": "2020-01-27T07:43:07.628147Z",
+     "start_time": "2020-01-27T07:43:05.571859Z"
     }
    },
    "outputs": [],
@@ -44,8 +44,8 @@
    "execution_count": 2,
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2020-01-27T07:40:38.163518Z",
-     "start_time": "2020-01-27T07:40:38.124298Z"
+     "end_time": "2020-01-27T07:43:07.669502Z",
+     "start_time": "2020-01-27T07:43:07.630969Z"
     }
    },
    "outputs": [],
@@ -60,8 +60,8 @@
    "execution_count": 3,
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2020-01-27T07:40:38.205268Z",
-     "start_time": "2020-01-27T07:40:38.166821Z"
+     "end_time": "2020-01-27T07:43:07.709662Z",
+     "start_time": "2020-01-27T07:43:07.672259Z"
     }
    },
    "outputs": [],
@@ -76,8 +76,8 @@
    "execution_count": 4,
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2020-01-27T07:40:38.312464Z",
-     "start_time": "2020-01-27T07:40:38.208303Z"
+     "end_time": "2020-01-27T07:43:07.814838Z",
+     "start_time": "2020-01-27T07:43:07.712319Z"
     }
    },
    "outputs": [],
@@ -94,8 +94,8 @@
    "execution_count": 5,
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2020-01-27T07:40:38.365279Z",
-     "start_time": "2020-01-27T07:40:38.314895Z"
+     "end_time": "2020-01-27T07:43:07.855003Z",
+     "start_time": "2020-01-27T07:43:07.816925Z"
     }
    },
    "outputs": [],
@@ -117,8 +117,8 @@
    "execution_count": 6,
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2020-01-27T07:40:47.424794Z",
-     "start_time": "2020-01-27T07:40:38.368144Z"
+     "end_time": "2020-01-27T07:43:16.639754Z",
+     "start_time": "2020-01-27T07:43:07.857131Z"
     }
    },
    "outputs": [],
@@ -157,15 +157,15 @@
    "execution_count": 7,
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2020-01-27T07:40:48.116414Z",
-     "start_time": "2020-01-27T07:40:47.427028Z"
+     "end_time": "2020-01-27T07:43:17.384406Z",
+     "start_time": "2020-01-27T07:43:16.642126Z"
     }
    },
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "<matplotlib.legend.Legend at 0x7f404c4607b8>"
+       "<matplotlib.legend.Legend at 0x7f74c55a8d30>"
       ]
      },
      "execution_count": 7,
@@ -217,8 +217,8 @@
    "execution_count": 8,
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2020-01-27T07:40:48.161001Z",
-     "start_time": "2020-01-27T07:40:48.120042Z"
+     "end_time": "2020-01-27T07:43:17.433751Z",
+     "start_time": "2020-01-27T07:43:17.388223Z"
     }
    },
    "outputs": [],
@@ -235,8 +235,8 @@
    "execution_count": 9,
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2020-01-27T07:40:48.216556Z",
-     "start_time": "2020-01-27T07:40:48.163566Z"
+     "end_time": "2020-01-27T07:43:17.484939Z",
+     "start_time": "2020-01-27T07:43:17.437011Z"
     }
    },
    "outputs": [],
@@ -275,11 +275,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": 10,
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2020-01-27T07:41:22.368105Z",
-     "start_time": "2020-01-27T07:41:22.312012Z"
+     "end_time": "2020-01-27T07:43:17.542735Z",
+     "start_time": "2020-01-27T07:43:17.487704Z"
     }
    },
    "outputs": [],
@@ -316,7 +316,7 @@
     "    # TensorBoard. We create a simple logger instead that holds the log in memory so that the\n",
     "    # final accuracy can be obtained after optimization. When using the default logger, the\n",
     "    # final accuracy could be stored in an attribute of the `Trainer` instead.\n",
-    "    logger = DictLogger(\"tb_logs\", name=\"anp\", version=trial.number)\n",
+    "    logger = DictLogger(MODEL_DIR, name=\"lstm\", version=trial.number)\n",
     "    print(\"log_dir\", logger.experiment.log_dir)\n",
     "\n",
     "    trainer = pl.Trainer(\n",
@@ -343,11 +343,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 11,
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2020-01-27T07:42:37.046477Z",
-     "start_time": "2020-01-27T07:41:22.538606Z"
+     "end_time": "2020-01-27T07:44:09.773041Z",
+     "start_time": "2020-01-27T07:43:17.545051Z"
     }
    },
    "outputs": [
@@ -355,7 +355,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "0 {'learning_rate': 1.961387604469747e-05, 'lstm_dropout': 0.1891197441240466, 'hidden_size': 1, 'lstm_layers': 16, 'bidirectional': False, 'window_length': 48, 'target_length': 24, 'max_nb_epochs': 2, 'num_workers': 4, 'grad_clip': 40, 'vis_i': 670, 'input_size': 17, 'batch_size': 16}\n",
+      "0 {'learning_rate': 0.0066558126890954965, 'lstm_dropout': 0.6404976003848417, 'hidden_size': 128, 'lstm_layers': 128, 'bidirectional': True, 'window_length': 48, 'target_length': 24, 'max_nb_epochs': 2, 'num_workers': 4, 'grad_clip': 40, 'vis_i': 670, 'input_size': 17, 'batch_size': 16}\n",
       "log_dir tb_logs/anp/version_0\n"
      ]
     },
@@ -367,8 +367,8 @@
       "INFO:root:VISIBLE GPUS: 0\n",
       "INFO:root:\n",
       "     Name    Type Params\n",
-      "0   lstm1    LSTM  320  \n",
-      "1  linear  Linear   17  \n"
+      "0   lstm1    LSTM   50 M\n",
+      "1  linear  Linear   32 K\n"
      ]
     },
     {
@@ -389,228 +389,14 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "step 0, {'val_loss': '0.2277267426252365'}\n",
+      "step 0, {'val_loss': '0.2659297585487366'}\n",
       "\r"
      ]
     },
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "fc75e999d348447bac2446d9636ca616",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=9.0, style=Prog…"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "step 32, {'val_loss': '0.23519690334796906'}\n"
-     ]
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=9.0, style=Prog…"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "step 65, {'val_loss': '0.23126697540283203'}\n",
-      "\n",
-      "[{'val_loss': 0.23126697540283203, 'epoch': 1}]\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "[I 2020-01-27 15:41:47,100] Finished trial#0 resulted in value: 0.23126697540283203. Current best value is 0.23126697540283203 with parameters: {'learning_rate': 1.961387604469747e-05, 'lstm_dropout': 0.1891197441240466, 'hidden_size': 1, 'lstm_layers': 16, 'bidirectional': False, 'window_length': 48, 'target_length': 24, 'max_nb_epochs': 2, 'num_workers': 4, 'grad_clip': 40, 'vis_i': 670, 'input_size': 17, 'batch_size': 16}.\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "1 {'learning_rate': 0.0001324964724521358, 'lstm_dropout': 0.6061575334689335, 'hidden_size': 16, 'lstm_layers': 16, 'bidirectional': True, 'window_length': 48, 'target_length': 24, 'max_nb_epochs': 2, 'num_workers': 4, 'grad_clip': 40, 'vis_i': 670, 'input_size': 17, 'batch_size': 16}\n",
-      "log_dir tb_logs/anp/version_1\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "INFO:root:gpu available: True, used: True\n",
-      "INFO:root:VISIBLE GPUS: 0\n",
-      "INFO:root:\n",
-      "     Name    Type Params\n",
-      "0   lstm1    LSTM  100 K\n",
-      "1  linear  Linear  513  \n"
-     ]
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "HBox(children=(FloatProgress(value=0.0, description='Validation sanity check', layout=Layout(flex='2'), max=5.…"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "step 0, {'val_loss': '0.23207616806030273'}\n",
-      "\r"
-     ]
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "ce266be8f5bb447d8c1f0e5126b68ecf",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=9.0, style=Prog…"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "step 32, {'val_loss': '0.0821017175912857'}\n"
-     ]
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=9.0, style=Prog…"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "step 65, {'val_loss': '0.09262563288211823'}\n",
-      "\n",
-      "[{'val_loss': 0.09262563288211823, 'epoch': 1}]\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "[I 2020-01-27 15:42:14,176] Finished trial#1 resulted in value: 0.09262563288211823. Current best value is 0.09262563288211823 with parameters: {'learning_rate': 0.0001324964724521358, 'lstm_dropout': 0.6061575334689335, 'hidden_size': 16, 'lstm_layers': 16, 'bidirectional': True, 'window_length': 48, 'target_length': 24, 'max_nb_epochs': 2, 'num_workers': 4, 'grad_clip': 40, 'vis_i': 670, 'input_size': 17, 'batch_size': 16}.\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "2 {'learning_rate': 0.000769512914491473, 'lstm_dropout': 0.4010391775925083, 'hidden_size': 4, 'lstm_layers': 64, 'bidirectional': True, 'window_length': 48, 'target_length': 24, 'max_nb_epochs': 2, 'num_workers': 4, 'grad_clip': 40, 'vis_i': 670, 'input_size': 17, 'batch_size': 16}\n",
-      "log_dir tb_logs/anp/version_2\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "INFO:root:gpu available: True, used: True\n",
-      "INFO:root:VISIBLE GPUS: 0\n",
-      "INFO:root:\n",
-      "     Name    Type Params\n",
-      "0   lstm1    LSTM   28 K\n",
-      "1  linear  Linear  513  \n"
-     ]
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "HBox(children=(FloatProgress(value=0.0, description='Validation sanity check', layout=Layout(flex='2'), max=5.…"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "step 0, {'val_loss': '0.41099223494529724'}\n",
-      "\r"
-     ]
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "439a6b749da849aba385fc562daf3c08",
+       "model_id": "b1d16df354bc423d8cd19062e56d406c",
        "version_major": 2,
        "version_minor": 0
       },
@@ -625,11 +411,22 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Exception ignored in: <function tqdm.__del__ at 0x7f4097607048>\n",
+      "Exception in thread Thread-4:\n",
       "Traceback (most recent call last):\n",
-      "  File \"/home/wassname/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/tqdm/std.py\", line 1061, in __del__\n",
-      "    def __del__(self):\n",
-      "KeyboardInterrupt: \n"
+      "  File \"/home/wassname/.pyenv/versions/3.7.3/lib/python3.7/threading.py\", line 917, in _bootstrap_inner\n",
+      "    self.run()\n",
+      "  File \"/home/wassname/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/tensorboard/summary/writer/event_file_writer.py\", line 211, in run\n",
+      "    self._record_writer.write(data)\n",
+      "  File \"/home/wassname/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/tensorboard/summary/writer/record_writer.py\", line 39, in write\n",
+      "    self._writer.write(header + header_crc + data + footer_crc)\n",
+      "  File \"/home/wassname/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/io/gfile.py\", line 507, in write\n",
+      "    self.fs.append(self.filename, file_content, self.binary_mode)\n",
+      "  File \"/home/wassname/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/io/gfile.py\", line 150, in append\n",
+      "    self._write(filename, file_content, \"ab\" if binary_mode else \"a\")\n",
+      "  File \"/home/wassname/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/io/gfile.py\", line 154, in _write\n",
+      "    with io.open(filename, mode, encoding=encoding) as f:\n",
+      "FileNotFoundError: [Errno 2] No such file or directory: b'tb_logs/anp/version_0/events.out.tfevents.1580110997.mjcdesktop.9368.0'\n",
+      "\n"
      ]
     },
     {
@@ -639,20 +436,26 @@
      "traceback": [
       "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
       "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
-      "\u001b[0;32m<ipython-input-16-6b19a7555e5a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0mstudy\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moptuna\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreate_study\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdirection\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'minimize'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpruner\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpruner\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m \u001b[0mstudy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobjective\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_trials\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m6000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Number of finished trials: {}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstudy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrials\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m<ipython-input-11-6b19a7555e5a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0mstudy\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moptuna\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreate_study\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdirection\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'minimize'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpruner\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpruner\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m \u001b[0mstudy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobjective\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_trials\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m6000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Number of finished trials: {}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstudy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrials\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
       "\u001b[0;32m~/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/optuna/study.py\u001b[0m in \u001b[0;36moptimize\u001b[0;34m(self, func, n_trials, timeout, n_jobs, catch, callbacks, gc_after_trial)\u001b[0m\n\u001b[1;32m    300\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mn_jobs\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    301\u001b[0m                 self._optimize_sequential(func, n_trials, timeout, catch, callbacks,\n\u001b[0;32m--> 302\u001b[0;31m                                           gc_after_trial, None)\n\u001b[0m\u001b[1;32m    303\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    304\u001b[0m                 \u001b[0mtime_start\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdatetime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdatetime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
       "\u001b[0;32m~/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/optuna/study.py\u001b[0m in \u001b[0;36m_optimize_sequential\u001b[0;34m(self, func, n_trials, timeout, catch, callbacks, gc_after_trial, time_start)\u001b[0m\n\u001b[1;32m    536\u001b[0m                     \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    537\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 538\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_trial_and_callbacks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgc_after_trial\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    539\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_storage\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mremove_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    540\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
       "\u001b[0;32m~/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/optuna/study.py\u001b[0m in \u001b[0;36m_run_trial_and_callbacks\u001b[0;34m(self, func, catch, callbacks, gc_after_trial)\u001b[0m\n\u001b[1;32m    548\u001b[0m         \u001b[0;31m# type: (...) -> None\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    549\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 550\u001b[0;31m         \u001b[0mtrial\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_trial\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgc_after_trial\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    551\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcallbacks\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    552\u001b[0m             \u001b[0mfrozen_trial\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_storage\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_trial\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_trial_id\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
       "\u001b[0;32m~/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/optuna/study.py\u001b[0m in \u001b[0;36m_run_trial\u001b[0;34m(self, func, catch, gc_after_trial)\u001b[0m\n\u001b[1;32m    567\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    568\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 569\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    570\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mexceptions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTrialPruned\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    571\u001b[0m             message = 'Setting status of trial#{} as {}. {}'.format(trial_number,\n",
-      "\u001b[0;32m<ipython-input-15-a1b77f69801f>\u001b[0m in \u001b[0;36mobjective\u001b[0;34m(trial)\u001b[0m\n\u001b[1;32m     44\u001b[0m     \u001b[0;32mglobal\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     45\u001b[0m     \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mLSTM_PL\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 46\u001b[0;31m     \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     47\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     48\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m<ipython-input-10-a1b77f69801f>\u001b[0m in \u001b[0;36mobjective\u001b[0;34m(trial)\u001b[0m\n\u001b[1;32m     44\u001b[0m     \u001b[0;32mglobal\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     45\u001b[0m     \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mLSTM_PL\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 46\u001b[0;31m     \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     47\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     48\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
       "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, model)\u001b[0m\n\u001b[1;32m    700\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    701\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msingle_gpu\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 702\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msingle_gpu_train\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    703\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    704\u001b[0m         \u001b[0;31m# ON CPU\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
       "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/trainer/distrib_parts.py\u001b[0m in \u001b[0;36msingle_gpu_train\u001b[0;34m(self, model)\u001b[0m\n\u001b[1;32m    439\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizers\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moptimizers\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    440\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 441\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_pretrain_routine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    442\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    443\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mdp_train\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
       "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36mrun_pretrain_routine\u001b[0;34m(self, model)\u001b[0m\n\u001b[1;32m    839\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    840\u001b[0m         \u001b[0;31m# CORE TRAINING LOOP\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 841\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    842\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    843\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mtest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
       "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/trainer/training_loop.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    331\u001b[0m             \u001b[0;31m# RUN TNG EPOCH\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    332\u001b[0m             \u001b[0;31m# -----------------\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 333\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_training_epoch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    334\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    335\u001b[0m             \u001b[0;31m# update LR schedulers\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
-      "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/trainer/training_loop.py\u001b[0m in \u001b[0;36mrun_training_epoch\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    386\u001b[0m             \u001b[0;31m# RUN TRAIN STEP\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    387\u001b[0m             \u001b[0;31m# ---------------\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 388\u001b[0;31m             \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_training_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    389\u001b[0m             \u001b[0mbatch_result\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_norm_dic\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_step_metrics\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    390\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
-      "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/trainer/training_loop.py\u001b[0m in \u001b[0;36mrun_training_batch\u001b[0;34m(self, batch, batch_idx)\u001b[0m\n\u001b[1;32m    536\u001b[0m                     \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    537\u001b[0m                     model.optimizer_step(self.current_epoch, batch_idx,\n\u001b[0;32m--> 538\u001b[0;31m                                          optimizer, opt_idx, optimizer_closure)\n\u001b[0m\u001b[1;32m    539\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    540\u001b[0m                     \u001b[0;31m# calculate running loss for display\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
-      "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/core/lightning.py\u001b[0m in \u001b[0;36moptimizer_step\u001b[0;34m(self, epoch, batch_idx, optimizer, optimizer_idx, second_order_closure)\u001b[0m\n\u001b[1;32m    790\u001b[0m             \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msecond_order_closure\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    791\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 792\u001b[0;31m             \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    793\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    794\u001b[0m         \u001b[0;31m# clear gradients\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
-      "\u001b[0;32m~/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/torch/optim/adam.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure)\u001b[0m\n\u001b[1;32m     93\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m                 \u001b[0;31m# Decay the first and second moment running average coefficient\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 95\u001b[0;31m                 \u001b[0mexp_avg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbeta1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mbeta1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     96\u001b[0m                 \u001b[0mexp_avg_sq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbeta2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddcmul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mbeta2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     97\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mamsgrad\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/trainer/training_loop.py\u001b[0m in \u001b[0;36mrun_training_epoch\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    417\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mshould_log_metrics\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfast_dev_run\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    418\u001b[0m                 \u001b[0;31m# logs user requested information to logger\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 419\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog_metrics\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_step_metrics\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_norm_dic\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    420\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    421\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mglobal_step\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/trainer/logging.py\u001b[0m in \u001b[0;36mlog_metrics\u001b[0;34m(self, metrics, grad_norm_dic, step)\u001b[0m\n\u001b[1;32m     46\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mproc_rank\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlogger\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     47\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog_metrics\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscalar_metrics\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstep\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 48\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     49\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0madd_tqdm_metrics\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetrics\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/logging/base.py\u001b[0m in \u001b[0;36mwrapped_fn\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mwrapped_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrank\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m             \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mwrapped_fn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/logging/tensorboard.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    100\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    101\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 102\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperiment\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mflush\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    103\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    104\u001b[0m             \u001b[0;31m# you are using PT version (<v1.2) which does not have implemented flush\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/torch/utils/tensorboard/writer.py\u001b[0m in \u001b[0;36mflush\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    956\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    957\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mwriter\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mall_writers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 958\u001b[0;31m             \u001b[0mwriter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mflush\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    959\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    960\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/torch/utils/tensorboard/writer.py\u001b[0m in \u001b[0;36mflush\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    131\u001b[0m         \u001b[0mdisk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    132\u001b[0m         \"\"\"\n\u001b[0;32m--> 133\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevent_writer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mflush\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    134\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    135\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/tensorboard/summary/writer/event_file_writer.py\u001b[0m in \u001b[0;36mflush\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    104\u001b[0m         \u001b[0mdisk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    105\u001b[0m         \"\"\"\n\u001b[0;32m--> 106\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_async_writer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mflush\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    107\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    108\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/tensorboard/summary/writer/event_file_writer.py\u001b[0m in \u001b[0;36mflush\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    153\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_closed\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    154\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mIOError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Writer is closed'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 155\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_byte_queue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    156\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_writer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mflush\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    157\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/.pyenv/versions/3.7.3/lib/python3.7/queue.py\u001b[0m in \u001b[0;36mjoin\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     87\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mall_tasks_done\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     88\u001b[0m             \u001b[0;32mwhile\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munfinished_tasks\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 89\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mall_tasks_done\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     90\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mqsize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/.pyenv/versions/3.7.3/lib/python3.7/threading.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    294\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m    \u001b[0;31m# restore state no matter what (e.g., KeyboardInterrupt)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    295\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 296\u001b[0;31m                 \u001b[0mwaiter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    297\u001b[0m                 \u001b[0mgotit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    298\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
       "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
      ]
     }