From 3b16eba5e4f33ba87be0bfedef0b035deea6055a Mon Sep 17 00:00:00 2001 From: wassname Date: Mon, 27 Jan 2020 17:19:05 +0800 Subject: [PATCH] fix log dirs and plot lstm --- smartmeters-anp-optuna.ipynb | 86 +-- smartmeters-lstm-optuna.ipynb | 1103 ++++----------------------------- src/models/lightning_anp.py | 1 + src/models/lstm.py | 235 ++++--- 4 files changed, 328 insertions(+), 1097 deletions(-) diff --git a/smartmeters-anp-optuna.ipynb b/smartmeters-anp-optuna.ipynb index b858c53..d3cda37 100644 --- a/smartmeters-anp-optuna.ipynb +++ b/smartmeters-anp-optuna.ipynb @@ -20,8 +20,8 @@ "execution_count": 1, "metadata": { "ExecuteTime": { - "end_time": "2020-01-27T08:42:29.539837Z", - "start_time": "2020-01-27T08:42:27.505676Z" + "end_time": "2020-01-27T09:16:15.104531Z", + "start_time": "2020-01-27T09:16:13.059814Z" } }, "outputs": [], @@ -50,8 +50,8 @@ "execution_count": 2, "metadata": { "ExecuteTime": { - "end_time": "2020-01-27T08:42:29.600991Z", - "start_time": "2020-01-27T08:42:29.544096Z" + "end_time": "2020-01-27T09:16:15.147806Z", + "start_time": "2020-01-27T09:16:15.107965Z" } }, "outputs": [], @@ -66,8 +66,8 @@ "execution_count": 3, "metadata": { "ExecuteTime": { - "end_time": "2020-01-27T08:42:29.648401Z", - "start_time": "2020-01-27T08:42:29.604654Z" + "end_time": "2020-01-27T09:16:15.205163Z", + "start_time": "2020-01-27T09:16:15.151170Z" } }, "outputs": [], @@ -82,8 +82,8 @@ "execution_count": 4, "metadata": { "ExecuteTime": { - "end_time": "2020-01-27T08:42:29.779009Z", - "start_time": "2020-01-27T08:42:29.651231Z" + "end_time": "2020-01-27T09:16:15.317804Z", + "start_time": "2020-01-27T09:16:15.208363Z" } }, "outputs": [], @@ -100,8 +100,8 @@ "execution_count": 5, "metadata": { "ExecuteTime": { - "end_time": "2020-01-27T08:42:29.837715Z", - "start_time": "2020-01-27T08:42:29.781031Z" + "end_time": "2020-01-27T09:16:15.363114Z", + "start_time": "2020-01-27T09:16:15.320595Z" } }, "outputs": [], @@ -123,8 +123,8 @@ "execution_count": 6, "metadata": { "ExecuteTime": { - "end_time": "2020-01-27T08:42:38.918046Z", - "start_time": "2020-01-27T08:42:29.840722Z" + "end_time": "2020-01-27T09:16:24.239293Z", + "start_time": "2020-01-27T09:16:15.365709Z" } }, "outputs": [], @@ -137,15 +137,15 @@ "execution_count": 7, "metadata": { "ExecuteTime": { - "end_time": "2020-01-27T08:42:39.642785Z", - "start_time": "2020-01-27T08:42:38.920381Z" + "end_time": "2020-01-27T09:16:24.922850Z", + "start_time": "2020-01-27T09:16:24.242339Z" } }, "outputs": [ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 7, @@ -197,8 +197,8 @@ "execution_count": 8, "metadata": { "ExecuteTime": { - "end_time": "2020-01-27T08:42:39.689983Z", - "start_time": "2020-01-27T08:42:39.646151Z" + "end_time": "2020-01-27T09:16:24.969589Z", + "start_time": "2020-01-27T09:16:24.926123Z" } }, "outputs": [ @@ -231,8 +231,8 @@ "execution_count": 9, "metadata": { "ExecuteTime": { - "end_time": "2020-01-27T08:42:39.776781Z", - "start_time": "2020-01-27T08:42:39.693083Z" + "end_time": "2020-01-27T09:16:25.034860Z", + "start_time": "2020-01-27T09:16:24.972165Z" } }, "outputs": [], @@ -283,8 +283,9 @@ " \n", " # PyTorch Lightning will try to restore model parameters from previous trials if checkpoint\n", " # filenames match. Therefore, the filenames for each trial must be made unique.\n", + " name = 'anp'\n", " checkpoint_callback = pl.callbacks.ModelCheckpoint(\n", - " os.path.join(MODEL_DIR, 'trial_{}'.format(trial.number)), monitor='val_loss', mode=\"min\")\n", + " os.path.join(MODEL_DIR, name, 'version_{}'.format(trial.number), \"chk\"), monitor='val_loss', mode=\"min\")\n", "\n", " # The default logger in PyTorch Lightning writes to event files to be consumed by\n", " # TensorBoard. We create a simple logger instead that holds the log in memory so that the\n", @@ -340,7 +341,7 @@ "execution_count": null, "metadata": { "ExecuteTime": { - "start_time": "2020-01-27T08:42:27.600Z" + "start_time": "2020-01-27T09:16:13.200Z" }, "scrolled": true }, @@ -349,7 +350,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "trial 0 params {'learning_rate': 5.775195863607715e-05, 'hidden_dim': 8, 'latent_dim': 512, 'attention_layers': 2, 'n_latent_encoder_layers': 4, 'n_det_encoder_layers': 8, 'n_decoder_layers': 8, 'dropout': 0.6651734011378423, 'attention_dropout': 0.7254712115642596, 'latent_enc_self_attn_type': 'ptmultihead', 'det_enc_self_attn_type': 'multihead', 'det_enc_cross_attn_type': 'multihead', 'use_lvar': False, 'use_deterministic_path': True, 'min_std': 0.005, 'grad_clip': 40, 'num_context': 48, 'num_extra_target': 24, 'max_nb_epochs': 20, 'num_workers': 4, 'batch_size': 16, 'num_heads': 8, 'x_dim': 16, 'y_dim': 1, 'vis_i': 670}\n" + "trial 0 params {'learning_rate': 0.008760313458256718, 'hidden_dim': 256, 'latent_dim': 64, 'attention_layers': 4, 'n_latent_encoder_layers': 2, 'n_det_encoder_layers': 16, 'n_decoder_layers': 1, 'dropout': 0.3977091380558382, 'attention_dropout': 0.7191709329577967, 'latent_enc_self_attn_type': 'dot', 'det_enc_self_attn_type': 'dot', 'det_enc_cross_attn_type': 'ptmultihead', 'use_lvar': False, 'use_deterministic_path': True, 'min_std': 0.005, 'grad_clip': 40, 'num_context': 48, 'num_extra_target': 24, 'max_nb_epochs': 20, 'num_workers': 4, 'batch_size': 16, 'num_heads': 8, 'x_dim': 16, 'y_dim': 1, 'vis_i': 670}\n" ] }, { @@ -361,18 +362,18 @@ "INFO:root:\n", " Name Type Params\n", "0 model LatentModel 2 M\n", - "1 model._latent_encoder LatentEncoder 10 K\n", - "2 model._latent_encoder._input_layer NPBlockRelu2d 160 \n", - "3 model._latent_encoder._input_layer.linear Linear 144 \n", + "1 model._latent_encoder LatentEncoder 236 K\n", + "2 model._latent_encoder._input_layer NPBlockRelu2d 5 K\n", + "3 model._latent_encoder._input_layer.linear Linear 4 K\n", "4 model._latent_encoder._input_layer.act ReLU 0 \n", ".. ... ... ...\n", - "168 model._decoder._decoder.7.act ReLU 0 \n", - "169 model._decoder._decoder.7.dropout Dropout2d 0 \n", - "170 model._decoder._decoder.7.norm BatchNorm2d 1 K\n", - "171 model._decoder._mean Linear 529 \n", - "172 model._decoder._std Linear 529 \n", + "179 model._decoder._decoder.0.act ReLU 0 \n", + "180 model._decoder._decoder.0.dropout Dropout2d 0 \n", + "181 model._decoder._decoder.0.norm BatchNorm2d 1 K\n", + "182 model._decoder._mean Linear 577 \n", + "183 model._decoder._std Linear 577 \n", "\n", - "[173 rows x 3 columns]\n" + "[184 rows x 3 columns]\n" ] }, { @@ -393,14 +394,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "step 0, {'val_loss': '0.8915367126464844', 'val/kl': '1.9502738268784015e-06', 'val/std': '0.6818246841430664', 'val/mse': '0.33060529828071594'}\n", + "step 0, {'val_loss': '0.6977585554122925', 'val/kl': '8.717864147911314e-06', 'val/std': '0.6536452174186707', 'val/mse': '0.17428502440452576'}\n", "\r" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "510857fe419e429d90eb5eca171118d3", + "model_id": "e56a48896f3a45a3b3644f280a076c53", "version_major": 2, "version_minor": 0 }, @@ -441,16 +442,27 @@ ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": { "ExecuteTime": { "end_time": "2020-01-27T07:35:33.470251Z", "start_time": "2020-01-27T07:34:47.800Z" } }, - "outputs": [], - "source": [] + "source": [ + "# View\n", + "\n", + "TODO" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Test\n", + "\n", + "TODO test best" + ] }, { "cell_type": "code", diff --git a/smartmeters-lstm-optuna.ipynb b/smartmeters-lstm-optuna.ipynb index d4b1076..451274e 100644 --- a/smartmeters-lstm-optuna.ipynb +++ b/smartmeters-lstm-optuna.ipynb @@ -20,8 +20,8 @@ "execution_count": 1, "metadata": { "ExecuteTime": { - "end_time": "2020-01-27T07:45:17.311457Z", - "start_time": "2020-01-27T07:45:15.216733Z" + "end_time": "2020-01-27T09:16:13.478035Z", + "start_time": "2020-01-27T09:16:11.443192Z" } }, "outputs": [], @@ -50,8 +50,8 @@ "execution_count": 2, "metadata": { "ExecuteTime": { - "end_time": "2020-01-27T07:45:17.354181Z", - "start_time": "2020-01-27T07:45:17.314415Z" + "end_time": "2020-01-27T09:16:13.518944Z", + "start_time": "2020-01-27T09:16:13.480810Z" } }, "outputs": [], @@ -66,8 +66,8 @@ "execution_count": 3, "metadata": { "ExecuteTime": { - "end_time": "2020-01-27T07:45:17.405000Z", - "start_time": "2020-01-27T07:45:17.359193Z" + "end_time": "2020-01-27T09:16:13.559666Z", + "start_time": "2020-01-27T09:16:13.521864Z" } }, "outputs": [], @@ -82,8 +82,8 @@ "execution_count": 4, "metadata": { "ExecuteTime": { - "end_time": "2020-01-27T07:45:17.512385Z", - "start_time": "2020-01-27T07:45:17.408270Z" + "end_time": "2020-01-27T09:16:13.663063Z", + "start_time": "2020-01-27T09:16:13.562115Z" } }, "outputs": [], @@ -100,8 +100,8 @@ "execution_count": 5, "metadata": { "ExecuteTime": { - "end_time": "2020-01-27T07:45:17.555331Z", - "start_time": "2020-01-27T07:45:17.515077Z" + "end_time": "2020-01-27T09:16:13.720549Z", + "start_time": "2020-01-27T09:16:13.665200Z" } }, "outputs": [], @@ -123,8 +123,8 @@ "execution_count": 6, "metadata": { "ExecuteTime": { - "end_time": "2020-01-27T07:45:26.369474Z", - "start_time": "2020-01-27T07:45:17.557877Z" + "end_time": "2020-01-27T09:16:22.429851Z", + "start_time": "2020-01-27T09:16:13.723294Z" } }, "outputs": [], @@ -132,46 +132,20 @@ "df_train, df_test = get_smartmeter_df()" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2020-01-27T06:04:43.170460Z", - "start_time": "2020-01-27T06:04:43.082672Z" - } - }, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, { "cell_type": "code", "execution_count": 7, "metadata": { "ExecuteTime": { - "end_time": "2020-01-27T07:45:27.062986Z", - "start_time": "2020-01-27T07:45:26.371775Z" + "end_time": "2020-01-27T09:16:23.088727Z", + "start_time": "2020-01-27T09:16:22.432605Z" } }, "outputs": [ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 7, @@ -220,11 +194,11 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "metadata": { "ExecuteTime": { - "end_time": "2020-01-27T07:46:23.858802Z", - "start_time": "2020-01-27T07:46:23.805320Z" + "end_time": "2020-01-27T09:16:23.132107Z", + "start_time": "2020-01-27T09:16:23.092355Z" } }, "outputs": [], @@ -238,14 +212,22 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 9, "metadata": { "ExecuteTime": { - "end_time": "2020-01-27T07:46:24.064322Z", - "start_time": "2020-01-27T07:46:24.015555Z" + "end_time": "2020-01-27T09:16:23.175803Z", + "start_time": "2020-01-27T09:16:23.134684Z" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "now run `tensorboard --logdir /media/wassname/Storage5/projects2/3ST/attentive-neural-processes/optuna_result/lstm\n" + ] + } + ], "source": [ "print(f\"now run `tensorboard --logdir {MODEL_DIR}\")" ] @@ -264,11 +246,11 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 10, "metadata": { "ExecuteTime": { - "end_time": "2020-01-27T07:46:24.375249Z", - "start_time": "2020-01-27T07:46:24.321671Z" + "end_time": "2020-01-27T09:16:23.236356Z", + "start_time": "2020-01-27T09:16:23.179477Z" } }, "outputs": [], @@ -298,14 +280,15 @@ " \n", " # PyTorch Lightning will try to restore model parameters from previous trials if checkpoint\n", " # filenames match. Therefore, the filenames for each trial must be made unique.\n", + " name = \"lstm\"\n", " checkpoint_callback = pl.callbacks.ModelCheckpoint(\n", - " os.path.join(MODEL_DIR, 'trial_{}'.format(trial.number)), monitor='val_loss', mode='min')\n", + " os.path.join(MODEL_DIR, name, 'version_{}'.format(trial.number), \"chk\"), monitor='val_loss', mode='min')\n", "\n", " # The default logger in PyTorch Lightning writes to event files to be consumed by\n", " # TensorBoard. We create a simple logger instead that holds the log in memory so that the\n", " # final accuracy can be obtained after optimization. When using the default logger, the\n", " # final accuracy could be stored in an attribute of the `Trainer` instead.\n", - " logger = DictLogger(MODEL_DIR, name=\"lstm\", version=trial.number)\n", + " logger = DictLogger(MODEL_DIR, name=name, version=trial.number)\n", "# print(\"log_dir\", logger.experiment.log_dir)\n", "\n", " trainer = pl.Trainer(\n", @@ -330,11 +313,10 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2020-01-27T08:41:55.620861Z", - "start_time": "2020-01-27T07:46:24.471638Z" + "start_time": "2020-01-27T09:16:11.600Z" } }, "outputs": [ @@ -342,8 +324,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "0 {'learning_rate': 0.000638083680184452, 'lstm_dropout': 0.40417652550416117, 'hidden_size': 16, 'lstm_layers': 4, 'bidirectional': True, 'window_length': 48, 'target_length': 24, 'max_nb_epochs': 2, 'num_workers': 4, 'grad_clip': 40, 'vis_i': 670, 'input_size': 17, 'batch_size': 16}\n", - "log_dir /media/wassname/Storage5/projects2/3ST/attentive-neural-processes/optuna_result/lstm/lstm/version_0\n" + "trial 0 params {'learning_rate': 0.00010146652064922795, 'lstm_dropout': 0.5524433594179708, 'hidden_size': 8, 'lstm_layers': 64, 'bidirectional': True, 'window_length': 48, 'target_length': 24, 'max_nb_epochs': 20, 'num_workers': 4, 'grad_clip': 40, 'vis_i': 670, 'input_size': 17, 'batch_size': 16}\n" ] }, { @@ -353,9 +334,10 @@ "INFO:root:gpu available: True, used: True\n", "INFO:root:VISIBLE GPUS: 0\n", "INFO:root:\n", - " Name Type Params\n", - "0 lstm1 LSTM 23 K\n", - "1 linear Linear 129 \n" + " Name Type Params\n", + "0 _model LSTMNet 107 K\n", + "1 _model.lstm1 LSTM 106 K\n", + "2 _model.linear Linear 1 K\n" ] }, { @@ -376,14 +358,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "step 0, {'val_loss': '0.28643688559532166'}\n", + "step 0, {'val_loss': '0.297493040561676'}\n", "\r" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "78a62b388c6b451da11c9bfcbb1915fe", + "model_id": "015bffde987142e7ad2e9d8ff3e11686", "version_major": 2, "version_minor": 0 }, @@ -393,918 +375,6 @@ }, "metadata": {}, "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=72.0, style=Pro…" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step 2201, {'val_loss': '0.010225002653896809'}\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=72.0, style=Pro…" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step 4403, {'val_loss': '0.006430376321077347'}\n", - "\n", - "[{'val_loss': 0.006430376321077347, 'epoch': 1}]\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[I 2020-01-27 15:50:44,126] Finished trial#0 resulted in value: 0.006430376321077347. Current best value is 0.006430376321077347 with parameters: {'learning_rate': 0.000638083680184452, 'lstm_dropout': 0.40417652550416117, 'hidden_size': 16, 'lstm_layers': 4, 'bidirectional': True, 'window_length': 48, 'target_length': 24, 'max_nb_epochs': 2, 'num_workers': 4, 'grad_clip': 40, 'vis_i': 670, 'input_size': 17, 'batch_size': 16}.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1 {'learning_rate': 0.002006580734997886, 'lstm_dropout': 0.04391486258894739, 'hidden_size': 8, 'lstm_layers': 8, 'bidirectional': True, 'window_length': 48, 'target_length': 24, 'max_nb_epochs': 2, 'num_workers': 4, 'grad_clip': 40, 'vis_i': 670, 'input_size': 17, 'batch_size': 16}\n", - "log_dir /media/wassname/Storage5/projects2/3ST/attentive-neural-processes/optuna_result/lstm/lstm/version_1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:root:gpu available: True, used: True\n", - "INFO:root:VISIBLE GPUS: 0\n", - "INFO:root:\n", - " Name Type Params\n", - "0 lstm1 LSTM 13 K\n", - "1 linear Linear 129 \n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Validation sanity check', layout=Layout(flex='2'), max=5.…" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step 0, {'val_loss': '0.1360984891653061'}\n", - "\r" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "e19165b8fd3b40869df54031cad0439a", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=72.0, style=Pro…" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step 2201, {'val_loss': '0.010522536002099514'}\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=72.0, style=Pro…" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step 4403, {'val_loss': '0.008737520314753056'}\n", - "\n", - "[{'val_loss': 0.008737520314753056, 'epoch': 1}]\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[I 2020-01-27 15:57:54,851] Finished trial#1 resulted in value: 0.008737520314753056. Current best value is 0.006430376321077347 with parameters: {'learning_rate': 0.000638083680184452, 'lstm_dropout': 0.40417652550416117, 'hidden_size': 16, 'lstm_layers': 4, 'bidirectional': True, 'window_length': 48, 'target_length': 24, 'max_nb_epochs': 2, 'num_workers': 4, 'grad_clip': 40, 'vis_i': 670, 'input_size': 17, 'batch_size': 16}.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2 {'learning_rate': 0.003387458294570852, 'lstm_dropout': 0.13275103800337396, 'hidden_size': 16, 'lstm_layers': 1, 'bidirectional': False, 'window_length': 48, 'target_length': 24, 'max_nb_epochs': 2, 'num_workers': 4, 'grad_clip': 40, 'vis_i': 670, 'input_size': 17, 'batch_size': 16}\n", - "log_dir /media/wassname/Storage5/projects2/3ST/attentive-neural-processes/optuna_result/lstm/lstm/version_2\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:root:gpu available: True, used: True\n", - "INFO:root:VISIBLE GPUS: 0\n", - "/home/wassname/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/torch/nn/modules/rnn.py:51: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.13275103800337396 and num_layers=1\n", - " \"num_layers={}\".format(dropout, num_layers))\n", - "INFO:root:\n", - " Name Type Params\n", - "0 lstm1 LSTM 2 K\n", - "1 linear Linear 17 \n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Validation sanity check', layout=Layout(flex='2'), max=5.…" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step 0, {'val_loss': '0.3834688067436218'}\n", - "\r" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "4e0b1ccf11cc49519287086a218ed012", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=72.0, style=Pro…" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step 2201, {'val_loss': '0.014188763685524464'}\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=72.0, style=Pro…" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step 4403, {'val_loss': '0.008127140812575817'}\n", - "\n", - "[{'val_loss': 0.008127140812575817, 'epoch': 1}]\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[I 2020-01-27 16:00:57,632] Finished trial#2 resulted in value: 0.008127140812575817. Current best value is 0.006430376321077347 with parameters: {'learning_rate': 0.000638083680184452, 'lstm_dropout': 0.40417652550416117, 'hidden_size': 16, 'lstm_layers': 4, 'bidirectional': True, 'window_length': 48, 'target_length': 24, 'max_nb_epochs': 2, 'num_workers': 4, 'grad_clip': 40, 'vis_i': 670, 'input_size': 17, 'batch_size': 16}.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3 {'learning_rate': 0.0002715141337319053, 'lstm_dropout': 0.3917594507436753, 'hidden_size': 128, 'lstm_layers': 1, 'bidirectional': True, 'window_length': 48, 'target_length': 24, 'max_nb_epochs': 2, 'num_workers': 4, 'grad_clip': 40, 'vis_i': 670, 'input_size': 17, 'batch_size': 16}\n", - "log_dir /media/wassname/Storage5/projects2/3ST/attentive-neural-processes/optuna_result/lstm/lstm/version_3\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:root:gpu available: True, used: True\n", - "INFO:root:VISIBLE GPUS: 0\n", - "/home/wassname/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/torch/nn/modules/rnn.py:51: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.3917594507436753 and num_layers=1\n", - " \"num_layers={}\".format(dropout, num_layers))\n", - "INFO:root:\n", - " Name Type Params\n", - "0 lstm1 LSTM 150 K\n", - "1 linear Linear 257 \n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Validation sanity check', layout=Layout(flex='2'), max=5.…" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step 0, {'val_loss': '0.3805188834667206'}\n", - "\r" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "2ab9f5e1cad744ec86c3dd61d6bb85ee", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=72.0, style=Pro…" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step 2201, {'val_loss': '0.0071783727034926414'}\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=72.0, style=Pro…" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step 4403, {'val_loss': '0.008768502622842789'}\n", - "\n", - "[{'val_loss': 0.008768502622842789, 'epoch': 1}]\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[I 2020-01-27 16:03:48,715] Finished trial#3 resulted in value: 0.008768502622842789. Current best value is 0.006430376321077347 with parameters: {'learning_rate': 0.000638083680184452, 'lstm_dropout': 0.40417652550416117, 'hidden_size': 16, 'lstm_layers': 4, 'bidirectional': True, 'window_length': 48, 'target_length': 24, 'max_nb_epochs': 2, 'num_workers': 4, 'grad_clip': 40, 'vis_i': 670, 'input_size': 17, 'batch_size': 16}.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "4 {'learning_rate': 0.004588567080154951, 'lstm_dropout': 0.2478702730829206, 'hidden_size': 8, 'lstm_layers': 8, 'bidirectional': True, 'window_length': 48, 'target_length': 24, 'max_nb_epochs': 2, 'num_workers': 4, 'grad_clip': 40, 'vis_i': 670, 'input_size': 17, 'batch_size': 16}\n", - "log_dir /media/wassname/Storage5/projects2/3ST/attentive-neural-processes/optuna_result/lstm/lstm/version_4\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:root:gpu available: True, used: True\n", - "INFO:root:VISIBLE GPUS: 0\n", - "INFO:root:\n", - " Name Type Params\n", - "0 lstm1 LSTM 13 K\n", - "1 linear Linear 129 \n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Validation sanity check', layout=Layout(flex='2'), max=5.…" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step 0, {'val_loss': '0.42670002579689026'}\n", - "\r" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "6f2357f5522f4be29950766c40367a44", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=72.0, style=Pro…" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step 2201, {'val_loss': '0.013795542530715466'}\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=72.0, style=Pro…" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step 4403, {'val_loss': '0.008892875164747238'}\n", - "\n", - "[{'val_loss': 0.008892875164747238, 'epoch': 1}]\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[I 2020-01-27 16:10:48,500] Finished trial#4 resulted in value: 0.008892875164747238. Current best value is 0.006430376321077347 with parameters: {'learning_rate': 0.000638083680184452, 'lstm_dropout': 0.40417652550416117, 'hidden_size': 16, 'lstm_layers': 4, 'bidirectional': True, 'window_length': 48, 'target_length': 24, 'max_nb_epochs': 2, 'num_workers': 4, 'grad_clip': 40, 'vis_i': 670, 'input_size': 17, 'batch_size': 16}.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "5 {'learning_rate': 0.004958811856881581, 'lstm_dropout': 0.5755318563895904, 'hidden_size': 32, 'lstm_layers': 32, 'bidirectional': False, 'window_length': 48, 'target_length': 24, 'max_nb_epochs': 2, 'num_workers': 4, 'grad_clip': 40, 'vis_i': 670, 'input_size': 17, 'batch_size': 16}\n", - "log_dir /media/wassname/Storage5/projects2/3ST/attentive-neural-processes/optuna_result/lstm/lstm/version_5\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:root:gpu available: True, used: True\n", - "INFO:root:VISIBLE GPUS: 0\n", - "INFO:root:\n", - " Name Type Params\n", - "0 lstm1 LSTM 268 K\n", - "1 linear Linear 1 K\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Validation sanity check', layout=Layout(flex='2'), max=5.…" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step 0, {'val_loss': '0.22852063179016113'}\n", - "\r" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "37766733232b4ee88534a8ffe6ae0d32", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=72.0, style=Pro…" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step 2201, {'val_loss': '0.010773574002087116'}\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=72.0, style=Pro…" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step 4403, {'val_loss': '0.01649811863899231'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[I 2020-01-27 16:29:02,527] Setting status of trial#5 as TrialState.PRUNED. Trial was pruned at epoch 1.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "6 {'learning_rate': 0.00010708601072581249, 'lstm_dropout': 0.6184462116956337, 'hidden_size': 64, 'lstm_layers': 2, 'bidirectional': False, 'window_length': 48, 'target_length': 24, 'max_nb_epochs': 2, 'num_workers': 4, 'grad_clip': 40, 'vis_i': 670, 'input_size': 17, 'batch_size': 16}\n", - "log_dir /media/wassname/Storage5/projects2/3ST/attentive-neural-processes/optuna_result/lstm/lstm/version_6\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:root:gpu available: True, used: True\n", - "INFO:root:VISIBLE GPUS: 0\n", - "INFO:root:\n", - " Name Type Params\n", - "0 lstm1 LSTM 54 K\n", - "1 linear Linear 129 \n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Validation sanity check', layout=Layout(flex='2'), max=5.…" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step 0, {'val_loss': '0.18932603299617767'}\n", - "\r" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "25161cdf3b9048ccb6638f21dc8c49a6", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=72.0, style=Pro…" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step 2201, {'val_loss': '0.031354598701000214'}\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=72.0, style=Pro…" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step 4403, {'val_loss': '0.00781797245144844'}\n", - "\n", - "[{'val_loss': 0.00781797245144844, 'epoch': 1}]\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[I 2020-01-27 16:31:52,305] Finished trial#6 resulted in value: 0.00781797245144844. Current best value is 0.006430376321077347 with parameters: {'learning_rate': 0.000638083680184452, 'lstm_dropout': 0.40417652550416117, 'hidden_size': 16, 'lstm_layers': 4, 'bidirectional': True, 'window_length': 48, 'target_length': 24, 'max_nb_epochs': 2, 'num_workers': 4, 'grad_clip': 40, 'vis_i': 670, 'input_size': 17, 'batch_size': 16}.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "7 {'learning_rate': 0.00020591102432589484, 'lstm_dropout': 0.4887320755159783, 'hidden_size': 2, 'lstm_layers': 4, 'bidirectional': True, 'window_length': 48, 'target_length': 24, 'max_nb_epochs': 2, 'num_workers': 4, 'grad_clip': 40, 'vis_i': 670, 'input_size': 17, 'batch_size': 16}\n", - "log_dir /media/wassname/Storage5/projects2/3ST/attentive-neural-processes/optuna_result/lstm/lstm/version_7\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:root:gpu available: True, used: True\n", - "INFO:root:VISIBLE GPUS: 0\n", - "INFO:root:\n", - " Name Type Params\n", - "0 lstm1 LSTM 720 \n", - "1 linear Linear 17 \n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Validation sanity check', layout=Layout(flex='2'), max=5.…" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step 0, {'val_loss': '0.12604233622550964'}\n", - "\r" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "c00aafd263394bbb9e6457f96e211e2d", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=72.0, style=Pro…" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step 2201, {'val_loss': '0.03605831786990166'}\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=72.0, style=Pro…" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step 4403, {'val_loss': '0.033860258758068085'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[I 2020-01-27 16:35:42,156] Setting status of trial#7 as TrialState.PRUNED. Trial was pruned at epoch 1.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "8 {'learning_rate': 0.0002114319194224485, 'lstm_dropout': 0.18956161922634335, 'hidden_size': 4, 'lstm_layers': 8, 'bidirectional': True, 'window_length': 48, 'target_length': 24, 'max_nb_epochs': 2, 'num_workers': 4, 'grad_clip': 40, 'vis_i': 670, 'input_size': 17, 'batch_size': 16}\n", - "log_dir /media/wassname/Storage5/projects2/3ST/attentive-neural-processes/optuna_result/lstm/lstm/version_8\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:root:gpu available: True, used: True\n", - "INFO:root:VISIBLE GPUS: 0\n", - "INFO:root:\n", - " Name Type Params\n", - "0 lstm1 LSTM 3 K\n", - "1 linear Linear 65 \n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Validation sanity check', layout=Layout(flex='2'), max=5.…" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step 0, {'val_loss': '0.23964658379554749'}\n", - "\r" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "c31ae32a33e84982a9775a647d21a514", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=72.0, style=Pro…" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step 2201, {'val_loss': '0.031666651368141174'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Exception ignored in: \n", - "Traceback (most recent call last):\n", - " File \"/home/wassname/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/tqdm/std.py\", line 1061, in __del__\n", - " def __del__(self):\n", - "KeyboardInterrupt: \n" - ] - }, - { - "ename": "KeyboardInterrupt", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0mstudy\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moptuna\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreate_study\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdirection\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'minimize'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpruner\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpruner\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m \u001b[0mstudy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobjective\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_trials\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m6000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 14\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Number of finished trials: {}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstudy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrials\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/optuna/study.py\u001b[0m in \u001b[0;36moptimize\u001b[0;34m(self, func, n_trials, timeout, n_jobs, catch, callbacks, gc_after_trial)\u001b[0m\n\u001b[1;32m 300\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mn_jobs\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 301\u001b[0m self._optimize_sequential(func, n_trials, timeout, catch, callbacks,\n\u001b[0;32m--> 302\u001b[0;31m gc_after_trial, None)\n\u001b[0m\u001b[1;32m 303\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 304\u001b[0m \u001b[0mtime_start\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdatetime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdatetime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/optuna/study.py\u001b[0m in \u001b[0;36m_optimize_sequential\u001b[0;34m(self, func, n_trials, timeout, catch, callbacks, gc_after_trial, time_start)\u001b[0m\n\u001b[1;32m 536\u001b[0m \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 537\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 538\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_trial_and_callbacks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgc_after_trial\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 539\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_storage\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mremove_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 540\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/optuna/study.py\u001b[0m in \u001b[0;36m_run_trial_and_callbacks\u001b[0;34m(self, func, catch, callbacks, gc_after_trial)\u001b[0m\n\u001b[1;32m 548\u001b[0m \u001b[0;31m# type: (...) -> None\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 549\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 550\u001b[0;31m \u001b[0mtrial\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_trial\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgc_after_trial\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 551\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcallbacks\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 552\u001b[0m \u001b[0mfrozen_trial\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_storage\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_trial\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_trial_id\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/optuna/study.py\u001b[0m in \u001b[0;36m_run_trial\u001b[0;34m(self, func, catch, gc_after_trial)\u001b[0m\n\u001b[1;32m 567\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 568\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 569\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 570\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mexceptions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTrialPruned\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 571\u001b[0m message = 'Setting status of trial#{} as {}. {}'.format(trial_number,\n", - "\u001b[0;32m\u001b[0m in \u001b[0;36mobjective\u001b[0;34m(trial)\u001b[0m\n\u001b[1;32m 44\u001b[0m \u001b[0;32mglobal\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 45\u001b[0m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mLSTM_PL\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 46\u001b[0;31m \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 47\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 48\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, model)\u001b[0m\n\u001b[1;32m 700\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 701\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msingle_gpu\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 702\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msingle_gpu_train\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 703\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 704\u001b[0m \u001b[0;31m# ON CPU\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/trainer/distrib_parts.py\u001b[0m in \u001b[0;36msingle_gpu_train\u001b[0;34m(self, model)\u001b[0m\n\u001b[1;32m 439\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizers\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moptimizers\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 440\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 441\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_pretrain_routine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 442\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 443\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mdp_train\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36mrun_pretrain_routine\u001b[0;34m(self, model)\u001b[0m\n\u001b[1;32m 839\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 840\u001b[0m \u001b[0;31m# CORE TRAINING LOOP\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 841\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 842\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 843\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mtest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/trainer/training_loop.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 331\u001b[0m \u001b[0;31m# RUN TNG EPOCH\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 332\u001b[0m \u001b[0;31m# -----------------\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 333\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_training_epoch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 334\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 335\u001b[0m \u001b[0;31m# update LR schedulers\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/trainer/training_loop.py\u001b[0m in \u001b[0;36mrun_training_epoch\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 386\u001b[0m \u001b[0;31m# RUN TRAIN STEP\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 387\u001b[0m \u001b[0;31m# ---------------\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 388\u001b[0;31m \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_training_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 389\u001b[0m \u001b[0mbatch_result\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_norm_dic\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_step_metrics\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 390\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/trainer/training_loop.py\u001b[0m in \u001b[0;36mrun_training_batch\u001b[0;34m(self, batch, batch_idx)\u001b[0m\n\u001b[1;32m 536\u001b[0m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 537\u001b[0m model.optimizer_step(self.current_epoch, batch_idx,\n\u001b[0;32m--> 538\u001b[0;31m optimizer, opt_idx, optimizer_closure)\n\u001b[0m\u001b[1;32m 539\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 540\u001b[0m \u001b[0;31m# calculate running loss for display\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/core/lightning.py\u001b[0m in \u001b[0;36moptimizer_step\u001b[0;34m(self, epoch, batch_idx, optimizer, optimizer_idx, second_order_closure)\u001b[0m\n\u001b[1;32m 790\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msecond_order_closure\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 791\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 792\u001b[0;31m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 793\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 794\u001b[0m \u001b[0;31m# clear gradients\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/torch/optim/adam.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure)\u001b[0m\n\u001b[1;32m 94\u001b[0m \u001b[0;31m# Decay the first and second moment running average coefficient\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 95\u001b[0m \u001b[0mexp_avg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbeta1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mbeta1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 96\u001b[0;31m \u001b[0mexp_avg_sq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbeta2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddcmul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mbeta2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 97\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mamsgrad\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 98\u001b[0m \u001b[0;31m# Maintains the maximum of all 2nd moment running avg. till now\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " - ] } ], "source": [ @@ -1340,7 +410,8 @@ "execution_count": null, "metadata": { "ExecuteTime": { - "start_time": "2020-01-27T06:59:13.000Z" + "end_time": "2020-01-27T09:05:22.547568Z", + "start_time": "2020-01-27T09:05:22.473558Z" } }, "outputs": [], @@ -1349,14 +420,90 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "ExecuteTime": { + "start_time": "2020-01-27T09:16:11.600Z" + } + }, + "outputs": [], + "source": [ + "# test" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "start_time": "2020-01-27T09:16:11.600Z" + } + }, + "outputs": [], + "source": [ + "trial = study.trials[0]\n", + "model = LSTM_PL(trial.params)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "start_time": "2020-01-27T09:16:11.600Z" + } + }, + "outputs": [], + "source": [ + "loader = model.val_dataloader()[0]\n", + "dset_test = loader.dataset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "start_time": "2020-01-27T09:16:11.600Z" + } + }, + "outputs": [], + "source": [ + "model.cuda()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-01-27T09:02:48.225225Z", + "start_time": "2020-01-27T09:02:48.174287Z" + } + }, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "ExecuteTime": { + "end_time": "2020-01-27T09:03:20.413623Z", + "start_time": "2020-01-27T09:03:20.313659Z" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-01-27T09:03:28.792465Z", + "start_time": "2020-01-27T09:03:28.346446Z" + } + }, "outputs": [], "source": [] }, diff --git a/src/models/lightning_anp.py b/src/models/lightning_anp.py index dda2f9d..2caa6ac 100644 --- a/src/models/lightning_anp.py +++ b/src/models/lightning_anp.py @@ -9,6 +9,7 @@ from src.plot import plot_from_loader_to_tensor from src.utils import ObjectDict + class LatentModelPL(pl.LightningModule): def __init__(self, hparams): super().__init__() diff --git a/src/models/lstm.py b/src/models/lstm.py index cea284d..07fc1d9 100644 --- a/src/models/lstm.py +++ b/src/models/lstm.py @@ -10,11 +10,17 @@ import torchvision.transforms as transforms from argparse import ArgumentParser import json import pytorch_lightning as pl +from matplotlib import pyplot as plt +import torch +import io +import PIL +from torchvision.transforms import ToTensor from src.data.smart_meter import get_smartmeter_df from src.utils import ObjectDict + class SequenceDfDataSet(torch.utils.data.Dataset): def __init__(self, df, hparams, label_names=None, train=True, transforms=None): super().__init__() @@ -22,102 +28,120 @@ class SequenceDfDataSet(torch.utils.data.Dataset): self.hparams = hparams self.label_names = label_names self.train = train - self.transforms=transforms - + self.transforms = transforms + def __len__(self): return len(self.data) - +self.hparams.window_length - self.hparams.target_length - + def iloc(self, idx): - k = idx+self.hparams.window_length+self.hparams.target_length - j = k-self.hparams.target_length - i = j-self.hparams.window_length - assert i>=0 - assert idx<=len(self.data) + k = idx + self.hparams.window_length + self.hparams.target_length + j = k - self.hparams.target_length + i = j - self.hparams.window_length + assert i >= 0 + assert idx <= len(self.data) x_rows = self.data.iloc[i:j].copy() y_rows = self.data.iloc[k].to_frame().T.copy() -# print(i,j,k) - + # print(i,j,k) + # add seconds since start of window index - x_rows['tstp'] = (x_rows['tstp'] - x_rows['tstp'].iloc[0]).dt.total_seconds() / 86400.0 - + x_rows["tstp"] = ( + x_rows["tstp"] - x_rows["tstp"].iloc[0] + ).dt.total_seconds() / 86400.0 + # TODO we could augment by removing and backfilling some return x_rows, y_rows def __getitem__(self, idx): x_rows, y_rows = self.iloc(idx) - -# if self.train: -# # zero and backfill some for augmentation -# drop_inds = np.random.randint(1, len(x_rows)-1, size=int(len(x_rows)*0.3)) -# x_rows.iloc[drop_inds] = np.nan -# x_rows = x_rows.bfill() - -# print(x_rows, y_rows) + y = y_rows[self.label_names].astype(np.float32).values x = x_rows.astype(np.float32).values -# print(x, y) - return self.transforms(x).squeeze(0).float(), self.transforms(y[:, None,])[:, 0, 0].float() - + return ( + self.transforms(x).squeeze(0).float(), + self.transforms(y[:, None,])[:, 0, 0].float(), + ) - - -class LSTM_PL(pl.LightningModule): - +class LSTMNet(nn.Module): def __init__(self, hparams): super().__init__() - self.hparams = ObjectDict() - self.hparams.update(hparams.__dict__ if hasattr(hparams, '__dict__') else hparams) + self.hparams = hparams + self.lstm1 = nn.LSTM( - input_size=self.hparams.input_size, - hidden_size=self.hparams.hidden_size, + input_size=self.hparams.input_size, + hidden_size=self.hparams.hidden_size, batch_first=True, num_layers=self.hparams.lstm_layers, bidirectional=self.hparams.bidirectional, dropout=self.hparams.lstm_dropout, ) - self.hidden_out_size = self.hparams.hidden_size * self.hparams.lstm_layers * (self.hparams.bidirectional + 1) + self.hidden_out_size = ( + self.hparams.hidden_size + * self.hparams.lstm_layers + * (self.hparams.bidirectional + 1) + ) self.linear = nn.Linear(self.hidden_out_size, 1) - self._dfs = None def forward(self, x): outputs, (h_out, _) = self.lstm1(x) h_out = h_out.permute((1, 0, 2)).reshape((-1, self.hidden_out_size)) return self.linear(h_out) + +class LSTM_PL(pl.LightningModule): + def __init__(self, hparams): + super().__init__() + self.hparams = ObjectDict() + self.hparams.update( + hparams.__dict__ if hasattr(hparams, "__dict__") else hparams + ) + self._model = LSTMNet(self.hparams) + self._dfs = None + + def forward(self, x): + return self._model(x) + def training_step(self, batch, batch_idx): # REQUIRED x, y = batch y_hat = self.forward(x) loss = F.mse_loss(y_hat, y) - tensorboard_logs = {'train_loss': loss} - return {'loss': loss, 'log': tensorboard_logs} + tensorboard_logs = {"train_loss": loss} + return {"loss": loss, "log": tensorboard_logs} def validation_step(self, batch, batch_idx): - # OPTIONAL x, y = batch y_hat = self.forward(x) loss = F.mse_loss(y_hat, y) - tensorboard_logs = {'val_loss': loss} - return {'val_loss': loss, 'log': tensorboard_logs} + tensorboard_logs = {"val_loss": loss} + return {"val_loss": loss, "log": tensorboard_logs} def validation_end(self, outputs): # TODO send an image to tensroboard, like in the lighting_anp.py file - # if self.hparams["vis_i"] > 0: - # self.logger.experiment.add_image('val/image', image, + if self.hparams["vis_i"] > 0: + loader = self.val_dataloader()[0] + vis_i = min(self.hparams["vis_i"], len(loader.dataset)) + image = plot_from_loader_to_tensor(loader, self, vis_i=vis_i) + self.logger.experiment.add_image( + "val/image", image, self.trainer.global_step + ) - # OPTIONAL - avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean() + avg_loss = torch.stack([x["val_loss"] for x in outputs]).mean() keys = outputs[0]["log"].keys() - tensorboard_logs = {k: torch.stack([x["log"][k] for x in outputs if k in x["log"]]).mean() for k in keys} - tensorboard_logs_str = {k: f'{v}' for k, v in tensorboard_logs.items()} + tensorboard_logs = { + k: torch.stack([x["log"][k] for x in outputs if k in x["log"]]).mean() + for k in keys + } + tensorboard_logs_str = {k: f"{v}" for k, v in tensorboard_logs.items()} print(f"step {self.trainer.global_step}, {tensorboard_logs_str}") - return {'avg_val_loss': avg_loss, 'log': tensorboard_logs} + return {"avg_val_loss": avg_loss, "log": tensorboard_logs} def configure_optimizers(self): - # REQUIRED - # can return multiple optimizers and learning_rate schedulers - return torch.optim.Adam(self.parameters(), lr=self.hparams.learning_rate) + optim = torch.optim.Adam(self.parameters(), lr=self.hparams["learning_rate"]) + scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau( + optim, patience=2, verbose=True, min_lr=1e-5 + ) # note early stopping has patient 3 + return [optim], [scheduler] def _get_cache_dfs(self): if self._dfs is None: @@ -128,25 +152,43 @@ class LSTM_PL(pl.LightningModule): @pl.data_loader def train_dataloader(self): - # REQUIRED - df_train = self._get_cache_dfs()['df_train'] - dset_train = SequenceDfDataSet(df_train, self.hparams, label_names=['energy(kWh/hh)'], transforms=transforms.ToTensor(), train=True) - return DataLoader(dset_train, batch_size=self.hparams.batch_size, - shuffle=True, - num_workers=self.hparams.num_workers) + df_train = self._get_cache_dfs()["df_train"] + dset_train = SequenceDfDataSet( + df_train, + self.hparams, + label_names=["energy(kWh/hh)"], + transforms=transforms.ToTensor(), + train=True, + ) + return DataLoader( + dset_train, + batch_size=self.hparams.batch_size, + shuffle=True, + num_workers=self.hparams.num_workers, + ) @pl.data_loader def val_dataloader(self): - # OPTIONAL - df_test = self._get_cache_dfs()['df_test'] - dset_test = SequenceDfDataSet(df_test, self.hparams, label_names=['energy(kWh/hh)'], train=False, transforms=transforms.ToTensor()) + df_test = self._get_cache_dfs()["df_test"] + dset_test = SequenceDfDataSet( + df_test, + self.hparams, + label_names=["energy(kWh/hh)"], + train=False, + transforms=transforms.ToTensor(), + ) return DataLoader(dset_test, batch_size=self.hparams.batch_size, shuffle=False) @pl.data_loader def test_dataloader(self): - # OPTIONAL - df_test = self._get_cache_dfs()['df_test'] - dset_test = SequenceDfDataSet(df_test, self.hparams, label_names=['energy(kWh/hh)'], train=False, transforms=transforms.ToTensor()) + df_test = self._get_cache_dfs()["df_test"] + dset_test = SequenceDfDataSet( + df_test, + self.hparams, + label_names=["energy(kWh/hh)"], + train=False, + transforms=transforms.ToTensor(), + ) return DataLoader(dset_test, batch_size=self.hparams.batch_size, shuffle=False) @staticmethod @@ -156,33 +198,62 @@ class LSTM_PL(pl.LightningModule): """ # MODEL specific parser = HyperOptArgumentParser(parents=[parent_parser]) - parser.add_argument('--learning_rate', default=0.02, type=float) - parser.add_argument('--batch_size', default=32, type=int) - parser.add_argument('--lstm_dropout', default=0, type=float) - parser.add_argument('--hidden_size', default=32, type=int) - parser.add_argument('--input_size', default=8, type=int) - parser.add_argument('--lstm_layers', default=4, type=int) - parser.add_argument('--bidirectional', default=False, type=bool) + parser.add_argument("--learning_rate", default=0.02, type=float) + parser.add_argument("--batch_size", default=32, type=int) + parser.add_argument("--lstm_dropout", default=0, type=float) + parser.add_argument("--hidden_size", default=32, type=int) + parser.add_argument("--input_size", default=8, type=int) + parser.add_argument("--lstm_layers", default=4, type=int) + parser.add_argument("--bidirectional", default=False, type=bool) # training specific (for this model) - parser.add_argument('--window_length', type=int, default=12) - parser.add_argument('--target_length', type=int, default=2) - parser.add_argument('--max_nb_epochs', default=10, type=int) - parser.add_argument('--num_workers', default=4, type=int) + parser.add_argument("--window_length", type=int, default=12) + parser.add_argument("--target_length", type=int, default=2) + parser.add_argument("--max_nb_epochs", default=10, type=int) + parser.add_argument("--num_workers", default=4, type=int) return parser -# dset_train = SequenceDfDataSet(df_train, hparams, transforms=transforms.ToTensor()) -# dset_test = SequenceDfDataSet(df_test, hparams, train=False, transforms=transforms.ToTensor()) -# dset_val = SequenceDfDataSet(df_test, hparams, train=False, transforms=transforms.ToTensor()) +def plot_from_loader(loader, model, vis_i=670): + dset_test = loader.dataset + label_names = dset_test.label_names + y_trues = [] + y_preds = [] + vis_i = min(vis_i, len(dset_test)) + for i in range(vis_i, vis_i + 50): + x_rows, y_rows = dset_test.iloc(i) + x, y = dset_test[i] + device = next(model.parameters()).device + x = x[None, :].to(device) + model.eval() + with torch.no_grad(): + y_hat = model.forward(x) + y_hat = y_hat.cpu().numpy() -# model = LSTM_PL(hparams) + dt = y_rows.iloc[0].name -# # most basic trainer, uses good defaults -# trainer = Trainer( -# max_nb_epochs=hparams.max_nb_epochs, -# gpus=hparams.gpus, -# nb_gpu_nodes=hparams.nodes, -# ) -# trainer.fit(model) + y_hat_rows = y_rows.copy() + y_hat_rows[label_names[0]] = y_hat + y_trues.append(y_rows) + y_preds.append(y_hat_rows) + + plt.figure() + pd.concat(y_trues)[label_names[0]].plot(label="y_true") + pd.concat(y_preds)[label_names[0]].plot(label="y_pred") + plt.legend() + t_ahead = pd.Timedelta("30T") * model.hparams.target_length + plt.title(f"predicting {t_ahead} ahead") + # plt.show() + + +def plot_from_loader_to_tensor(*args, **kwargs): + plot_from_loader(*args, **kwargs) + + # Send fig to tensorboard + buf = io.BytesIO() + plt.savefig(buf, format="jpeg") + buf.seek(0) + image = PIL.Image.open(buf) + image = ToTensor()(image) # .unsqueeze(0) + return image