From be268d21bf87ebe9c658655f8d89c80cfa1b9e91 Mon Sep 17 00:00:00 2001 From: wassname Date: Mon, 27 Jan 2020 09:08:09 +0800 Subject: [PATCH] ptmultihead, random_search --- run_lightning_anp.py | 27 +- run_lightning_hparams.py | 48 +++ smartmeters-lightning.ipynb | 603 +++++++++++++++++++----------------- src/models/attention.py | 120 +++++++ src/models/lightning_anp.py | 50 +-- src/models/model.py | 4 + src/models/modules.py | 35 ++- 7 files changed, 561 insertions(+), 326 deletions(-) create mode 100644 run_lightning_hparams.py create mode 100644 src/models/attention.py diff --git a/run_lightning_anp.py b/run_lightning_anp.py index bcf4455..5dd25d8 100644 --- a/run_lightning_anp.py +++ b/run_lightning_anp.py @@ -20,16 +20,11 @@ def add_default_args(parser): help='evaluate model on validation set', default=False) return parser -if __name__ == "__main__": - parser = HyperOptArgumentParser(add_help=False) - - parser = add_default_args(parser) - - # give the module a chance to add own params - parser = LatentModelPL.add_model_specific_args(parser) - +def parse_args(parser, argv=None): # parse params - hyperparams = parser.parse_args() + # Set our params here, in a way compatible with cli + argv = argv.replace('\n','').strip().split(' ') + hyperparams = parser.parse_args(argv) import copy hparams = copy.deepcopy(hyperparams) @@ -39,7 +34,19 @@ if __name__ == "__main__": v = getattr(hparams, k) if not isinstance(v, (int, float, str, bool, torch.Tensor)): delattr(hparams, k) - + + + return hyperparams, hparams + +if __name__ == "__main__": + parser = HyperOptArgumentParser(add_help=False) + + parser = add_default_args(parser) + + # give the module a chance to add own params + parser = LatentModelPL.add_model_specific_args(parser) + + hyperparams, hparams = parse_args(parser) print(hparams) model = LatentModelPL(hparams) diff --git a/run_lightning_hparams.py b/run_lightning_hparams.py new file mode 100644 index 0000000..111defb --- /dev/null +++ b/run_lightning_hparams.py @@ -0,0 +1,48 @@ +import pytorch_lightning as pl +from pytorch_lightning import Trainer +import torch +from argparse import ArgumentParser +from test_tube import Experiment, HyperOptArgumentParser +from src.models.lightning_anp import LatentModelPL +from run_lightning_anp import add_default_args + + + +def main(hparams): + if hparams.seed is not None: + random.seed(hparams.seed) + torch.manual_seed(hparams.seed) + torch.backends.cudnn.deterministic = True + + print(hparams) + # build model + model = LatentModelPL(hparams) + # configure trainer + trainer = Trainer( +# default_save_path=hparams.save_path, + max_epochs=hparams.max_nb_epochs, + gpus=hparams.gpus, + gradient_clip_val=hparams.grad_clip, + # track_grad_norm=1, + ) + # train model + if hparams.evaluate: + trainer.run_evaluation() + else: + trainer.fit(model) + +if __name__ == "__main__": + parser = HyperOptArgumentParser(add_help=False) + + parser = add_default_args(parser) + + # give the module a chance to add own params + parser = LatentModelPL.add_model_specific_args(parser) + + # parse params + hyperparams = parser.parse_args() + + # Run some trials on a single cpu. You can view them in tensorboard, and see the hyper params + trials = hyperparams.generate_trials(3) + for trial in trials: + main(trial) diff --git a/smartmeters-lightning.ipynb b/smartmeters-lightning.ipynb index 8864a01..6666190 100644 --- a/smartmeters-lightning.ipynb +++ b/smartmeters-lightning.ipynb @@ -12,8 +12,8 @@ "execution_count": 1, "metadata": { "ExecuteTime": { - "end_time": "2020-01-26T02:22:22.309517Z", - "start_time": "2020-01-26T02:22:20.133629Z" + "end_time": "2020-01-27T00:47:28.989470Z", + "start_time": "2020-01-27T00:47:27.436007Z" } }, "outputs": [], @@ -38,8 +38,8 @@ "execution_count": 2, "metadata": { "ExecuteTime": { - "end_time": "2020-01-26T02:22:22.347319Z", - "start_time": "2020-01-26T02:22:22.313335Z" + "end_time": "2020-01-27T00:47:29.023293Z", + "start_time": "2020-01-27T00:47:28.992339Z" } }, "outputs": [], @@ -54,8 +54,8 @@ "execution_count": 3, "metadata": { "ExecuteTime": { - "end_time": "2020-01-26T02:22:22.387637Z", - "start_time": "2020-01-26T02:22:22.352952Z" + "end_time": "2020-01-27T00:47:29.056361Z", + "start_time": "2020-01-27T00:47:29.025949Z" } }, "outputs": [], @@ -70,8 +70,8 @@ "execution_count": 4, "metadata": { "ExecuteTime": { - "end_time": "2020-01-26T02:22:22.530943Z", - "start_time": "2020-01-26T02:22:22.390294Z" + "end_time": "2020-01-27T00:47:29.173980Z", + "start_time": "2020-01-27T00:47:29.060560Z" } }, "outputs": [], @@ -87,8 +87,8 @@ "execution_count": 5, "metadata": { "ExecuteTime": { - "end_time": "2020-01-26T02:22:22.567732Z", - "start_time": "2020-01-26T02:22:22.533322Z" + "end_time": "2020-01-27T00:47:29.215450Z", + "start_time": "2020-01-27T00:47:29.176561Z" } }, "outputs": [], @@ -110,8 +110,8 @@ "execution_count": 6, "metadata": { "ExecuteTime": { - "end_time": "2020-01-26T02:22:31.555013Z", - "start_time": "2020-01-26T02:22:22.570186Z" + "end_time": "2020-01-27T00:47:38.318681Z", + "start_time": "2020-01-27T00:47:29.217953Z" } }, "outputs": [], @@ -124,15 +124,15 @@ "execution_count": 7, "metadata": { "ExecuteTime": { - "end_time": "2020-01-26T02:22:32.181569Z", - "start_time": "2020-01-26T02:22:31.557102Z" + "end_time": "2020-01-27T00:47:38.905765Z", + "start_time": "2020-01-27T00:47:38.321536Z" } }, "outputs": [ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 7, @@ -203,25 +203,39 @@ "outputs": [], "source": [] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-01-26T03:28:36.003974Z", + "start_time": "2020-01-26T03:28:35.969452Z" + } + }, + "outputs": [], + "source": [] + }, { "cell_type": "code", "execution_count": 8, "metadata": { "ExecuteTime": { - "end_time": "2020-01-26T02:22:32.231362Z", - "start_time": "2020-01-26T02:22:32.185920Z" + "end_time": "2020-01-27T00:47:38.948352Z", + "start_time": "2020-01-27T00:47:38.909346Z" } }, "outputs": [], "source": [ - "# Set our params here, in a way compatible with cli\n", - "argv = f\"\"\"\n", - "--x_dim {df_train.shape[-1]-1} \\\n", - "--y_dim 1 \\\n", - "--max_nb_epochs 40 \\\n", - "--vis_i 670 \\\n", - "--gpus 0 \\\n", - "\"\"\".replace('\\n','').strip().split(' ')" + "from pytorch_lightning import Trainer\n", + "from argparse import ArgumentParser \n", + "from run_lightning_anp import add_default_args\n", + "from test_tube import HyperOptArgumentParser\n", + "\n", + "parser = HyperOptArgumentParser(strategy='random_search', add_help=False)\n", + "parser = add_default_args(parser)\n", + "# give the module a chance to add own params\n", + "parser = LatentModelPL.add_model_specific_args(parser)\n", + "\n" ] }, { @@ -229,8 +243,38 @@ "execution_count": 9, "metadata": { "ExecuteTime": { - "end_time": "2020-01-26T02:22:32.282878Z", - "start_time": "2020-01-26T02:22:32.233850Z" + "end_time": "2020-01-27T00:47:38.988856Z", + "start_time": "2020-01-27T00:47:38.951026Z" + } + }, + "outputs": [], + "source": [ + "def parse_args(argv):\n", + " # parse params\n", + " # Set our params here, in a way compatible with cli\n", + " argv = argv.replace('\\n','').strip().split(' ')\n", + " hyperparams = parser.parse_args(argv)\n", + "\n", + " import copy\n", + " hparams = copy.deepcopy(hyperparams)\n", + " for k in dir(hparams):\n", + " if k.startswith('_'):\n", + " continue\n", + " v = getattr(hparams, k)\n", + " if not isinstance(v, (int, float, str, bool, torch.Tensor)):\n", + " delattr(hparams, k)\n", + "\n", + "\n", + " return hyperparams, hparams" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "ExecuteTime": { + "end_time": "2020-01-27T00:47:39.032966Z", + "start_time": "2020-01-27T00:47:38.991941Z" } }, "outputs": [ @@ -240,8 +284,11 @@ "text": [ "----------------------------------------------------------------------------------------------------\n", "Hyperparameters:\n", - "gpus : 0\n", + "gpus : -1\n", "nodes : 1\n", + "hpc_exp_number : 1\n", + "seed : 42\n", + "evaluate : False\n", "learning_rate : 0.0001\n", "batch_size : 16\n", "x_dim : 16\n", @@ -250,87 +297,36 @@ "hidden_dim : 128\n", "latent_dim : 128\n", "num_heads : 8\n", + "attention_layers : 1\n", "n_latent_encoder_layers: 4\n", "n_det_encoder_layers: 4\n", "n_decoder_layers : 2\n", "dropout : 0\n", "attention_dropout : 0\n", - "min_std : 0.01\n", + "min_std : 0.005\n", "latent_enc_self_attn_type: multihead\n", "det_enc_self_attn_type: multihead\n", "det_enc_cross_attn_type: multihead\n", "use_lvar : False\n", "use_deterministic_path: True\n", - "grad_clip : 0\n", + "grad_clip : 40.0\n", "num_context : 48\n", "num_extra_target : 24\n", - "max_nb_epochs : 40\n", + "max_nb_epochs : 20\n", "num_workers : 4\n", - "hpc_exp_number : None\n", - "trials : , conflict_handler='error', add_help=True)>\n", - "optimize_parallel : , conflict_handler='error', add_help=True)>\n", - "optimize_parallel_gpu: , conflict_handler='error', add_help=True)>\n", - "optimize_parallel_cpu: , conflict_handler='error', add_help=True)>\n", - "generate_trials : , conflict_handler='error', add_help=True)>\n", - "optimize_trials_parallel_gpu: , conflict_handler='error', add_help=True)>\n", "\n" ] } ], "source": [ - "from pytorch_lightning import Trainer\n", - "from argparse import ArgumentParser \n", - "\n", - "parser = ArgumentParser(add_help=False)\n", - "parser.add_argument('--gpus', type=str, default=None)\n", - "parser.add_argument('--nodes', type=int, default=1)\n", - "\n", - "# give the module a chance to add own params\n", - "parser = LatentModelPL.add_model_specific_args(parser)\n", - "\n", - "# parse params\n", - "hparams = parser.parse_args(argv)\n", + "hyperparams, hparams = parse_args(f\"\"\"\n", + "--x_dim {df_train.shape[-1]-1} \\\n", + "--y_dim 1 \\\n", + "--grad_clip 40 \\\n", + "\"\"\")\n", "print(hparams)" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Run" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2020-01-26T02:48:24.261610Z", - "start_time": "2020-01-26T02:48:19.200Z" - }, - "scrolled": true - }, - "outputs": [], - "source": [ - "# import os\n", - "# from test_tube import Experiment\n", - "\n", - "# model = LatentModelPL(hparams)\n", - "\n", - "\n", - "# # most basic trainer, uses good defaults\n", - "# trainer = Trainer(\n", - "# max_epochs=hparams.max_nb_epochs,\n", - "# gpus=hparams.gpus,\n", - "# nb_gpu_nodes=hparams.nodes,\n", - "# gradient_clip_val=hparams.grad_clip,\n", - "# track_grad_norm=1,\n", - "# show_progress_bar=True\n", - "# )\n", - "# # model.trainer = trainer\n", - "# trainer.fit(model)" - ] - }, { "cell_type": "code", "execution_count": null, @@ -345,7 +341,7 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2020-01-26T02:43:08.911937Z", @@ -364,11 +360,11 @@ }, { "cell_type": "code", - "execution_count": 90, + "execution_count": 11, "metadata": { "ExecuteTime": { - "end_time": "2020-01-26T02:55:36.164163Z", - "start_time": "2020-01-26T02:55:36.097064Z" + "end_time": "2020-01-27T00:47:39.090630Z", + "start_time": "2020-01-27T00:47:39.035815Z" }, "scrolled": true }, @@ -394,7 +390,7 @@ " max_epochs=hparams.max_nb_epochs,\n", " gpus=hparams.gpus,\n", " gradient_clip_val=hparams.grad_clip,\n", - " track_grad_norm=1,\n", + "# track_grad_norm=1,\n", " )\n", " # train model\n", " if hparams.evaluate:\n", @@ -406,156 +402,116 @@ "# root_dir = os.path.split(os.path.dirname(sys.modules['__main__'].__file__))[0]\n", "root_dir = os.path.split(os.getcwd())[0]\n", "\n", - "parent = HyperOptArgumentParser(strategy='random_search', add_help=False)\n", + "parser = HyperOptArgumentParser(strategy='random_search', add_help=False)\n", "\n", - "parent = add_default_args(parent)\n", + "parser = add_default_args(parser)\n", "\n", "# allow model to overwrite or extend args\n", - "parser = LatentModelPL.add_model_specific_args(parent)\n", - "\n" + "parser = LatentModelPL.add_model_specific_args(parser)\n", + "\n", + "hyperparams = parser.parse_args([])" ] }, { "cell_type": "code", - "execution_count": 103, + "execution_count": 12, "metadata": { "ExecuteTime": { - "end_time": "2020-01-26T03:03:41.752611Z", - "start_time": "2020-01-26T03:03:41.698482Z" - } - }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'gpus': -1,\n", - " 'nodes': 1,\n", - " 'hpc_exp_number': 1,\n", - " 'seed': 42,\n", - " 'evaluate': False,\n", - " 'learning_rate': 0.0001,\n", - " 'batch_size': 16,\n", - " 'x_dim': 16,\n", - " 'y_dim': 1,\n", - " 'vis_i': 670,\n", - " 'hidden_dim': 128,\n", - " 'latent_dim': 128,\n", - " 'num_heads': 8,\n", - " 'n_latent_encoder_layers': 4,\n", - " 'n_det_encoder_layers': 4,\n", - " 'n_decoder_layers': 2,\n", - " 'dropout': 0,\n", - " 'attention_dropout': 0,\n", - " 'min_std': 0.01,\n", - " 'latent_enc_self_attn_type': 'multihead',\n", - " 'det_enc_self_attn_type': 'multihead',\n", - " 'det_enc_cross_attn_type': 'multihead',\n", - " 'use_lvar': False,\n", - " 'use_deterministic_path': True,\n", - " 'grad_clip': 0,\n", - " 'num_context': 48,\n", - " 'num_extra_target': 24,\n", - " 'max_nb_epochs': 20,\n", - " 'num_workers': 4,\n", - " 'trials': , conflict_handler='error', add_help=True)>,\n", - " 'optimize_parallel': , conflict_handler='error', add_help=True)>,\n", - " 'optimize_parallel_gpu': , conflict_handler='error', add_help=True)>,\n", - " 'optimize_parallel_cpu': , conflict_handler='error', add_help=True)>,\n", - " 'generate_trials': , conflict_handler='error', add_help=True)>,\n", - " 'optimize_trials_parallel_gpu': , conflict_handler='error', add_help=True)>}" - ] - }, - "execution_count": 103, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "hyperparams.__dict__" - ] - }, - { - "cell_type": "code", - "execution_count": 91, - "metadata": { - "ExecuteTime": { - "end_time": "2020-01-26T02:55:36.508582Z", - "start_time": "2020-01-26T02:55:36.470898Z" + "end_time": "2020-01-27T00:47:39.134945Z", + "start_time": "2020-01-27T00:47:39.093368Z" } }, "outputs": [], "source": [ - "hyperparams = parser.parse_args([])\n", - "\n", - "# train model\n", - "# main(hyperparams)" + "# # Run some trials on a single cpu. You can view them in tensorboard, and see the hyper params\n", + "# trials = hyperparams.generate_trials(16)\n", + "# for trial in tqdm(trials, desc='trials'):\n", + "# try:\n", + "# main(trial)\n", + "# except KeyboardInterrupt:\n", + "# break\n", + "# except:\n", + "# logging.exception(\"failed to run\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Run" ] }, { "cell_type": "code", - "execution_count": 109, + "execution_count": 13, "metadata": { "ExecuteTime": { - "end_time": "2020-01-26T03:08:52.581129Z", - "start_time": "2020-01-26T03:08:52.513317Z" + "end_time": "2020-01-27T00:47:39.174912Z", + "start_time": "2020-01-27T00:47:39.137458Z" } }, "outputs": [ { - "ename": "TypeError", - "evalue": "__class__ must be set to a class, not 'NoneType' object", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfloat\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbool\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0msetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 7\u001b[0m \u001b[0mhparams\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mTypeError\u001b[0m: __class__ must be set to a class, not 'NoneType' object" + "name": "stdout", + "output_type": "stream", + "text": [ + "----------------------------------------------------------------------------------------------------\n", + "Hyperparameters:\n", + "gpus : -1\n", + "nodes : 1\n", + "hpc_exp_number : 1\n", + "seed : 42\n", + "evaluate : False\n", + "learning_rate : 0.0001\n", + "batch_size : 16\n", + "x_dim : 16\n", + "y_dim : 1\n", + "vis_i : 670\n", + "hidden_dim : 128\n", + "latent_dim : 128\n", + "num_heads : 8\n", + "attention_layers : 2\n", + "n_latent_encoder_layers: 4\n", + "n_det_encoder_layers: 4\n", + "n_decoder_layers : 2\n", + "dropout : 0\n", + "attention_dropout : 0\n", + "min_std : 0.005\n", + "latent_enc_self_attn_type: ptmultihead\n", + "det_enc_self_attn_type: ptmultihead\n", + "det_enc_cross_attn_type: ptmultihead\n", + "use_lvar : False\n", + "use_deterministic_path: True\n", + "grad_clip : 40.0\n", + "num_context : 48\n", + "num_extra_target : 24\n", + "max_nb_epochs : 20\n", + "num_workers : 4\n", + "\n" ] } ], "source": [ - "import copy\n", - "hparams = copy.deepcopy(hyperparams)\n", - "for k in dir(hparams):\n", - " v = getattr(hparams, k)\n", - " if not isinstance(v, (int, float, str, bool, torch.Tensor)):\n", - " setattr(hparams, k, None)\n", - "hparams" + "hyperparams, hparams = parse_args(f\"\"\"\n", + "--x_dim {df_train.shape[-1]-1} \\\n", + "--y_dim 1 \\\n", + "--grad_clip 40 \\\n", + "--latent_enc_self_attn_type ptmultihead \\\n", + "--det_enc_self_attn_type ptmultihead \\\n", + "--det_enc_cross_attn_type ptmultihead \\\n", + "--attention_layers 2 \\\n", + "\"\"\")\n", + "print(hparams)" ] }, { "cell_type": "code", - "execution_count": 94, + "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2020-01-26T02:56:37.274166Z", - "start_time": "2020-01-26T02:56:37.230029Z" - } - }, - "outputs": [ - { - "data": { - "text/plain": [ - "24" - ] - }, - "execution_count": 94, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "72-48" - ] - }, - { - "cell_type": "code", - "execution_count": 96, - "metadata": { - "ExecuteTime": { - "end_time": "2020-01-26T02:57:32.050149Z", - "start_time": "2020-01-26T02:57:22.400348Z" - } + "start_time": "2020-01-27T00:47:27.500Z" + }, + "scrolled": true }, "outputs": [ { @@ -563,72 +519,28 @@ "output_type": "stream", "text": [ "INFO:root:gpu available: True, used: True\n", - "INFO:root:VISIBLE GPUS: 0\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "----------------------------------------------------------------------------------------------------\n", - "Hyperparameters:\n", - "learning_rate : 0.0037216107638185568\n", - "hidden_dim : 128\n", - "latent_dim : 512\n", - "n_latent_encoder_layers: 8\n", - "n_det_encoder_layers: 1\n", - "n_decoder_layers : 8\n", - "dropout : 0.3553951704743503\n", - "attention_dropout : 0.10436943022123696\n", - "latent_enc_self_attn_type: laplace\n", - "det_enc_self_attn_type: multihead\n", - "det_enc_cross_attn_type: laplace\n", - "use_lvar : True\n", - "use_deterministic_path: True\n", - "gpus : -1\n", - "nodes : 1\n", - "hpc_exp_number : 1\n", - "seed : 42\n", - "evaluate : False\n", - "batch_size : 16\n", - "x_dim : 16\n", - "y_dim : 1\n", - "vis_i : 670\n", - "num_heads : 8\n", - "min_std : 0.01\n", - "grad_clip : 0\n", - "num_context : 48\n", - "num_extra_target : 24\n", - "max_nb_epochs : 20\n", - "num_workers : 4\n", - "\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ + "INFO:root:VISIBLE GPUS: 0\n", "INFO:root:\n", " Name Type Params\n", - "0 model LatentModel 5 M\n", - "1 model._latent_encoder LatentEncoder 285 K\n", + "0 model LatentModel 987 K\n", + "1 model._latent_encoder LatentEncoder 284 K\n", "2 model._latent_encoder._input_layer NPBlockRelu2d 2 K\n", "3 model._latent_encoder._input_layer.linear Linear 2 K\n", "4 model._latent_encoder._input_layer.act ReLU 0 \n", ".. ... ... ...\n", - "118 model._decoder._decoder.7.act ReLU 0 \n", - "119 model._decoder._decoder.7.dropout Dropout2d 0 \n", - "120 model._decoder._decoder.7.norm BatchNorm2d 1 K\n", - "121 model._decoder._mean Linear 769 \n", - "122 model._decoder._std Linear 769 \n", + "162 model._decoder._decoder.1.act ReLU 0 \n", + "163 model._decoder._decoder.1.dropout Dropout2d 0 \n", + "164 model._decoder._decoder.1.norm BatchNorm2d 768 \n", + "165 model._decoder._mean Linear 385 \n", + "166 model._decoder._std Linear 385 \n", "\n", - "[123 rows x 3 columns]\n" + "[167 rows x 3 columns]\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "591cb5ee0f3f4feb8d486b3ac06dfe6a", + "model_id": "", "version_major": 2, "version_minor": 0 }, @@ -640,34 +552,145 @@ "output_type": "display_data" }, { - "ename": "RuntimeError", - "evalue": "invalid argument 0: Sizes of tensors must match except in dimension 2. Got 72 and 48 in dimension 1 at /pytorch/aten/src/THC/generic/THCTensorMath.cu:71", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0mtrials\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhyperparams\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgenerate_trials\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m8\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mtrial\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtrials\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mmain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m\u001b[0m in \u001b[0;36mmain\u001b[0;34m(hparams)\u001b[0m\n\u001b[1;32m 25\u001b[0m \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_evaluation\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 26\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 29\u001b[0m \u001b[0;31m# use default args given by lightning\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, model)\u001b[0m\n\u001b[1;32m 700\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 701\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msingle_gpu\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 702\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msingle_gpu_train\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 703\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 704\u001b[0m \u001b[0;31m# ON CPU\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/trainer/distrib_parts.py\u001b[0m in \u001b[0;36msingle_gpu_train\u001b[0;34m(self, model)\u001b[0m\n\u001b[1;32m 439\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizers\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moptimizers\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 440\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 441\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_pretrain_routine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 442\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 443\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mdp_train\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36mrun_pretrain_routine\u001b[0;34m(self, model)\u001b[0m\n\u001b[1;32m 818\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 819\u001b[0m eval_results = self.evaluate(model, self.get_val_dataloaders(),\n\u001b[0;32m--> 820\u001b[0;31m self.num_sanity_val_steps, False)\n\u001b[0m\u001b[1;32m 821\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback_metrics\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprocess_output\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0meval_results\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 822\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/trainer/evaluation_loop.py\u001b[0m in \u001b[0;36mevaluate\u001b[0;34m(self, model, dataloaders, max_batches, test)\u001b[0m\n\u001b[1;32m 232\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 233\u001b[0m \u001b[0mdataloader_idx\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 234\u001b[0;31m test)\n\u001b[0m\u001b[1;32m 235\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 236\u001b[0m \u001b[0;31m# track outputs for collation\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/trainer/evaluation_loop.py\u001b[0m in \u001b[0;36mevaluation_forward\u001b[0;34m(self, model, batch, batch_idx, dataloader_idx, test)\u001b[0m\n\u001b[1;32m 363\u001b[0m \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtest_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 364\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 365\u001b[0;31m \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalidation_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 366\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 367\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/attentive-neural-processes/src/models/lightning_anp.py\u001b[0m in \u001b[0;36mvalidation_step\u001b[0;34m(self, batch, batch_idx)\u001b[0m\n\u001b[1;32m 33\u001b[0m \u001b[0;32massert\u001b[0m \u001b[0mall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misfinite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0md\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0md\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 34\u001b[0m \u001b[0mcontext_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontext_y\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_y\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 35\u001b[0;31m \u001b[0my_pred\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_std\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcontext_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontext_y\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_y\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 36\u001b[0m tensorboard_logs = {\n\u001b[1;32m 37\u001b[0m \u001b[0;34m\"val/loss\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/attentive-neural-processes/src/models/lightning_anp.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, context_x, context_y, target_x, target_y)\u001b[0m\n\u001b[1;32m 17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 18\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontext_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontext_y\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_y\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcontext_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontext_y\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_y\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 20\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 21\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mtraining_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 539\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 540\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 541\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 542\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 543\u001b[0m \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/attentive-neural-processes/src/models/model.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, context_x, context_y, target_x, target_y)\u001b[0m\n\u001b[1;32m 121\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 122\u001b[0m \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 123\u001b[0;31m \u001b[0mdist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlog_sigma\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_decoder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mz\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_x\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 124\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 125\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mtarget_y\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 539\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 540\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 541\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 542\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 543\u001b[0m \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/media/wassname/Storage5/projects2/3ST/attentive-neural-processes/src/models/modules.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, r, z, target_x)\u001b[0m\n\u001b[1;32m 255\u001b[0m \u001b[0;31m# concatenate target_x and representation\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 256\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muse_deterministic_path\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 257\u001b[0;31m \u001b[0mz\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mz\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 258\u001b[0m \u001b[0mrepresentation\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mz\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 259\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mRuntimeError\u001b[0m: invalid argument 0: Sizes of tensors must match except in dimension 2. Got 72 and 48 in dimension 1 at /pytorch/aten/src/THC/generic/THCTensorMath.cu:71" + "name": "stdout", + "output_type": "stream", + "text": [ + "step 0, {'val_loss': tensor(0.7484, device='cuda:0'), 'val/kl': tensor(0.0002, device='cuda:0'), 'val/std': tensor(0.7129, device='cuda:0'), 'val/mse': tensor(0.1703, device='cuda:0')}\n", + "\r" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "f0c6de3ff0a849878a30c8e19e04a552", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=241.0, style=Pr…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step 2201, {'val_loss': tensor(-0.6897, device='cuda:0'), 'val/kl': tensor(0.0036, device='cuda:0'), 'val/std': tensor(0.2597, device='cuda:0'), 'val/mse': tensor(0.0670, device='cuda:0')}\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=241.0, style=Pr…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step 4403, {'val_loss': tensor(-0.8933, device='cuda:0'), 'val/kl': tensor(0.0012, device='cuda:0'), 'val/std': tensor(0.1486, device='cuda:0'), 'val/mse': tensor(0.0167, device='cuda:0')}\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=241.0, style=Pr…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step 6605, {'val_loss': tensor(-1.2594, device='cuda:0'), 'val/kl': tensor(0.0007, device='cuda:0'), 'val/std': tensor(0.0727, device='cuda:0'), 'val/mse': tensor(0.0054, device='cuda:0')}\n" ] } ], "source": [ - "# Run some trials on a single cpu. You can view them in tensorboard, and see the hyper params\n", - "trials = hyperparams.generate_trials(8)\n", - "for trial in trials:\n", - " main(trial)" + "import os\n", + "from test_tube import Experiment\n", + "\n", + "model = LatentModelPL(hparams)\n", + "\n", + "# most basic trainer, uses good defaults\n", + "trainer = Trainer(\n", + " max_epochs=hparams.max_nb_epochs,\n", + " gpus=hparams.gpus,\n", + " nb_gpu_nodes=hparams.nodes,\n", + " gradient_clip_val=hparams.grad_clip,\n", + ")\n", + "trainer.fit(model)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "start_time": "2020-01-27T00:47:27.600Z" + } + }, + "outputs": [], + "source": [ + "%debug" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "start_time": "2020-01-27T00:47:27.600Z" + } + }, + "outputs": [], + "source": [ + "model.model._latent_encoder.self_attention" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "start_time": "2020-01-27T00:47:27.600Z" + } + }, + "outputs": [], + "source": [ + "import torch\n", + "torch.nn.MultiheadAttention?" ] }, { @@ -675,9 +698,7 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [ - "%debug" - ] + "source": [] } ], "metadata": { diff --git a/src/models/attention.py b/src/models/attention.py new file mode 100644 index 0000000..5af54e9 --- /dev/null +++ b/src/models/attention.py @@ -0,0 +1,120 @@ +import torch as t +import torch.nn as nn +import torch.nn.functional as F +import torch +import math + +class Linear(nn.Module): + """ + Linear Module + """ + def __init__(self, in_dim, out_dim, bias=True, w_init='linear'): + """ + :param in_dim: dimension of input + :param out_dim: dimension of output + :param bias: boolean. if True, bias is included. + :param w_init: str. weight inits with xavier initialization. + """ + super(Linear, self).__init__() + self.linear_layer = nn.Linear(in_dim, out_dim, bias=bias) + + nn.init.xavier_uniform_( + self.linear_layer.weight, + gain=nn.init.calculate_gain(w_init)) + + def forward(self, x): + return self.linear_layer(x) + +class MultiheadAttention(nn.Module): + """ + Multihead attention mechanism (dot attention) + """ + def __init__(self, num_hidden_k): + """ + :param num_hidden_k: dimension of hidden + """ + super(MultiheadAttention, self).__init__() + + self.num_hidden_k = num_hidden_k + self.attn_dropout = nn.Dropout(p=0.1) + + def forward(self, key, value, query): + # Get attention score + attn = t.bmm(query, key.transpose(1, 2)) + attn = attn / math.sqrt(self.num_hidden_k) + + attn = t.softmax(attn, dim=-1) + + # Dropout + attn = self.attn_dropout(attn) + + # Get Context Vector + result = t.bmm(attn, value) + + return result, attn + + +class Attention(nn.Module): + """ + Attention Network + """ + def __init__(self, num_hidden, h=4): + """ + :param num_hidden: dimension of hidden + :param h: num of heads + """ + super(Attention, self).__init__() + + self.num_hidden = num_hidden + self.num_hidden_per_attn = num_hidden // h + self.h = h + + self.key = Linear(num_hidden, num_hidden, bias=False) + self.value = Linear(num_hidden, num_hidden, bias=False) + self.query = Linear(num_hidden, num_hidden, bias=False) + + self.multihead = MultiheadAttention(self.num_hidden_per_attn) + + self.residual_dropout = nn.Dropout(p=0.1) + + self.final_linear = Linear(num_hidden * 2, num_hidden) + + self.layer_norm = nn.LayerNorm(num_hidden) + + def forward(self, key, value, query): + + batch_size = key.size(0) + seq_k = key.size(1) + seq_q = query.size(1) + residual = query + + # Make multihead + key = self.key(key).view(batch_size, seq_k, self.h, self.num_hidden_per_attn) + value = self.value(value).view(batch_size, seq_k, self.h, self.num_hidden_per_attn) + query = self.query(query).view(batch_size, seq_q, self.h, self.num_hidden_per_attn) + + key = key.permute(2, 0, 1, 3).contiguous().view(-1, seq_k, self.num_hidden_per_attn) + value = value.permute(2, 0, 1, 3).contiguous().view(-1, seq_k, self.num_hidden_per_attn) + query = query.permute(2, 0, 1, 3).contiguous().view(-1, seq_q, self.num_hidden_per_attn) + + # Get context vector + result, attns = self.multihead(key, value, query) + + # Concatenate all multihead context vector + result = result.view(self.h, batch_size, seq_q, self.num_hidden_per_attn) + result = result.permute(1, 2, 0, 3).contiguous().view(batch_size, seq_q, -1) + + # Concatenate context vector with input (most important) + result = t.cat([residual, result], dim=-1) + + # Final linear + result = self.final_linear(result) + + # Residual dropout & connection + result = self.residual_dropout(result) + result = result + residual + + # Layer normalization + result = self.layer_norm(result) + + return result, attns diff --git a/src/models/lightning_anp.py b/src/models/lightning_anp.py index f067177..3e9a8fb 100644 --- a/src/models/lightning_anp.py +++ b/src/models/lightning_anp.py @@ -1,5 +1,6 @@ import pytorch_lightning as pl import torch +import torch.nn.functional as F from argparse import ArgumentParser from test_tube import Experiment, HyperOptArgumentParser from src.models.model import LatentModel @@ -23,9 +24,10 @@ class LatentModelPL(pl.LightningModule): context_x, context_y, target_x, target_y = batch y_pred, kl, loss, y_std = self.forward(context_x, context_y, target_x, target_y) tensorboard_logs = { - "train_loss": loss, - "train_kl": kl.mean(), + "train/loss": loss, + "train/kl": kl.mean(), "train/std": y_std.mean(), + "train/mse": F.mse_loss(y_pred, target_y).mean(), } return {"loss": loss, "log": tensorboard_logs} @@ -33,10 +35,12 @@ class LatentModelPL(pl.LightningModule): assert all(torch.isfinite(d).all() for d in batch) context_x, context_y, target_x, target_y = batch y_pred, kl, loss, y_std = self.forward(context_x, context_y, target_x, target_y) + tensorboard_logs = { - "val/loss": loss, + "val_loss": loss, "val/kl": kl.mean(), - "val/std": y_std.mean() + "val/std": y_std.mean(), + "val/mse": F.mse_loss(y_pred, target_y).mean(), } return {"val_loss": loss, "log": tensorboard_logs} @@ -47,14 +51,23 @@ class LatentModelPL(pl.LightningModule): image = plot_from_loader_to_tensor(loader, self.model, i=self.hparams.vis_i) self.logger.experiment.add_image('val/image', image, self.trainer.global_step) + keys = outputs[0]["log"].keys() + # tensorboard_logs = {} + # for k in keys: + # tensorboard_logs[k] = torch.stack([x["log"][k] for x in outputs if k in x["log"]]).mean() + tensorboard_logs = {k: torch.stack([x["log"][k] for x in outputs if k in x["log"]]).mean() for k in keys} + avg_loss = torch.stack([x["val_loss"] for x in outputs]).mean() - tensorboard_logs = {"val_loss": avg_loss} - print(self.trainer.global_step, tensorboard_logs) + print(f"step {self.trainer.global_step}, {tensorboard_logs}") + + # Log hparams with metric, doesn't work + # self.logger.experiment.add_hparams(self.hparams.__dict__, {"avg_val_loss": avg_loss}) + return {"avg_val_loss": avg_loss, "log": tensorboard_logs} def configure_optimizers(self): optim = torch.optim.Adam(self.parameters(), lr=self.hparams.learning_rate) - scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optim) + scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optim, patience=2, verbose=True, min_lr=1e-5) # note early stopping has patient 3 return [optim], [scheduler] def _get_cache_dfs(self): @@ -123,22 +136,23 @@ class LatentModelPL(pl.LightningModule): parser.add_argument("--y_dim", default=1, type=int) parser.add_argument("--vis_i", default=670, type=int) - parser.opt_list("--hidden_dim", default=128, type=int, tunable=True, options=[8*2**i for i in range(7)]) - parser.opt_list("--latent_dim", default=256, type=int, tunable=True, options=[8*2**i for i in range(7)]) + parser.opt_list("--hidden_dim", default=128, type=int, tunable=True, options=[8*2**i for i in range(8)]) + parser.opt_list("--latent_dim", default=128, type=int, tunable=True, options=[8*2**i for i in range(8)]) parser.add_argument("--num_heads", default=8, type=int) - parser.opt_list("--n_latent_encoder_layers", default=4, type=int, tunable=True, options=[1, 2, 4, 8]) - parser.opt_list("--n_det_encoder_layers", default=4, type=int, tunable=True, options=[1, 2, 4, 8]) - parser.opt_list("--n_decoder_layers", default=2, type=int, tunable=True, options=[1, 2, 4, 8]) + parser.add_argument("--attention_layers", default=1, type=int) + parser.opt_list("--n_latent_encoder_layers", default=4, type=int, tunable=True, options=[1, 2, 4, 8, 16]) + parser.opt_list("--n_det_encoder_layers", default=4, type=int, tunable=True, options=[1, 2, 4, 8, 16]) + parser.opt_list("--n_decoder_layers", default=2, type=int, tunable=True, options=[1, 2, 4, 8, 16]) - parser.opt_range("--dropout", default=0, type=float, tunable=True, low=0, high=0.5) - parser.opt_range("--attention_dropout", default=0, type=float, tunable=True, low=0, high=0.5) - parser.add_argument("--min_std", default=0.01, type=float) + parser.opt_range("--dropout", default=0, type=float, tunable=True, low=0, high=0.75) + parser.opt_range("--attention_dropout", default=0, type=float, tunable=True, low=0, high=0.75) + parser.add_argument("--min_std", default=0.005, type=float) parser.opt_list( - "--latent_enc_self_attn_type", default="multihead", type=str, tunable=True, options=['uniform', 'dot', 'multihead', 'laplace'] + "--latent_enc_self_attn_type", default="multihead", type=str, tunable=True, options=['uniform', 'dot', 'multihead', 'ptmultihead'] ) - parser.opt_list("--det_enc_self_attn_type", default="multihead", type=str, tunable=True, options=['uniform', 'dot', 'multihead', 'laplace']) - parser.opt_list("--det_enc_cross_attn_type", default="multihead", type=str, tunable=True, options=['uniform', 'dot', 'multihead', 'laplace']) + parser.opt_list("--det_enc_self_attn_type", default="multihead", type=str, tunable=True, options=['uniform', 'dot', 'multihead', 'ptmultihead']) + parser.opt_list("--det_enc_cross_attn_type", default="multihead", type=str, tunable=True, options=['uniform', 'dot', 'multihead', 'ptmultihead']) parser.opt_list("--use_lvar", default=False, type=bool, tunable=True, options=[False, True]) parser.opt_list("--use_deterministic_path", default=True, tunable=True, type=bool, options=[False, True]) diff --git a/src/models/model.py b/src/models/model.py index 63a46a6..ff2a126 100644 --- a/src/models/model.py +++ b/src/models/model.py @@ -53,6 +53,7 @@ class LatentModel(nn.Module): min_std=0.1, use_lvar=False, use_deterministic_path=True, + attention_layers=2, **kwargs ): @@ -71,6 +72,7 @@ class LatentModel(nn.Module): n_heads=num_heads, min_std=min_std, use_lvar=use_lvar, + attention_layers=attention_layers, ) self._deterministic_encoder = DeterministicEncoder( @@ -83,6 +85,7 @@ class LatentModel(nn.Module): dropout=dropout, attention_dropout=attention_dropout, n_heads=num_heads, + attention_layers=attention_layers, ) self._decoder = Decoder( @@ -141,6 +144,7 @@ class LatentModel(nn.Module): log_p = None kl_loss = None loss = None + mse_loss = None y_pred = dist.rsample() if self.training else dist.loc return y_pred, kl_loss, loss, dist.scale diff --git a/src/models/modules.py b/src/models/modules.py index 1d6f556..db3bded 100644 --- a/src/models/modules.py +++ b/src/models/modules.py @@ -2,6 +2,7 @@ import torch from torch import nn import torch.nn.functional as F import math +from .attention import Attention as PtAttention class NPBlockRelu2d(nn.Module): """Block for Neural Processes.""" @@ -39,7 +40,7 @@ def block_relu(in_dim, out_dim, dropout=0, inplace=False): class Attention(nn.Module): - def __init__(self, hidden_dim, attention_type, n_heads=8, dropout=0): + def __init__(self, hidden_dim, attention_type, attention_layers=1, n_heads=8, dropout=0): super().__init__() if attention_type == "uniform": self._attention_func = self._uniform_attention @@ -48,11 +49,17 @@ class Attention(nn.Module): elif attention_type == "dot": self._attention_func = self._dot_attention elif attention_type == "multihead": - self._mattn = torch.nn.MultiheadAttention( + self._mattn = nn.ModuleList([torch.nn.MultiheadAttention( hidden_dim, n_heads, bias=False, dropout=dropout - ) + ) for _ in range(attention_layers)]) self._attention_func = self._pytorch_multihead_attention self.n_heads = n_heads + elif attention_type == "ptmultihead": + self._mattn = nn.ModuleList([PtAttention( + hidden_dim, n_heads + ) for _ in range(attention_layers)]) + self._attention_func = self._ptmultihead_fn + self.n_heads = n_heads else: raise NotImplementedError @@ -85,9 +92,21 @@ class Attention(nn.Module): def _pytorch_multihead_attention(self, k, v, q): # Pytorch multiheaded attention takes inputs if diff order and permutation - o = self._mattn(q.permute(1, 0, 2), k.permute(1, 0, 2), v.permute(1, 0, 2))[0] + q = q.permute(1, 0, 2) + k = k.permute(1, 0, 2) + v = v.permute(1, 0, 2) + for attention in self._mattn: + o = attention(q, k, v)[0] + q, k, v = o, o, o return o.permute(1, 0, 2) + def _ptmultihead_fn(self, k, v, q): + for attention in self._mattn: + o = attention(k, v, q)[0] + # print(k.shape, v.shape, q.shape, o.shape) + q, k, v = o, o, o + return o + class LatentEncoder(nn.Module): """ @@ -105,6 +124,7 @@ class LatentEncoder(nn.Module): dropout=0, attention_dropout=0, use_lvar=False, + attention_layers=2, ): super().__init__() self.use_lvar = use_lvar @@ -116,7 +136,7 @@ class LatentEncoder(nn.Module): ] ) self._self_attention = Attention( - hidden_dim, self_attention_type, n_heads=n_heads, dropout=attention_dropout + hidden_dim, self_attention_type, n_heads=n_heads, dropout=attention_dropout, attention_layers=attention_layers ) self._penultimate_layer = block_relu(hidden_dim, hidden_dim, dropout) self._mean = nn.Linear(hidden_dim, latent_dim) @@ -180,6 +200,7 @@ class DeterministicEncoder(nn.Module): dropout=0, attention_dropout=0, n_heads=8, + attention_layers=2, ): super().__init__() self._input_layer = NPBlockRelu2d(input_dim, hidden_dim, dropout) @@ -190,10 +211,10 @@ class DeterministicEncoder(nn.Module): ] ) self._self_attention = Attention( - hidden_dim, self_attention_type, dropout=attention_dropout, n_heads=n_heads + hidden_dim, self_attention_type, dropout=attention_dropout, n_heads=n_heads, attention_layers=attention_layers ) self._cross_attention = Attention( - hidden_dim, cross_attention_type, dropout=attention_dropout, n_heads=n_heads + hidden_dim, cross_attention_type, dropout=attention_dropout, n_heads=n_heads, attention_layers=attention_layers ) self._target_transform = nn.Linear(x_dim, hidden_dim) self._context_transform = nn.Linear(x_dim, hidden_dim)