mirror of
https://github.com/wassname/attentive-neural-processes.git
synced 2026-08-12 11:50:46 +08:00
ptmultihead, random_search
This commit is contained in:
wassname
+17
-10
@@ -20,16 +20,11 @@ def add_default_args(parser):
|
||||
help='evaluate model on validation set', default=False)
|
||||
return parser
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = HyperOptArgumentParser(add_help=False)
|
||||
|
||||
parser = add_default_args(parser)
|
||||
|
||||
# give the module a chance to add own params
|
||||
parser = LatentModelPL.add_model_specific_args(parser)
|
||||
|
||||
def parse_args(parser, argv=None):
|
||||
# parse params
|
||||
hyperparams = parser.parse_args()
|
||||
# Set our params here, in a way compatible with cli
|
||||
argv = argv.replace('\n','').strip().split(' ')
|
||||
hyperparams = parser.parse_args(argv)
|
||||
|
||||
import copy
|
||||
hparams = copy.deepcopy(hyperparams)
|
||||
@@ -39,7 +34,19 @@ if __name__ == "__main__":
|
||||
v = getattr(hparams, k)
|
||||
if not isinstance(v, (int, float, str, bool, torch.Tensor)):
|
||||
delattr(hparams, k)
|
||||
|
||||
|
||||
|
||||
return hyperparams, hparams
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = HyperOptArgumentParser(add_help=False)
|
||||
|
||||
parser = add_default_args(parser)
|
||||
|
||||
# give the module a chance to add own params
|
||||
parser = LatentModelPL.add_model_specific_args(parser)
|
||||
|
||||
hyperparams, hparams = parse_args(parser)
|
||||
print(hparams)
|
||||
|
||||
model = LatentModelPL(hparams)
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
import pytorch_lightning as pl
|
||||
from pytorch_lightning import Trainer
|
||||
import torch
|
||||
from argparse import ArgumentParser
|
||||
from test_tube import Experiment, HyperOptArgumentParser
|
||||
from src.models.lightning_anp import LatentModelPL
|
||||
from run_lightning_anp import add_default_args
|
||||
|
||||
|
||||
|
||||
def main(hparams):
|
||||
if hparams.seed is not None:
|
||||
random.seed(hparams.seed)
|
||||
torch.manual_seed(hparams.seed)
|
||||
torch.backends.cudnn.deterministic = True
|
||||
|
||||
print(hparams)
|
||||
# build model
|
||||
model = LatentModelPL(hparams)
|
||||
# configure trainer
|
||||
trainer = Trainer(
|
||||
# default_save_path=hparams.save_path,
|
||||
max_epochs=hparams.max_nb_epochs,
|
||||
gpus=hparams.gpus,
|
||||
gradient_clip_val=hparams.grad_clip,
|
||||
# track_grad_norm=1,
|
||||
)
|
||||
# train model
|
||||
if hparams.evaluate:
|
||||
trainer.run_evaluation()
|
||||
else:
|
||||
trainer.fit(model)
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = HyperOptArgumentParser(add_help=False)
|
||||
|
||||
parser = add_default_args(parser)
|
||||
|
||||
# give the module a chance to add own params
|
||||
parser = LatentModelPL.add_model_specific_args(parser)
|
||||
|
||||
# parse params
|
||||
hyperparams = parser.parse_args()
|
||||
|
||||
# Run some trials on a single cpu. You can view them in tensorboard, and see the hyper params
|
||||
trials = hyperparams.generate_trials(3)
|
||||
for trial in trials:
|
||||
main(trial)
|
||||
+312
-291
@@ -12,8 +12,8 @@
|
||||
"execution_count": 1,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-01-26T02:22:22.309517Z",
|
||||
"start_time": "2020-01-26T02:22:20.133629Z"
|
||||
"end_time": "2020-01-27T00:47:28.989470Z",
|
||||
"start_time": "2020-01-27T00:47:27.436007Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
@@ -38,8 +38,8 @@
|
||||
"execution_count": 2,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-01-26T02:22:22.347319Z",
|
||||
"start_time": "2020-01-26T02:22:22.313335Z"
|
||||
"end_time": "2020-01-27T00:47:29.023293Z",
|
||||
"start_time": "2020-01-27T00:47:28.992339Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
@@ -54,8 +54,8 @@
|
||||
"execution_count": 3,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-01-26T02:22:22.387637Z",
|
||||
"start_time": "2020-01-26T02:22:22.352952Z"
|
||||
"end_time": "2020-01-27T00:47:29.056361Z",
|
||||
"start_time": "2020-01-27T00:47:29.025949Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
@@ -70,8 +70,8 @@
|
||||
"execution_count": 4,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-01-26T02:22:22.530943Z",
|
||||
"start_time": "2020-01-26T02:22:22.390294Z"
|
||||
"end_time": "2020-01-27T00:47:29.173980Z",
|
||||
"start_time": "2020-01-27T00:47:29.060560Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
@@ -87,8 +87,8 @@
|
||||
"execution_count": 5,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-01-26T02:22:22.567732Z",
|
||||
"start_time": "2020-01-26T02:22:22.533322Z"
|
||||
"end_time": "2020-01-27T00:47:29.215450Z",
|
||||
"start_time": "2020-01-27T00:47:29.176561Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
@@ -110,8 +110,8 @@
|
||||
"execution_count": 6,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-01-26T02:22:31.555013Z",
|
||||
"start_time": "2020-01-26T02:22:22.570186Z"
|
||||
"end_time": "2020-01-27T00:47:38.318681Z",
|
||||
"start_time": "2020-01-27T00:47:29.217953Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
@@ -124,15 +124,15 @@
|
||||
"execution_count": 7,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-01-26T02:22:32.181569Z",
|
||||
"start_time": "2020-01-26T02:22:31.557102Z"
|
||||
"end_time": "2020-01-27T00:47:38.905765Z",
|
||||
"start_time": "2020-01-27T00:47:38.321536Z"
|
||||
}
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"<matplotlib.legend.Legend at 0x7fc892ebfb00>"
|
||||
"<matplotlib.legend.Legend at 0x7fe320db56d8>"
|
||||
]
|
||||
},
|
||||
"execution_count": 7,
|
||||
@@ -203,25 +203,39 @@
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-01-26T03:28:36.003974Z",
|
||||
"start_time": "2020-01-26T03:28:35.969452Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-01-26T02:22:32.231362Z",
|
||||
"start_time": "2020-01-26T02:22:32.185920Z"
|
||||
"end_time": "2020-01-27T00:47:38.948352Z",
|
||||
"start_time": "2020-01-27T00:47:38.909346Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Set our params here, in a way compatible with cli\n",
|
||||
"argv = f\"\"\"\n",
|
||||
"--x_dim {df_train.shape[-1]-1} \\\n",
|
||||
"--y_dim 1 \\\n",
|
||||
"--max_nb_epochs 40 \\\n",
|
||||
"--vis_i 670 \\\n",
|
||||
"--gpus 0 \\\n",
|
||||
"\"\"\".replace('\\n','').strip().split(' ')"
|
||||
"from pytorch_lightning import Trainer\n",
|
||||
"from argparse import ArgumentParser \n",
|
||||
"from run_lightning_anp import add_default_args\n",
|
||||
"from test_tube import HyperOptArgumentParser\n",
|
||||
"\n",
|
||||
"parser = HyperOptArgumentParser(strategy='random_search', add_help=False)\n",
|
||||
"parser = add_default_args(parser)\n",
|
||||
"# give the module a chance to add own params\n",
|
||||
"parser = LatentModelPL.add_model_specific_args(parser)\n",
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -229,8 +243,38 @@
|
||||
"execution_count": 9,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-01-26T02:22:32.282878Z",
|
||||
"start_time": "2020-01-26T02:22:32.233850Z"
|
||||
"end_time": "2020-01-27T00:47:38.988856Z",
|
||||
"start_time": "2020-01-27T00:47:38.951026Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def parse_args(argv):\n",
|
||||
" # parse params\n",
|
||||
" # Set our params here, in a way compatible with cli\n",
|
||||
" argv = argv.replace('\\n','').strip().split(' ')\n",
|
||||
" hyperparams = parser.parse_args(argv)\n",
|
||||
"\n",
|
||||
" import copy\n",
|
||||
" hparams = copy.deepcopy(hyperparams)\n",
|
||||
" for k in dir(hparams):\n",
|
||||
" if k.startswith('_'):\n",
|
||||
" continue\n",
|
||||
" v = getattr(hparams, k)\n",
|
||||
" if not isinstance(v, (int, float, str, bool, torch.Tensor)):\n",
|
||||
" delattr(hparams, k)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
" return hyperparams, hparams"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-01-27T00:47:39.032966Z",
|
||||
"start_time": "2020-01-27T00:47:38.991941Z"
|
||||
}
|
||||
},
|
||||
"outputs": [
|
||||
@@ -240,8 +284,11 @@
|
||||
"text": [
|
||||
"----------------------------------------------------------------------------------------------------\n",
|
||||
"Hyperparameters:\n",
|
||||
"gpus : 0\n",
|
||||
"gpus : -1\n",
|
||||
"nodes : 1\n",
|
||||
"hpc_exp_number : 1\n",
|
||||
"seed : 42\n",
|
||||
"evaluate : False\n",
|
||||
"learning_rate : 0.0001\n",
|
||||
"batch_size : 16\n",
|
||||
"x_dim : 16\n",
|
||||
@@ -250,87 +297,36 @@
|
||||
"hidden_dim : 128\n",
|
||||
"latent_dim : 128\n",
|
||||
"num_heads : 8\n",
|
||||
"attention_layers : 1\n",
|
||||
"n_latent_encoder_layers: 4\n",
|
||||
"n_det_encoder_layers: 4\n",
|
||||
"n_decoder_layers : 2\n",
|
||||
"dropout : 0\n",
|
||||
"attention_dropout : 0\n",
|
||||
"min_std : 0.01\n",
|
||||
"min_std : 0.005\n",
|
||||
"latent_enc_self_attn_type: multihead\n",
|
||||
"det_enc_self_attn_type: multihead\n",
|
||||
"det_enc_cross_attn_type: multihead\n",
|
||||
"use_lvar : False\n",
|
||||
"use_deterministic_path: True\n",
|
||||
"grad_clip : 0\n",
|
||||
"grad_clip : 40.0\n",
|
||||
"num_context : 48\n",
|
||||
"num_extra_target : 24\n",
|
||||
"max_nb_epochs : 40\n",
|
||||
"max_nb_epochs : 20\n",
|
||||
"num_workers : 4\n",
|
||||
"hpc_exp_number : None\n",
|
||||
"trials : <bound method HyperOptArgumentParser.opt_trials of HyperOptArgumentParser(prog='ipykernel_launcher.py', usage=None, description=None, formatter_class=<class 'argparse.HelpFormatter'>, conflict_handler='error', add_help=True)>\n",
|
||||
"optimize_parallel : <bound method HyperOptArgumentParser.optimize_parallel of HyperOptArgumentParser(prog='ipykernel_launcher.py', usage=None, description=None, formatter_class=<class 'argparse.HelpFormatter'>, conflict_handler='error', add_help=True)>\n",
|
||||
"optimize_parallel_gpu: <bound method HyperOptArgumentParser.optimize_parallel_gpu of HyperOptArgumentParser(prog='ipykernel_launcher.py', usage=None, description=None, formatter_class=<class 'argparse.HelpFormatter'>, conflict_handler='error', add_help=True)>\n",
|
||||
"optimize_parallel_cpu: <bound method HyperOptArgumentParser.optimize_parallel_cpu of HyperOptArgumentParser(prog='ipykernel_launcher.py', usage=None, description=None, formatter_class=<class 'argparse.HelpFormatter'>, conflict_handler='error', add_help=True)>\n",
|
||||
"generate_trials : <bound method HyperOptArgumentParser.generate_trials of HyperOptArgumentParser(prog='ipykernel_launcher.py', usage=None, description=None, formatter_class=<class 'argparse.HelpFormatter'>, conflict_handler='error', add_help=True)>\n",
|
||||
"optimize_trials_parallel_gpu: <bound method HyperOptArgumentParser.optimize_trials_parallel_gpu of HyperOptArgumentParser(prog='ipykernel_launcher.py', usage=None, description=None, formatter_class=<class 'argparse.HelpFormatter'>, conflict_handler='error', add_help=True)>\n",
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from pytorch_lightning import Trainer\n",
|
||||
"from argparse import ArgumentParser \n",
|
||||
"\n",
|
||||
"parser = ArgumentParser(add_help=False)\n",
|
||||
"parser.add_argument('--gpus', type=str, default=None)\n",
|
||||
"parser.add_argument('--nodes', type=int, default=1)\n",
|
||||
"\n",
|
||||
"# give the module a chance to add own params\n",
|
||||
"parser = LatentModelPL.add_model_specific_args(parser)\n",
|
||||
"\n",
|
||||
"# parse params\n",
|
||||
"hparams = parser.parse_args(argv)\n",
|
||||
"hyperparams, hparams = parse_args(f\"\"\"\n",
|
||||
"--x_dim {df_train.shape[-1]-1} \\\n",
|
||||
"--y_dim 1 \\\n",
|
||||
"--grad_clip 40 \\\n",
|
||||
"\"\"\")\n",
|
||||
"print(hparams)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Run"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-01-26T02:48:24.261610Z",
|
||||
"start_time": "2020-01-26T02:48:19.200Z"
|
||||
},
|
||||
"scrolled": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# import os\n",
|
||||
"# from test_tube import Experiment\n",
|
||||
"\n",
|
||||
"# model = LatentModelPL(hparams)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# # most basic trainer, uses good defaults\n",
|
||||
"# trainer = Trainer(\n",
|
||||
"# max_epochs=hparams.max_nb_epochs,\n",
|
||||
"# gpus=hparams.gpus,\n",
|
||||
"# nb_gpu_nodes=hparams.nodes,\n",
|
||||
"# gradient_clip_val=hparams.grad_clip,\n",
|
||||
"# track_grad_norm=1,\n",
|
||||
"# show_progress_bar=True\n",
|
||||
"# )\n",
|
||||
"# # model.trainer = trainer\n",
|
||||
"# trainer.fit(model)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
@@ -345,7 +341,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 60,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-01-26T02:43:08.911937Z",
|
||||
@@ -364,11 +360,11 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 90,
|
||||
"execution_count": 11,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-01-26T02:55:36.164163Z",
|
||||
"start_time": "2020-01-26T02:55:36.097064Z"
|
||||
"end_time": "2020-01-27T00:47:39.090630Z",
|
||||
"start_time": "2020-01-27T00:47:39.035815Z"
|
||||
},
|
||||
"scrolled": true
|
||||
},
|
||||
@@ -394,7 +390,7 @@
|
||||
" max_epochs=hparams.max_nb_epochs,\n",
|
||||
" gpus=hparams.gpus,\n",
|
||||
" gradient_clip_val=hparams.grad_clip,\n",
|
||||
" track_grad_norm=1,\n",
|
||||
"# track_grad_norm=1,\n",
|
||||
" )\n",
|
||||
" # train model\n",
|
||||
" if hparams.evaluate:\n",
|
||||
@@ -406,156 +402,116 @@
|
||||
"# root_dir = os.path.split(os.path.dirname(sys.modules['__main__'].__file__))[0]\n",
|
||||
"root_dir = os.path.split(os.getcwd())[0]\n",
|
||||
"\n",
|
||||
"parent = HyperOptArgumentParser(strategy='random_search', add_help=False)\n",
|
||||
"parser = HyperOptArgumentParser(strategy='random_search', add_help=False)\n",
|
||||
"\n",
|
||||
"parent = add_default_args(parent)\n",
|
||||
"parser = add_default_args(parser)\n",
|
||||
"\n",
|
||||
"# allow model to overwrite or extend args\n",
|
||||
"parser = LatentModelPL.add_model_specific_args(parent)\n",
|
||||
"\n"
|
||||
"parser = LatentModelPL.add_model_specific_args(parser)\n",
|
||||
"\n",
|
||||
"hyperparams = parser.parse_args([])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 103,
|
||||
"execution_count": 12,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-01-26T03:03:41.752611Z",
|
||||
"start_time": "2020-01-26T03:03:41.698482Z"
|
||||
}
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"{'gpus': -1,\n",
|
||||
" 'nodes': 1,\n",
|
||||
" 'hpc_exp_number': 1,\n",
|
||||
" 'seed': 42,\n",
|
||||
" 'evaluate': False,\n",
|
||||
" 'learning_rate': 0.0001,\n",
|
||||
" 'batch_size': 16,\n",
|
||||
" 'x_dim': 16,\n",
|
||||
" 'y_dim': 1,\n",
|
||||
" 'vis_i': 670,\n",
|
||||
" 'hidden_dim': 128,\n",
|
||||
" 'latent_dim': 128,\n",
|
||||
" 'num_heads': 8,\n",
|
||||
" 'n_latent_encoder_layers': 4,\n",
|
||||
" 'n_det_encoder_layers': 4,\n",
|
||||
" 'n_decoder_layers': 2,\n",
|
||||
" 'dropout': 0,\n",
|
||||
" 'attention_dropout': 0,\n",
|
||||
" 'min_std': 0.01,\n",
|
||||
" 'latent_enc_self_attn_type': 'multihead',\n",
|
||||
" 'det_enc_self_attn_type': 'multihead',\n",
|
||||
" 'det_enc_cross_attn_type': 'multihead',\n",
|
||||
" 'use_lvar': False,\n",
|
||||
" 'use_deterministic_path': True,\n",
|
||||
" 'grad_clip': 0,\n",
|
||||
" 'num_context': 48,\n",
|
||||
" 'num_extra_target': 24,\n",
|
||||
" 'max_nb_epochs': 20,\n",
|
||||
" 'num_workers': 4,\n",
|
||||
" 'trials': <bound method HyperOptArgumentParser.opt_trials of HyperOptArgumentParser(prog='ipykernel_launcher.py', usage=None, description=None, formatter_class=<class 'argparse.HelpFormatter'>, conflict_handler='error', add_help=True)>,\n",
|
||||
" 'optimize_parallel': <bound method HyperOptArgumentParser.optimize_parallel of HyperOptArgumentParser(prog='ipykernel_launcher.py', usage=None, description=None, formatter_class=<class 'argparse.HelpFormatter'>, conflict_handler='error', add_help=True)>,\n",
|
||||
" 'optimize_parallel_gpu': <bound method HyperOptArgumentParser.optimize_parallel_gpu of HyperOptArgumentParser(prog='ipykernel_launcher.py', usage=None, description=None, formatter_class=<class 'argparse.HelpFormatter'>, conflict_handler='error', add_help=True)>,\n",
|
||||
" 'optimize_parallel_cpu': <bound method HyperOptArgumentParser.optimize_parallel_cpu of HyperOptArgumentParser(prog='ipykernel_launcher.py', usage=None, description=None, formatter_class=<class 'argparse.HelpFormatter'>, conflict_handler='error', add_help=True)>,\n",
|
||||
" 'generate_trials': <bound method HyperOptArgumentParser.generate_trials of HyperOptArgumentParser(prog='ipykernel_launcher.py', usage=None, description=None, formatter_class=<class 'argparse.HelpFormatter'>, conflict_handler='error', add_help=True)>,\n",
|
||||
" 'optimize_trials_parallel_gpu': <bound method HyperOptArgumentParser.optimize_trials_parallel_gpu of HyperOptArgumentParser(prog='ipykernel_launcher.py', usage=None, description=None, formatter_class=<class 'argparse.HelpFormatter'>, conflict_handler='error', add_help=True)>}"
|
||||
]
|
||||
},
|
||||
"execution_count": 103,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"hyperparams.__dict__"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 91,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-01-26T02:55:36.508582Z",
|
||||
"start_time": "2020-01-26T02:55:36.470898Z"
|
||||
"end_time": "2020-01-27T00:47:39.134945Z",
|
||||
"start_time": "2020-01-27T00:47:39.093368Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"hyperparams = parser.parse_args([])\n",
|
||||
"\n",
|
||||
"# train model\n",
|
||||
"# main(hyperparams)"
|
||||
"# # Run some trials on a single cpu. You can view them in tensorboard, and see the hyper params\n",
|
||||
"# trials = hyperparams.generate_trials(16)\n",
|
||||
"# for trial in tqdm(trials, desc='trials'):\n",
|
||||
"# try:\n",
|
||||
"# main(trial)\n",
|
||||
"# except KeyboardInterrupt:\n",
|
||||
"# break\n",
|
||||
"# except:\n",
|
||||
"# logging.exception(\"failed to run\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Run"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 109,
|
||||
"execution_count": 13,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-01-26T03:08:52.581129Z",
|
||||
"start_time": "2020-01-26T03:08:52.513317Z"
|
||||
"end_time": "2020-01-27T00:47:39.174912Z",
|
||||
"start_time": "2020-01-27T00:47:39.137458Z"
|
||||
}
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"ename": "TypeError",
|
||||
"evalue": "__class__ must be set to a class, not 'NoneType' object",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)",
|
||||
"\u001b[0;32m<ipython-input-109-fc3f99954fa5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfloat\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbool\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0msetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 7\u001b[0m \u001b[0mhparams\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;31mTypeError\u001b[0m: __class__ must be set to a class, not 'NoneType' object"
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"----------------------------------------------------------------------------------------------------\n",
|
||||
"Hyperparameters:\n",
|
||||
"gpus : -1\n",
|
||||
"nodes : 1\n",
|
||||
"hpc_exp_number : 1\n",
|
||||
"seed : 42\n",
|
||||
"evaluate : False\n",
|
||||
"learning_rate : 0.0001\n",
|
||||
"batch_size : 16\n",
|
||||
"x_dim : 16\n",
|
||||
"y_dim : 1\n",
|
||||
"vis_i : 670\n",
|
||||
"hidden_dim : 128\n",
|
||||
"latent_dim : 128\n",
|
||||
"num_heads : 8\n",
|
||||
"attention_layers : 2\n",
|
||||
"n_latent_encoder_layers: 4\n",
|
||||
"n_det_encoder_layers: 4\n",
|
||||
"n_decoder_layers : 2\n",
|
||||
"dropout : 0\n",
|
||||
"attention_dropout : 0\n",
|
||||
"min_std : 0.005\n",
|
||||
"latent_enc_self_attn_type: ptmultihead\n",
|
||||
"det_enc_self_attn_type: ptmultihead\n",
|
||||
"det_enc_cross_attn_type: ptmultihead\n",
|
||||
"use_lvar : False\n",
|
||||
"use_deterministic_path: True\n",
|
||||
"grad_clip : 40.0\n",
|
||||
"num_context : 48\n",
|
||||
"num_extra_target : 24\n",
|
||||
"max_nb_epochs : 20\n",
|
||||
"num_workers : 4\n",
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import copy\n",
|
||||
"hparams = copy.deepcopy(hyperparams)\n",
|
||||
"for k in dir(hparams):\n",
|
||||
" v = getattr(hparams, k)\n",
|
||||
" if not isinstance(v, (int, float, str, bool, torch.Tensor)):\n",
|
||||
" setattr(hparams, k, None)\n",
|
||||
"hparams"
|
||||
"hyperparams, hparams = parse_args(f\"\"\"\n",
|
||||
"--x_dim {df_train.shape[-1]-1} \\\n",
|
||||
"--y_dim 1 \\\n",
|
||||
"--grad_clip 40 \\\n",
|
||||
"--latent_enc_self_attn_type ptmultihead \\\n",
|
||||
"--det_enc_self_attn_type ptmultihead \\\n",
|
||||
"--det_enc_cross_attn_type ptmultihead \\\n",
|
||||
"--attention_layers 2 \\\n",
|
||||
"\"\"\")\n",
|
||||
"print(hparams)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 94,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-01-26T02:56:37.274166Z",
|
||||
"start_time": "2020-01-26T02:56:37.230029Z"
|
||||
}
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"24"
|
||||
]
|
||||
},
|
||||
"execution_count": 94,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"72-48"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 96,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-01-26T02:57:32.050149Z",
|
||||
"start_time": "2020-01-26T02:57:22.400348Z"
|
||||
}
|
||||
"start_time": "2020-01-27T00:47:27.500Z"
|
||||
},
|
||||
"scrolled": true
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -563,72 +519,28 @@
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"INFO:root:gpu available: True, used: True\n",
|
||||
"INFO:root:VISIBLE GPUS: 0\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"----------------------------------------------------------------------------------------------------\n",
|
||||
"Hyperparameters:\n",
|
||||
"learning_rate : 0.0037216107638185568\n",
|
||||
"hidden_dim : 128\n",
|
||||
"latent_dim : 512\n",
|
||||
"n_latent_encoder_layers: 8\n",
|
||||
"n_det_encoder_layers: 1\n",
|
||||
"n_decoder_layers : 8\n",
|
||||
"dropout : 0.3553951704743503\n",
|
||||
"attention_dropout : 0.10436943022123696\n",
|
||||
"latent_enc_self_attn_type: laplace\n",
|
||||
"det_enc_self_attn_type: multihead\n",
|
||||
"det_enc_cross_attn_type: laplace\n",
|
||||
"use_lvar : True\n",
|
||||
"use_deterministic_path: True\n",
|
||||
"gpus : -1\n",
|
||||
"nodes : 1\n",
|
||||
"hpc_exp_number : 1\n",
|
||||
"seed : 42\n",
|
||||
"evaluate : False\n",
|
||||
"batch_size : 16\n",
|
||||
"x_dim : 16\n",
|
||||
"y_dim : 1\n",
|
||||
"vis_i : 670\n",
|
||||
"num_heads : 8\n",
|
||||
"min_std : 0.01\n",
|
||||
"grad_clip : 0\n",
|
||||
"num_context : 48\n",
|
||||
"num_extra_target : 24\n",
|
||||
"max_nb_epochs : 20\n",
|
||||
"num_workers : 4\n",
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"INFO:root:VISIBLE GPUS: 0\n",
|
||||
"INFO:root:\n",
|
||||
" Name Type Params\n",
|
||||
"0 model LatentModel 5 M\n",
|
||||
"1 model._latent_encoder LatentEncoder 285 K\n",
|
||||
"0 model LatentModel 987 K\n",
|
||||
"1 model._latent_encoder LatentEncoder 284 K\n",
|
||||
"2 model._latent_encoder._input_layer NPBlockRelu2d 2 K\n",
|
||||
"3 model._latent_encoder._input_layer.linear Linear 2 K\n",
|
||||
"4 model._latent_encoder._input_layer.act ReLU 0 \n",
|
||||
".. ... ... ...\n",
|
||||
"118 model._decoder._decoder.7.act ReLU 0 \n",
|
||||
"119 model._decoder._decoder.7.dropout Dropout2d 0 \n",
|
||||
"120 model._decoder._decoder.7.norm BatchNorm2d 1 K\n",
|
||||
"121 model._decoder._mean Linear 769 \n",
|
||||
"122 model._decoder._std Linear 769 \n",
|
||||
"162 model._decoder._decoder.1.act ReLU 0 \n",
|
||||
"163 model._decoder._decoder.1.dropout Dropout2d 0 \n",
|
||||
"164 model._decoder._decoder.1.norm BatchNorm2d 768 \n",
|
||||
"165 model._decoder._mean Linear 385 \n",
|
||||
"166 model._decoder._std Linear 385 \n",
|
||||
"\n",
|
||||
"[123 rows x 3 columns]\n"
|
||||
"[167 rows x 3 columns]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "591cb5ee0f3f4feb8d486b3ac06dfe6a",
|
||||
"model_id": "",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
@@ -640,34 +552,145 @@
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"ename": "RuntimeError",
|
||||
"evalue": "invalid argument 0: Sizes of tensors must match except in dimension 2. Got 72 and 48 in dimension 1 at /pytorch/aten/src/THC/generic/THCTensorMath.cu:71",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)",
|
||||
"\u001b[0;32m<ipython-input-96-726df5c33ea2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0mtrials\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhyperparams\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgenerate_trials\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m8\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mtrial\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtrials\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mmain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
|
||||
"\u001b[0;32m<ipython-input-90-f749382ebe64>\u001b[0m in \u001b[0;36mmain\u001b[0;34m(hparams)\u001b[0m\n\u001b[1;32m 25\u001b[0m \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_evaluation\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 26\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 29\u001b[0m \u001b[0;31m# use default args given by lightning\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;32m/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, model)\u001b[0m\n\u001b[1;32m 700\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 701\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msingle_gpu\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 702\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msingle_gpu_train\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 703\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 704\u001b[0m \u001b[0;31m# ON CPU\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;32m/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/trainer/distrib_parts.py\u001b[0m in \u001b[0;36msingle_gpu_train\u001b[0;34m(self, model)\u001b[0m\n\u001b[1;32m 439\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizers\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moptimizers\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 440\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 441\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_pretrain_routine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 442\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 443\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mdp_train\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;32m/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36mrun_pretrain_routine\u001b[0;34m(self, model)\u001b[0m\n\u001b[1;32m 818\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 819\u001b[0m eval_results = self.evaluate(model, self.get_val_dataloaders(),\n\u001b[0;32m--> 820\u001b[0;31m self.num_sanity_val_steps, False)\n\u001b[0m\u001b[1;32m 821\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback_metrics\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprocess_output\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0meval_results\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 822\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;32m/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/trainer/evaluation_loop.py\u001b[0m in \u001b[0;36mevaluate\u001b[0;34m(self, model, dataloaders, max_batches, test)\u001b[0m\n\u001b[1;32m 232\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 233\u001b[0m \u001b[0mdataloader_idx\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 234\u001b[0;31m test)\n\u001b[0m\u001b[1;32m 235\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 236\u001b[0m \u001b[0;31m# track outputs for collation\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;32m/media/wassname/Storage5/projects2/3ST/pytorch-lightning/pytorch_lightning/trainer/evaluation_loop.py\u001b[0m in \u001b[0;36mevaluation_forward\u001b[0;34m(self, model, batch, batch_idx, dataloader_idx, test)\u001b[0m\n\u001b[1;32m 363\u001b[0m \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtest_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 364\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 365\u001b[0;31m \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalidation_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 366\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 367\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;32m/media/wassname/Storage5/projects2/3ST/attentive-neural-processes/src/models/lightning_anp.py\u001b[0m in \u001b[0;36mvalidation_step\u001b[0;34m(self, batch, batch_idx)\u001b[0m\n\u001b[1;32m 33\u001b[0m \u001b[0;32massert\u001b[0m \u001b[0mall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misfinite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0md\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0md\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 34\u001b[0m \u001b[0mcontext_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontext_y\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_y\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 35\u001b[0;31m \u001b[0my_pred\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_std\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcontext_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontext_y\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_y\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 36\u001b[0m tensorboard_logs = {\n\u001b[1;32m 37\u001b[0m \u001b[0;34m\"val/loss\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;32m/media/wassname/Storage5/projects2/3ST/attentive-neural-processes/src/models/lightning_anp.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, context_x, context_y, target_x, target_y)\u001b[0m\n\u001b[1;32m 17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 18\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontext_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontext_y\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_y\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcontext_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontext_y\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_y\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 20\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 21\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mtraining_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;32m~/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 539\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 540\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 541\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 542\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 543\u001b[0m \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;32m/media/wassname/Storage5/projects2/3ST/attentive-neural-processes/src/models/model.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, context_x, context_y, target_x, target_y)\u001b[0m\n\u001b[1;32m 121\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 122\u001b[0m \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 123\u001b[0;31m \u001b[0mdist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlog_sigma\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_decoder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mz\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_x\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 124\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 125\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mtarget_y\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;32m~/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 539\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 540\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 541\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 542\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 543\u001b[0m \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;32m/media/wassname/Storage5/projects2/3ST/attentive-neural-processes/src/models/modules.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, r, z, target_x)\u001b[0m\n\u001b[1;32m 255\u001b[0m \u001b[0;31m# concatenate target_x and representation\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 256\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muse_deterministic_path\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 257\u001b[0;31m \u001b[0mz\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mz\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 258\u001b[0m \u001b[0mrepresentation\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mz\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 259\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;31mRuntimeError\u001b[0m: invalid argument 0: Sizes of tensors must match except in dimension 2. Got 72 and 48 in dimension 1 at /pytorch/aten/src/THC/generic/THCTensorMath.cu:71"
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"step 0, {'val_loss': tensor(0.7484, device='cuda:0'), 'val/kl': tensor(0.0002, device='cuda:0'), 'val/std': tensor(0.7129, device='cuda:0'), 'val/mse': tensor(0.1703, device='cuda:0')}\n",
|
||||
"\r"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "f0c6de3ff0a849878a30c8e19e04a552",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=241.0, style=Pr…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"step 2201, {'val_loss': tensor(-0.6897, device='cuda:0'), 'val/kl': tensor(0.0036, device='cuda:0'), 'val/std': tensor(0.2597, device='cuda:0'), 'val/mse': tensor(0.0670, device='cuda:0')}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=241.0, style=Pr…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"step 4403, {'val_loss': tensor(-0.8933, device='cuda:0'), 'val/kl': tensor(0.0012, device='cuda:0'), 'val/std': tensor(0.1486, device='cuda:0'), 'val/mse': tensor(0.0167, device='cuda:0')}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=241.0, style=Pr…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"step 6605, {'val_loss': tensor(-1.2594, device='cuda:0'), 'val/kl': tensor(0.0007, device='cuda:0'), 'val/std': tensor(0.0727, device='cuda:0'), 'val/mse': tensor(0.0054, device='cuda:0')}\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Run some trials on a single cpu. You can view them in tensorboard, and see the hyper params\n",
|
||||
"trials = hyperparams.generate_trials(8)\n",
|
||||
"for trial in trials:\n",
|
||||
" main(trial)"
|
||||
"import os\n",
|
||||
"from test_tube import Experiment\n",
|
||||
"\n",
|
||||
"model = LatentModelPL(hparams)\n",
|
||||
"\n",
|
||||
"# most basic trainer, uses good defaults\n",
|
||||
"trainer = Trainer(\n",
|
||||
" max_epochs=hparams.max_nb_epochs,\n",
|
||||
" gpus=hparams.gpus,\n",
|
||||
" nb_gpu_nodes=hparams.nodes,\n",
|
||||
" gradient_clip_val=hparams.grad_clip,\n",
|
||||
")\n",
|
||||
"trainer.fit(model)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"start_time": "2020-01-27T00:47:27.600Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%debug"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"start_time": "2020-01-27T00:47:27.600Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"model.model._latent_encoder.self_attention"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"start_time": "2020-01-27T00:47:27.600Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import torch\n",
|
||||
"torch.nn.MultiheadAttention?"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -675,9 +698,7 @@
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%debug"
|
||||
]
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
import torch as t
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
import torch
|
||||
import math
|
||||
|
||||
class Linear(nn.Module):
|
||||
"""
|
||||
Linear Module
|
||||
"""
|
||||
def __init__(self, in_dim, out_dim, bias=True, w_init='linear'):
|
||||
"""
|
||||
:param in_dim: dimension of input
|
||||
:param out_dim: dimension of output
|
||||
:param bias: boolean. if True, bias is included.
|
||||
:param w_init: str. weight inits with xavier initialization.
|
||||
"""
|
||||
super(Linear, self).__init__()
|
||||
self.linear_layer = nn.Linear(in_dim, out_dim, bias=bias)
|
||||
|
||||
nn.init.xavier_uniform_(
|
||||
self.linear_layer.weight,
|
||||
gain=nn.init.calculate_gain(w_init))
|
||||
|
||||
def forward(self, x):
|
||||
return self.linear_layer(x)
|
||||
|
||||
class MultiheadAttention(nn.Module):
|
||||
"""
|
||||
Multihead attention mechanism (dot attention)
|
||||
"""
|
||||
def __init__(self, num_hidden_k):
|
||||
"""
|
||||
:param num_hidden_k: dimension of hidden
|
||||
"""
|
||||
super(MultiheadAttention, self).__init__()
|
||||
|
||||
self.num_hidden_k = num_hidden_k
|
||||
self.attn_dropout = nn.Dropout(p=0.1)
|
||||
|
||||
def forward(self, key, value, query):
|
||||
# Get attention score
|
||||
attn = t.bmm(query, key.transpose(1, 2))
|
||||
attn = attn / math.sqrt(self.num_hidden_k)
|
||||
|
||||
attn = t.softmax(attn, dim=-1)
|
||||
|
||||
# Dropout
|
||||
attn = self.attn_dropout(attn)
|
||||
|
||||
# Get Context Vector
|
||||
result = t.bmm(attn, value)
|
||||
|
||||
return result, attn
|
||||
|
||||
|
||||
class Attention(nn.Module):
|
||||
"""
|
||||
Attention Network
|
||||
"""
|
||||
def __init__(self, num_hidden, h=4):
|
||||
"""
|
||||
:param num_hidden: dimension of hidden
|
||||
:param h: num of heads
|
||||
"""
|
||||
super(Attention, self).__init__()
|
||||
|
||||
self.num_hidden = num_hidden
|
||||
self.num_hidden_per_attn = num_hidden // h
|
||||
self.h = h
|
||||
|
||||
self.key = Linear(num_hidden, num_hidden, bias=False)
|
||||
self.value = Linear(num_hidden, num_hidden, bias=False)
|
||||
self.query = Linear(num_hidden, num_hidden, bias=False)
|
||||
|
||||
self.multihead = MultiheadAttention(self.num_hidden_per_attn)
|
||||
|
||||
self.residual_dropout = nn.Dropout(p=0.1)
|
||||
|
||||
self.final_linear = Linear(num_hidden * 2, num_hidden)
|
||||
|
||||
self.layer_norm = nn.LayerNorm(num_hidden)
|
||||
|
||||
def forward(self, key, value, query):
|
||||
|
||||
batch_size = key.size(0)
|
||||
seq_k = key.size(1)
|
||||
seq_q = query.size(1)
|
||||
residual = query
|
||||
|
||||
# Make multihead
|
||||
key = self.key(key).view(batch_size, seq_k, self.h, self.num_hidden_per_attn)
|
||||
value = self.value(value).view(batch_size, seq_k, self.h, self.num_hidden_per_attn)
|
||||
query = self.query(query).view(batch_size, seq_q, self.h, self.num_hidden_per_attn)
|
||||
|
||||
key = key.permute(2, 0, 1, 3).contiguous().view(-1, seq_k, self.num_hidden_per_attn)
|
||||
value = value.permute(2, 0, 1, 3).contiguous().view(-1, seq_k, self.num_hidden_per_attn)
|
||||
query = query.permute(2, 0, 1, 3).contiguous().view(-1, seq_q, self.num_hidden_per_attn)
|
||||
|
||||
# Get context vector
|
||||
result, attns = self.multihead(key, value, query)
|
||||
|
||||
# Concatenate all multihead context vector
|
||||
result = result.view(self.h, batch_size, seq_q, self.num_hidden_per_attn)
|
||||
result = result.permute(1, 2, 0, 3).contiguous().view(batch_size, seq_q, -1)
|
||||
|
||||
# Concatenate context vector with input (most important)
|
||||
result = t.cat([residual, result], dim=-1)
|
||||
|
||||
# Final linear
|
||||
result = self.final_linear(result)
|
||||
|
||||
# Residual dropout & connection
|
||||
result = self.residual_dropout(result)
|
||||
result = result + residual
|
||||
|
||||
# Layer normalization
|
||||
result = self.layer_norm(result)
|
||||
|
||||
return result, attns
|
||||
+32
-18
@@ -1,5 +1,6 @@
|
||||
import pytorch_lightning as pl
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from argparse import ArgumentParser
|
||||
from test_tube import Experiment, HyperOptArgumentParser
|
||||
from src.models.model import LatentModel
|
||||
@@ -23,9 +24,10 @@ class LatentModelPL(pl.LightningModule):
|
||||
context_x, context_y, target_x, target_y = batch
|
||||
y_pred, kl, loss, y_std = self.forward(context_x, context_y, target_x, target_y)
|
||||
tensorboard_logs = {
|
||||
"train_loss": loss,
|
||||
"train_kl": kl.mean(),
|
||||
"train/loss": loss,
|
||||
"train/kl": kl.mean(),
|
||||
"train/std": y_std.mean(),
|
||||
"train/mse": F.mse_loss(y_pred, target_y).mean(),
|
||||
}
|
||||
return {"loss": loss, "log": tensorboard_logs}
|
||||
|
||||
@@ -33,10 +35,12 @@ class LatentModelPL(pl.LightningModule):
|
||||
assert all(torch.isfinite(d).all() for d in batch)
|
||||
context_x, context_y, target_x, target_y = batch
|
||||
y_pred, kl, loss, y_std = self.forward(context_x, context_y, target_x, target_y)
|
||||
|
||||
tensorboard_logs = {
|
||||
"val/loss": loss,
|
||||
"val_loss": loss,
|
||||
"val/kl": kl.mean(),
|
||||
"val/std": y_std.mean()
|
||||
"val/std": y_std.mean(),
|
||||
"val/mse": F.mse_loss(y_pred, target_y).mean(),
|
||||
}
|
||||
return {"val_loss": loss, "log": tensorboard_logs}
|
||||
|
||||
@@ -47,14 +51,23 @@ class LatentModelPL(pl.LightningModule):
|
||||
image = plot_from_loader_to_tensor(loader, self.model, i=self.hparams.vis_i)
|
||||
self.logger.experiment.add_image('val/image', image, self.trainer.global_step)
|
||||
|
||||
keys = outputs[0]["log"].keys()
|
||||
# tensorboard_logs = {}
|
||||
# for k in keys:
|
||||
# tensorboard_logs[k] = torch.stack([x["log"][k] for x in outputs if k in x["log"]]).mean()
|
||||
tensorboard_logs = {k: torch.stack([x["log"][k] for x in outputs if k in x["log"]]).mean() for k in keys}
|
||||
|
||||
avg_loss = torch.stack([x["val_loss"] for x in outputs]).mean()
|
||||
tensorboard_logs = {"val_loss": avg_loss}
|
||||
print(self.trainer.global_step, tensorboard_logs)
|
||||
print(f"step {self.trainer.global_step}, {tensorboard_logs}")
|
||||
|
||||
# Log hparams with metric, doesn't work
|
||||
# self.logger.experiment.add_hparams(self.hparams.__dict__, {"avg_val_loss": avg_loss})
|
||||
|
||||
return {"avg_val_loss": avg_loss, "log": tensorboard_logs}
|
||||
|
||||
def configure_optimizers(self):
|
||||
optim = torch.optim.Adam(self.parameters(), lr=self.hparams.learning_rate)
|
||||
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optim)
|
||||
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optim, patience=2, verbose=True, min_lr=1e-5) # note early stopping has patient 3
|
||||
return [optim], [scheduler]
|
||||
|
||||
def _get_cache_dfs(self):
|
||||
@@ -123,22 +136,23 @@ class LatentModelPL(pl.LightningModule):
|
||||
parser.add_argument("--y_dim", default=1, type=int)
|
||||
parser.add_argument("--vis_i", default=670, type=int)
|
||||
|
||||
parser.opt_list("--hidden_dim", default=128, type=int, tunable=True, options=[8*2**i for i in range(7)])
|
||||
parser.opt_list("--latent_dim", default=256, type=int, tunable=True, options=[8*2**i for i in range(7)])
|
||||
parser.opt_list("--hidden_dim", default=128, type=int, tunable=True, options=[8*2**i for i in range(8)])
|
||||
parser.opt_list("--latent_dim", default=128, type=int, tunable=True, options=[8*2**i for i in range(8)])
|
||||
parser.add_argument("--num_heads", default=8, type=int)
|
||||
parser.opt_list("--n_latent_encoder_layers", default=4, type=int, tunable=True, options=[1, 2, 4, 8])
|
||||
parser.opt_list("--n_det_encoder_layers", default=4, type=int, tunable=True, options=[1, 2, 4, 8])
|
||||
parser.opt_list("--n_decoder_layers", default=2, type=int, tunable=True, options=[1, 2, 4, 8])
|
||||
parser.add_argument("--attention_layers", default=1, type=int)
|
||||
parser.opt_list("--n_latent_encoder_layers", default=4, type=int, tunable=True, options=[1, 2, 4, 8, 16])
|
||||
parser.opt_list("--n_det_encoder_layers", default=4, type=int, tunable=True, options=[1, 2, 4, 8, 16])
|
||||
parser.opt_list("--n_decoder_layers", default=2, type=int, tunable=True, options=[1, 2, 4, 8, 16])
|
||||
|
||||
parser.opt_range("--dropout", default=0, type=float, tunable=True, low=0, high=0.5)
|
||||
parser.opt_range("--attention_dropout", default=0, type=float, tunable=True, low=0, high=0.5)
|
||||
parser.add_argument("--min_std", default=0.01, type=float)
|
||||
parser.opt_range("--dropout", default=0, type=float, tunable=True, low=0, high=0.75)
|
||||
parser.opt_range("--attention_dropout", default=0, type=float, tunable=True, low=0, high=0.75)
|
||||
parser.add_argument("--min_std", default=0.005, type=float)
|
||||
|
||||
parser.opt_list(
|
||||
"--latent_enc_self_attn_type", default="multihead", type=str, tunable=True, options=['uniform', 'dot', 'multihead', 'laplace']
|
||||
"--latent_enc_self_attn_type", default="multihead", type=str, tunable=True, options=['uniform', 'dot', 'multihead', 'ptmultihead']
|
||||
)
|
||||
parser.opt_list("--det_enc_self_attn_type", default="multihead", type=str, tunable=True, options=['uniform', 'dot', 'multihead', 'laplace'])
|
||||
parser.opt_list("--det_enc_cross_attn_type", default="multihead", type=str, tunable=True, options=['uniform', 'dot', 'multihead', 'laplace'])
|
||||
parser.opt_list("--det_enc_self_attn_type", default="multihead", type=str, tunable=True, options=['uniform', 'dot', 'multihead', 'ptmultihead'])
|
||||
parser.opt_list("--det_enc_cross_attn_type", default="multihead", type=str, tunable=True, options=['uniform', 'dot', 'multihead', 'ptmultihead'])
|
||||
|
||||
parser.opt_list("--use_lvar", default=False, type=bool, tunable=True, options=[False, True])
|
||||
parser.opt_list("--use_deterministic_path", default=True, tunable=True, type=bool, options=[False, True])
|
||||
|
||||
@@ -53,6 +53,7 @@ class LatentModel(nn.Module):
|
||||
min_std=0.1,
|
||||
use_lvar=False,
|
||||
use_deterministic_path=True,
|
||||
attention_layers=2,
|
||||
**kwargs
|
||||
):
|
||||
|
||||
@@ -71,6 +72,7 @@ class LatentModel(nn.Module):
|
||||
n_heads=num_heads,
|
||||
min_std=min_std,
|
||||
use_lvar=use_lvar,
|
||||
attention_layers=attention_layers,
|
||||
)
|
||||
|
||||
self._deterministic_encoder = DeterministicEncoder(
|
||||
@@ -83,6 +85,7 @@ class LatentModel(nn.Module):
|
||||
dropout=dropout,
|
||||
attention_dropout=attention_dropout,
|
||||
n_heads=num_heads,
|
||||
attention_layers=attention_layers,
|
||||
)
|
||||
|
||||
self._decoder = Decoder(
|
||||
@@ -141,6 +144,7 @@ class LatentModel(nn.Module):
|
||||
log_p = None
|
||||
kl_loss = None
|
||||
loss = None
|
||||
mse_loss = None
|
||||
|
||||
y_pred = dist.rsample() if self.training else dist.loc
|
||||
return y_pred, kl_loss, loss, dist.scale
|
||||
|
||||
+28
-7
@@ -2,6 +2,7 @@ import torch
|
||||
from torch import nn
|
||||
import torch.nn.functional as F
|
||||
import math
|
||||
from .attention import Attention as PtAttention
|
||||
|
||||
class NPBlockRelu2d(nn.Module):
|
||||
"""Block for Neural Processes."""
|
||||
@@ -39,7 +40,7 @@ def block_relu(in_dim, out_dim, dropout=0, inplace=False):
|
||||
|
||||
|
||||
class Attention(nn.Module):
|
||||
def __init__(self, hidden_dim, attention_type, n_heads=8, dropout=0):
|
||||
def __init__(self, hidden_dim, attention_type, attention_layers=1, n_heads=8, dropout=0):
|
||||
super().__init__()
|
||||
if attention_type == "uniform":
|
||||
self._attention_func = self._uniform_attention
|
||||
@@ -48,11 +49,17 @@ class Attention(nn.Module):
|
||||
elif attention_type == "dot":
|
||||
self._attention_func = self._dot_attention
|
||||
elif attention_type == "multihead":
|
||||
self._mattn = torch.nn.MultiheadAttention(
|
||||
self._mattn = nn.ModuleList([torch.nn.MultiheadAttention(
|
||||
hidden_dim, n_heads, bias=False, dropout=dropout
|
||||
)
|
||||
) for _ in range(attention_layers)])
|
||||
self._attention_func = self._pytorch_multihead_attention
|
||||
self.n_heads = n_heads
|
||||
elif attention_type == "ptmultihead":
|
||||
self._mattn = nn.ModuleList([PtAttention(
|
||||
hidden_dim, n_heads
|
||||
) for _ in range(attention_layers)])
|
||||
self._attention_func = self._ptmultihead_fn
|
||||
self.n_heads = n_heads
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -85,9 +92,21 @@ class Attention(nn.Module):
|
||||
|
||||
def _pytorch_multihead_attention(self, k, v, q):
|
||||
# Pytorch multiheaded attention takes inputs if diff order and permutation
|
||||
o = self._mattn(q.permute(1, 0, 2), k.permute(1, 0, 2), v.permute(1, 0, 2))[0]
|
||||
q = q.permute(1, 0, 2)
|
||||
k = k.permute(1, 0, 2)
|
||||
v = v.permute(1, 0, 2)
|
||||
for attention in self._mattn:
|
||||
o = attention(q, k, v)[0]
|
||||
q, k, v = o, o, o
|
||||
return o.permute(1, 0, 2)
|
||||
|
||||
def _ptmultihead_fn(self, k, v, q):
|
||||
for attention in self._mattn:
|
||||
o = attention(k, v, q)[0]
|
||||
# print(k.shape, v.shape, q.shape, o.shape)
|
||||
q, k, v = o, o, o
|
||||
return o
|
||||
|
||||
|
||||
class LatentEncoder(nn.Module):
|
||||
"""
|
||||
@@ -105,6 +124,7 @@ class LatentEncoder(nn.Module):
|
||||
dropout=0,
|
||||
attention_dropout=0,
|
||||
use_lvar=False,
|
||||
attention_layers=2,
|
||||
):
|
||||
super().__init__()
|
||||
self.use_lvar = use_lvar
|
||||
@@ -116,7 +136,7 @@ class LatentEncoder(nn.Module):
|
||||
]
|
||||
)
|
||||
self._self_attention = Attention(
|
||||
hidden_dim, self_attention_type, n_heads=n_heads, dropout=attention_dropout
|
||||
hidden_dim, self_attention_type, n_heads=n_heads, dropout=attention_dropout, attention_layers=attention_layers
|
||||
)
|
||||
self._penultimate_layer = block_relu(hidden_dim, hidden_dim, dropout)
|
||||
self._mean = nn.Linear(hidden_dim, latent_dim)
|
||||
@@ -180,6 +200,7 @@ class DeterministicEncoder(nn.Module):
|
||||
dropout=0,
|
||||
attention_dropout=0,
|
||||
n_heads=8,
|
||||
attention_layers=2,
|
||||
):
|
||||
super().__init__()
|
||||
self._input_layer = NPBlockRelu2d(input_dim, hidden_dim, dropout)
|
||||
@@ -190,10 +211,10 @@ class DeterministicEncoder(nn.Module):
|
||||
]
|
||||
)
|
||||
self._self_attention = Attention(
|
||||
hidden_dim, self_attention_type, dropout=attention_dropout, n_heads=n_heads
|
||||
hidden_dim, self_attention_type, dropout=attention_dropout, n_heads=n_heads, attention_layers=attention_layers
|
||||
)
|
||||
self._cross_attention = Attention(
|
||||
hidden_dim, cross_attention_type, dropout=attention_dropout, n_heads=n_heads
|
||||
hidden_dim, cross_attention_type, dropout=attention_dropout, n_heads=n_heads, attention_layers=attention_layers
|
||||
)
|
||||
self._target_transform = nn.Linear(x_dim, hidden_dim)
|
||||
self._context_transform = nn.Linear(x_dim, hidden_dim)
|
||||
|
||||
Reference in New Issue
Block a user