mirror of
https://github.com/wassname/attentive-neural-processes.git
synced 2026-08-16 11:16:53 +08:00
working
This commit is contained in:
wassname
+22
-18
@@ -87,6 +87,7 @@ def run_trial(
|
||||
user_attrs: dict = {},
|
||||
MODEL_DIR: Path = Path("./lightning_logs"),
|
||||
plot_from_loader=plot_from_loader,
|
||||
number=None,
|
||||
):
|
||||
print(f"now run `tensorboard --logdir {MODEL_DIR}`")
|
||||
(MODEL_DIR / name).mkdir(parents=True, exist_ok=True)
|
||||
@@ -101,32 +102,35 @@ def run_trial(
|
||||
trial = optuna.trial.FixedTrial(params=params)
|
||||
trial = PL_MODEL_CLS.add_suggest(trial)
|
||||
|
||||
# Add auto number
|
||||
trial = add_number(trial, MODEL_DIR / name)
|
||||
if number is None:
|
||||
trial = add_number(trial, MODEL_DIR / name)
|
||||
else:
|
||||
trial.number = number
|
||||
|
||||
# Add user attributes
|
||||
trial._user_attrs.update(user_attrs)
|
||||
print('trial', trial, trial.params, trial.user_attrs)
|
||||
print('trial', trial.number, trial, trial.params, trial.user_attrs)
|
||||
|
||||
model, trainer = main(
|
||||
trial, PL_MODEL_CLS, name=name, MODEL_DIR=MODEL_DIR, train=False, prune=False
|
||||
)
|
||||
try:
|
||||
trainer.fit(model)
|
||||
except KeyboardInterrupt:
|
||||
print('KeyboardInterrupt, skipping rest of training')
|
||||
pass
|
||||
if number is None:
|
||||
try:
|
||||
trainer.fit(model)
|
||||
except KeyboardInterrupt:
|
||||
print('KeyboardInterrupt, skipping rest of training')
|
||||
pass
|
||||
|
||||
# Plot
|
||||
loader = model.val_dataloader()
|
||||
dset_test = loader.dataset
|
||||
label_names = dset_test.label_names
|
||||
plot_from_loader(model.val_dataloader(), model, i=670, title='overfit val 670')
|
||||
plt.show()
|
||||
plot_from_loader(model.train_dataloader(), model, i=670, title='overfit train 670')
|
||||
plt.show()
|
||||
plot_from_loader(model.test_dataloader(), model, i=670, title='overfit test 670')
|
||||
plt.show()
|
||||
# Plot
|
||||
loader = model.val_dataloader()
|
||||
dset_test = loader.dataset
|
||||
label_names = dset_test.label_names
|
||||
plot_from_loader(model.val_dataloader(), model, i=670, title='overfit val 670')
|
||||
plt.show()
|
||||
plot_from_loader(model.train_dataloader(), model, i=670, title='overfit train 670')
|
||||
plt.show()
|
||||
plot_from_loader(model.test_dataloader(), model, i=670, title='overfit test 670')
|
||||
plt.show()
|
||||
|
||||
# Load checkpoint
|
||||
checkpoints = sorted(Path(trainer.checkpoint_callback.dirpath).glob("*.ckpt"))
|
||||
|
||||
@@ -13,7 +13,7 @@ def init_random_seed(seed):
|
||||
np.random.seed(seed)
|
||||
torch.random.manual_seed(seed)
|
||||
torch.backends.cudnn.deterministic = True
|
||||
torch.backends.cudnn.benchmark = False
|
||||
torch.backends.cudnn.benchmark = False
|
||||
|
||||
|
||||
class PyTorchLightningPruningCallback(EarlyStopping):
|
||||
@@ -57,32 +57,6 @@ class PyTorchLightningPruningCallback(EarlyStopping):
|
||||
raise optuna.exceptions.TrialPruned(message)
|
||||
|
||||
|
||||
# class ObjectDict(dict):
|
||||
# """
|
||||
# Interface similar to an argparser
|
||||
# """
|
||||
|
||||
# def __init__(self):
|
||||
# pass
|
||||
|
||||
# def __setattr__(self, attr, value):
|
||||
# self[attr] = value
|
||||
# return self[attr]
|
||||
|
||||
# def __getattr__(self, attr):
|
||||
# if attr.startswith("_"):
|
||||
# # https://stackoverflow.com/questions/10364332/how-to-pickle-python-object-derived-from-dict
|
||||
# raise AttributeError
|
||||
# try:
|
||||
# return super().__getitem__(attr)
|
||||
# except KeyError:
|
||||
# # cPickle expects __getattr__ to raise AttributeError, not KeyError.
|
||||
# raise AttributeError(self._KeyErrorString(name))
|
||||
|
||||
# @property
|
||||
# def __dict__(self):
|
||||
# return dict(self)
|
||||
|
||||
class ObjectDict(dict):
|
||||
"""
|
||||
easy way to represent (hyper)parameters.
|
||||
@@ -102,6 +76,10 @@ class ObjectDict(dict):
|
||||
def copy(self, **extra_params):
|
||||
return ObjectDict(**self, **extra_params)
|
||||
|
||||
@property
|
||||
def __dict__(self):
|
||||
return dict(self)
|
||||
|
||||
|
||||
def hparams_power(hparams):
|
||||
"""Some value we want to go up in powers of 2
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -1,963 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# This notebook uses pytorch lightning & optuna & a Recurrent Attentive Neural Process for Sequential Data (RANPfSQ)\n",
|
||||
"\n",
|
||||
"This notebook trains an Attentional Neural Network on timeseries data from smartmeters.\n",
|
||||
"\n",
|
||||
"It uses pytorch lighting for the training loop. And Optuna for the hyperparameter optimisation.\n",
|
||||
"\n",
|
||||
"It also pushes results to the tensorboard hyperparameter dashboard for examination.\n",
|
||||
"\n",
|
||||
"- https://github.com/optuna/optuna/blob/master/examples/pytorch_lightning_simple.py\n",
|
||||
"\n",
|
||||
"- similar to https://arxiv.org/abs/1910.09323"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-02-15T11:09:58.775974Z",
|
||||
"start_time": "2020-02-15T11:09:43.765111Z"
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"Results on *Smartmeter* prediction\n",
|
||||
"\n",
|
||||
"|Model|val_loss|\n",
|
||||
"|--|--| \n",
|
||||
"|ANP-RNN|-1.27|\n",
|
||||
"|ANP-RNN_imp|-1.38|\n",
|
||||
"|ANP|-1.3|\n",
|
||||
"|ANP_impr|-1.2|\n",
|
||||
"|NP|-1.3|"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-04-11T10:55:16.549550Z",
|
||||
"start_time": "2020-04-11T10:55:14.083795Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import sys, re, os, itertools, functools, collections\n",
|
||||
"import pandas as pd\n",
|
||||
"import numpy as np\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"import collections\n",
|
||||
"from pathlib import Path\n",
|
||||
"from tqdm.auto import tqdm\n",
|
||||
"\n",
|
||||
"import optuna\n",
|
||||
"import pytorch_lightning as pl\n",
|
||||
"from optuna.integration import PyTorchLightningPruningCallback\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"import math\n",
|
||||
"%matplotlib inline\n",
|
||||
"%reload_ext autoreload\n",
|
||||
"%autoreload 2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-04-11T10:55:16.601275Z",
|
||||
"start_time": "2020-04-11T10:55:16.553482Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import torch\n",
|
||||
"from torch import nn\n",
|
||||
"import torch.nn.functional as F"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-04-11T10:55:16.723806Z",
|
||||
"start_time": "2020-04-11T10:55:16.604381Z"
|
||||
}
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/home/wassname/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/pytorch_lightning/core/decorators.py:13: UserWarning:\n",
|
||||
"\n",
|
||||
"data_loader decorator deprecated in 0.7.0. Will remove 0.9.0\n",
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"\n",
|
||||
"from neural_processes.data.smart_meter import collate_fns, SmartMeterDataSet, get_smartmeter_df\n",
|
||||
"from neural_processes.plot import plot_from_loader\n",
|
||||
"\n",
|
||||
"from neural_processes.dict_logger import DictLogger\n",
|
||||
"from neural_processes.utils import PyTorchLightningPruningCallback\n",
|
||||
"from neural_processes.train import main, objective, add_number, run_trial\n",
|
||||
"\n",
|
||||
"from neural_processes.models.neural_process import PL_NeuralProcess\n",
|
||||
"\n",
|
||||
"from neural_processes.models.transformer import PL_Transformer\n",
|
||||
"from neural_processes.models.transformer_seq2seq import TransformerSeq2Seq_PL\n",
|
||||
"\n",
|
||||
"from neural_processes.models.lstm_seqseq import LSTMSeq2Seq_PL\n",
|
||||
"from neural_processes.models.lstm_std import LSTM_PL_STD"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-04-11T10:55:16.777646Z",
|
||||
"start_time": "2020-04-11T10:55:16.726460Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Params\n",
|
||||
"device='cuda'\n",
|
||||
"use_logy=False"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-04-11T10:55:16.831267Z",
|
||||
"start_time": "2020-04-11T10:55:16.780549Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import logging\n",
|
||||
"logging.basicConfig(stream=sys.stdout, level=logging.INFO)\n",
|
||||
"logger = logging.getLogger(\"RANP.ipynb\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Load kaggle smart meter data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-04-11T10:55:25.924795Z",
|
||||
"start_time": "2020-04-11T10:55:16.834009Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"df_train, df_val, df_test = get_smartmeter_df()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"start_time": "2020-04-11T10:55:14.100Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Show split\n",
|
||||
"df_train['energy(kWh/hh)'].plot(label='train')\n",
|
||||
"df_val['energy(kWh/hh)'].plot(label='val')\n",
|
||||
"df_test['energy(kWh/hh)'].plot(label='test')\n",
|
||||
"plt.title('energy(kWh/hh)')\n",
|
||||
"plt.legend()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Default params"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"start_time": "2020-04-11T10:55:14.100Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"default_user_attrs = {\n",
|
||||
" 'context_in_target': False,\n",
|
||||
" 'x_dim': 17,\n",
|
||||
" 'y_dim': 1,\n",
|
||||
" 'vis_i': '670',\n",
|
||||
" 'num_workers': 3,\n",
|
||||
" 'num_context': 24*4,\n",
|
||||
" 'num_extra_target': 24*4,\n",
|
||||
" 'max_nb_epochs': 200,\n",
|
||||
" 'min_std': 0.005,\n",
|
||||
" 'grad_clip': 40,\n",
|
||||
" 'batch_size': 16,\n",
|
||||
" 'patience': 2\n",
|
||||
"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"start_time": "2020-04-11T10:55:14.100Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"results = {}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-04-11T01:56:38.106876Z",
|
||||
"start_time": "2020-04-11T01:56:38.100361Z"
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"# Models"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"start_time": "2020-04-11T10:55:14.100Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# trial, trainer, model = run_trial(\n",
|
||||
"# name=\"anp-rnn\",\n",
|
||||
"# params={\n",
|
||||
"# },\n",
|
||||
"# user_attrs = default_user_attrs,\n",
|
||||
"# PL_MODEL_CLS=PL_NeuralProcess\n",
|
||||
"# )"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-04-11T07:47:21.609656Z",
|
||||
"start_time": "2020-04-11T07:47:10.900Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"start_time": "2020-04-11T10:55:14.100Z"
|
||||
},
|
||||
"scrolled": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# trial, trainer, model = run_trial(name=\"TransformerSeq2Seq_PL\",\n",
|
||||
"# params={},\n",
|
||||
"# user_attrs=default_user_attrs,\n",
|
||||
"# PL_MODEL_CLS=TransformerSeq2Seq_PL)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"start_time": "2020-04-11T10:55:14.100Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# trial, trainer, model = run_trial(name=\"LSTMSeq2Seq_PL\",\n",
|
||||
"# params={},\n",
|
||||
"# user_attrs=default_user_attrs,\n",
|
||||
"# PL_MODEL_CLS=LSTMSeq2Seq_PL)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"start_time": "2020-04-11T10:55:14.100Z"
|
||||
},
|
||||
"scrolled": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# trial, trainer, model = run_trial(name=\"PL_Transformer\",\n",
|
||||
"# params={\n",
|
||||
"# },\n",
|
||||
"# user_attrs={**default_user_attrs, 'context_in_target': False},\n",
|
||||
"# PL_MODEL_CLS=PL_Transformer)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"start_time": "2020-04-11T10:55:14.200Z"
|
||||
},
|
||||
"scrolled": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"trial, trainer, model = run_trial(name=\"LSTM_PL_STD\",\n",
|
||||
" params={},\n",
|
||||
" user_attrs=default_user_attrs,\n",
|
||||
" PL_MODEL_CLS=LSTM_PL_STD)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## ANP-RNN"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"start_time": "2020-04-11T10:55:14.200Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"trial, trainer, model = run_trial(\n",
|
||||
" name=\"anp-rnn\",\n",
|
||||
" params={\n",
|
||||
" 'det_enc_cross_attn_type': 'multihead',\n",
|
||||
" 'det_enc_self_attn_type': 'uniform',\n",
|
||||
" 'latent_enc_self_attn_type': 'uniform',\n",
|
||||
" 'use_deterministic_path': True,\n",
|
||||
" 'use_rnn': True,\n",
|
||||
" 'use_lvar': False,\n",
|
||||
" },\n",
|
||||
" user_attrs = default_user_attrs,\n",
|
||||
" PL_MODEL_CLS=PL_NeuralProcess\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"start_time": "2020-04-11T10:55:14.200Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"name = trainer.logger.name\n",
|
||||
"results[f\"{name}_{trial.number}\"]=dict(\n",
|
||||
" **trainer.logger.metrics[-1],\n",
|
||||
" **(trainer.logger.metrics[-2] if len(trainer.logger.metrics)>1 else {})\n",
|
||||
")\n",
|
||||
"results[f\"{name}_{trial.number}\"]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-04-11T05:46:41.864057Z",
|
||||
"start_time": "2020-04-11T05:46:41.787999Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-02-15T07:40:36.819548Z",
|
||||
"start_time": "2020-02-15T07:40:07.785566Z"
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"## ANP-RNN 2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"start_time": "2020-04-11T10:55:14.200Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"trial, trainer, model = run_trial(\n",
|
||||
" name=\"anp-rnn2\",\n",
|
||||
" params={ \n",
|
||||
" 'det_enc_self_attn_type': 'multihead',\n",
|
||||
" 'latent_enc_self_attn_type': 'multihead',\n",
|
||||
" 'use_deterministic_path': False,\n",
|
||||
" 'use_rnn': True,\n",
|
||||
" 'use_lvar': False,\n",
|
||||
" },\n",
|
||||
" user_attrs=default_user_attrs,\n",
|
||||
" PL_MODEL_CLS=PL_NeuralProcess)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"start_time": "2020-04-11T10:55:14.200Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"name = trainer.logger.name\n",
|
||||
"results[f\"{name}_{trial.number}\"]=dict(\n",
|
||||
" **trainer.logger.metrics[-1],\n",
|
||||
" **(trainer.logger.metrics[-2] if len(trainer.logger.metrics)>1 else {})\n",
|
||||
")\n",
|
||||
"results[f\"{name}_{trial.number}\"]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-02-15T07:00:04.268233Z",
|
||||
"start_time": "2020-02-15T06:57:58.700Z"
|
||||
},
|
||||
"scrolled": true
|
||||
},
|
||||
"source": [
|
||||
"## ANP"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"start_time": "2020-04-11T10:55:14.200Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"trial, trainer, model = run_trial(name=\"anp_c\",\n",
|
||||
" params={\n",
|
||||
" 'det_enc_cross_attn_type': 'multihead',\n",
|
||||
" 'det_enc_self_attn_type': 'multihead',\n",
|
||||
" 'latent_enc_self_attn_type': 'multihead',\n",
|
||||
" 'use_deterministic_path': False,\n",
|
||||
" 'use_lvar': False,\n",
|
||||
" },\n",
|
||||
" user_attrs=default_user_attrs,\n",
|
||||
" PL_MODEL_CLS=PL_NeuralProcess)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"start_time": "2020-04-11T10:55:14.200Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"name = trainer.logger.name\n",
|
||||
"results[f\"{name}_{trial.number}\"]=dict(\n",
|
||||
" **trainer.logger.metrics[-1],\n",
|
||||
" **(trainer.logger.metrics[-2] if len(trainer.logger.metrics)>1 else {})\n",
|
||||
")\n",
|
||||
"results[f\"{name}_{trial.number}\"]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## NP"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"start_time": "2020-04-11T10:55:14.200Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"trial, trainer, model = run_trial(name=\"np\",\n",
|
||||
" params={\n",
|
||||
" 'det_enc_cross_attn_type': 'uniform',\n",
|
||||
" 'det_enc_self_attn_type': 'uniform',\n",
|
||||
" 'latent_enc_self_attn_type': 'uniform',\n",
|
||||
" 'use_deterministic_path': False,\n",
|
||||
" 'use_lvar': False,\n",
|
||||
" },\n",
|
||||
" user_attrs=default_user_attrs,\n",
|
||||
" PL_MODEL_CLS=PL_NeuralProcess)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"start_time": "2020-04-11T10:55:14.200Z"
|
||||
},
|
||||
"scrolled": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"name = trainer.logger.name\n",
|
||||
"results[f\"{name}_{trial.number}\"]=dict(\n",
|
||||
" **trainer.logger.metrics[-1],\n",
|
||||
" **(trainer.logger.metrics[-2] if len(trainer.logger.metrics)>1 else {})\n",
|
||||
")\n",
|
||||
"results[f\"{name}_{trial.number}\"]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-04-11T01:24:32.163872Z",
|
||||
"start_time": "2020-04-11T01:24:32.160011Z"
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"## Transformer"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"start_time": "2020-04-11T10:55:14.200Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"trial, trainer, model = run_trial(name=\"PL_Transformer\",\n",
|
||||
" params={\n",
|
||||
" },\n",
|
||||
" user_attrs={**default_user_attrs, 'context_in_target': False},\n",
|
||||
" PL_MODEL_CLS=PL_Transformer)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"start_time": "2020-04-11T10:55:14.200Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# %debug"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"start_time": "2020-04-11T10:55:14.200Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"name = trainer.logger.name\n",
|
||||
"results[f\"{name}_{trial.number}\"]=dict(\n",
|
||||
" **trainer.logger.metrics[-1],\n",
|
||||
" **(trainer.logger.metrics[-2] if len(trainer.logger.metrics)>1 else {})\n",
|
||||
")\n",
|
||||
"results[f\"{name}_{trial.number}\"]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Transformer Seq2Seq"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"start_time": "2020-04-11T10:55:14.200Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"trial, trainer, model = run_trial(name=\"TransformerSeq2Seq_PL\",\n",
|
||||
" params={},\n",
|
||||
" user_attrs=default_user_attrs,\n",
|
||||
" PL_MODEL_CLS=TransformerSeq2Seq_PL)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"start_time": "2020-04-11T10:55:14.300Z"
|
||||
},
|
||||
"scrolled": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"name = trainer.logger.name\n",
|
||||
"results[f\"{name}_{trial.number}\"]=dict(\n",
|
||||
" **trainer.logger.metrics[-1],\n",
|
||||
" **(trainer.logger.metrics[-2] if len(trainer.logger.metrics)>1 else {})\n",
|
||||
")\n",
|
||||
"results[f\"{name}_{trial.number}\"]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## LSTM"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"start_time": "2020-04-11T10:55:14.300Z"
|
||||
},
|
||||
"scrolled": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"trial, trainer, model = run_trial(name=\"LSTM_PL_STD\",\n",
|
||||
" params={},\n",
|
||||
" user_attrs=default_user_attrs,\n",
|
||||
" PL_MODEL_CLS=LSTM_PL_STD)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"start_time": "2020-04-11T10:55:14.300Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"name = trainer.logger.name\n",
|
||||
"results[f\"{name}_{trial.number}\"]=dict(\n",
|
||||
" **trainer.logger.metrics[-1],\n",
|
||||
" **(trainer.logger.metrics[-2] if len(trainer.logger.metrics)>1 else {})\n",
|
||||
")\n",
|
||||
"results[f\"{name}_{trial.number}\"]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## LSTM seq2seq"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"start_time": "2020-04-11T10:55:14.300Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"trial, trainer, model = run_trial(name=\"LSTMSeq2Seq_PL\",\n",
|
||||
" params={},\n",
|
||||
" user_attrs=default_user_attrs,\n",
|
||||
" PL_MODEL_CLS=LSTMSeq2Seq_PL)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"start_time": "2020-04-11T10:55:14.300Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"name = trainer.logger.name\n",
|
||||
"results[f\"{name}_{trial.number}\"]=dict(\n",
|
||||
" **trainer.logger.metrics[-1],\n",
|
||||
" **(trainer.logger.metrics[-2] if len(trainer.logger.metrics)>1 else {})\n",
|
||||
")\n",
|
||||
"results[f\"{name}_{trial.number}\"]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-01-27T04:27:28.582523Z",
|
||||
"start_time": "2020-01-27T04:27:28.445326Z"
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"# Hyperparam"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"start_time": "2020-04-11T10:55:14.300Z"
|
||||
},
|
||||
"scrolled": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import argparse \n",
|
||||
"\n",
|
||||
"parser = argparse.ArgumentParser(description='PyTorch Lightning example.')\n",
|
||||
"parser.add_argument('--pruning', '-p', action='store_true',\n",
|
||||
" help='Activate the pruning feature. `MedianPruner` stops unpromising '\n",
|
||||
" 'trials at the early stages of training.')\n",
|
||||
"args = parser.parse_args(['-p'])\n",
|
||||
"\n",
|
||||
"pruner = optuna.pruners.MedianPruner(n_warmup_steps=1, n_startup_trials=20) if args.pruning else optuna.pruners.NopPruner()\n",
|
||||
"pruner = optuna.pruners.PercentilePruner(75.0)\n",
|
||||
"name = 'anp-rnn1'\n",
|
||||
"study = optuna.create_study(direction='minimize', pruner=pruner, storage=f'sqlite:///optuna_result/{name}.db', study_name=name, load_if_exists=True)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"start_time": "2020-04-11T10:55:14.300Z"
|
||||
},
|
||||
"scrolled": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"study.optimize(objective, n_trials=200, timeout=pd.Timedelta('3d').total_seconds())"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-02-16T04:12:26.187191Z",
|
||||
"start_time": "2020-02-16T04:12:26.081801Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"start_time": "2020-04-11T10:55:14.300Z"
|
||||
},
|
||||
"scrolled": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"\n",
|
||||
"print('Number of finished trials: {}'.format(len(study.trials)))\n",
|
||||
"\n",
|
||||
"print('Best trial:')\n",
|
||||
"trial = study.best_trial\n",
|
||||
"\n",
|
||||
"print(' Value: {}'.format(trial.value))\n",
|
||||
"\n",
|
||||
"print(' Params: ')\n",
|
||||
"for key, value in trial.params.items():\n",
|
||||
" print(' {}: {}'.format(key, value))\n",
|
||||
"\n",
|
||||
"# shutil.rmtree(MODEL_DIR)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-01-27T07:45:29.794794Z",
|
||||
"start_time": "2020-01-27T07:45:29.733450Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2020-01-27T07:35:33.470251Z",
|
||||
"start_time": "2020-01-27T07:34:47.800Z"
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"## View\n",
|
||||
"\n",
|
||||
"TODO"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"start_time": "2020-04-11T10:55:14.300Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"df = study.trials_dataframe(attrs=('number', 'value', 'params', 'state'))\n",
|
||||
"df.sort_values('value')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"start_time": "2020-04-11T10:55:14.300Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"df.sort_values('value').head(17).T"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"file_extension": ".py",
|
||||
"kernelspec": {
|
||||
"display_name": "jup3.7.3",
|
||||
"language": "python",
|
||||
"name": "jup3.7.3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.3"
|
||||
},
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"npconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"toc": {
|
||||
"base_numbering": 1,
|
||||
"nav_menu": {},
|
||||
"number_sections": true,
|
||||
"sideBar": true,
|
||||
"skip_h1_title": false,
|
||||
"title_cell": "Table of Contents",
|
||||
"title_sidebar": "Contents",
|
||||
"toc_cell": false,
|
||||
"toc_position": {
|
||||
"height": "calc(100% - 180px)",
|
||||
"left": "10px",
|
||||
"top": "150px",
|
||||
"width": "384px"
|
||||
},
|
||||
"toc_section_display": true,
|
||||
"toc_window_display": true
|
||||
},
|
||||
"version": 3
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+5055
File diff suppressed because one or more lines are too long
@@ -0,0 +1,11 @@
|
||||
import neural_processes.utils
|
||||
import pickle, json
|
||||
import tempfile
|
||||
|
||||
def test_obectdict(tmpdir):
|
||||
o = neural_processes.utils.ObjectDict(z=1, b=4, test="g", w=0)
|
||||
pickle.dump(o, open(tmpdir+'/test.pkl', 'wb'))
|
||||
o2 = pickle.load(open(tmpdir+'/test.pkl', 'rb'))
|
||||
|
||||
o3 = json.loads(json.dumps(o))
|
||||
print(o, o2, o3)
|
||||
Reference in New Issue
Block a user