wassname/attentive-neural-processes
1
0 Fork 0
mirror of https://github.com/wassname/attentive-neural-processes.git synced 2026-10-02 12:01:31 +08:00
Code Issues Packages Projects Releases Wiki Activity

misc

Browse Source
This commit is contained in:
wassname committed 2020-04-11 15:37:55 +08:00
1 parent 4957e04bdc
commit 0a393b4e02
11 files changed
+606 -329
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

neural_processes/models/lstm_seqseq.py
+61 -41
View File
@@ -8,7 +8,11 @@ from torch.nn import functional as F
from torch.utils.data import DataLoader
from torchvision.datasets import MNIST
from test_tube import Experiment, HyperOptArgumentParser
from neural_processes.data.smart_meter import collate_fns, SmartMeterDataSet, get_smartmeter_df
from neural_processes.data.smart_meter import (
collate_fns,
SmartMeterDataSet,
get_smartmeter_df,
)
import torchvision.transforms as transforms
from neural_processes.plot import plot_from_loader_to_tensor, plot_from_loader
from argparse import ArgumentParser
@@ -20,20 +24,23 @@ import torch
import io
import PIL
from torchvision.transforms import ToTensor
from neural_processes.modules import BatchNormSequence
from neural_processes.data.smart_meter import get_smartmeter_df
from neural_processes.utils import ObjectDict
from neural_processes.lightning import PL_Seq2Seq
from ..logger import logger
from ..utils import hparams_power
class Seq2SeqNet(nn.Module):
def __init__(self, hparams, _min_std = 0.05):
def __init__(self, hparams, _min_std=0.05):
super().__init__()
hparams = hparams_power(hparams)
self.hparams = hparams
self._min_std = _min_std
self.norm_input = BatchNormSequence(self.hparams.input_size)
self.encoder = nn.LSTM(
input_size=self.hparams.input_size,
@@ -43,7 +50,9 @@ class Seq2SeqNet(nn.Module):
bidirectional=self.hparams.bidirectional,
dropout=self.hparams.lstm_dropout,
)
self.multihead_attn = nn.MultiheadAttention(self.hparams.hidden_size, num_heads=8)
self.multihead_attn = nn.MultiheadAttention(
self.hparams.hidden_size, num_heads=8
)
self.norm_target = BatchNormSequence(self.hparams.input_size_decoder)
self.decoder = nn.LSTM(
@@ -54,9 +63,8 @@ class Seq2SeqNet(nn.Module):
bidirectional=self.hparams.bidirectional,
dropout=self.hparams.lstm_dropout,
)
self.hidden_out_size = (
self.hparams.hidden_size
* (self.hparams.bidirectional + 1)
self.hidden_out_size = self.hparams.hidden_size * (
self.hparams.bidirectional + 1
)
self.mean = nn.Linear(self.hidden_out_size, self.hparams.output_size)
self.std = nn.Linear(self.hidden_out_size, self.hparams.output_size)
@@ -76,74 +84,86 @@ class Seq2SeqNet(nn.Module):
# context_x, d_encoded, target_x = k, v, q
# query, key, value = target_x, context_x, d_encoded
attn_output, _ = self.multihead_attn(h_out.permute(1, 0, 2), h_out.permute(1, 0, 2), h_out.permute(1, 0, 2))
attn_output, _ = self.multihead_attn(
h_out.permute(1, 0, 2), h_out.permute(1, 0, 2), h_out.permute(1, 0, 2)
)
h_out = attn_output.permute(1, 0, 2).contiguous()
attn_output, _ = self.multihead_attn(cell.permute(1, 0, 2), cell.permute(1, 0, 2), cell.permute(1, 0, 2))
attn_output, _ = self.multihead_attn(
cell.permute(1, 0, 2), cell.permute(1, 0, 2), cell.permute(1, 0, 2)
)
cell = attn_output.permute(1, 0, 2).contiguous()
outputs, (_, _) = self.decoder(target_x, (h_out, cell))
# output = [batch size, seq len, hid dim * n directions]
# outputs: [B, T, num_direction * H]
mean = self.mean(outputs)
log_sigma = self.std(outputs)
if self._use_lvar:
log_sigma = torch.clamp(log_sigma, math.log(self._min_std), -math.log(self._min_std))
log_sigma = torch.clamp(
log_sigma, math.log(self._min_std), -math.log(self._min_std)
)
sigma = torch.exp(log_sigma)
else:
sigma = self._min_std + (1 - self._min_std) * F.softplus(log_sigma)
y_dist=torch.distributions.Normal(mean, sigma)
y_dist = torch.distributions.Normal(mean, sigma)
# Loss
loss_mse = loss_p = None
if target_y is not None:
loss_mse = F.mse_loss(mean, target_y, reduction='none')
loss_mse = F.mse_loss(mean, target_y, reduction="none")
if self._use_lvar:
loss_p = -log_prob_sigma(target_y, mean, log_sigma)
else:
loss_p = -y_dist.log_prob(target_y).mean(-1)
if self.hparams["context_in_target"]:
loss_p[:context_x.size(1)] /= 100
loss_mse[:context_x.size(1)] /= 100
loss_p[: context_x.size(1)] /= 100
loss_mse[: context_x.size(1)] /= 100
# # Don't catch loss on context window
# mean = mean[:, self.hparams.num_context:]
# log_sigma = log_sigma[:, self.hparams.num_context:]
y_pred = y_dist.rsample if self.training else y_dist.loc
return y_pred, dict(loss_p=loss_p.mean(), loss_mse=loss_mse.mean()), dict(log_sigma=log_sigma, dist=y_dist)
return (
y_pred,
dict(loss_p=loss_p.mean(), loss_mse=loss_mse.mean()),
dict(log_sigma=log_sigma, dist=y_dist),
)
class LSTMSeq2Seq_PL(PL_Seq2Seq):
def __init__(self, hparams,
MODEL_CLS=Seq2SeqNet, **kwargs):
super().__init__(hparams,
MODEL_CLS=MODEL_CLS, **kwargs)
def __init__(self, hparams, MODEL_CLS=Seq2SeqNet, **kwargs):
super().__init__(hparams, MODEL_CLS=MODEL_CLS, **kwargs)
DEFAULT_ARGS = {
"agg": "mean",
"lstm_dropout": 0.22,
"hidden_size_power": 4.0,
"learning_rate": 0.001,
"lstm_layers": 4,
'bidirectional': False
}
DEFAULT_ARGS = {'agg': 'mean', 'lstm_dropout': 0.12013231612195126, 'hidden_out_size_power': 4.0, 'hidden_size_power': 7.0, 'learning_rate': 0.0022924639229335475, 'nhead_power': 2.0, 'nlayers_power': 4.0}
@staticmethod
def add_suggest(trial):
# TODO make label name configurable
# TODO make data source configurable
trial.suggest_loguniform("learning_rate", 1e-5, 1e-2)
trial.suggest_uniform("lstm_dropout", 0, 0.75)
trial.suggest_categorical("hidden_size", [1, 2, 4, 8, 16, 32, 64, 128, 256, 512])
trial.suggest_categorical("lstm_layers", [1, 2, 4, 8])
trial.suggest_categorical("bidirectional", [False, True])
trial.suggest_discrete_uniform("hidden_size_power", 3, 9, 1)
trial.suggest_int("lstm_layers", 1, 8)
trial.suggest_categorical("bidirectional", [False, True])
trial._user_attrs = {
'batch_size': 16,
'grad_clip': 40,
'max_nb_epochs': 200,
'num_workers': 4,
'num_extra_target': 24*4,
'vis_i': '670',
'num_context': 24*4,
'input_size': 18,
'input_size_decoder': 17,
'context_in_target': True,
'output_size': 1
"batch_size": 16,
"grad_clip": 40,
"max_nb_epochs": 200,
"num_workers": 4,
"num_extra_target": 24 * 4,
"vis_i": "670",
"num_context": 24 * 4,
"input_size": 18,
"input_size_decoder": 17,
"context_in_target": False,
"output_size": 1,
}
return trial
neural_processes/models/lstm_std.py
+38 -31
View File
@@ -24,15 +24,14 @@ from neural_processes.data.smart_meter import get_smartmeter_df
from neural_processes.utils import ObjectDict
from ..lightning import PL_Seq2Seq
from torch.utils.data._utils.collate import default_collate
def collate_fn(batch, sample=None):
return default_collate(batch)
from ..logger import logger
from ..utils import hparams_power
class LSTMNet(nn.Module):
def __init__(self, hparams, _min_std = 0.05):
def __init__(self, hparams, _min_std=0.05):
super().__init__()
hparams = hparams_power(hparams)
self.hparams = hparams
self._min_std = _min_std
@@ -44,9 +43,8 @@ class LSTMNet(nn.Module):
bidirectional=self.hparams.bidirectional,
dropout=self.hparams.lstm_dropout,
)
self.hidden_out_size = (
self.hparams.hidden_size
* (self.hparams.bidirectional + 1)
self.hidden_out_size = self.hparams.hidden_size * (
self.hparams.bidirectional + 1
)
self.mean = nn.Linear(self.hidden_out_size, 1)
self.std = nn.Linear(self.hidden_out_size, 1)
@@ -62,7 +60,12 @@ class LSTMNet(nn.Module):
loss = None
if target_y is not None:
loss = F.mse_loss(y_pred * loss_scale, y[:, -steps:, :] * loss_scale, reduction='none') / loss_scale
loss = (
F.mse_loss(
y_pred * loss_scale, y[:, -steps:, :] * loss_scale, reduction="none"
)
/ loss_scale
)
assert torch.isfinite(loss)
@@ -70,35 +73,39 @@ class LSTMNet(nn.Module):
class LSTM_PL_STD(PL_Seq2Seq):
def __init__(self, hparams,
MODEL_CLS=LSTMNet, **kwargs):
super().__init__(hparams,
MODEL_CLS=MODEL_CLS, **kwargs)
def __init__(self, hparams, MODEL_CLS=LSTMNet, **kwargs):
super().__init__(hparams, MODEL_CLS=MODEL_CLS, **kwargs)
DEFAULT_ARGS = {'bidirectional': False, 'hidden_size_power': 4, 'learning_rate': 0.0010825329363784934, 'lstm_dropout': 0.3905792111699782, 'lstm_layers': 4}
DEFAULT_ARGS = {
"bidirectional": False,
"hidden_size_power": 4,
"learning_rate": 0.001,
"lstm_dropout": 0.39,
"lstm_layers": 4,
"bidirectional": False,
}
@staticmethod
def add_suggest(trial):
trial.suggest_loguniform("learning_rate", 1e-5, 1e-2)
trial.suggest_uniform("lstm_dropout", 0, 0.75)
trial.suggest_categorical("hidden_size", [1, 2, 4, 8, 16, 32, 64, 128, 256, 512])
trial.suggest_categorical("lstm_layers", [1, 2, 4, 8])
trial.suggest_uniform("lstm_dropout", 0, 0.85)
trial.suggest_discrete_uniform("hidden_size_power", 3, 9, 1)
trial.suggest_int("lstm_layers", 1, 8)
trial.suggest_categorical("bidirectional", [False, True])
# constants
trial._user_attrs = {
'batch_size': 16,
'grad_clip': 40,
'max_nb_epochs': 200,
'num_workers': 4,
'num_extra_target': 24*4,
'vis_i': '670',
'num_context': 24*4,
'input_size': 18,
'input_size_decoder': 17,
'context_in_target': True,
'output_size': 1,
'patience': 3,
"batch_size": 16,
"grad_clip": 40,
"max_nb_epochs": 200,
"num_workers": 4,
"num_extra_target": 24 * 4,
"vis_i": "670",
"num_context": 24 * 4,
"input_size": 18,
"input_size_decoder": 17,
"context_in_target": False,
"output_size": 1,
"patience": 3,
}
return trial
neural_processes/models/neural_process/lightning.py
+21 -14
View File
@@ -21,14 +21,14 @@ class PL_NeuralProcess(PL_Seq2Seq):
'det_enc_cross_attn_type': 'multihead',
'det_enc_self_attn_type': 'uniform',
'dropout': 0,
'hidden_dim': 128,
'latent_dim': 128,
'hidden_dim_power': 7,
'latent_dim_power': 7,
'latent_enc_self_attn_type': 'uniform',
'learning_rate': 0.002,
'n_decoder_layers': 4,
'n_det_encoder_layers': 4,
'n_latent_encoder_layers': 2,
'num_heads': 8,
'n_latent_encoder_layers_power': 1,
'num_heads_power': 3,
'use_deterministic_path': True,
'use_lvar': True,
'use_self_attn': True,
@@ -37,16 +37,23 @@ class PL_NeuralProcess(PL_Seq2Seq):
@staticmethod
def add_suggest(trial):
trial.suggest_loguniform("learning_rate", 1e-5, 1e-2)
trial.suggest_categorical("hidden_dim", [8*2**i for i in range(8)])
trial.suggest_categorical("latent_dim", [8*2**i for i in range(8)])
trial.suggest_loguniform("learning_rate", 1e-6, 1e-2)
trial.suggest_int("attention_layers", 1, 4)
trial.suggest_categorical("n_latent_encoder_layers", [1, 2, 4, 6, 8, 12])
trial.suggest_categorical("n_det_encoder_layers", [1, 2, 4, 6, 8, 12])
trial.suggest_categorical("n_decoder_layers", [1, 2, 4, 6, 8, 12])
trial.suggest_int("num_heads", 8, 8)
trial.suggest_discrete_uniform("num_heads_power", 2, 4, 1)
trial.suggest_discrete_uniform(
"hidden_dim_power", 3, 11, 1
)
trial.suggest_discrete_uniform(
"latent_dim_power", 3, 11, 1
)
trial.suggest_int(
"n_latent_encoder_layers", 1, 11
)
trial.suggest_int("n_latent_encoder_layers", 1, 12)
trial.suggest_int("n_det_encoder_layers", 1, 12)
trial.suggest_int("n_decoder_layers", 1, 12)
trial.suggest_uniform("dropout", 0, 0.9)
trial.suggest_uniform("attention_dropout", 0, 0.9)
@@ -72,7 +79,7 @@ class PL_NeuralProcess(PL_Seq2Seq):
'vis_i': '670',
'num_extra_target': 24*4,
'x_dim': 18,
'context_in_target': True,
'context_in_target': False,
'y_dim': 1,
'patience': 3,
'min_std': 0.005,
neural_processes/models/neural_process/model.py
+2 -1
View File
@@ -6,7 +6,7 @@ import math
from neural_processes.modules import BatchNormSequence, BatchMLP, Attention, LSTMBlock
from neural_processes.utils import kl_loss_var, log_prob_sigma
from neural_processes.utils import hparams_power
class LatentEncoder(nn.Module):
def __init__(
@@ -195,6 +195,7 @@ class NeuralProcess(nn.Module):
@staticmethod
def FROM_HPARAMS(hparams):
hparams = hparams_power(hparams)
return NeuralProcess(**hparams)
def __init__(self,
neural_processes/models/transformer.py
+9 -6
View File
@@ -9,15 +9,15 @@ from torch.nn import functional as F
from torch.utils.data import DataLoader
from neural_processes.lightning import PL_Seq2Seq
from ..logger import logger
class NetTransformer(nn.Module):
def __init__(self, hparams):
super().__init__()
hparams["nlayers"] = int(2 ** hparams["nlayers_power"])
hparams["hidden_size"] = int(2**hparams["hidden_size_power"])
hparams["hidden_out_size"] = int(2 ** hparams["hidden_out_size_power"])
hparams["nhead"] = int(2 ** hparams["nhead_power"])
logger.debug(f"{type(self)} hparams {hparams}")
for k in hparams.keys():
if k.endswith("_power"):
k_new = k.replace("_power", "")
hparams[k_new] = int(2 ** hparams[k])
self.hparams = hparams
hidden_out_size = self.hparams.hidden_out_size
@@ -148,7 +148,7 @@ class PL_Transformer(PL_Seq2Seq):
)
trial.suggest_discrete_uniform("hidden_out_size_power", 2, 9, 1)
trial.suggest_discrete_uniform("nhead_power", 1, 4, 1)
trial.suggest_discrete_uniform("nlayers_power", 1, 5, 1)
trial.suggest_int("nlayers_power", 1, 12)
user_attrs_default = {
"batch_size": 16,
@@ -159,6 +159,9 @@ class PL_Transformer(PL_Seq2Seq):
"input_size": 6,
"output_size": 1,
"label_steps": 24,
"nan_value": -99.9,
'context_in_target': False,
'patience': 3,
}
[trial.set_user_attr(k, v) for k, v in user_attrs_default.items()]
[trial.set_user_attr(k, v) for k, v in user_attrs.items()]
neural_processes/models/transformer_seq2seq.py
+11 -6
View File
@@ -27,10 +27,13 @@ from neural_processes.modules import BatchNormSequence
from neural_processes.utils import ObjectDict
from neural_processes.lightning import PL_Seq2Seq
from ..logger import logger
from ..utils import hparams_power
class TransformerSeq2SeqNet(nn.Module):
def __init__(self, hparams, _min_std = 0.05):
super().__init__()
hparams = hparams_power(hparams)
self.hparams = hparams
self._min_std = _min_std
@@ -134,7 +137,7 @@ class TransformerSeq2Seq_PL(PL_Seq2Seq):
super().__init__(hparams,
MODEL_CLS=MODEL_CLS, **kwargs)
DEFAULT_ARGS = {'agg': 'mean', 'attention_dropout': 0.12013231612195126, 'hidden_out_size_power': 4.0, 'hidden_size_power': 7.0, 'learning_rate': 0.0022924639229335475, 'nhead_power': 2.0, 'nlayers_power': 4.0}
DEFAULT_ARGS = {'agg': 'mean', 'attention_dropout': 0.12, 'hidden_out_size_power': 4, 'hidden_size_power': 7, 'learning_rate': 0.0023, 'nhead_power': 2, 'nlayers_power': 4}
@staticmethod
def add_suggest(trial: optuna.Trial):
@@ -155,10 +158,12 @@ class TransformerSeq2Seq_PL(PL_Seq2Seq):
"""
trial.suggest_loguniform("learning_rate", 1e-6, 1e-2)
trial.suggest_uniform("attention_dropout", 0, 0.75)
trial.suggest_categorical("hidden_size", [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048])
trial.suggest_categorical("hidden_out_size", [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048])
trial.suggest_categorical("nlayers", [1, 2, 4, 6, 8, 16, 32])
trial.suggest_categorical("nhead", [1, 2, 8, 16])
trial.suggest_discrete_uniform(
"hidden_size_power", 2, 10, 1
)
trial.suggest_discrete_uniform("hidden_out_size_power", 2, 9, 1)
trial.suggest_discrete_uniform("nhead_power", 1, 4, 1)
trial.suggest_int("nlayers", 1, 12)
trial._user_attrs = {
'batch_size': 16,
@@ -170,7 +175,7 @@ class TransformerSeq2Seq_PL(PL_Seq2Seq):
'num_context': 24*4,
'input_size': 18,
'input_size_decoder': 17,
'context_in_target': True,
'context_in_target': False,
'output_size': 1,
'patience': 3,
}
neural_processes/train.py
+1 -1
View File
@@ -106,7 +106,7 @@ def run_trial(
# Add user attributes
trial._user_attrs.update(user_attrs)
print('trial', trial)
print('trial', trial, trial.params, trial.user_attrs)
model, trainer = main(
trial, PL_MODEL_CLS, name=name, MODEL_DIR=MODEL_DIR, train=False, prune=False
neural_processes/utils.py
+49 -14
View File
@@ -55,28 +55,63 @@ class PyTorchLightningPruningCallback(EarlyStopping):
raise optuna.exceptions.TrialPruned(message)
# class ObjectDict(dict):
# """
# Interface similar to an argparser
# """
# def __init__(self):
# pass
# def __setattr__(self, attr, value):
# self[attr] = value
# return self[attr]
# def __getattr__(self, attr):
# if attr.startswith("_"):
# # https://stackoverflow.com/questions/10364332/how-to-pickle-python-object-derived-from-dict
# raise AttributeError
# try:
# return super().__getitem__(attr)
# except KeyError:
# # cPickle expects __getattr__ to raise AttributeError, not KeyError.
# raise AttributeError(self._KeyErrorString(name))
# @property
# def __dict__(self):
# return dict(self)
class ObjectDict(dict):
"""
Interface similar to an argparser
easy way to represent (hyper)parameters.
https://stackoverflow.com/a/50613966/221742
"""
__getattr__ = dict.__getitem__
__setattr__ = dict.__setitem__
__delattr__ = dict.__delitem__
def __init__(self):
pass
def __getstate__(self):
return self
def __setattr__(self, attr, value):
self[attr] = value
return self[attr]
def __setstate__(self, state):
self.update(state)
def __getattr__(self, attr):
if attr.startswith("_"):
# https://stackoverflow.com/questions/10364332/how-to-pickle-python-object-derived-from-dict
raise AttributeError
return dict(self)[attr]
def copy(self, **extra_params):
return ObjectDict(**self, **extra_params)
@property
def __dict__(self):
return dict(self)
def hparams_power(hparams):
"""Some value we want to go up in powers of 2
So any hyper param that ends in power will be used this way.
"""
hparams_old = hparams.copy()
for k in hparams_old.keys():
if k.endswith("_power"):
k_new = k.replace("_power", "")
hparams[k_new] = int(2 ** hparams[k])
return hparams
def log_prob_sigma(value, loc, log_scale):
"""A slightly more stable (not confirmed yet) log prob taking in log_var instead of scale.
requirements.txt
+4
View File
@@ -1,3 +1,7 @@
# local package
-e .
# external requirements
torch>=1.3.0
tqdm
pandas
setup.py
+10
View File
@@ -0,0 +1,10 @@
from setuptools import find_packages, setup
setup(
name='neural_processes',
packages=find_packages(),
version='0.1.0',
description='Attentive Neural Processes',
author='wassname',
license='Apachev2',
)
smartmeters-ANP-RNN.ipynb
+400 -215
View File
@@ -42,8 +42,8 @@
"execution_count": 1,
"metadata": {
"ExecuteTime": {
"end_time": "2020-04-11T05:31:36.604028Z",
"start_time": "2020-04-11T05:31:34.179270Z"
"end_time": "2020-04-11T07:33:41.440376Z",
"start_time": "2020-04-11T07:33:39.040281Z"
}
},
"outputs": [],
@@ -72,8 +72,8 @@
"execution_count": 2,
"metadata": {
"ExecuteTime": {
"end_time": "2020-04-11T05:31:36.672837Z",
"start_time": "2020-04-11T05:31:36.609199Z"
"end_time": "2020-04-11T07:33:41.490347Z",
"start_time": "2020-04-11T07:33:41.442977Z"
}
},
"outputs": [],
@@ -88,8 +88,8 @@
"execution_count": 3,
"metadata": {
"ExecuteTime": {
"end_time": "2020-04-11T05:31:36.811071Z",
"start_time": "2020-04-11T05:31:36.675979Z"
"end_time": "2020-04-11T07:33:41.618208Z",
"start_time": "2020-04-11T07:33:41.493249Z"
}
},
"outputs": [
@@ -127,8 +127,8 @@
"execution_count": 4,
"metadata": {
"ExecuteTime": {
"end_time": "2020-04-11T05:31:36.865423Z",
"start_time": "2020-04-11T05:31:36.814579Z"
"end_time": "2020-04-11T07:33:41.670012Z",
"start_time": "2020-04-11T07:33:41.621030Z"
}
},
"outputs": [],
@@ -143,8 +143,8 @@
"execution_count": 5,
"metadata": {
"ExecuteTime": {
"end_time": "2020-04-11T05:31:36.918980Z",
"start_time": "2020-04-11T05:31:36.868490Z"
"end_time": "2020-04-11T07:33:41.721306Z",
"start_time": "2020-04-11T07:33:41.672405Z"
}
},
"outputs": [],
@@ -166,8 +166,8 @@
"execution_count": 6,
"metadata": {
"ExecuteTime": {
"end_time": "2020-04-11T05:31:45.902849Z",
"start_time": "2020-04-11T05:31:36.921263Z"
"end_time": "2020-04-11T07:33:50.843115Z",
"start_time": "2020-04-11T07:33:41.723616Z"
}
},
"outputs": [],
@@ -180,15 +180,15 @@
"execution_count": 7,
"metadata": {
"ExecuteTime": {
"end_time": "2020-04-11T05:31:46.602152Z",
"start_time": "2020-04-11T05:31:45.906531Z"
"end_time": "2020-04-11T07:34:06.000515Z",
"start_time": "2020-04-11T07:33:50.846196Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"<matplotlib.legend.Legend at 0x7f27ce25c4a8>"
"<matplotlib.legend.Legend at 0x7f2b15a3ffd0>"
]
},
"execution_count": 7,
@@ -229,8 +229,8 @@
"execution_count": 8,
"metadata": {
"ExecuteTime": {
"end_time": "2020-04-11T05:31:46.658638Z",
"start_time": "2020-04-11T05:31:46.606018Z"
"end_time": "2020-04-11T07:34:06.056421Z",
"start_time": "2020-04-11T07:34:06.004945Z"
}
},
"outputs": [],
@@ -256,8 +256,8 @@
"execution_count": 9,
"metadata": {
"ExecuteTime": {
"end_time": "2020-04-11T05:31:46.715140Z",
"start_time": "2020-04-11T05:31:46.661863Z"
"end_time": "2020-04-11T07:34:06.108308Z",
"start_time": "2020-04-11T07:34:06.059793Z"
}
},
"outputs": [],
@@ -278,10 +278,46 @@
]
},
{
"cell_type": "markdown",
"metadata": {},
"cell_type": "code",
"execution_count": 11,
"metadata": {
"ExecuteTime": {
"end_time": "2020-04-11T07:34:41.904630Z",
"start_time": "2020-04-11T07:34:41.800002Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"now run `tensorboard --logdir lightning_logs`\n"
]
},
{
"ename": "ValueError",
"evalue": "The value 1 of the parameter 'n_latent_encoder_layers_power' is out of the range of the distribution DiscreteUniformDistribution(high=11, low=3, q=1).",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-11-280de67a1f89>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 4\u001b[0m },\n\u001b[1;32m 5\u001b[0m \u001b[0muser_attrs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdefault_user_attrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mPL_MODEL_CLS\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mPL_NeuralProcess\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 7\u001b[0m )\n",
"\u001b[0;32m/media/wassname/Storage5/projects2/3ST/attentive-neural-processes/neural_processes/train.py\u001b[0m in \u001b[0;36mrun_trial\u001b[0;34m(name, PL_MODEL_CLS, params, user_attrs, MODEL_DIR, plot_from_loader)\u001b[0m\n\u001b[1;32m 100\u001b[0m \u001b[0;31m# Make trial\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 101\u001b[0m \u001b[0mtrial\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moptuna\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mFixedTrial\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 102\u001b[0;31m \u001b[0mtrial\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mPL_MODEL_CLS\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_suggest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 103\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 104\u001b[0m \u001b[0;31m# Add auto number\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/media/wassname/Storage5/projects2/3ST/attentive-neural-processes/neural_processes/models/neural_process/lightning.py\u001b[0m in \u001b[0;36madd_suggest\u001b[0;34m(trial)\u001b[0m\n\u001b[1;32m 49\u001b[0m )\n\u001b[1;32m 50\u001b[0m trial.suggest_discrete_uniform(\n\u001b[0;32m---> 51\u001b[0;31m \u001b[0;34m\"n_latent_encoder_layers_power\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m11\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 52\u001b[0m )\n\u001b[1;32m 53\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/optuna/trial.py\u001b[0m in \u001b[0;36msuggest_discrete_uniform\u001b[0;34m(self, name, low, high, q)\u001b[0m\n\u001b[1;32m 681\u001b[0m \u001b[0mhigh\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_adjust_discrete_uniform_high\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlow\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhigh\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mq\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 682\u001b[0m \u001b[0mdiscrete\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdistributions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDiscreteUniformDistribution\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlow\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlow\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhigh\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mhigh\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mq\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mq\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 683\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_suggest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdiscrete\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 684\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 685\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0msuggest_int\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlow\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhigh\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/optuna/trial.py\u001b[0m in \u001b[0;36m_suggest\u001b[0;34m(self, name, distribution)\u001b[0m\n\u001b[1;32m 705\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mdistribution\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_contains\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparam_value_in_internal_repr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 706\u001b[0m raise ValueError(\"The value {} of the parameter '{}' is out of \"\n\u001b[0;32m--> 707\u001b[0;31m \"the range of the distribution {}.\".format(value, name, distribution))\n\u001b[0m\u001b[1;32m 708\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 709\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_distributions\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mValueError\u001b[0m: The value 1 of the parameter 'n_latent_encoder_layers_power' is out of the range of the distribution DiscreteUniformDistribution(high=11, low=3, q=1)."
]
}
],
"source": [
"## ANP-RNN"
"trial, trainer, model = run_trial(\n",
" name=\"anp-rnn\",\n",
" params={\n",
" },\n",
" user_attrs = default_user_attrs,\n",
" PL_MODEL_CLS=PL_NeuralProcess\n",
")"
]
},
{
@@ -289,7 +325,60 @@
"execution_count": null,
"metadata": {
"ExecuteTime": {
"start_time": "2020-04-11T05:31:34.200Z"
"end_time": "2020-04-11T07:34:41.908866Z",
"start_time": "2020-04-11T07:33:39.100Z"
}
},
"outputs": [],
"source": [
"trial, trainer, model = run_trial(name=\"PL_Transformer\",\n",
" params={\n",
" },\n",
" user_attrs={**default_user_attrs, 'context_in_target': False},\n",
" PL_MODEL_CLS=PL_Transformer)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2020-04-11T07:34:41.910251Z",
"start_time": "2020-04-11T07:33:39.200Z"
}
},
"outputs": [],
"source": [
"trial, trainer, model = run_trial(name=\"TransformerSeq2Seq_PL\",\n",
" params={},\n",
" user_attrs=default_user_attrs,\n",
" PL_MODEL_CLS=TransformerSeq2Seq_PL)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2020-04-11T07:34:41.911846Z",
"start_time": "2020-04-11T07:33:39.200Z"
}
},
"outputs": [],
"source": [
"trial, trainer, model = run_trial(name=\"LSTM_PL_STD\",\n",
" params={},\n",
" user_attrs=default_user_attrs,\n",
" PL_MODEL_CLS=LSTM_PL_STD)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"ExecuteTime": {
"end_time": "2020-04-11T07:34:41.796127Z",
"start_time": "2020-04-11T07:34:06.110618Z"
}
},
"outputs": [
@@ -298,136 +387,24 @@
"output_type": "stream",
"text": [
"now run `tensorboard --logdir lightning_logs`\n",
"trial.number -31\n",
"trial <optuna.trial.FixedTrial object at 0x7f27cf19b860>\n",
"trial.number -4\n",
"trial <optuna.trial.FixedTrial object at 0x7f2b11fb2278> {'learning_rate': 0.001, 'lstm_dropout': 0.22, 'hidden_size_power': 4.0, 'lstm_layers': 4, 'bidirectional': False} {'batch_size': 16, 'grad_clip': 40, 'max_nb_epochs': 200, 'num_workers': 3, 'num_extra_target': 96, 'vis_i': '670', 'num_context': 96, 'input_size': 18, 'input_size_decoder': 17, 'context_in_target': True, 'output_size': 1, 'x_dim': 17, 'y_dim': 1, 'min_std': 0.005, 'patience': 2}\n",
"INFO:root:GPU available: True, used: True\n",
"INFO:root:VISIBLE GPUS: 0\n",
"INFO:root:\n",
" | Name | Type | Params\n",
"----------------------------------------------------------------------------------------------------------------\n",
"0 | _model | NeuralProcess | 1 M \n",
"1 | _model.norm_x | BatchNormSequence | 34 \n",
"2 | _model.norm_x.norm | BatchNorm1d | 34 \n",
"3 | _model.norm_y | BatchNormSequence | 2 \n",
"4 | _model.norm_y.norm | BatchNorm1d | 2 \n",
"5 | _model._lstm_x | LSTM | 207 K \n",
"6 | _model._lstm_y | LSTM | 199 K \n",
"7 | _model._latent_encoder | LatentEncoder | 98 K \n",
"8 | _model._latent_encoder._encoder | BatchMLP | 49 K \n",
"9 | _model._latent_encoder._encoder.initial | NPBlockRelu2d | 32 K \n",
"10 | _model._latent_encoder._encoder.initial.linear | Linear | 32 K \n",
"11 | _model._latent_encoder._encoder.initial.act | ReLU | 0 \n",
"12 | _model._latent_encoder._encoder.initial.dropout | Dropout2d | 0 \n",
"13 | _model._latent_encoder._encoder.encoder | Sequential | 0 \n",
"14 | _model._latent_encoder._encoder.final | Linear | 16 K \n",
"15 | _model._latent_encoder._self_attention | Attention | 0 \n",
"16 | _model._latent_encoder._penultimate_layer | Linear | 16 K \n",
"17 | _model._latent_encoder._mean | Linear | 16 K \n",
"18 | _model._latent_encoder._log_var | Linear | 16 K \n",
"19 | _model._deterministic_encoder | DeterministicEncoder | 672 K \n",
"20 | _model._deterministic_encoder._d_encoder | BatchMLP | 82 K \n",
"21 | _model._deterministic_encoder._d_encoder.initial | NPBlockRelu2d | 32 K \n",
"22 | _model._deterministic_encoder._d_encoder.initial.linear | Linear | 32 K \n",
"23 | _model._deterministic_encoder._d_encoder.initial.act | ReLU | 0 \n",
"24 | _model._deterministic_encoder._d_encoder.initial.dropout | Dropout2d | 0 \n",
"25 | _model._deterministic_encoder._d_encoder.encoder | Sequential | 32 K \n",
"26 | _model._deterministic_encoder._d_encoder.encoder.0 | NPBlockRelu2d | 16 K \n",
"27 | _model._deterministic_encoder._d_encoder.encoder.0.linear | Linear | 16 K \n",
"28 | _model._deterministic_encoder._d_encoder.encoder.0.act | ReLU | 0 \n",
"29 | _model._deterministic_encoder._d_encoder.encoder.0.dropout | Dropout2d | 0 \n",
"30 | _model._deterministic_encoder._d_encoder.encoder.1 | NPBlockRelu2d | 16 K \n",
"31 | _model._deterministic_encoder._d_encoder.encoder.1.linear | Linear | 16 K \n",
"32 | _model._deterministic_encoder._d_encoder.encoder.1.act | ReLU | 0 \n",
"33 | _model._deterministic_encoder._d_encoder.encoder.1.dropout | Dropout2d | 0 \n",
"34 | _model._deterministic_encoder._d_encoder.final | Linear | 16 K \n",
"35 | _model._deterministic_encoder._self_attention | Attention | 0 \n",
"36 | _model._deterministic_encoder._cross_attention | Attention | 590 K \n",
"37 | _model._deterministic_encoder._cross_attention.batch_mlp_k | BatchMLP | 32 K \n",
"38 | _model._deterministic_encoder._cross_attention.batch_mlp_k.initial | NPBlockRelu2d | 16 K \n",
"39 | _model._deterministic_encoder._cross_attention.batch_mlp_k.initial.linear | Linear | 16 K \n",
"40 | _model._deterministic_encoder._cross_attention.batch_mlp_k.initial.act | ReLU | 0 \n",
"41 | _model._deterministic_encoder._cross_attention.batch_mlp_k.initial.dropout | Dropout2d | 0 \n",
"42 | _model._deterministic_encoder._cross_attention.batch_mlp_k.encoder | Sequential | 0 \n",
"43 | _model._deterministic_encoder._cross_attention.batch_mlp_k.final | Linear | 16 K \n",
"44 | _model._deterministic_encoder._cross_attention.batch_mlp_q | BatchMLP | 32 K \n",
"45 | _model._deterministic_encoder._cross_attention.batch_mlp_q.initial | NPBlockRelu2d | 16 K \n",
"46 | _model._deterministic_encoder._cross_attention.batch_mlp_q.initial.linear | Linear | 16 K \n",
"47 | _model._deterministic_encoder._cross_attention.batch_mlp_q.initial.act | ReLU | 0 \n",
"48 | _model._deterministic_encoder._cross_attention.batch_mlp_q.initial.dropout | Dropout2d | 0 \n",
"49 | _model._deterministic_encoder._cross_attention.batch_mlp_q.encoder | Sequential | 0 \n",
"50 | _model._deterministic_encoder._cross_attention.batch_mlp_q.final | Linear | 16 K \n",
"51 | _model._deterministic_encoder._cross_attention._W_k | ModuleList | 131 K \n",
"52 | _model._deterministic_encoder._cross_attention._W_k.0 | AttnLinear | 16 K \n",
"53 | _model._deterministic_encoder._cross_attention._W_k.0.linear | Linear | 16 K \n",
"54 | _model._deterministic_encoder._cross_attention._W_k.1 | AttnLinear | 16 K \n",
"55 | _model._deterministic_encoder._cross_attention._W_k.1.linear | Linear | 16 K \n",
"56 | _model._deterministic_encoder._cross_attention._W_k.2 | AttnLinear | 16 K \n",
"57 | _model._deterministic_encoder._cross_attention._W_k.2.linear | Linear | 16 K \n",
"58 | _model._deterministic_encoder._cross_attention._W_k.3 | AttnLinear | 16 K \n",
"59 | _model._deterministic_encoder._cross_attention._W_k.3.linear | Linear | 16 K \n",
"60 | _model._deterministic_encoder._cross_attention._W_k.4 | AttnLinear | 16 K \n",
"61 | _model._deterministic_encoder._cross_attention._W_k.4.linear | Linear | 16 K \n",
"62 | _model._deterministic_encoder._cross_attention._W_k.5 | AttnLinear | 16 K \n",
"63 | _model._deterministic_encoder._cross_attention._W_k.5.linear | Linear | 16 K \n",
"64 | _model._deterministic_encoder._cross_attention._W_k.6 | AttnLinear | 16 K \n",
"65 | _model._deterministic_encoder._cross_attention._W_k.6.linear | Linear | 16 K \n",
"66 | _model._deterministic_encoder._cross_attention._W_k.7 | AttnLinear | 16 K \n",
"67 | _model._deterministic_encoder._cross_attention._W_k.7.linear | Linear | 16 K \n",
"68 | _model._deterministic_encoder._cross_attention._W_v | ModuleList | 131 K \n",
"69 | _model._deterministic_encoder._cross_attention._W_v.0 | AttnLinear | 16 K \n",
"70 | _model._deterministic_encoder._cross_attention._W_v.0.linear | Linear | 16 K \n",
"71 | _model._deterministic_encoder._cross_attention._W_v.1 | AttnLinear | 16 K \n",
"72 | _model._deterministic_encoder._cross_attention._W_v.1.linear | Linear | 16 K \n",
"73 | _model._deterministic_encoder._cross_attention._W_v.2 | AttnLinear | 16 K \n",
"74 | _model._deterministic_encoder._cross_attention._W_v.2.linear | Linear | 16 K \n",
"75 | _model._deterministic_encoder._cross_attention._W_v.3 | AttnLinear | 16 K \n",
"76 | _model._deterministic_encoder._cross_attention._W_v.3.linear | Linear | 16 K \n",
"77 | _model._deterministic_encoder._cross_attention._W_v.4 | AttnLinear | 16 K \n",
"78 | _model._deterministic_encoder._cross_attention._W_v.4.linear | Linear | 16 K \n",
"79 | _model._deterministic_encoder._cross_attention._W_v.5 | AttnLinear | 16 K \n",
"80 | _model._deterministic_encoder._cross_attention._W_v.5.linear | Linear | 16 K \n",
"81 | _model._deterministic_encoder._cross_attention._W_v.6 | AttnLinear | 16 K \n",
"82 | _model._deterministic_encoder._cross_attention._W_v.6.linear | Linear | 16 K \n",
"83 | _model._deterministic_encoder._cross_attention._W_v.7 | AttnLinear | 16 K \n",
"84 | _model._deterministic_encoder._cross_attention._W_v.7.linear | Linear | 16 K \n",
"85 | _model._deterministic_encoder._cross_attention._W_q | ModuleList | 131 K \n",
"86 | _model._deterministic_encoder._cross_attention._W_q.0 | AttnLinear | 16 K \n",
"87 | _model._deterministic_encoder._cross_attention._W_q.0.linear | Linear | 16 K \n",
"88 | _model._deterministic_encoder._cross_attention._W_q.1 | AttnLinear | 16 K \n",
"89 | _model._deterministic_encoder._cross_attention._W_q.1.linear | Linear | 16 K \n",
"90 | _model._deterministic_encoder._cross_attention._W_q.2 | AttnLinear | 16 K \n",
"91 | _model._deterministic_encoder._cross_attention._W_q.2.linear | Linear | 16 K \n",
"92 | _model._deterministic_encoder._cross_attention._W_q.3 | AttnLinear | 16 K \n",
"93 | _model._deterministic_encoder._cross_attention._W_q.3.linear | Linear | 16 K \n",
"94 | _model._deterministic_encoder._cross_attention._W_q.4 | AttnLinear | 16 K \n",
"95 | _model._deterministic_encoder._cross_attention._W_q.4.linear | Linear | 16 K \n",
"96 | _model._deterministic_encoder._cross_attention._W_q.5 | AttnLinear | 16 K \n",
"97 | _model._deterministic_encoder._cross_attention._W_q.5.linear | Linear | 16 K \n",
"98 | _model._deterministic_encoder._cross_attention._W_q.6 | AttnLinear | 16 K \n",
"99 | _model._deterministic_encoder._cross_attention._W_q.6.linear | Linear | 16 K \n",
"100 | _model._deterministic_encoder._cross_attention._W_q.7 | AttnLinear | 16 K \n",
"101 | _model._deterministic_encoder._cross_attention._W_q.7.linear | Linear | 16 K \n",
"102 | _model._deterministic_encoder._cross_attention._W | AttnLinear | 131 K \n",
"103 | _model._deterministic_encoder._cross_attention._W.linear | Linear | 131 K \n",
"104 | _model._decoder | Decoder | 607 K \n",
"105 | _model._decoder._target_transform | Linear | 16 K \n",
"106 | _model._decoder._decoder | BatchMLP | 590 K \n",
"107 | _model._decoder._decoder.initial | NPBlockRelu2d | 147 K \n",
"108 | _model._decoder._decoder.initial.linear | Linear | 147 K \n",
"109 | _model._decoder._decoder.initial.act | ReLU | 0 \n",
"110 | _model._decoder._decoder.initial.dropout | Dropout2d | 0 \n",
"111 | _model._decoder._decoder.encoder | Sequential | 294 K \n",
"112 | _model._decoder._decoder.encoder.0 | NPBlockRelu2d | 147 K \n",
"113 | _model._decoder._decoder.encoder.0.linear | Linear | 147 K \n",
"114 | _model._decoder._decoder.encoder.0.act | ReLU | 0 \n",
"115 | _model._decoder._decoder.encoder.0.dropout | Dropout2d | 0 \n",
"116 | _model._decoder._decoder.encoder.1 | NPBlockRelu2d | 147 K \n",
"117 | _model._decoder._decoder.encoder.1.linear | Linear | 147 K \n",
"118 | _model._decoder._decoder.encoder.1.act | ReLU | 0 \n",
"119 | _model._decoder._decoder.encoder.1.dropout | Dropout2d | 0 \n",
"120 | _model._decoder._decoder.final | Linear | 147 K \n",
"121 | _model._decoder._mean | Linear | 385 \n",
"122 | _model._decoder._std | Linear | 385 \n"
" | Name | Type | Params\n",
"------------------------------------------------------------------\n",
"0 | _model | Seq2SeqNet | 18 K \n",
"1 | _model.norm_input | BatchNormSequence | 36 \n",
"2 | _model.norm_input.norm | BatchNorm1d | 36 \n",
"3 | _model.encoder | LSTM | 8 K \n",
"4 | _model.multihead_attn | MultiheadAttention | 1 K \n",
"5 | _model.multihead_attn.out_proj | Linear | 272 \n",
"6 | _model.norm_target | BatchNormSequence | 34 \n",
"7 | _model.norm_target.norm | BatchNorm1d | 34 \n",
"8 | _model.decoder | LSTM | 8 K \n",
"9 | _model.mean | Linear | 17 \n",
"10 | _model.std | Linear | 17 \n"
]
},
{
@@ -446,7 +423,7 @@
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOydeViUVRfAfxcQEFARcEFQcDc19y3DNDWz8tMyzQXXUstKbTEzLTVNU7PFXDOXTEhNS7QyTctUzNyX3HdFcQNEZF/mfn+8Aw4wGzDDMPD+nmcemfve994z7zj33HvPuecIKSUqKioqKiUXB1sLoKKioqJiW1RFoKKiolLCURWBioqKSglHVQQqKioqJRxVEaioqKiUcFRFoKKiolLCURVBMUQIIYUQtWwth4qKin2gKgKVQkEI4SKEWC6EiBNC3BJCvGPmfX9qFZuTTllbIcR+IcQDIcRxIUSQkfsHCyEOafu9LoSYnaMtLyHEBiFEghDiqhCiv841XyHEJiFEpFaGwBxtzxZCRGjbviqEmGDis7wkhPhHCJEohPg7x7U6QoiNQoi7QogYIcRWIURdI22ZXV/fM9RTp4n2OSVq/22S43ozIcQuIUS8EOK2EGKMkbZ2aOWKE0IcE0L0yHG9v/Z5JQghwoQQXkbaMvj95LUtFcOoikClsJgC1AYCgCeBcUKIrsZuEEIEA6VylHkBvwCfAZ7AbOAXIUR5A824AW8BPkBroBMwVuf6AiAVqAQEA4uEEA201zTAFuBFA20vA+pJKcsCbYFgIURPIx8pBvgKmKnnmiewCairlWU/sNFIW2bV1/cM9dRx1t4bApQHVgIbteUIIXxQnsM3gDdQC/jDSJNjAF/tcxkBhAghfLVtNdC2M1ArdyKw0EhbBr+ffLSlYggppfoqZi9AArW0f5cDvgfuAleBDwEH7bVawE7gPhAFrNWWC+BL4A4QB/wHNCygTJFAF53304A1RuqXA84BbbSfx0lb3g04maPuOeAVM+V4B/hF+7c7yiBTR+f6KmBmjnuctDIEGmnXT/ucxpkhwzDgbxN1vLR9epv5uXLVN/QM9dzbBbgBCJ2ya0BX7d8zgFX5/N5bAclAK522ftC5XlP7HZTRc6/R7ycvbakv4y91RVD8mYcyINQA2gODgKHaa9NQZnblAX9tXVAGhieAOtp7XwKiAYQQ44UQsYZe+gTQztZ9gWM6xceABvrqa5kBLAJu6WtSz/uGRtrS5QngpPbvOkC6lPJcHuTK3rHyPOKB6ygD1w/m3muGnLeklNEFqG/sGerSADgutaOpluM8fA5tgBjtttYdIcQvQohqmRWFEL8KIcbrNqgtSwb2AX8DB3X6yvp/IKW8iHaw1963UAiROas39f0YbUvFfFRFUIwRQjgCfYEPpJQPpJRXgM9RltIAaShbNVWklMlSynCd8jJAPZRZ4mkp5U0AKeVMKaWnoZcBUTy0/97XKbuv7UOf3C2Ax3momHTZC1QRQvQTQpQSQgxGmQm6mfE8XgZaAHN05IrLUc2gXPqQUs7U1m+GMlu9b/wO0wgh/FG2RMy1o+Sqb+IZ5sSD3HLrPgd/YDDKlk814DKwOrOilLKb9jmgW6a9/1ngDymlxpy+pJSvSylf16lr7PsxJbeKmaiKoHjjg7I/fFWn7CrKNgbAOJTZ9H4hxEntQImU8i9gPsrgckcIsUQIUbYAcsRr/9VtoyzwIGdFIYQDyj7vGClles7r2hlvD5RB7zbQFdiOMiM3iBDieeBT4BkpZZSOXDk/l165jCEVjgBJwMfa/hZrDavxpozIOeSsgLJKWyilXK1THq/zqmasvqlnqAdTzyEJ2CClPCClTNZ+xrZCiHLGGpVSpkkpfwe6CCG6m9lXXuSyyPenoiqC4k4UD2f9mVRD2Q9GSnlLSjlcSlkFeBVYKLRup1LKr6WUzYH6KEvt9wCEEBNyDErZXvqEkFLeA24CjXWKG/Nwi0aXsiiz9rVCiFvAAW35dSFEO217O6WULaWUXiirm3ooxlK9aI3S3wL/k1L+p3PpHOAkhKhthlzm4ISyOkFK+ZqU0kP7mmHOzdottD+ATVLK6brXdNrykFJeM1Hf5DPMwUmgkRBCd8utEQ+fw3EUG0OWOOZ8Hh2ynou2zaz/B0KIGoALyneRE1PfT17aUjGGrY0U6svyL7Ibi0OADSjL5QDgDDBMe6034K/9uwHKzK8G0BLFw6YUyr73FuDjAso0E8UwXR5l4L6J1hiZo54AKuu8Wmo/jx/grK3TVCtbWRQvnD1G+u2IYt94wsD1NSjbHO4oWyn3gQY611211ySKh46rttwBRXmW18rcSvuZRhuRxVHb3mvALu3fpbTXyqIos/lmPk+D9c15hjnqO6OsFMegDKRvat9nPu+OwD2gifa5fwnsNiBXPeAZoLS27gCUfftmOv/P4oB22ucagnGnAYPfT17bUl9G/j/ZWgD1ZYUvNbsiKK/9gdwFIoBJPPQamo2yOogHLgIjtOWdUGaB8SirilDAo4AyuQDLtT/c28A7Oteqafuqpue+QHJ4vGgHhvva11qgopF+dwDp2vYzX7/rXPcCwoAEFE+Z/nqeZbaXttwBRUHGaNs8B0xAx/NGjyxD9LT3nfbaYO37hByy5nomea2v7xnqqdMUOIQyGTgMNM1xfaT2/8o9FPfdqjrXfgcmaP9+BMVA/ACIRVmNvJCjrf7aZ52A4rbqpXNtMbA4D9+PwbbUl/kvoX2YKioqKiolFNVGoKKiolLCURWBioqKSglHVQQqKioqJRxVEaioqKiUcFRFoKKiolLCMRiWtqji4+MjAwMD83VvWhqoTlIqKir2iIMDOBVgxD506FCUlLKCvmt2pwgCAwM5ePCg6Yp6uH0bkpIsLJCKiopKIeDuDhX0DuPmIYS4auiaujWkoqKiUsJRFYGKiopKCUdVBCoqKiolHFURqKioqJRwVEWgoqKiUsKxqiIQQnQVQpwVQlzImcpOez1ACPGnEOK4EOJvbaYlFRUVFZVCxGqKQJsmcQFKbPL6QD8hRP0c1eYA30spGwFTUTJIqaioqKgUItZcEbQCLkgpL0kpU1ESTPTIUac+8Jf27x16rquoqKgUKU6dOsaRI/vQaDSmK9sJ1lQEfiiJUDK5zsNcuZkcA3pq/34BKCOE8LaiTCoqKioFYtGiT+nZsw3vvDOQuLhYW4tjEWxtLB4LtBdCHAHao2RAyshZSQgxQghxUAhx8O7du4Uto4qKikoW1aop6Zc3bvyBI0f+tbE0lsGaISZuAFV13vtry7KQUkaiXREIITyAF6WUuVSslHIJsASgRYsWarQgFRUVm/Hee9NJSUni4ME9ODraXZQevVjzUxwAagshqqMogL4o+UWzEEL4ADFSSg3wAUpOWxUVFRWDaDQaHBxsu5nx4Ydf2LR/S2O1pymlTAfeBLYCp4EfpZQnhRBThRDdtdU6AGeFEOeASsB0a8mjoqJSPAgJWcSzzzbhjz/CCr3vGzeukZAQX+j9WhurqlUp5WYpZR0pZU0p5XRt2SQp5Sbt3+ullLW1dYZJKVOsKY+Kior9s23bRk6fPsbYsUM4f/6Uyfo7dmxm6tS3SElJLnDfgwZ1oWHDMpw7dxKg2HgO2dpYrKKiopInli79BYAHD+5z+fJ5M+p/zooVc1m3bkWB+3Z1LY2LiyvJyUm0aeNHt27NCtxmUaB4WDpUVFRKDC4uLixZspFbt65Tr96jJusPGvQmV65coFWrJwrc92+/HUFKyb170dy+HWmRVUZRQEg7S9nVokULqSamUVEpmUgpEULYWgyklERGRlCxoi+lSpUqlD4tkJjmkJSyhb5r6taQioqKXXD9+hXatavOZ59NNFk3IyODWbPGc/jwXqwx2RVC4OdXrdCUgLVRFYGKiopdsGPHZm7cuMqVK+e5c+cmO3du5cSJw3rr7t+/i8WLZ/HWW8EAJCYm8MMP37Bq1cJ8979mzVKefroh33+/IN9tFFVUG4GKiopdEBz8Gg0bNsPZ2YVdu7by3ntD6dlzEJ9/vjJX3SpVqjFs2LtUruyHEILU1BQmTnyNMmXKMXDg6/nq/+LFM5w7d5KEhAcA/PzzKrZt20ifPq/QocMzBfpstkZVBCoqKnaBg4MDTZu2AeDBgziCgp6idu0GeusGBNRk4sQ5We/LlClHnz7D8PT0yredYdSoj3jhhQGUL+8DwNmz/7Fly080bNjM7hWBaixWUVFRyQcnThzmypXzNGzYnMDAWlbvz5rGYnVFoKKiUuS5du0SCxbMoE6dBrzyyttG68bFxXLo0D9Uq1aDmjXrWU2mhg2b0bBh8ThHoBqLVVRUijzXr1/hxx+XsX37pmzlKSm5gxGcOnWUl19+jvHjh2Urj4mJ4uLFMyQmJuS5/9TUVN5/fxjz5xfPKDiqIlBRUSnyVK9ehxkzljBkyGhAsRHUretKixa590pcXFwJCnqKpk0fy1b+xhu96dz5EY4e3Zfn/m/fvsGPPy5j9epvssri4u4TFhbKpk2r89xeUUPdGlJRKeacPn2c8eOHMXXqAho3bmlrcfKFr68//foNz3rv7u5BWloqIMnIyMDR0THrWtOmbVi16o9cbfj7BxIYWDtf/bu5eTB58te4ublnlcXGRvP22wPw8wuge/d++Wq3qKAqAhWVYk5YWAjHjx9gzJh+/P33BVuLYxEcHBw4dSoBFxdXsz2APvss/7GGvL0rMGTIqBxlFenWrQ+VK/vnu92igqoIbERYWChz5kwkMvIaVapUY+zY6Tz/fLCtxVIphvTsOYjbtyPp2PE5W4uSb06dOkZk5DXq1XsUf/9AQAkAp4/k5KQ8KYj84u7uwbx5a6zaR2Gh2ghsQFhYKBMmjODGjatIKblx4yoTJowgLCzU1qKpFEPq1m3IV1+F2PX2xbp1yxk+vLtZOQj69GlPgwYeBk8d54erVy+ya9cfXLt2yWJtFiVLine truncated
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOydd3wU1fbAvzedJJRUCAESIHSQFsASioKIyIP3sACGqoCigGBFsCCCgB2pIuUhXfwJoo8i8GjhgQLSe4dQAkkIIb3s/f0xu2GTbEuyJZvM9/OZT3Zn7pw5O5u95849554jpJSoqKioqJRfXBytgIqKioqKY1ENgYqKiko5RzUEKioqKuUc1RCoqKiolHNUQ6CioqJSzlENgYqKiko5RzUE5RghhBRCRDhaDxUVFceiGgKVUoUQwlMIsUgIkSyEuCWEeNNE22eEEDFCiCRt2wVCiIqWyBJCeAghfhZCXNYaxE4FZI8VQlzUnntDCPGNEMLNhC6PCyG2CyHuCSEuFzgWLIRYqZVzTwixRwjRzoQsi9trP59Jgy6ECNfqliaEOC2E6FLgeB0hxO9CiPtCiHghxOcmZC0TQtzU3pezQoihBY531l4jTXvNMBOyTH7XRZGlUjJUQ6BS2pgI1APCgMeBd4UQ3Yy0rQxMBqoDjYBQ4IsiyIoB+gO3DMheD7SSUlYCmgLNgdEm9E4FFgHvGDjmC+wHWgP+wBLgP0IIXyOyLGovhIgC6prQScdK4BAQAEwAfhZCBGlleABbgP8C1YAawDITsqYC4dr70hOYLIRorZUVCPwCfKjV+wCw2oSsiRj5foohS6UkSCnVrZxugAQitK8rAz8Cd4ArwAeAi/ZYBLATuAfEA6u1+wXwDXAbSAaOAU1LqNMNoKve+0+BVRae2xs4VlRZQCzQyYTcAGArMMcCHboAly1olwy0LsJ9ydcecEPp3B/S/x4NnFcfyAQq6u3bDbyqfT0c2F3M76oBcBN4QU/W//SO+wDpQMOiftdFlaVuJdvUJwIVHTNRjEEdoCMwEBiiPfYp8AfghzJinKnd3xXogNLZVAZeABIAhBDjtFM2BjdDCggh/IAQ4Ije7iNAEws/QwfghJVkIYR4UQiRjGL8mgPfW3quGbktAA/gfAnajwV2SSmPmjm9CXBRSnlfb5/+fXgYuCyE2KidFtohhGimd+05Qog5BfSZI4RIA06jGIINetfKu99SylTggu5a2v+J37WvzX0/JmWpWBejc54q5QchhCvQF2ih7TDuCyG+AgYAC4FslMf36lLKWJQpFbT7KwINgb+klKd0MqWU04BpRVRFN/VxT2/fPe01zH2GJ4FBgG4uvdiydEgpVwArhBD1UAxjnKXnmtCzErAU+ERKea847YUQNYFXUKaOzOFL/nuA9n2o9nUNlGmZnsA24A3gVyFEQylllpTytYICpZSvCSFGAY8AnVCeOHTXumPgWhW15+n/P5j7fkzKUrEu6hOBCkAg4I4yJaTjCg86i3dRpoH+EkKcEEK8BCCl/C8wC5gN3BZCzNd2XMUlRftXX0Yl4L6BtnkIIR4GVgDPSSnPlkSWIaSU51CeNOZorzdeCJGi3eZZKkcIUQH4DdgnpZyqt/+Enrz25toD3wKTLDEkKPeh4Heifx/SgRgp5UYpZRbwJcpUWCNTQqWUuVLKGBRDMsLCaxXUS3fcUNuiyFIpIaohUAFl6kM36tdRC7gOIKW8JaUcJqWsjjISnaOLUpFSfielbA00RpkiegcKdZaFNkNKSCnvokw1NNfb3RztdI8hhBAtURy7L0kpt5VElhnc0DpmpZSfSSl9tdurlpwshPAE1qH4I17RPyalbKInb7e59kBn4AttpI3O0b1XCPGigUufAOoIvWgq8t+Hoyg+huKSd1+0MvPutxDCR3us0D234PuxWJaKFXC0k0LdHLeR31m8DFiL8ugdhjL/O1R77HmghvZ1E5RRZB2gDcpUjDuKM28TyhRGSXSahuKY9kOZcroJdDPStinKdE2f4sgCPAEvlM62q/a10B4bCgRrXzdG6YC+NqG3i/b8p1GeprwAD+0xd5SR/TrAzYJ7YLI9EIwS4aPbJMpcfwUj8vahjPS9gH8BSUCQ9lgDIA3Fye2K4nu4oNPdwHX7okzbuAJPoURL9dQeD0KZvnlWe63pKE8zRf6uiypL3Uq2OVwBdXPgl5/fEPhpjcEd4BrwEQ+ihj5HeTpI0XYSw7X7O6OMKFNQniqWA74l1MkTJQwzWdvJv2mi7WJAo72+bjthqSzgsvYe6G/herLjtB3dZZSwVC8TunQyIGuH9lhH7fu0Arq2NyKrqO2NRg1pj4cDO1AM+BmgS4HjvVEc0cnadk30js0D5mlfB2k77iQeRIkNKyCrC8ogIl0rK1zv2HhgYxG+H6Oy1M26m270o6KioqJSTlF9BCoqKirlHNUQqKioqJRzVEOgoqKiUs5RDYGKiopKOUc1BCoqKirlHKdLMREYGCjDw8OLdW52NqhBUioqKs6Iiwu4laDHPnjwYLyUMsjQMaczBOHh4Rw4cKBY58bFQXq6lRVSUVFRsQM+PhBksBu3DCHEFWPH1KkhFRUVlXKOaghUVFRUyjmqIVBRUVEp56iGQEVFRaWcoxoCFRUVlXKOTQ2BEKKbEOKMEOK8EGKcgeNhQohtQoij2hJ5NWypj4qKiopKYWxmCLTlD2ej5GdvDPQTQjQu0OxL4Ecp5UPAJGAqKioqKip2xZZPBG2B81LKi1IpgbcK6FWgTWPgv9rX2w0cV1FRUSlVnDx5hEOH/kSj0ThaFathS0MQilLgREcsD2rg6jiCUhQDlMpJFYUQATbUSUVFRaVEzJ07ld69H+bNNweQnJzkaHWsgqOdxW8DHYUQh1CqMl0Hcgs2EkIMF0IcEEIcuHPnjr11VFFRUcmjVi2lRPOvv67g0KF9DtbGOtgyxcR1oKbe+xrafXlIKW+gfSIQQvgCz0opC5lYKeV8YD5AZGSkmi1IRUXFYbzzzhQyM9M5cGAPrq5Ol6XHILb8FPuBekKI2igGoC/won4DIUQgkCil1ADvo9QvVVFRUTGKRqPBxcWxkxkffPC1Q69vbWx2N6WUOcBIYDNwCvhJSnlCCDFJCNFT26wTcEYIcRaoCkyxlT4qKiplg2XL5tK9ewv++GOd3a99/fpVUlNT7H5dW2NTsyql3CClrC+lrCulnKLd95GUcr329c9SynraNkOllJm21EdFRcX52bLlV06dOsLbbw/m3LmTZttv376BSZPGkJmZUeJrDxzYlaZNK3L27AmAMhM55GhnsYqKikqRWLDgNwDu37/HpUvnLGj/FYsXz2DNmsUlvraXVwU8Pb3IyEjn4YdD6dGjVYlllgbKhqdDRUWl3ODp6cn8+b9y61YsDRs2M9t+4MCRXL58nrZtO5T42v/5zyGklNy9m0Bc3A2rPGWUBoR0spJdkZGRUi1Mo6JSPpFSIoRwtBpIKblx4xrBwSG4u7vb5ZpWKExzUEoZaehYuZoacjKbp6Kiokds7GXat6/NF19MMNs2NzeX6dPH8fffe7HFYFcIQWhoLbsZAVtTrgxBcrKjNVBRUSku27dv4Pr1K1y+fI7bt2+yc+dmjh//22Dbv/7axbx50xkzJhqAtLRUVqz4nqVL5xT7+qtWLeCpp5ry44+ziy2jtFKufASJiVCtmqO1UFFRKQ7R0a/StGkrPDw82bVrM++8M4TevQfy1VdLCrWtXr0WQ4e+RbVqoQghyMrKZMKEV6lYsTIDBrxWrOtfuHCas2dPkJp6H4BfflnKli2/0qfPy3Tq9HSJPpujKVeGICMD0tLA29vRmqioqBQVFxcXWrZ8GID795OJinqSevWaGGwbFlaXCRO+zHtfsWJl+vQZSpUq/sX2M4wa9SH/+ld//PwCAThz5hibNv0fTZu2cnpDUK6cxTt2gJsb1FCrHqioqJSQ48f/5vLlczRt2prw8AibX091FluRu3cdrYGKikpRuXr1Iu+9N5SFC78x2zY5OYnt2zdw4cJpm+rUtGkrevToYxcjYGvKnSFITVWNgYqKsxEbe5mfflrI1q3r8+3PzCycjODkycO89NIzjBs3NN/+xMR4Llw4TVpaapGvn5WVxXvvDWXWrLKZBafcGQKAK1fUUFIVFWeidu36fPbZfAYPHg0oPoIGDbyIjCw8V+Lp6UVU1JO0bPlIvv2vv/48Xbo04vDhP4t8/bi46/z000JWrvw+b19y8j3WrVvO+vUriyyvtFGunMU6MjLg+nXVV6BSPjh16ijjxg1l0qTZNG/extHqFIuQkBr06zcs772Pjy/Z2VmAJDc3F1dX17xjLVs+zNKlfxSSUaNGOOHh9Yp1fW9vXz7++Du8vX3y9iUlJTB2bH9CQ8Po2bNfseSWFsqlIQCIjVUiiGrWhAoV8h/LzYWcHNBolCcHKUEIZYMHf+HB8dxc5a9G82Az9F63gbJPHyFA9//s4qK8dnHJ/1qnh34WXp0+Bc8pzejuje61/hOa7rWhpzb9z6q7D6VgoWmpZt26ZRw9up833ujHjh3nHa2OVXBxceHkyVQ8Pb0sjgD64ovi5xoKCAhi8OBRBfYF06NHH6pVc/4RZbk1BKCsK0hMfNB5ajRKh24PYmKWs3r1BOLjrxIYWIs+faYQFRVt1Wu4uT34bPqGRP+17pi+oSuIsWk0fcOm69gLGr/c3AdGUv+1tdEZP31jaejzFNTZ0KbDmNFxdX1wz3SvdfdRt+nuva6dvgx92fp66eunP3DQv6/GjKOxDaBr14Fcv36DDh2eITXV8IBGXwf9a+sPigoOjPSvrW+QCx4zt6/g/TbU7vTpI9y6dZX69ZtRo0a4tlUFsrIK34+MjPQ8A2FogFHwtf77gt+D/vHCmy8ffbQKgDt3TMss+D3qfgu6zdSAUfc3NLRkUUOmKNeGQIfuy7AXMTHL+eGH4WRlpQEQH3+FH34Line truncated
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
@@ -460,14 +437,14 @@
"name": "stdout",
"output_type": "stream",
"text": [
"step 0, {'val_loss': '0.44150611758232117', 'val/loss_mse': '0.2931194007396698', 'val/loss_p': '0.44150611758232117', 'val/sigma': '1.0362932682037354'}\n",
"step 0, {'val_loss': '0.005343312863260508', 'val/loss_mse': '0.0009845279855653644', 'val/loss_p': '0.005343312863260508', 'val/sigma': '0.767909824848175'}\n",
"\r"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "c1091128cecf44d5a9a0973151c2f7e7",
"model_id": "a5b3f808530c47fb96d58b9302472726",
"version_major": 2,
"version_minor": 0
},
@@ -477,8 +454,97 @@
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"INFO:root:Detected KeyboardInterrupt, attempting graceful shutdown...\n",
"\n"
]
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOydd3hUxdrAf5NOEiAkISEESIDQO9LUCIiKiH5gQQFDVcSG2JULV0UBFduVKiJFmoB4FVERBEUgXJQqvQcSQgkkIZDedr4/ztmw2exuNsludjc5v+c5z+6ZM2fmPW3eqe8rpJRoaGhoaFRf3BwtgIaGhoaGY9EUgYaGhkY1R1MEGhoaGtUcTRFoaGhoVHM0RaChoaFRzdEUgYaGhkY1R1MELoIQ4iEhxHkhRIYQopMQ4ogQoncl5NtbCJFo73yskGOUECLW0XJoaFRFNEXgOnwCjJNS+ksp90sp20gp/wQQQkwWQix3lGBCiBeFEGeFEJlCiGNCiOZq+ERVcem3bCGETggRrB73FkIsEkLcEEJcFkK84qhrKC9CiI5CiL1CiCz1t6OFuJ8IIU4JIdKFEMeFECOsTUsIcacQYosQ4roQ4pyJtLcIIa6q9/KAEGJgKXJPEUIcEkIUCCEmGx27XwgRK4RIU5/LAiFETQtpWRVfCBGoymhRoQshHhdCxKvv01ohRKDR8SHqe5YphDgjhLjDTDqthRB7hBDX1G2zEKK1wXEhhJguhEhRt+lCCGFBLkvPp0xpOR1SSm1z4g3wUH8LgCgzcSYDy+2Uf28g0cLxMcBBoDUggKZAoAU5/zDY/wDYDtQBWgGXgX5mzh0FxDr6eRjJ5AXEAy8D3sB4dd/LTPx3gZYoFbDuwDXgNmvSAroBw4GxwDkTabc3eFe6A+lAmAXZRwL3AT8Ck42OPQ70A3zVZ/MrMM9CWlbFB74Ctll6jkAbVfaegD/wDbDK4Pg96n3pod7HcCDcTFoBQKT6Xrqr9/SgwfGngRNAAzWdo8Az5XnWZUnLGTeHC+DKm1p4/QmkAUeAAWp4d7VQczeI+5D+JVRf4AnAGSAF+Ba18FRfXAk8CSQAO4EMNSwTOKPGOwfcrX6AeUC+Gu+ACTnfBL4zCpsBzFT/jwaOqR9gHPC0QbzemFEE6nWcB+6y4l4JNe2RBmEXgb4G+1MMP3qj80cZFiDAbcBu4Lr6e5tR3Dj1es4CMWp4FLBVPScZWF3B598XuAAIg7AEzCgzE+evA14tS1rqMz9XSrrdgBygmxUyLMdIEZiI8zBwqAz3pUR89XntVN81S4rgfeAbg/2m6vtdU93/H/BkOZ6VB/A8kGUQ9j9grMH+k8Bf5XnWZUnLGTeta6icCCE8gZ+A34AQ4AVghRCihZTyb5RCu4/BKY+j1G5Q4z4I9ALqo9QM5xhl0QtF0fSRUvqrYR2klE0NI0kpN6B8PKul0m3UwYS4q4D++ua6EMIdeMxAnivAA0AtlA/1P0KIzlbchgbq1lYdvzgrhHhXCGHqvboD5T79V5WhDhAGHDCIcwClRmgRtavgF2AmEAR8BvwihAgSQvip4fdJKWuiFED/qKdOQXledVS5ZxmkeVDt2jC1zTUjShsU5W5op+WglddQA+iKUoGoUFoGaf4shMgB/kapoOyx9txS6GkgZ5njq+/bbGAcSoXGEm0weCeklGdQFEFzNZ0uQF0hxGkhRKIQYrZ6L/V5pQkhog0TFEKkoSjGWSjfism8MHr/1HficYO4lp6PxbScHU0RlJ8eKE3XD6WUeVLKP4CfgaHq8ZX6/2oB3F8NA3gGmCSlTJRS5qJ0mQwSQngYpD9ZSpkppcyuqKBSynhgH0qrBBQFlSWl/Es9/ouU8oxU2IpSWJrsdzWigfrbF2gH3IlyzU+aiDsSpVWSoe7rldt1gzjXAbN90QbcD5ySUi6TUhZIKVcCx4H/U4/rUJRTDSnlJSmlvlDKByKA+lLKHCllUV+1lLK9lDLAzPacGTn8jeQvyzXMQyksNtogLQCklA+o8fsDv0kpddaeaw4hxD0oz+7tCsQfD/wtpdxrRRKW7kMo4AkMQnk/OwKdgH/rI6rPq9gYhJQyAKiNooj2W8jrOuCv79tX34lvzMQ1lKvUtJwdTRGUn/rAeaOPLR6lfxCU2vbDQghvlKbyPrVABqUw+kFf40TplilEedH1nLexvN9wU0kZtk4QQtwnhPhLCJGqytMfCLYiTb2S+khKmSalPAd8qZ5fhBDCF3gUWGIQrFcItQzCaqF055RGfZR7bUg8Sl9xJjAYRdleEkL8IoRoqcZ5A6WLapdQZl09YUVelsiguPxgxTUIIT4G2gKPGdQwy5WWMVLKfCnlr0BfIcQANb8j4uaAvTUKXi9nD5T3ZJCU8qQadodBWkesiF8fRRFMsjJbS/dB/77NUhV8MkprsD+loL4X84ClQogQM3nVAjKMav3WyFXWtJwOTRGUn4tAQ6NukEYo/YhIKY+iFE73YVTwohTy9xnVOn2klBcM4pTlBbIm7hqgtxCiAUrL4BtQZu6gdNd8AoSqtaf1KAVmaZxAabYb5m9KloeAVJTuCiWSlNeAS4BhV1YHrOuCuIiiTA0xvPcbpZT3oHQ9HUcZpERKeVlK+ZSUsj7K4N5cIUQUlCgsjbd5ZuQ4ArQ3qvW1t3QNQoh3Ud6JvlLKGxVJqxQ8UPrXkcoMM391227NyUKITihjGE9IKX/Xh0sptxuk1aa0+CjjFWHAUSHEZZSxqW5CmV3kbiLrIxi8E0KIJiiDsyfVdyaR0t83c7ihDGjrK2vF8sLy+1fa8ylLWs6HowcpXHVDmUUQhzLo64kyqJoOtDSI8yawBaUmE2wQ/jJKoRih7tcFBqr/I1Febg+j/CQGs4ZQB4vV/88AsYBbKTL/CmwC9huE1URpjfRCKfzvA7KAqerx3lieNbQUpUusJkpX0XGMBvNQupreM3HuhyiDt3VQZtNcwopZQyjjAmkoCtYDpQWQhtKKCQUGAn4oH/67wFb1vEeBBur/NupzaVLBdyAeeBGlsBqH5VlD/wJOAfXKmpZ6LT7q84lX/+uPtVTDa6jv4jAUBd3ZguyeahrfAFPV/+7qsbZAEjDYyvtgNr56LfUMthdRxjBK3AOD53IDpevHD2Uw23DW0HsokwNC1PdmOzDFTFr3oHQduaPU0GeiVCJ8DL6bYyiKoT5KwV3arCFzz8fqtJxxc7gArrypL61+FspR4CGj441Q+qt/MQp3A15BqVGno8weel89FknZFUEQiiK4htIFZU7e4Wo6rxuFP69+yGnAMpTBZWsVQS01fjpKS+dtis+sCMfM1Ff1g1qkfvhJwCsW8hlF8VlD0cBe9d7vBaLV8DCDZ5KGonBbq8c+Qmk1ZKj3fKy5/MrwDnRS889GGYfpZCGuBHLV/PXbRGvSUp+DNNr+VI+1Qilc09Vr3m38LpqQ5WsT6Y1Sjy1W31tDOY9YSMvq+MbP0Uycx1Fm5GSiTG8NNDjmCcxVr/MySuHuY3A8A7hD/f8oSsUkA7iKMsGgvUFcob4Tqer2kdG7ewR1xpkVz8diWs6+CfUiNDQ0NDSqKdoYgYaGhkY1R1MEGhoaGtUcTRFoaGhoVHM0RaChoaFRzdEUgYaGhkY1x6P0KM5FcHCwjIyMrHA6eXkVl0VDQ0OjsnBzA48KlNh79+5NllLWNXXM5RRBZGQke/ZUzJZWYSGct7UBBw0NDQ074ucHdU0W49YhhDA2y1JEteways93tAQaGhoazkO1VAQFBY6WQENDQ8N5qJaKQGsRaGhoaNykWioCrUWgoaGhcZNqqQi0FoGGhobGTeyqCIQQ/YQQJ1S3chNMHI8QQvyuuoT7U7WVb3e0FoGGhobGTeymCFSnE3NQ7KS3BoYKIVobRfsEWCqlbI9iZ/wDe8mjp7AQdBV24KehoaFRdbBni6AbcFpKGSelzEOxWT/QKE5r4A/1/xYTx21OYaG9c9DQ0KjKHD16gP37/0ZXhWqU9lQE4RT3u5vITRdxeg6g+PMFxZ1hTSFEkB1l0hSBhoZGhfjiiw94+OEevPLKcG7cSHO0ODbB0YPFrwG9hBD7UVwlXkBxm1gMIcRYIcQeIcSeq1evVijDKqTENTQ0HECjRk0B+PHHb9i//y8HS2Mb7Gli4gLQ0GC/gRpWhJTyImqLQAjhDzwipSyhYqWU84H5AF26dKmQSzWtRaChoVERXn99Grm52ezZswN3d5ez0mMSe17FbqCZEKIxigIYguKLtAghRDCQKqXUoTj2XmRHeQBNEWhouDo6nQ43N8d2Zvz73585NH9bY7e7KaUsAMYBG4FjwLdSyiNCiPeEEAPUaL2BE0KIk0AoMM1e8ujRFIGGhmuzfPkX9O/fkd9+W1vpeV+4kEBmZkal52tv7KpWpZTrpZTNpZRNpZTT1LC3pZTr1P/fSSmbqXHGSClz7SkPaIpAQ8PV2bTpR44dO8Brr43i1KmjpcbfsmU97733Erm5ORXOe8SIvrRtW5OTJ48AVJmZQ44eLK50qshz09CotixY8BMA6enXOXv2lBXxP2Xx4hmsWbO4wnn7+NTA29uHnJxsevQI54EHOlc4TWegaox0lAGtRaCh4dp4e3szf/6PXL6cSMuW7UqNP2LEOM6dO023bj0rnPcvv+xHSsm1aykkJV20SSvDGRBSVmgSTqXTpUsXWRHHNPHx4GKXrKGhoSKlRAjhaDGQUnLx4nlCQsLw9PSslDxt4Jhmr5Syi6lj1aprSKfTlICGhquSmHiOO+5ozMcfTyo1bmFhIdOnT2Dfvp3Yo7IrhCA8vFGlKQF7U60UgdYtpKHhumzZsp4LF+I5d+4UV65cYuvWjRw+vM9k3F27tjFv3nReeikGgKysTL755kuWLZtb7vxXrVrAvfe2ZenSOeVOw1mpVmME2kCxhobrEhPzDG3bdsbLy5tt2zby+uujefjhEXz66ZIScevXb8SYMa9Line truncated
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOydd3hURdfAf5NOCCQhoQVIAkhvUqQICkgRfFEQQcSIVKOiiCKCEkEEgw1fP0CqiigEBBQQUVERpClSBZHeQgmkkk76fH/c3bBJdjebTXY3eZnf8+yT3Xvnzpxbcs/MOWfOCCklCoVCobhzcXK0AAqFQqFwLEoRKBQKxR2OUgQKhUJxh6MUgUKhUNzhKEWgUCgUdzhKESgUCsUdjlIE5RAhxKNCiCtCiFQhRFshxL9CiB52lmGJEGK6PdssKUKIYCGEFEK4OFoWhaIioxRB+WQu8KKU0ktKeURK2UJK+TuAEGKmEGKVuYOFEJeEEL1LI4CU8jkp5WxrjxdC9BZCHBZCpAkhrgohHtdtv0+n4Aw/UgjxmMGxrwghbgghkoUQy4UQ7qU5F3sjhKgmhNioO/dIIcSTZsqOFEIc0p3rVSHEB4aKzVxdQojaQojNQogo3TUMLlT3XCHEWSFEihDilBDi6WLkdtdd72Td9Z9kotwMXXsmnzELZKsjhPhOCJGgO+/nzNTVWQjxq65srBBivRCitsH+nkKIHUKIJCHEJXPnqCu/Q1dPshDiqBBiYKH9T+qudZoQYpMQopqZusze65LU5UiUIihHGLwAgoB/7dCOrepvDqwGwgBvoA1wCEBKuVun4LyklF7AACAV2Ko79kHgdaAX2nVoALxtS3ltwEIgC6gJhACLhRAtTJT1BF4G/IFOaOc92cK68tCu22MYJw14GO0ejATmCSHuNSP3TKAR2nXvCUwRQvQzLCCEaAgMBa6bqccS2VYBF9HO6z/AHCFETxNlfYFlQLBOthTgC4P9acBy4LViZNIzEagtpawKhAKr9IpFd22XAiN0sqUDi8zUZfL+WFGX45BSqo8FH6AZ8DuQiPaSfkS3vRNwA3A2KPsocEz33QntxXYeiAfWAdV0+4IBCYwFLgN/or0UJdrDfV5X7hLQG+iH9tBl68odNSLnSrR/wlu6MlOMtLNLV3a9TvYkYBfQwqCeFcA7uu89gKvAq0AM2ktgtJlrtRqYbeF1/QL4otCxcwx+9wJumDhWf14uut8BwGYgATgHPGNQtiNwEEgGooH/6rZ7oL2U4nX39gBQsxTPSWXdPWpc6J68Z+Hxk4DvS1IX4KK7DsHF1L0ZeNXM/iigr8Hv2cDXhcpsBR7SP5MWnE8R2QAv3bbqBtuWASstvEbtgBQj23sDl0p4vzoCGUBH3e85wGqD/Q1196BKSe91Sepy9EeNCCxACOEKfA/8AtQAJgARQogmUsq/0F7aDxgc8iTaCw1d2UFAd7QX1U20XoQh3dEUzQNS6yUDtJFSNjQsJKXcivZwrZVaj7pNYVmllCPQXvYP68p8YKSdB3W/f0LrAdYADgMRZi5DLbSeZR00hbJQCOFromxnACHEP0KI60KIVcaGxEKIysAQ4EuDzS2Aowa/jwI1hRB+ZmTT8zWawgrQ1TtHCKG/L/OAeVLrBTZEU8ig9ZS9gXqAH/AcmhJFCLFICJFo4nPMhAyNgRwp5ZlC52BqRFCY+7k9GixtXfkIISoB92BipKm7l7Upeu1bGJQZCmRKKX8safuFmyv0V/+9pYXHG16j4hsTYosQ4nUj2zKAv9A6eAd1uwo8f1LK8+he9rrjFgkh9L364u6P2brKE0oRWEZntF7Me1LKLCnldmALMFy3f43+uxCiClqPaY1u33NAmJTyqpQyE234PaSQeWamlDJNSnnLxudRoB0p5XIpZYqBXG2EEN4mjs0GZkkps3UvglSgiYmyddGGw4+hKZpKwAIj5QYDccBOg21eaCMUPfrvVcydmBCiHtAVmCqlzJBS/g18Bujt4tnAXUIIfyllqpRyn8F2P+AuKWWulPKQlDIZQEo5XkrpY+LT2oQoXmijDkOSipNfdw5jgA5oPqJS1WWEJWgvpZ9N7Nd3QApf+yo62aqgdUImWtF2AaSUKcBeYLoQwkMI0Q7tWfEs7lghRGtgBpabgZBSDpBSvld4G9q5PQT8IqXM0+0q/PyBwXXQPRPjDcqauz9m6ypPKEVgGQHAFYOHBSASrXcMWu9/sNCcmoOBw1LKSN2+IGCjvicJnARy0WyGeq7YVHoj7QghnIUQ7wkhzgshktGG+qDZqo0RL6XMMfidzu2XR2FuoZl7zkgpU9FeIA8ZKTcS+Erqxs06UoGqBr/131NMtKUnAEjQvWT0GN6jsWg9sVNCiANCiAG67SvRXo5f6xybH+hGgNZSWH50v83KL4QYBLwL9JdSxpWmLiN1f4jW235cf62FFhWmd9ZP07Wlr99YWzPRTDeXjNRvGABgaU89BKiP9kwuRjPPXS3mPO5CG8VOlFLutrAdk+g6NT8BfYUQj+g2l+SaF1e2TO6fPVCKwDKigHpCCMPrFQhcA5BSnkB76fSnoFkItAe9f6HepIeU8ppBmZKkgLWkrKkyhtufBAai2VW90eztUHC4bi3HCrVVRB5dD74H8FWhXf+iOZf1tAGipZTxxbQZBVTT9Vz1GN6js1LK4WhmsPeBb4QQlXUvg7ellM2Be9Gc10/rZDR8WRb+mHrhnQFchBCNCp2DyRekziH7KZo575/S1GWk7rfRnsu++pEO5EeF6Z32c6SUN9F8P4Wvvb6tXsBLQosmuoFmSlsnhJgqCwYAWGS2klJG6nrq1aWUndA6IPvNnEcQsA3N97TS0vO3EBc0cyEUev6EEA0Ad7R7UZji7k9J6nIsjnZSVIQP4AZcQHP6uqK9wFKApgZlpgI70HrD/gbbX0GzQQbpflcHBuq+B2Pg7DQ4RqKZKvS/L6FzzKGZmvYATmbk3QeEGvwu0g4wHvgbrYdSGS2aIb9djDiLC7WRL5OR9segRYQ0QBvur6OQIxCYhs5pXWh7PzQHdnPAB9iOCUdr4fMCdgOfoDmAW6M5hfXX7Sl0zkk05ZeBZrLqCbQCnIFqaOYTk45wC5+Xr9FMg5XRzFVJGDjiC5V9AM1Rfb81denOtbLuOjQBPAz2vQGcBWpZKPd7aGY6X6ApmmLop9vnh+Yn0n+uoEUPeZmpz5xszdBMJG66exOHgfO4UD110IItJpvY76Rrqz9ah8wDcDNRtqmuXCW0/+Wn0Oz27XT7W6CZe+7Tyb6KQg5zS+9PSety5MfhAlSUj+6m7tTd6BPAo4X2B6JF6/xQaLsTWiTIaTTlcR5dVAzWKQI/NEVwE80EZUzWgWgO40S0UMQi7aCZdb7TyRSJ1gsuE0Wg2/82EKv7rAR8C+0/BYw1cewktJd4MlpUkbuJcgXOC803sQUtaug88JxB2VVoEU+paD21Qbrtw3X3Jk3X5vzC98OKZ6UasElX52XgSTNldwA5Orn0n58srUt3/gU+hfZlFqp7mhlZ3NHCMPWRVZPMlDV7/y2Q7WXds5Gme547mKnnLd3xhueRarC/h5G2fjfY/5P+vNEU0F9oz70+Sqzw//KTumudhvY/Us1g3xJgSQnuj8m6ytNH6IRVKBQKxR2K8hEoFArFHY5SBAqFQnGHoxSBQqFQ3OEoRaBQKBR3OEoRKBQKxR1Ohcvj7u/vL4ODgx0thkKhUFQoDh06FCelrG5sX4VTBMHBwRw8eLD4ggqFQqHIRwgRaWqfMg0pFArFHY5SBAqFQnGHoxSBQqFQ3OEoRaBQKBR3OEoRKBRWEhERQXBwME5OTgQHBxMRYW6BN4Wi/FLhooYUivJAREQEoaGhpKenAxAZGUloaCgAISEhjhRNoSgxakSgUFhBWFhYvhLQk56eTlhYmIMkUiisRykChcIKLl++XKLtCkV5RikChcIKAgMDS7RdoSjPKEWgUFhBeHg4np6eBbZ5enoSHh7uIIkUCutRikChsIKQkBCWLl2a/7tevXosW7ZMOYoVFZIKt1Rlhw4dpMo1pCgPpKen07x5c7KysoiKinK0OAqFWYQQh6SUHYztUyMChcJKPD09efbZZ+ncuTM//fSTo8VRKKzGZopACLFcCBEjhDheTLl7hBA5QoghtpJFobAVS5cuZePGjXz//feOFkWhsBpbjghWAP3MFRBCOAPvA7/YUA6Fwiakp6dzzz330KpVKzp16uRocRQKq7GZIpBS7gISiik2AfgWiLGVHAqFrVi2bBnffPMNly9fZuTIkY4WR6GwGof5CIQQdYBHgcWOkkGhKA36mcXu7u4OlkShKB2OdBb/HzBVSplXXEEhRKgQ4qAQ4mBsbKwdRFMoimfatGlcunSJNWvWcPHiRUeLo1BYjSMVQQfgayHEJWAIsEgIMchYQSnlMillByllh+rVjS65qVA4hMcff5xevXoxfvx4R4uiUFiNw7KPSinr678LIVYAW6SUmxwlj0JhDf7+/ri4uFClShVHi6JQWI3NFIEQYg3QA/AXQlwF3gJcAaSUS2zVrkJhL0JCQvjjjz94/vnnmT9/vqPFUSisxmaKQEo5vARlR9lKDoXCVpw4cYLExEQiIyMdLYpCUSrUwjQKhZV8/PHHHDhwgC5dujhaFIWiVKgUEwqFlfTo0YMrV67Qu3dvevXq5WhxFAqrUYpAoSgFcXFxZGZmqqRzigqNMg0pFFYyZswYrly5wty5c9WIQFGhUSMChcJKVq9ezZ49ewgKCuLuu+92tDgKhdWoEYFCYSV9+vThypUrNGvWzNGiKBSlQikChcJKvv/+e7Zs2cK4ceNo1qwZy5cvd7RICoVVKEWgUJSC/fv3s2/fPs6fP+9oURQKq1GKQKGwgqysLDZv3oy3tzd9+/alZcuWjhZJobAapQgUCis4e/YsQ4cORQhBXl6xCXQVinKNUgQKhRXcunULd3dLine truncated
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOydd3hURdfAf7PpvRNCSKF3UKQTiiBSVEAREWmCwouCvqK8CIKCIoICKiJS5JNiAtgRESvSey/SSxJSICEhvSfz/XF3wybZDWmbTcj9Pc99sjszd+bsvTf3TDlzjpBSoqKioqJSc9GYWwAVFRUVFfOiKgIVFRWVGo6qCFRUVFRqOKoiUFFRUanhqIpARUVFpYajKgIVFRWVGo6qCKo4QognhRA3hBApQogHhRD/CiF6mlsucyKECBRCSCGEpbllUVG5H1AVQdVnETBZSukopTwhpWwhpdwJIISYI4QILu5kIUSoEOKR8gohhHheCLG3BOUeEUIcF0KkCiEihBDPaNO7aZWZ/iGFEEP0zp0ihLgphEgSQnwlhLApr9yViRDCXQjxk/a3hwkhnium7BghxDHtb40QQnykr9iKq0sI4SOE2CKEiNJew8BCdT8jhNgvhEgTQuwsgdxGywshGgshfhZCxAoh4oUQfwghmtyjvlVCiItCiDwhxPOF8myEEJ9oZb8jhPhCCGFVTF2BQogdWtku6D/LQuF9IUSkECJRCLFTCNGimLpstM9VkvY5e71Qfm9tG2naNgMqo66qgKoIqih6L4UA4F9zylJShBDNgQ3ATMAFaAMcA5BS7tEqM0cppSPwOJAC/K49ty8wHeiN8pvrA+9W+o8oH8uALMAbGAEsL+bFZA+8BngCHVF+99QS1pWHct2GYJh44FNgQQnlLq68K7AFaKKV5TDw8z3qOwW8DBw3kDcdaAe0BBoDbYFZxdS1ETgBeKA8V98LIby0eUOBcUA3wB04AHxdTF1zgEYoz9fDwDQhRD8AIYQn8CPwtrauo8A3lVSX+ZFSqkcpD6AZsBNIQHlJD9SmdwRuAhZ6ZZ8ETms/a1D+Ea4CccC3gLs2LxCQwAtAOMpDnaJNSwWuasuFAo8A/VBeFNnacqcMyPk1yksjXVtmmja9E7BfK/8poKfeOc8D14Bk4DrKS6gZkAHkautJMHJdNgBzS3gN1wBrCp37gd733sBNI+fqrpWl9nsdlJdVPHAFGK9XtgPKP2IScAv4WJtuCwRr70MCcATwLscz4aC9H40LXf8FJTz/deCX0tQFWGqvQ6CROl8EdpbiN9yzPMqLTQIeJahvL/B8obSjwFC9788BN4yc3xjIBJz00vYAE7Wf3wS+1ctrAWQUI08U8Kje97nAJu3nCcD+QvczHWhq6rqqwqGOCEqJdhj7C/AnUAt4BQgRQjSRUh5CeWn30jvlOZSXHNqyg4EeKC+vOyg9P316oLx4e0ml5wzQRkrZQL+QlPJ34APgG6n0stsUllVKOQpFqTyhLfOREMIX+BV4H+WfeirwgxDCSwjhAHwG9JdSOgFdgJNSyvPAROCAth5XI5enk/YanRFCRAshgoUQ7gauoQPwNLBOL7kFilLScQrwFkJ4GGlLn01ABMo1fRr4QAihuwdLgCVSSmegAYryBRiDMmrxQ+ltTkT5Z0U7XZFg5DhtRIbGQI6U8lKh32B0qqIQ3bk78itvXaakO4qCjitHHaLQ57pCCBcD5VoA16SUyXpp+tdhE9BAO31lhXJPf8+vWIjpQoit2s9ugA9FnzFdXQWePyllKkqHrUVF11UVURVB6ekEOKL0zrKklP8AW4Hh2vyNus9CCCdggDYNlJfNTCllhJQyE2V4+bQouOg5R0qZKqVMN5H8I4FtUsptUso8KeVfKL20Adr8PKClEMJOShktpSzNtFRdYBTKlEUjwA5YaqDcU8BtYJdemiOQqPdd99mpuAaFEH5AV+BNKWWGlPIksBoYrS2SDTQUQnhKKVOklAf10j2AhlLKXCnlMSllEoCU8mUppauRo7URURxRRh36JN5Lfu1vGIcyXbKovHWZEiFEXZSOy+v3KlsMvwP/1XY8agOvatPtDZQt/ExAwesQjTLquIiixIcCU3QFpZQLpJSP69WlO99QXcW2VZF1VUVURVB66qAMZfP00sIAX+3nDcBTQlnofAo4LqUM0+YFAD/pepfAeZTpFm+9um6YVHpFhqH6vVwgCPDR9lyGoSisaCHEr0KIpqWoOx1luueSlDIFZcQywEC5McB6qR03a0kBnPW+6z7r9wYNUQeIL9Rr1L8fL6D0sC8IIY4IIXT/zF8DfwCbtAuXHxW3aFkCCsuP9nux8gshBgPzUUZht8tTV0kQQqwQdxfr3yrFeV4oo+AvpJQb9dL1F//9S1DVPJQ5/5Mo05ObUZTyLQNl73Ud3gHao4zqbFHWlP4RQhhSKil65xuqqzTXvCLrqhKoiqD0RAF+Qgj9a+cPRAJIKc+hvIj6U3BaCJSXfP9CPUxbKWWkXpnSuIMtSdnCZW4AXxeSwUFKuUAr/x9Syj4oQ98LwJelaOt0oXJFztH24HsC6wtl/YuyuKyjDXCrBFMQUYC7dvSlQ/9+XJZSDkeZxvsQZbHRQUqZLaV8V0rZHGUK7HG0o4hCL8vCh7ER0iXAUgjRqNBvMDqi0i4ufokydXemPHWVFCnlRHl30f6DkpyjnQr5E9gipZxXqD5HvSO8BO2nSyknSyl9pZT1UdZojhXqWOn4F6hf6N7qX4cHUKZGI6SUOVLKtYAb0NxAu3dQRhCFnzFdXQWeP+30ZQMMXPOKrKvKUNmLEtX9AKxRFlOnA1YoL7Vk9BaCUBaxdqD0kD310qegLDIHaL97AYO0nwPRWwDVO0eiTF/ovocCj2g/T0QZGmuKkfcgMEHvux/KgnZfwAKlJ9UTZVrHGxiEsrilQelh7dKe10/btnUxbY1DWWCujzLU/xZF6eiXeQvYbeDcflq5mqNYqvyDkYXWwtcKZQHxc+1vaY3Su9Rdo5GAl/bzIyiL3nYolh6ttNfAHWVOd2w5n41NKNOADijTVYlACyNle6G8BLuXpS7tb3XQXocmgK1enu6+TgR2az9bFSO30fIoPdnDwOel/B+xBfYB47WfNdo8X5RRnECZZr2B3qKrked3kbaOJ1EW9nX3czbK8++tfV5HoazRuRqpawHKdKQb0BTlZd5P738xEWVa0xal03CwGLkqrK6qcJhdgOp4oCz67NLe7HPAk4Xy/VHm2n8tlK5BmV+9iKI8rqK1lKFsisBD+49wB2UKypCsg1AWjBOAqdq0jlr544FYlMVjf5RRgO53JaAorebac6y15eKB28Vcm3e1dcaiTL+4Fcq/ALxg5NzXUV7iSShWRTZGyhW4VihKbKtWtqtorUq0ecFADMpw/V9gsDZ9uPY+pGrb/KzwtS/Dc+GOMtWRqr3mzxVTdgeQo5VLd/xW0rq0v7/AoZf3vIH8tcXIYrQ8yjSe1MqhL6t/MfXtNFBfT21ed5RnOE17/Ufc45oGautL15Z/RC/PFmXNIlr7zBxH+zLW5r9V6JraAF9x14Ls9UJtPaJ9PtO1bQaaoq6qeAit0CoqKioqNRR1jUBFRUWlhqMqAhUVFZUajqoIVFRUVGo4qiJQUVFRqeGoikBFRUWlhlPt/Ll7enrKwMBAc4uhoqKiUq04duzYbSmll6G8aqcIAgMDOXr0qLnFUFFRUalWCCHCjOWpU0MqKioqNRxVEaioqKjUcFRFoKKiolLDURWBioqKSg3HZIpAG9g5Rghx9h7l2gshcoQQT5tKFkOEhIQQGBiIRqMhMDCQkJCQymxeRUVFpcpgyhHBWhTXwkYRQliguGj904RyFCEkJIQJEyYQFhaGlJKwsDAmTJigKgMVFZUaickUgZRyN4pb4OJ4BfgBxU1wpTFz5kzS0tIKpKWlpTFz5szKFENFRUWlSmC2NQJtEPUngeUlKDtBCHFUCHE0Nja23G2HhxsOpGQsXUVFReV+xpyLxZ+iBBw3FKKuAFLKVVLKdlLKdl5eBjfGlQp/f8OhVY2lq6ioqNzPmFMRtEMJHB4KPA18oQ3kbXLmzZuHvX3B+Nb29vbMmzfPyBkqKioq9y9mczEhpayn+yyEWAtslVJuroy2R4wYAcCbb75JZGQkVlZWrFq1Kj9dRUVFpSZhSvPRjcABoIkQIkII8YIQYqIQYqKp2iwNI0aMoH79+gBkZ2czaNAgM0ukUlZUU2AVlfJhshGBlHJ4Kco+byo5jJGXl8eePXsAeOWVV8jLu+dShUoVRGcKrLMC05kCA+oIT0WlhNTYncV5eXkEBQVRr149tmzZgqurq9qbrIaopsAqKuWnxioCS0tLJk6cyK1bt9SNZdUY1RRYRaX81FhFAGpv8n5ANQVWUSk/NVYRJCUlqb3J+wDVFFhFpfzUWEWwd+9epJQG89TeZPVhxIgRdOvWLf97QECAagqsolJKql2oyorCwsICS0tLcnJyCqSrvcnqh4eHBwDNmjXj3LlzZpZGRaX6UWNHBH379iU7O5vg4GACAgIQQqi9yWpKSEgIXl5eXLhwgS1btphbHBWVakeNHRHoGD58OA8//DCJiYk0a9bM3OKolJG0tDSklNy8edPcoqioVDtqvCLIyMjA19cXwOiagUrVZ8uWLUgpadeunblFUVGpdtTYqaF3330XS0tLOnToAIAQoogpqUr1oFu3bowZM4a4uDhcXFzMLY6KSrWjxo4Ibt++TW5uLsnJyepIoJpz8uRJUlJSuH37trlFUVGpltTYEcHs2bPZv38/v/zyi7lFUSknCxcuxN/fn8WLF7N06VJzi6OiUu2osYrA09OTzp0707p1a3OLolJOJk6ciJubG9euXWP79u3mFkdFpdpRYxWBPo0aNcLR0VF9iVRjRo8ezcCBAxk6dKi5RVFRqXbUWEUwZ84cWrduzbvvvktUVBSpqalERESLine truncated
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "8bbc198529744059b51c19cbca291808",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0, description='Testing', layout=Layout(flex='2'), max=356.0, style=Progr…"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"KeyboardInterrupt, skipping rest of testing\n"
]
}
],
"source": [
"trial, trainer, model = run_trial(name=\"LSTMSeq2Seq_PL\",\n",
" params={},\n",
" user_attrs=default_user_attrs,\n",
" PL_MODEL_CLS=LSTMSeq2Seq_PL)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## ANP-RNN"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2020-04-11T07:34:41.913720Z",
"start_time": "2020-04-11T07:33:39.200Z"
}
},
"outputs": [],
"source": [
"trial, trainer, model = run_trial(\n",
" name=\"anp-rnn\",\n",
@@ -500,12 +566,18 @@
"execution_count": null,
"metadata": {
"ExecuteTime": {
"start_time": "2020-04-11T05:31:34.200Z"
"end_time": "2020-04-11T07:34:41.915176Z",
"start_time": "2020-04-11T07:33:39.200Z"
}
},
"outputs": [],
"source": [
"%debug"
"name = trainer.logger.name\n",
"results[f\"{name}_{trial.number}\"]=dict(\n",
" **trainer.logger.metrics[-1],\n",
" **(trainer.logger.metrics[-2] if len(trainer.logger.metrics)>1 else {})\n",
")\n",
"results[f\"{name}_{trial.number}\"]"
]
},
{
@@ -513,26 +585,12 @@
"execution_count": null,
"metadata": {
"ExecuteTime": {
"start_time": "2020-04-11T05:31:34.200Z"
"end_time": "2020-04-11T05:46:41.864057Z",
"start_time": "2020-04-11T05:46:41.787999Z"
}
},
"outputs": [],
"source": [
"results"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"start_time": "2020-04-11T05:31:34.200Z"
}
},
"outputs": [],
"source": [
"trainer.logger.metrics"
]
"source": []
},
{
"cell_type": "markdown",
@@ -551,7 +609,8 @@
"execution_count": null,
"metadata": {
"ExecuteTime": {
"start_time": "2020-04-11T05:31:34.300Z"
"end_time": "2020-04-11T07:34:41.916492Z",
"start_time": "2020-04-11T07:33:39.200Z"
}
},
"outputs": [],
@@ -574,12 +633,19 @@
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2020-02-15T07:01:21.281138Z",
"start_time": "2020-02-15T07:01:20.834255Z"
"end_time": "2020-04-11T07:34:41.918079Z",
"start_time": "2020-04-11T07:33:39.200Z"
}
},
"outputs": [],
"source": []
"source": [
"name = trainer.logger.name\n",
"results[f\"{name}_{trial.number}\"]=dict(\n",
" **trainer.logger.metrics[-1],\n",
" **(trainer.logger.metrics[-2] if len(trainer.logger.metrics)>1 else {})\n",
")\n",
"results[f\"{name}_{trial.number}\"]"
]
},
{
"cell_type": "markdown",
@@ -599,7 +665,8 @@
"execution_count": null,
"metadata": {
"ExecuteTime": {
"start_time": "2020-04-11T05:31:34.300Z"
"end_time": "2020-04-11T07:34:41.919374Z",
"start_time": "2020-04-11T07:33:39.200Z"
}
},
"outputs": [],
@@ -616,6 +683,25 @@
" PL_MODEL_CLS=PL_NeuralProcess)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2020-04-11T07:34:41.920622Z",
"start_time": "2020-04-11T07:33:39.200Z"
}
},
"outputs": [],
"source": [
"name = trainer.logger.name\n",
"results[f\"{name}_{trial.number}\"]=dict(\n",
" **trainer.logger.metrics[-1],\n",
" **(trainer.logger.metrics[-2] if len(trainer.logger.metrics)>1 else {})\n",
")\n",
"results[f\"{name}_{trial.number}\"]"
]
},
{
"cell_type": "markdown",
"metadata": {},
@@ -628,7 +714,8 @@
"execution_count": null,
"metadata": {
"ExecuteTime": {
"start_time": "2020-04-11T05:31:34.300Z"
"end_time": "2020-04-11T07:34:41.921916Z",
"start_time": "2020-04-11T07:33:39.200Z"
}
},
"outputs": [],
@@ -645,6 +732,26 @@
" PL_MODEL_CLS=PL_NeuralProcess)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2020-04-11T07:34:41.923211Z",
"start_time": "2020-04-11T07:33:39.200Z"
},
"scrolled": true
},
"outputs": [],
"source": [
"name = trainer.logger.name\n",
"results[f\"{name}_{trial.number}\"]=dict(\n",
" **trainer.logger.metrics[-1],\n",
" **(trainer.logger.metrics[-2] if len(trainer.logger.metrics)>1 else {})\n",
")\n",
"results[f\"{name}_{trial.number}\"]"
]
},
{
"cell_type": "markdown",
"metadata": {
@@ -662,23 +769,52 @@
"execution_count": null,
"metadata": {
"ExecuteTime": {
"start_time": "2020-04-11T05:31:34.300Z"
"end_time": "2020-04-11T07:34:41.924517Z",
"start_time": "2020-04-11T07:33:39.200Z"
}
},
"outputs": [],
"source": [
"trial, trainer, model = run_trial(name=\"PL_Transformer\",\n",
" params={\n",
" 'det_enc_cross_attn_type': 'uniform',\n",
" 'det_enc_self_attn_type': 'uniform',\n",
" 'latent_enc_self_attn_type': 'uniform',\n",
" 'use_deterministic_path': False,\n",
" 'use_lvar': False,\n",
" },\n",
" user_attrs=default_user_attrs,\n",
" user_attrs={**default_user_attrs, 'context_in_target': False},\n",
" PL_MODEL_CLS=PL_Transformer)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2020-04-11T07:34:41.925801Z",
"start_time": "2020-04-11T07:33:39.200Z"
}
},
"outputs": [],
"source": [
"# %debug"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2020-04-11T07:34:41.927063Z",
"start_time": "2020-04-11T07:33:39.300Z"
}
},
"outputs": [],
"source": [
"name = trainer.logger.name\n",
"results[f\"{name}_{trial.number}\"]=dict(\n",
" **trainer.logger.metrics[-1],\n",
" **(trainer.logger.metrics[-2] if len(trainer.logger.metrics)>1 else {})\n",
")\n",
"results[f\"{name}_{trial.number}\"]"
]
},
{
"cell_type": "markdown",
"metadata": {},
@@ -691,23 +827,38 @@
"execution_count": null,
"metadata": {
"ExecuteTime": {
"start_time": "2020-04-11T05:31:34.300Z"
"end_time": "2020-04-11T07:34:41.928494Z",
"start_time": "2020-04-11T07:33:39.300Z"
}
},
"outputs": [],
"source": [
"trial, trainer, model = run_trial(name=\"TransformerSeq2Seq_PL\",\n",
" params={\n",
" 'det_enc_cross_attn_type': 'uniform',\n",
" 'det_enc_self_attn_type': 'uniform',\n",
" 'latent_enc_self_attn_type': 'uniform',\n",
" 'use_deterministic_path': False,\n",
" 'use_lvar': False,\n",
" },\n",
" params={},\n",
" user_attrs=default_user_attrs,\n",
" PL_MODEL_CLS=TransformerSeq2Seq_PL)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2020-04-11T07:34:41.929842Z",
"start_time": "2020-04-11T07:33:39.300Z"
},
"scrolled": true
},
"outputs": [],
"source": [
"name = trainer.logger.name\n",
"results[f\"{name}_{trial.number}\"]=dict(\n",
" **trainer.logger.metrics[-1],\n",
" **(trainer.logger.metrics[-2] if len(trainer.logger.metrics)>1 else {})\n",
")\n",
"results[f\"{name}_{trial.number}\"]"
]
},
{
"cell_type": "markdown",
"metadata": {},
@@ -720,23 +871,38 @@
"execution_count": null,
"metadata": {
"ExecuteTime": {
"start_time": "2020-04-11T05:31:34.300Z"
}
"end_time": "2020-04-11T07:34:41.931210Z",
"start_time": "2020-04-11T07:33:39.300Z"
},
"scrolled": true
},
"outputs": [],
"source": [
"trial, trainer, model = run_trial(name=\"LSTM_PL_STD\",\n",
" params={\n",
" 'det_enc_cross_attn_type': 'uniform',\n",
" 'det_enc_self_attn_type': 'uniform',\n",
" 'latent_enc_self_attn_type': 'uniform',\n",
" 'use_deterministic_path': False,\n",
" 'use_lvar': False,\n",
" },\n",
" params={},\n",
" user_attrs=default_user_attrs,\n",
" PL_MODEL_CLS=LSTM_PL_STD)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2020-04-11T07:34:41.932472Z",
"start_time": "2020-04-11T07:33:39.300Z"
}
},
"outputs": [],
"source": [
"name = trainer.logger.name\n",
"results[f\"{name}_{trial.number}\"]=dict(\n",
" **trainer.logger.metrics[-1],\n",
" **(trainer.logger.metrics[-2] if len(trainer.logger.metrics)>1 else {})\n",
")\n",
"results[f\"{name}_{trial.number}\"]"
]
},
{
"cell_type": "markdown",
"metadata": {},
@@ -749,23 +915,37 @@
"execution_count": null,
"metadata": {
"ExecuteTime": {
"start_time": "2020-04-11T05:31:34.300Z"
"end_time": "2020-04-11T07:34:41.934047Z",
"start_time": "2020-04-11T07:33:39.300Z"
}
},
"outputs": [],
"source": [
"trial, trainer, model = run_trial(name=\"LSTMSeq2Seq_PL\",\n",
" params={\n",
" 'det_enc_cross_attn_type': 'uniform',\n",
" 'det_enc_self_attn_type': 'uniform',\n",
" 'latent_enc_self_attn_type': 'uniform',\n",
" 'use_deterministic_path': False,\n",
" 'use_lvar': False,\n",
" },\n",
" params={},\n",
" user_attrs=default_user_attrs,\n",
" PL_MODEL_CLS=LSTMSeq2Seq_PL)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2020-04-11T07:34:41.935335Z",
"start_time": "2020-04-11T07:33:39.300Z"
}
},
"outputs": [],
"source": [
"name = trainer.logger.name\n",
"results[f\"{name}_{trial.number}\"]=dict(\n",
" **trainer.logger.metrics[-1],\n",
" **(trainer.logger.metrics[-2] if len(trainer.logger.metrics)>1 else {})\n",
")\n",
"results[f\"{name}_{trial.number}\"]"
]
},
{
"cell_type": "markdown",
"metadata": {
@@ -783,7 +963,8 @@
"execution_count": null,
"metadata": {
"ExecuteTime": {
"start_time": "2020-04-11T05:31:34.300Z"
"end_time": "2020-04-11T07:34:41.936575Z",
"start_time": "2020-04-11T07:33:39.300Z"
},
"scrolled": true
},
@@ -808,7 +989,8 @@
"execution_count": null,
"metadata": {
"ExecuteTime": {
"start_time": "2020-04-11T05:31:34.300Z"
"end_time": "2020-04-11T07:34:41.937868Z",
"start_time": "2020-04-11T07:33:39.300Z"
},
"scrolled": true
},
@@ -834,7 +1016,8 @@
"execution_count": null,
"metadata": {
"ExecuteTime": {
"start_time": "2020-04-11T05:31:34.300Z"
"end_time": "2020-04-11T07:34:41.939222Z",
"start_time": "2020-04-11T07:33:39.300Z"
},
"scrolled": true
},
@@ -886,7 +1069,8 @@
"execution_count": null,
"metadata": {
"ExecuteTime": {
"start_time": "2020-04-11T05:31:34.300Z"
"end_time": "2020-04-11T07:34:41.940518Z",
"start_time": "2020-04-11T07:33:39.300Z"
}
},
"outputs": [],
@@ -900,7 +1084,8 @@
"execution_count": null,
"metadata": {
"ExecuteTime": {
"start_time": "2020-04-11T05:31:34.300Z"
"end_time": "2020-04-11T07:34:41.941905Z",
"start_time": "2020-04-11T07:33:39.300Z"
}
},
"outputs": [],