wassname/attentive-neural-processes
1
0
Fork 0
mirror of https://github.com/wassname/attentive-neural-processes.git synced 2026-08-14 12:10:10 +08:00
Code Issues Packages Projects Releases Wiki Activity
Files
45117ff491d0ea6811045a7dfa67df5767f10f95
attentive-neural-processes/smartmeters-ANP-RNN.ipynb
T
wassname 2ec7b99bb7 MCDropout
2020-02-16 21:40:39 +08:00

1.9 MiB
Raw Blame History

This notebook uses pytorch lightning & optuna & a Recurrent Attentive Neural Process for Sequential Data (RANPfSQ)

This notebook trains an Attentional Neural Network on timeseries data from smartmeters.

It uses pytorch lighting for the training loop. And Optuna for the hyperparameter optimisation.

It also pushes results to the tensorboard hyperparameter dashboard for examination.

Results on Smartmeter prediction

Model val_loss
ANP-RNN -1.27
ANP-RNN_imp -1.38
ANP -1.3
ANP_impr -1.2
NP -1.3
In [1]:
import sys, re, os, itertools, functools, collections
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import collections
from pathlib import Path
from tqdm.auto import tqdm

import optuna
import pytorch_lightning as pl
from optuna.integration import PyTorchLightningPruningCallback


import math
%matplotlib inline
%reload_ext autoreload
%autoreload 2
In [2]:
import logging
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
logger = logging.getLogger("RANP.ipynb")
In [3]:
import torch
from torch import nn
import torch.nn.functional as F
In [4]:
from src.models.model import LatentModel
from src.data.smart_meter import collate_fns, SmartMeterDataSet, get_smartmeter_df
from src.plot import plot_from_loader
from src.models.lightning_anp import LatentModelPL
from src.dict_logger import DictLogger
In [5]:
# Params
device='cuda'
use_logy=False

Load kaggle smart meter data

In [6]:
df_train, df_test = get_smartmeter_df()
In [7]:
# Show split
df_train['energy(kWh/hh)'].plot(label='train')
df_test['energy(kWh/hh)'].plot(label='test')
plt.title('energy(kWh/hh)')
plt.legend()
Out [7]:
<matplotlib.legend.Legend at 0x7f739446e828>

Train helpers

In [ ]:
In [8]:
PERCENT_TEST_EXAMPLES = 0.5
# EPOCHS = 5
DIR = Path(os.getcwd())
MODEL_DIR = DIR/ 'optuna_result'/ 'anp-rnn2'
name = 'anp-rnn2' # study name
MODEL_DIR.mkdir(parents=True, exist_ok=True)
print(f"now run `tensorboard --logdir {MODEL_DIR}")
now run `tensorboard --logdir /media/wassname/Storage5/projects2/3ST/attentive-neural-processes/optuna_result/anp-rnn2
In [ ]:
In [9]:
def main(trial, train=True):    
    checkpoint_callback = pl.callbacks.ModelCheckpoint(
        os.path.join(MODEL_DIR, name, 'version_{}'.format(trial.number), "chk"), monitor='val_loss', mode="min")

    # The default logger in PyTorch Lightning writes to event files to be consumed by
    # TensorBoard. We create a simple logger instead that holds the log in memory so that the
    # final accuracy can be obtained after optimization. When using the default logger, the
    # final accuracy could be stored in an attribute of the `Trainer` instead.
    logger = DictLogger(MODEL_DIR, name="anp-rnn", version=trial.number)

    trainer = pl.Trainer(
        logger=logger,
        val_percent_check=PERCENT_TEST_EXAMPLES,
        gradient_clip_val=trial.params["grad_clip"],
        checkpoint_callback=checkpoint_callback,
        max_epochs=trial.params['max_nb_epochs'],
        gpus=-1 if torch.cuda.is_available() else None,
        early_stop_callback=PyTorchLightningPruningCallback(trial, monitor='val_loss')
    )
    model = LatentModelPL(trial.params)
    if train:
        trainer.fit(model)
    
    return model, trainer


def add_sugg(trial):
    
    trial.suggest_loguniform("learning_rate", 1e-5, 1e-2)

    trial.suggest_categorical("hidden_dim", [8*2**i for i in range(6)])
    trial.suggest_categorical("latent_dim", [8*2**i for i in range(6)])
    
    trial.suggest_int("attention_layers", 1, 4)
    trial.suggest_categorical("n_latent_encoder_layers", [1, 2, 4, 8])
    trial.suggest_categorical("n_det_encoder_layers", [1, 2, 4, 8])
    trial.suggest_categorical("n_decoder_layers", [1, 2, 4, 8])

    trial.suggest_categorical("dropout", [0, 0.2, 0.5])
    trial.suggest_categorical("attention_dropout", [0, 0.2, 0.5])

    trial.suggest_categorical(
        "latent_enc_self_attn_type", ['uniform', 'multihead', 'ptmultihead']
    )
    trial.suggest_categorical("det_enc_self_attn_type",  ['uniform', 'multihead', 'ptmultihead'])
    trial.suggest_categorical("det_enc_cross_attn_type", ['uniform', 'multihead', 'ptmultihead'])

    trial.suggest_categorical("batchnorm", [False, True])
    trial.suggest_categorical("use_self_attn", [False, True])
    trial.suggest_categorical("use_lvar", [False, True])
    trial.suggest_categorical("use_deterministic_path", [False, True])
    trial.suggest_categorical("use_rnn", [True, False])

    # training specific (for this model)
    trial.suggest_uniform("min_std", 0.005, 0.005)
    trial.suggest_int("grad_clip", 40, 40)
    trial.suggest_int("num_context", 24 * 4, 24 * 4)
    trial.suggest_int("num_extra_target", 24*4, 24*4)
    trial.suggest_int("max_nb_epochs", 10, 10)
    trial.suggest_int("num_workers", 3, 3)
    trial.suggest_int("batch_size", 16, 16)
    trial.suggest_int("num_heads", 8, 8)

    trial.suggest_int("x_dim", 17, 17)
    trial.suggest_int("y_dim", 1, 1)
    trial.suggest_int("vis_i", 670, 670)
    
    trial.suggest_categorical("context_in_target", [True, True])
    
    return trial

def objective(trial):
    # see https://github.com/optuna/optuna/blob/cf6f02d/examples/pytorch_lightning_simple.py
    
    trial = add_sugg(trial)
    
    print('trial', trial.number, 'params', trial.params)
    
    
    # PyTorch Lightning will try to restore model parameters from previous trials if checkpoint
    # filenames match. Therefore, the filenames for each trial must be made unique.
    model, trainer = main(trial)
    
    # also report to tensorboard & print
    print('logger.metrics', model.logger.metrics[-1:])
    model.logger.experiment.add_hparams(trial.params, model.logger.metrics[-1])
    
    return model.logger.metrics[-1]['val_loss']

Default params

In [10]:
default_params = {
 'attention_dropout': 0,
 'attention_layers': 2,
 'batch_size': 16,
 'batchnorm': False,
 'det_enc_cross_attn_type': 'multihead',
 'det_enc_self_attn_type': 'uniform',
 'dropout': 0,
 'grad_clip': 40,
 'hidden_dim': 128,
 'latent_dim': 128,
 'latent_enc_self_attn_type': 'uniform',
 'learning_rate': 0.002,
 'max_nb_epochs': 10,
 'min_std': 0.005,
 'n_decoder_layers': 4,
 'n_det_encoder_layers': 4,
 'n_latent_encoder_layers': 2,
 'num_context': 24*4,
 'num_extra_target': 24*4,
 'num_heads': 8,
 'num_workers': 3,
 'use_deterministic_path': True,
 'use_lvar': True,
 'use_self_attn': True,
 'vis_i': '670',
 'x_dim': 17,
 'y_dim': 1,
 'use_rnn': False,
 'context_in_target': True
}
In [ ]:
In [ ]:

Train ANP-RNN

In [11]:
name = 'anp-rnn'

params =default_params.copy()
params.update({
 'det_enc_cross_attn_type': 'multihead',
 'det_enc_self_attn_type': 'uniform',
 'latent_enc_self_attn_type': 'uniform',
 'use_deterministic_path': True,
 'use_rnn': True
})
trial = optuna.trial.FixedTrial(params)
trial = add_sugg(trial)
trial.number = 2

checkpoint_callback = pl.callbacks.ModelCheckpoint(
    os.path.join(MODEL_DIR, name, 'version_{}'.format(trial.number), "chk"), monitor='val_loss', mode="min")

logger = DictLogger(MODEL_DIR, name="anp", version=trial.number)

trainer = pl.Trainer(
    gradient_clip_val=trial.params["grad_clip"],
    checkpoint_callback=checkpoint_callback,
    max_epochs=trial.params['max_nb_epochs'],
    gpus=-1 if torch.cuda.is_available() else None,
    early_stop_callback=True
)
model = LatentModelPL(trial.params)

trainer.fit(model)

# plot, main metric
loader = model.val_dataloader()[0]
vis_i=670
plot_from_loader(loader, model, i=vis_i)

print(trainer.test(model))
INFO:root:gpu available: True, used: True
INFO:root:VISIBLE GPUS: 0
INFO:root:
                                   Name           Type Params
0                                 model    LatentModel    1 M
1                           model._lstm           LSTM  207 K
2                 model._latent_encoder  LatentEncoder   98 K
3    model._latent_encoder._input_layer         Linear   16 K
4        model._latent_encoder._encoder     ModuleList   32 K
..                                  ...            ...    ...
121    model._decoder._decoder.3.linear         Linear  147 K
122       model._decoder._decoder.3.act           ReLU    0  
123   model._decoder._decoder.3.dropout      Dropout2d    0  
124                model._decoder._mean         Linear  385  
125                 model._decoder._std         Linear  385  

[126 rows x 3 columns]

HBox(children=(FloatProgress(value=0.0, description='Validation sanity check', layout=Layout(flex='2'), max=5.…
step 0, {'val_loss': '1.5693567991256714', 'val/kl': '0.5020207762718201', 'val/mse': '0.2388661950826645', 'val/std': '1.0383363962173462'}


HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …
HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=234.0, style=Pr…
step 2194, {'val_loss': '0.5343308448791504', 'val/kl': '0.0009442351292818785', 'val/mse': '0.007357416208833456', 'val/std': '0.07601924240589142'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=234.0, style=Pr…
step 4389, {'val_loss': '-1.1690877676010132', 'val/kl': '0.00022915728914085776', 'val/mse': '0.005337832495570183', 'val/std': '0.049246110022068024'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=234.0, style=Pr…
step 6584, {'val_loss': '-1.273390769958496', 'val/kl': '0.00022587954299524426', 'val/mse': '0.004663755185902119', 'val/std': '0.050486624240875244'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=234.0, style=Pr…
step 8779, {'val_loss': '-0.9049668312072754', 'val/kl': '0.000260441389400512', 'val/mse': '0.005967526696622372', 'val/std': '0.054492030292749405'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=234.0, style=Pr…
step 10974, {'val_loss': '-1.0017932653427124', 'val/kl': '0.0001905120152514428', 'val/mse': '0.004810153506696224', 'val/std': '0.0477420799434185'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=234.0, style=Pr…
step 13169, {'val_loss': '-0.9930673241615295', 'val/kl': '0.0002287834940943867', 'val/mse': '0.004163599573075771', 'val/std': '0.037404779344797134'}
Epoch     5: reducing learning rate of group 0 to 2.0000e-04.

INFO:root:Epoch 00006: early stopping

INFO:root:
                                   Name           Type Params
0                                 model    LatentModel    1 M
1                           model._lstm           LSTM  207 K
2                 model._latent_encoder  LatentEncoder   98 K
3    model._latent_encoder._input_layer         Linear   16 K
4        model._latent_encoder._encoder     ModuleList   32 K
..                                  ...            ...    ...
121    model._decoder._decoder.3.linear         Linear  147 K
122       model._decoder._decoder.3.act           ReLU    0  
123   model._decoder._decoder.3.dropout      Dropout2d    0  
124                model._decoder._mean         Linear  385  
125                 model._decoder._std         Linear  385  

[126 rows x 3 columns]
INFO:root:model and trainer restored from checkpoint: /media/wassname/Storage5/projects2/3ST/attentive-neural-processes/optuna_result/anp-rnn2/anp-rnn/version_2/chk/_ckpt_epoch_2.ckpt

HBox(children=(FloatProgress(value=0.0, description='Testing', layout=Layout(flex='2'), max=234.0, style=Progr…
step 6584, {'val_loss': '-1.273390769958496', 'val/kl': '0.00022587954299524426', 'val/mse': '0.004663755185902119', 'val/std': '0.050486624240875244'}

None




	

		

		
ANP-RNN 2



		

	


In [13]:
name = 'anp-rnn3'

params =default_params.copy()
params.update({
 'det_enc_cross_attn_type': 'ptmultihead',
 'det_enc_self_attn_type': 'uniform',
 'latent_enc_self_attn_type': 'uniform',
 'use_deterministic_path': False,
 'use_rnn': True
})
trial = optuna.trial.FixedTrial(params)
trial = add_sugg(trial)
trial.number = 2

checkpoint_callback = pl.callbacks.ModelCheckpoint(
    os.path.join(MODEL_DIR, name, 'version_{}'.format(trial.number), "chk"), monitor='val_loss', mode="min")

logger = DictLogger(MODEL_DIR, name="anp", version=trial.number)

trainer = pl.Trainer(
    gradient_clip_val=trial.params["grad_clip"],
    checkpoint_callback=checkpoint_callback,
    max_epochs=trial.params['max_nb_epochs'],
    gpus=-1 if torch.cuda.is_available() else None,
    early_stop_callback=True
)
model = LatentModelPL(trial.params)

trainer.fit(model)

# plot, main metric
loader = model.val_dataloader()[0]
vis_i=670
plot_from_loader(loader, model, i=vis_i)

print(trainer.test(model))
INFO:root:gpu available: True, used: True
INFO:root:VISIBLE GPUS: 0
INFO:root:
                                  Name           Type Params
0                                model    LatentModel  798 K
1                          model._lstm           LSTM  207 K
2                model._latent_encoder  LatentEncoder   98 K
3   model._latent_encoder._input_layer         Linear   16 K
4       model._latent_encoder._encoder     ModuleList   32 K
..                                 ...            ...    ...
70    model._decoder._decoder.3.linear         Linear   65 K
71       model._decoder._decoder.3.act           ReLU    0  
72   model._decoder._decoder.3.dropout      Dropout2d    0  
73                model._decoder._mean         Linear  257  
74                 model._decoder._std         Linear  257  

[75 rows x 3 columns]

HBox(children=(FloatProgress(value=0.0, description='Validation sanity check', layout=Layout(flex='2'), max=5.…
step 0, {'val_loss': '1.6190637350082397', 'val/kl': '0.5031337738037109', 'val/mse': '0.3089987337589264', 'val/std': '1.0617789030075073'}


HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …
HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=234.0, style=Pr…
step 2194, {'val_loss': '-1.3857086896896362', 'val/kl': '0.00048115666140802205', 'val/mse': '0.004232240840792656', 'val/std': '0.07312013953924179'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=234.0, style=Pr…
step 4389, {'val_loss': '-1.344988465309143', 'val/kl': '0.00020843140373472124', 'val/mse': '0.0055994573049247265', 'val/std': '0.07038313895463943'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=234.0, style=Pr…
step 6584, {'val_loss': '-1.0654047727584839', 'val/kl': '0.0002335252647753805', 'val/mse': '0.006818015594035387', 'val/std': '0.06158881261944771'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=234.0, style=Pr…
step 8779, {'val_loss': '-1.3153259754180908', 'val/kl': '0.00018677482148632407', 'val/mse': '0.003928155638277531', 'val/std': '0.046489667147397995'}
Epoch     3: reducing learning rate of group 0 to 2.0000e-04.

INFO:root:Epoch 00004: early stopping

INFO:root:
                                  Name           Type Params
0                                model    LatentModel  798 K
1                          model._lstm           LSTM  207 K
2                model._latent_encoder  LatentEncoder   98 K
3   model._latent_encoder._input_layer         Linear   16 K
4       model._latent_encoder._encoder     ModuleList   32 K
..                                 ...            ...    ...
70    model._decoder._decoder.3.linear         Linear   65 K
71       model._decoder._decoder.3.act           ReLU    0  
72   model._decoder._decoder.3.dropout      Dropout2d    0  
73                model._decoder._mean         Linear  257  
74                 model._decoder._std         Linear  257  

[75 rows x 3 columns]
INFO:root:model and trainer restored from checkpoint: /media/wassname/Storage5/projects2/3ST/attentive-neural-processes/optuna_result/anp-rnn2/anp-rnn3/version_2/chk/_ckpt_epoch_0.ckpt

HBox(children=(FloatProgress(value=0.0, description='Testing', layout=Layout(flex='2'), max=234.0, style=Progr…
step 2194, {'val_loss': '-1.3857086896896362', 'val/kl': '0.00048115666140802205', 'val/mse': '0.004232240840792656', 'val/std': '0.07312013953924179'}

None

In [ ]:
In [14]:
# # plot lots of metrics
# loader = model.val_dataloader()[0]
# for i in range(0, len(loader), 10):
#     plot_from_loader(loader, model, i=i)
#     plt.show()



	

		

		
ANP



		

	


In [15]:
name = 'anp'

params =default_params.copy()
params.update({
 'det_enc_cross_attn_type': 'multihead',
 'det_enc_self_attn_type': 'multihead',
 'latent_enc_self_attn_type': 'multihead',
 'use_deterministic_path': True,
})
trial = optuna.trial.FixedTrial(params)
trial = add_sugg(trial)
trial.number = 3

checkpoint_callback = pl.callbacks.ModelCheckpoint(
    os.path.join(MODEL_DIR, name, 'version_{}'.format(trial.number), "chk"), monitor='val_loss', mode="min")

logger = DictLogger(MODEL_DIR, name="anp", version=trial.number)

trainer = pl.Trainer(
    gradient_clip_val=trial.params["grad_clip"],
    checkpoint_callback=checkpoint_callback,
    max_epochs=trial.params['max_nb_epochs'],
    gpus=-1 if torch.cuda.is_available() else None,
    early_stop_callback=True
)
model = LatentModelPL(trial.params)

trainer.fit(model)

# plot, main metric
loader = model.val_dataloader()[0]
vis_i=670
plot_from_loader(loader, model, i=vis_i)

print(trainer.test(model))
INFO:root:gpu available: True, used: True
INFO:root:VISIBLE GPUS: 0
INFO:root:
                                   Name           Type Params
0                                 model    LatentModel    2 M
1                 model._latent_encoder  LatentEncoder  609 K
2    model._latent_encoder._input_layer         Linear    2 K
3        model._latent_encoder._encoder     ModuleList   32 K
4      model._latent_encoder._encoder.0  NPBlockRelu2d   16 K
..                                  ...            ...    ...
226    model._decoder._decoder.3.linear         Linear  147 K
227       model._decoder._decoder.3.act           ReLU    0  
228   model._decoder._decoder.3.dropout      Dropout2d    0  
229                model._decoder._mean         Linear  385  
230                 model._decoder._std         Linear  385  

[231 rows x 3 columns]

HBox(children=(FloatProgress(value=0.0, description='Validation sanity check', layout=Layout(flex='2'), max=5.…
step 0, {'val_loss': '1.5840158462524414', 'val/kl': '0.4973420202732086', 'val/mse': '0.25478214025497437', 'val/std': '1.0546205043792725'}


HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …
HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=234.0, style=Pr…
step 2194, {'val_loss': '-1.2122845649719238', 'val/kl': '0.0010215543443337083', 'val/mse': '0.007163152098655701', 'val/std': '0.07716882228851318'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=234.0, style=Pr…
step 4389, {'val_loss': '-1.028235673904419', 'val/kl': '0.000353723211446777', 'val/mse': '0.005656410474330187', 'val/std': '0.05565137043595314'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=234.0, style=Pr…
step 6584, {'val_loss': '-0.7582443356513977', 'val/kl': '0.0002442289551254362', 'val/mse': '0.0062664104625582695', 'val/std': '0.04417317360639572'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=234.0, style=Pr…
step 8779, {'val_loss': '-0.9393323659896851', 'val/kl': '0.0001381170586682856', 'val/mse': '0.007535985670983791', 'val/std': '0.05241641029715538'}
Epoch     3: reducing learning rate of group 0 to 2.0000e-04.

INFO:root:Epoch 00004: early stopping

INFO:root:
                                   Name           Type Params
0                                 model    LatentModel    2 M
1                 model._latent_encoder  LatentEncoder  609 K
2    model._latent_encoder._input_layer         Linear    2 K
3        model._latent_encoder._encoder     ModuleList   32 K
4      model._latent_encoder._encoder.0  NPBlockRelu2d   16 K
..                                  ...            ...    ...
226    model._decoder._decoder.3.linear         Linear  147 K
227       model._decoder._decoder.3.act           ReLU    0  
228   model._decoder._decoder.3.dropout      Dropout2d    0  
229                model._decoder._mean         Linear  385  
230                 model._decoder._std         Linear  385  

[231 rows x 3 columns]
INFO:root:model and trainer restored from checkpoint: /media/wassname/Storage5/projects2/3ST/attentive-neural-processes/optuna_result/anp-rnn2/anp/version_3/chk/_ckpt_epoch_0.ckpt

HBox(children=(FloatProgress(value=0.0, description='Testing', layout=Layout(flex='2'), max=234.0, style=Progr…
step 2194, {'val_loss': '-1.2122845649719238', 'val/kl': '0.0010215543443337083', 'val/mse': '0.007163152098655701', 'val/std': '0.07716882228851318'}

None




	

		

		
NP



		

	


In [16]:
name = 'np'

params =default_params.copy()
params.update({
 'det_enc_cross_attn_type': 'uniform',
 'det_enc_self_attn_type': 'uniform',
 'latent_enc_self_attn_type': 'uniform',
    'use_deterministic_path': False,
})
trial = optuna.trial.FixedTrial(params)
trial = add_sugg(trial)
trial.number = 2

checkpoint_callback = pl.callbacks.ModelCheckpoint(
    os.path.join(MODEL_DIR, name, 'version_{}'.format(trial.number), "chk"), monitor='val_loss', mode="min")

logger = DictLogger(MODEL_DIR, name="anp", version=trial.number)

trainer = pl.Trainer(
    gradient_clip_val=trial.params["grad_clip"],
    checkpoint_callback=checkpoint_callback,
    max_epochs=trial.params['max_nb_epochs'],
    gpus=-1 if torch.cuda.is_available() else None,
    early_stop_callback=True
)
model = LatentModelPL(trial.params)

trainer.fit(model)

# plot, main metric
loader = model.val_dataloader()[0]
vis_i=670
plot_from_loader(loader, model, i=vis_i)

print(trainer.test(model))
INFO:root:gpu available: True, used: True
INFO:root:VISIBLE GPUS: 0
INFO:root:
                                  Name           Type Params
0                                model    LatentModel  455 K
1                model._latent_encoder  LatentEncoder   84 K
2   model._latent_encoder._input_layer         Linear    2 K
3       model._latent_encoder._encoder     ModuleList   32 K
4     model._latent_encoder._encoder.0  NPBlockRelu2d   16 K
..                                 ...            ...    ...
67    model._decoder._decoder.3.linear         Linear   65 K
68       model._decoder._decoder.3.act           ReLU    0  
69   model._decoder._decoder.3.dropout      Dropout2d    0  
70                model._decoder._mean         Linear  257  
71                 model._decoder._std         Linear  257  

[72 rows x 3 columns]

HBox(children=(FloatProgress(value=0.0, description='Validation sanity check', layout=Layout(flex='2'), max=5.…
step 0, {'val_loss': '1.5374459028244019', 'val/kl': '0.4961514472961426', 'val/mse': '0.24871626496315002', 'val/std': '0.9974575042724609'}


HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …
HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=234.0, style=Pr…
step 2194, {'val_loss': '-1.3054770231246948', 'val/kl': '0.0014658418949693441', 'val/mse': '0.004981751553714275', 'val/std': '0.059659067541360855'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=234.0, style=Pr…
step 4389, {'val_loss': '-1.3078840970993042', 'val/kl': '0.0003726059221662581', 'val/mse': '0.004040098283439875', 'val/std': '0.04867682605981827'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=234.0, style=Pr…
step 6584, {'val_loss': '-1.023400068283081', 'val/kl': '0.00030945712933316827', 'val/mse': '0.005101347342133522', 'val/std': '0.05553557351231575'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=234.0, style=Pr…
step 8779, {'val_loss': '-0.5684605836868286', 'val/kl': '0.00023415201576426625', 'val/mse': '0.004800163209438324', 'val/std': '0.03970134258270264'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=234.0, style=Pr…
step 10974, {'val_loss': '0.206074520945549', 'val/kl': '0.0002613919787108898', 'val/mse': '0.0049077728763222694', 'val/std': '0.032850708812475204'}
Epoch     4: reducing learning rate of group 0 to 2.0000e-04.

INFO:root:Epoch 00005: early stopping

INFO:root:
                                  Name           Type Params
0                                model    LatentModel  455 K
1                model._latent_encoder  LatentEncoder   84 K
2   model._latent_encoder._input_layer         Linear    2 K
3       model._latent_encoder._encoder     ModuleList   32 K
4     model._latent_encoder._encoder.0  NPBlockRelu2d   16 K
..                                 ...            ...    ...
67    model._decoder._decoder.3.linear         Linear   65 K
68       model._decoder._decoder.3.act           ReLU    0  
69   model._decoder._decoder.3.dropout      Dropout2d    0  
70                model._decoder._mean         Linear  257  
71                 model._decoder._std         Linear  257  

[72 rows x 3 columns]
INFO:root:model and trainer restored from checkpoint: /media/wassname/Storage5/projects2/3ST/attentive-neural-processes/optuna_result/anp-rnn2/np/version_2/chk/_ckpt_epoch_1.ckpt

HBox(children=(FloatProgress(value=0.0, description='Testing', layout=Layout(flex='2'), max=234.0, style=Progr…
step 4389, {'val_loss': '-1.3078840970993042', 'val/kl': '0.0003726059221662581', 'val/mse': '0.004040098283439875', 'val/std': '0.04867682605981827'}

None




	

		

		
Hyperparam



		

	


In [18]:
import argparse 

parser = argparse.ArgumentParser(description='PyTorch Lightning example.')
parser.add_argument('--pruning', '-p', action='store_true',
                    help='Activate the pruning feature. `MedianPruner` stops unpromising '
                         'trials at the early stages of training.')
args = parser.parse_args(['-p'])

pruner = optuna.pruners.MedianPruner(n_warmup_steps=1, n_startup_trials=20) if args.pruning else optuna.pruners.NopPruner()
pruner = optuna.pruners.PercentilePruner(75.0)
name = 'anp-rnn1'
study = optuna.create_study(direction='minimize', pruner=pruner, storage=f'sqlite:///optuna_result/{name}.db', study_name=name, load_if_exists=True)

[I 2020-02-16 12:17:40,768] Using an existing study with name 'anp-rnn1' instead of creating a new one.

In [ ]:
study.optimize(objective, n_trials=200, timeout=pd.Timedelta('3d').total_seconds())
trial 40 params {'attention_dropout': 0.2, 'attention_layers': 1, 'batch_size': 16, 'batchnorm': True, 'context_in_target': True, 'det_enc_cross_attn_type': 'ptmultihead', 'det_enc_self_attn_type': 'multihead', 'dropout': 0, 'grad_clip': 40, 'hidden_dim': 32, 'latent_dim': 64, 'latent_enc_self_attn_type': 'ptmultihead', 'learning_rate': 0.009979117958707866, 'max_nb_epochs': 10, 'min_std': 0.005, 'n_decoder_layers': 8, 'n_det_encoder_layers': 2, 'n_latent_encoder_layers': 2, 'num_context': 96, 'num_extra_target': 96, 'num_heads': 8, 'num_workers': 3, 'use_deterministic_path': False, 'use_lvar': True, 'use_rnn': False, 'use_self_attn': False, 'vis_i': 670, 'x_dim': 17, 'y_dim': 1}

INFO:root:gpu available: True, used: True
INFO:root:VISIBLE GPUS: 0
INFO:root:
                                  Name           Type Params
0                                model    LatentModel   94 K
1                model._latent_encoder  LatentEncoder    8 K
2   model._latent_encoder._input_layer         Linear  608  
3       model._latent_encoder._encoder     ModuleList    2 K
4     model._latent_encoder._encoder.0  NPBlockRelu2d    1 K
..                                 ...            ...    ...
87       model._decoder._decoder.7.act           ReLU    0  
88   model._decoder._decoder.7.dropout      Dropout2d    0  
89      model._decoder._decoder.7.norm    BatchNorm2d  192  
90                model._decoder._mean         Linear   97  
91                 model._decoder._std         Linear   97  

[92 rows x 3 columns]

HBox(children=(FloatProgress(value=0.0, description='Validation sanity check', layout=Layout(flex='2'), max=5.…
step 0, {'val_loss': '1.53555166721344', 'val/kl': '0.5122055411338806', 'val/mse': '0.29894599318504333', 'val/std': '0.9359065890312195'}


HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …
HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 2194, {'val_loss': '-1.3771708011627197', 'val/kl': '0.0009297789074480534', 'val/mse': '0.004502739757299423', 'val/std': '0.06497536599636078'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 4389, {'val_loss': '-1.4605064392089844', 'val/kl': '0.000608345028012991', 'val/mse': '0.004014032427221537', 'val/std': '0.06412609666585922'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 6584, {'val_loss': '8.564395904541016', 'val/kl': '0.006522171664983034', 'val/mse': '24820252672.0', 'val/std': '1585.9449462890625'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 8779, {'val_loss': '-1.5419411659240723', 'val/kl': '0.0009605502709746361', 'val/mse': '0.0035048630088567734', 'val/std': '0.05785220488905907'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 10974, {'val_loss': '-1.4154374599456787', 'val/kl': '0.0018635217566043139', 'val/mse': '0.004522555973380804', 'val/std': '0.05777614936232567'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 13169, {'val_loss': '-1.4261882305145264', 'val/kl': '0.0008237186702899635', 'val/mse': '0.0040297918021678925', 'val/std': '0.05604660511016846'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 15364, {'val_loss': '-1.4118669033050537', 'val/kl': '0.0005158120184205472', 'val/mse': '0.00455522770062089', 'val/std': '0.05931553989648819'}
Epoch     6: reducing learning rate of group 0 to 9.9791e-04.

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 17559, {'val_loss': '-1.5386149883270264', 'val/kl': '0.0005057148518972099', 'val/mse': '0.0036372821778059006', 'val/std': '0.05127168074250221'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 19754, {'val_loss': '-1.5309010744094849', 'val/kl': '0.0003634715103544295', 'val/mse': '0.0035632983781397343', 'val/std': '0.0472351610660553'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 21949, {'val_loss': '-1.5311908721923828', 'val/kl': '0.00033483945298939943', 'val/mse': '0.003650220111012459', 'val/std': '0.04958202689886093'}
Epoch     9: reducing learning rate of group 0 to 9.9791e-05.

logger.metrics [{'val_loss': -1.5311908721923828, 'val/kl': 0.00033483945298939943, 'val/mse': 0.003650220111012459, 'val/std': 0.04958202689886093, 'epoch': 9}]

[I 2020-02-16 12:48:05,046] Finished trial#40 resulted in value: -1.5311908721923828. Current best value is -1.5311908721923828 with parameters: {'attention_dropout': 0.2, 'attention_layers': 1, 'batch_size': 16, 'batchnorm': True, 'context_in_target': True, 'det_enc_cross_attn_type': 'ptmultihead', 'det_enc_self_attn_type': 'multihead', 'dropout': 0, 'grad_clip': 40, 'hidden_dim': 32, 'latent_dim': 64, 'latent_enc_self_attn_type': 'ptmultihead', 'learning_rate': 0.009979117958707866, 'max_nb_epochs': 10, 'min_std': 0.005, 'n_decoder_layers': 8, 'n_det_encoder_layers': 2, 'n_latent_encoder_layers': 2, 'num_context': 96, 'num_extra_target': 96, 'num_heads': 8, 'num_workers': 3, 'use_deterministic_path': False, 'use_lvar': True, 'use_rnn': False, 'use_self_attn': False, 'vis_i': 670, 'x_dim': 17, 'y_dim': 1}.

trial 41 params {'attention_dropout': 0.2, 'attention_layers': 1, 'batch_size': 16, 'batchnorm': True, 'context_in_target': True, 'det_enc_cross_attn_type': 'ptmultihead', 'det_enc_self_attn_type': 'multihead', 'dropout': 0, 'grad_clip': 40, 'hidden_dim': 32, 'latent_dim': 64, 'latent_enc_self_attn_type': 'ptmultihead', 'learning_rate': 0.00825221580671433, 'max_nb_epochs': 10, 'min_std': 0.005, 'n_decoder_layers': 8, 'n_det_encoder_layers': 2, 'n_latent_encoder_layers': 2, 'num_context': 96, 'num_extra_target': 96, 'num_heads': 8, 'num_workers': 3, 'use_deterministic_path': False, 'use_lvar': True, 'use_rnn': False, 'use_self_attn': False, 'vis_i': 670, 'x_dim': 17, 'y_dim': 1}

INFO:root:gpu available: True, used: True
INFO:root:VISIBLE GPUS: 0
INFO:root:
                                  Name           Type Params
0                                model    LatentModel   94 K
1                model._latent_encoder  LatentEncoder    8 K
2   model._latent_encoder._input_layer         Linear  608  
3       model._latent_encoder._encoder     ModuleList    2 K
4     model._latent_encoder._encoder.0  NPBlockRelu2d    1 K
..                                 ...            ...    ...
87       model._decoder._decoder.7.act           ReLU    0  
88   model._decoder._decoder.7.dropout      Dropout2d    0  
89      model._decoder._decoder.7.norm    BatchNorm2d  192  
90                model._decoder._mean         Linear   97  
91                 model._decoder._std         Linear   97  

[92 rows x 3 columns]

HBox(children=(FloatProgress(value=0.0, description='Validation sanity check', layout=Layout(flex='2'), max=5.…
step 0, {'val_loss': '1.5108540058135986', 'val/kl': '0.5089762806892395', 'val/mse': '0.2757703363895416', 'val/std': '0.9246620535850525'}


HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …
HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 2194, {'val_loss': '-1.20550537109375', 'val/kl': '0.0006790390470996499', 'val/mse': '0.006131823640316725', 'val/std': '0.07889589667320251'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 4389, {'val_loss': '-1.2357312440872192', 'val/kl': '0.0007603747071698308', 'val/mse': '0.005223762709647417', 'val/std': '0.05958130210638046'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 6584, {'val_loss': '-1.5108121633529663', 'val/kl': '0.0008597009000368416', 'val/mse': '0.003719923784956336', 'val/std': '0.05571730062365532'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 8779, {'val_loss': '-1.564136266708374', 'val/kl': '0.000897054560482502', 'val/mse': '0.0033463523723185062', 'val/std': '0.05561968311667442'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 10974, {'val_loss': '-1.4187949895858765', 'val/kl': '0.0005841613747179508', 'val/mse': '0.0040751127526164055', 'val/std': '0.05243219807744026'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 13169, {'val_loss': '-1.4812263250350952', 'val/kl': '0.0007523014210164547', 'val/mse': '0.0037672489415854216', 'val/std': '0.049325522035360336'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 15364, {'val_loss': '-1.2123048305511475', 'val/kl': '0.0014134023804217577', 'val/mse': '0.006530694663524628', 'val/std': '0.07238736003637314'}
Epoch     6: reducing learning rate of group 0 to 8.2522e-04.

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 17559, {'val_loss': '-1.4875582456588745', 'val/kl': '0.0005257382290437818', 'val/mse': '0.003740638727322221', 'val/std': '0.044684037566185'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 19754, {'val_loss': '-1.4734346866607666', 'val/kl': '0.00039242839557118714', 'val/mse': '0.003947222139686346', 'val/std': '0.04682338610291481'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 21949, {'val_loss': '-1.4678099155426025', 'val/kl': '0.0003087829682044685', 'val/mse': '0.0037555007729679346', 'val/std': '0.04366892948746681'}
Epoch     9: reducing learning rate of group 0 to 8.2522e-05.

logger.metrics [{'val_loss': -1.4678099155426025, 'val/kl': 0.0003087829682044685, 'val/mse': 0.0037555007729679346, 'val/std': 0.04366892948746681, 'epoch': 9}]

[I 2020-02-16 13:19:27,299] Finished trial#41 resulted in value: -1.4678099155426025. Current best value is -1.5311908721923828 with parameters: {'attention_dropout': 0.2, 'attention_layers': 1, 'batch_size': 16, 'batchnorm': True, 'context_in_target': True, 'det_enc_cross_attn_type': 'ptmultihead', 'det_enc_self_attn_type': 'multihead', 'dropout': 0, 'grad_clip': 40, 'hidden_dim': 32, 'latent_dim': 64, 'latent_enc_self_attn_type': 'ptmultihead', 'learning_rate': 0.009979117958707866, 'max_nb_epochs': 10, 'min_std': 0.005, 'n_decoder_layers': 8, 'n_det_encoder_layers': 2, 'n_latent_encoder_layers': 2, 'num_context': 96, 'num_extra_target': 96, 'num_heads': 8, 'num_workers': 3, 'use_deterministic_path': False, 'use_lvar': True, 'use_rnn': False, 'use_self_attn': False, 'vis_i': 670, 'x_dim': 17, 'y_dim': 1}.

trial 42 params {'attention_dropout': 0.2, 'attention_layers': 1, 'batch_size': 16, 'batchnorm': True, 'context_in_target': True, 'det_enc_cross_attn_type': 'ptmultihead', 'det_enc_self_attn_type': 'multihead', 'dropout': 0, 'grad_clip': 40, 'hidden_dim': 32, 'latent_dim': 64, 'latent_enc_self_attn_type': 'ptmultihead', 'learning_rate': 0.004675772626655719, 'max_nb_epochs': 10, 'min_std': 0.005, 'n_decoder_layers': 8, 'n_det_encoder_layers': 4, 'n_latent_encoder_layers': 2, 'num_context': 96, 'num_extra_target': 96, 'num_heads': 8, 'num_workers': 3, 'use_deterministic_path': False, 'use_lvar': True, 'use_rnn': False, 'use_self_attn': False, 'vis_i': 670, 'x_dim': 17, 'y_dim': 1}

INFO:root:gpu available: True, used: True
INFO:root:VISIBLE GPUS: 0
INFO:root:
                                   Name           Type Params
0                                 model    LatentModel   96 K
1                 model._latent_encoder  LatentEncoder    8 K
2    model._latent_encoder._input_layer         Linear  608  
3        model._latent_encoder._encoder     ModuleList    2 K
4      model._latent_encoder._encoder.0  NPBlockRelu2d    1 K
..                                  ...            ...    ...
97        model._decoder._decoder.7.act           ReLU    0  
98    model._decoder._decoder.7.dropout      Dropout2d    0  
99       model._decoder._decoder.7.norm    BatchNorm2d  192  
100                model._decoder._mean         Linear   97  
101                 model._decoder._std         Linear   97  

[102 rows x 3 columns]

HBox(children=(FloatProgress(value=0.0, description='Validation sanity check', layout=Layout(flex='2'), max=5.…
step 0, {'val_loss': '1.6367992162704468', 'val/kl': '0.5042330622673035', 'val/mse': '0.32986244559288025', 'val/std': '1.0728768110275269'}


HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …
HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 2194, {'val_loss': '-1.3915525674819946', 'val/kl': '0.0012034019455313683', 'val/mse': '0.003971943166106939', 'val/std': '0.07428822666406631'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 4389, {'val_loss': '-1.3431251049041748', 'val/kl': '0.0008934412035159767', 'val/mse': '0.005613301880657673', 'val/std': '0.07552073895931244'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 6584, {'val_loss': '-1.4798474311828613', 'val/kl': '0.0007770339143462479', 'val/mse': '0.004133238922804594', 'val/std': '0.05946439132094383'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 8779, {'val_loss': '-1.4810700416564941', 'val/kl': '0.0005357018671929836', 'val/mse': '0.003644324606284499', 'val/std': '0.05902468413114548'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 10974, {'val_loss': '-1.569221019744873', 'val/kl': '0.0005315328016877174', 'val/mse': '0.003372111590579152', 'val/std': '0.0521407350897789'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 13169, {'val_loss': '-1.3642808198928833', 'val/kl': '0.00046485415077768266', 'val/mse': '0.004272697493433952', 'val/std': '0.04868315905332565'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 15364, {'val_loss': '-1.293749213218689', 'val/kl': '0.0004639705002773553', 'val/mse': '0.00423781294375658', 'val/std': '0.04431043565273285'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 17559, {'val_loss': '-1.3339027166366577', 'val/kl': '0.00040979773621074855', 'val/mse': '0.0038757165893912315', 'val/std': '0.04089698940515518'}
Epoch     7: reducing learning rate of group 0 to 4.6758e-04.

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 19754, {'val_loss': '-1.4316049814224243', 'val/kl': '0.00032472642487846315', 'val/mse': '0.003270149929448962', 'val/std': '0.0393795520067215'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 21949, {'val_loss': '-1.4260458946228027', 'val/kl': '0.0002844082482624799', 'val/mse': '0.0033221307676285505', 'val/std': '0.03993909806013107'}

logger.metrics [{'val_loss': -1.4260458946228027, 'val/kl': 0.0002844082482624799, 'val/mse': 0.0033221307676285505, 'val/std': 0.03993909806013107, 'epoch': 9}]

[I 2020-02-16 13:49:52,848] Finished trial#42 resulted in value: -1.4260458946228027. Current best value is -1.5311908721923828 with parameters: {'attention_dropout': 0.2, 'attention_layers': 1, 'batch_size': 16, 'batchnorm': True, 'context_in_target': True, 'det_enc_cross_attn_type': 'ptmultihead', 'det_enc_self_attn_type': 'multihead', 'dropout': 0, 'grad_clip': 40, 'hidden_dim': 32, 'latent_dim': 64, 'latent_enc_self_attn_type': 'ptmultihead', 'learning_rate': 0.009979117958707866, 'max_nb_epochs': 10, 'min_std': 0.005, 'n_decoder_layers': 8, 'n_det_encoder_layers': 2, 'n_latent_encoder_layers': 2, 'num_context': 96, 'num_extra_target': 96, 'num_heads': 8, 'num_workers': 3, 'use_deterministic_path': False, 'use_lvar': True, 'use_rnn': False, 'use_self_attn': False, 'vis_i': 670, 'x_dim': 17, 'y_dim': 1}.

trial 43 params {'attention_dropout': 0.2, 'attention_layers': 1, 'batch_size': 16, 'batchnorm': True, 'context_in_target': True, 'det_enc_cross_attn_type': 'ptmultihead', 'det_enc_self_attn_type': 'multihead', 'dropout': 0, 'grad_clip': 40, 'hidden_dim': 32, 'latent_dim': 64, 'latent_enc_self_attn_type': 'ptmultihead', 'learning_rate': 0.004368391916372206, 'max_nb_epochs': 10, 'min_std': 0.005, 'n_decoder_layers': 1, 'n_det_encoder_layers': 2, 'n_latent_encoder_layers': 1, 'num_context': 96, 'num_extra_target': 96, 'num_heads': 8, 'num_workers': 3, 'use_deterministic_path': False, 'use_lvar': True, 'use_rnn': False, 'use_self_attn': False, 'vis_i': 670, 'x_dim': 17, 'y_dim': 1}

INFO:root:gpu available: True, used: True
INFO:root:VISIBLE GPUS: 0
INFO:root:
                                                 Name                  Type  \
0                                               model           LatentModel   
1                               model._latent_encoder         LatentEncoder   
2                  model._latent_encoder._input_layer                Linear   
3                      model._latent_encoder._encoder            ModuleList   
4                    model._latent_encoder._encoder.0         NPBlockRelu2d   
5             model._latent_encoder._encoder.0.linear                Linear   
6                model._latent_encoder._encoder.0.act                  ReLU   
7            model._latent_encoder._encoder.0.dropout             Dropout2d   
8               model._latent_encoder._encoder.0.norm           BatchNorm2d   
9            model._latent_encoder._penultimate_layer                Linear   
10                        model._latent_encoder._mean                Linear   
11                     model._latent_encoder._log_var                Linear   
12                       model._deterministic_encoder  DeterministicEncoder   
13          model._deterministic_encoder._input_layer                Linear   
14            model._deterministic_encoder._d_encoder            ModuleList   
15          model._deterministic_encoder._d_encoder.0         NPBlockRelu2d   
16   model._deterministic_encoder._d_encoder.0.linear                Linear   
17      model._deterministic_encoder._d_encoder.0.act                  ReLU   
18  model._deterministic_encoder._d_encoder.0.dropout             Dropout2d   
19     model._deterministic_encoder._d_encoder.0.norm           BatchNorm2d   
20          model._deterministic_encoder._d_encoder.1         NPBlockRelu2d   
21   model._deterministic_encoder._d_encoder.1.linear                Linear   
22      model._deterministic_encoder._d_encoder.1.act                  ReLU   
23  model._deterministic_encoder._d_encoder.1.dropout             Dropout2d   
24     model._deterministic_encoder._d_encoder.1.norm           BatchNorm2d   
25      model._deterministic_encoder._cross_attention             Attention   
26  model._deterministic_encoder._cross_attention....              BatchMLP   
27  model._deterministic_encoder._cross_attention....         NPBlockRelu2d   
28  model._deterministic_encoder._cross_attention....                Linear   
29  model._deterministic_encoder._cross_attention....                  ReLU   
30  model._deterministic_encoder._cross_attention....             Dropout2d   
31  model._deterministic_encoder._cross_attention....            Sequential   
32  model._deterministic_encoder._cross_attention....                Linear   
33  model._deterministic_encoder._cross_attention....              BatchMLP   
34  model._deterministic_encoder._cross_attention....         NPBlockRelu2d   
35  model._deterministic_encoder._cross_attention....                Linear   
36  model._deterministic_encoder._cross_attention....                  ReLU   
37  model._deterministic_encoder._cross_attention....             Dropout2d   
38  model._deterministic_encoder._cross_attention....            Sequential   
39  model._deterministic_encoder._cross_attention....                Linear   
40   model._deterministic_encoder._cross_attention._W    MultiheadAttention   
41  model._deterministic_encoder._cross_attention....                Linear   
42                                     model._decoder               Decoder   
43                   model._decoder._target_transform                Linear   
44                            model._decoder._decoder            ModuleList   
45                          model._decoder._decoder.0         NPBlockRelu2d   
46                   model._decoder._decoder.0.linear                Linear   
47                      model._decoder._decoder.0.act                  ReLU   
48                  model._decoder._decoder.0.dropout             Dropout2d   
49                     model._decoder._decoder.0.norm           BatchNorm2d   
50                               model._decoder._mean                Linear   
51                                model._decoder._std                Linear   

   Params  
0    27 K  
1     6 K  
2   608    
3     1 K  
4     1 K  
5     1 K  
6     0    
7     0    
8    64    
9     1 K  
10    2 K  
11    2 K  
12   10 K  
13  608    
14    2 K  
15    1 K  
16    1 K  
17    0    
18    0    
19   64    
20    1 K  
21    1 K  
22    0    
23    0    
24   64    
25    7 K  
26    1 K  
27  544    
28  544    
29    0    
30    0    
31    0    
32    1 K  
33    1 K  
34  544    
35  544    
36    0    
37    0    
38    0    
39    1 K  
40    4 K  
41    1 K  
42   10 K  
43  576    
44    9 K  
45    9 K  
46    9 K  
47    0    
48    0    
49  192    
50   97    
51   97    

HBox(children=(FloatProgress(value=0.0, description='Validation sanity check', layout=Layout(flex='2'), max=5.…
step 0, {'val_loss': '1.6595051288604736', 'val/kl': '0.5012072920799255', 'val/mse': '0.2947925627231598', 'val/std': '1.1311085224151611'}


HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …
HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 2194, {'val_loss': '-1.0242871046066284', 'val/kl': '0.0017947274027392268', 'val/mse': '0.007185660302639008', 'val/std': '0.09871241450309753'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 4389, {'val_loss': '-1.2936391830444336', 'val/kl': '0.0012684506364166737', 'val/mse': '0.006113472394645214', 'val/std': '0.0684652253985405'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 6584, {'val_loss': '-1.3037265539169312', 'val/kl': '0.0007404569769278169', 'val/mse': '0.004260512068867683', 'val/std': '0.06514627486467361'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 8779, {'val_loss': '-1.4981956481933594', 'val/kl': '0.001371236750856042', 'val/mse': '0.0036893989890813828', 'val/std': '0.06575974076986313'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 10974, {'val_loss': '-1.1481499671936035', 'val/kl': '0.004259718116372824', 'val/mse': '0.0038308214861899614', 'val/std': '0.046711280941963196'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 13169, {'val_loss': '-1.4938817024230957', 'val/kl': '0.0014279276365414262', 'val/mse': '0.0036505504976958036', 'val/std': '0.05014738067984581'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 15364, {'val_loss': '-1.3433947563171387', 'val/kl': '0.001294697285629809', 'val/mse': '0.004627563524991274', 'val/std': '0.060922276228666306'}
Epoch     6: reducing learning rate of group 0 to 4.3684e-04.

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 17559, {'val_loss': '-1.573096513748169', 'val/kl': '0.0014650896191596985', 'val/mse': '0.0032621892169117928', 'val/std': '0.04872477054595947'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 19754, {'val_loss': '-1.568063497543335', 'val/kl': '0.0015724594704806805', 'val/mse': '0.003314936999231577', 'val/std': '0.04916192963719368'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 21949, {'val_loss': '-1.5752214193344116', 'val/kl': '0.001221427577547729', 'val/mse': '0.0032312502153217793', 'val/std': '0.04751383513212204'}

logger.metrics [{'val_loss': -1.5752214193344116, 'val/kl': 0.001221427577547729, 'val/mse': 0.0032312502153217793, 'val/std': 0.04751383513212204, 'epoch': 9}]

[I 2020-02-16 14:10:01,348] Finished trial#43 resulted in value: -1.5752214193344116. Current best value is -1.5752214193344116 with parameters: {'attention_dropout': 0.2, 'attention_layers': 1, 'batch_size': 16, 'batchnorm': True, 'context_in_target': True, 'det_enc_cross_attn_type': 'ptmultihead', 'det_enc_self_attn_type': 'multihead', 'dropout': 0, 'grad_clip': 40, 'hidden_dim': 32, 'latent_dim': 64, 'latent_enc_self_attn_type': 'ptmultihead', 'learning_rate': 0.004368391916372206, 'max_nb_epochs': 10, 'min_std': 0.005, 'n_decoder_layers': 1, 'n_det_encoder_layers': 2, 'n_latent_encoder_layers': 1, 'num_context': 96, 'num_extra_target': 96, 'num_heads': 8, 'num_workers': 3, 'use_deterministic_path': False, 'use_lvar': True, 'use_rnn': False, 'use_self_attn': False, 'vis_i': 670, 'x_dim': 17, 'y_dim': 1}.

trial 44 params {'attention_dropout': 0, 'attention_layers': 1, 'batch_size': 16, 'batchnorm': True, 'context_in_target': True, 'det_enc_cross_attn_type': 'ptmultihead', 'det_enc_self_attn_type': 'ptmultihead', 'dropout': 0.5, 'grad_clip': 40, 'hidden_dim': 64, 'latent_dim': 32, 'latent_enc_self_attn_type': 'multihead', 'learning_rate': 0.0039055042332039043, 'max_nb_epochs': 10, 'min_std': 0.005, 'n_decoder_layers': 1, 'n_det_encoder_layers': 2, 'n_latent_encoder_layers': 1, 'num_context': 96, 'num_extra_target': 96, 'num_heads': 8, 'num_workers': 3, 'use_deterministic_path': False, 'use_lvar': True, 'use_rnn': False, 'use_self_attn': False, 'vis_i': 670, 'x_dim': 17, 'y_dim': 1}

INFO:root:gpu available: True, used: True
INFO:root:VISIBLE GPUS: 0
INFO:root:
                                                 Name                  Type  \
0                                               model           LatentModel   
1                               model._latent_encoder         LatentEncoder   
2                  model._latent_encoder._input_layer                Linear   
3                      model._latent_encoder._encoder            ModuleList   
4                    model._latent_encoder._encoder.0         NPBlockRelu2d   
5             model._latent_encoder._encoder.0.linear                Linear   
6                model._latent_encoder._encoder.0.act                  ReLU   
7            model._latent_encoder._encoder.0.dropout             Dropout2d   
8               model._latent_encoder._encoder.0.norm           BatchNorm2d   
9            model._latent_encoder._penultimate_layer                Linear   
10                        model._latent_encoder._mean                Linear   
11                     model._latent_encoder._log_var                Linear   
12                       model._deterministic_encoder  DeterministicEncoder   
13          model._deterministic_encoder._input_layer                Linear   
14            model._deterministic_encoder._d_encoder            ModuleList   
15          model._deterministic_encoder._d_encoder.0         NPBlockRelu2d   
16   model._deterministic_encoder._d_encoder.0.linear                Linear   
17      model._deterministic_encoder._d_encoder.0.act                  ReLU   
18  model._deterministic_encoder._d_encoder.0.dropout             Dropout2d   
19     model._deterministic_encoder._d_encoder.0.norm           BatchNorm2d   
20          model._deterministic_encoder._d_encoder.1         NPBlockRelu2d   
21   model._deterministic_encoder._d_encoder.1.linear                Linear   
22      model._deterministic_encoder._d_encoder.1.act                  ReLU   
23  model._deterministic_encoder._d_encoder.1.dropout             Dropout2d   
24     model._deterministic_encoder._d_encoder.1.norm           BatchNorm2d   
25      model._deterministic_encoder._cross_attention             Attention   
26  model._deterministic_encoder._cross_attention....              BatchMLP   
27  model._deterministic_encoder._cross_attention....         NPBlockRelu2d   
28  model._deterministic_encoder._cross_attention....                Linear   
29  model._deterministic_encoder._cross_attention....                  ReLU   
30  model._deterministic_encoder._cross_attention....             Dropout2d   
31  model._deterministic_encoder._cross_attention....            Sequential   
32  model._deterministic_encoder._cross_attention....                Linear   
33  model._deterministic_encoder._cross_attention....              BatchMLP   
34  model._deterministic_encoder._cross_attention....         NPBlockRelu2d   
35  model._deterministic_encoder._cross_attention....                Linear   
36  model._deterministic_encoder._cross_attention....                  ReLU   
37  model._deterministic_encoder._cross_attention....             Dropout2d   
38  model._deterministic_encoder._cross_attention....            Sequential   
39  model._deterministic_encoder._cross_attention....                Linear   
40   model._deterministic_encoder._cross_attention._W    MultiheadAttention   
41  model._deterministic_encoder._cross_attention....                Linear   
42                                     model._decoder               Decoder   
43                   model._decoder._target_transform                Linear   
44                            model._decoder._decoder            ModuleList   
45                          model._decoder._decoder.0         NPBlockRelu2d   
46                   model._decoder._decoder.0.linear                Linear   
47                      model._decoder._decoder.0.act                  ReLU   
48                  model._decoder._decoder.0.dropout             Dropout2d   
49                     model._decoder._decoder.0.norm           BatchNorm2d   
50                               model._decoder._mean                Linear   
51                                model._decoder._std                Linear   

   Params  
0    61 K  
1    13 K  
2     1 K  
3     4 K  
4     4 K  
5     4 K  
6     0    
7     0    
8   128    
9     4 K  
10    2 K  
11    2 K  
12   36 K  
13    1 K  
14    8 K  
15    4 K  
16    4 K  
17    0    
18    0    
19  128    
20    4 K  
21    4 K  
22    0    
23    0    
24  128    
25   26 K  
26    5 K  
27    1 K  
28    1 K  
29    0    
30    0    
31    0    
32    4 K  
33    5 K  
34    1 K  
35    1 K  
36    0    
37    0    
38    0    
39    4 K  
40   16 K  
41    4 K  
42   10 K  
43    1 K  
44    9 K  
45    9 K  
46    9 K  
47    0    
48    0    
49  192    
50   97    
51   97    

HBox(children=(FloatProgress(value=0.0, description='Validation sanity check', layout=Layout(flex='2'), max=5.…
step 0, {'val_loss': '1.720916748046875', 'val/kl': '0.49156513810157776', 'val/mse': '0.2860490083694458', 'val/std': '1.2384977340698242'}


HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …
HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 2194, {'val_loss': '-0.958224356174469', 'val/kl': '0.0005997503758408129', 'val/mse': '0.008417556993663311', 'val/std': '0.11981521546840668'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 4389, {'val_loss': '-1.047938585281372', 'val/kl': '0.0007705810130573809', 'val/mse': '0.00699149863794446', 'val/std': '0.1092245802283287'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 6584, {'val_loss': '-0.970568835735321', 'val/kl': '0.001158003113232553', 'val/mse': '0.0072646429762244225', 'val/std': '0.11499355733394623'}

[I 2020-02-16 14:16:11,306] Setting status of trial#44 as TrialState.PRUNED. Trial was pruned at epoch 2.

trial 45 params {'attention_dropout': 0.2, 'attention_layers': 1, 'batch_size': 16, 'batchnorm': True, 'context_in_target': True, 'det_enc_cross_attn_type': 'multihead', 'det_enc_self_attn_type': 'multihead', 'dropout': 0, 'grad_clip': 40, 'hidden_dim': 32, 'latent_dim': 64, 'latent_enc_self_attn_type': 'ptmultihead', 'learning_rate': 0.0019041317749200822, 'max_nb_epochs': 10, 'min_std': 0.005, 'n_decoder_layers': 1, 'n_det_encoder_layers': 4, 'n_latent_encoder_layers': 1, 'num_context': 96, 'num_extra_target': 96, 'num_heads': 8, 'num_workers': 3, 'use_deterministic_path': False, 'use_lvar': True, 'use_rnn': False, 'use_self_attn': False, 'vis_i': 670, 'x_dim': 17, 'y_dim': 1}

INFO:root:gpu available: True, used: True
INFO:root:VISIBLE GPUS: 0
INFO:root:
                                   Name           Type Params
0                                 model    LatentModel   58 K
1                 model._latent_encoder  LatentEncoder    6 K
2    model._latent_encoder._input_layer         Linear  608  
3        model._latent_encoder._encoder     ModuleList    1 K
4      model._latent_encoder._encoder.0  NPBlockRelu2d    1 K
..                                  ...            ...    ...
108       model._decoder._decoder.0.act           ReLU    0  
109   model._decoder._decoder.0.dropout      Dropout2d    0  
110      model._decoder._decoder.0.norm    BatchNorm2d  192  
111                model._decoder._mean         Linear   97  
112                 model._decoder._std         Linear   97  

[113 rows x 3 columns]

HBox(children=(FloatProgress(value=0.0, description='Validation sanity check', layout=Layout(flex='2'), max=5.…
step 0, {'val_loss': '1.687723994255066', 'val/kl': '0.5016866326332092', 'val/mse': '0.2728389501571655', 'val/std': '1.18678879737854'}


HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …
HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 2194, {'val_loss': '-1.02152419090271', 'val/kl': '0.010310388170182705', 'val/mse': '0.007663198281079531', 'val/std': '0.11175099015235901'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 4389, {'val_loss': '-1.4133368730545044', 'val/kl': '0.0045659178867936134', 'val/mse': '0.004236111883074045', 'val/std': '0.0618896484375'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 6584, {'val_loss': '-1.3207731246948242', 'val/kl': '0.004010752309113741', 'val/mse': '0.004357465542852879', 'val/std': '0.057210419327020645'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 8779, {'val_loss': '-1.3673999309539795', 'val/kl': '0.001945743802934885', 'val/mse': '0.00422749063000083', 'val/std': '0.06024956330657005'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 10974, {'val_loss': '-1.1022073030471802', 'val/kl': '0.0028823784086853266', 'val/mse': '0.004391775466501713', 'val/std': '0.055085066705942154'}
Epoch     4: reducing learning rate of group 0 to 1.9041e-04.

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 13169, {'val_loss': '-0.7505188584327698', 'val/kl': '0.003391333855688572', 'val/mse': '0.004430678673088551', 'val/std': '0.051771990954875946'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 15364, {'val_loss': '-1.0067957639694214', 'val/kl': '0.003156122751533985', 'val/mse': '0.004495357163250446', 'val/std': '0.05695757642388344'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 17559, {'val_loss': '-1.3625431060791016', 'val/kl': '0.0014195841504260898', 'val/mse': '0.0037416103295981884', 'val/std': '0.0520796924829483'}
Epoch     7: reducing learning rate of group 0 to 1.9041e-05.

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 19754, {'val_loss': '49.18473815917969', 'val/kl': '0.032739244401454926', 'val/mse': '0.020993946120142937', 'val/std': '0.04838518053293228'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 21949, {'val_loss': '-1.3830543756484985', 'val/kl': '0.0012432830408215523', 'val/mse': '0.003595026209950447', 'val/std': '0.05097007006406784'}

logger.metrics [{'val_loss': -1.3830543756484985, 'val/kl': 0.0012432830408215523, 'val/mse': 0.003595026209950447, 'val/std': 0.05097007006406784, 'epoch': 9}]

[I 2020-02-16 14:36:15,872] Finished trial#45 resulted in value: -1.3830543756484985. Current best value is -1.5752214193344116 with parameters: {'attention_dropout': 0.2, 'attention_layers': 1, 'batch_size': 16, 'batchnorm': True, 'context_in_target': True, 'det_enc_cross_attn_type': 'ptmultihead', 'det_enc_self_attn_type': 'multihead', 'dropout': 0, 'grad_clip': 40, 'hidden_dim': 32, 'latent_dim': 64, 'latent_enc_self_attn_type': 'ptmultihead', 'learning_rate': 0.004368391916372206, 'max_nb_epochs': 10, 'min_std': 0.005, 'n_decoder_layers': 1, 'n_det_encoder_layers': 2, 'n_latent_encoder_layers': 1, 'num_context': 96, 'num_extra_target': 96, 'num_heads': 8, 'num_workers': 3, 'use_deterministic_path': False, 'use_lvar': True, 'use_rnn': False, 'use_self_attn': False, 'vis_i': 670, 'x_dim': 17, 'y_dim': 1}.

trial 46 params {'attention_dropout': 0.2, 'attention_layers': 1, 'batch_size': 16, 'batchnorm': True, 'context_in_target': True, 'det_enc_cross_attn_type': 'ptmultihead', 'det_enc_self_attn_type': 'multihead', 'dropout': 0, 'grad_clip': 40, 'hidden_dim': 32, 'latent_dim': 8, 'latent_enc_self_attn_type': 'ptmultihead', 'learning_rate': 0.00976440374071206, 'max_nb_epochs': 10, 'min_std': 0.005, 'n_decoder_layers': 1, 'n_det_encoder_layers': 2, 'n_latent_encoder_layers': 1, 'num_context': 96, 'num_extra_target': 96, 'num_heads': 8, 'num_workers': 3, 'use_deterministic_path': False, 'use_lvar': False, 'use_rnn': True, 'use_self_attn': False, 'vis_i': 670, 'x_dim': 17, 'y_dim': 1}

INFO:root:gpu available: True, used: True
INFO:root:VISIBLE GPUS: 0
INFO:root:
                                                 Name                  Type  \
0                                               model           LatentModel   
1                                         model._lstm                  LSTM   
2                               model._latent_encoder         LatentEncoder   
3                  model._latent_encoder._input_layer                Linear   
4                      model._latent_encoder._encoder            ModuleList   
5                    model._latent_encoder._encoder.0         NPBlockRelu2d   
6             model._latent_encoder._encoder.0.linear                Linear   
7                model._latent_encoder._encoder.0.act                  ReLU   
8            model._latent_encoder._encoder.0.dropout             Dropout2d   
9               model._latent_encoder._encoder.0.norm           BatchNorm2d   
10           model._latent_encoder._penultimate_layer                Linear   
11                        model._latent_encoder._mean                Linear   
12                     model._latent_encoder._log_var                Linear   
13                       model._deterministic_encoder  DeterministicEncoder   
14          model._deterministic_encoder._input_layer                Linear   
15            model._deterministic_encoder._d_encoder            ModuleList   
16          model._deterministic_encoder._d_encoder.0         NPBlockRelu2d   
17   model._deterministic_encoder._d_encoder.0.linear                Linear   
18      model._deterministic_encoder._d_encoder.0.act                  ReLU   
19  model._deterministic_encoder._d_encoder.0.dropout             Dropout2d   
20     model._deterministic_encoder._d_encoder.0.norm           BatchNorm2d   
21          model._deterministic_encoder._d_encoder.1         NPBlockRelu2d   
22   model._deterministic_encoder._d_encoder.1.linear                Linear   
23      model._deterministic_encoder._d_encoder.1.act                  ReLU   
24  model._deterministic_encoder._d_encoder.1.dropout             Dropout2d   
25     model._deterministic_encoder._d_encoder.1.norm           BatchNorm2d   
26      model._deterministic_encoder._cross_attention             Attention   
27  model._deterministic_encoder._cross_attention....              BatchMLP   
28  model._deterministic_encoder._cross_attention....         NPBlockRelu2d   
29  model._deterministic_encoder._cross_attention....                Linear   
30  model._deterministic_encoder._cross_attention....                  ReLU   
31  model._deterministic_encoder._cross_attention....             Dropout2d   
32  model._deterministic_encoder._cross_attention....            Sequential   
33  model._deterministic_encoder._cross_attention....                Linear   
34  model._deterministic_encoder._cross_attention....              BatchMLP   
35  model._deterministic_encoder._cross_attention....         NPBlockRelu2d   
36  model._deterministic_encoder._cross_attention....                Linear   
37  model._deterministic_encoder._cross_attention....                  ReLU   
38  model._deterministic_encoder._cross_attention....             Dropout2d   
39  model._deterministic_encoder._cross_attention....            Sequential   
40  model._deterministic_encoder._cross_attention....                Linear   
41   model._deterministic_encoder._cross_attention._W    MultiheadAttention   
42  model._deterministic_encoder._cross_attention....                Linear   
43                                     model._decoder               Decoder   
44                   model._decoder._target_transform                Linear   
45                            model._decoder._decoder            ModuleList   
46                          model._decoder._decoder.0         NPBlockRelu2d   
47                   model._decoder._decoder.0.linear                Linear   
48                      model._decoder._decoder.0.act                  ReLU   
49                  model._decoder._decoder.0.dropout             Dropout2d   
50                     model._decoder._decoder.0.norm           BatchNorm2d   
51                               model._decoder._mean                Linear   
52                                model._decoder._std                Linear   

   Params  
0    24 K  
1     6 K  
2     3 K  
3     1 K  
4     1 K  
5     1 K  
6     1 K  
7     0    
8     0    
9    64    
10    1 K  
11  264    
12  264    
13   11 K  
14    1 K  
15    2 K  
16    1 K  
17    1 K  
18    0    
19    0    
20   64    
21    1 K  
22    1 K  
23    0    
24    0    
25   64    
26    8 K  
27    2 K  
28    1 K  
29    1 K  
30    0    
31    0    
32    0    
33    1 K  
34    2 K  
35    1 K  
36    1 K  
37    0    
38    0    
39    0    
40    1 K  
41    4 K  
42    1 K  
43    2 K  
44    1 K  
45    1 K  
46    1 K  
47    1 K  
48    0    
49    0    
50   80    
51   41    
52   41    

HBox(children=(FloatProgress(value=0.0, description='Validation sanity check', layout=Layout(flex='2'), max=5.…
step 0, {'val_loss': '0.731191098690033', 'val/kl': '6.823187050031265e-06', 'val/mse': '0.20255911350250244', 'val/std': '0.656095027923584'}


HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …
HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 2194, {'val_loss': '-1.278185486793518', 'val/kl': '0.0012379628606140614', 'val/mse': '0.005873736459761858', 'val/std': '0.08137188106775284'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 4389, {'val_loss': '-1.1814210414886475', 'val/kl': '0.0008792407461442053', 'val/mse': '0.005760528147220612', 'val/std': '0.09953497350215912'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 6584, {'val_loss': '-1.2550276517868042', 'val/kl': '0.0017982054268941283', 'val/mse': '0.004931292030960321', 'val/std': '0.05580282211303711'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 8779, {'val_loss': '-1.052121639251709', 'val/kl': '0.0018478278070688248', 'val/mse': '0.0065329116769135', 'val/std': '0.052982039749622345'}
Epoch     3: reducing learning rate of group 0 to 9.7644e-04.

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 10974, {'val_loss': '-1.322417974472046', 'val/kl': '0.0014159505954012275', 'val/mse': '0.004260997287929058', 'val/std': '0.04744671657681465'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 13169, {'val_loss': '-1.2634797096252441', 'val/kl': '0.001777377910912037', 'val/mse': '0.00419056648388505', 'val/std': '0.05102219060063362'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 15364, {'val_loss': '-1.2476696968078613', 'val/kl': '0.001151483622379601', 'val/mse': '0.004098773002624512', 'val/std': '0.046411026269197464'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 17559, {'val_loss': '-1.2291877269744873', 'val/kl': '0.0011489269090816379', 'val/mse': '0.004238182213157415', 'val/std': '0.04569748416543007'}
Epoch     7: reducing learning rate of group 0 to 9.7644e-05.

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 19754, {'val_loss': '-1.2095108032226562', 'val/kl': '0.0013189006131142378', 'val/mse': '0.004299731459468603', 'val/std': '0.045542292296886444'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 21949, {'val_loss': '-1.116992712020874', 'val/kl': '0.0013356534764170647', 'val/mse': '0.00445883022621274', 'val/std': '0.045566070824861526'}

logger.metrics [{'val_loss': -1.116992712020874, 'val/kl': 0.0013356534764170647, 'val/mse': 0.00445883022621274, 'val/std': 0.045566070824861526, 'epoch': 9}]

[I 2020-02-16 14:57:35,834] Finished trial#46 resulted in value: -1.116992712020874. Current best value is -1.5752214193344116 with parameters: {'attention_dropout': 0.2, 'attention_layers': 1, 'batch_size': 16, 'batchnorm': True, 'context_in_target': True, 'det_enc_cross_attn_type': 'ptmultihead', 'det_enc_self_attn_type': 'multihead', 'dropout': 0, 'grad_clip': 40, 'hidden_dim': 32, 'latent_dim': 64, 'latent_enc_self_attn_type': 'ptmultihead', 'learning_rate': 0.004368391916372206, 'max_nb_epochs': 10, 'min_std': 0.005, 'n_decoder_layers': 1, 'n_det_encoder_layers': 2, 'n_latent_encoder_layers': 1, 'num_context': 96, 'num_extra_target': 96, 'num_heads': 8, 'num_workers': 3, 'use_deterministic_path': False, 'use_lvar': True, 'use_rnn': False, 'use_self_attn': False, 'vis_i': 670, 'x_dim': 17, 'y_dim': 1}.

trial 47 params {'attention_dropout': 0, 'attention_layers': 1, 'batch_size': 16, 'batchnorm': True, 'context_in_target': True, 'det_enc_cross_attn_type': 'ptmultihead', 'det_enc_self_attn_type': 'multihead', 'dropout': 0, 'grad_clip': 40, 'hidden_dim': 64, 'latent_dim': 64, 'latent_enc_self_attn_type': 'multihead', 'learning_rate': 0.005578860461082355, 'max_nb_epochs': 10, 'min_std': 0.005, 'n_decoder_layers': 4, 'n_det_encoder_layers': 2, 'n_latent_encoder_layers': 1, 'num_context': 96, 'num_extra_target': 96, 'num_heads': 8, 'num_workers': 3, 'use_deterministic_path': False, 'use_lvar': True, 'use_rnn': False, 'use_self_attn': False, 'vis_i': 670, 'x_dim': 17, 'y_dim': 1}

INFO:root:gpu available: True, used: True
INFO:root:VISIBLE GPUS: 0
INFO:root:
                                  Name           Type Params
0                                model    LatentModel  122 K
1                model._latent_encoder  LatentEncoder   17 K
2   model._latent_encoder._input_layer         Linear    1 K
3       model._latent_encoder._encoder     ModuleList    4 K
4     model._latent_encoder._encoder.0  NPBlockRelu2d    4 K
..                                 ...            ...    ...
62       model._decoder._decoder.3.act           ReLU    0  
63   model._decoder._decoder.3.dropout      Dropout2d    0  
64      model._decoder._decoder.3.norm    BatchNorm2d  256  
65                model._decoder._mean         Linear  129  
66                 model._decoder._std         Linear  129  

[67 rows x 3 columns]

HBox(children=(FloatProgress(value=0.0, description='Validation sanity check', layout=Layout(flex='2'), max=5.…
step 0, {'val_loss': '1.5845451354980469', 'val/kl': '0.5105976462364197', 'val/mse': '0.32177481055259705', 'val/std': '0.9911500811576843'}


HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …
HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 2194, {'val_loss': '-1.2954986095428467', 'val/kl': '0.0017858344363048673', 'val/mse': '0.00484514981508255', 'val/std': '0.058575764298439026'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 4389, {'val_loss': '-1.3119332790374756', 'val/kl': '0.010542633943259716', 'val/mse': '0.0046494826674461365', 'val/std': '0.08081772178411484'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 6584, {'val_loss': '-1.4577152729034424', 'val/kl': '0.0015178888570517302', 'val/mse': '0.003925465513020754', 'val/std': '0.062119293957948685'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 8779, {'val_loss': '-1.3628203868865967', 'val/kl': '0.005198372062295675', 'val/mse': '0.004551110789179802', 'val/std': '0.06566743552684784'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 10974, {'val_loss': '-1.2501224279403687', 'val/kl': '0.0016630118479952216', 'val/mse': '0.0055082859471440315', 'val/std': '0.07815753668546677'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 13169, {'val_loss': '-1.3890389204025269', 'val/kl': '0.0008469870081171393', 'val/mse': '0.003969137091189623', 'val/std': '0.047608278691768646'}
Epoch     5: reducing learning rate of group 0 to 5.5789e-04.

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 15364, {'val_loss': '-1.3288187980651855', 'val/kl': '0.0007632175111211836', 'val/mse': '0.003451170166954398', 'val/std': '0.03932596743106842'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 17559, {'val_loss': '-1.2176209688186646', 'val/kl': '0.0007735801045782864', 'val/mse': '0.0036017836537212133', 'val/std': '0.03742733225226402'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 19754, {'val_loss': '-1.2690728902816772', 'val/kl': '0.0007958909845910966', 'val/mse': '0.0034207545686513186', 'val/std': '0.03738531842827797'}
Epoch     8: reducing learning rate of group 0 to 5.5789e-05.

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 21949, {'val_loss': '-1.0883088111877441', 'val/kl': '0.0007551736780442297', 'val/mse': '0.00359739875420928', 'val/std': '0.034629255533218384'}

logger.metrics [{'val_loss': -1.0883088111877441, 'val/kl': 0.0007551736780442297, 'val/mse': 0.00359739875420928, 'val/std': 0.034629255533218384, 'epoch': 9}]

[I 2020-02-16 15:18:42,703] Finished trial#47 resulted in value: -1.0883088111877441. Current best value is -1.5752214193344116 with parameters: {'attention_dropout': 0.2, 'attention_layers': 1, 'batch_size': 16, 'batchnorm': True, 'context_in_target': True, 'det_enc_cross_attn_type': 'ptmultihead', 'det_enc_self_attn_type': 'multihead', 'dropout': 0, 'grad_clip': 40, 'hidden_dim': 32, 'latent_dim': 64, 'latent_enc_self_attn_type': 'ptmultihead', 'learning_rate': 0.004368391916372206, 'max_nb_epochs': 10, 'min_std': 0.005, 'n_decoder_layers': 1, 'n_det_encoder_layers': 2, 'n_latent_encoder_layers': 1, 'num_context': 96, 'num_extra_target': 96, 'num_heads': 8, 'num_workers': 3, 'use_deterministic_path': False, 'use_lvar': True, 'use_rnn': False, 'use_self_attn': False, 'vis_i': 670, 'x_dim': 17, 'y_dim': 1}.

trial 48 params {'attention_dropout': 0.2, 'attention_layers': 1, 'batch_size': 16, 'batchnorm': True, 'context_in_target': True, 'det_enc_cross_attn_type': 'multihead', 'det_enc_self_attn_type': 'ptmultihead', 'dropout': 0.2, 'grad_clip': 40, 'hidden_dim': 32, 'latent_dim': 16, 'latent_enc_self_attn_type': 'uniform', 'learning_rate': 0.0007489973570107983, 'max_nb_epochs': 10, 'min_std': 0.005, 'n_decoder_layers': 1, 'n_det_encoder_layers': 8, 'n_latent_encoder_layers': 1, 'num_context': 96, 'num_extra_target': 96, 'num_heads': 8, 'num_workers': 3, 'use_deterministic_path': False, 'use_lvar': True, 'use_rnn': False, 'use_self_attn': False, 'vis_i': 670, 'x_dim': 17, 'y_dim': 1}

INFO:root:gpu available: True, used: True
INFO:root:VISIBLE GPUS: 0
INFO:root:
                                   Name           Type Params
0                                 model    LatentModel   52 K
1                 model._latent_encoder  LatentEncoder    3 K
2    model._latent_encoder._input_layer         Linear  608  
3        model._latent_encoder._encoder     ModuleList    1 K
4      model._latent_encoder._encoder.0  NPBlockRelu2d    1 K
..                                  ...            ...    ...
128       model._decoder._decoder.0.act           ReLU    0  
129   model._decoder._decoder.0.dropout      Dropout2d    0  
130      model._decoder._decoder.0.norm    BatchNorm2d   96  
131                model._decoder._mean         Linear   49  
132                 model._decoder._std         Linear   49  

[133 rows x 3 columns]

HBox(children=(FloatProgress(value=0.0, description='Validation sanity check', layout=Layout(flex='2'), max=5.…
step 0, {'val_loss': '1.7075859308242798', 'val/kl': '0.5058380365371704', 'val/mse': '0.22975042462348938', 'val/std': '1.2318896055221558'}


HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …
HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 2194, {'val_loss': '-0.8427137732505798', 'val/kl': '0.0019351966911926866', 'val/mse': '0.011691011488437653', 'val/std': '0.12049313634634018'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 4389, {'val_loss': '-0.8563151955604553', 'val/kl': '0.001034051994793117', 'val/mse': '0.010801645927131176', 'val/std': '0.12607762217521667'}

[I 2020-02-16 15:23:24,637] Setting status of trial#48 as TrialState.PRUNED. Trial was pruned at epoch 1.

trial 49 params {'attention_dropout': 0.2, 'attention_layers': 1, 'batch_size': 16, 'batchnorm': True, 'context_in_target': True, 'det_enc_cross_attn_type': 'ptmultihead', 'det_enc_self_attn_type': 'multihead', 'dropout': 0.5, 'grad_clip': 40, 'hidden_dim': 16, 'latent_dim': 64, 'latent_enc_self_attn_type': 'ptmultihead', 'learning_rate': 0.003667694327569568, 'max_nb_epochs': 10, 'min_std': 0.005, 'n_decoder_layers': 1, 'n_det_encoder_layers': 2, 'n_latent_encoder_layers': 8, 'num_context': 96, 'num_extra_target': 96, 'num_heads': 8, 'num_workers': 3, 'use_deterministic_path': False, 'use_lvar': True, 'use_rnn': False, 'use_self_attn': False, 'vis_i': 670, 'x_dim': 17, 'y_dim': 1}

INFO:root:gpu available: True, used: True
INFO:root:VISIBLE GPUS: 0
INFO:root:
                                  Name           Type Params
0                                model    LatentModel   15 K
1                model._latent_encoder  LatentEncoder    5 K
2   model._latent_encoder._input_layer         Linear  304  
3       model._latent_encoder._encoder     ModuleList    2 K
4     model._latent_encoder._encoder.0  NPBlockRelu2d  288  
..                                 ...            ...    ...
82       model._decoder._decoder.0.act           ReLU    0  
83   model._decoder._decoder.0.dropout      Dropout2d    0  
84      model._decoder._decoder.0.norm    BatchNorm2d  160  
85                model._decoder._mean         Linear   81  
86                 model._decoder._std         Linear   81  

[87 rows x 3 columns]

HBox(children=(FloatProgress(value=0.0, description='Validation sanity check', layout=Layout(flex='2'), max=5.…
step 0, {'val_loss': '1.5659270286560059', 'val/kl': '0.5043103098869324', 'val/mse': '0.2756112515926361', 'val/std': '1.0073413848876953'}


HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …
HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 2194, {'val_loss': '-0.5222434997558594', 'val/kl': '0.0002722125791478902', 'val/mse': '0.018109584227204323', 'val/std': '0.19142760336399078'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 4389, {'val_loss': '-0.42250725626945496', 'val/kl': '5.115848398418166e-05', 'val/mse': '0.02268870547413826', 'val/std': '0.20704622566699982'}

[I 2020-02-16 15:29:34,381] Setting status of trial#49 as TrialState.PRUNED. Trial was pruned at epoch 1.

trial 50 params {'attention_dropout': 0.2, 'attention_layers': 3, 'batch_size': 16, 'batchnorm': True, 'context_in_target': True, 'det_enc_cross_attn_type': 'ptmultihead', 'det_enc_self_attn_type': 'multihead', 'dropout': 0, 'grad_clip': 40, 'hidden_dim': 32, 'latent_dim': 64, 'latent_enc_self_attn_type': 'ptmultihead', 'learning_rate': 3.1088917887340893e-05, 'max_nb_epochs': 10, 'min_std': 0.005, 'n_decoder_layers': 4, 'n_det_encoder_layers': 2, 'n_latent_encoder_layers': 2, 'num_context': 96, 'num_extra_target': 96, 'num_heads': 8, 'num_workers': 3, 'use_deterministic_path': False, 'use_lvar': True, 'use_rnn': False, 'use_self_attn': False, 'vis_i': 670, 'x_dim': 17, 'y_dim': 1}

INFO:root:gpu available: True, used: True
INFO:root:VISIBLE GPUS: 0
INFO:root:
                                  Name           Type Params
0                                model    LatentModel   58 K
1                model._latent_encoder  LatentEncoder    8 K
2   model._latent_encoder._input_layer         Linear  608  
3       model._latent_encoder._encoder     ModuleList    2 K
4     model._latent_encoder._encoder.0  NPBlockRelu2d    1 K
..                                 ...            ...    ...
75       model._decoder._decoder.3.act           ReLU    0  
76   model._decoder._decoder.3.dropout      Dropout2d    0  
77      model._decoder._decoder.3.norm    BatchNorm2d  192  
78                model._decoder._mean         Linear   97  
79                 model._decoder._std         Linear   97  

[80 rows x 3 columns]

HBox(children=(FloatProgress(value=0.0, description='Validation sanity check', layout=Layout(flex='2'), max=5.…
step 0, {'val_loss': '1.4470834732055664', 'val/kl': '0.49991416931152344', 'val/mse': '0.18826858699321747', 'val/std': '0.9204845428466797'}


HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …
HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 2194, {'val_loss': '0.7972294092178345', 'val/kl': '0.3388958275318146', 'val/mse': '0.06819967180490494', 'val/std': '0.39780858159065247'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 4389, {'val_loss': '0.12696629762649536', 'val/kl': '0.14482451975345612', 'val/mse': '0.044183190912008286', 'val/std': '0.2921614944934845'}

[I 2020-02-16 15:33:53,417] Setting status of trial#50 as TrialState.PRUNED. Trial was pruned at epoch 1.

trial 51 params {'attention_dropout': 0.2, 'attention_layers': 2, 'batch_size': 16, 'batchnorm': True, 'context_in_target': True, 'det_enc_cross_attn_type': 'ptmultihead', 'det_enc_self_attn_type': 'multihead', 'dropout': 0, 'grad_clip': 40, 'hidden_dim': 32, 'latent_dim': 64, 'latent_enc_self_attn_type': 'ptmultihead', 'learning_rate': 0.007193614773934935, 'max_nb_epochs': 10, 'min_std': 0.005, 'n_decoder_layers': 8, 'n_det_encoder_layers': 2, 'n_latent_encoder_layers': 1, 'num_context': 96, 'num_extra_target': 96, 'num_heads': 8, 'num_workers': 3, 'use_deterministic_path': False, 'use_lvar': True, 'use_rnn': False, 'use_self_attn': False, 'vis_i': 670, 'x_dim': 17, 'y_dim': 1}

INFO:root:gpu available: True, used: True
INFO:root:VISIBLE GPUS: 0
INFO:root:
                                  Name           Type Params
0                                model    LatentModel   93 K
1                model._latent_encoder  LatentEncoder    6 K
2   model._latent_encoder._input_layer         Linear  608  
3       model._latent_encoder._encoder     ModuleList    1 K
4     model._latent_encoder._encoder.0  NPBlockRelu2d    1 K
..                                 ...            ...    ...
82       model._decoder._decoder.7.act           ReLU    0  
83   model._decoder._decoder.7.dropout      Dropout2d    0  
84      model._decoder._decoder.7.norm    BatchNorm2d  192  
85                model._decoder._mean         Linear   97  
86                 model._decoder._std         Linear   97  

[87 rows x 3 columns]

HBox(children=(FloatProgress(value=0.0, description='Validation sanity check', layout=Layout(flex='2'), max=5.…
step 0, {'val_loss': '1.4573177099227905', 'val/kl': '0.5117107033729553', 'val/mse': '0.19515596330165863', 'val/std': '0.9137517809867859'}


HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …
HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 2194, {'val_loss': '170.95159912109375', 'val/kl': '0.000982249970547855', 'val/mse': '0.11342734098434448', 'val/std': '0.07711008191108704'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 4389, {'val_loss': '-1.5134085416793823', 'val/kl': '0.0010569699807092547', 'val/mse': '0.003368363017216325', 'val/std': '0.06244845688343048'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 6584, {'val_loss': '-1.5219111442565918', 'val/kl': '0.001321342191658914', 'val/mse': '0.00357621256262064', 'val/std': '0.06076379492878914'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 8779, {'val_loss': '-1.5059326887130737', 'val/kl': '0.001544301863759756', 'val/mse': '0.0037375360261648893', 'val/std': '0.05373244360089302'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 10974, {'val_loss': '-1.4931892156600952', 'val/kl': '0.00041607089224271476', 'val/mse': '0.004085092805325985', 'val/std': '0.06542228907346725'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 13169, {'val_loss': '-1.4815890789031982', 'val/kl': '0.0005998624837957323', 'val/mse': '0.0038775834254920483', 'val/std': '0.05528929457068443'}
Epoch     5: reducing learning rate of group 0 to 7.1936e-04.

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 15364, {'val_loss': '-1.5160049200057983', 'val/kl': '0.0004562124377116561', 'val/mse': '0.003399617737159133', 'val/std': '0.048332419246435165'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 17559, {'val_loss': '-1.4910939931869507', 'val/kl': '0.00036651312257163227', 'val/mse': '0.0037599860224872828', 'val/std': '0.04610859602689743'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 19754, {'val_loss': '-1.4462004899978638', 'val/kl': '0.0003672659513540566', 'val/mse': '0.003462857799604535', 'val/std': '0.04291912913322449'}
Epoch     8: reducing learning rate of group 0 to 7.1936e-05.

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 21949, {'val_loss': '-1.3219516277313232', 'val/kl': '0.00033365757553838193', 'val/mse': '0.00348706915974617', 'val/std': '0.04272124543786049'}

logger.metrics [{'val_loss': -1.3219516277313232, 'val/kl': 0.00033365757553838193, 'val/mse': 0.00348706915974617, 'val/std': 0.04272124543786049, 'epoch': 9}]

[I 2020-02-16 15:56:42,878] Finished trial#51 resulted in value: -1.3219516277313232. Current best value is -1.5752214193344116 with parameters: {'attention_dropout': 0.2, 'attention_layers': 1, 'batch_size': 16, 'batchnorm': True, 'context_in_target': True, 'det_enc_cross_attn_type': 'ptmultihead', 'det_enc_self_attn_type': 'multihead', 'dropout': 0, 'grad_clip': 40, 'hidden_dim': 32, 'latent_dim': 64, 'latent_enc_self_attn_type': 'ptmultihead', 'learning_rate': 0.004368391916372206, 'max_nb_epochs': 10, 'min_std': 0.005, 'n_decoder_layers': 1, 'n_det_encoder_layers': 2, 'n_latent_encoder_layers': 1, 'num_context': 96, 'num_extra_target': 96, 'num_heads': 8, 'num_workers': 3, 'use_deterministic_path': False, 'use_lvar': True, 'use_rnn': False, 'use_self_attn': False, 'vis_i': 670, 'x_dim': 17, 'y_dim': 1}.

trial 52 params {'attention_dropout': 0.2, 'attention_layers': 1, 'batch_size': 16, 'batchnorm': True, 'context_in_target': True, 'det_enc_cross_attn_type': 'ptmultihead', 'det_enc_self_attn_type': 'multihead', 'dropout': 0, 'grad_clip': 40, 'hidden_dim': 32, 'latent_dim': 64, 'latent_enc_self_attn_type': 'ptmultihead', 'learning_rate': 0.006974889906878952, 'max_nb_epochs': 10, 'min_std': 0.005, 'n_decoder_layers': 8, 'n_det_encoder_layers': 2, 'n_latent_encoder_layers': 2, 'num_context': 96, 'num_extra_target': 96, 'num_heads': 8, 'num_workers': 3, 'use_deterministic_path': False, 'use_lvar': True, 'use_rnn': False, 'use_self_attn': False, 'vis_i': 670, 'x_dim': 17, 'y_dim': 1}

INFO:root:gpu available: True, used: True
INFO:root:VISIBLE GPUS: 0
INFO:root:
                                  Name           Type Params
0                                model    LatentModel   94 K
1                model._latent_encoder  LatentEncoder    8 K
2   model._latent_encoder._input_layer         Linear  608  
3       model._latent_encoder._encoder     ModuleList    2 K
4     model._latent_encoder._encoder.0  NPBlockRelu2d    1 K
..                                 ...            ...    ...
87       model._decoder._decoder.7.act           ReLU    0  
88   model._decoder._decoder.7.dropout      Dropout2d    0  
89      model._decoder._decoder.7.norm    BatchNorm2d  192  
90                model._decoder._mean         Linear   97  
91                 model._decoder._std         Linear   97  

[92 rows x 3 columns]

HBox(children=(FloatProgress(value=0.0, description='Validation sanity check', layout=Layout(flex='2'), max=5.…
step 0, {'val_loss': '1.4809108972549438', 'val/kl': '0.49705177545547485', 'val/mse': '0.19410839676856995', 'val/std': '0.9605253338813782'}


HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …
HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 2194, {'val_loss': '-1.3835784196853638', 'val/kl': '0.0008006279822438955', 'val/mse': '0.004557789769023657', 'val/std': '0.063215471804142'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 4389, {'val_loss': '-1.5923731327056885', 'val/kl': '0.000711232831235975', 'val/mse': '0.0032738428562879562', 'val/std': '0.057123053818941116'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 6584, {'val_loss': '-1.5143545866012573', 'val/kl': '0.0006580971530638635', 'val/mse': '0.003498892532661557', 'val/std': '0.05259685590863228'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 8779, {'val_loss': '-1.3843765258789062', 'val/kl': '0.0006477347924374044', 'val/mse': '0.00456392765045166', 'val/std': '0.053285516798496246'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 10974, {'val_loss': '-1.4790021181106567', 'val/kl': '0.0006617918843403459', 'val/mse': '0.0036568765062838793', 'val/std': '0.054161760956048965'}
Epoch     4: reducing learning rate of group 0 to 6.9749e-04.

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 13169, {'val_loss': '-1.6128934621810913', 'val/kl': '0.0004931865842081606', 'val/mse': '0.0029674884863197803', 'val/std': '0.04655424878001213'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 15364, {'val_loss': '-1.5709235668182373', 'val/kl': '0.0004162131517659873', 'val/mse': '0.00316846021451056', 'val/std': '0.04433130845427513'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 17559, {'val_loss': '-1.5288468599319458', 'val/kl': '0.0004013367579318583', 'val/mse': '0.0031520789489150047', 'val/std': '0.04289095103740692'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 19754, {'val_loss': '-1.4257675409317017', 'val/kl': '0.0003335531509947032', 'val/mse': '0.0034472807310521603', 'val/std': '0.04128710553050041'}
Epoch     8: reducing learning rate of group 0 to 6.9749e-05.

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 21949, {'val_loss': '-1.5207812786102295', 'val/kl': '0.00031931945704855025', 'val/mse': '0.0032018099445849657', 'val/std': '0.04250483587384224'}

logger.metrics [{'val_loss': -1.5207812786102295, 'val/kl': 0.00031931945704855025, 'val/mse': 0.0032018099445849657, 'val/std': 0.04250483587384224, 'epoch': 9}]

[I 2020-02-16 16:21:09,177] Finished trial#52 resulted in value: -1.5207812786102295. Current best value is -1.5752214193344116 with parameters: {'attention_dropout': 0.2, 'attention_layers': 1, 'batch_size': 16, 'batchnorm': True, 'context_in_target': True, 'det_enc_cross_attn_type': 'ptmultihead', 'det_enc_self_attn_type': 'multihead', 'dropout': 0, 'grad_clip': 40, 'hidden_dim': 32, 'latent_dim': 64, 'latent_enc_self_attn_type': 'ptmultihead', 'learning_rate': 0.004368391916372206, 'max_nb_epochs': 10, 'min_std': 0.005, 'n_decoder_layers': 1, 'n_det_encoder_layers': 2, 'n_latent_encoder_layers': 1, 'num_context': 96, 'num_extra_target': 96, 'num_heads': 8, 'num_workers': 3, 'use_deterministic_path': False, 'use_lvar': True, 'use_rnn': False, 'use_self_attn': False, 'vis_i': 670, 'x_dim': 17, 'y_dim': 1}.

trial 53 params {'attention_dropout': 0.2, 'attention_layers': 1, 'batch_size': 16, 'batchnorm': True, 'context_in_target': True, 'det_enc_cross_attn_type': 'ptmultihead', 'det_enc_self_attn_type': 'multihead', 'dropout': 0, 'grad_clip': 40, 'hidden_dim': 32, 'latent_dim': 64, 'latent_enc_self_attn_type': 'ptmultihead', 'learning_rate': 0.004945595289855884, 'max_nb_epochs': 10, 'min_std': 0.005, 'n_decoder_layers': 1, 'n_det_encoder_layers': 2, 'n_latent_encoder_layers': 2, 'num_context': 96, 'num_extra_target': 96, 'num_heads': 8, 'num_workers': 3, 'use_deterministic_path': False, 'use_lvar': True, 'use_rnn': False, 'use_self_attn': False, 'vis_i': 670, 'x_dim': 17, 'y_dim': 1}

INFO:root:gpu available: True, used: True
INFO:root:VISIBLE GPUS: 0
INFO:root:
                                                 Name                  Type  \
0                                               model           LatentModel   
1                               model._latent_encoder         LatentEncoder   
2                  model._latent_encoder._input_layer                Linear   
3                      model._latent_encoder._encoder            ModuleList   
4                    model._latent_encoder._encoder.0         NPBlockRelu2d   
5             model._latent_encoder._encoder.0.linear                Linear   
6                model._latent_encoder._encoder.0.act                  ReLU   
7            model._latent_encoder._encoder.0.dropout             Dropout2d   
8               model._latent_encoder._encoder.0.norm           BatchNorm2d   
9                    model._latent_encoder._encoder.1         NPBlockRelu2d   
10            model._latent_encoder._encoder.1.linear                Linear   
11               model._latent_encoder._encoder.1.act                  ReLU   
12           model._latent_encoder._encoder.1.dropout             Dropout2d   
13              model._latent_encoder._encoder.1.norm           BatchNorm2d   
14           model._latent_encoder._penultimate_layer                Linear   
15                        model._latent_encoder._mean                Linear   
16                     model._latent_encoder._log_var                Linear   
17                       model._deterministic_encoder  DeterministicEncoder   
18          model._deterministic_encoder._input_layer                Linear   
19            model._deterministic_encoder._d_encoder            ModuleList   
20          model._deterministic_encoder._d_encoder.0         NPBlockRelu2d   
21   model._deterministic_encoder._d_encoder.0.linear                Linear   
22      model._deterministic_encoder._d_encoder.0.act                  ReLU   
23  model._deterministic_encoder._d_encoder.0.dropout             Dropout2d   
24     model._deterministic_encoder._d_encoder.0.norm           BatchNorm2d   
25          model._deterministic_encoder._d_encoder.1         NPBlockRelu2d   
26   model._deterministic_encoder._d_encoder.1.linear                Linear   
27      model._deterministic_encoder._d_encoder.1.act                  ReLU   
28  model._deterministic_encoder._d_encoder.1.dropout             Dropout2d   
29     model._deterministic_encoder._d_encoder.1.norm           BatchNorm2d   
30      model._deterministic_encoder._cross_attention             Attention   
31  model._deterministic_encoder._cross_attention....              BatchMLP   
32  model._deterministic_encoder._cross_attention....         NPBlockRelu2d   
33  model._deterministic_encoder._cross_attention....                Linear   
34  model._deterministic_encoder._cross_attention....                  ReLU   
35  model._deterministic_encoder._cross_attention....             Dropout2d   
36  model._deterministic_encoder._cross_attention....            Sequential   
37  model._deterministic_encoder._cross_attention....                Linear   
38  model._deterministic_encoder._cross_attention....              BatchMLP   
39  model._deterministic_encoder._cross_attention....         NPBlockRelu2d   
40  model._deterministic_encoder._cross_attention....                Linear   
41  model._deterministic_encoder._cross_attention....                  ReLU   
42  model._deterministic_encoder._cross_attention....             Dropout2d   
43  model._deterministic_encoder._cross_attention....            Sequential   
44  model._deterministic_encoder._cross_attention....                Linear   
45   model._deterministic_encoder._cross_attention._W    MultiheadAttention   
46  model._deterministic_encoder._cross_attention....                Linear   
47                                     model._decoder               Decoder   
48                   model._decoder._target_transform                Linear   
49                            model._decoder._decoder            ModuleList   
50                          model._decoder._decoder.0         NPBlockRelu2d   
51                   model._decoder._decoder.0.linear                Linear   
52                      model._decoder._decoder.0.act                  ReLU   
53                  model._decoder._decoder.0.dropout             Dropout2d   
54                     model._decoder._decoder.0.norm           BatchNorm2d   
55                               model._decoder._mean                Linear   
56                                model._decoder._std                Linear   

   Params  
0    28 K  
1     8 K  
2   608    
3     2 K  
4     1 K  
5     1 K  
6     0    
7     0    
8    64    
9     1 K  
10    1 K  
11    0    
12    0    
13   64    
14    1 K  
15    2 K  
16    2 K  
17   10 K  
18  608    
19    2 K  
20    1 K  
21    1 K  
22    0    
23    0    
24   64    
25    1 K  
26    1 K  
27    0    
28    0    
29   64    
30    7 K  
31    1 K  
32  544    
33  544    
34    0    
35    0    
36    0    
37    1 K  
38    1 K  
39  544    
40  544    
41    0    
42    0    
43    0    
44    1 K  
45    4 K  
46    1 K  
47   10 K  
48  576    
49    9 K  
50    9 K  
51    9 K  
52    0    
53    0    
54  192    
55   97    
56   97    

HBox(children=(FloatProgress(value=0.0, description='Validation sanity check', layout=Layout(flex='2'), max=5.…
step 0, {'val_loss': '1.6228408813476562', 'val/kl': '0.5012664198875427', 'val/mse': '0.2795771658420563', 'val/std': '1.088279128074646'}


HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …
HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 2194, {'val_loss': '-1.3279961347579956', 'val/kl': '0.0006853163940832019', 'val/mse': '0.005132939200848341', 'val/std': '0.07134196162223816'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 4389, {'val_loss': '-1.4325536489486694', 'val/kl': '0.0006263595423661172', 'val/mse': '0.004049481358379126', 'val/std': '0.056351326406002045'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 6584, {'val_loss': '-1.5361480712890625', 'val/kl': '0.0007605483988299966', 'val/mse': '0.0035171343479305506', 'val/std': '0.055043116211891174'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 8779, {'val_loss': '-1.477478265762329', 'val/kl': '0.0007185991271398962', 'val/mse': '0.003651161678135395', 'val/std': '0.054432306438684464'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 10974, {'val_loss': '-1.4673835039138794', 'val/kl': '0.0007735468680039048', 'val/mse': '0.003901753108948469', 'val/std': '0.05738849192857742'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 13169, {'val_loss': '-1.4404581785202026', 'val/kl': '0.001055828994140029', 'val/mse': '0.004138254560530186', 'val/std': '0.07071433961391449'}
Epoch     5: reducing learning rate of group 0 to 4.9456e-04.

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 15364, {'val_loss': '-1.6021909713745117', 'val/kl': '0.0007687730831094086', 'val/mse': '0.003044766141101718', 'val/std': '0.05109384283423424'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 17559, {'val_loss': '-1.5778357982635498', 'val/kl': '0.0007317148265428841', 'val/mse': '0.0031571618746966124', 'val/std': '0.05000822991132736'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 19754, {'val_loss': '-1.5999406576156616', 'val/kl': '0.0005693243583664298', 'val/mse': '0.0031292077619582415', 'val/std': '0.05022908002138138'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 21949, {'val_loss': '-1.56947660446167', 'val/kl': '0.0005845829145982862', 'val/mse': '0.003213117830455303', 'val/std': '0.05039476230740547'}
Epoch     9: reducing learning rate of group 0 to 4.9456e-05.

logger.metrics [{'val_loss': -1.56947660446167, 'val/kl': 0.0005845829145982862, 'val/mse': 0.003213117830455303, 'val/std': 0.05039476230740547, 'epoch': 9}]

[I 2020-02-16 16:41:27,996] Finished trial#53 resulted in value: -1.56947660446167. Current best value is -1.5752214193344116 with parameters: {'attention_dropout': 0.2, 'attention_layers': 1, 'batch_size': 16, 'batchnorm': True, 'context_in_target': True, 'det_enc_cross_attn_type': 'ptmultihead', 'det_enc_self_attn_type': 'multihead', 'dropout': 0, 'grad_clip': 40, 'hidden_dim': 32, 'latent_dim': 64, 'latent_enc_self_attn_type': 'ptmultihead', 'learning_rate': 0.004368391916372206, 'max_nb_epochs': 10, 'min_std': 0.005, 'n_decoder_layers': 1, 'n_det_encoder_layers': 2, 'n_latent_encoder_layers': 1, 'num_context': 96, 'num_extra_target': 96, 'num_heads': 8, 'num_workers': 3, 'use_deterministic_path': False, 'use_lvar': True, 'use_rnn': False, 'use_self_attn': False, 'vis_i': 670, 'x_dim': 17, 'y_dim': 1}.

trial 54 params {'attention_dropout': 0.2, 'attention_layers': 1, 'batch_size': 16, 'batchnorm': True, 'context_in_target': True, 'det_enc_cross_attn_type': 'ptmultihead', 'det_enc_self_attn_type': 'multihead', 'dropout': 0, 'grad_clip': 40, 'hidden_dim': 128, 'latent_dim': 64, 'latent_enc_self_attn_type': 'ptmultihead', 'learning_rate': 0.005270631834316311, 'max_nb_epochs': 10, 'min_std': 0.005, 'n_decoder_layers': 1, 'n_det_encoder_layers': 8, 'n_latent_encoder_layers': 2, 'num_context': 96, 'num_extra_target': 96, 'num_heads': 8, 'num_workers': 3, 'use_deterministic_path': False, 'use_lvar': True, 'use_rnn': False, 'use_self_attn': False, 'vis_i': 670, 'x_dim': 17, 'y_dim': 1}

INFO:root:gpu available: True, used: True
INFO:root:VISIBLE GPUS: 0
INFO:root:
                                  Name           Type Params
0                                model    LatentModel  347 K
1                model._latent_encoder  LatentEncoder   68 K
2   model._latent_encoder._input_layer         Linear    2 K
3       model._latent_encoder._encoder     ModuleList   33 K
4     model._latent_encoder._encoder.0  NPBlockRelu2d   16 K
..                                 ...            ...    ...
82       model._decoder._decoder.0.act           ReLU    0  
83   model._decoder._decoder.0.dropout      Dropout2d    0  
84      model._decoder._decoder.0.norm    BatchNorm2d  384  
85                model._decoder._mean         Linear  193  
86                 model._decoder._std         Linear  193  

[87 rows x 3 columns]

HBox(children=(FloatProgress(value=0.0, description='Validation sanity check', layout=Layout(flex='2'), max=5.…
step 0, {'val_loss': '1.522579550743103', 'val/kl': '0.5025149583816528', 'val/mse': '0.31865665316581726', 'val/std': '0.9117485284805298'}


HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …
HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 2194, {'val_loss': '-1.3701834678649902', 'val/kl': '0.0027001516427844763', 'val/mse': '0.00422711344435811', 'val/std': '0.06300579011440277'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 4389, {'val_loss': '-1.4492722749710083', 'val/kl': '0.002561358967795968', 'val/mse': '0.004086161497980356', 'val/std': '0.060740407556295395'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 6584, {'val_loss': '-1.3625338077545166', 'val/kl': '0.002082530641928315', 'val/mse': '0.004729835782200098', 'val/std': '0.057794589549303055'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 8779, {'val_loss': '-1.2864140272140503', 'val/kl': '0.003959427587687969', 'val/mse': '0.004717061761766672', 'val/std': '0.06390085816383362'}

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
step 10974, {'val_loss': '-1.2884516716003418', 'val/kl': '0.003093295730650425', 'val/mse': '0.005015104543417692', 'val/std': '0.04962388053536415'}
Epoch     4: reducing learning rate of group 0 to 5.2706e-04.

HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=117.0, style=Pr…
In [ ]:
In [15]:

print('Number of finished trials: {}'.format(len(study.trials)))

print('Best trial:')
trial = study.best_trial

print('  Value: {}'.format(trial.value))

print('  Params: ')
for key, value in trial.params.items():
    print('    {}: {}'.format(key, value))

# shutil.rmtree(MODEL_DIR)
Number of finished trials: 39
Best trial:
  Value: -1.5058155059814453
  Params: 
    attention_dropout: 0.2
    attention_layers: 1
    batch_size: 16
    batchnorm: True
    context_in_target: True
    det_enc_cross_attn_type: ptmultihead
    det_enc_self_attn_type: multihead
    dropout: 0
    grad_clip: 40
    hidden_dim: 32
    latent_dim: 64
    latent_enc_self_attn_type: ptmultihead
    learning_rate: 0.008663362578308754
    max_nb_epochs: 10
    min_std: 0.005
    n_decoder_layers: 8
    n_det_encoder_layers: 2
    n_latent_encoder_layers: 2
    num_context: 96
    num_extra_target: 96
    num_heads: 8
    num_workers: 3
    use_deterministic_path: False
    use_lvar: True
    use_rnn: False
    use_self_attn: False
    vis_i: 670
    x_dim: 17
    y_dim: 1

In [ ]:



	

		

		
View


TODO



		

	


In [ ]:
df = study.trials_dataframe(attrs=('number', 'value', 'params', 'state'))
df.sort_values('value')
In [ ]:
df.sort_values('value').head(17).T
In [ ]:
In [ ]:
In [ ]:

			
		


		

			
			Reference in New Issue
			
			View Git Blame
			Copy Permalink