DeepTime/scratch-run_exp.ipynb

• try just one predictor
  • multi input, single output
• comparem ulti
• losses:
  • try logp? nah
  • mae?
• make my own csv with 5m data (maybe 10k rows)
• backtest?

In [1]:

import warnings
warnings.simplefilter("ignore")

# autoreload import your package
%load_ext autoreload
%autoreload 2

In [2]:

import os
from os.path import join
import math
import logging
from typing import Callable, Optional, Union, Dict, Tuple

from matplotlib import pyplot as plt
from pathlib import Path
import matplotlib.colors as mcolors

import gin
from fire import Fire
import numpy as np
import torch
from torch.utils.data import DataLoader
from torch import optim
from torch import nn

from experiments.base import Experiment
from data.datasets import ForecastDataset
from models import get_model
from utils.checkpoint import Checkpoint
from utils.ops import default_device, to_tensor
from utils.losses import get_loss_fn
from utils.metrics import calc_metrics

from experiments.forecast import get_data
gin.enter_interactive_mode()

In [3]:

import logging
logging.root.setLevel(logging.INFO)

from loguru import logger
logger.remove()
logger.add(os.sys.stdout, level="INFO", colorize=True, format="<level>{time} | {message}</level>")

Out [3]:

1

auto

In [4]:

def plot(model_name="deeptime", save_path=Path("storage/experiments/Exchange/96M/repeat=0"), i=200, title=None, plot=True):

    gin.clear_config()
    gin.parse_config(open(save_path/"config.gin"))

    train_set, train_loader = get_data(flag='train', batch_size=2)

    model = get_model(model_name,
                        dim_size=train_set.data_x.shape[1],
                        datetime_feats=train_set.timestamps.shape[-1]).to(default_device())
    model.load_state_dict(torch.load(save_path/'model.pth'))
    model = model.eval()


    b = train_set[i]
    b = [bb[None, :] for bb in b]
    b2 = list(map(to_tensor, b))
    context_past_x, context_y, query_past_x, query_y, context_time, query_time = b2
    with torch.no_grad():
        forecast = model(*b2)

    if title is None:
        title = str(save_path).split('/')[-3:]
        title = "-".join(title)
    
    colors = list(mcolors.BASE_COLORS.keys())
    l = x.shape[1]
    forecast2 = forecast[0].detach().cpu().numpy()
    x2 = x[0].cpu()
    y2 = y[0].cpu()
    l2 = y.shape[1]
    i_past = list(range(l))
    i_future = list(range(l, l+l2))
    
    if plot:
        plt.title(title)
        for i in range(x.shape[-1]):
            plt.plot(i_past, x2[:, i], c=colors[i])
        for i in range(x.shape[-1]):
            plt.plot(i_future, y2[:, i], c=colors[i])
        for i in range(x.shape[-1]):
            plt.plot(i_future, forecast2[:, i], c=colors[i], linestyle='--')
    return x2, y2, forecast2, i_past, i_future

In [ ]:

In [5]:

list(mcolors.BASE_COLORS.keys())

Out [5]:

['b', 'g', 'r', 'c', 'm', 'y', 'k', 'w']

view model

run exps

In [ ]:

In [6]:

# list the models we have run...
configs=sorted(Path("storage/experiments/Stocks").glob("**/config.gin"))
import random
random.shuffle(configs)
# print(configs)

In [7]:

from experiments.forecast import ForecastExperiment
from tqdm.auto import tqdm

In [ ]:

for config in tqdm(configs):
    save_path = config.parent

    exp = ForecastExperiment(config_path=config)
    print(config)
    try:
        exp.run()
    except KeyboardInterrupt:
        raise
    except Exception as e:
        raise
        print(e)
        pass

  0%|          | 0/42 [00:00<?, ?it/s]

storage/experiments/Stocks/96M2S/base_learner=Transformer,inr=INR,encoder=none,repeat=0/config.gin

INFO:root:epochs: 1, iters: 100 | training loss: 1.84

In [ ]:

%debug

In [ ]:

exp.instance()

In [ ]:

# save_path = Path('storage/experiments/Stocks/96M2S/repeat=0')

In [ ]:

# gin.clear_config()
# config_path = save_path/"config.gin"
# gin.parse_config(open(config_path))
# model_name = gin.query_parameter("instance.model_type")
# model_name

In [ ]:

# exp = ForecastExperiment(config_path=config_path)
# # exp.run()

In [ ]:

def save_path2name(save_path: Path) -> str:
    """
    Path('storage/experiments/Stocks/96M2S/base_learner=None,inr=INR,encoder=mlp,repeat=0')
    to 
    '96M2S-None_INR_mlp_0'
    """
    mtitle = str(save_path).split('/')[-2:]
    tags = mtitle[-1]
    tags = [x.split('=')[-1] for x in tags.split(',')]
    mtitle[-1] = '_'.join(tags)
    mtitle = "-".join(mtitle)
    return mtitle

# save_path2name(save_path)
# save_path

view all

In [ ]:

from torchsummaryX import summary

def plot_multi(save_paths=[Path("storage/experiments/Exchange/96M/repeat=0")], i=200, title=None, plot=True, verbose=1,):
    assert len(save_paths)>0
    for j in range(len(save_paths)):
        save_path = save_paths[j]

        gin.clear_config()
        gin.parse_config(open(save_path/"config.gin"))
        model_name = gin.query_parameter("instance.model_type")

        train_set, train_loader = get_data(flag='test', batch_size=3)
        seq_len = train_set[0][1].shape[0]
        model = get_model(model_name,
                            dim_size=train_set.data_x.shape[1],
                          seq_len=seq_len,
                            datetime_feats=train_set.timestamps.shape[-1]).to(default_device())
        model.load_state_dict(torch.load(save_path/'model.pth'))
        model = model.eval()
        
        


        b = train_set[i]
        b = [bb[None, :] for bb in b]
        b2 = list(map(to_tensor, b))
        
#         b = next(iter(train_loader))
#         print([s.shape for s in b]
        
        if verbose>1:
            
#             print(model)
            summary(model, *b2)
            print(save_path)
        
        context_past_x, context_y, query_past_x, query_y, context_time, query_time = b2
        with torch.no_grad():
            forecast = model(*b2)
       
        colors = list(mcolors.BASE_COLORS.keys())
        l = context_time.shape[1]
        forecast2 = forecast[0].detach().cpu().numpy()
        x2 = context_y[0].cpu()
        y2 = query_y[0].cpu()
        l2 = query_time.shape[1]
        i_past = list(range(l))
        i_future = list(range(l, l+l2))
        
        

        if plot:
        
            if j==0:
                plt.plot(i_past, x2[:, 0], c=colors[0], label=f"past")
                plt.plot(i_future, y2[:, 0], c=colors[0], label="future true", alpha=0.3)
            mtitle = save_path2name(save_path)
            plt.plot(i_future, forecast2[:, 0], linestyle='--', label=f"{mtitle}") # c=colors[j], 
            

    plt.legend()
    plt.title(title)
    return x2, y2, forecast2, i_past, i_future

In [ ]:

# list the models we have run...
m=sorted(Path("storage/experiments/Stocks/96M2S").glob("**/_SUCCESS"))
print(m)

In [ ]:

for mm in m:
    mtitle = save_path2name(mm.parent)
    print(mtitle)
    m3 = np.load(mm.parent/'metrics.npy', allow_pickle=1)
    m3 = eval(str(m3))
    print(m3['val']['mape'])

In [ ]:

train_set, train_loader = get_data(flag='train')
train_set[0][1].shape

In [ ]:

save_paths = [mm.parent for mm in m]
for mm in m:
    try:
        plot_multi(
            save_paths=[mm.parent],
            i=600,
            verbose=2,
        )
    except:
        print('failed', mm)
#         mm.unlink()
        pass
1

In [ ]:

save_paths = [mm.parent for mm in m]
plot_multi(
    save_paths=save_paths,
    i=200,
    verbose=0,
)
1

In [ ]:

256/24

In [ ]:

check positions in dl

In [ ]:

train_set, train_loader = get_data(flag='test', batch_size=3)
b = context_past_x, context_y, query_past_x, query_y, context_time, query_time = train_set[100]
print([bb.shape for bb in b])
# context_y, query_y

In [ ]:

cx_start, cx_end, c_start, c_end, qx_start, qx_end, q_start, q_end = train_set.get_inds(100)
cx_start, cx_end, c_start, c_end, qx_start, qx_end, q_start, q_end

In [ ]:

plt.hlines(1, cx_start, cx_end, color='green', alpha=0.5, label='context_past_x')
plt.hlines(2, c_start, c_end, color='green', label='context_labels')
plt.hlines(3, qx_start, qx_end, alpha=0.5, label='query_past_x')
plt.hlines(4, q_start, q_end, label='query_labels/target')
plt.legend(loc='upper left')

In [ ]:

In [ ]:

In [ ]: