DeepTime/scratch-run_exp.ipynb

• try just one predictor
  • multi input, single output
• comparem ulti
• losses:
  • try logp? nah
  • mae?
• make my own csv with 5m data (maybe 10k rows)
• backtest?

In [1]:

import warnings
warnings.simplefilter("ignore")

# autoreload import your package
%load_ext autoreload
%autoreload 2

In [2]:

import os
from os.path import join
import math
import logging
from typing import Callable, Optional, Union, Dict, Tuple

from matplotlib import pyplot as plt
from pathlib import Path
import matplotlib.colors as mcolors

import gin
from fire import Fire
import numpy as np
import torch
from torch.utils.data import DataLoader
from torch import optim
from torch import nn

from experiments.base import Experiment
from data.datasets import ForecastDataset
from models import get_model
from utils.checkpoint import Checkpoint
from utils.ops import default_device, to_tensor
from utils.losses import get_loss_fn
from utils.metrics import calc_metrics

from experiments.forecast import get_data
gin.enter_interactive_mode()

In [ ]:

auto

In [19]:

def plot(model_name="deeptime", save_path=Path("storage/experiments/Exchange/96M/repeat=0"), i=200, title=None, plot=True):

    gin.clear_config()
    gin.parse_config(open(save_path/"config.gin"))

    train_set, train_loader = get_data(flag='train', batch_size=2)

    model = get_model(model_name,
                        dim_size=train_set.data_x.shape[1],
                        datetime_feats=train_set.timestamps.shape[-1]).to(default_device())
    model.load_state_dict(torch.load(save_path/'model.pth'))
    model = model.eval()


    b = train_set[i]
    b = [bb[None, :] for bb in b]
    b2 = list(map(to_tensor, b))
    context_past_x, context_y, query_past_x, query_y, context_time, query_time = b2
    with torch.no_grad():
        forecast = model(*b2)

    if title is None:
        title = str(save_path).split('/')[-3:]
        title = "-".join(title)
    
    colors = list(mcolors.BASE_COLORS.keys())
    l = x.shape[1]
    forecast2 = forecast[0].detach().cpu().numpy()
    x2 = x[0].cpu()
    y2 = y[0].cpu()
    l2 = y.shape[1]
    i_past = list(range(l))
    i_future = list(range(l, l+l2))
    
    if plot:
        plt.title(title)
        for i in range(x.shape[-1]):
            plt.plot(i_past, x2[:, i], c=colors[i])
        for i in range(x.shape[-1]):
            plt.plot(i_future, y2[:, i], c=colors[i])
        for i in range(x.shape[-1]):
            plt.plot(i_future, forecast2[:, i], c=colors[i], linestyle='--')
    return x2, y2, forecast2, i_past, i_future

Out [19]:

<data.datasets.ForecastDataset at 0x7fe2a3a58b50>

In [ ]:

In [ ]:

In [135]:

def plot_multi(save_paths=[Path("storage/experiments/Exchange/96M/repeat=0")], i=200, title=None, plot=True):
    assert len(save_paths)>0
    for j in range(len(save_paths)):
        save_path = save_paths[j]

        gin.clear_config()
        gin.parse_config(open(save_path/"config.gin"))
        model_name = gin.query_parameter("instance.model_type")

        train_set, train_loader = get_data(flag='test', batch_size=3)

        model = get_model(model_name,
                            dim_size=train_set.data_x.shape[1],
                            datetime_feats=train_set.timestamps.shape[-1]).to(default_device())
        model.load_state_dict(torch.load(save_path/'model.pth'))
        model = model.eval()


        b = train_set[i]
        b = [bb[None, :] for bb in b]
        b2 = list(map(to_tensor, b))
        
#         b = next(iter(train_loader))
#         print([s.shape for s in b])
        
        context_past_x, context_y, query_past_x, query_y, context_time, query_time = b2
        with torch.no_grad():
            forecast = model(*b2)
       
        colors = list(mcolors.BASE_COLORS.keys())
        l = context_time.shape[1]
        forecast2 = forecast[0].detach().cpu().numpy()
        x2 = context_y[0].cpu()
        y2 = query_y[0].cpu()
        l2 = query_time.shape[1]
        i_past = list(range(l))
        i_future = list(range(l, l+l2))

        if plot:
            
            mtitle = str(save_path).split('/')[-2:]
        
            tags = mtitle[-1]
            tags = [x.split('=')[-1] for x in tags.split(',')]
            
            mtitle[-1] = ' '.join(tags)
            mtitle = "-".join(mtitle)
            plt.plot(i_future, forecast2[:, 0], c=colors[j], linestyle='--', label=f"{mtitle}")
            
    if plot:
        plt.plot(i_past, x2[:, 0], c=colors[0], label=f"past")
        plt.plot(i_future, y2[:, 0], c=colors[0], label="future true", alpha=0.3)
    plt.legend()
    plt.title(title)
    return x2, y2, forecast2, i_past, i_future

In [ ]:

run exp

In [91]:

save_path = Path('storage/experiments/Stocks/96M2S/repeat=0')

In [92]:

gin.clear_config()
config_path = save_path/"config.gin"
gin.parse_config(open(config_path))
model_name = gin.query_parameter("instance.model_type")
model_name

---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
Cell In [92], line 3
      1 gin.clear_config()
      2 config_path = save_path/"config.gin"
----> 3 gin.parse_config(open(config_path))
      4 model_name = gin.query_parameter("instance.model_type")
      5 model_name

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/IPython/core/interactiveshell.py:282, in _modified_open(file, *args, **kwargs)
    275 if file in {0, 1, 2}:
    276     raise ValueError(
    277         f"IPython won't let you open fd={file} by default "
    278         "as it is likely to crash IPython. If you know what you are doing, "
    279         "you can use builtins' open."
    280     )
--> 282 return io_open(file, *args, **kwargs)

FileNotFoundError: [Errno 2] No such file or directory: 'storage/experiments/Stocks/96M2S/repeat=0/config.gin'

In [93]:

from experiments.forecast import ForecastExperiment
exp = ForecastExperiment(config_path=config_path)
exp

ERROR:root:Path not found: storage/experiments/Stocks/96M2S/repeat=0/config.gin

---------------------------------------------------------------------------
OSError                                   Traceback (most recent call last)
Cell In [93], line 2
      1 from experiments.forecast import ForecastExperiment
----> 2 exp = ForecastExperiment(config_path=config_path)
      3 exp

File /media/wassname/SGIronWolf/projects5/investing/DeepTime/experiments/base.py:24, in Experiment.__init__(self, config_path)
     22 self.config_path = config_path
     23 self.root = Path(config_path).parent
---> 24 gin.parse_config_file(self.config_path)

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/gin/config.py:2457, in parse_config_file(config_file, skip_unknown, print_includes_and_imports)
   2455         return results
   2456 err_str = 'Unable to open file: {}. Searched config paths: {}.'
-> 2457 raise IOError(err_str.format(config_file, prefixes))

OSError: Unable to open file: storage/experiments/Stocks/96M2S/repeat=0/config.gin. Searched config paths: [''].

In [94]:

# exp.run()

view all

In [151]:

# list the models we have run...
m=sorted(Path("storage/experiments/Stocks").glob("**/_SUCCESS"))
print(m)

[Path('storage/experiments/Stocks/96M2S/base_learner=None,inr=INR,encoder=inception,repeat=0/_SUCCESS'), Path('storage/experiments/Stocks/96M2S/base_learner=None,inr=INRPlus2,encoder=inception,repeat=0/_SUCCESS'), Path('storage/experiments/Stocks/96M2S/base_learner=Ridge,inr=INR,encoder=inception,repeat=0/_SUCCESS')]

In [152]:

# m = [
# #     Path('storage/experiments/Stocks/96M/repeat=0/_SUCCESS'),
#     Path(
#         'storage/experiments/Stocks/96M2S/base_learner=None,inr=INRPlus2,encoder=inception,repeat=0/_SUCCESS'
#     ),
#     Path(
#         'storage/experiments/Stocks/96M2S/base_learner=Ridge,inr=INR,encoder=inception,repeat=0/_SUCCESS'
#     ),
#     Path(
#         'storage/experiments/Stocks/96M2S/base_learner=Ridge,inr=INRPlus2,encoder=inception,repeat=0/_SUCCESS'
#     ),
#     Path(
#         'storage/experiments/Stocks/96M2S/base_learner=Transformer,inr=INR,encoder=inception,repeat=0/_SUCCESS'
#     ),
#     Path(
#         'storage/experiments/Stocks/96M2S/base_learner=Transformer,inr=INRPlus2,encoder=inception,repeat=0/_SUCCESS'
#     ),
# #     Path('storage/experiments/Stocks/96S/repeat=0/_SUCCESS'),
# #     Path('storage/experiments/Stocks/96Splus/repeat=0/_SUCCESS'),
# #     Path('storage/experiments/Stocks/96Splusshort/repeat=0/_SUCCESS'),
# #     Path('storage/experiments/Stocks/96Sshort/repeat=0/_SUCCESS')
# ]

In [153]:

save_paths = [mm.parent for mm in m]
plot_multi(
    save_paths=save_paths,
    i=1000,
)
1

Out [153]:

receptive field [114 126  42]=[38 18  2]*[[ 1  1  1]
 [ 1  2  4]
 [ 1  4 16]]
receptive field [114 126  42]=[38 18  2]*[[ 1  1  1]
 [ 1  2  4]
 [ 1  4 16]]
receptive field [690 378 242]=[138  18   2]*[[ 1  1  1]
 [ 1  2  4]
 [ 1  4 16]
 [ 1  6 36]
 [ 1  8 64]]
receptive field [114 126  42]=[38 18  2]*[[ 1  1  1]
 [ 1  2  4]
 [ 1  4 16]]

1

In [ ]:

In [ ]:

check positions in dl

In [139]:

train_set, train_loader = get_data(flag='test', batch_size=3)
b = context_past_x, context_y, query_past_x, query_y, context_time, query_time = train_set[100]
print([bb.shape for bb in b])
# context_y, query_y

[(92, 5), (46, 1), (92, 5), (46, 1), (46, 7), (46, 7)]

In [80]:

cx_start, cx_end, c_start, c_end, qx_start, qx_end, q_start, q_end = train_set.get_inds(100)
cx_start, cx_end, c_start, c_end, qx_start, qx_end, q_start, q_end

Out [80]:

(100, 192, 192, 238, 146, 238, 238, 284)

In [81]:

plt.hlines(1, cx_start, cx_end)
plt.hlines(2, c_start, c_end)
plt.hlines(3, qx_start, qx_end)
plt.hlines(4, q_start, q_end)

Out [81]:

<matplotlib.collections.LineCollection at 0x7fe2a0938f10>