DeepTime/scratch-single-stocks.ipynb

• try just one predictor
  • multi input, single output
• comparem ulti
• losses:
  • try logp? nah
  • mae?
• make my own csv with 5m data (maybe 10k rows)
• backtest?

In [1]:

import warnings
warnings.simplefilter("ignore")

# autoreload import your package
%load_ext autoreload
%autoreload 2

In [2]:

import os
from os.path import join
import math
import logging
from typing import Callable, Optional, Union, Dict, Tuple

from matplotlib import pyplot as plt
from pathlib import Path
import matplotlib.colors as mcolors

import gin
from fire import Fire
import numpy as np
import torch
from torch.utils.data import DataLoader
from torch import optim
from torch import nn

from experiments.base import Experiment
from data.datasets import ForecastDataset
from models import get_model
from utils.checkpoint import Checkpoint
from utils.ops import default_device, to_tensor
from utils.losses import get_loss_fn
from utils.metrics import calc_metrics

from experiments.forecast import get_data
gin.enter_interactive_mode()

In [ ]:

auto

In [3]:

def plot(model_name="deeptime", save_path=Path("storage/experiments/Exchange/96M/repeat=0"), i=200, title=None, plot=True):

    gin.clear_config()
    gin.parse_config(open(save_path/"config.gin"))

    train_set, train_loader = get_data(flag='train', batch_size=2)

    model = get_model(model_name,
                        dim_size=train_set.data_x.shape[1],
                        datetime_feats=train_set.timestamps.shape[-1]).to(default_device())
    model.load_state_dict(torch.load(save_path/'model.pth'))
    model = model.eval()


    b = train_set[i]
    b = [bb[None, :] for bb in b]
    x, y, x_time, y_time = map(to_tensor, b)
    with torch.no_grad():
        forecast = model(x, x_time, y_time)

    if title is None:
        title = str(save_path).split('/')[-3:]
        title = "-".join(title)
    
    colors = list(mcolors.BASE_COLORS.keys())
    l = x.shape[1]
    forecast2 = forecast[0].detach().cpu().numpy()
    x2 = x[0].cpu()
    y2 = y[0].cpu()
    l2 = y.shape[1]
    i_past = list(range(l))
    i_future = list(range(l, l+l2))
    
    if plot:
        plt.title(title)
        for i in range(x.shape[-1]):
            plt.plot(i_past, x2[:, i], c=colors[i])
        for i in range(x.shape[-1]):
            plt.plot(i_future, y2[:, i], c=colors[i])
        for i in range(x.shape[-1]):
            plt.plot(i_future, forecast2[:, i], c=colors[i], linestyle='--')
    return x2, y2, forecast2, i_past, i_future

In [39]:

def plot_multi(save_paths=[Path("storage/experiments/Exchange/96M/repeat=0")], i=200, title=None, plot=True):
    for j in range(len(save_paths)):
        save_path = save_paths[j]

        gin.clear_config()
        gin.parse_config(open(save_path/"config.gin"))
        model_name = gin.query_parameter("instance.model_type")

        train_set, train_loader = get_data(flag='test', batch_size=3)

        model = get_model(model_name,
                            dim_size=train_set.data_x.shape[1],
                            datetime_feats=train_set.timestamps.shape[-1]).to(default_device())
        model.load_state_dict(torch.load(save_path/'model.pth'))
        model = model.eval()


        b = train_set[i]
        b = [bb[None, :] for bb in b]
        
        b = next(iter(train_loader))
        print([s.shape for s in b])
        
        x, y, x_time, y_time = map(to_tensor, b)
#         print(b)
        with torch.no_grad():
            forecast = model(x, x_time, y_time)
       
        colors = list(mcolors.BASE_COLORS.keys())
        l = x.shape[1]
        forecast2 = forecast[0].detach().cpu().numpy()
        x2 = x[0].cpu()
        y2 = y[0].cpu()
        l2 = y.shape[1]
        i_past = list(range(l))
        i_future = list(range(l, l+l2))

        if plot:
            plt.plot(i_past, x2[:, 0], c=colors[0], label=f"past")
            plt.plot(i_future, y2[:, 0], c=colors[0], label="future true", alpha=0.3)
            
            mtitle = str(save_path).split('/')[-2:-1]
            mtitle = "-".join(mtitle)
            plt.plot(i_future, forecast2[:, 0], c=colors[j], linestyle='--', label=f"{mtitle}")
    plt.legend()
    plt.title(title)
    return x2, y2, forecast2, i_past, i_future

In [40]:

# list the models we have run...
m=sorted(Path("storage/experiments/Stocks").glob("**/_SUCCESS"))
print(m)

[Path('storage/experiments/Stocks/96M/repeat=0/_SUCCESS'), Path('storage/experiments/Stocks/96S/repeat=0/_SUCCESS'), Path('storage/experiments/Stocks/96Splus/repeat=0/_SUCCESS'), Path('storage/experiments/Stocks/96Splusshort/repeat=0/_SUCCESS'), Path('storage/experiments/Stocks/96Sshort/repeat=0/_SUCCESS')]

In [42]:

for mm in m:
    plot_multi(
               save_paths=[
                   mm.parent
               ],
               i=160
              )
    plt.show()
1

---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
Cell In [42], line 2
      1 for mm in m:
----> 2     plot_multi(
      3                save_paths=[
      4                    mm.parent
      5                ],
      6                i=160
      7               )
      8     plt.show()
      9 1

Cell In [39], line 9, in plot_multi(save_paths, i, title, plot)
      6 gin.parse_config(open(save_path/"config.gin"))
      7 model_name = gin.query_parameter("instance.model_type")
----> 9 train_set, train_loader = get_data(flag='test', batch_size=3)
     11 model = get_model(model_name,
     12                     dim_size=train_set.data_x.shape[1],
     13                     datetime_feats=train_set.timestamps.shape[-1]).to(default_device())
     14 model.load_state_dict(torch.load(save_path/'model.pth'))

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/gin/config.py:1605, in _make_gin_wrapper.<locals>.gin_wrapper(*args, **kwargs)
   1603 scope_info = " in scope '{}'".format(scope_str) if scope_str else ''
   1604 err_str = err_str.format(name, fn_or_cls, scope_info)
-> 1605 utils.augment_exception_message_and_reraise(e, err_str)

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/gin/utils.py:41, in augment_exception_message_and_reraise(exception, message)
     39 proxy = ExceptionProxy()
     40 ExceptionProxy.__qualname__ = type(exception).__qualname__
---> 41 raise proxy.with_traceback(exception.__traceback__) from None

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/gin/config.py:1582, in _make_gin_wrapper.<locals>.gin_wrapper(*args, **kwargs)
   1579 new_kwargs.update(kwargs)
   1581 try:
-> 1582   return fn(*new_args, **new_kwargs)
   1583 except Exception as e:  # pylint: disable=broad-except
   1584   err_str = ''

File /media/wassname/SGIronWolf/projects5/investing/DeepTime/experiments/forecast.py:115, in get_data(flag, batch_size)
    113 else:
    114     raise ValueError(f'no such flag {flag}')
--> 115 dataset = ForecastDataset(flag)
    116 data_loader = DataLoader(dataset,
    117                          batch_size=batch_size,
    118                          shuffle=shuffle,
    119                          drop_last=drop_last)
    120 return dataset, data_loader

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/gin/config.py:1605, in _make_gin_wrapper.<locals>.gin_wrapper(*args, **kwargs)
   1603 scope_info = " in scope '{}'".format(scope_str) if scope_str else ''
   1604 err_str = err_str.format(name, fn_or_cls, scope_info)
-> 1605 utils.augment_exception_message_and_reraise(e, err_str)

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/gin/utils.py:41, in augment_exception_message_and_reraise(exception, message)
     39 proxy = ExceptionProxy()
     40 ExceptionProxy.__qualname__ = type(exception).__qualname__
---> 41 raise proxy.with_traceback(exception.__traceback__) from None

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/gin/config.py:1582, in _make_gin_wrapper.<locals>.gin_wrapper(*args, **kwargs)
   1579 new_kwargs.update(kwargs)
   1581 try:
-> 1582   return fn(*new_args, **new_kwargs)
   1583 except Exception as e:  # pylint: disable=broad-except
   1584   err_str = ''

File /media/wassname/SGIronWolf/projects5/investing/DeepTime/data/datasets.py:71, in ForecastDataset.__init__(self, flag, horizon_len, scale, cross_learn, data_path, root_path, features, target, lookback_len, lookback_aux_len, lookback_mult, time_features, normalise_time_features)
     69 self.timestamps = None
     70 self.n_time = None
---> 71 self.n_time_samples = None
     72 self.load_data()

File /media/wassname/SGIronWolf/projects5/investing/DeepTime/data/datasets.py:105, in ForecastDataset.load_data(self)
    103 self.data_x = data[border1:border2] 
    104 # y is just the col we predict
--> 105 self.data_y = data[border1:border2][[self.target]]
    106 self.timestamps = get_time_features(pd.to_datetime(df_raw.date[border1:border2].values),
    107                                     normalise=self.normalise_time_features,
    108                                     features=self.time_features)
    109 self.n_time = len(self.data_x)

IndexError: only integers, slices (`:`), ellipsis (`...`), numpy.newaxis (`None`) and integer or boolean arrays are valid indices
  In call to configurable 'ForecastDataset' (<class 'data.datasets.ForecastDataset'>)
  In call to configurable 'get_data' (<function get_data at 0x7fb53c141160>)

In [49]:

plot_multi(
           save_paths=[
               Path("storage/experiments/Stocks/96M/repeat=0"),
#                Path("storage/experiments/Stocks/96Mplus/repeat=0"),
           ],
           i=60
          )
1

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
Cell In [49], line 1
----> 1 plot_multi(
      2            save_paths=[
      3                Path("storage/experiments/Stocks/96M/repeat=0"),
      4 #                Path("storage/experiments/Stocks/96Mplus/repeat=0"),
      5            ],
      6            i=60
      7           )
      8 1

Cell In [39], line 18, in plot_multi(save_paths, i, title, plot)
     14 model.load_state_dict(torch.load(save_path/'model.pth'))
     15 model = model.eval()
---> 18 b = train_set[i]
     19 b = [bb[None, :] for bb in b]
     21 b = next(iter(train_loader))

File /media/wassname/SGIronWolf/projects5/investing/DeepTime/data/datasets.py:145, in ForecastDataset.__getitem__(self, idx)
    143 cx_start = idx
    144 cx_end = cx_start + self.lookback_len
--> 145 c_start = cx_end + self.gap
    146 c_end = c_start + self.horizon_len
    148 qx_start = cx_end + self.gap

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/torch/utils/data/dataset.py:83, in Dataset.__getattr__(self, attribute_name)
     81     return function
     82 else:
---> 83     raise AttributeError

AttributeError: 

In [50]:

%debug

> /home/wassname/miniforge3/envs/deeptime/lib/python3.8/site-packages/torch/utils/data/dataset.py(83)__getattr__()
     81             return function
     82         else:
---> 83             raise AttributeError
     84 
     85     @classmethod

ipdb> u
> /media/wassname/SGIronWolf/projects5/investing/DeepTime/data/datasets.py(145)__getitem__()
    143         cx_start = idx
    144         cx_end = cx_start + self.lookback_len
--> 145         c_start = cx_end + self.gap
    146         c_end = c_start + self.horizon_len
    147 

ipdb> self.gap
*** AttributeError
ipdb> q

In [43]:

plot_multi(
           save_paths=[
               Path("storage/experiments/Stocks/96S/repeat=0"),
               Path("storage/experiments/Stocks/96Splus/repeat=0"),
           ],
           i=60
          )
1

---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
Cell In [43], line 1
----> 1 plot_multi(
      2            save_paths=[
      3                Path("storage/experiments/Stocks/96S/repeat=0"),
      4                Path("storage/experiments/Stocks/96Splus/repeat=0"),
      5            ],
      6            i=60
      7           )
      8 1

Cell In [39], line 9, in plot_multi(save_paths, i, title, plot)
      6 gin.parse_config(open(save_path/"config.gin"))
      7 model_name = gin.query_parameter("instance.model_type")
----> 9 train_set, train_loader = get_data(flag='test', batch_size=3)
     11 model = get_model(model_name,
     12                     dim_size=train_set.data_x.shape[1],
     13                     datetime_feats=train_set.timestamps.shape[-1]).to(default_device())
     14 model.load_state_dict(torch.load(save_path/'model.pth'))

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/gin/config.py:1605, in _make_gin_wrapper.<locals>.gin_wrapper(*args, **kwargs)
   1603 scope_info = " in scope '{}'".format(scope_str) if scope_str else ''
   1604 err_str = err_str.format(name, fn_or_cls, scope_info)
-> 1605 utils.augment_exception_message_and_reraise(e, err_str)

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/gin/utils.py:41, in augment_exception_message_and_reraise(exception, message)
     39 proxy = ExceptionProxy()
     40 ExceptionProxy.__qualname__ = type(exception).__qualname__
---> 41 raise proxy.with_traceback(exception.__traceback__) from None

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/gin/config.py:1582, in _make_gin_wrapper.<locals>.gin_wrapper(*args, **kwargs)
   1579 new_kwargs.update(kwargs)
   1581 try:
-> 1582   return fn(*new_args, **new_kwargs)
   1583 except Exception as e:  # pylint: disable=broad-except
   1584   err_str = ''

File /media/wassname/SGIronWolf/projects5/investing/DeepTime/experiments/forecast.py:115, in get_data(flag, batch_size)
    113 else:
    114     raise ValueError(f'no such flag {flag}')
--> 115 dataset = ForecastDataset(flag)
    116 data_loader = DataLoader(dataset,
    117                          batch_size=batch_size,
    118                          shuffle=shuffle,
    119                          drop_last=drop_last)
    120 return dataset, data_loader

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/gin/config.py:1605, in _make_gin_wrapper.<locals>.gin_wrapper(*args, **kwargs)
   1603 scope_info = " in scope '{}'".format(scope_str) if scope_str else ''
   1604 err_str = err_str.format(name, fn_or_cls, scope_info)
-> 1605 utils.augment_exception_message_and_reraise(e, err_str)

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/gin/utils.py:41, in augment_exception_message_and_reraise(exception, message)
     39 proxy = ExceptionProxy()
     40 ExceptionProxy.__qualname__ = type(exception).__qualname__
---> 41 raise proxy.with_traceback(exception.__traceback__) from None

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/gin/config.py:1582, in _make_gin_wrapper.<locals>.gin_wrapper(*args, **kwargs)
   1579 new_kwargs.update(kwargs)
   1581 try:
-> 1582   return fn(*new_args, **new_kwargs)
   1583 except Exception as e:  # pylint: disable=broad-except
   1584   err_str = ''

File /media/wassname/SGIronWolf/projects5/investing/DeepTime/data/datasets.py:71, in ForecastDataset.__init__(self, flag, horizon_len, scale, cross_learn, data_path, root_path, features, target, lookback_len, lookback_aux_len, lookback_mult, time_features, normalise_time_features)
     69 self.timestamps = None
     70 self.n_time = None
---> 71 self.n_time_samples = None
     72 self.load_data()

File /media/wassname/SGIronWolf/projects5/investing/DeepTime/data/datasets.py:105, in ForecastDataset.load_data(self)
    103 self.data_x = data[border1:border2] 
    104 # y is just the col we predict
--> 105 self.data_y = data[border1:border2][[self.target]]
    106 self.timestamps = get_time_features(pd.to_datetime(df_raw.date[border1:border2].values),
    107                                     normalise=self.normalise_time_features,
    108                                     features=self.time_features)
    109 self.n_time = len(self.data_x)

IndexError: only integers, slices (`:`), ellipsis (`...`), numpy.newaxis (`None`) and integer or boolean arrays are valid indices
  In call to configurable 'ForecastDataset' (<class 'data.datasets.ForecastDataset'>)
  In call to configurable 'get_data' (<function get_data at 0x7fb53c141160>)

In [44]:

%debug

> /media/wassname/SGIronWolf/projects5/investing/DeepTime/data/datasets.py(105)load_data()
    103         self.data_x = data[border1:border2]
    104         # y is just the col we predict
--> 105         self.data_y = data[border1:border2][[self.target]]
    106         self.timestamps = get_time_features(pd.to_datetime(df_raw.date[border1:border2].values),
    107                                             normalise=self.normalise_time_features,

ipdb> self.target
'RSMKs_18_144_72'
ipdb> data[border1:border2]
array([[-0.31080395],
       [-0.30072371],
       [-0.29116544],
       ...,
       [-0.14084614],
       [-0.15264537],
       [-0.16396183]])
ipdb> data
array([[ 0.08264075],
       [ 0.08548946],
       [ 0.08766026],
       ...,
       [-0.14084614],
       [-0.15264537],
       [-0.16396183]])
ipdb> data.shape
(53398, 1)
ipdb> q

In [38]:

plot_multi(
           save_paths=[
               Path("storage/experiments/Stocks/96Sshort/repeat=0"),
               Path("storage/experiments/Stocks/96Splusshort/repeat=0"),
           ], i=60)
1

Out [38]:

torch.Size([1, 54, 1]) torch.Size([1, 54, 256]) 3 torch.Size([3, 54, 256])
receptive field [690 378 242]=[138  18   2]*[[ 1  1  1]
 [ 1  2  4]
 [ 1  4 16]
 [ 1  6 36]
 [ 1  8 64]]

1

In [ ]:

# plot("deeptime", Path("storage/experiments/Exchange/96S/repeat=0"));

multi

In [ ]:

def plot_multiM(save_paths=[Path("storage/experiments/Exchange/96M/repeat=0")], i=200, title=None, plot=True):
    for j in range(len(save_paths)):
        save_path = save_paths[j]

        gin.clear_config()
        gin.parse_config(open(save_path/"config.gin"))
        model_name = gin.query_parameter("instance.model_type")
        train_set, train_loader = get_data(flag='train', batch_size=2)

        model = get_model(model_name,
                            dim_size=train_set.data_x.shape[1],
                            datetime_feats=train_set.timestamps.shape[-1]).to(default_device())
        model.load_state_dict(torch.load(save_path/'model.pth'))
        model = model.eval()


        b = train_set[i]
        b = [bb[None, :] for bb in b]
        x, y, x_time, y_time = map(to_tensor, b)
        with torch.no_grad():
            forecast = model(x, x_time, y_time)

       
        colors = list(mcolors.BASE_COLORS.keys())
        l = x.shape[1]
        forecast2 = forecast[0].detach().cpu().numpy()
        x2 = x[0].cpu()
        y2 = y[0].cpu()
        l2 = y.shape[1]
        i_past = list(range(l))
        i_future = list(range(l, l+l2))

        linestyles =  [ '--', '-.', '-', ':']
        if plot:
            ls = linestyles[j]
            print(ls)
            if j==0:
                
                
                for k in range(x.shape[-1]):
                    plt.plot(i_past, x2[:, k], c=colors[k], label=f"var {k}")
                for k in range(x.shape[-1]):
                    plt.plot(i_future, y2[:, k], c=colors[k], alpha=0.3)
            
            mtitle = str(save_path).split('/')[-2:-1]
            mtitle = "-".join(mtitle)
            for k in range(x.shape[-1]):
                plt.plot(i_future, forecast2[:, k], c=colors[k], linestyle=ls, label=f"{mtitle}" if k==0 else None)
    plt.legend()
    plt.title(title)
    return x2, y2, forecast2, i_past, i_future

In [ ]:

plot_multiM([Path("storage/experiments/Stocks/96M/repeat=0")]);

In [ ]:

In [ ]: