DeepTime/scratch-run_exp.ipynb

• try just one predictor
  • multi input, single output
• comparem ulti
• losses:
  • try logp? nah
  • mae?
• make my own csv with 5m data (maybe 10k rows)
• backtest?

In [1]:

import warnings
warnings.simplefilter("ignore")

# autoreload import your package
%load_ext autoreload
%autoreload 2

In [2]:

import os
from os.path import join
import math
import logging
from typing import Callable, Optional, Union, Dict, Tuple

from matplotlib import pyplot as plt
from pathlib import Path
import matplotlib.colors as mcolors

import gin
from fire import Fire
import numpy as np
import torch
from torch.utils.data import DataLoader
from torch import optim
from torch import nn

from experiments.base import Experiment
from data.datasets import ForecastDataset
from models import get_model
from utils.checkpoint import Checkpoint
from utils.ops import default_device, to_tensor
from utils.losses import get_loss_fn
from utils.metrics import calc_metrics

from experiments.forecast import get_data
gin.enter_interactive_mode()

In [ ]:

auto

In [3]:

def plot(model_name="deeptime", save_path=Path("storage/experiments/Exchange/96M/repeat=0"), i=200, title=None, plot=True):

    gin.clear_config()
    gin.parse_config(open(save_path/"config.gin"))

    train_set, train_loader = get_data(flag='train', batch_size=2)

    model = get_model(model_name,
                        dim_size=train_set.data_x.shape[1],
                        datetime_feats=train_set.timestamps.shape[-1]).to(default_device())
    model.load_state_dict(torch.load(save_path/'model.pth'))
    model = model.eval()


    b = train_set[i]
    b = [bb[None, :] for bb in b]
    x, y, x_time, y_time = map(to_tensor, b)
    with torch.no_grad():
        forecast = model(x, x_time, y_time)

    if title is None:
        title = str(save_path).split('/')[-3:]
        title = "-".join(title)
    
    colors = list(mcolors.BASE_COLORS.keys())
    l = x.shape[1]
    forecast2 = forecast[0].detach().cpu().numpy()
    x2 = x[0].cpu()
    y2 = y[0].cpu()
    l2 = y.shape[1]
    i_past = list(range(l))
    i_future = list(range(l, l+l2))
    
    if plot:
        plt.title(title)
        for i in range(x.shape[-1]):
            plt.plot(i_past, x2[:, i], c=colors[i])
        for i in range(x.shape[-1]):
            plt.plot(i_future, y2[:, i], c=colors[i])
        for i in range(x.shape[-1]):
            plt.plot(i_future, forecast2[:, i], c=colors[i], linestyle='--')
    return x2, y2, forecast2, i_past, i_future

In [4]:

def plot_multi(save_paths=[Path("storage/experiments/Exchange/96M/repeat=0")], i=200, title=None, plot=True):
    for j in range(len(save_paths)):
        save_path = save_paths[j]

        gin.clear_config()
        gin.parse_config(open(save_path/"config.gin"))
        model_name = gin.query_parameter("instance.model_type")

        train_set, train_loader = get_data(flag='test', batch_size=3)

        model = get_model(model_name,
                            dim_size=train_set.data_x.shape[1],
                            datetime_feats=train_set.timestamps.shape[-1]).to(default_device())
        model.load_state_dict(torch.load(save_path/'model.pth'))
        model = model.eval()


        b = train_set[i]
        b = [bb[None, :] for bb in b]
        
        b = next(iter(train_loader))
        print([s.shape for s in b])
        
        x, y, x_time, y_time = map(to_tensor, b)
#         print(b)
        with torch.no_grad():
            forecast = model(x, x_time, y_time)
       
        colors = list(mcolors.BASE_COLORS.keys())
        l = x.shape[1]
        forecast2 = forecast[0].detach().cpu().numpy()
        x2 = x[0].cpu()
        y2 = y[0].cpu()
        l2 = y.shape[1]
        i_past = list(range(l))
        i_future = list(range(l, l+l2))

        if plot:
            plt.plot(i_past, x2[:, 0], c=colors[0], label=f"past")
            plt.plot(i_future, y2[:, 0], c=colors[0], label="future true", alpha=0.3)
            
            mtitle = str(save_path).split('/')[-2:-1]
            mtitle = "-".join(mtitle)
            plt.plot(i_future, forecast2[:, 0], c=colors[j], linestyle='--', label=f"{mtitle}")
    plt.legend()
    plt.title(title)
    return x2, y2, forecast2, i_past, i_future

In [5]:

# list the models we have run...
m=sorted(Path("storage/experiments/Stocks").glob("**/_SUCCESS"))
print(m)

[Path('storage/experiments/Stocks/96M/repeat=0/_SUCCESS'), Path('storage/experiments/Stocks/96S/repeat=0/_SUCCESS'), Path('storage/experiments/Stocks/96Splus/repeat=0/_SUCCESS'), Path('storage/experiments/Stocks/96Splusshort/repeat=0/_SUCCESS'), Path('storage/experiments/Stocks/96Sshort/repeat=0/_SUCCESS')]

In [6]:

save_path = Path('storage/experiments/Stocks/96M2S/repeat=0')

In [7]:

gin.clear_config()
config_path = save_path/"config.gin"
gin.parse_config(open(config_path))
model_name = gin.query_parameter("instance.model_type")
model_name

Out [7]:

'deeptime3'

In [8]:

from experiments.forecast import ForecastExperiment
exp = ForecastExperiment(config_path=config_path)
exp

Out [8]:

<experiments.forecast.ForecastExperiment at 0x7f4044fc3fa0>

In [9]:

exp.run()

receptive field [114  72  12]=[38 18  2]*[[1 1 1]
 [1 1 1]
 [1 2 4]]
129 in_feats
receptive field [690 378 242]=[138  18   2]*[[ 1  1  1]
 [ 1  2  4]
 [ 1  4 16]
 [ 1  6 36]
 [ 1  8 64]]
torch.Size([256, 96, 129])
torch.Size([256, 96, 129])

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
Cell In [9], line 1
----> 1 exp.run()

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/gin/config.py:1605, in _make_gin_wrapper.<locals>.gin_wrapper(*args, **kwargs)
   1603 scope_info = " in scope '{}'".format(scope_str) if scope_str else ''
   1604 err_str = err_str.format(name, fn_or_cls, scope_info)
-> 1605 utils.augment_exception_message_and_reraise(e, err_str)

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/gin/utils.py:41, in augment_exception_message_and_reraise(exception, message)
     39 proxy = ExceptionProxy()
     40 ExceptionProxy.__qualname__ = type(exception).__qualname__
---> 41 raise proxy.with_traceback(exception.__traceback__) from None

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/gin/config.py:1582, in _make_gin_wrapper.<locals>.gin_wrapper(*args, **kwargs)
   1579 new_kwargs.update(kwargs)
   1581 try:
-> 1582   return fn(*new_args, **new_kwargs)
   1583 except Exception as e:  # pylint: disable=broad-except
   1584   err_str = ''

File /media/wassname/SGIronWolf/projects5/investing/DeepTime/experiments/base.py:96, in Experiment.run(self, timer)
     94 except Exception as e:
     95     Path(running_flag).unlink()
---> 96     raise e
     97 except KeyboardInterrupt:
     98     Path(running_flag).unlink()

File /media/wassname/SGIronWolf/projects5/investing/DeepTime/experiments/base.py:93, in Experiment.run(self, timer)
     90     Path(running_flag).touch()
     92 try:
---> 93     self.instance()
     94 except Exception as e:
     95     Path(running_flag).unlink()

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/gin/config.py:1605, in _make_gin_wrapper.<locals>.gin_wrapper(*args, **kwargs)
   1603 scope_info = " in scope '{}'".format(scope_str) if scope_str else ''
   1604 err_str = err_str.format(name, fn_or_cls, scope_info)
-> 1605 utils.augment_exception_message_and_reraise(e, err_str)

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/gin/utils.py:41, in augment_exception_message_and_reraise(exception, message)
     39 proxy = ExceptionProxy()
     40 ExceptionProxy.__qualname__ = type(exception).__qualname__
---> 41 raise proxy.with_traceback(exception.__traceback__) from None

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/gin/config.py:1582, in _make_gin_wrapper.<locals>.gin_wrapper(*args, **kwargs)
   1579 new_kwargs.update(kwargs)
   1581 try:
-> 1582   return fn(*new_args, **new_kwargs)
   1583 except Exception as e:  # pylint: disable=broad-except
   1584   err_str = ''

File /media/wassname/SGIronWolf/projects5/investing/DeepTime/experiments/forecast.py:40, in ForecastExperiment.instance(self, model_type, save_vals)
     37 checkpoint = Checkpoint(self.root)
     39 # train forecasting task
---> 40 model = train(model, checkpoint, train_loader, val_loader, test_loader)
     42 # testing
     43 val_metrics = validate(model, loader=val_loader, report_metrics=True)

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/gin/config.py:1605, in _make_gin_wrapper.<locals>.gin_wrapper(*args, **kwargs)
   1603 scope_info = " in scope '{}'".format(scope_str) if scope_str else ''
   1604 err_str = err_str.format(name, fn_or_cls, scope_info)
-> 1605 utils.augment_exception_message_and_reraise(e, err_str)

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/gin/utils.py:41, in augment_exception_message_and_reraise(exception, message)
     39 proxy = ExceptionProxy()
     40 ExceptionProxy.__qualname__ = type(exception).__qualname__
---> 41 raise proxy.with_traceback(exception.__traceback__) from None

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/gin/config.py:1582, in _make_gin_wrapper.<locals>.gin_wrapper(*args, **kwargs)
   1579 new_kwargs.update(kwargs)
   1581 try:
-> 1582   return fn(*new_args, **new_kwargs)
   1583 except Exception as e:  # pylint: disable=broad-except
   1584   err_str = ''

File /media/wassname/SGIronWolf/projects5/investing/DeepTime/experiments/forecast.py:144, in train(model, checkpoint, train_loader, val_loader, test_loader, loss_name, epochs, clip)
    142 data2 = map(to_tensor, data)
    143 context_past_x, context_y, query_past_x, query_y, context_time, query_time = data2
--> 144 forecast = model(context_past_x, context_y, query_past_x, context_time, query_time)
    146 if isinstance(forecast, tuple):
    147     # for models which require reconstruction + forecast loss
    148     loss = training_loss_fn(forecast[0], context_y) + \
    149            training_loss_fn(forecast[1], query_y)

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/torch/nn/modules/module.py:1102, in Module._call_impl(self, *input, **kwargs)
   1098 # If we don't have any hooks, we want to skip the rest of the logic in
   1099 # this function, and just call forward.
   1100 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1101         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1102     return forward_call(*input, **kwargs)
   1103 # Do not call functions when jit is used
   1104 full_backward_hooks, non_full_backward_hooks = [], []

File /media/wassname/SGIronWolf/projects5/investing/DeepTime/models/DeepTIMe3.py:77, in DeepTIMe3.forward(self, context_past_x, context_y, query_past_x, context_time, query_time)
     74 def forward(self, context_past_x, context_y, query_past_x, context_time, query_time) -> Tensor:
     76     context_reprs = self.encode_and_decode(context_past_x, context_time)
---> 77     query_reprs = self.encode_and_decode(query_past_x, query_time, offset=context_reprs.shape[1])
     79     w, b = self.adaptive_weights(context_reprs, context_y)
     80     preds = self.forecast(query_reprs, w, b)

File /media/wassname/SGIronWolf/projects5/investing/DeepTime/models/DeepTIMe3.py:71, in DeepTIMe3.encode_and_decode(self, past_x, time, offset)
     68 context_input = torch.cat([encoded_x, coords, time], dim=-1)
     70 print(context_input.shape)
---> 71 context_repr = self.inr(context_input)
     72 return context_repr

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/torch/nn/modules/module.py:1102, in Module._call_impl(self, *input, **kwargs)
   1098 # If we don't have any hooks, we want to skip the rest of the logic in
   1099 # this function, and just call forward.
   1100 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1101         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1102     return forward_call(*input, **kwargs)
   1103 # Do not call functions when jit is used
   1104 full_backward_hooks, non_full_backward_hooks = [], []

File /media/wassname/SGIronWolf/projects5/investing/DeepTime/models/modules/inrplus2.py:45, in INRPlus2.forward(self, x)
     43 if self.n_fourier_feats>0:
     44     f = torch.concat([f, x], -1)
---> 45 return self.layers(f.permute((0, 2, 1))).permute((0, 2, 1))

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/torch/nn/modules/module.py:1102, in Module._call_impl(self, *input, **kwargs)
   1098 # If we don't have any hooks, we want to skip the rest of the logic in
   1099 # this function, and just call forward.
   1100 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1101         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1102     return forward_call(*input, **kwargs)
   1103 # Do not call functions when jit is used
   1104 full_backward_hooks, non_full_backward_hooks = [], []

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/torch/nn/modules/container.py:141, in Sequential.forward(self, input)
    139 def forward(self, input):
    140     for module in self:
--> 141         input = module(input)
    142     return input

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/torch/nn/modules/module.py:1102, in Module._call_impl(self, *input, **kwargs)
   1098 # If we don't have any hooks, we want to skip the rest of the logic in
   1099 # this function, and just call forward.
   1100 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1101         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1102     return forward_call(*input, **kwargs)
   1103 # Do not call functions when jit is used
   1104 full_backward_hooks, non_full_backward_hooks = [], []

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/torch/nn/modules/container.py:141, in Sequential.forward(self, input)
    139 def forward(self, input):
    140     for module in self:
--> 141         input = module(input)
    142     return input

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/torch/nn/modules/module.py:1102, in Module._call_impl(self, *input, **kwargs)
   1098 # If we don't have any hooks, we want to skip the rest of the logic in
   1099 # this function, and just call forward.
   1100 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1101         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1102     return forward_call(*input, **kwargs)
   1103 # Do not call functions when jit is used
   1104 full_backward_hooks, non_full_backward_hooks = [], []

File /media/wassname/SGIronWolf/projects5/investing/DeepTime/models/modules/causalinception.py:92, in InceptionBlockPlus.forward(self, x)
     90 for i in range(self.depth):
     91     if self.keep_prob[i] > random.random() or not self.training:
---> 92         x = self.inception[i](x)
     93     if self.residual and i % 3 == 2:
     94         res = x = self.act[i//3](self.add(x, self.shortcut[i//3](res)))

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/torch/nn/modules/module.py:1102, in Module._call_impl(self, *input, **kwargs)
   1098 # If we don't have any hooks, we want to skip the rest of the logic in
   1099 # this function, and just call forward.
   1100 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1101         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1102     return forward_call(*input, **kwargs)
   1103 # Do not call functions when jit is used
   1104 full_backward_hooks, non_full_backward_hooks = [], []

File /media/wassname/SGIronWolf/projects5/investing/DeepTime/models/modules/causalinception.py:52, in InceptionModulePlus.forward(self, x)
     50 input_tensor = x
     51 x = self.bottleneck(x)
---> 52 x = self.concat([l(x) for l in self.convs] + [self.mp_conv(input_tensor)])
     53 x = self.norm(x)
     54 x = self.conv_dropout(x)

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/torch/nn/modules/module.py:1102, in Module._call_impl(self, *input, **kwargs)
   1098 # If we don't have any hooks, we want to skip the rest of the logic in
   1099 # this function, and just call forward.
   1100 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1101         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1102     return forward_call(*input, **kwargs)
   1103 # Do not call functions when jit is used
   1104 full_backward_hooks, non_full_backward_hooks = [], []

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/torch/nn/modules/container.py:141, in Sequential.forward(self, input)
    139 def forward(self, input):
    140     for module in self:
--> 141         input = module(input)
    142     return input

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/torch/nn/modules/module.py:1102, in Module._call_impl(self, *input, **kwargs)
   1098 # If we don't have any hooks, we want to skip the rest of the logic in
   1099 # this function, and just call forward.
   1100 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1101         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1102     return forward_call(*input, **kwargs)
   1103 # Do not call functions when jit is used
   1104 full_backward_hooks, non_full_backward_hooks = [], []

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/torch/nn/modules/container.py:141, in Sequential.forward(self, input)
    139 def forward(self, input):
    140     for module in self:
--> 141         input = module(input)
    142     return input

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/torch/nn/modules/module.py:1102, in Module._call_impl(self, *input, **kwargs)
   1098 # If we don't have any hooks, we want to skip the rest of the logic in
   1099 # this function, and just call forward.
   1100 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1101         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1102     return forward_call(*input, **kwargs)
   1103 # Do not call functions when jit is used
   1104 full_backward_hooks, non_full_backward_hooks = [], []

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/tsai/models/layers.py:148, in CausalConv1d.forward(self, input)
    147 def forward(self, input):
--> 148     return super(CausalConv1d, self).forward(F.pad(input, (self.__padding, 0)))

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/torch/nn/modules/conv.py:301, in Conv1d.forward(self, input)
    300 def forward(self, input: Tensor) -> Tensor:
--> 301     return self._conv_forward(input, self.weight, self.bias)

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/torch/nn/modules/conv.py:297, in Conv1d._conv_forward(self, input, weight, bias)
    293 if self.padding_mode != 'zeros':
    294     return F.conv1d(F.pad(input, self._reversed_padding_repeated_twice, mode=self.padding_mode),
    295                     weight, bias, self.stride,
    296                     _single(0), self.dilation, self.groups)
--> 297 return F.conv1d(input, weight, bias, self.stride,
    298                 self.padding, self.dilation, self.groups)

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/fastai/torch_core.py:378, in TensorBase.__torch_function__(cls, func, types, args, kwargs)
    376 if cls.debug and func.__name__ not in ('__str__','__repr__'): print(func, types, args, kwargs)
    377 if _torch_handled(args, cls._opt, func): types = (torch.Tensor,)
--> 378 res = super().__torch_function__(func, types, args, ifnone(kwargs, {}))
    379 dict_objs = _find_args(args) if args else _find_args(list(kwargs.values()))
    380 if issubclass(type(res),TensorBase) and dict_objs: res.set_meta(dict_objs[0],as_copy=True)

File ~/miniforge3/envs/deeptime/lib/python3.8/site-packages/torch/_tensor.py:1051, in Tensor.__torch_function__(cls, func, types, args, kwargs)
   1048     return NotImplemented
   1050 with _C.DisableTorchFunction():
-> 1051     ret = func(*args, **kwargs)
   1052     if func in get_default_nowrap_functions():
   1053         return ret

RuntimeError: CUDA out of memory. Tried to allocate 24.00 MiB (GPU 0; 10.74 GiB total capacity; 8.00 GiB already allocated; 50.12 MiB free; 8.16 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF
  In call to configurable 'train' (<function train at 0x7f4045284ee0>)
  In call to configurable 'instance' (<function ForecastExperiment.instance at 0x7f4045284550>)
  In call to configurable 'run' (<function Experiment.run at 0x7f4092a49550>)

In [ ]:

%debug

In [ ]: