mirror of
https://github.com/wassname/DeepTime.git
synced 2026-08-20 12:00:07 +08:00
7.1 MiB
7.1 MiB
In [1]:
import warnings
warnings.simplefilter("ignore")
# autoreload import your package
%load_ext autoreload
%autoreload 2In [2]:
import os
from os.path import join
import math
import logging
from typing import Callable, Optional, Union, Dict, Tuple
from matplotlib import pyplot as plt
from pathlib import Path
import matplotlib.colors as mcolors
import gin
from fire import Fire
import numpy as np
import torch
from torch.utils.data import DataLoader
from torch import optim
from torch import nn
from experiments.base import Experiment
from data.datasets import ForecastDataset
from models import get_model
from utils.checkpoint import Checkpoint
from utils.ops import default_device, to_tensor
from utils.losses import get_loss_fn
from utils.metrics import calc_metrics
from experiments.forecast import get_data
gin.enter_interactive_mode()In [3]:
import logging
logging.root.setLevel(logging.INFO)
from loguru import logger
logger.remove()
logger.add(os.sys.stdout, level="INFO", colorize=True, format="<level>{time} | {message}</level>")Out [3]:
1
In [4]:
def plot(model_name="deeptime", save_path=Path("storage/experiments/Exchange/96M/repeat=0"), i=200, title=None, plot=True):
gin.clear_config()
gin.parse_config(open(save_path/"config.gin"))
train_set, train_loader = get_data(flag='train', batch_size=2)
model = get_model(model_name,
dim_size=train_set.data_x.shape[1],
datetime_feats=train_set.timestamps.shape[-1]).to(default_device())
model.load_state_dict(torch.load(save_path/'model.pth'))
model = model.eval()
b = train_set[i]
b = [bb[None, :] for bb in b]
b2 = list(map(to_tensor, b))
context_past_x, context_y, query_past_x, query_y, context_time, query_time = b2
with torch.no_grad():
forecast = model(*b2)
if title is None:
title = str(save_path).split('/')[-3:]
title = "-".join(title)
colors = list(mcolors.BASE_COLORS.keys())
l = x.shape[1]
forecast2 = forecast[0].detach().cpu().numpy()
x2 = x[0].cpu()
y2 = y[0].cpu()
l2 = y.shape[1]
i_past = list(range(l))
i_future = list(range(l, l+l2))
if plot:
plt.title(title)
for i in range(x.shape[-1]):
plt.plot(i_past, x2[:, i], c=colors[i])
for i in range(x.shape[-1]):
plt.plot(i_future, y2[:, i], c=colors[i])
for i in range(x.shape[-1]):
plt.plot(i_future, forecast2[:, i], c=colors[i], linestyle='--')
return x2, y2, forecast2, i_past, i_future
In [ ]:
In [5]:
list(mcolors.BASE_COLORS.keys())Out [5]:
['b', 'g', 'r', 'c', 'm', 'y', 'k', 'w']
In [ ]:
In [6]:
# list the models we have run...
configs=sorted(Path("storage/experiments/Stocks").glob("**/config.gin"))
import random
random.shuffle(configs)
# print(configs)In [7]:
from experiments.forecast import ForecastExperiment
from tqdm.auto import tqdmIn [8]:
# for config in tqdm(configs):
# save_path = config.parent
# exp = ForecastExperiment(config_path=config)
# print(config)
# try:
# exp.run()
# except KeyboardInterrupt:
# raise
# except Exception as e:
# raise
# print(e)
# passIn [9]:
# save_path = Path('storage/experiments/Stocks/96M2S/repeat=0')In [10]:
# gin.clear_config()
# config_path = save_path/"config.gin"
# gin.parse_config(open(config_path))
# model_name = gin.query_parameter("instance.model_type")
# model_nameIn [11]:
# exp = ForecastExperiment(config_path=config_path)
# # exp.run()In [12]:
def save_path2name(save_path: Path) -> str:
"""
Path('storage/experiments/Stocks/96M2S/base_learner=None,inr=INR,encoder=mlp,repeat=0')
to
'96M2S-None_INR_mlp_0'
"""
mtitle = str(save_path).split('/')[-2:]
tags = mtitle[-1]
tags = [x.split('=')[-1] for x in tags.split(',')]
mtitle[-1] = '_'.join(tags)
mtitle = "-".join(mtitle)
return mtitle
# save_path2name(save_path)
# save_pathIn [140]:
from torchsummaryX import summary
from torchinfo import summary
def plot_multi(save_paths=[Path("storage/experiments/Exchange/96M/repeat=0")], i=200, title=None, plot=True, verbose=1,):
assert len(save_paths)>0
for j in range(len(save_paths)):
save_path = save_paths[j]
gin.clear_config()
gin.parse_config(open(save_path/"config.gin"))
model_name = gin.query_parameter("instance.model_type")
train_set, train_loader = get_data(flag='test', batch_size=3)
seq_len = train_set[0][0].shape[0]
model = get_model(model_name,
dim_size=train_set.data_x.shape[1],
seq_len=seq_len,
datetime_feats=train_set.timestamps.shape[-1]).to(default_device())
model.load_state_dict(torch.load(save_path/'model.pth'))
model = model.eval()
b = train_set[i]
b = [bb[None, :] for bb in b]
b2 = list(map(to_tensor, b))
# b = next(iter(train_loader))
# print([s.shape for s in b]
if verbose>1:
print(summary(model, input_data=b2, depth=2))
print(save_path)
context_past_x, context_y, query_past_x, query_y, context_time, query_time = b2
with torch.no_grad():
forecast = model(*b2)
colors = list(mcolors.BASE_COLORS.keys())
l = context_time.shape[1]
forecast2 = forecast[0].detach().cpu().numpy()
x2 = context_y[0].cpu()
y2 = query_y[0].cpu()
l2 = query_time.shape[1]
i_past = list(range(l))
i_future = list(range(l, l+l2))
if plot:
if j==0:
plt.plot(i_past, x2[:, 0], c=colors[0], label=f"past")
plt.plot(i_future, y2[:, 0], c=colors[0], label="future true", alpha=0.3)
mtitle = save_path2name(save_path)
plt.plot(i_future, forecast2[:, 0], linestyle='--', label=f"{mtitle}") # c=colors[j],
plt.legend()
plt.title(title)
return x2, y2, forecast2, i_past, i_future
In [472]:
# list the models we have run...
m=sorted(Path("storage/experiments/Stocks/96M2S").glob("**/_SUCCESS"))
print(m)[Path('storage/experiments/Stocks/96M2S/lrn=None,inr=INR,enc=inception,repeat=0/_SUCCESS'), Path('storage/experiments/Stocks/96M2S/lrn=None,inr=INR,enc=lstm,repeat=0/_SUCCESS'), Path('storage/experiments/Stocks/96M2S/lrn=None,inr=INR,enc=lstm2,repeat=0/_SUCCESS'), Path('storage/experiments/Stocks/96M2S/lrn=None,inr=INRPlus2,enc=inception,repeat=0/_SUCCESS'), Path('storage/experiments/Stocks/96M2S/lrn=None,inr=INRPlus2,enc=lstm,repeat=0/_SUCCESS'), Path('storage/experiments/Stocks/96M2S/lrn=None,inr=INRPlus2,enc=lstm2,repeat=0/_SUCCESS'), Path('storage/experiments/Stocks/96M2S/lrn=None,inr=INRPlus2,enc=none,repeat=0/_SUCCESS'), Path('storage/experiments/Stocks/96M2S/lrn=Ridge,inr=INR,enc=inception,repeat=0/_SUCCESS'), Path('storage/experiments/Stocks/96M2S/lrn=Ridge,inr=INR,enc=mlp,repeat=0/_SUCCESS'), Path('storage/experiments/Stocks/96M2S/lrn=Ridge,inr=INR,enc=none,repeat=0/_SUCCESS'), Path('storage/experiments/Stocks/96M2S/lrn=Ridge,inr=INR,enc=transformer,repeat=0/_SUCCESS'), Path('storage/experiments/Stocks/96M2S/lrn=Ridge,inr=INR,enc=transformer2,repeat=0/_SUCCESS'), Path('storage/experiments/Stocks/96M2S/lrn=Ridge,inr=INRPlus2,enc=inception,repeat=0/_SUCCESS'), Path('storage/experiments/Stocks/96M2S/lrn=Ridge,inr=INRPlus2,enc=lstm2,repeat=0/_SUCCESS'), Path('storage/experiments/Stocks/96M2S/lrn=Ridge,inr=INRPlus2,enc=transformer2,repeat=0/_SUCCESS'), Path('storage/experiments/Stocks/96M2S/lrn=Transformer,inr=INR,enc=inception,repeat=0/_SUCCESS'), Path('storage/experiments/Stocks/96M2S/lrn=Transformer,inr=INR,enc=lstm2,repeat=0/_SUCCESS'), Path('storage/experiments/Stocks/96M2S/lrn=Transformer,inr=INRPlus2,enc=lstm,repeat=0/_SUCCESS'), Path('storage/experiments/Stocks/96M2S/lrn=Transformer,inr=INRPlus2,enc=mlp,repeat=0/_SUCCESS'), Path('storage/experiments/Stocks/96M2S/lrn=Transformer,inr=INRPlus2,enc=none,repeat=0/_SUCCESS'), Path('storage/experiments/Stocks/96M2S/lrn=Transformer,inr=INRPlus2,enc=transformer,repeat=0/_SUCCESS'), Path('storage/experiments/Stocks/96M2S/lrn=Transformer,inr=INRPlus2,enc=transformer2,repeat=0/_SUCCESS')]
In [473]:
# show score
good_m = []
for mm in m:
mtitle = save_path2name(mm.parent)
# print(mtitle)
m3 = json.load((mm.parent/'metrics.json').open())
m3 = eval(str(m3))
print(f"mape={m3['val']['mape']:2.2f}, {mtitle}")
s = m3['val']['mape']
if s < 1:
good_m.append(mm)mape=2.82, 96M2S-None_INR_inception_0 mape=1.74, 96M2S-None_INR_lstm_0 mape=3.13, 96M2S-None_INR_lstm2_0 mape=2.06, 96M2S-None_INRPlus2_inception_0 mape=2.26, 96M2S-None_INRPlus2_lstm_0 mape=2.13, 96M2S-None_INRPlus2_lstm2_0 mape=1.38, 96M2S-None_INRPlus2_none_0 mape=1.41, 96M2S-Ridge_INR_inception_0 mape=1.83, 96M2S-Ridge_INR_mlp_0 mape=0.92, 96M2S-Ridge_INR_none_0 mape=1.36, 96M2S-Ridge_INR_transformer_0 mape=1.51, 96M2S-Ridge_INR_transformer2_0 mape=0.86, 96M2S-Ridge_INRPlus2_inception_0 mape=0.84, 96M2S-Ridge_INRPlus2_lstm2_0 mape=0.85, 96M2S-Ridge_INRPlus2_transformer2_0 mape=1.76, 96M2S-Transformer_INR_inception_0 mape=2.39, 96M2S-Transformer_INR_lstm2_0 mape=1.95, 96M2S-Transformer_INRPlus2_lstm_0 mape=1.80, 96M2S-Transformer_INRPlus2_mlp_0 mape=1.89, 96M2S-Transformer_INRPlus2_none_0 mape=1.94, 96M2S-Transformer_INRPlus2_transformer_0 mape=1.95, 96M2S-Transformer_INRPlus2_transformer2_0
In [474]:
1/0[0;31m---------------------------------------------------------------------------[0m [0;31mZeroDivisionError[0m Traceback (most recent call last) Cell [0;32mIn [474], line 1[0m [0;32m----> 1[0m [38;5;241;43m1[39;49m[38;5;241;43m/[39;49m[38;5;241;43m0[39;49m [0;31mZeroDivisionError[0m: division by zero
In [479]:
# # show size
# save_paths = [mm.parent for mm in m]
# for mm in m:
# try:
# plot_multi(
# save_paths=[mm.parent],
# i=600,
# verbose=2,
# )
# except Exception as e:
# print('failed', mm, e)
# # mm.unlink()
# pass
# 1In [476]:
save_paths = [mm.parent for mm in good_m]
plot_multi(
save_paths=save_paths,
i=669,
verbose=0,
)
plt.title('just the good ones')
1Out [476]:
receptive field [114 342 362] receptive field [690 378 242] receptive field [690 378 242] receptive field [690 378 242]
1
In [477]:
save_paths = [mm.parent for mm in good_m]
plot_multi(
save_paths=save_paths,
i=679,
verbose=0,
)
plt.title('just the good ones')
1Out [477]:
receptive field [114 342 362] receptive field [690 378 242] receptive field [690 378 242] receptive field [690 378 242]
1
In [478]:
256/24Out [478]:
10.666666666666666
In [ ]:
In [83]:
train_set, train_loader = get_data(flag='test', batch_size=3)
b = context_past_x, context_y, query_past_x, query_y, context_time, query_time = train_set[0]
print([bb.shape for bb in b])
# context_y, query_y[(138, 5), (46, 1), (138, 5), (46, 1), (46, 0), (46, 0)]
In [84]:
cx_start, cx_end, c_start, c_end, qx_start, qx_end, q_start, q_end = train_set.get_inds(0)
cx_start, cx_end, c_start, c_end, qx_start, qx_end, q_start, q_endOut [84]:
(0, 138, 138, 184, 46, 184, 184, 230)
In [ ]:
In [89]:
plt.vlinesOut [89]:
<function matplotlib.pyplot.vlines(x, ymin, ymax, colors=None, linestyles='solid', label='', *, data=None, **kwargs)>
In [94]:
plt.hlines(1, cx_start, cx_end, color='green', alpha=0.5, label='context_past_x')
plt.hlines(2, c_start, c_end, color='green', label='context_labels')
plt.hlines(3, qx_start, qx_end, alpha=0.5, label='query_past_x')
plt.hlines(4, q_start, q_end, label='query_labels/target')
o = train_set.horizon_len+train_set.lookback_len
plt.vlines(o, *plt.ylim(), alpha=0.3, ls='--', label='now')
plt.legend(loc='upper left')Out [94]:
<matplotlib.legend.Legend at 0x7fd91c18e7c0>
In [ ]:
In [493]:
save_path = Path('storage/experiments/Stocks/96M2S/lrn=Ridge,inr=INRPlus2,enc=lstm2,repeat=0')
save_path = Path('storage/experiments/Stocks/96M2S/lrn=Ridge,inr=INRPlus2,enc=inception,repeat=0')
# save_path = Path('storage/experiments/Stocks/96M2S/lrn=Ridge,inr=INRPlus2,enc=transformer2,repeat=0')
# save_path = Path('storage/experiments/Stocks/96M2S/lrn=Ridge,inr=INR,enc=none,repeat=0')In [494]:
gin.clear_config()
gin.parse_config(open(save_path/"config.gin"))
model_name = gin.query_parameter("instance.model_type")
train_set, train_loader = get_data(flag='train', batch_size=256)
test_set, test_loader = get_data(flag='test', batch_size=256)
seq_len = train_set[0][0].shape[0]
model = get_model(model_name,
dim_size=train_set.data_x.shape[1],
seq_len=seq_len,
datetime_feats=train_set.timestamps.shape[-1]).to(default_device())
model.load_state_dict(torch.load(save_path/'model.pth'))
model = model.eval()
receptive field [114 342 362] receptive field [690 378 242]
In [495]:
test_set, test_loader = get_data(flag='test', batch_size=256)
test_set.indexOut [495]:
42765 2020-09-01 16:10:00
42766 2020-09-01 16:20:00
42767 2020-09-01 16:35:00
42768 2020-09-01 16:40:00
42769 2020-09-01 16:45:00
...
53256 2020-12-30 15:30:00
53257 2020-12-30 15:35:00
53258 2020-12-30 15:40:00
53259 2020-12-30 15:45:00
53260 2020-12-30 15:50:00
Name: date, Length: 10496, dtype: objectIn [ ]:
In [496]:
def pred(test_loader, model):
isinstance(test_loader.sampler, torch.utils.data.sampler.SequentialSampler)
forecasts = []
with torch.no_grad():
for it, data in enumerate(tqdm(test_loader)):
data2 = list(map(to_tensor, data))
context_past_x, context_y, query_past_x, query_y, context_time, query_time = data2
if context_past_x.shape[0] == 1:
# skip final batch if batch_size == 1
# due to bug in torch.linalg.solve which raises error when batch_size == 1
continue
forecast = model(*data2)
forecasts.append(forecast.cpu().numpy())
return np.concatenate(forecasts, 0)In [497]:
import pandas as pd
df = pd.read_csv(Path('storage/datasets')/test_loader.dataset.data_path).set_index('date')
close = df.close.reindex(test_set.index)
closeOut [497]:
date
2020-09-01 16:10:00 12.480
2020-09-01 16:20:00 12.480
2020-09-01 16:35:00 12.500
2020-09-01 16:40:00 12.500
2020-09-01 16:45:00 12.500
...
2020-12-30 15:30:00 17.845
2020-12-30 15:35:00 17.875
2020-12-30 15:40:00 17.840
2020-12-30 15:45:00 17.800
2020-12-30 15:50:00 17.895
Name: close, Length: 10496, dtype: float64In [498]:
y_pred = pred(test_loader, model)[:, :, 0]
y_predOut [498]:
0%| | 0/41 [00:00<?, ?it/s]
array([[-0.6312843 , -0.6324215 , -0.63075 , ..., -0.57064354,
-0.57067883, -0.5687439 ],
[-0.6315764 , -0.6330354 , -0.6340139 , ..., -0.5467202 ,
-0.5472645 , -0.546636 ],
[-0.62531006, -0.63098663, -0.6323066 , ..., -0.5397977 ,
-0.537265 , -0.53612953],
...,
[ 0.5385701 , 0.5380618 , 0.55043346, ..., 0.6225661 ,
0.62645656, 0.6287976 ],
[ 0.56083286, 0.56980455, 0.5735802 , ..., 0.6431285 ,
0.6496463 , 0.649108 ],
[ 0.57005936, 0.57283384, 0.5839323 , ..., 0.67222625,
0.674573 , 0.673405 ]], dtype=float32)In [499]:
fees = 0.0025
l = 33
nn = 3
d = y_pred[:, 0] - y_pred[:, l]
d = pd.Series(d).rolling(nn).mean().fillna(0)
# we need some idea of the confidence. We will get this by looking at how much the prediction is varying
# and only taking the top 5%
conf = 1-pd.Series(d).rolling(n).std().fillna(0)
thresh = conf.quantile(0.97)
n = 20
entries = (d>fees*n) & (conf>thresh)
exits = (d<fees) & (conf>thresh)
short_entries = (d<-fees*n) & (conf>thresh)
short_exits = (d>-fees) & (conf>thresh)
size = 0.5In [ ]:
In [500]:
import vectorbt as vbt
pf = vbt.Portfolio.from_signals(
close,
entries=entries,
exits=exits,
short_entries=short_entries,
short_exits=short_exits,
fees=fees,
freq="5T",
size=size # size=1 and short leads to inf's
)
s = pf.stats()[
["Win Rate [%]", "Profit Factor", "Total Trades", "Sharpe Ratio"]
]
print(s)
pf.stats()Out [500]:
Win Rate [%] 70.0 Profit Factor 3.045164 Total Trades 21 Sharpe Ratio 5.686471 dtype: object
Start 2020-09-01 16:10:00 End 2020-12-30 15:50:00 Period 36 days 10:40:00 Start Value 100.0 End Value 105.777266 Total Return [%] 5.777266 Benchmark Return [%] 43.389423 Max Gross Exposure [%] 10.348995 Total Fees Paid 0.676384 Max Drawdown [%] 1.929388 Max Drawdown Duration 10 days 01:15:00 Total Trades 21 Total Closed Trades 20 Total Open Trades 1 Open Trade PnL 0.758175 Win Rate [%] 70.0 Best Trade [%] 24.925 Worst Trade [%] -24.555761 Avg Winning Trade [%] 7.587012 Avg Losing Trade [%] -7.813665 Avg Winning Trade Duration 1 days 08:21:25.714285714 Avg Losing Trade Duration 2 days 04:45:50 Profit Factor 3.045164 Expectancy 0.250955 Sharpe Ratio 5.686471 Calmar Ratio 39.135368 Omega Ratio 1.061663 Sortino Ratio 8.288093 dtype: object
In [501]:
pf.plot().show()In [ ]:
In [ ]:
In [ ]: