Volt/experiments/calibration/plotter.ipynb

In [1]:

import pickle as pkl
import pandas as pd
import numpy as np
import torch
import matplotlib.pyplot as plt
import seaborn as sns
import os
from voltron.data import make_ticker_list, GetStockHistory

sns.set_style('white')
palette = ["#1b4079", "#C6DDF0", "#048A81", "#B9E28C", "#8C2155", "#AF7595", "#E6480F", "#FA9500"]
sns.set(palette = palette, font_scale=2.0, style="white", rc={"lines.linewidth": 4.0})

Warning no robinhood utils.

In [2]:

matern_df = pd.read_pickle("./matern_calib.pkl")
volt_df = pd.read_pickle("./volt_calib.pkl")
matern_df.columns = volt_df.columns

---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
<ipython-input-2-18f65b61756d> in <module>
----> 1 matern_df = pd.read_pickle("./matern_calib.pkl")
      2 volt_df = pd.read_pickle("./volt_calib.pkl")
      3 matern_df.columns = volt_df.columns

~/miniconda3/envs/rpp/lib/python3.8/site-packages/pandas/io/pickle.py in read_pickle(filepath_or_buffer, compression, storage_options)
    183     """
    184     excs_to_catch = (AttributeError, ImportError, ModuleNotFoundError, TypeError)
--> 185     with get_handle(
    186         filepath_or_buffer,
    187         "rb",

~/miniconda3/envs/rpp/lib/python3.8/site-packages/pandas/io/common.py in get_handle(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)
    649         else:
    650             # Binary mode
--> 651             handle = open(handle, ioargs.mode)
    652         handles.append(handle)
    653 

FileNotFoundError: [Errno 2] No such file or directory: './matern_calib.pkl'

In [6]:

matern_df.Mean.unique()

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-6-5b36a26c82e9> in <module>
----> 1 matern_df.Mean.unique()

NameError: name 'matern_df' is not defined

In [7]:

def ECDF(sample_pxs, true_px):    
    return (torch.sum(sample_pxs < true_px, 0)/sample_pxs.shape[0])
 
def Calibration(pcts, percentile=0.95):
    in_band = np.where((pcts < percentile))[0].shape[0]
    return in_band/pcts.shape[0]

def GetCalibration(model, horizon=np.arange(75,100), 
                   logger=[], exp=True):
    

    ntrain = 400
    n_test_times = 20
    ntest = 100
    pcts = torch.tensor([])
    for tckr in ticker_list:
        data = GetStockHistory(tckr, history=1000, end_date=end_date)
        for idx, date in enumerate(data.index):

            fpath = "./saved-outputs/"+ tckr + "/"
            fname = model + "_"
            if model == 'volt':
                fname +=  "constant"

            fname += str(date.date()) + ".pt"


            if os.path.exists(fpath + fname): 
                preds = torch.load(fpath + fname)
                if isinstance(preds, tuple):
                    preds = preds[0]
                
                if preds.shape[-1] == 100:
                    preds = preds[:, horizon]

                    test_y = torch.tensor(data.iloc[idx:idx+100].Close.to_numpy())
                    if test_y.shape[0] == 100:
                        if exp:
                            preds = preds.exp()
                        pcts = torch.cat((pcts, ECDF(preds, test_y[horizon])))
            
    if pcts.numel() == 0:
        return logger
    
    pcts = pcts.flatten().numpy()
    percentiles = np.linspace(0.05, 0.95, 19)
    for pct in percentiles:
        clb = Calibration(pcts, pct)
        logger.append([clb, np.round(pct, 2), model, "Constant", 100])
        
    return logger

In [8]:

data_path = "../../voltron/data/"
ticker_list = make_ticker_list(data_path + "test_tickers.txt")

In [9]:

end_date = "2022-01-13"

In [10]:

log = []
for model in ['volt', 'lstm']:
    log = GetCalibration(model, horizon=np.arange(75,100), 
                   logger=log, exp=True)

In [11]:

df = pd.DataFrame(log)
df.columns = ['Calibration', 'Percentile', "Model", "Mean", "k"]

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-11-e23ecfc68e6a> in <module>
      1 df = pd.DataFrame(log)
----> 2 df.columns = ['Calibration', 'Percentile', "Model", "Mean", "k"]

~/miniconda3/envs/rpp/lib/python3.8/site-packages/pandas/core/generic.py in __setattr__(self, name, value)
   5476         try:
   5477             object.__getattribute__(self, name)
-> 5478             return object.__setattr__(self, name, value)
   5479         except AttributeError:
   5480             pass

pandas/_libs/properties.pyx in pandas._libs.properties.AxisProperty.__set__()

~/miniconda3/envs/rpp/lib/python3.8/site-packages/pandas/core/generic.py in _set_axis(self, axis, labels)
    668     def _set_axis(self, axis: int, labels: Index) -> None:
    669         labels = ensure_index(labels)
--> 670         self._mgr.set_axis(axis, labels)
    671         self._clear_item_cache()
    672 

~/miniconda3/envs/rpp/lib/python3.8/site-packages/pandas/core/internals/managers.py in set_axis(self, axis, new_labels)
    218 
    219         if new_len != old_len:
--> 220             raise ValueError(
    221                 f"Length mismatch: Expected axis has {old_len} elements, new "
    222                 f"values have {new_len} elements"

ValueError: Length mismatch: Expected axis has 0 elements, new values have 5 elements

In [15]:

df = pd.concat([df, matern_df, volt_df])

In [16]:

mat_df = df[(df['Model'] == 'matern') & (df['Mean'] == 'tewma') & (df['k'] == 400)]
lstm_df = df[df['Model']=='lstm']
volt_df = df[(df['Model'] == 'volt') & (df['Mean'] == 'ewma') & (df['k']==100)]
plt_df = pd.concat([lstm_df, mat_df, volt_df])

In [27]:

mat_df = df[(df['Model'] == 'matern') & (df['Mean'] == 'constant')]
volt_df = df[(df['Model'] == 'volt') & (df['Mean'] == 'Constant')]
const_df = pd.concat([mat_df, volt_df])

In [41]:

from matplotlib.lines import Line2D
fig, ax = plt.subplots(1,1,dpi=150, figsize=(6, 3))

percentiles = np.linspace(0.05, 0.95, 19)


pal = [palette[5], palette[7]]
# pal = [palette[5]]
sns.lineplot(x='Percentile', y="Calibration", hue='Model', data=const_df, ax=ax, alpha=0.2,
            palette=pal, legend=True)
sns.scatterplot(x='Percentile', y="Calibration", hue='Model', data=const_df, ax=ax, s=120, legend=False, zorder=4,
               palette=pal, alpha=0.35)

pal = [ palette[0], palette[4], palette[6]]
sns.lineplot(x='Percentile', y="Calibration", hue='Model', data=plt_df, ax=ax, alpha=0.5,
            palette=pal, legend=True)
sns.scatterplot(x='Percentile', y="Calibration", hue='Model', data=plt_df, ax=ax, s=120, legend=False, zorder=4,
               palette=pal)
x = np.linspace(0.05,0.95)
y = np.linspace(0, len(percentiles))
ax.plot(x, x, color="gray", lw=1., ls="--")
ax.set_title("Stock Price Calibration")



plt.tick_params(labelsize=16)
sns.despine()

custom_lines = [Line2D([0], [0], color=palette[0], lw=2),
               Line2D([0], [0], color=palette[4], lw=2),
               Line2D([0], [0], color=palette[6], lw=2),
               Line2D([0], [0], color=palette[5], lw=2),
               Line2D([0], [0], color=palette[7], lw=2)]


plt.legend(custom_lines, ['LSTM', r"Matérn + Magpie", "Volt + Magpie", "Matérn + Constant",
                         "Volt + Constant"],
           fontsize=14, frameon=False, bbox_to_anchor=(1., 0.85))
# ax.legend(fontsize=14, bbox_to_anchor=(1., 0.75))
# plt.label("Percentile")
plt.savefig("./stock_calibration.pdf", bbox_inches="tight")
plt.show()

In [ ]:

In [ ]: