pytorch-ts/examples/m5.ipynb

In [1]:

import matplotlib.pyplot as plt
import json

In [2]:

import torch

In [3]:

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

In [4]:

from gluonts.dataset.repository.datasets import get_dataset
from gluonts.dataset.util import to_pandas
from gluonts.evaluation import Evaluator
from gluonts.evaluation.backtest import make_evaluation_predictions

In [5]:

from pts.model.deepar import DeepAREstimator
from pts.dataset.repository.datasets import dataset_recipes
from pts.modules import ZeroInflatedNegativeBinomialOutput
from pts import Trainer

Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.

In [6]:

dataset = get_dataset("pts_m5", regenerate=False)

In [7]:

entry = next(iter(dataset.train))
train_series = to_pandas(entry)
train_series.plot()
plt.grid(which="both")
plt.legend(["train series"], loc="upper left")
plt.title(entry['item_id'])
plt.show()

In [8]:

entry = next(iter(dataset.test))
test_series = to_pandas(entry)
test_series.plot()
plt.axvline(train_series.index[-1], color='r') # end of train dataset
plt.grid(which="both")
plt.legend(["test series", "end of train series"], loc="upper left")
plt.title(entry['item_id'])
plt.show()

In [9]:

print(f"Recommended prediction horizon: {dataset.metadata.prediction_length}")
print(f"Frequency of the time series: {dataset.metadata.freq}")

Recommended prediction horizon: 28
Frequency of the time series: D

In [21]:

estimator = DeepAREstimator(
    distr_output=ZeroInflatedNegativeBinomialOutput(),
    cell_type='GRU',
    input_size=62,
    num_cells=64,
    num_layers=3,
    dropout_rate=0.1,
    use_feat_dynamic_real=True,
    use_feat_static_cat=True,
    cardinality=[int(cat_feat_info.cardinality) for cat_feat_info in dataset.metadata.feat_static_cat],
    embedding_dimension = [4, 4, 4, 4, 16],
    prediction_length=dataset.metadata.prediction_length,
    context_length=dataset.metadata.prediction_length*2,
    freq=dataset.metadata.freq,
    scaling=True,
    trainer=Trainer(device=device,
                    epochs=25,
                    learning_rate=1e-3,
                    num_batches_per_epoch=120,
                    batch_size=256,
                   )
)

In [22]:

predictor = estimator.train(dataset.train, num_workers=8, shuffle_buffer_length=512)

119it [00:07, 16.35it/s, avg_epoch_loss=1.31, epoch=0]
119it [00:07, 16.06it/s, avg_epoch_loss=1.23, epoch=1]
119it [00:07, 16.48it/s, avg_epoch_loss=1.2, epoch=2] 
119it [00:07, 16.67it/s, avg_epoch_loss=1.19, epoch=3]
119it [00:07, 16.24it/s, avg_epoch_loss=1.19, epoch=4]
119it [00:07, 16.17it/s, avg_epoch_loss=1.19, epoch=5]
119it [00:07, 16.75it/s, avg_epoch_loss=1.18, epoch=6]
119it [00:07, 16.94it/s, avg_epoch_loss=1.18, epoch=7]
119it [00:07, 16.29it/s, avg_epoch_loss=1.18, epoch=8]
119it [00:07, 15.81it/s, avg_epoch_loss=1.18, epoch=9]
119it [00:07, 16.00it/s, avg_epoch_loss=1.17, epoch=10]
119it [00:07, 15.83it/s, avg_epoch_loss=1.17, epoch=11]
119it [00:07, 16.10it/s, avg_epoch_loss=1.17, epoch=12]
119it [00:07, 16.05it/s, avg_epoch_loss=1.17, epoch=13]
119it [00:07, 16.64it/s, avg_epoch_loss=1.17, epoch=14]
119it [00:07, 16.47it/s, avg_epoch_loss=1.17, epoch=15]
119it [00:07, 16.29it/s, avg_epoch_loss=1.16, epoch=16]
119it [00:07, 16.56it/s, avg_epoch_loss=1.16, epoch=17]
119it [00:07, 16.01it/s, avg_epoch_loss=1.16, epoch=18]
119it [00:07, 15.97it/s, avg_epoch_loss=1.16, epoch=19]
119it [00:07, 16.25it/s, avg_epoch_loss=1.16, epoch=20]
119it [00:07, 16.52it/s, avg_epoch_loss=1.16, epoch=21]
119it [00:07, 15.89it/s, avg_epoch_loss=1.16, epoch=22]
119it [00:07, 15.66it/s, avg_epoch_loss=1.16, epoch=23]
119it [00:07, 15.78it/s, avg_epoch_loss=1.15, epoch=24]

In [23]:

forecast_it, ts_it = make_evaluation_predictions(
    dataset=dataset.test,  # test dataset
    predictor=predictor,  # predictor
    num_samples=100,  # number of sample paths we want for evaluation
)

In [24]:

forecasts = list(forecast_it)
tss = list(ts_it)

In [25]:

evaluator = Evaluator()
agg_metrics, item_metrics = evaluator(iter(tss), iter(forecasts), num_series=len(dataset.test))

Running evaluation: 100%|██████████| 30490/30490 [00:00<00:00, 72825.91it/s]

In [26]:

print(json.dumps(agg_metrics, indent=4))

{
    "MSE": 4.9831022580643065,
    "abs_error": 837355.0,
    "abs_target_sum": 1231764.0,
    "abs_target_mean": 1.4428196598416343,
    "seasonal_error": 1.1272178349378457,
    "MASE": 0.9153679082981667,
    "MAPE": 0.7800716220446275,
    "sMAPE": 1.5507582107846607,
    "OWA": NaN,
    "MSIS": 8.01609371056141,
    "QuantileLoss[0.1]": 232214.80000000002,
    "Coverage[0.1]": 0.006127301691421073,
    "QuantileLoss[0.2]": 432821.2,
    "Coverage[0.2]": 0.019133908072904462,
    "QuantileLoss[0.3]": 603271.6,
    "Coverage[0.3]": 0.04252096706180011,
    "QuantileLoss[0.4]": 739940.8,
    "Coverage[0.4]": 0.0828257508316544,
    "QuantileLoss[0.5]": 837355.0,
    "Coverage[0.5]": 0.14760460104015366,
    "QuantileLoss[0.6]": 889230.8,
    "Coverage[0.6]": 0.2332122944290868,
    "QuantileLoss[0.7]": 880199.2,
    "Coverage[0.7]": 0.3683186524855924,
    "QuantileLoss[0.8]": 790916.0000000001,
    "Coverage[0.8]": 0.553399240968936,
    "QuantileLoss[0.9]": 578007.2,
    "Coverage[0.9]": 0.771742491683456,
    "RMSE": 2.232286329766929,
    "NRMSE": 1.547169332314163,
    "ND": 0.6798014879473665,
    "wQuantileLoss[0.1]": 0.1885221519706697,
    "wQuantileLoss[0.2]": 0.35138321951282875,
    "wQuantileLoss[0.3]": 0.4897623246011411,
    "wQuantileLoss[0.4]": 0.6007163709931448,
    "wQuantileLoss[0.5]": 0.6798014879473665,
    "wQuantileLoss[0.6]": 0.7219165359598105,
    "wQuantileLoss[0.7]": 0.7145842872498303,
    "wQuantileLoss[0.8]": 0.6421002724547885,
    "wQuantileLoss[0.9]": 0.46925157741255624,
    "mean_absolute_QuantileLoss": 664884.0666666668,
    "mean_wQuantileLoss": 0.5397820253446818,
    "MAE_Coverage": 0.2527905324149995
}

In [27]:

item_metrics.plot(x='MSIS', y='MASE', kind='scatter')
plt.grid(which="both")
plt.show()

In [ ]: