pytorch-transformer-ts/tft/tft.ipynb

In [1]:

%matplotlib inline
from matplotlib import pyplot as plt
import matplotlib.dates as mdates

from itertools import islice

In [2]:

from gluonts.evaluation import make_evaluation_predictions, Evaluator
from gluonts.dataset.repository.datasets import get_dataset

from estimator import TFTEstimator

/home/kashif/.env/pytorch/lib/python3.8/site-packages/xgboost/compat.py:36: FutureWarning: pandas.Int64Index is deprecated and will be removed from pandas in a future version. Use pandas.Index with the appropriate dtype instead.
  from pandas import MultiIndex, Int64Index

In [3]:

dataset = get_dataset("traffic")

In [5]:

dataset

Out [5]:

TrainDatasets(metadata=MetaData(freq='H', target=None, feat_static_cat=[CategoricalFeatureInfo(name='feat_static_cat', cardinality='862')], feat_static_real=[], feat_dynamic_real=[], feat_dynamic_cat=[], prediction_length=24), train=<gluonts.dataset.common.FileDataset object at 0x7fe8c7dafc40>, test=<gluonts.dataset.common.FileDataset object at 0x7fe8c7dafe20>)

In [16]:

estimator = TFTEstimator(
    freq=dataset.metadata.freq,
    prediction_length=dataset.metadata.prediction_length,
    num_feat_static_cat=1,
    cardinality=[862],
    
    scaling=False,

    batch_size=128,
    num_batches_per_epoch=100,
    trainer_kwargs=dict(max_epochs=50, accelerator='gpu', gpus=1),
)

In [17]:

predictor = estimator.train(
    training_data=dataset.train,
    num_workers=8,
    shuffle_buffer_length=1024
)

/home/kashif/.env/pytorch/lib/python3.8/site-packages/pytorch_lightning/utilities/parsing.py:244: UserWarning: Attribute 'model' is an instance of `nn.Module` and is already saved during checkpointing. It is recommended to ignore them using `self.save_hyperparameters(ignore=['model'])`.
  rank_zero_warn(
/home/kashif/.env/pytorch/lib/python3.8/site-packages/pytorch_lightning/utilities/parsing.py:244: UserWarning: Attribute 'loss' is an instance of `nn.Module` and is already saved during checkpointing. It is recommended to ignore them using `self.save_hyperparameters(ignore=['loss'])`.
  rank_zero_warn(
GPU available: True, used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
/home/kashif/.env/pytorch/lib/python3.8/site-packages/pytorch_lightning/trainer/configuration_validator.py:133: UserWarning: You defined a `validation_step` but have no `val_dataloader`. Skipping val loop.
  rank_zero_warn("You defined a `validation_step` but have no `val_dataloader`. Skipping val loop.")
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

  | Name  | Type                  | Params
------------------------------------------------
0 | model | TFTModel              | 121 K 
1 | loss  | NegativeLogLikelihood | 0     
------------------------------------------------
121 K     Trainable params
0         Non-trainable params
121 K     Total params
0.487     Total estimated model params size (MB)
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base = start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  return _shift_timestamp_helper(ts, ts.freq, offset)
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base = start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  return _shift_timestamp_helper(ts, ts.freq, offset)
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base = start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  return _shift_timestamp_helper(ts, ts.freq, offset)
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base = start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base = start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  ..., i0 : i0 + length * start.freq.n : start.freq.n
/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  return _shift_timestamp_helper(ts, ts.freq, offset)
/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  return _shift_timestamp_helper(ts, ts.freq, offset)
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  ..., i0 : i0 + length * start.freq.n : start.freq.n
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base = start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  ..., i0 : i0 + length * start.freq.n : start.freq.n
/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  return _shift_timestamp_helper(ts, ts.freq, offset)
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base is None or self._freq_base == start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base is None or self._freq_base == start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base is None or self._freq_base == start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  ..., i0 : i0 + length * start.freq.n : start.freq.n
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  ..., i0 : i0 + length * start.freq.n : start.freq.n
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base = start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  ..., i0 : i0 + length * start.freq.n : start.freq.n
/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  return _shift_timestamp_helper(ts, ts.freq, offset)
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base is None or self._freq_base == start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base is None or self._freq_base == start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base is None or self._freq_base == start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  ..., i0 : i0 + length * start.freq.n : start.freq.n
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base is None or self._freq_base == start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base = start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  return _shift_timestamp_helper(ts, ts.freq, offset)
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  ..., i0 : i0 + length * start.freq.n : start.freq.n
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base is None or self._freq_base == start.freq.base

Training: 0it [00:00, ?it/s]

Epoch 0, global step 100: 'train_loss' reached -1.76367 (best -1.76367), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_40/checkpoints/epoch=0-step=100.ckpt' as top 1
Epoch 1, global step 200: 'train_loss' reached -2.67806 (best -2.67806), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_40/checkpoints/epoch=1-step=200.ckpt' as top 1
Epoch 2, global step 300: 'train_loss' reached -2.82263 (best -2.82263), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_40/checkpoints/epoch=2-step=300.ckpt' as top 1
Epoch 3, global step 400: 'train_loss' reached -2.91125 (best -2.91125), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_40/checkpoints/epoch=3-step=400.ckpt' as top 1
Epoch 4, global step 500: 'train_loss' reached -3.00680 (best -3.00680), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_40/checkpoints/epoch=4-step=500.ckpt' as top 1
Epoch 5, global step 600: 'train_loss' reached -3.03799 (best -3.03799), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_40/checkpoints/epoch=5-step=600.ckpt' as top 1
Epoch 6, global step 700: 'train_loss' reached -3.08277 (best -3.08277), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_40/checkpoints/epoch=6-step=700.ckpt' as top 1
Epoch 7, global step 800: 'train_loss' reached -3.11023 (best -3.11023), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_40/checkpoints/epoch=7-step=800.ckpt' as top 1
Epoch 8, global step 900: 'train_loss' reached -3.15504 (best -3.15504), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_40/checkpoints/epoch=8-step=900.ckpt' as top 1
Epoch 9, global step 1000: 'train_loss' reached -3.19840 (best -3.19840), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_40/checkpoints/epoch=9-step=1000.ckpt' as top 1
Epoch 10, global step 1100: 'train_loss' reached -3.23382 (best -3.23382), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_40/checkpoints/epoch=10-step=1100.ckpt' as top 1
Epoch 11, global step 1200: 'train_loss' reached -3.26279 (best -3.26279), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_40/checkpoints/epoch=11-step=1200.ckpt' as top 1
Epoch 12, global step 1300: 'train_loss' reached -3.29968 (best -3.29968), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_40/checkpoints/epoch=12-step=1300.ckpt' as top 1
Epoch 13, global step 1400: 'train_loss' was not in top 1
Epoch 14, global step 1500: 'train_loss' reached -3.37084 (best -3.37084), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_40/checkpoints/epoch=14-step=1500.ckpt' as top 1
Epoch 15, global step 1600: 'train_loss' was not in top 1
Epoch 16, global step 1700: 'train_loss' was not in top 1
Epoch 17, global step 1800: 'train_loss' reached -3.45659 (best -3.45659), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_40/checkpoints/epoch=17-step=1800.ckpt' as top 1
Epoch 18, global step 1900: 'train_loss' reached -3.47860 (best -3.47860), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_40/checkpoints/epoch=18-step=1900.ckpt' as top 1
Epoch 19, global step 2000: 'train_loss' reached -3.48826 (best -3.48826), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_40/checkpoints/epoch=19-step=2000.ckpt' as top 1
Epoch 20, global step 2100: 'train_loss' reached -3.51585 (best -3.51585), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_40/checkpoints/epoch=20-step=2100.ckpt' as top 1
Epoch 21, global step 2200: 'train_loss' reached -3.53465 (best -3.53465), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_40/checkpoints/epoch=21-step=2200.ckpt' as top 1
Epoch 22, global step 2300: 'train_loss' was not in top 1
Epoch 23, global step 2400: 'train_loss' was not in top 1
Epoch 24, global step 2500: 'train_loss' was not in top 1
Epoch 25, global step 2600: 'train_loss' reached -3.57969 (best -3.57969), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_40/checkpoints/epoch=25-step=2600.ckpt' as top 1
Epoch 26, global step 2700: 'train_loss' was not in top 1
Epoch 27, global step 2800: 'train_loss' reached -3.61853 (best -3.61853), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_40/checkpoints/epoch=27-step=2800.ckpt' as top 1
Epoch 28, global step 2900: 'train_loss' was not in top 1
Epoch 29, global step 3000: 'train_loss' was not in top 1
Epoch 30, global step 3100: 'train_loss' reached -3.63641 (best -3.63641), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_40/checkpoints/epoch=30-step=3100.ckpt' as top 1
Epoch 31, global step 3200: 'train_loss' was not in top 1
Epoch 32, global step 3300: 'train_loss' reached -3.65598 (best -3.65598), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_40/checkpoints/epoch=32-step=3300.ckpt' as top 1
Epoch 33, global step 3400: 'train_loss' was not in top 1
Epoch 34, global step 3500: 'train_loss' was not in top 1
Epoch 35, global step 3600: 'train_loss' reached -3.66342 (best -3.66342), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_40/checkpoints/epoch=35-step=3600.ckpt' as top 1
Epoch 36, global step 3700: 'train_loss' was not in top 1
Epoch 37, global step 3800: 'train_loss' reached -3.67218 (best -3.67218), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_40/checkpoints/epoch=37-step=3800.ckpt' as top 1
Epoch 38, global step 3900: 'train_loss' was not in top 1
Epoch 39, global step 4000: 'train_loss' reached -3.69459 (best -3.69459), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_40/checkpoints/epoch=39-step=4000.ckpt' as top 1
Epoch 40, global step 4100: 'train_loss' reached -3.69594 (best -3.69594), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_40/checkpoints/epoch=40-step=4100.ckpt' as top 1
Epoch 41, global step 4200: 'train_loss' reached -3.70282 (best -3.70282), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_40/checkpoints/epoch=41-step=4200.ckpt' as top 1
Epoch 42, global step 4300: 'train_loss' was not in top 1
Epoch 43, global step 4400: 'train_loss' reached -3.71828 (best -3.71828), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_40/checkpoints/epoch=43-step=4400.ckpt' as top 1
Epoch 44, global step 4500: 'train_loss' reached -3.71923 (best -3.71923), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_40/checkpoints/epoch=44-step=4500.ckpt' as top 1
Epoch 45, global step 4600: 'train_loss' was not in top 1
Epoch 46, global step 4700: 'train_loss' reached -3.72760 (best -3.72760), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_40/checkpoints/epoch=46-step=4700.ckpt' as top 1
Epoch 47, global step 4800: 'train_loss' reached -3.73313 (best -3.73313), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_40/checkpoints/epoch=47-step=4800.ckpt' as top 1
Epoch 48, global step 4900: 'train_loss' was not in top 1
Epoch 49, global step 5000: 'train_loss' was not in top 1

In [18]:

forecast_it, ts_it = make_evaluation_predictions(
    dataset=dataset.test, 
    predictor=predictor
)

In [19]:

forecasts = list(forecast_it)

/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base = start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  return _shift_timestamp_helper(ts, ts.freq, offset)
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  ..., i0 : i0 + length * start.freq.n : start.freq.n
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base is None or self._freq_base == start.freq.base

In [10]:

tss = list(ts_it)

In [20]:

evaluator = Evaluator()

In [21]:

agg_metrics, ts_metrics = evaluator(iter(tss), iter(forecasts))

Running evaluation: 6034it [00:00, 10921.04it/s]/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  date_before_forecast = forecast.index[0] - forecast.index[0].freq
/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  date_before_forecast = forecast.index[0] - forecast.index[0].freq
/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  date_before_forecast = forecast.index[0] - forecast.index[0].freq
/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  date_before_forecast = forecast.index[0] - forecast.index[0].freq
/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  date_before_forecast = forecast.index[0] - forecast.index[0].freq
/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  date_before_forecast = forecast.index[0] - forecast.index[0].freq
/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  date_before_forecast = forecast.index[0] - forecast.index[0].freq
/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  date_before_forecast = forecast.index[0] - forecast.index[0].freq
/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  date_before_forecast = forecast.index[0] - forecast.index[0].freq
/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  date_before_forecast = forecast.index[0] - forecast.index[0].freq
/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  date_before_forecast = forecast.index[0] - forecast.index[0].freq
/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  date_before_forecast = forecast.index[0] - forecast.index[0].freq
/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  date_before_forecast = forecast.index[0] - forecast.index[0].freq
/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  date_before_forecast = forecast.index[0] - forecast.index[0].freq

/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  date_before_forecast = forecast.index[0] - forecast.index[0].freq
/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  date_before_forecast = forecast.index[0] - forecast.index[0].freq
/home/kashif/.env/pytorch/lib/python3.8/site-packages/pandas/core/construction.py:781: UserWarning: Warning: converting a masked element to nan.
  subarr = np.array(arr, dtype=dtype, copy=copy)

In [22]:

agg_metrics

Out [22]:

{'MSE': 0.000649330164079505,
 'abs_error': 1432.974686332047,
 'abs_target_sum': 8672.5710073933,
 'abs_target_mean': 0.0598868288545002,
 'seasonal_error': 0.015220711169889631,
 'MASE': 0.648407474736938,
 'MAPE': 0.2660992923204324,
 'sMAPE': 0.2139038286564818,
 'MSIS': 7.822711779756906,
 'QuantileLoss[0.1]': 758.046953115314,
 'Coverage[0.1]': 0.035451883769749205,
 'QuantileLoss[0.2]': 1091.1472629882426,
 'Coverage[0.2]': 0.07841674953043862,
 'QuantileLoss[0.3]': 1288.6517700438721,
 'Coverage[0.3]': 0.1443072588664236,
 'QuantileLoss[0.4]': 1394.0904395095831,
 'Coverage[0.4]': 0.23985609324936472,
 'QuantileLoss[0.5]': 1432.9746841261403,
 'Coverage[0.5]': 0.366036901999779,
 'QuantileLoss[0.6]': 1426.1454076143127,
 'Coverage[0.6]': 0.49707905203844877,
 'QuantileLoss[0.7]': 1358.4131771848538,
 'Coverage[0.7]': 0.6520481162302508,
 'QuantileLoss[0.8]': 1218.5885981488536,
 'Coverage[0.8]': 0.8036611976577173,
 'QuantileLoss[0.9]': 944.6292773525245,
 'Coverage[0.9]': 0.9196428571428571,
 'RMSE': 0.02548195761866629,
 'NRMSE': 0.4255018692102854,
 'ND': 0.16523066632841021,
 'wQuantileLoss[0.1]': 0.08740740807646137,
 'wQuantileLoss[0.2]': 0.12581589266413015,
 'wQuantileLoss[0.3]': 0.14858935936590273,
 'wQuantileLoss[0.4]': 0.16074707699955776,
 'wQuantileLoss[0.5]': 0.16523066607405582,
 'wQuantileLoss[0.6]': 0.16444320910126128,
 'wQuantileLoss[0.7]': 0.1566332724202335,
 'wQuantileLoss[0.8]': 0.14051065100649118,
 'wQuantileLoss[0.9]': 0.10892148090194191,
 'mean_absolute_QuantileLoss': 1212.5208411204105,
 'mean_wQuantileLoss': 0.1398110018455595,
 'MAE_Coverage': 0.09001199990179105,
 'OWA': nan}

In [13]:

agg_metrics

Out [13]:

{'MSE': 0.0008602817230717296,
 'abs_error': 1617.4472325854003,
 'abs_target_sum': 8672.5710073933,
 'abs_target_mean': 0.0598868288545002,
 'seasonal_error': 0.015220711169889631,
 'MASE': 0.7225524827690298,
 'MAPE': 0.21059109221974529,
 'sMAPE': 0.17548061777599777,
 'MSIS': 7.981380574607891,
 'QuantileLoss[0.1]': 798.1957122146287,
 'Coverage[0.1]': 0.05049856369461938,
 'QuantileLoss[0.2]': 1163.7939581929065,
 'Coverage[0.2]': 0.1206703679151475,
 'QuantileLoss[0.3]': 1394.4469597546472,
 'Coverage[0.3]': 0.19998480830847423,
 'QuantileLoss[0.4]': 1541.6169008201734,
 'Coverage[0.4]': 0.2855209369130483,
 'QuantileLoss[0.5]': 1617.447230679832,
 'Coverage[0.5]': 0.37727184841453987,
 'QuantileLoss[0.6]': 1641.2588065576676,
 'Coverage[0.6]': 0.46351922439509446,
 'QuantileLoss[0.7]': 1565.8052165220274,
 'Coverage[0.7]': 0.5644956358413434,
 'QuantileLoss[0.8]': 1386.698439198728,
 'Coverage[0.8]': 0.6784195116561705,
 'QuantileLoss[0.9]': 1048.4756686581238,
 'Coverage[0.9]': 0.8173613412882555,
 'RMSE': 0.029330559542424854,
 'NRMSE': 0.489766449542449,
 'ND': 0.1865014689653782,
 'wQuantileLoss[0.1]': 0.09203680333480958,
 'wQuantileLoss[0.2]': 0.13419249691940038,
 'wQuantileLoss[0.3]': 0.16078818594461686,
 'wQuantileLoss[0.4]': 0.17775777211924315,
 'wQuantileLoss[0.5]': 0.18650146874565462,
 'wQuantileLoss[0.6]': 0.18924708776192287,
 'wQuantileLoss[0.7]': 0.18054683152057105,
 'wQuantileLoss[0.8]': 0.15989473456217057,
 'wQuantileLoss[0.9]': 0.12089559921323288,
 'mean_absolute_QuantileLoss': 1350.859876955415,
 'mean_wQuantileLoss': 0.15576233112462465,
 'MAE_Coverage': 0.10469530684147853,
 'OWA': nan}

In [23]:

plt.figure(figsize=(20, 15))
date_formater = mdates.DateFormatter('%b, %d')
plt.rcParams.update({'font.size': 15})

for idx, (forecast, ts) in islice(enumerate(zip(forecasts, tss)), 9):
    ax = plt.subplot(3, 3, idx+1)

    plt.plot(ts[-4 * dataset.metadata.prediction_length:], label="target", )
    forecast.plot( color='g')
    plt.xticks(rotation=60)
    plt.title(forecast.item_id)
    ax.xaxis.set_major_formatter(date_formater)

plt.gcf().tight_layout()
plt.legend()
plt.show()

In [ ]: