wassname/pytorch-transformer-ts
1
0
Fork 0
mirror of https://github.com/wassname/pytorch-transformer-ts.git synced 2026-08-15 12:44:52 +08:00
Code Issues Packages Projects Releases Wiki Activity
Files
cfc8daa4fa0d136ee501f24ed6bd37136fd6d770
pytorch-transformer-ts/tft/tft.ipynb
T
Kashif Rasul 75ef832caa updated notebook
2022-03-31 12:17:27 +02:00

391 KiB
Raw Blame History

In [1]:
%matplotlib inline
from matplotlib import pyplot as plt
import matplotlib.dates as mdates

from itertools import islice
In [2]:
from gluonts.evaluation import make_evaluation_predictions, Evaluator
from gluonts.dataset.repository.datasets import get_dataset

from estimator import TFTEstimator
/home/kashif/.env/pytorch/lib/python3.8/site-packages/xgboost/compat.py:36: FutureWarning: pandas.Int64Index is deprecated and will be removed from pandas in a future version. Use pandas.Index with the appropriate dtype instead.
  from pandas import MultiIndex, Int64Index
In [3]:
dataset = get_dataset("electricity")
In [4]:
estimator = TFTEstimator(
    freq=dataset.metadata.freq,
    prediction_length=dataset.metadata.prediction_length,
    num_feat_static_cat=1,
    cardinality=[321],

    batch_size=128,
    num_batches_per_epoch=100,
    trainer_kwargs=dict(max_epochs=50, accelerator='gpu', gpus=1),
)
In [5]:
predictor = estimator.train(
    training_data=dataset.train,
    num_workers=8,
    shuffle_buffer_length=1024
)
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:324: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version.
  timestamp = pd.Timestamp(timestamp_input, freq=freq)
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:324: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version.
  timestamp = pd.Timestamp(timestamp_input, freq=freq)
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:324: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version.
  timestamp = pd.Timestamp(timestamp_input, freq=freq)
GPU available: True, used: True
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:327: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  if isinstance(timestamp.freq, Tick):
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:327: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  if isinstance(timestamp.freq, Tick):
TPU available: False, using: 0 TPU cores
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:327: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  if isinstance(timestamp.freq, Tick):
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:329: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  timestamp.floor(timestamp.freq), timestamp.freq
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:329: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  timestamp.floor(timestamp.freq), timestamp.freq
IPU available: False, using: 0 IPUs
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:329: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  timestamp.floor(timestamp.freq), timestamp.freq
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:328: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version.
  return pd.Timestamp(
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:328: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version.
  return pd.Timestamp(
/home/kashif/.env/pytorch/lib/python3.8/site-packages/pytorch_lightning/trainer/configuration_validator.py:122: UserWarning: You defined a `validation_step` but have no `val_dataloader`. Skipping val loop.
  rank_zero_warn("You defined a `validation_step` but have no `val_dataloader`. Skipping val loop.")
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:324: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version.
  timestamp = pd.Timestamp(timestamp_input, freq=freq)
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:328: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version.
  return pd.Timestamp(
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:324: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version.
  timestamp = pd.Timestamp(timestamp_input, freq=freq)
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:324: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version.
  timestamp = pd.Timestamp(timestamp_input, freq=freq)
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:327: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  if isinstance(timestamp.freq, Tick):
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:327: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  if isinstance(timestamp.freq, Tick):
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:327: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  if isinstance(timestamp.freq, Tick):
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:329: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  timestamp.floor(timestamp.freq), timestamp.freq
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base = start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base = start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base = start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:328: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version.
  return pd.Timestamp(
/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  return _shift_timestamp_helper(ts, ts.freq, offset)
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:329: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  timestamp.floor(timestamp.freq), timestamp.freq
/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  return _shift_timestamp_helper(ts, ts.freq, offset)
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:329: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  timestamp.floor(timestamp.freq), timestamp.freq
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:328: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version.
  return pd.Timestamp(
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base = start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:328: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version.
  return pd.Timestamp(
/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  return _shift_timestamp_helper(ts, ts.freq, offset)
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:324: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version.
  timestamp = pd.Timestamp(timestamp_input, freq=freq)
/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  return _shift_timestamp_helper(ts, ts.freq, offset)
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:324: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version.
  timestamp = pd.Timestamp(timestamp_input, freq=freq)
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:327: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  if isinstance(timestamp.freq, Tick):
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:329: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  timestamp.floor(timestamp.freq), timestamp.freq
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base = start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:328: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version.
  return pd.Timestamp(
/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  return _shift_timestamp_helper(ts, ts.freq, offset)
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base = start.freq.base

/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base = start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  return _shift_timestamp_helper(ts, ts.freq, offset)
/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  return _shift_timestamp_helper(ts, ts.freq, offset)
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:327: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  if isinstance(timestamp.freq, Tick):
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  ..., i0 : i0 + length * start.freq.n : start.freq.n
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:329: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  timestamp.floor(timestamp.freq), timestamp.freq
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  ..., i0 : i0 + length * start.freq.n : start.freq.n
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base is None or self._freq_base == start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:328: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version.
  return pd.Timestamp(
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  ..., i0 : i0 + length * start.freq.n : start.freq.n
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  ..., i0 : i0 + length * start.freq.n : start.freq.n
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  ..., i0 : i0 + length * start.freq.n : start.freq.n
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  ..., i0 : i0 + length * start.freq.n : start.freq.n
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base = start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  return _shift_timestamp_helper(ts, ts.freq, offset)
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base is None or self._freq_base == start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base is None or self._freq_base == start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base is None or self._freq_base == start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base is None or self._freq_base == start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base is None or self._freq_base == start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  ..., i0 : i0 + length * start.freq.n : start.freq.n
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base is None or self._freq_base == start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  ..., i0 : i0 + length * start.freq.n : start.freq.n
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base is None or self._freq_base == start.freq.base

  | Name  | Type                  | Params
------------------------------------------------
0 | model | TFTModel              | 104 K 
1 | loss  | NegativeLogLikelihood | 0     
------------------------------------------------
104 K     Trainable params
0         Non-trainable params
104 K     Total params
0.417     Total estimated model params size (MB)

Training: 0it [00:00, ?it/s]
Epoch 0, global step 99: train_loss reached 6.38547 (best 6.38547), saving model to "/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_38/checkpoints/epoch=0-step=99.ckpt" as top 1
Epoch 1, global step 199: train_loss reached 5.69424 (best 5.69424), saving model to "/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_38/checkpoints/epoch=1-step=199.ckpt" as top 1
Epoch 2, global step 299: train_loss reached 5.52260 (best 5.52260), saving model to "/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_38/checkpoints/epoch=2-step=299.ckpt" as top 1
Epoch 3, global step 399: train_loss reached 5.43439 (best 5.43439), saving model to "/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_38/checkpoints/epoch=3-step=399.ckpt" as top 1
Epoch 4, global step 499: train_loss reached 5.38715 (best 5.38715), saving model to "/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_38/checkpoints/epoch=4-step=499.ckpt" as top 1
Epoch 5, global step 599: train_loss reached 5.28788 (best 5.28788), saving model to "/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_38/checkpoints/epoch=5-step=599.ckpt" as top 1
Epoch 6, global step 699: train_loss was not in top 1
Epoch 7, global step 799: train_loss reached 5.27511 (best 5.27511), saving model to "/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_38/checkpoints/epoch=7-step=799.ckpt" as top 1
Epoch 8, global step 899: train_loss reached 5.23092 (best 5.23092), saving model to "/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_38/checkpoints/epoch=8-step=899.ckpt" as top 1
Epoch 9, global step 999: train_loss was not in top 1
Epoch 10, global step 1099: train_loss reached 5.21828 (best 5.21828), saving model to "/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_38/checkpoints/epoch=10-step=1099.ckpt" as top 1
Epoch 11, global step 1199: train_loss reached 5.18606 (best 5.18606), saving model to "/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_38/checkpoints/epoch=11-step=1199.ckpt" as top 1
Epoch 12, global step 1299: train_loss was not in top 1
Epoch 13, global step 1399: train_loss reached 5.17699 (best 5.17699), saving model to "/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_38/checkpoints/epoch=13-step=1399.ckpt" as top 1
Epoch 14, global step 1499: train_loss reached 5.17665 (best 5.17665), saving model to "/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_38/checkpoints/epoch=14-step=1499.ckpt" as top 1
Epoch 15, global step 1599: train_loss reached 5.15192 (best 5.15192), saving model to "/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_38/checkpoints/epoch=15-step=1599.ckpt" as top 1
Epoch 16, global step 1699: train_loss reached 5.13906 (best 5.13906), saving model to "/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_38/checkpoints/epoch=16-step=1699.ckpt" as top 1
Epoch 17, global step 1799: train_loss was not in top 1
Epoch 18, global step 1899: train_loss reached 5.13224 (best 5.13224), saving model to "/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_38/checkpoints/epoch=18-step=1899.ckpt" as top 1
Epoch 19, global step 1999: train_loss was not in top 1
Epoch 20, global step 2099: train_loss reached 5.09653 (best 5.09653), saving model to "/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_38/checkpoints/epoch=20-step=2099.ckpt" as top 1
Epoch 21, global step 2199: train_loss was not in top 1
Epoch 22, global step 2299: train_loss was not in top 1
Epoch 23, global step 2399: train_loss was not in top 1
Epoch 24, global step 2499: train_loss was not in top 1
Epoch 25, global step 2599: train_loss reached 5.09542 (best 5.09542), saving model to "/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_38/checkpoints/epoch=25-step=2599.ckpt" as top 1
Epoch 26, global step 2699: train_loss reached 5.07425 (best 5.07425), saving model to "/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_38/checkpoints/epoch=26-step=2699.ckpt" as top 1
Epoch 27, global step 2799: train_loss was not in top 1
Epoch 28, global step 2899: train_loss was not in top 1
Epoch 29, global step 2999: train_loss was not in top 1
Epoch 30, global step 3099: train_loss was not in top 1
Epoch 31, global step 3199: train_loss was not in top 1
Epoch 32, global step 3299: train_loss reached 5.06753 (best 5.06753), saving model to "/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_38/checkpoints/epoch=32-step=3299.ckpt" as top 1
Epoch 33, global step 3399: train_loss was not in top 1
Epoch 34, global step 3499: train_loss was not in top 1
Epoch 35, global step 3599: train_loss reached 5.06716 (best 5.06716), saving model to "/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_38/checkpoints/epoch=35-step=3599.ckpt" as top 1
Epoch 36, global step 3699: train_loss reached 5.05435 (best 5.05435), saving model to "/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_38/checkpoints/epoch=36-step=3699.ckpt" as top 1
Epoch 37, global step 3799: train_loss was not in top 1
Epoch 38, global step 3899: train_loss reached 5.03846 (best 5.03846), saving model to "/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_38/checkpoints/epoch=38-step=3899.ckpt" as top 1
Epoch 39, global step 3999: train_loss was not in top 1
Epoch 40, global step 4099: train_loss was not in top 1
Epoch 41, global step 4199: train_loss was not in top 1
Epoch 42, global step 4299: train_loss reached 5.03356 (best 5.03356), saving model to "/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_38/checkpoints/epoch=42-step=4299.ckpt" as top 1
Epoch 43, global step 4399: train_loss was not in top 1
Epoch 44, global step 4499: train_loss was not in top 1
Epoch 45, global step 4599: train_loss was not in top 1
Epoch 46, global step 4699: train_loss reached 5.03011 (best 5.03011), saving model to "/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_38/checkpoints/epoch=46-step=4699.ckpt" as top 1
Epoch 47, global step 4799: train_loss was not in top 1
Epoch 48, global step 4899: train_loss was not in top 1
Epoch 49, global step 4999: train_loss reached 5.01233 (best 5.01233), saving model to "/mnt/scratch/kashif/pytorch-transformer-ts/tft/lightning_logs/version_38/checkpoints/epoch=49-step=4999.ckpt" as top 1
In [6]:
forecast_it, ts_it = make_evaluation_predictions(
    dataset=dataset.test, 
    predictor=predictor
)
In [7]:
forecasts = list(forecast_it)
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:324: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version.
  timestamp = pd.Timestamp(timestamp_input, freq=freq)
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:327: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  if isinstance(timestamp.freq, Tick):
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:329: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  timestamp.floor(timestamp.freq), timestamp.freq
/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:328: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version.
  return pd.Timestamp(
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base = start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  return _shift_timestamp_helper(ts, ts.freq, offset)
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  ..., i0 : i0 + length * start.freq.n : start.freq.n
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  self._freq_base is None or self._freq_base == start.freq.base
In [8]:
tss = list(ts_it)
In [9]:
evaluator = Evaluator()
In [10]:
agg_metrics, ts_metrics = evaluator(iter(tss), iter(forecasts))
Running evaluation: 2247it [00:00, 5399.22it/s]/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  date_before_forecast = forecast.index[0] - forecast.index[0].freq
/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  date_before_forecast = forecast.index[0] - forecast.index[0].freq
/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  date_before_forecast = forecast.index[0] - forecast.index[0].freq
/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  date_before_forecast = forecast.index[0] - forecast.index[0].freq
/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  date_before_forecast = forecast.index[0] - forecast.index[0].freq
/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  date_before_forecast = forecast.index[0] - forecast.index[0].freq
/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  date_before_forecast = forecast.index[0] - forecast.index[0].freq
/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  date_before_forecast = forecast.index[0] - forecast.index[0].freq
/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  date_before_forecast = forecast.index[0] - forecast.index[0].freq
/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  date_before_forecast = forecast.index[0] - forecast.index[0].freq
/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  date_before_forecast = forecast.index[0] - forecast.index[0].freq

/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  date_before_forecast = forecast.index[0] - forecast.index[0].freq
/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  date_before_forecast = forecast.index[0] - forecast.index[0].freq
/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  date_before_forecast = forecast.index[0] - forecast.index[0].freq
/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  date_before_forecast = forecast.index[0] - forecast.index[0].freq
/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
  date_before_forecast = forecast.index[0] - forecast.index[0].freq
/home/kashif/.env/pytorch/lib/python3.8/site-packages/pandas/core/construction.py:781: UserWarning: Warning: converting a masked element to nan.
  subarr = np.array(arr, dtype=dtype, copy=copy)
In [11]:
agg_metrics
Out [11]:
{'MSE': 2288155.4595733206,
 'abs_error': 8677549.890345097,
 'abs_target_sum': 128632956.0,
 'abs_target_mean': 2385.272140631954,
 'seasonal_error': 189.49338196116761,
 'MASE': 0.7392301619000542,
 'MAPE': 0.0942166421951502,
 'sMAPE': 0.10727499540855988,
 'MSIS': 5.90130373159719,
 'QuantileLoss[0.1]': 3652603.570823352,
 'Coverage[0.1]': 0.09373609256786827,
 'QuantileLoss[0.2]': 5729977.868776519,
 'Coverage[0.2]': 0.21504598724224894,
 'QuantileLoss[0.3]': 7170568.846287252,
 'Coverage[0.3]': 0.342215546654799,
 'QuantileLoss[0.4]': 8147483.8875474585,
 'Coverage[0.4]': 0.46387776294318356,
 'QuantileLoss[0.5]': 8677549.828375284,
 'Coverage[0.5]': 0.5856697819314641,
 'QuantileLoss[0.6]': 8664060.21840468,
 'Coverage[0.6]': 0.6839860554813825,
 'QuantileLoss[0.7]': 8038668.19911083,
 'Coverage[0.7]': 0.7789645453196856,
 'QuantileLoss[0.8]': 6666073.66160029,
 'Coverage[0.8]': 0.8620753597389113,
 'QuantileLoss[0.9]': 4274899.861139778,
 'Coverage[0.9]': 0.9331886960391633,
 'RMSE': 1512.6650189560544,
 'NRMSE': 0.634168736216107,
 'ND': 0.06745977205363372,
 'wQuantileLoss[0.1]': 0.028395550288243022,
 'wQuantileLoss[0.2]': 0.04454517758867734,
 'wQuantileLoss[0.3]': 0.055744414722827734,
 'wQuantileLoss[0.4]': 0.06333900845400349,
 'wQuantileLoss[0.5]': 0.06745977157187684,
 'wQuantileLoss[0.6]': 0.06735490256792886,
 'wQuantileLoss[0.7]': 0.062493069032098045,
 'wQuantileLoss[0.8]': 0.0518224401342397,
 'wQuantileLoss[0.9]': 0.033233317448910824,
 'mean_absolute_QuantileLoss': 6780209.549118383,
 'mean_wQuantileLoss': 0.05270973908986732,
 'MAE_Coverage': 0.05236529364255222,
 'OWA': nan}
In [12]:
plt.figure(figsize=(20, 15))
date_formater = mdates.DateFormatter('%b, %d')
plt.rcParams.update({'font.size': 15})

for idx, (forecast, ts) in islice(enumerate(zip(forecasts, tss)), 9):
    ax = plt.subplot(3, 3, idx+1)

    plt.plot(ts[-4 * dataset.metadata.prediction_length:], label="target", )
    forecast.plot( color='g')
    plt.xticks(rotation=60)
    ax.xaxis.set_major_formatter(date_formater)

plt.gcf().tight_layout()
plt.legend()
plt.show()
In [ ]:
Reference in New Issue View Git Blame Copy Permalink