pytorch-transformer-ts/perceiverar/perceiverar.ipynb

In [1]:

%matplotlib inline
from matplotlib import pyplot as plt
import matplotlib.dates as mdates

from itertools import islice

In [2]:

from gluonts.evaluation import make_evaluation_predictions, Evaluator
from gluonts.dataset.repository.datasets import get_dataset

from estimator import PerceiverAREstimator

In [3]:

dataset = get_dataset("electricity")

In [13]:

estimator = PerceiverAREstimator(
    depth=2,
    heads=1,
    freq=dataset.metadata.freq,
    prediction_length=dataset.metadata.prediction_length,
    context_length=dataset.metadata.prediction_length*10,
    num_feat_static_cat=1,
    cardinality=[321],
    embedding_dimension=[3],

    batch_size=128,
    num_batches_per_epoch=100,
    trainer_kwargs=dict(max_epochs=25, accelerator='cpu'),
)

In [14]:

predictor = estimator.train(
    training_data=dataset.train,
    num_workers=8,
    shuffle_buffer_length=1024
)

/opt/homebrew/lib/python3.9/site-packages/pytorch_lightning/utilities/parsing.py:261: UserWarning: Attribute 'model' is an instance of `nn.Module` and is already saved during checkpointing. It is recommended to ignore them using `self.save_hyperparameters(ignore=['model'])`.
  rank_zero_warn(
GPU available: True (mps), used: False
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
/opt/homebrew/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py:1788: UserWarning: MPS available but not used. Set `accelerator` and `devices` using `Trainer(accelerator='mps', devices=1)`.
  rank_zero_warn(
/opt/homebrew/lib/python3.9/site-packages/pytorch_lightning/trainer/configuration_validator.py:117: PossibleUserWarning: You defined a `validation_step` but have no `val_dataloader`. Skipping val loop.
  rank_zero_warn(

  | Name  | Type             | Params
-------------------------------------------
0 | model | PerceiverARModel | 36.1 K
-------------------------------------------
36.1 K    Trainable params
0         Non-trainable params
36.1 K    Total params
0.144     Total estimated model params size (MB)

Training: 0it [00:00, ?it/s]

Epoch 0, global step 100: 'train_loss' reached 6.42776 (best 6.42776), saving model to '/Users/kashif/Documents/GitHub/pytorch-transformer-ts/perceiverar/lightning_logs/version_22/checkpoints/epoch=0-step=100.ckpt' as top 1
Epoch 1, global step 200: 'train_loss' reached 5.74860 (best 5.74860), saving model to '/Users/kashif/Documents/GitHub/pytorch-transformer-ts/perceiverar/lightning_logs/version_22/checkpoints/epoch=1-step=200.ckpt' as top 1
Epoch 2, global step 300: 'train_loss' reached 5.53532 (best 5.53532), saving model to '/Users/kashif/Documents/GitHub/pytorch-transformer-ts/perceiverar/lightning_logs/version_22/checkpoints/epoch=2-step=300.ckpt' as top 1
Epoch 3, global step 400: 'train_loss' was not in top 1
Epoch 4, global step 500: 'train_loss' reached 5.51218 (best 5.51218), saving model to '/Users/kashif/Documents/GitHub/pytorch-transformer-ts/perceiverar/lightning_logs/version_22/checkpoints/epoch=4-step=500.ckpt' as top 1
Epoch 5, global step 600: 'train_loss' reached 5.48945 (best 5.48945), saving model to '/Users/kashif/Documents/GitHub/pytorch-transformer-ts/perceiverar/lightning_logs/version_22/checkpoints/epoch=5-step=600.ckpt' as top 1
Epoch 6, global step 700: 'train_loss' reached 5.46006 (best 5.46006), saving model to '/Users/kashif/Documents/GitHub/pytorch-transformer-ts/perceiverar/lightning_logs/version_22/checkpoints/epoch=6-step=700.ckpt' as top 1
Epoch 7, global step 800: 'train_loss' was not in top 1
Epoch 8, global step 900: 'train_loss' reached 5.45789 (best 5.45789), saving model to '/Users/kashif/Documents/GitHub/pytorch-transformer-ts/perceiverar/lightning_logs/version_22/checkpoints/epoch=8-step=900.ckpt' as top 1
Epoch 9, global step 1000: 'train_loss' reached 5.43128 (best 5.43128), saving model to '/Users/kashif/Documents/GitHub/pytorch-transformer-ts/perceiverar/lightning_logs/version_22/checkpoints/epoch=9-step=1000.ckpt' as top 1
Epoch 10, global step 1100: 'train_loss' reached 5.42391 (best 5.42391), saving model to '/Users/kashif/Documents/GitHub/pytorch-transformer-ts/perceiverar/lightning_logs/version_22/checkpoints/epoch=10-step=1100.ckpt' as top 1
Epoch 11, global step 1200: 'train_loss' reached 5.37426 (best 5.37426), saving model to '/Users/kashif/Documents/GitHub/pytorch-transformer-ts/perceiverar/lightning_logs/version_22/checkpoints/epoch=11-step=1200.ckpt' as top 1
Epoch 12, global step 1300: 'train_loss' was not in top 1
Epoch 13, global step 1400: 'train_loss' reached 5.34587 (best 5.34587), saving model to '/Users/kashif/Documents/GitHub/pytorch-transformer-ts/perceiverar/lightning_logs/version_22/checkpoints/epoch=13-step=1400.ckpt' as top 1
Epoch 14, global step 1500: 'train_loss' reached 5.34270 (best 5.34270), saving model to '/Users/kashif/Documents/GitHub/pytorch-transformer-ts/perceiverar/lightning_logs/version_22/checkpoints/epoch=14-step=1500.ckpt' as top 1
Epoch 15, global step 1600: 'train_loss' was not in top 1
Epoch 16, global step 1700: 'train_loss' reached 5.33624 (best 5.33624), saving model to '/Users/kashif/Documents/GitHub/pytorch-transformer-ts/perceiverar/lightning_logs/version_22/checkpoints/epoch=16-step=1700.ckpt' as top 1
Epoch 17, global step 1800: 'train_loss' reached 5.32782 (best 5.32782), saving model to '/Users/kashif/Documents/GitHub/pytorch-transformer-ts/perceiverar/lightning_logs/version_22/checkpoints/epoch=17-step=1800.ckpt' as top 1
Epoch 18, global step 1900: 'train_loss' was not in top 1
Epoch 19, global step 2000: 'train_loss' reached 5.28012 (best 5.28012), saving model to '/Users/kashif/Documents/GitHub/pytorch-transformer-ts/perceiverar/lightning_logs/version_22/checkpoints/epoch=19-step=2000.ckpt' as top 1
Epoch 20, global step 2100: 'train_loss' was not in top 1
Epoch 21, global step 2200: 'train_loss' was not in top 1
Epoch 22, global step 2300: 'train_loss' was not in top 1
Epoch 23, global step 2400: 'train_loss' was not in top 1
Epoch 24, global step 2500: 'train_loss' reached 5.27051 (best 5.27051), saving model to '/Users/kashif/Documents/GitHub/pytorch-transformer-ts/perceiverar/lightning_logs/version_22/checkpoints/epoch=24-step=2500.ckpt' as top 1
`Trainer.fit` stopped: `max_epochs=25` reached.

In [15]:

forecast_it, ts_it = make_evaluation_predictions(
    dataset=dataset.test,
    predictor=predictor
)

In [16]:

forecasts = list(forecast_it)

In [8]:

tss = list(ts_it)

In [17]:

evaluator = Evaluator()
agg_metrics, ts_metrics = evaluator(iter(tss), iter(forecasts), num_series=len(dataset.test))

Running evaluation: 100%|██████████████████████████████████████████████████████████████████████████████████████████| 2247/2247 [00:00<00:00, 25805.41it/s]
/Users/kashif/Github/gluon-ts/src/gluonts/evaluation/_base.py:361: RuntimeWarning: divide by zero encountered in float_scalars
  metrics["ND"] = cast(float, metrics["abs_error"]) / cast(
/opt/homebrew/lib/python3.9/site-packages/pandas/core/dtypes/cast.py:1181: UserWarning: Warning: converting a masked element to nan.
  return arr.astype(dtype, copy=True)

In [18]:

agg_metrics

Out [18]:

{'MSE': 54659388.626912884,
 'abs_error': 45449271.4559927,
 'abs_target_sum': 128632956.0,
 'abs_target_mean': 2385.272140631954,
 'seasonal_error': 189.49338196116761,
 'MASE': 3.402492133476373,
 'MAPE': 0.5195859642334041,
 'sMAPE': 0.344615968371725,
 'MSIS': 85.79713867275944,
 'QuantileLoss[0.1]': 59259097.99589533,
 'Coverage[0.1]': 0.49907283785788464,
 'QuantileLoss[0.2]': 57816227.81258023,
 'Coverage[0.2]': 0.5358070019284973,
 'QuantileLoss[0.3]': 54536714.14192347,
 'Coverage[0.3]': 0.5650126094051329,
 'QuantileLoss[0.4]': 50351281.862691015,
 'Coverage[0.4]': 0.5897307521139297,
 'QuantileLoss[0.5]': 45449271.678276934,
 'Coverage[0.5]': 0.6161363299213767,
 'QuantileLoss[0.6]': 39958118.98481089,
 'Coverage[0.6]': 0.6402425456163774,
 'QuantileLoss[0.7]': 33818848.94476284,
 'Coverage[0.7]': 0.6704124017208128,
 'QuantileLoss[0.8]': 26983896.70083867,
 'Coverage[0.8]': 0.7084260495475448,
 'QuantileLoss[0.9]': 18850785.382875618,
 'Coverage[0.9]': 0.7596981160065271,
 'RMSE': 7393.198808831863,
 'NRMSE': 3.099520043391405,
 'ND': 0.3533252509255303,
 'wQuantileLoss[0.1]': 0.4606836369048017,
 'wQuantileLoss[0.2]': 0.44946668109360893,
 'wQuantileLoss[0.3]': 0.4239715531525488,
 'wQuantileLoss[0.4]': 0.3914337618323178,
 'wQuantileLoss[0.5]': 0.3533252526535807,
 'wQuantileLoss[0.6]': 0.3106367157170118,
 'wQuantileLoss[0.7]': 0.26290967724292086,
 'wQuantileLoss[0.8]': 0.20977436529437035,
 'wQuantileLoss[0.9]': 0.14654709002314786,
 'mean_absolute_QuantileLoss': 43002693.722739436,
 'mean_wQuantileLoss': 0.33430541487936766,
 'MAE_Coverage': 0.17860727884092376,
 'OWA': nan}

In [19]:

ts_metrics.plot(x='MSIS', y='MASE', kind='scatter')
plt.grid(which="both")
plt.show()

In [20]:

plt.figure(figsize=(20, 15))
date_formater = mdates.DateFormatter('%b, %d')
plt.rcParams.update({'font.size': 15})

for idx, (forecast, ts) in islice(enumerate(zip(forecasts, tss)), 9):
    ax = plt.subplot(3, 3, idx+1)

    plt.plot(ts[-4 * dataset.metadata.prediction_length:].to_timestamp(), label="target", )
    forecast.plot( color='g')
    plt.xticks(rotation=60)
    ax.xaxis.set_major_formatter(date_formater)

plt.gcf().tight_layout()
plt.legend()
plt.show()

In [ ]: