pytorch-transformer-ts/switch/switch.ipynb

In [1]:

%matplotlib inline
from matplotlib import pyplot as plt
import matplotlib.dates as mdates

from itertools import islice

In [2]:

from gluonts.evaluation import make_evaluation_predictions, Evaluator
from gluonts.dataset.repository.datasets import get_dataset

from estimator import SwitchTransformerEstimator

WARNING:root:Pytorch pre-release version 1.13.0a0+git9e80661 - assuming intent to test it
/home/kashif/.env/pytorch/lib/python3.8/site-packages/horovod/common/util.py:258: UserWarning: Framework pytorch installed with version 1.12.0a0+git689df63 but found version 1.13.0a0+git9e80661.
             This can result in unexpected behavior including runtime errors.
             Reinstall Horovod using `pip install --no-cache-dir` to build with the new version.
  warnings.warn(get_version_mismatch_message(name, version, installed_version))

In [3]:

dataset = get_dataset("electricity")

In [4]:

estimator = SwitchTransformerEstimator(
    freq=dataset.metadata.freq,
    prediction_length=dataset.metadata.prediction_length,
    num_feat_static_cat=1,
    cardinality=[321],
    embedding_dimension=[3],
    
    dim_feedforward=16,
    num_encoder_layers=2,
    num_decoder_layers=2,
    nhead=2,
    n_experts = 4,
    capacity_factor = 0.2,
    
    activation="relu",

    batch_size=128,
    num_batches_per_epoch=100,
    trainer_kwargs=dict(max_epochs=50, accelerator='gpu', gpus=1),
)

In [5]:

predictor = estimator.train(
    training_data=dataset.train,
    num_workers=8,
    shuffle_buffer_length=1024
)

/home/kashif/.env/pytorch/lib/python3.8/site-packages/pytorch_lightning/utilities/parsing.py:261: UserWarning: Attribute 'model' is an instance of `nn.Module` and is already saved during checkpointing. It is recommended to ignore them using `self.save_hyperparameters(ignore=['model'])`.
  rank_zero_warn(
/home/kashif/.env/pytorch/lib/python3.8/site-packages/pytorch_lightning/utilities/parsing.py:261: UserWarning: Attribute 'loss' is an instance of `nn.Module` and is already saved during checkpointing. It is recommended to ignore them using `self.save_hyperparameters(ignore=['loss'])`.
  rank_zero_warn(
GPU available: True, used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
/home/kashif/.env/pytorch/lib/python3.8/site-packages/pytorch_lightning/trainer/configuration_validator.py:133: UserWarning: You defined a `validation_step` but have no `val_dataloader`. Skipping val loop.
  rank_zero_warn("You defined a `validation_step` but have no `val_dataloader`. Skipping val loop.")
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

  | Name  | Type                   | Params
-------------------------------------------------
0 | model | SwitchTransformerModel | 75.5 K
1 | loss  | NegativeLogLikelihood  | 0     
-------------------------------------------------
75.5 K    Trainable params
0         Non-trainable params
75.5 K    Total params
0.302     Total estimated model params size (MB)

Training: 0it [00:00, ?it/s]

/home/kashif/.env/pytorch/lib/python3.8/site-packages/torchmetrics/utilities/prints.py:36: UserWarning: Torchmetrics v0.9 introduced a new argument class property called `full_state_update` that has
                not been set for this class (_ResultMetric). The property determines if `update` by
                default needs access to the full metric state. If this is not the case, significant speedups can be
                achieved and we recommend setting this to `False`.
                We provide an checking function
                `from torchmetrics.utilities import check_forward_no_full_state`
                that can be used to check if the `full_state_update=True` (old and potential slower behaviour,
                default for now) or if `full_state_update=False` can be used safely.
                
  warnings.warn(*args, **kwargs)
Epoch 0, global step 100: 'train_loss' reached 6.53932 (best 6.53932), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/switch/lightning_logs/version_1/checkpoints/epoch=0-step=100.ckpt' as top 1
Epoch 1, global step 200: 'train_loss' reached 6.06170 (best 6.06170), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/switch/lightning_logs/version_1/checkpoints/epoch=1-step=200.ckpt' as top 1
Epoch 2, global step 300: 'train_loss' reached 5.85441 (best 5.85441), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/switch/lightning_logs/version_1/checkpoints/epoch=2-step=300.ckpt' as top 1
Epoch 3, global step 400: 'train_loss' reached 5.71498 (best 5.71498), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/switch/lightning_logs/version_1/checkpoints/epoch=3-step=400.ckpt' as top 1
Epoch 4, global step 500: 'train_loss' reached 5.63095 (best 5.63095), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/switch/lightning_logs/version_1/checkpoints/epoch=4-step=500.ckpt' as top 1
Epoch 5, global step 600: 'train_loss' reached 5.59845 (best 5.59845), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/switch/lightning_logs/version_1/checkpoints/epoch=5-step=600.ckpt' as top 1
Epoch 6, global step 700: 'train_loss' reached 5.50794 (best 5.50794), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/switch/lightning_logs/version_1/checkpoints/epoch=6-step=700.ckpt' as top 1
Epoch 7, global step 800: 'train_loss' reached 5.50530 (best 5.50530), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/switch/lightning_logs/version_1/checkpoints/epoch=7-step=800.ckpt' as top 1
Epoch 8, global step 900: 'train_loss' reached 5.46209 (best 5.46209), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/switch/lightning_logs/version_1/checkpoints/epoch=8-step=900.ckpt' as top 1
Epoch 9, global step 1000: 'train_loss' reached 5.44430 (best 5.44430), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/switch/lightning_logs/version_1/checkpoints/epoch=9-step=1000.ckpt' as top 1
Epoch 10, global step 1100: 'train_loss' reached 5.44166 (best 5.44166), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/switch/lightning_logs/version_1/checkpoints/epoch=10-step=1100.ckpt' as top 1
Epoch 11, global step 1200: 'train_loss' reached 5.36817 (best 5.36817), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/switch/lightning_logs/version_1/checkpoints/epoch=11-step=1200.ckpt' as top 1
Epoch 12, global step 1300: 'train_loss' was not in top 1
Epoch 13, global step 1400: 'train_loss' was not in top 1
Epoch 14, global step 1500: 'train_loss' reached 5.32576 (best 5.32576), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/switch/lightning_logs/version_1/checkpoints/epoch=14-step=1500.ckpt' as top 1
Epoch 15, global step 1600: 'train_loss' reached 5.29555 (best 5.29555), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/switch/lightning_logs/version_1/checkpoints/epoch=15-step=1600.ckpt' as top 1
Epoch 16, global step 1700: 'train_loss' was not in top 1
Epoch 17, global step 1800: 'train_loss' reached 5.29322 (best 5.29322), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/switch/lightning_logs/version_1/checkpoints/epoch=17-step=1800.ckpt' as top 1
Epoch 18, global step 1900: 'train_loss' was not in top 1
Epoch 19, global step 2000: 'train_loss' reached 5.26062 (best 5.26062), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/switch/lightning_logs/version_1/checkpoints/epoch=19-step=2000.ckpt' as top 1
Epoch 20, global step 2100: 'train_loss' was not in top 1
Epoch 21, global step 2200: 'train_loss' reached 5.25029 (best 5.25029), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/switch/lightning_logs/version_1/checkpoints/epoch=21-step=2200.ckpt' as top 1
Epoch 22, global step 2300: 'train_loss' reached 5.23372 (best 5.23372), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/switch/lightning_logs/version_1/checkpoints/epoch=22-step=2300.ckpt' as top 1
Epoch 23, global step 2400: 'train_loss' was not in top 1
Epoch 24, global step 2500: 'train_loss' reached 5.21814 (best 5.21814), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/switch/lightning_logs/version_1/checkpoints/epoch=24-step=2500.ckpt' as top 1
Epoch 25, global step 2600: 'train_loss' was not in top 1
Epoch 26, global step 2700: 'train_loss' reached 5.21784 (best 5.21784), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/switch/lightning_logs/version_1/checkpoints/epoch=26-step=2700.ckpt' as top 1
Epoch 27, global step 2800: 'train_loss' reached 5.18084 (best 5.18084), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/switch/lightning_logs/version_1/checkpoints/epoch=27-step=2800.ckpt' as top 1
Epoch 28, global step 2900: 'train_loss' was not in top 1
Epoch 29, global step 3000: 'train_loss' was not in top 1
Epoch 30, global step 3100: 'train_loss' was not in top 1
Epoch 31, global step 3200: 'train_loss' was not in top 1
Epoch 32, global step 3300: 'train_loss' reached 5.16294 (best 5.16294), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/switch/lightning_logs/version_1/checkpoints/epoch=32-step=3300.ckpt' as top 1
Epoch 33, global step 3400: 'train_loss' was not in top 1
Epoch 34, global step 3500: 'train_loss' was not in top 1
Epoch 35, global step 3600: 'train_loss' was not in top 1
Epoch 36, global step 3700: 'train_loss' was not in top 1
Epoch 37, global step 3800: 'train_loss' was not in top 1
Epoch 38, global step 3900: 'train_loss' reached 5.16191 (best 5.16191), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/switch/lightning_logs/version_1/checkpoints/epoch=38-step=3900.ckpt' as top 1
Epoch 39, global step 4000: 'train_loss' was not in top 1
Epoch 40, global step 4100: 'train_loss' was not in top 1
Epoch 41, global step 4200: 'train_loss' reached 5.12905 (best 5.12905), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/switch/lightning_logs/version_1/checkpoints/epoch=41-step=4200.ckpt' as top 1
Epoch 42, global step 4300: 'train_loss' was not in top 1
Epoch 43, global step 4400: 'train_loss' reached 5.11128 (best 5.11128), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/switch/lightning_logs/version_1/checkpoints/epoch=43-step=4400.ckpt' as top 1
Epoch 44, global step 4500: 'train_loss' was not in top 1
Epoch 45, global step 4600: 'train_loss' was not in top 1
Epoch 46, global step 4700: 'train_loss' was not in top 1
Epoch 47, global step 4800: 'train_loss' was not in top 1
Epoch 48, global step 4900: 'train_loss' was not in top 1
Epoch 49, global step 5000: 'train_loss' was not in top 1

In [6]:

forecast_it, ts_it = make_evaluation_predictions(
    dataset=dataset.test, 
    predictor=predictor
)

In [7]:

forecasts = list(forecast_it)

In [8]:

tss = list(ts_it)

In [9]:

evaluator = Evaluator()

In [10]:

agg_metrics, ts_metrics = evaluator(iter(tss), iter(forecasts))

Running evaluation: 2247it [00:00, 5481.81it/s]
/home/kashif/.env/pytorch/lib/python3.8/site-packages/pandas/core/construction.py:781: UserWarning: Warning: converting a masked element to nan.
  subarr = np.array(arr, dtype=dtype, copy=copy)

In [11]:

agg_metrics

Out [11]:

{'MSE': 2308789.695387119,
 'abs_error': 9913947.80090332,
 'abs_target_sum': 128632956.0,
 'abs_target_mean': 2385.272140631954,
 'seasonal_error': 189.49338196116761,
 'MASE': 0.782350009562839,
 'MAPE': 0.10093158438717875,
 'sMAPE': 0.11210628837831137,
 'MSIS': 6.093644166414652,
 'QuantileLoss[0.1]': 4243983.490897928,
 'Coverage[0.1]': 0.12959872422489244,
 'QuantileLoss[0.2]': 6717838.2415222265,
 'Coverage[0.2]': 0.24310191366266132,
 'QuantileLoss[0.3]': 8384472.076653192,
 'Coverage[0.3]': 0.3441625871532414,
 'QuantileLoss[0.4]': 9423126.726327974,
 'Coverage[0.4]': 0.4417742174751521,
 'QuantileLoss[0.5]': 9913947.8229811,
 'Coverage[0.5]': 0.5353619641002819,
 'QuantileLoss[0.6]': 9806506.761595396,
 'Coverage[0.6]': 0.6134661029520843,
 'QuantileLoss[0.7]': 9047604.118984528,
 'Coverage[0.7]': 0.6995067497403945,
 'QuantileLoss[0.8]': 7602965.167928445,
 'Coverage[0.8]': 0.7845460614152203,
 'QuantileLoss[0.9]': 5178201.723310344,
 'Coverage[0.9]': 0.8760384215991693,
 'RMSE': 1519.4702022044128,
 'NRMSE': 0.637021736983791,
 'ND': 0.07707160053838241,
 'wQuantileLoss[0.1]': 0.03299297180807948,
 'wQuantileLoss[0.2]': 0.05222486095648946,
 'wQuantileLoss[0.3]': 0.06518136826967727,
 'wQuantileLoss[0.4]': 0.07325592926845259,
 'wQuantileLoss[0.5]': 0.07707160071001634,
 'wQuantileLoss[0.6]': 0.07623634771788496,
 'wQuantileLoss[0.7]': 0.07033659491572694,
 'wQuantileLoss[0.8]': 0.05910588860236132,
 'wQuantileLoss[0.9]': 0.0402556380909908,
 'mean_absolute_QuantileLoss': 7813182.903355681,
 'mean_wQuantileLoss': 0.06074013337107546,
 'MAE_Coverage': 0.02748603075705881,
 'OWA': nan}

In [14]:

plt.figure(figsize=(20, 15))
date_formater = mdates.DateFormatter('%b, %d')
plt.rcParams.update({'font.size': 15})

for idx, (forecast, ts) in islice(enumerate(zip(forecasts, tss)), 9):
    ax = plt.subplot(3, 3, idx+1)

    ts[-4 * dataset.metadata.prediction_length:].plot(ax=ax, label="target",)
    forecast.plot( color='g')
    plt.xticks(rotation=60)
    plt.title(forecast.item_id)
    ax.xaxis.set_major_formatter(date_formater)

plt.gcf().tight_layout()
plt.legend()
plt.show()

In [ ]: