mirror of
https://github.com/wassname/pytorch-transformer-ts.git
synced 2026-08-07 11:27:21 +08:00
392 KiB
392 KiB
In [1]:
%matplotlib inline
from matplotlib import pyplot as plt
import matplotlib.dates as mdates
from itertools import isliceIn [2]:
from gluonts.evaluation import make_evaluation_predictions, Evaluator
from gluonts.dataset.repository.datasets import get_dataset
from estimator import TFTEstimator/home/kashif/.env/pytorch/lib/python3.8/site-packages/xgboost/compat.py:36: FutureWarning: pandas.Int64Index is deprecated and will be removed from pandas in a future version. Use pandas.Index with the appropriate dtype instead. from pandas import MultiIndex, Int64Index
In [3]:
dataset = get_dataset("electricity")In [14]:
estimator = TFTEstimator(
freq=dataset.metadata.freq,
prediction_length=dataset.metadata.prediction_length,
num_feat_static_cat=1,
cardinality=[321],
batch_size=128,
num_batches_per_epoch=100,
trainer_kwargs=dict(max_epochs=20, accelerator='gpu', gpus=1),
)In [ ]:
predictor = estimator.train(
training_data=dataset.train,
num_workers=16,
shuffle_buffer_length=1024
)GPU available: True, used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
/home/kashif/.env/pytorch/lib/python3.8/site-packages/pytorch_lightning/trainer/configuration_validator.py:122: UserWarning: You defined a `validation_step` but have no `val_dataloader`. Skipping val loop.
rank_zero_warn("You defined a `validation_step` but have no `val_dataloader`. Skipping val loop.")
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
self._freq_base = start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
self._freq_base = start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
self._freq_base = start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
self._freq_base = start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
self._freq_base = start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
self._freq_base = start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
self._freq_base = start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
self._freq_base = start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
self._freq_base = start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
self._freq_base = start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
return _shift_timestamp_helper(ts, ts.freq, offset)
/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
return _shift_timestamp_helper(ts, ts.freq, offset)
/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
return _shift_timestamp_helper(ts, ts.freq, offset)
/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
return _shift_timestamp_helper(ts, ts.freq, offset)
/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
return _shift_timestamp_helper(ts, ts.freq, offset)
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
self._freq_base = start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
return _shift_timestamp_helper(ts, ts.freq, offset)
/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
return _shift_timestamp_helper(ts, ts.freq, offset)
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
self._freq_base = start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
return _shift_timestamp_helper(ts, ts.freq, offset)
/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
return _shift_timestamp_helper(ts, ts.freq, offset)
/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
return _shift_timestamp_helper(ts, ts.freq, offset)
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
self._freq_base = start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
return _shift_timestamp_helper(ts, ts.freq, offset)
/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
return _shift_timestamp_helper(ts, ts.freq, offset)
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
..., i0 : i0 + length * start.freq.n : start.freq.n
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
..., i0 : i0 + length * start.freq.n : start.freq.n
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
self._freq_base = start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
..., i0 : i0 + length * start.freq.n : start.freq.n
/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
return _shift_timestamp_helper(ts, ts.freq, offset)
/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
return _shift_timestamp_helper(ts, ts.freq, offset)
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
..., i0 : i0 + length * start.freq.n : start.freq.n
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
..., i0 : i0 + length * start.freq.n : start.freq.n
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
self._freq_base is None or self._freq_base == start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
..., i0 : i0 + length * start.freq.n : start.freq.n
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
..., i0 : i0 + length * start.freq.n : start.freq.n
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
..., i0 : i0 + length * start.freq.n : start.freq.n
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
self._freq_base is None or self._freq_base == start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
self._freq_base is None or self._freq_base == start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.
self._freq_base is None or self._freq_base == start.freq.base
/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. ..., i0 : i0 + length * start.freq.n : start.freq.n /home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. self._freq_base is None or self._freq_base == start.freq.base /home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. ..., i0 : i0 + length * start.freq.n : start.freq.n /home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. ..., i0 : i0 + length * start.freq.n : start.freq.n /home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. ..., i0 : i0 + length * start.freq.n : start.freq.n /home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. self._freq_base is None or self._freq_base == start.freq.base /home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. ..., i0 : i0 + length * start.freq.n : start.freq.n /home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. self._freq_base is None or self._freq_base == start.freq.base /home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. ..., i0 : i0 + length * start.freq.n : start.freq.n /home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. self._freq_base is None or self._freq_base == start.freq.base /home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. self._freq_base is None or self._freq_base == start.freq.base /home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. self._freq_base = start.freq.base /home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. self._freq_base is None or self._freq_base == start.freq.base /home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. self._freq_base is None or self._freq_base == start.freq.base /home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. self._freq_base is None or self._freq_base == start.freq.base /home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. self._freq_base is None or self._freq_base == start.freq.base /home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. return _shift_timestamp_helper(ts, ts.freq, offset) /home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. self._freq_base is None or self._freq_base == start.freq.base /home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. self._freq_base = start.freq.base /home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. return _shift_timestamp_helper(ts, ts.freq, offset) /home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. ..., i0 : i0 + length * start.freq.n : start.freq.n /home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. ..., i0 : i0 + length * start.freq.n : start.freq.n /home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. self._freq_base is None or self._freq_base == start.freq.base /home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. self._freq_base is None or self._freq_base == start.freq.base | Name | Type | Params ------------------------------------------------ 0 | model | TFTModel | 104 K 1 | loss | NegativeLogLikelihood | 0 ------------------------------------------------ 104 K Trainable params 0 Non-trainable params 104 K Total params 0.417 Total estimated model params size (MB)
Training: 0it [00:00, ?it/s]
In [6]:
forecast_it, ts_it = make_evaluation_predictions(
dataset=dataset.test,
predictor=predictor
)In [7]:
forecasts = list(forecast_it)/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:324: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version. timestamp = pd.Timestamp(timestamp_input, freq=freq) /home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:327: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. if isinstance(timestamp.freq, Tick): /home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:329: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. timestamp.floor(timestamp.freq), timestamp.freq /home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:328: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version. return pd.Timestamp( /home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. self._freq_base = start.freq.base /home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. return _shift_timestamp_helper(ts, ts.freq, offset) /home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. ..., i0 : i0 + length * start.freq.n : start.freq.n /home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. self._freq_base is None or self._freq_base == start.freq.base
In [8]:
tss = list(ts_it)In [9]:
evaluator = Evaluator()In [10]:
agg_metrics, ts_metrics = evaluator(iter(tss), iter(forecasts))Running evaluation: 2247it [00:00, 3696.80it/s]/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. date_before_forecast = forecast.index[0] - forecast.index[0].freq /home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. date_before_forecast = forecast.index[0] - forecast.index[0].freq /home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. date_before_forecast = forecast.index[0] - forecast.index[0].freq /home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. date_before_forecast = forecast.index[0] - forecast.index[0].freq /home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. date_before_forecast = forecast.index[0] - forecast.index[0].freq /home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. date_before_forecast = forecast.index[0] - forecast.index[0].freq /home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. date_before_forecast = forecast.index[0] - forecast.index[0].freq /home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. date_before_forecast = forecast.index[0] - forecast.index[0].freq /home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. date_before_forecast = forecast.index[0] - forecast.index[0].freq /home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. date_before_forecast = forecast.index[0] - forecast.index[0].freq /home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. date_before_forecast = forecast.index[0] - forecast.index[0].freq /home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. date_before_forecast = forecast.index[0] - forecast.index[0].freq /home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. date_before_forecast = forecast.index[0] - forecast.index[0].freq /home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. date_before_forecast = forecast.index[0] - forecast.index[0].freq /home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. date_before_forecast = forecast.index[0] - forecast.index[0].freq /home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version. date_before_forecast = forecast.index[0] - forecast.index[0].freq /home/kashif/.env/pytorch/lib/python3.8/site-packages/pandas/core/construction.py:781: UserWarning: Warning: converting a masked element to nan. subarr = np.array(arr, dtype=dtype, copy=copy)
In [12]:
agg_metricsOut [12]:
{'MSE': 2268670.185238203,
'abs_error': 12053610.225944996,
'abs_target_sum': 128632956.0,
'abs_target_mean': 2385.272140631954,
'seasonal_error': 189.49338196116761,
'MASE': 1.115612490152452,
'MAPE': 0.15282696886510425,
'sMAPE': 0.15643239726127742,
'MSIS': 8.31069379824802,
'QuantileLoss[0.1]': 6807481.895313849,
'Coverage[0.1]': 0.318331849873906,
'QuantileLoss[0.2]': 9639934.200819714,
'Coverage[0.2]': 0.46717846016911435,
'QuantileLoss[0.3]': 11212656.535200916,
'Coverage[0.3]': 0.5766021361815754,
'QuantileLoss[0.4]': 11978553.100825835,
'Coverage[0.4]': 0.6592679127725857,
'QuantileLoss[0.5]': 12053610.195398472,
'Coverage[0.5]': 0.7272288977896454,
'QuantileLoss[0.6]': 11435207.664433796,
'Coverage[0.6]': 0.7807261533897047,
'QuantileLoss[0.7]': 10287538.289494911,
'Coverage[0.7]': 0.8345386441180834,
'QuantileLoss[0.8]': 8401266.733424887,
'Coverage[0.8]': 0.8838265835929388,
'QuantileLoss[0.9]': 5519886.29474154,
'Coverage[0.9]': 0.931538347426198,
'RMSE': 1506.210538151358,
'NRMSE': 0.6314627637215025,
'ND': 0.09370545932214289,
'wQuantileLoss[0.1]': 0.052921755878126975,
'wQuantileLoss[0.2]': 0.07494140304775172,
'wQuantileLoss[0.3]': 0.08716783695152676,
'wQuantileLoss[0.4]': 0.09312196091354563,
'wQuantileLoss[0.5]': 0.09370545908467245,
'wQuantileLoss[0.6]': 0.08889796223320715,
'wQuantileLoss[0.7]': 0.0799759144887793,
'wQuantileLoss[0.8]': 0.06531193089759119,
'wQuantileLoss[0.9]': 0.042911913605884484,
'mean_absolute_QuantileLoss': 9704014.989961546,
'mean_wQuantileLoss': 0.07543957078900951,
'MAE_Coverage': 0.18658210947930576,
'OWA': nan}In [13]:
plt.figure(figsize=(20, 15))
date_formater = mdates.DateFormatter('%b, %d')
plt.rcParams.update({'font.size': 15})
for idx, (forecast, ts) in islice(enumerate(zip(forecasts, tss)), 9):
ax = plt.subplot(3, 3, idx+1)
plt.plot(ts[-4 * dataset.metadata.prediction_length:], label="target", )
forecast.plot( color='g')
plt.xticks(rotation=60)
ax.xaxis.set_major_formatter(date_formater)
plt.gcf().tight_layout()
plt.legend()
plt.show()In [ ]: