pytorch-ts/examples/Multivariate-Flow-Solar.ipynb

In [1]:

import numpy as np
import pandas as pd

import torch

In [2]:

from gluonts.dataset.multivariate_grouper import MultivariateGrouper
from gluonts.dataset.repository.datasets import dataset_recipes, get_dataset
from pts.model.tempflow import TempFlowEstimator
from pts.model.transformer_tempflow import TransformerTempFlowEstimator
from pts import Trainer
from gluonts.evaluation.backtest import make_evaluation_predictions
from gluonts.evaluation import MultivariateEvaluator

In [3]:

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

Prepeare data set

In [4]:

dataset = get_dataset("solar_nips", regenerate=False)

In [5]:

dataset.metadata

Out [5]:

MetaData(freq='H', target=None, feat_static_cat=[CategoricalFeatureInfo(name='feat_static_cat_0', cardinality='137')], feat_static_real=[], feat_dynamic_real=[], feat_dynamic_cat=[], prediction_length=24)

In [6]:

train_grouper = MultivariateGrouper(max_target_dim=int(dataset.metadata.feat_static_cat[0].cardinality))

test_grouper = MultivariateGrouper(num_test_dates=int(len(dataset.test)/len(dataset.train)), 
                                   max_target_dim=int(dataset.metadata.feat_static_cat[0].cardinality))

In [7]:

dataset_train = train_grouper(dataset.train)
dataset_test = test_grouper(dataset.test)

/home/wassname/miniforge3/envs/glounts/lib/python3.9/site-packages/gluonts/dataset/multivariate_grouper.py:191: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.
  return {FieldName.TARGET: np.array([funcs(data) for data in dataset])}

Evaluator

In [8]:

evaluator = MultivariateEvaluator(quantiles=(np.arange(20)/20.0)[1:],
                                  target_agg_funcs={'sum': np.sum})

GRU-Real-NVP

In [9]:

estimator = TempFlowEstimator(
    target_dim=int(dataset.metadata.feat_static_cat[0].cardinality),
    prediction_length=dataset.metadata.prediction_length,
    cell_type='GRU',
    input_size=552,
    freq=dataset.metadata.freq,
    scaling=True,
    dequantize=True,
    n_blocks=4,
    trainer=Trainer(device=device,
                    epochs=45,
                    learning_rate=1e-3,
                    num_batches_per_epoch=100,
                    batch_size=64)
)

In [10]:

predictor = estimator.train(dataset_train)
forecast_it, ts_it = make_evaluation_predictions(dataset=dataset_test,
                                             predictor=predictor,
                                             num_samples=100)
forecasts = list(forecast_it)
targets = list(ts_it)

agg_metric, _ = evaluator(targets, forecasts, num_series=len(dataset_test))

  0%|          | 0/99 [00:00<?, ?it/s]

---------------------------------------------------------------------------
Exception                                 Traceback (most recent call last)
/media/wassname/SGIronWolf/projects5/timeseries/pytorch-ts/examples/Multivariate-Flow-Solar.ipynb Cell 13 in <cell line: 1>()
----> <a href='vscode-notebook-cell:/media/wassname/SGIronWolf/projects5/timeseries/pytorch-ts/examples/Multivariate-Flow-Solar.ipynb#X15sZmlsZQ%3D%3D?line=0'>1</a> predictor = estimator.train(dataset_train)
      <a href='vscode-notebook-cell:/media/wassname/SGIronWolf/projects5/timeseries/pytorch-ts/examples/Multivariate-Flow-Solar.ipynb#X15sZmlsZQ%3D%3D?line=1'>2</a> forecast_it, ts_it = make_evaluation_predictions(dataset=dataset_test,
      <a href='vscode-notebook-cell:/media/wassname/SGIronWolf/projects5/timeseries/pytorch-ts/examples/Multivariate-Flow-Solar.ipynb#X15sZmlsZQ%3D%3D?line=2'>3</a>                                              predictor=predictor,
      <a href='vscode-notebook-cell:/media/wassname/SGIronWolf/projects5/timeseries/pytorch-ts/examples/Multivariate-Flow-Solar.ipynb#X15sZmlsZQ%3D%3D?line=3'>4</a>                                              num_samples=100)
      <a href='vscode-notebook-cell:/media/wassname/SGIronWolf/projects5/timeseries/pytorch-ts/examples/Multivariate-Flow-Solar.ipynb#X15sZmlsZQ%3D%3D?line=4'>5</a> forecasts = list(forecast_it)

File /media/wassname/SGIronWolf/projects5/timeseries/pytorch-ts/pts/model/estimator.py:179, in PyTorchEstimator.train(self, training_data, validation_data, num_workers, prefetch_factor, shuffle_buffer_length, cache_data, **kwargs)
    169 def train(
    170     self,
    171     training_data: Dataset,
   (...)
    177     **kwargs,
    178 ) -> PyTorchPredictor:
--> 179     return self.train_model(
    180         training_data,
    181         validation_data,
    182         num_workers=num_workers,
    183         prefetch_factor=prefetch_factor,
    184         shuffle_buffer_length=shuffle_buffer_length,
    185         cache_data=cache_data,
    186         **kwargs,
    187     ).predictor

File /media/wassname/SGIronWolf/projects5/timeseries/pytorch-ts/pts/model/estimator.py:151, in PyTorchEstimator.train_model(self, training_data, validation_data, num_workers, prefetch_factor, shuffle_buffer_length, cache_data, **kwargs)
    133     validation_iter_dataset = TransformedIterableDataset(
    134         dataset=validation_data,
    135         transform=transformation
   (...)
    139         cache_data=cache_data,
    140     )
    141     validation_data_loader = DataLoader(
    142         validation_iter_dataset,
    143         batch_size=self.trainer.batch_size,
   (...)
    148         **kwargs,
    149     )
--> 151 self.trainer(
    152     net=trained_net,
    153     train_iter=training_data_loader,
    154     validation_iter=validation_data_loader,
    155 )
    157 return TrainOutput(
    158     transformation=transformation,
    159     trained_net=trained_net,
   (...)
    162     ),
    163 )

File /media/wassname/SGIronWolf/projects5/timeseries/pytorch-ts/pts/trainer.py:63, in Trainer.__call__(self, net, train_iter, validation_iter)
     61 # training loop
     62 with tqdm(train_iter, total=total) as it:
---> 63     for batch_no, data_entry in enumerate(it, start=1):
     64         optimizer.zero_grad()
     66         inputs = [v.to(self.device) for v in data_entry.values()]

File ~/miniforge3/envs/glounts/lib/python3.9/site-packages/tqdm/notebook.py:259, in tqdm_notebook.__iter__(self)
    257 try:
    258     it = super(tqdm_notebook, self).__iter__()
--> 259     for obj in it:
    260         # return super(tqdm...) will not catch exception
    261         yield obj
    262 # NB: except ... [ as ...] breaks IPython async KeyboardInterrupt

File ~/miniforge3/envs/glounts/lib/python3.9/site-packages/tqdm/std.py:1195, in tqdm.__iter__(self)
   1192 time = self._time
   1194 try:
-> 1195     for obj in iterable:
   1196         yield obj
   1197         # Update and possibly print the progressbar.
   1198         # Note: does not call self.update(1) for speed optimisation.

File ~/miniforge3/envs/glounts/lib/python3.9/site-packages/torch/utils/data/dataloader.py:628, in _BaseDataLoaderIter.__next__(self)
    625 if self._sampler_iter is None:
    626     # TODO(https://github.com/pytorch/pytorch/issues/76750)
    627     self._reset()  # type: ignore[call-arg]
--> 628 data = self._next_data()
    629 self._num_yielded += 1
    630 if self._dataset_kind == _DatasetKind.Iterable and \
    631         self._IterableDataset_len_called is not None and \
    632         self._num_yielded > self._IterableDataset_len_called:

File ~/miniforge3/envs/glounts/lib/python3.9/site-packages/torch/utils/data/dataloader.py:671, in _SingleProcessDataLoaderIter._next_data(self)
    669 def _next_data(self):
    670     index = self._next_index()  # may raise StopIteration
--> 671     data = self._dataset_fetcher.fetch(index)  # may raise StopIteration
    672     if self._pin_memory:
    673         data = _utils.pin_memory.pin_memory(data, self._pin_memory_device)

File ~/miniforge3/envs/glounts/lib/python3.9/site-packages/torch/utils/data/_utils/fetch.py:34, in _IterableDatasetFetcher.fetch(self, possibly_batched_index)
     32 for _ in possibly_batched_index:
     33     try:
---> 34         data.append(next(self.dataset_iter))
     35     except StopIteration:
     36         self.ended = True

File ~/miniforge3/envs/glounts/lib/python3.9/site-packages/gluonts/transform/_base.py:103, in TransformedDataset.__iter__(self)
    102 def __iter__(self) -> Iterator[DataEntry]:
--> 103     yield from self.transformation(
    104         self.base_dataset, is_train=self.is_train
    105     )

File ~/miniforge3/envs/glounts/lib/python3.9/site-packages/gluonts/transform/_base.py:124, in MapTransformation.__call__(self, data_it, is_train)
    121 def __call__(
    122     self, data_it: Iterable[DataEntry], is_train: bool
    123 ) -> Iterator:
--> 124     for data_entry in data_it:
    125         try:
    126             yield self.map_transform(data_entry.copy(), is_train)

File ~/miniforge3/envs/glounts/lib/python3.9/site-packages/gluonts/transform/_base.py:124, in MapTransformation.__call__(self, data_it, is_train)
    121 def __call__(
    122     self, data_it: Iterable[DataEntry], is_train: bool
    123 ) -> Iterator:
--> 124     for data_entry in data_it:
    125         try:
    126             yield self.map_transform(data_entry.copy(), is_train)

File ~/miniforge3/envs/glounts/lib/python3.9/site-packages/gluonts/transform/_base.py:189, in FlatMapTransformation.__call__(self, data_it, is_train)
    182     yield result
    184 if (
    185     # negative values disable the check
    186     self.max_idle_transforms > 0
    187     and num_idle_transforms > self.max_idle_transforms
    188 ):
--> 189     raise Exception(
    190         "Reached maximum number of idle transformation"
    191         " calls.\nThis means the transformation looped over"
    192         f" {self.max_idle_transforms} inputs without returning any"
    193         " output.\nThis occurred in the following"
    194         f" transformation:\n{self}"
    195     )

Exception: Reached maximum number of idle transformation calls.
This means the transformation looped over 1 inputs without returning any output.
This occurred in the following transformation:
gluonts.transform.split.InstanceSplitter(dummy_value=0.0, forecast_start_field="forecast_start", future_length=24, instance_sampler=gluonts.transform.sampler.ExpectedNumInstanceSampler(axis=-1, min_past=192, min_future=24, num_instances=1.0, total_length=20382, n=3), is_pad_field="is_pad", lead_time=0, output_NTC=True, past_length=192, start_field="start", target_field="target", time_series_fields=["time_feat", "observed_values"])

Metrics

In [ ]:

print("CRPS: {}".format(agg_metric['mean_wQuantileLoss']))
print("ND: {}".format(agg_metric['ND']))
print("NRMSE: {}".format(agg_metric['NRMSE']))
print("MSE: {}".format(agg_metric['MSE']))

CRPS: 0.36531966950112466
ND: 0.45434020382814283
NRMSE: 0.9820216603495642
MSE: 914.7868680304274

In [ ]:

print("CRPS-Sum: {}".format(agg_metric['m_sum_mean_wQuantileLoss']))
print("ND-Sum: {}".format(agg_metric['m_sum_ND']))
print("NRMSE-Sum: {}".format(agg_metric['m_sum_NRMSE']))
print("MSE-Sum: {}".format(agg_metric['m_sum_MSE']))

CRPS-Sum: 0.2873863376280519
ND-Sum: 0.35970480888579265
NRMSE-Sum: 0.7184166842326591
MSE-Sum: 9189074.285714285

GRU-MAF

In [ ]:

estimator = TempFlowEstimator(
    target_dim=int(dataset.metadata.feat_static_cat[0].cardinality),
    prediction_length=dataset.metadata.prediction_length,
    cell_type='GRU',
    input_size=552,
    freq=dataset.metadata.freq,
    scaling=True,
    dequantize=True,
    flow_type='MAF',
    trainer=Trainer(device=device,
                    epochs=25,
                    learning_rate=1e-3,
                    num_batches_per_epoch=100,
                    batch_size=64)
)

In [ ]:

predictor = estimator.train(dataset_train)
forecast_it, ts_it = make_evaluation_predictions(dataset=dataset_test,
                                             predictor=predictor,
                                             num_samples=100)
forecasts = list(forecast_it)
targets = list(ts_it)

agg_metric, _ = evaluator(targets, forecasts, num_series=len(dataset_test))

98it [00:10,  9.05it/s, avg_epoch_loss=-7.36, epoch=0]
99it [00:10,  9.19it/s, avg_epoch_loss=-136, epoch=1]
99it [00:10,  9.12it/s, avg_epoch_loss=-164, epoch=2]
98it [00:10,  8.91it/s, avg_epoch_loss=-179, epoch=3]
98it [00:10,  9.09it/s, avg_epoch_loss=-188, epoch=4]
99it [00:10,  9.05it/s, avg_epoch_loss=-194, epoch=5]
98it [00:10,  9.04it/s, avg_epoch_loss=-198, epoch=6]
98it [00:10,  8.97it/s, avg_epoch_loss=-201, epoch=7]
97it [00:10,  8.90it/s, avg_epoch_loss=-204, epoch=8]
99it [00:10,  9.07it/s, avg_epoch_loss=-206, epoch=9]
99it [00:10,  9.09it/s, avg_epoch_loss=-207, epoch=10]
98it [00:11,  8.90it/s, avg_epoch_loss=-209, epoch=11]
99it [00:10,  9.02it/s, avg_epoch_loss=-210, epoch=12]
98it [00:10,  8.95it/s, avg_epoch_loss=-211, epoch=13]
99it [00:10,  9.21it/s, avg_epoch_loss=-212, epoch=14]
98it [00:10,  9.00it/s, avg_epoch_loss=-213, epoch=15]
99it [00:10,  9.21it/s, avg_epoch_loss=-214, epoch=16]
98it [00:10,  8.95it/s, avg_epoch_loss=-215, epoch=17]
98it [00:11,  8.88it/s, avg_epoch_loss=-216, epoch=18]
99it [00:10,  9.08it/s, avg_epoch_loss=-216, epoch=19]
98it [00:10,  8.96it/s, avg_epoch_loss=-217, epoch=20]
98it [00:10,  8.98it/s, avg_epoch_loss=-218, epoch=21]
97it [00:10,  8.88it/s, avg_epoch_loss=-218, epoch=22]
97it [00:10,  8.83it/s, avg_epoch_loss=-219, epoch=23]
98it [00:10,  8.97it/s, avg_epoch_loss=-219, epoch=24]
  0%|          | 0/137 [00:00<?, ?it/s]
Running evaluation: 7it [00:00, 79.30it/s]
  1%|          | 1/137 [00:00<00:14,  9.67it/s]
Running evaluation: 7it [00:00, 80.37it/s]
  1%|▏         | 2/137 [00:00<00:13,  9.69it/s]
Running evaluation: 7it [00:00, 80.99it/s]
  2%|▏         | 3/137 [00:00<00:13,  9.72it/s]
Running evaluation: 7it [00:00, 80.16it/s]
  3%|▎         | 4/137 [00:00<00:13,  9.74it/s]
Running evaluation: 7it [00:00, 81.87it/s]
  4%|▎         | 5/137 [00:00<00:15,  8.38it/s]
Running evaluation: 7it [00:00, 80.00it/s]
  4%|▍         | 6/137 [00:00<00:15,  8.73it/s]
Running evaluation: 7it [00:00, 80.20it/s]
  5%|▌         | 7/137 [00:00<00:14,  9.02it/s]
Running evaluation: 7it [00:00, 81.10it/s]
  6%|▌         | 8/137 [00:00<00:13,  9.26it/s]
Running evaluation: 7it [00:00, 78.81it/s]
  7%|▋         | 9/137 [00:00<00:13,  9.37it/s]
Running evaluation: 7it [00:00, 80.94it/s]
  7%|▋         | 10/137 [00:01<00:13,  9.51it/s]
Running evaluation: 7it [00:00, 80.86it/s]
  8%|▊         | 11/137 [00:01<00:13,  9.61it/s]
Running evaluation: 7it [00:00, 79.54it/s]
  9%|▉         | 12/137 [00:01<00:12,  9.65it/s]
Running evaluation: 7it [00:00, 80.68it/s]
  9%|▉         | 13/137 [00:01<00:12,  9.70it/s]
Running evaluation: 7it [00:00, 80.79it/s]
 10%|█         | 14/137 [00:01<00:12,  9.67it/s]
Running evaluation: 7it [00:00, 79.82it/s]
 11%|█         | 15/137 [00:01<00:12,  9.69it/s]
Running evaluation: 7it [00:00, 80.53it/s]
 12%|█▏        | 16/137 [00:01<00:12,  9.73it/s]
Running evaluation: 7it [00:00, 76.42it/s]
 12%|█▏        | 17/137 [00:01<00:12,  9.59it/s]
Running evaluation: 7it [00:00, 79.49it/s]
 13%|█▎        | 18/137 [00:01<00:12,  9.61it/s]
Running evaluation: 7it [00:00, 80.33it/s]
 14%|█▍        | 19/137 [00:02<00:12,  9.66it/s]
Running evaluation: 7it [00:00, 81.06it/s]
 15%|█▍        | 20/137 [00:02<00:12,  9.70it/s]
Running evaluation: 7it [00:00, 80.93it/s]
 15%|█▌        | 21/137 [00:02<00:11,  9.74it/s]
Running evaluation: 7it [00:00, 81.20it/s]
 16%|█▌        | 22/137 [00:02<00:11,  9.78it/s]
Running evaluation: 7it [00:00, 80.83it/s]
 17%|█▋        | 23/137 [00:02<00:11,  9.79it/s]
Running evaluation: 7it [00:00, 81.33it/s]
 18%|█▊        | 24/137 [00:02<00:11,  9.82it/s]
Running evaluation: 7it [00:00, 80.91it/s]
 18%|█▊        | 25/137 [00:02<00:11,  9.83it/s]
Running evaluation: 7it [00:00, 81.40it/s]
 19%|█▉        | 26/137 [00:02<00:11,  9.85it/s]
Running evaluation: 7it [00:00, 81.20it/s]
 20%|█▉        | 27/137 [00:02<00:11,  9.86it/s]
Running evaluation: 7it [00:00, 82.38it/s]
 20%|██        | 28/137 [00:02<00:11,  9.90it/s]
Running evaluation: 7it [00:00, 82.08it/s]
 21%|██        | 29/137 [00:03<00:10,  9.92it/s]
Running evaluation: 7it [00:00, 81.48it/s]
 22%|██▏       | 30/137 [00:03<00:10,  9.91it/s]
Running evaluation: 7it [00:00, 80.91it/s]
 23%|██▎       | 31/137 [00:03<00:10,  9.89it/s]
Running evaluation: 7it [00:00, 81.74it/s]
 23%|██▎       | 32/137 [00:03<00:10,  9.89it/s]
Running evaluation: 7it [00:00, 82.56it/s]
 24%|██▍       | 33/137 [00:03<00:10,  9.91it/s]
Running evaluation: 7it [00:00, 82.94it/s]

Running evaluation: 7it [00:00, 81.87it/s]
 26%|██▌       | 35/137 [00:03<00:10,  9.93it/s]
Running evaluation: 7it [00:00, 82.36it/s]

Running evaluation: 7it [00:00, 82.48it/s]
 27%|██▋       | 37/137 [00:03<00:10,  9.96it/s]
Running evaluation: 7it [00:00, 81.42it/s]
 28%|██▊       | 38/137 [00:03<00:09,  9.93it/s]
Running evaluation: 7it [00:00, 81.16it/s]
 28%|██▊       | 39/137 [00:04<00:09,  9.91it/s]
Running evaluation: 7it [00:00, 80.18it/s]
 29%|██▉       | 40/137 [00:04<00:09,  9.86it/s]
Running evaluation: 7it [00:00, 79.44it/s]
 30%|██▉       | 41/137 [00:04<00:09,  9.77it/s]
Running evaluation: 7it [00:00, 78.49it/s]
 31%|███       | 42/137 [00:04<00:09,  9.71it/s]
Running evaluation: 7it [00:00, 81.14it/s]
 31%|███▏      | 43/137 [00:04<00:09,  9.76it/s]
Running evaluation: 7it [00:00, 80.89it/s]
 32%|███▏      | 44/137 [00:04<00:09,  9.77it/s]
Running evaluation: 7it [00:00, 79.56it/s]
 33%|███▎      | 45/137 [00:04<00:09,  9.75it/s]
Running evaluation: 7it [00:00, 80.37it/s]
 34%|███▎      | 46/137 [00:04<00:09,  9.77it/s]
Running evaluation: 7it [00:00, 81.06it/s]
 34%|███▍      | 47/137 [00:04<00:09,  9.80it/s]
Running evaluation: 7it [00:00, 81.00it/s]
 35%|███▌      | 48/137 [00:04<00:09,  9.82it/s]
Running evaluation: 7it [00:00, 79.59it/s]
 36%|███▌      | 49/137 [00:05<00:08,  9.78it/s]
Running evaluation: 7it [00:00, 80.16it/s]
 36%|███▋      | 50/137 [00:05<00:08,  9.78it/s]
Running evaluation: 7it [00:00, 81.13it/s]
 37%|███▋      | 51/137 [00:05<00:08,  9.81it/s]
Running evaluation: 7it [00:00, 80.09it/s]
 38%|███▊      | 52/137 [00:05<00:08,  9.80it/s]
Running evaluation: 7it [00:00, 81.05it/s]
 39%|███▊      | 53/137 [00:05<00:08,  9.82it/s]
Running evaluation: 7it [00:00, 79.80it/s]
 39%|███▉      | 54/137 [00:05<00:08,  9.80it/s]
Running evaluation: 7it [00:00, 79.99it/s]
 40%|████      | 55/137 [00:05<00:08,  9.79it/s]
Running evaluation: 7it [00:00, 80.74it/s]
 41%|████      | 56/137 [00:05<00:08,  9.80it/s]
Running evaluation: 7it [00:00, 80.38it/s]
 42%|████▏     | 57/137 [00:05<00:08,  9.80it/s]
Running evaluation: 7it [00:00, 80.81it/s]
 42%|████▏     | 58/137 [00:05<00:08,  9.81it/s]
Running evaluation: 7it [00:00, 80.07it/s]
 43%|████▎     | 59/137 [00:06<00:07,  9.80it/s]
Running evaluation: 7it [00:00, 79.89it/s]
 44%|████▍     | 60/137 [00:06<00:07,  9.79it/s]
Running evaluation: 7it [00:00, 80.78it/s]
 45%|████▍     | 61/137 [00:06<00:07,  9.81it/s]
Running evaluation: 7it [00:00, 80.28it/s]
 45%|████▌     | 62/137 [00:06<00:07,  9.80it/s]
Running evaluation: 7it [00:00, 80.50it/s]
 46%|████▌     | 63/137 [00:06<00:07,  9.81it/s]
Running evaluation: 7it [00:00, 80.07it/s]
 47%|████▋     | 64/137 [00:06<00:07,  9.76it/s]
Running evaluation: 7it [00:00, 77.93it/s]
 47%|████▋     | 65/137 [00:06<00:07,  9.67it/s]
Running evaluation: 7it [00:00, 80.83it/s]
 48%|████▊     | 66/137 [00:06<00:07,  9.72it/s]
Running evaluation: 7it [00:00, 81.68it/s]
 49%|████▉     | 67/137 [00:06<00:07,  9.79it/s]
Running evaluation: 7it [00:00, 81.13it/s]
 50%|████▉     | 68/137 [00:06<00:07,  9.81it/s]
Running evaluation: 7it [00:00, 81.32it/s]
 50%|█████     | 69/137 [00:07<00:06,  9.84it/s]
Running evaluation: 7it [00:00, 80.92it/s]
 51%|█████     | 70/137 [00:07<00:06,  9.83it/s]
Running evaluation: 7it [00:00, 81.30it/s]
 52%|█████▏    | 71/137 [00:07<00:06,  9.85it/s]
Running evaluation: 7it [00:00, 82.09it/s]
 53%|█████▎    | 72/137 [00:07<00:06,  9.89it/s]
Running evaluation: 7it [00:00, 81.77it/s]
 53%|█████▎    | 73/137 [00:07<00:06,  9.87it/s]
Running evaluation: 7it [00:00, 81.08it/s]
 54%|█████▍    | 74/137 [00:07<00:06,  9.87it/s]
Running evaluation: 7it [00:00, 81.81it/s]
 55%|█████▍    | 75/137 [00:07<00:06,  9.89it/s]
Running evaluation: 7it [00:00, 80.63it/s]
 55%|█████▌    | 76/137 [00:07<00:06,  9.87it/s]
Running evaluation: 7it [00:00, 80.91it/s]
 56%|█████▌    | 77/137 [00:07<00:06,  9.87it/s]
Running evaluation: 7it [00:00, 80.83it/s]
 57%|█████▋    | 78/137 [00:08<00:05,  9.86it/s]
Running evaluation: 7it [00:00, 80.90it/s]
 58%|█████▊    | 79/137 [00:08<00:05,  9.86it/s]
Running evaluation: 7it [00:00, 80.50it/s]
 58%|█████▊    | 80/137 [00:08<00:05,  9.84it/s]
Running evaluation: 7it [00:00, 80.69it/s]
 59%|█████▉    | 81/137 [00:08<00:05,  9.84it/s]
Running evaluation: 7it [00:00, 78.67it/s]
 60%|█████▉    | 82/137 [00:08<00:05,  9.74it/s]
Running evaluation: 0it [00:00, ?it/s]
Running evaluation: 7it [00:00, 57.62it/s]
 61%|██████    | 83/137 [00:08<00:06,  8.73it/s]
Running evaluation: 7it [00:00, 79.47it/s]
 61%|██████▏   | 84/137 [00:08<00:05,  8.97it/s]
Running evaluation: 7it [00:00, 77.83it/s]
 62%|██████▏   | 85/137 [00:08<00:05,  9.12it/s]
Running evaluation: 7it [00:00, 79.50it/s]
 63%|██████▎   | 86/137 [00:08<00:05,  9.26it/s]
Running evaluation: 7it [00:00, 78.65it/s]
 64%|██████▎   | 87/137 [00:08<00:05,  9.33it/s]
Running evaluation: 7it [00:00, 77.89it/s]
 64%|██████▍   | 88/137 [00:09<00:05,  9.37it/s]
Running evaluation: 7it [00:00, 77.88it/s]
 65%|██████▍   | 89/137 [00:09<00:05,  9.39it/s]
Running evaluation: 7it [00:00, 78.58it/s]
 66%|██████▌   | 90/137 [00:09<00:04,  9.43it/s]
Running evaluation: 7it [00:00, 78.98it/s]
 66%|██████▋   | 91/137 [00:09<00:04,  9.45it/s]
Running evaluation: 7it [00:00, 79.47it/s]
 67%|██████▋   | 92/137 [00:09<00:04,  9.47it/s]
Running evaluation: 7it [00:00, 75.97it/s]
 68%|██████▊   | 93/137 [00:09<00:04,  9.37it/s]
Running evaluation: 7it [00:00, 81.28it/s]
 69%|██████▊   | 94/137 [00:09<00:04,  9.47it/s]
Running evaluation: 7it [00:00, 81.42it/s]
 69%|██████▉   | 95/137 [00:09<00:04,  9.57it/s]
Running evaluation: 7it [00:00, 78.29it/s]
 70%|███████   | 96/137 [00:09<00:04,  9.51it/s]
Running evaluation: 7it [00:00, 78.14it/s]
 71%|███████   | 97/137 [00:10<00:04,  9.49it/s]
Running evaluation: 7it [00:00, 79.59it/s]
 72%|███████▏  | 98/137 [00:10<00:04,  9.50it/s]
Running evaluation: 7it [00:00, 81.20it/s]
 72%|███████▏  | 99/137 [00:10<00:03,  9.56it/s]
Running evaluation: 7it [00:00, 81.43it/s]
 73%|███████▎  | 100/137 [00:10<00:03,  9.66it/s]
Running evaluation: 7it [00:00, 82.14it/s]
 74%|███████▎  | 101/137 [00:10<00:03,  9.76it/s]
Running evaluation: 7it [00:00, 80.95it/s]
 74%|███████▍  | 102/137 [00:10<00:03,  9.79it/s]
Running evaluation: 7it [00:00, 75.02it/s]
 75%|███████▌  | 103/137 [00:10<00:03,  9.58it/s]
Running evaluation: 7it [00:00, 80.46it/s]
 76%|███████▌  | 104/137 [00:10<00:03,  9.63it/s]
Running evaluation: 7it [00:00, 81.08it/s]
 77%|███████▋  | 105/137 [00:10<00:03,  9.70it/s]
Running evaluation: 7it [00:00, 82.26it/s]
 77%|███████▋  | 106/137 [00:10<00:03,  9.77it/s]
Running evaluation: 7it [00:00, 81.13it/s]
 78%|███████▊  | 107/137 [00:11<00:03,  9.80it/s]
Running evaluation: 7it [00:00, 81.79it/s]
 79%|███████▉  | 108/137 [00:11<00:02,  9.81it/s]
Running evaluation: 7it [00:00, 81.81it/s]
 80%|███████▉  | 109/137 [00:11<00:02,  9.83it/s]
Running evaluation: 7it [00:00, 79.83it/s]
 80%|████████  | 110/137 [00:11<00:02,  9.78it/s]
Running evaluation: 7it [00:00, 81.44it/s]
 81%|████████  | 111/137 [00:11<00:02,  9.81it/s]
Running evaluation: 7it [00:00, 81.57it/s]
 82%|████████▏ | 112/137 [00:11<00:02,  9.85it/s]
Running evaluation: 7it [00:00, 79.45it/s]
 82%|████████▏ | 113/137 [00:11<00:02,  9.77it/s]
Running evaluation: 7it [00:00, 77.78it/s]
 83%|████████▎ | 114/137 [00:11<00:02,  9.68it/s]
Running evaluation: 7it [00:00, 81.29it/s]
 84%|████████▍ | 115/137 [00:11<00:02,  9.74it/s]
Running evaluation: 7it [00:00, 81.49it/s]
 85%|████████▍ | 116/137 [00:11<00:02,  9.80it/s]
Running evaluation: 7it [00:00, 80.46it/s]
 85%|████████▌ | 117/137 [00:12<00:02,  9.80it/s]
Running evaluation: 7it [00:00, 80.67it/s]
 86%|████████▌ | 118/137 [00:12<00:01,  9.81it/s]
Running evaluation: 7it [00:00, 81.37it/s]
 87%|████████▋ | 119/137 [00:12<00:01,  9.84it/s]
Running evaluation: 7it [00:00, 78.34it/s]
 88%|████████▊ | 120/137 [00:12<00:01,  9.74it/s]
Running evaluation: 7it [00:00, 79.76it/s]
 88%|████████▊ | 121/137 [00:12<00:01,  9.70it/s]
Running evaluation: 7it [00:00, 79.34it/s]
 89%|████████▉ | 122/137 [00:12<00:01,  9.64it/s]
Running evaluation: 7it [00:00, 78.06it/s]
 90%|████████▉ | 123/137 [00:12<00:01,  9.58it/s]
Running evaluation: 7it [00:00, 81.37it/s]
 91%|█████████ | 124/137 [00:12<00:01,  9.67it/s]
Running evaluation: 7it [00:00, 80.66it/s]
 91%|█████████ | 125/137 [00:12<00:01,  9.72it/s]
Running evaluation: 7it [00:00, 81.24it/s]
 92%|█████████▏| 126/137 [00:13<00:01,  9.77it/s]
Running evaluation: 7it [00:00, 82.04it/s]
 93%|█████████▎| 127/137 [00:13<00:01,  9.83it/s]
Running evaluation: 7it [00:00, 81.05it/s]
 93%|█████████▎| 128/137 [00:13<00:00,  9.82it/s]
Running evaluation: 7it [00:00, 83.03it/s]

Running evaluation: 7it [00:00, 82.04it/s]
 95%|█████████▍| 130/137 [00:13<00:00,  9.88it/s]
Running evaluation: 7it [00:00, 80.86it/s]
 96%|█████████▌| 131/137 [00:13<00:00,  9.86it/s]
Running evaluation: 7it [00:00, 81.55it/s]
 96%|█████████▋| 132/137 [00:13<00:00,  9.87it/s]
Running evaluation: 7it [00:00, 81.48it/s]
 97%|█████████▋| 133/137 [00:13<00:00,  9.88it/s]
Running evaluation: 7it [00:00, 81.72it/s]
 98%|█████████▊| 134/137 [00:13<00:00,  9.90it/s]
Running evaluation: 7it [00:00, 81.09it/s]
 99%|█████████▊| 135/137 [00:13<00:00,  9.82it/s]
Running evaluation: 7it [00:00, 76.88it/s]
 99%|█████████▉| 136/137 [00:14<00:00,  9.65it/s]
Running evaluation: 7it [00:00, 81.92it/s]
100%|██████████| 137/137 [00:14<00:00,  9.70it/s]
Running evaluation: 7it [00:00, 61.47it/s]

Metrics

In [ ]:

print("CRPS: {}".format(agg_metric['mean_wQuantileLoss']))
print("ND: {}".format(agg_metric['ND']))
print("NRMSE: {}".format(agg_metric['NRMSE']))
print("MSE: {}".format(agg_metric['MSE']))

CRPS: 0.3855313301520275
ND: 0.48820539490099113
NRMSE: 1.018839692673421
MSE: 984.6672641166102

In [ ]:

print("CRPS-Sum: {}".format(agg_metric['m_sum_mean_wQuantileLoss']))
print("ND-Sum: {}".format(agg_metric['m_sum_ND']))
print("NRMSE-Sum: {}".format(agg_metric['m_sum_NRMSE']))
print("MSE-Sum: {}".format(agg_metric['m_sum_MSE']))

CRPS-Sum: 0.3268739166960563
ND-Sum: 0.40321702146475014
NRMSE-Sum: 0.75586334994103
MSE-Sum: 10171980.5

Transformer-MAF

In [ ]:

estimator = TransformerTempFlowEstimator(
    d_model=16,
    num_heads=4,
    input_size=552,
    target_dim=int(dataset.metadata.feat_static_cat[0].cardinality),
    prediction_length=dataset.metadata.prediction_length,
    context_length=dataset.metadata.prediction_length*4,
    flow_type='MAF',
    dequantize=True,
    freq=dataset.metadata.freq,
    trainer=Trainer(
        device=device,
        epochs=14,
        learning_rate=1e-3,
        num_batches_per_epoch=100,
        batch_size=64,
    )
)

In [ ]:

predictor = estimator.train(dataset_train)
forecast_it, ts_it = make_evaluation_predictions(dataset=dataset_test,
                                             predictor=predictor,
                                             num_samples=100)
forecasts = list(forecast_it)
targets = list(ts_it)

agg_metric, _ = evaluator(targets, forecasts, num_series=len(dataset_test))

99it [00:26,  3.70it/s, avg_epoch_loss=-82.7, epoch=0]
Running evaluation: 7it [00:00, 121.58it/s]
Running evaluation: 7it [00:00, 129.89it/s]
Running evaluation: 7it [00:00, 133.02it/s]
Running evaluation: 7it [00:00, 133.71it/s]
Running evaluation: 7it [00:00, 129.68it/s]
Running evaluation: 7it [00:00, 130.11it/s]
Running evaluation: 7it [00:00, 135.91it/s]
Running evaluation: 7it [00:00, 134.94it/s]
Running evaluation: 7it [00:00, 127.03it/s]
Running evaluation: 7it [00:00, 131.79it/s]
Running evaluation: 7it [00:00, 131.80it/s]
Running evaluation: 7it [00:00, 129.62it/s]
Running evaluation: 7it [00:00, 130.80it/s]
Running evaluation: 7it [00:00, 134.32it/s]
Running evaluation: 7it [00:00, 135.98it/s]
Running evaluation: 7it [00:00, 132.59it/s]
Running evaluation: 7it [00:00, 132.17it/s]
Running evaluation: 7it [00:00, 131.03it/s]
Running evaluation: 7it [00:00, 130.69it/s]
Running evaluation: 7it [00:00, 130.72it/s]
Running evaluation: 7it [00:00, 132.53it/s]
Running evaluation: 7it [00:00, 129.63it/s]
Running evaluation: 7it [00:00, 65.78it/s]
Running evaluation: 7it [00:00, 133.88it/s]
Running evaluation: 7it [00:00, 129.27it/s]
Running evaluation: 7it [00:00, 134.82it/s]
Running evaluation: 7it [00:00, 133.96it/s]
Running evaluation: 7it [00:00, 130.77it/s]
Running evaluation: 7it [00:00, 130.26it/s]
Running evaluation: 7it [00:00, 130.87it/s]
Running evaluation: 7it [00:00, 129.28it/s]
Running evaluation: 7it [00:00, 129.81it/s]
Running evaluation: 7it [00:00, 132.89it/s]
Running evaluation: 7it [00:00, 132.76it/s]
Running evaluation: 7it [00:00, 131.64it/s]
Running evaluation: 7it [00:00, 133.07it/s]
Running evaluation: 7it [00:00, 128.50it/s]
Running evaluation: 7it [00:00, 135.86it/s]
Running evaluation: 7it [00:00, 130.13it/s]
Running evaluation: 7it [00:00, 129.31it/s]
Running evaluation: 7it [00:00, 128.67it/s]
Running evaluation: 7it [00:00, 134.41it/s]
Running evaluation: 7it [00:00, 128.88it/s]
Running evaluation: 7it [00:00, 134.21it/s]
Running evaluation: 7it [00:00, 134.11it/s]
Running evaluation: 7it [00:00, 133.82it/s]
Running evaluation: 7it [00:00, 131.31it/s]
Running evaluation: 7it [00:00, 128.78it/s]
Running evaluation: 7it [00:00, 128.76it/s]
Running evaluation: 7it [00:00, 127.98it/s]
Running evaluation: 7it [00:00, 130.26it/s]
Running evaluation: 7it [00:00, 120.39it/s]
Running evaluation: 7it [00:00, 134.66it/s]
Running evaluation: 7it [00:00, 134.51it/s]
Running evaluation: 7it [00:00, 125.47it/s]
Running evaluation: 7it [00:00, 133.05it/s]
Running evaluation: 7it [00:00, 129.13it/s]
Running evaluation: 7it [00:00, 131.84it/s]
Running evaluation: 7it [00:00, 130.52it/s]
Running evaluation: 7it [00:00, 136.95it/s]
Running evaluation: 7it [00:00, 135.88it/s]
Running evaluation: 7it [00:00, 137.97it/s]
Running evaluation: 7it [00:00, 136.48it/s]
Running evaluation: 7it [00:00, 137.81it/s]
Running evaluation: 7it [00:00, 138.88it/s]
Running evaluation: 7it [00:00, 140.48it/s]
Running evaluation: 7it [00:00, 139.82it/s]
Running evaluation: 7it [00:00, 137.45it/s]
Running evaluation: 7it [00:00, 139.96it/s]
Running evaluation: 7it [00:00, 139.87it/s]
Running evaluation: 7it [00:00, 137.53it/s]
Running evaluation: 7it [00:00, 136.43it/s]
Running evaluation: 7it [00:00, 129.52it/s]
Running evaluation: 7it [00:00, 134.57it/s]
Running evaluation: 7it [00:00, 136.23it/s]
Running evaluation: 7it [00:00, 141.61it/s]
Running evaluation: 7it [00:00, 137.81it/s]
Running evaluation: 7it [00:00, 137.27it/s]
Running evaluation: 7it [00:00, 138.90it/s]
Running evaluation: 7it [00:00, 138.50it/s]
Running evaluation: 7it [00:00, 136.98it/s]
Running evaluation: 7it [00:00, 121.52it/s]
Running evaluation: 7it [00:00, 129.19it/s]
Running evaluation: 7it [00:00, 136.95it/s]
Running evaluation: 7it [00:00, 138.76it/s]
Running evaluation: 7it [00:00, 135.65it/s]
Running evaluation: 7it [00:00, 137.78it/s]
Running evaluation: 7it [00:00, 130.82it/s]
Running evaluation: 7it [00:00, 129.72it/s]
Running evaluation: 7it [00:00, 133.23it/s]
Running evaluation: 7it [00:00, 136.86it/s]
Running evaluation: 7it [00:00, 140.74it/s]
Running evaluation: 7it [00:00, 134.70it/s]
Running evaluation: 7it [00:00, 138.99it/s]
Running evaluation: 7it [00:00, 133.51it/s]
Running evaluation: 7it [00:00, 129.42it/s]
Running evaluation: 7it [00:00, 133.08it/s]
Running evaluation: 7it [00:00, 132.08it/s]
Running evaluation: 7it [00:00, 135.16it/s]
Running evaluation: 7it [00:00, 135.33it/s]
Running evaluation: 7it [00:00, 132.90it/s]
Running evaluation: 7it [00:00, 129.54it/s]
Running evaluation: 7it [00:00, 128.03it/s]
Running evaluation: 7it [00:00, 129.57it/s]
Running evaluation: 7it [00:00, 129.91it/s]
Running evaluation: 7it [00:00, 131.76it/s]
Running evaluation: 7it [00:00, 130.74it/s]
Running evaluation: 7it [00:00, 131.16it/s]
Running evaluation: 7it [00:00, 126.87it/s]
Running evaluation: 7it [00:00, 133.84it/s]
Running evaluation: 7it [00:00, 130.86it/s]
Running evaluation: 7it [00:00, 126.03it/s]
Running evaluation: 7it [00:00, 125.04it/s]
Running evaluation: 7it [00:00, 128.97it/s]
Running evaluation: 7it [00:00, 130.28it/s]
Running evaluation: 7it [00:00, 127.62it/s]
Running evaluation: 7it [00:00, 134.83it/s]
Running evaluation: 7it [00:00, 135.22it/s]
Running evaluation: 7it [00:00, 118.29it/s]
Running evaluation: 7it [00:00, 124.76it/s]
Running evaluation: 7it [00:00, 128.80it/s]
Running evaluation: 7it [00:00, 129.55it/s]
Running evaluation: 7it [00:00, 129.08it/s]
Running evaluation: 7it [00:00, 130.09it/s]
Running evaluation: 7it [00:00, 126.55it/s]
Running evaluation: 7it [00:00, 128.26it/s]
Running evaluation: 7it [00:00, 132.86it/s]
Running evaluation: 7it [00:00, 128.60it/s]
Running evaluation: 7it [00:00, 128.98it/s]
Running evaluation: 7it [00:00, 127.39it/s]
Running evaluation: 7it [00:00, 133.66it/s]
Running evaluation: 7it [00:00, 128.62it/s]
Running evaluation: 7it [00:00, 130.26it/s]
Running evaluation: 7it [00:00, 70.39it/s]
Running evaluation: 7it [00:00, 126.12it/s]
Running evaluation: 7it [00:00, 128.30it/s]
Running evaluation: 7it [00:00, 129.34it/s]
Running evaluation: 7it [00:00, 54.24it/s]

Metrics

In [ ]:

print("CRPS: {}".format(agg_metric['mean_wQuantileLoss']))
print("ND: {}".format(agg_metric['ND']))
print("NRMSE: {}".format(agg_metric['NRMSE']))
print("MSE: {}".format(agg_metric['MSE']))

CRPS: 0.37264046134993567
ND: 0.5043621354947913
NRMSE: 0.9928759300158241
MSE: 935.1208752979203

In [ ]:

print("CRPS-Sum: {}".format(agg_metric['m_sum_mean_wQuantileLoss']))
print("ND-Sum: {}".format(agg_metric['m_sum_ND']))
print("NRMSE-Sum: {}".format(agg_metric['m_sum_NRMSE']))
print("MSE-Sum: {}".format(agg_metric['m_sum_MSE']))

CRPS-Sum: 0.30787625107438427
ND-Sum: 0.4188356756894787
NRMSE-Sum: 0.7504274205713227
MSE-Sum: 10026199.285714285

In [ ]: