pytorch-ts/examples/Time-Grad-Electricity.ipynb

In [1]:

%matplotlib inline
import matplotlib.pyplot as plt

import numpy as np
import pandas as pd

import torch

In [2]:

from gluonts.dataset.multivariate_grouper import MultivariateGrouper
from gluonts.dataset.repository.datasets import dataset_recipes, get_dataset
from gluonts.evaluation.backtest import make_evaluation_predictions
from gluonts.evaluation import MultivariateEvaluator

In [3]:

from pts.model.tempflow import TempFlowEstimator
from pts.model.time_grad import TimeGradEstimator
from pts.model.transformer_tempflow import TransformerTempFlowEstimator
from pts import Trainer

In [6]:

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

In [7]:

def plot(target, forecast, prediction_length, prediction_intervals=(50.0, 90.0), color='g', fname=None):
    label_prefix = ""
    rows = 4
    cols = 4
    fig, axs = plt.subplots(rows, cols, figsize=(24, 24))
    axx = axs.ravel()
    seq_len, target_dim = target.shape
    
    ps = [50.0] + [
            50.0 + f * c / 2.0 for c in prediction_intervals for f in [-1.0, +1.0]
        ]
        
    percentiles_sorted = sorted(set(ps))
    
    def alpha_for_percentile(p):
        return (p / 100.0) ** 0.3
        
    for dim in range(0, min(rows * cols, target_dim)):
        ax = axx[dim]

        target[-2 * prediction_length :][dim].plot(ax=ax)
        
        ps_data = [forecast.quantile(p / 100.0)[:,dim] for p in percentiles_sorted]
        i_p50 = len(percentiles_sorted) // 2
        
        p50_data = ps_data[i_p50]
        p50_series = pd.Series(data=p50_data, index=forecast.index)
        p50_series.plot(color=color, ls="-", label=f"{label_prefix}median", ax=ax)
        
        for i in range(len(percentiles_sorted) // 2):
            ptile = percentiles_sorted[i]
            alpha = alpha_for_percentile(ptile)
            ax.fill_between(
                forecast.index,
                ps_data[i],
                ps_data[-i - 1],
                facecolor=color,
                alpha=alpha,
                interpolate=True,
            )
            # Hack to create labels for the error intervals.
            # Doesn't actually plot anything, because we only pass a single data point
            pd.Series(data=p50_data[:1], index=forecast.index[:1]).plot(
                color=color,
                alpha=alpha,
                linewidth=10,
                label=f"{label_prefix}{100 - ptile * 2}%",
                ax=ax,
            )

    legend = ["observations", "median prediction"] + [f"{k}% prediction interval" for k in prediction_intervals][::-1]    
    axx[0].legend(legend, loc="upper left")
    
    if fname is not None:
        plt.savefig(fname, bbox_inches='tight', pad_inches=0.05)

In [8]:

print(f"Available datasets: {list(dataset_recipes.keys())}")

Available datasets: ['constant', 'exchange_rate', 'solar-energy', 'electricity', 'traffic', 'exchange_rate_nips', 'electricity_nips', 'traffic_nips', 'solar_nips', 'wiki-rolling_nips', 'taxi_30min', 'm3_monthly', 'm3_quarterly', 'm3_yearly', 'm3_other', 'm4_hourly', 'm4_daily', 'm4_weekly', 'm4_monthly', 'm4_quarterly', 'm4_yearly', 'm5']

In [9]:

# exchange_rate_nips, electricity_nips, traffic_nips, solar_nips, wiki-rolling_nips, ## taxi_30min is buggy still
dataset = get_dataset("electricity_nips", regenerate=False)

In [10]:

dataset.metadata

Out [10]:

MetaData(freq='H', target=None, feat_static_cat=[CategoricalFeatureInfo(name='feat_static_cat', cardinality='370')], feat_static_real=[], feat_dynamic_real=[], feat_dynamic_cat=[], prediction_length=24)

In [11]:

train_grouper = MultivariateGrouper(max_target_dim=min(2000, int(dataset.metadata.feat_static_cat[0].cardinality)))

test_grouper = MultivariateGrouper(num_test_dates=int(len(dataset.test)/len(dataset.train)), 
                                   max_target_dim=min(2000, int(dataset.metadata.feat_static_cat[0].cardinality)))

In [12]:

dataset_train = train_grouper(dataset.train)
dataset_test = test_grouper(dataset.test)

/home/krasul/.env/pytorch/lib/python3.7/site-packages/gluonts-0.5.1.dev236+gb4d8144.d20210210-py3.7.egg/gluonts/dataset/multivariate_grouper.py:182: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.
  return {FieldName.TARGET: np.array([funcs(data) for data in dataset])}

In [21]:

estimator = TimeGradEstimator(
    target_dim=int(dataset.metadata.feat_static_cat[0].cardinality),
    prediction_length=dataset.metadata.prediction_length,
    context_length=dataset.metadata.prediction_length,
    cell_type='GRU',
    input_size=1484,
    freq=dataset.metadata.freq,
    loss_type='l2',
    scaling=True,
    diff_steps=100,
    beta_end=0.1,
    beta_schedule="linear",
    trainer=Trainer(device=device,
                    epochs=20,
                    learning_rate=1e-3,
                    num_batches_per_epoch=100,
                    batch_size=64,)
)

In [22]:

predictor = estimator.train(dataset_train, num_workers=8)

99it [00:36,  2.68it/s, avg_epoch_loss=0.78, epoch=0] 
99it [00:37,  2.67it/s, avg_epoch_loss=0.171, epoch=1]
99it [00:36,  2.69it/s, avg_epoch_loss=0.113, epoch=2]
99it [00:36,  2.70it/s, avg_epoch_loss=0.103, epoch=3]
99it [00:37,  2.65it/s, avg_epoch_loss=0.103, epoch=4] 
99it [00:36,  2.68it/s, avg_epoch_loss=0.0922, epoch=5]
99it [00:37,  2.66it/s, avg_epoch_loss=0.0864, epoch=6]
99it [00:36,  2.71it/s, avg_epoch_loss=0.0831, epoch=7]
99it [00:36,  2.68it/s, avg_epoch_loss=0.0809, epoch=8]
99it [00:37,  2.67it/s, avg_epoch_loss=0.0787, epoch=9]
99it [00:36,  2.70it/s, avg_epoch_loss=0.0769, epoch=10]
99it [00:36,  2.72it/s, avg_epoch_loss=0.0752, epoch=11]
99it [00:36,  2.69it/s, avg_epoch_loss=0.0747, epoch=12]
99it [00:36,  2.70it/s, avg_epoch_loss=0.0734, epoch=13]
99it [00:36,  2.70it/s, avg_epoch_loss=0.0732, epoch=14]
99it [00:36,  2.69it/s, avg_epoch_loss=0.0724, epoch=15]
99it [00:36,  2.68it/s, avg_epoch_loss=0.0713, epoch=16]
99it [00:36,  2.70it/s, avg_epoch_loss=0.0711, epoch=17]
99it [00:36,  2.69it/s, avg_epoch_loss=0.0709, epoch=18]
99it [00:36,  2.71it/s, avg_epoch_loss=0.0705, epoch=19]

In [23]:

forecast_it, ts_it = make_evaluation_predictions(dataset=dataset_test,
                                                 predictor=predictor,
                                                 num_samples=100)

In [24]:

forecasts = list(forecast_it)
targets = list(ts_it)

In [25]:

plot(
    target=targets[0],
    forecast=forecasts[0],
    prediction_length=dataset.metadata.prediction_length,
)
plt.show()

In [26]:

evaluator = MultivariateEvaluator(quantiles=(np.arange(20)/20.0)[1:], 
                                  target_agg_funcs={'sum': np.sum})

In [27]:

agg_metric, item_metrics = evaluator(targets, forecasts, num_series=len(dataset_test))

Running evaluation: 7it [00:00, 72.38it/s]
Running evaluation: 7it [00:00, 77.71it/s]
Running evaluation: 7it [00:00, 75.46it/s]
Running evaluation: 7it [00:00, 76.83it/s]
Running evaluation: 7it [00:00, 73.96it/s]
Running evaluation: 7it [00:00, 77.31it/s]
Running evaluation: 7it [00:00, 76.99it/s]
Running evaluation: 7it [00:00, 77.94it/s]
Running evaluation: 7it [00:00, 77.07it/s]
Running evaluation: 7it [00:00, 76.45it/s]
Running evaluation: 7it [00:00, 76.61it/s]
Running evaluation: 7it [00:00, 75.67it/s]
Running evaluation: 7it [00:00, 77.38it/s]
Running evaluation: 7it [00:00, 76.56it/s]
Running evaluation: 7it [00:00, 76.79it/s]
Running evaluation: 7it [00:00, 77.81it/s]
Running evaluation: 7it [00:00, 77.01it/s]
Running evaluation: 7it [00:00, 75.84it/s]
Running evaluation: 7it [00:00, 75.33it/s]
Running evaluation: 7it [00:00, 76.27it/s]
Running evaluation: 7it [00:00, 74.73it/s]
Running evaluation: 7it [00:00, 77.41it/s]
Running evaluation: 7it [00:00, 76.45it/s]
Running evaluation: 7it [00:00, 77.14it/s]
Running evaluation: 7it [00:00, 76.62it/s]
Running evaluation: 7it [00:00, 76.83it/s]
Running evaluation: 7it [00:00, 76.54it/s]
Running evaluation: 7it [00:00, 76.47it/s]
Running evaluation: 7it [00:00, 76.61it/s]
Running evaluation: 7it [00:00, 76.61it/s]
Running evaluation: 7it [00:00, 75.72it/s]
Running evaluation: 7it [00:00, 75.76it/s]
Running evaluation: 7it [00:00, 74.84it/s]
Running evaluation: 7it [00:00, 74.49it/s]
Running evaluation: 7it [00:00, 74.05it/s]
Running evaluation: 7it [00:00, 75.60it/s]
Running evaluation: 7it [00:00, 71.55it/s]
Running evaluation: 7it [00:00, 76.41it/s]
Running evaluation: 7it [00:00, 75.78it/s]
Running evaluation: 7it [00:00, 74.05it/s]
Running evaluation: 7it [00:00, 75.00it/s]
Running evaluation: 7it [00:00, 76.55it/s]
Running evaluation: 7it [00:00, 75.55it/s]
Running evaluation: 7it [00:00, 75.17it/s]
Running evaluation: 7it [00:00, 76.29it/s]
Running evaluation: 7it [00:00, 75.67it/s]
Running evaluation: 7it [00:00, 75.91it/s]
Running evaluation: 7it [00:00, 76.78it/s]
Running evaluation: 7it [00:00, 76.55it/s]
Running evaluation: 7it [00:00, 76.62it/s]
Running evaluation: 7it [00:00, 77.14it/s]
Running evaluation: 7it [00:00, 77.05it/s]
Running evaluation: 7it [00:00, 76.62it/s]
Running evaluation: 7it [00:00, 76.77it/s]
Running evaluation: 7it [00:00, 75.57it/s]
Running evaluation: 7it [00:00, 75.76it/s]
Running evaluation: 7it [00:00, 76.86it/s]
Running evaluation: 7it [00:00, 77.20it/s]
Running evaluation: 7it [00:00, 76.78it/s]
Running evaluation: 7it [00:00, 76.01it/s]
Running evaluation: 7it [00:00, 76.60it/s]
Running evaluation: 7it [00:00, 57.32it/s]
Running evaluation: 7it [00:00, 74.88it/s]
Running evaluation: 7it [00:00, 76.47it/s]
Running evaluation: 7it [00:00, 76.41it/s]
Running evaluation: 7it [00:00, 75.55it/s]
Running evaluation: 7it [00:00, 75.49it/s]
Running evaluation: 7it [00:00, 73.49it/s]
Running evaluation: 7it [00:00, 77.10it/s]
Running evaluation: 7it [00:00, 76.59it/s]
Running evaluation: 7it [00:00, 76.85it/s]
Running evaluation: 7it [00:00, 76.77it/s]
Running evaluation: 7it [00:00, 74.23it/s]
Running evaluation: 7it [00:00, 74.60it/s]
Running evaluation: 7it [00:00, 77.11it/s]
Running evaluation: 7it [00:00, 78.20it/s]
Running evaluation: 7it [00:00, 78.37it/s]
Running evaluation: 7it [00:00, 76.93it/s]
Running evaluation: 7it [00:00, 76.91it/s]
Running evaluation: 7it [00:00, 76.60it/s]
Running evaluation: 7it [00:00, 76.01it/s]
Running evaluation: 7it [00:00, 75.52it/s]
Running evaluation: 7it [00:00, 75.61it/s]
Running evaluation: 7it [00:00, 75.47it/s]
Running evaluation: 7it [00:00, 76.67it/s]
Running evaluation: 7it [00:00, 76.75it/s]
Running evaluation: 7it [00:00, 75.94it/s]
Running evaluation: 7it [00:00, 76.72it/s]
Running evaluation: 7it [00:00, 76.35it/s]
Running evaluation: 7it [00:00, 76.55it/s]
Running evaluation: 7it [00:00, 75.42it/s]
Running evaluation: 7it [00:00, 75.54it/s]
Running evaluation: 7it [00:00, 74.10it/s]
Running evaluation: 7it [00:00, 76.50it/s]
Running evaluation: 7it [00:00, 75.45it/s]
Running evaluation: 7it [00:00, 74.88it/s]
Running evaluation: 7it [00:00, 74.57it/s]
Running evaluation: 7it [00:00, 76.75it/s]
Running evaluation: 7it [00:00, 76.03it/s]
Running evaluation: 7it [00:00, 76.02it/s]
Running evaluation: 7it [00:00, 75.24it/s]
Running evaluation: 7it [00:00, 75.56it/s]
Running evaluation: 7it [00:00, 75.52it/s]
Running evaluation: 7it [00:00, 77.11it/s]
Running evaluation: 7it [00:00, 76.98it/s]
Running evaluation: 7it [00:00, 75.59it/s]
Running evaluation: 7it [00:00, 76.75it/s]
Running evaluation: 7it [00:00, 76.76it/s]
Running evaluation: 7it [00:00, 76.82it/s]
Running evaluation: 7it [00:00, 76.16it/s]
Running evaluation: 7it [00:00, 71.23it/s]
Running evaluation: 7it [00:00, 36.27it/s]
Running evaluation: 7it [00:00, 76.96it/s]
Running evaluation: 7it [00:00, 75.86it/s]
Running evaluation: 7it [00:00, 75.41it/s]
Running evaluation: 7it [00:00, 75.07it/s]
Running evaluation: 7it [00:00, 75.36it/s]
Running evaluation: 7it [00:00, 74.79it/s]
Running evaluation: 7it [00:00, 75.88it/s]
Running evaluation: 7it [00:00, 75.87it/s]
Running evaluation: 7it [00:00, 76.54it/s]
Running evaluation: 7it [00:00, 76.72it/s]
Running evaluation: 7it [00:00, 76.56it/s]
Running evaluation: 7it [00:00, 76.97it/s]
Running evaluation: 7it [00:00, 76.15it/s]
Running evaluation: 7it [00:00, 75.78it/s]
Running evaluation: 7it [00:00, 76.68it/s]
Running evaluation: 7it [00:00, 76.68it/s]
Running evaluation: 7it [00:00, 76.78it/s]
Running evaluation: 7it [00:00, 72.47it/s]
Running evaluation: 7it [00:00, 76.30it/s]
Running evaluation: 7it [00:00, 75.78it/s]
Running evaluation: 7it [00:00, 75.62it/s]
Running evaluation: 7it [00:00, 76.77it/s]
Running evaluation: 7it [00:00, 75.80it/s]
Running evaluation: 7it [00:00, 77.45it/s]
Running evaluation: 7it [00:00, 77.08it/s]
Running evaluation: 7it [00:00, 76.47it/s]
Running evaluation: 7it [00:00, 76.08it/s]
Running evaluation: 7it [00:00, 76.44it/s]
Running evaluation: 7it [00:00, 76.37it/s]
Running evaluation: 7it [00:00, 76.13it/s]
Running evaluation: 7it [00:00, 74.55it/s]
Running evaluation: 7it [00:00, 76.56it/s]
Running evaluation: 7it [00:00, 75.98it/s]
Running evaluation: 7it [00:00, 76.66it/s]
Running evaluation: 7it [00:00, 74.98it/s]
Running evaluation: 7it [00:00, 76.65it/s]
Running evaluation: 7it [00:00, 75.65it/s]
Running evaluation: 7it [00:00, 76.61it/s]
Running evaluation: 7it [00:00, 75.90it/s]
Running evaluation: 7it [00:00, 76.39it/s]
Running evaluation: 7it [00:00, 76.65it/s]
Running evaluation: 7it [00:00, 76.23it/s]
Running evaluation: 7it [00:00, 76.94it/s]
Running evaluation: 7it [00:00, 76.94it/s]
Running evaluation: 7it [00:00, 76.86it/s]
Running evaluation: 7it [00:00, 76.76it/s]
Running evaluation: 7it [00:00, 76.05it/s]
Running evaluation: 7it [00:00, 76.69it/s]
Running evaluation: 7it [00:00, 76.81it/s]
Running evaluation: 7it [00:00, 70.44it/s]
Running evaluation: 7it [00:00, 78.37it/s]
Running evaluation: 7it [00:00, 76.43it/s]
Running evaluation: 7it [00:00, 76.66it/s]
Running evaluation: 7it [00:00, 75.95it/s]
Running evaluation: 7it [00:00, 76.63it/s]
Running evaluation: 7it [00:00, 75.95it/s]
Running evaluation: 7it [00:00, 76.75it/s]
Running evaluation: 7it [00:00, 77.93it/s]
Running evaluation: 7it [00:00, 76.41it/s]
Running evaluation: 7it [00:00, 75.30it/s]
Running evaluation: 7it [00:00, 76.12it/s]
Running evaluation: 7it [00:00, 77.12it/s]
Running evaluation: 7it [00:00, 76.87it/s]
Running evaluation: 7it [00:00, 76.04it/s]
Running evaluation: 7it [00:00, 75.31it/s]
Running evaluation: 7it [00:00, 75.77it/s]
Running evaluation: 7it [00:00, 76.42it/s]
Running evaluation: 7it [00:00, 75.20it/s]
Running evaluation: 7it [00:00, 74.34it/s]
Running evaluation: 7it [00:00, 75.60it/s]
Running evaluation: 7it [00:00, 75.49it/s]
Running evaluation: 7it [00:00, 76.64it/s]
Running evaluation: 7it [00:00, 76.40it/s]
Running evaluation: 7it [00:00, 77.15it/s]
Running evaluation: 7it [00:00, 76.11it/s]
Running evaluation: 7it [00:00, 75.78it/s]
Running evaluation: 7it [00:00, 76.64it/s]
Running evaluation: 7it [00:00, 76.86it/s]
Running evaluation: 7it [00:00, 76.80it/s]
Running evaluation: 7it [00:00, 75.24it/s]
Running evaluation: 7it [00:00, 72.19it/s]
Running evaluation: 7it [00:00, 75.85it/s]
Running evaluation: 7it [00:00, 75.68it/s]
Running evaluation: 7it [00:00, 76.51it/s]
Running evaluation: 7it [00:00, 75.67it/s]
Running evaluation: 7it [00:00, 74.39it/s]
Running evaluation: 7it [00:00, 76.15it/s]
Running evaluation: 7it [00:00, 75.85it/s]
Running evaluation: 7it [00:00, 75.26it/s]
Running evaluation: 7it [00:00, 73.00it/s]
Running evaluation: 7it [00:00, 75.26it/s]
Running evaluation: 7it [00:00, 75.92it/s]
Running evaluation: 7it [00:00, 76.78it/s]
Running evaluation: 7it [00:00, 77.05it/s]
Running evaluation: 7it [00:00, 77.08it/s]
Running evaluation: 7it [00:00, 75.77it/s]
Running evaluation: 7it [00:00, 75.86it/s]
Running evaluation: 7it [00:00, 74.57it/s]
Running evaluation: 7it [00:00, 73.82it/s]
Running evaluation: 7it [00:00, 75.11it/s]
Running evaluation: 7it [00:00, 75.77it/s]
Running evaluation: 7it [00:00, 76.73it/s]
Running evaluation: 7it [00:00, 74.64it/s]
Running evaluation: 7it [00:00, 74.36it/s]
Running evaluation: 7it [00:00, 76.38it/s]
Running evaluation: 7it [00:00, 76.24it/s]
Running evaluation: 7it [00:00, 76.69it/s]
Running evaluation: 7it [00:00, 76.70it/s]
Running evaluation: 7it [00:00, 76.50it/s]
Running evaluation: 7it [00:00, 69.48it/s]
Running evaluation: 7it [00:00, 27.96it/s]
Running evaluation: 7it [00:00, 68.76it/s]
Running evaluation: 7it [00:00, 69.15it/s]
Running evaluation: 7it [00:00, 67.74it/s]
Running evaluation: 7it [00:00, 67.86it/s]
Running evaluation: 7it [00:00, 74.84it/s]
Running evaluation: 7it [00:00, 76.08it/s]
Running evaluation: 7it [00:00, 75.69it/s]
Running evaluation: 7it [00:00, 76.70it/s]
Running evaluation: 7it [00:00, 76.64it/s]
Running evaluation: 7it [00:00, 76.16it/s]
Running evaluation: 7it [00:00, 76.74it/s]
Running evaluation: 7it [00:00, 75.95it/s]
Running evaluation: 7it [00:00, 77.02it/s]
Running evaluation: 7it [00:00, 76.65it/s]
Running evaluation: 7it [00:00, 76.28it/s]
Running evaluation: 7it [00:00, 75.46it/s]
Running evaluation: 7it [00:00, 76.30it/s]
Running evaluation: 7it [00:00, 75.03it/s]
Running evaluation: 7it [00:00, 75.19it/s]
Running evaluation: 7it [00:00, 75.84it/s]
Running evaluation: 7it [00:00, 75.39it/s]
Running evaluation: 7it [00:00, 76.46it/s]
Running evaluation: 7it [00:00, 76.84it/s]
Running evaluation: 7it [00:00, 77.07it/s]
Running evaluation: 7it [00:00, 76.85it/s]
Running evaluation: 7it [00:00, 76.01it/s]
Running evaluation: 7it [00:00, 73.57it/s]
Running evaluation: 7it [00:00, 74.73it/s]
Running evaluation: 7it [00:00, 76.63it/s]
Running evaluation: 7it [00:00, 75.02it/s]
Running evaluation: 7it [00:00, 76.59it/s]
Running evaluation: 7it [00:00, 75.61it/s]
Running evaluation: 7it [00:00, 74.89it/s]
Running evaluation: 7it [00:00, 76.42it/s]
Running evaluation: 7it [00:00, 75.86it/s]
Running evaluation: 7it [00:00, 75.88it/s]
Running evaluation: 7it [00:00, 76.86it/s]
Running evaluation: 7it [00:00, 76.98it/s]
Running evaluation: 7it [00:00, 77.07it/s]
Running evaluation: 7it [00:00, 75.41it/s]
Running evaluation: 7it [00:00, 76.85it/s]
Running evaluation: 7it [00:00, 76.78it/s]
Running evaluation: 7it [00:00, 76.14it/s]
Running evaluation: 7it [00:00, 75.72it/s]
Running evaluation: 7it [00:00, 75.73it/s]
Running evaluation: 7it [00:00, 76.32it/s]
Running evaluation: 7it [00:00, 76.33it/s]
Running evaluation: 7it [00:00, 76.39it/s]
Running evaluation: 7it [00:00, 77.03it/s]
Running evaluation: 7it [00:00, 75.15it/s]
Running evaluation: 7it [00:00, 76.48it/s]
Running evaluation: 7it [00:00, 76.64it/s]
Running evaluation: 7it [00:00, 77.05it/s]
Running evaluation: 7it [00:00, 76.19it/s]
Running evaluation: 7it [00:00, 76.93it/s]
Running evaluation: 7it [00:00, 77.60it/s]
Running evaluation: 7it [00:00, 76.80it/s]
Running evaluation: 7it [00:00, 76.79it/s]
Running evaluation: 7it [00:00, 77.85it/s]
Running evaluation: 7it [00:00, 75.65it/s]
Running evaluation: 7it [00:00, 76.66it/s]
Running evaluation: 7it [00:00, 77.83it/s]
Running evaluation: 7it [00:00, 76.59it/s]
Running evaluation: 7it [00:00, 76.81it/s]
Running evaluation: 7it [00:00, 76.04it/s]
Running evaluation: 7it [00:00, 77.66it/s]
Running evaluation: 7it [00:00, 76.94it/s]
Running evaluation: 7it [00:00, 76.80it/s]
Running evaluation: 7it [00:00, 76.44it/s]
Running evaluation: 7it [00:00, 75.45it/s]
Running evaluation: 7it [00:00, 76.43it/s]
Running evaluation: 7it [00:00, 77.12it/s]
Running evaluation: 7it [00:00, 76.83it/s]
Running evaluation: 7it [00:00, 75.56it/s]
Running evaluation: 7it [00:00, 75.45it/s]
Running evaluation: 7it [00:00, 76.62it/s]
Running evaluation: 7it [00:00, 76.89it/s]
Running evaluation: 7it [00:00, 76.53it/s]
Running evaluation: 7it [00:00, 75.48it/s]
Running evaluation: 7it [00:00, 75.23it/s]
Running evaluation: 7it [00:00, 75.21it/s]
Running evaluation: 7it [00:00, 76.55it/s]
Running evaluation: 7it [00:00, 76.51it/s]
Running evaluation: 7it [00:00, 76.49it/s]
Running evaluation: 7it [00:00, 76.02it/s]
Running evaluation: 7it [00:00, 76.00it/s]
Running evaluation: 7it [00:00, 75.48it/s]
Running evaluation: 7it [00:00, 75.43it/s]
Running evaluation: 7it [00:00, 75.48it/s]
Running evaluation: 7it [00:00, 75.48it/s]
Running evaluation: 7it [00:00, 76.29it/s]
Running evaluation: 7it [00:00, 76.31it/s]
Running evaluation: 7it [00:00, 75.40it/s]
Running evaluation: 7it [00:00, 76.67it/s]
Running evaluation: 7it [00:00, 75.97it/s]
Running evaluation: 7it [00:00, 76.89it/s]
Running evaluation: 7it [00:00, 77.87it/s]
Running evaluation: 7it [00:00, 76.58it/s]
Running evaluation: 7it [00:00, 76.24it/s]
Running evaluation: 7it [00:00, 79.08it/s]
/home/krasul/.env/pytorch/lib/python3.7/site-packages/gluonts-0.5.1.dev236+gb4d8144.d20210210-py3.7.egg/gluonts/evaluation/_base.py:397: RuntimeWarning: divide by zero encountered in double_scalars
  totals["NRMSE"] = totals["RMSE"] / totals["abs_target_mean"]
/home/krasul/.env/pytorch/lib/python3.7/site-packages/gluonts-0.5.1.dev236+gb4d8144.d20210210-py3.7.egg/gluonts/evaluation/_base.py:398: RuntimeWarning: divide by zero encountered in double_scalars
  totals["ND"] = totals["abs_error"] / totals["abs_target_sum"]
/home/krasul/.env/pytorch/lib/python3.7/site-packages/gluonts-0.5.1.dev236+gb4d8144.d20210210-py3.7.egg/gluonts/evaluation/_base.py:402: RuntimeWarning: divide by zero encountered in double_scalars
  totals[quantile.loss_name] / totals["abs_target_sum"]
Running evaluation: 7it [00:00, 75.39it/s]
Running evaluation: 7it [00:00, 76.68it/s]
Running evaluation: 7it [00:00, 76.40it/s]
Running evaluation: 7it [00:00, 75.92it/s]
Running evaluation: 7it [00:00, 76.58it/s]
Running evaluation: 7it [00:00, 76.74it/s]
Running evaluation: 7it [00:00, 76.74it/s]
Running evaluation: 7it [00:00, 76.61it/s]
Running evaluation: 7it [00:00, 76.95it/s]
Running evaluation: 7it [00:00, 75.36it/s]
Running evaluation: 7it [00:00, 76.78it/s]
Running evaluation: 7it [00:00, 76.83it/s]
Running evaluation: 7it [00:00, 76.17it/s]
Running evaluation: 7it [00:00, 76.61it/s]
Running evaluation: 7it [00:00, 76.83it/s]
Running evaluation: 7it [00:00, 75.64it/s]
Running evaluation: 7it [00:00, 75.53it/s]
Running evaluation: 7it [00:00, 75.69it/s]
Running evaluation: 7it [00:00, 76.99it/s]
Running evaluation: 7it [00:00, 76.39it/s]
Running evaluation: 7it [00:00, 77.64it/s]
Running evaluation: 7it [00:00, 76.86it/s]
Running evaluation: 7it [00:00, 57.64it/s]
Running evaluation: 7it [00:00, 66.44it/s]
Running evaluation: 7it [00:00, 75.57it/s]
Running evaluation: 7it [00:00, 76.37it/s]
Running evaluation: 7it [00:00, 75.56it/s]
Running evaluation: 7it [00:00, 76.91it/s]
Running evaluation: 7it [00:00, 77.78it/s]
Running evaluation: 7it [00:00, 75.51it/s]
Running evaluation: 7it [00:00, 76.79it/s]
Running evaluation: 7it [00:00, 76.42it/s]
Running evaluation: 7it [00:00, 76.92it/s]
Running evaluation: 7it [00:00, 76.83it/s]
Running evaluation: 7it [00:00, 77.58it/s]
Running evaluation: 7it [00:00, 72.64it/s]
Running evaluation: 7it [00:00, 75.89it/s]
Running evaluation: 7it [00:00, 76.64it/s]
Running evaluation: 7it [00:00, 77.58it/s]
Running evaluation: 7it [00:00, 75.58it/s]
Running evaluation: 7it [00:00, 28.73it/s]
Running evaluation: 7it [00:00, 71.01it/s]
Running evaluation: 7it [00:00, 77.34it/s]
Running evaluation: 7it [00:00, 76.68it/s]
Running evaluation: 7it [00:00, 75.96it/s]
Running evaluation: 7it [00:00, 76.78it/s]
Running evaluation: 7it [00:00, 77.01it/s]
Running evaluation: 7it [00:00, 41.50it/s]

In [28]:

print("CRPS:", agg_metric["mean_wQuantileLoss"])
print("ND:", agg_metric["ND"])
print("NRMSE:", agg_metric["NRMSE"])
print("")
print("CRPS-Sum:", agg_metric["m_sum_mean_wQuantileLoss"])
print("ND-Sum:", agg_metric["m_sum_ND"])
print("NRMSE-Sum:", agg_metric["m_sum_NRMSE"])

CRPS: 0.05053423468691888
ND: 0.06425762218850603
NRMSE: 0.49794260559943126

CRPS-Sum: 0.02014583598885778
ND-Sum: 0.0265341333895826
NRMSE-Sum: 0.03624724117126602

In [ ]: