pytorch-ts/examples/Time-Grad-Electricity.ipynb at d418d978b0d784a42cab9f8116eec0968fee6c48

mirror of https://github.com/wassname/pytorch-ts.git synced 2026-07-18 12:40:51 +08:00

Files

T

Kashif RasulandGitHub Enterprise 908945b422 Time grad (#28 )

* initial uncond image gaussian diff

TODO make it work for multivariate vector
add conditioning

* remove tqdm

* initial unet

TODO convert to 1d conv

* initial time grad estimator

* initial training

* initial sampling

* added huber loss

* use SinusoidalPosEmb from wavegrad

* use time diff network

* fix reshaping

* fix missing property

* clip false

* updated api

* added padding

* added circular padding

* use linear schedule

* added more schedules

* added back cosine schedule

* Delete Solar-time-grad.ipynb

* updated estimator API

* not tuple

* renamed to EpsilonTheta

* removed

* added example notebook

* removed some output

* fix requirements

* formatting

* added more options to time-grad

* added article

2021-02-11 10:09:25 +01:00

519 KiB

Raw Blame History

In [1]:

%matplotlib inline
import matplotlib.pyplot as plt

import numpy as np
import pandas as pd

import torch

In [2]:

from gluonts.dataset.multivariate_grouper import MultivariateGrouper
from gluonts.dataset.repository.datasets import dataset_recipes, get_dataset
from gluonts.evaluation.backtest import make_evaluation_predictions
from gluonts.evaluation import MultivariateEvaluator

In [ ]:

from pts.model.tempflow import TempFlowEstimator
from pts.model.time_grad import TimeGradEstimator
from pts.model.transformer_tempflow import TransformerTempFlowEstimator
from pts import Trainer

In [4]:

device = torch.device("cuda:1" if torch.cuda.is_available() else "cpu")

In [5]:

def plot(target, forecast, prediction_length, prediction_intervals=(50.0, 90.0), color='g', fname=None):
    label_prefix = ""
    rows = 4
    cols = 4
    fig, axs = plt.subplots(rows, cols, figsize=(24, 24))
    axx = axs.ravel()
    seq_len, target_dim = target.shape
    
    ps = [50.0] + [
            50.0 + f * c / 2.0 for c in prediction_intervals for f in [-1.0, +1.0]
        ]
        
    percentiles_sorted = sorted(set(ps))
    
    def alpha_for_percentile(p):
        return (p / 100.0) ** 0.3
        
    for dim in range(0, min(rows * cols, target_dim)):
        ax = axx[dim]

        target[-2 * prediction_length :][dim].plot(ax=ax)
        
        ps_data = [forecast.quantile(p / 100.0)[:,dim] for p in percentiles_sorted]
        i_p50 = len(percentiles_sorted) // 2
        
        p50_data = ps_data[i_p50]
        p50_series = pd.Series(data=p50_data, index=forecast.index)
        p50_series.plot(color=color, ls="-", label=f"{label_prefix}median", ax=ax)
        
        for i in range(len(percentiles_sorted) // 2):
            ptile = percentiles_sorted[i]
            alpha = alpha_for_percentile(ptile)
            ax.fill_between(
                forecast.index,
                ps_data[i],
                ps_data[-i - 1],
                facecolor=color,
                alpha=alpha,
                interpolate=True,
            )
            # Hack to create labels for the error intervals.
            # Doesn't actually plot anything, because we only pass a single data point
            pd.Series(data=p50_data[:1], index=forecast.index[:1]).plot(
                color=color,
                alpha=alpha,
                linewidth=10,
                label=f"{label_prefix}{100 - ptile * 2}%",
                ax=ax,
            )

    legend = ["observations", "median prediction"] + [f"{k}% prediction interval" for k in prediction_intervals][::-1]    
    axx[0].legend(legend, loc="upper left")
    
    if fname is not None:
        plt.savefig(fname, bbox_inches='tight', pad_inches=0.05)

In [6]:

print(f"Available datasets: {list(dataset_recipes.keys())}")

Available datasets: ['constant', 'exchange_rate', 'solar-energy', 'electricity', 'traffic', 'exchange_rate_nips', 'electricity_nips', 'traffic_nips', 'solar_nips', 'wiki-rolling_nips', 'taxi_30min', 'm3_monthly', 'm3_quarterly', 'm3_yearly', 'm3_other', 'm4_hourly', 'm4_daily', 'm4_weekly', 'm4_monthly', 'm4_quarterly', 'm4_yearly', 'm5']

In [7]:

# exchange_rate_nips, electricity_nips, traffic_nips, solar_nips, wiki-rolling_nips, ## taxi_30min is buggy still
dataset = get_dataset("electricity_nips", regenerate=False)

In [8]:

dataset.metadata

Out [8]:

MetaData(freq='H', target=None, feat_static_cat=[CategoricalFeatureInfo(name='feat_static_cat', cardinality='370')], feat_static_real=[], feat_dynamic_real=[], feat_dynamic_cat=[], prediction_length=24)

In [9]:

train_grouper = MultivariateGrouper(max_target_dim=min(2000, int(dataset.metadata.feat_static_cat[0].cardinality)))

test_grouper = MultivariateGrouper(num_test_dates=int(len(dataset.test)/len(dataset.train)), 
                                   max_target_dim=min(2000, int(dataset.metadata.feat_static_cat[0].cardinality)))

In [10]:

dataset_train = train_grouper(dataset.train)
dataset_test = test_grouper(dataset.test)

/home/krasul/gluon-ts/src/gluonts/dataset/multivariate_grouper.py:182: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.
  return {FieldName.TARGET: np.array([funcs(data) for data in dataset])}

In [ ]:

estimator = TimeGradEstimator(
    target_dim=int(dataset.metadata.feat_static_cat[0].cardinality),
    prediction_length=dataset.metadata.prediction_length,
    context_length=dataset.metadata.prediction_length,
    cell_type='GRU',
    input_size=1484,
    freq=dataset.metadata.freq,
    loss_type='l2',
    scaling=True,
    diff_steps=100,
    beta_end=0.07,
    beta_schedule="quad",
    trainer=Trainer(device=device,
                    epochs=20,
                    learning_rate=1e-3,
                    num_batches_per_epoch=100,
                    batch_size=64,)
)

In [49]:

predictor = estimator.train(dataset_train, num_workers=8)

99it [00:15,  6.36it/s, avg_epoch_loss=0.863, epoch=0]
99it [00:15,  6.34it/s, avg_epoch_loss=0.317, epoch=1]
99it [00:15,  6.40it/s, avg_epoch_loss=0.244, epoch=2]
99it [00:15,  6.30it/s, avg_epoch_loss=0.226, epoch=3]
99it [00:15,  6.36it/s, avg_epoch_loss=0.216, epoch=4]
99it [00:15,  6.38it/s, avg_epoch_loss=0.205, epoch=5]
99it [00:15,  6.41it/s, avg_epoch_loss=0.192, epoch=6]
99it [00:15,  6.44it/s, avg_epoch_loss=0.183, epoch=7]
99it [00:15,  6.40it/s, avg_epoch_loss=0.181, epoch=8]
99it [00:15,  6.32it/s, avg_epoch_loss=0.175, epoch=9]
99it [00:15,  6.36it/s, avg_epoch_loss=0.171, epoch=10]
99it [00:15,  6.31it/s, avg_epoch_loss=0.171, epoch=11]
99it [00:15,  6.37it/s, avg_epoch_loss=0.168, epoch=12]
99it [00:15,  6.39it/s, avg_epoch_loss=0.167, epoch=13]
99it [00:15,  6.34it/s, avg_epoch_loss=0.164, epoch=14]
99it [00:15,  6.46it/s, avg_epoch_loss=0.163, epoch=15]
99it [00:15,  6.34it/s, avg_epoch_loss=0.162, epoch=16]
99it [00:15,  6.35it/s, avg_epoch_loss=0.162, epoch=17]
99it [00:15,  6.35it/s, avg_epoch_loss=0.16, epoch=18] 
99it [00:15,  6.34it/s, avg_epoch_loss=0.159, epoch=19]

In [16]:

forecast_it, ts_it = make_evaluation_predictions(dataset=dataset_test,
                                                 predictor=predictor,
                                                 num_samples=100)

In [17]:

forecasts = list(forecast_it)
targets = list(ts_it)

In [18]:

plot(
    target=targets[0],
    forecast=forecasts[0],
    prediction_length=dataset.metadata.prediction_length,
)
plt.show()

In [19]:

evaluator = MultivariateEvaluator(quantiles=(np.arange(20)/20.0)[1:], 
                                  target_agg_funcs={'sum': np.sum})

In [20]:

agg_metric, item_metrics = evaluator(targets, forecasts, num_series=len(dataset_test))

Running evaluation: 7it [00:00, 110.66it/s]
Running evaluation: 7it [00:00, 116.69it/s]
Running evaluation: 7it [00:00, 119.24it/s]
Running evaluation: 7it [00:00, 121.21it/s]
Running evaluation: 7it [00:00, 118.23it/s]
Running evaluation: 7it [00:00, 118.84it/s]
Running evaluation: 7it [00:00, 118.73it/s]
Running evaluation: 7it [00:00, 120.08it/s]
Running evaluation: 7it [00:00, 119.11it/s]
Running evaluation: 7it [00:00, 118.41it/s]
Running evaluation: 7it [00:00, 118.33it/s]
Running evaluation: 7it [00:00, 118.97it/s]
Running evaluation: 7it [00:00, 107.63it/s]
Running evaluation: 7it [00:00, 119.74it/s]
Running evaluation: 7it [00:00, 119.90it/s]
Running evaluation: 7it [00:00, 118.61it/s]
Running evaluation: 7it [00:00, 118.41it/s]
Running evaluation: 7it [00:00, 101.37it/s]
Running evaluation: 7it [00:00, 118.14it/s]
Running evaluation: 7it [00:00, 115.88it/s]
Running evaluation: 7it [00:00, 28.32it/s]
Running evaluation: 7it [00:00, 121.84it/s]
Running evaluation: 7it [00:00, 118.24it/s]
Running evaluation: 7it [00:00, 120.95it/s]
Running evaluation: 7it [00:00, 117.94it/s]
Running evaluation: 7it [00:00, 118.51it/s]
Running evaluation: 7it [00:00, 117.05it/s]
Running evaluation: 7it [00:00, 117.06it/s]
Running evaluation: 7it [00:00, 117.98it/s]
Running evaluation: 7it [00:00, 120.32it/s]
Running evaluation: 7it [00:00, 118.40it/s]
Running evaluation: 7it [00:00, 119.10it/s]
Running evaluation: 7it [00:00, 118.09it/s]
Running evaluation: 7it [00:00, 108.91it/s]
Running evaluation: 7it [00:00, 118.27it/s]
Running evaluation: 7it [00:00, 118.40it/s]
Running evaluation: 7it [00:00, 120.10it/s]
Running evaluation: 7it [00:00, 118.36it/s]
Running evaluation: 7it [00:00, 118.73it/s]
Running evaluation: 7it [00:00, 118.87it/s]
Running evaluation: 7it [00:00, 115.69it/s]
Running evaluation: 7it [00:00, 120.14it/s]
Running evaluation: 7it [00:00, 118.37it/s]
Running evaluation: 7it [00:00, 118.03it/s]
Running evaluation: 7it [00:00, 117.47it/s]
Running evaluation: 7it [00:00, 118.26it/s]
Running evaluation: 7it [00:00, 118.34it/s]
Running evaluation: 7it [00:00, 120.49it/s]
Running evaluation: 7it [00:00, 118.57it/s]
Running evaluation: 7it [00:00, 116.97it/s]
Running evaluation: 7it [00:00, 118.93it/s]
Running evaluation: 7it [00:00, 116.76it/s]
Running evaluation: 7it [00:00, 118.32it/s]
Running evaluation: 7it [00:00, 118.01it/s]
Running evaluation: 7it [00:00, 115.84it/s]
Running evaluation: 7it [00:00, 117.90it/s]
Running evaluation: 7it [00:00, 117.89it/s]
Running evaluation: 7it [00:00, 118.50it/s]
Running evaluation: 7it [00:00, 108.53it/s]
Running evaluation: 7it [00:00, 117.70it/s]
Running evaluation: 7it [00:00, 115.02it/s]
Running evaluation: 7it [00:00, 117.77it/s]
Running evaluation: 7it [00:00, 117.95it/s]
Running evaluation: 7it [00:00, 116.74it/s]
Running evaluation: 7it [00:00, 119.25it/s]
Running evaluation: 7it [00:00, 116.94it/s]
Running evaluation: 7it [00:00, 118.49it/s]
Running evaluation: 7it [00:00, 118.38it/s]
Running evaluation: 7it [00:00, 118.03it/s]
Running evaluation: 7it [00:00, 118.57it/s]
Running evaluation: 7it [00:00, 116.57it/s]
Running evaluation: 7it [00:00, 117.70it/s]
Running evaluation: 7it [00:00, 118.13it/s]
Running evaluation: 7it [00:00, 118.92it/s]
Running evaluation: 7it [00:00, 118.76it/s]
Running evaluation: 7it [00:00, 116.82it/s]
Running evaluation: 7it [00:00, 117.33it/s]
Running evaluation: 7it [00:00, 117.37it/s]
Running evaluation: 7it [00:00, 118.40it/s]
Running evaluation: 7it [00:00, 117.78it/s]
Running evaluation: 7it [00:00, 115.65it/s]
Running evaluation: 7it [00:00, 117.72it/s]
Running evaluation: 7it [00:00, 109.61it/s]
Running evaluation: 7it [00:00, 117.79it/s]
Running evaluation: 7it [00:00, 117.47it/s]
Running evaluation: 7it [00:00, 119.03it/s]
Running evaluation: 7it [00:00, 119.23it/s]
Running evaluation: 7it [00:00, 118.11it/s]
Running evaluation: 7it [00:00, 119.11it/s]
Running evaluation: 7it [00:00, 117.49it/s]
Running evaluation: 7it [00:00, 116.72it/s]
Running evaluation: 7it [00:00, 116.95it/s]
Running evaluation: 7it [00:00, 118.80it/s]
Running evaluation: 7it [00:00, 118.67it/s]
Running evaluation: 7it [00:00, 117.89it/s]
Running evaluation: 7it [00:00, 117.69it/s]
Running evaluation: 7it [00:00, 120.28it/s]
Running evaluation: 7it [00:00, 116.86it/s]
Running evaluation: 7it [00:00, 118.22it/s]
Running evaluation: 7it [00:00, 118.36it/s]
Running evaluation: 7it [00:00, 114.60it/s]
Running evaluation: 7it [00:00, 118.03it/s]
Running evaluation: 7it [00:00, 118.58it/s]
Running evaluation: 7it [00:00, 119.84it/s]
Running evaluation: 7it [00:00, 117.54it/s]
Running evaluation: 7it [00:00, 117.31it/s]
Running evaluation: 7it [00:00, 96.81it/s]
Running evaluation: 7it [00:00, 119.03it/s]
Running evaluation: 7it [00:00, 118.68it/s]
Running evaluation: 7it [00:00, 116.88it/s]
Running evaluation: 7it [00:00, 117.91it/s]
Running evaluation: 7it [00:00, 117.48it/s]
Running evaluation: 7it [00:00, 117.79it/s]
Running evaluation: 7it [00:00, 116.98it/s]
Running evaluation: 7it [00:00, 118.16it/s]
Running evaluation: 7it [00:00, 117.49it/s]
Running evaluation: 7it [00:00, 116.87it/s]
Running evaluation: 7it [00:00, 118.42it/s]
Running evaluation: 7it [00:00, 118.02it/s]
Running evaluation: 7it [00:00, 118.17it/s]
Running evaluation: 7it [00:00, 114.16it/s]
Running evaluation: 7it [00:00, 115.98it/s]
Running evaluation: 7it [00:00, 120.77it/s]
Running evaluation: 7it [00:00, 120.67it/s]
Running evaluation: 7it [00:00, 117.72it/s]
Running evaluation: 7it [00:00, 116.98it/s]
Running evaluation: 7it [00:00, 117.48it/s]
Running evaluation: 7it [00:00, 118.91it/s]
Running evaluation: 7it [00:00, 119.93it/s]
Running evaluation: 7it [00:00, 118.85it/s]
Running evaluation: 7it [00:00, 107.11it/s]
Running evaluation: 7it [00:00, 40.64it/s]
Running evaluation: 7it [00:00, 118.55it/s]
Running evaluation: 7it [00:00, 121.31it/s]
Running evaluation: 7it [00:00, 117.04it/s]
Running evaluation: 7it [00:00, 118.61it/s]
Running evaluation: 7it [00:00, 118.11it/s]
Running evaluation: 7it [00:00, 117.37it/s]
Running evaluation: 7it [00:00, 118.33it/s]
Running evaluation: 7it [00:00, 120.00it/s]
Running evaluation: 7it [00:00, 118.55it/s]
Running evaluation: 7it [00:00, 115.61it/s]
Running evaluation: 7it [00:00, 119.91it/s]
Running evaluation: 7it [00:00, 118.26it/s]
Running evaluation: 7it [00:00, 121.18it/s]
Running evaluation: 7it [00:00, 118.07it/s]
Running evaluation: 7it [00:00, 118.80it/s]
Running evaluation: 7it [00:00, 118.59it/s]
Running evaluation: 7it [00:00, 118.67it/s]
Running evaluation: 7it [00:00, 116.73it/s]
Running evaluation: 7it [00:00, 116.35it/s]
Running evaluation: 7it [00:00, 118.74it/s]
Running evaluation: 7it [00:00, 110.16it/s]
Running evaluation: 7it [00:00, 117.06it/s]
Running evaluation: 7it [00:00, 117.80it/s]
Running evaluation: 7it [00:00, 116.70it/s]
Running evaluation: 7it [00:00, 116.65it/s]
Running evaluation: 7it [00:00, 120.57it/s]
Running evaluation: 7it [00:00, 116.61it/s]
Running evaluation: 7it [00:00, 118.10it/s]
Running evaluation: 7it [00:00, 116.66it/s]
Running evaluation: 7it [00:00, 116.98it/s]
Running evaluation: 7it [00:00, 117.34it/s]
Running evaluation: 7it [00:00, 116.52it/s]
Running evaluation: 7it [00:00, 115.50it/s]
Running evaluation: 7it [00:00, 117.97it/s]
Running evaluation: 7it [00:00, 115.41it/s]
Running evaluation: 7it [00:00, 118.62it/s]
Running evaluation: 7it [00:00, 116.48it/s]
Running evaluation: 7it [00:00, 117.51it/s]
Running evaluation: 7it [00:00, 116.68it/s]
Running evaluation: 7it [00:00, 115.80it/s]
Running evaluation: 7it [00:00, 120.82it/s]
Running evaluation: 7it [00:00, 117.80it/s]
Running evaluation: 7it [00:00, 118.11it/s]
Running evaluation: 7it [00:00, 120.39it/s]
Running evaluation: 7it [00:00, 112.03it/s]
Running evaluation: 7it [00:00, 116.52it/s]
Running evaluation: 7it [00:00, 118.08it/s]
Running evaluation: 7it [00:00, 118.06it/s]
Running evaluation: 7it [00:00, 116.41it/s]
Running evaluation: 7it [00:00, 116.53it/s]
Running evaluation: 7it [00:00, 118.24it/s]
Running evaluation: 7it [00:00, 116.86it/s]
Running evaluation: 7it [00:00, 116.87it/s]
Running evaluation: 7it [00:00, 116.78it/s]
Running evaluation: 7it [00:00, 117.23it/s]
Running evaluation: 7it [00:00, 116.57it/s]
Running evaluation: 7it [00:00, 118.17it/s]
Running evaluation: 7it [00:00, 116.79it/s]
Running evaluation: 7it [00:00, 117.72it/s]
Running evaluation: 7it [00:00, 114.68it/s]
Running evaluation: 7it [00:00, 116.86it/s]
Running evaluation: 7it [00:00, 117.34it/s]
Running evaluation: 7it [00:00, 119.74it/s]
Running evaluation: 7it [00:00, 116.96it/s]
Running evaluation: 7it [00:00, 118.04it/s]
Running evaluation: 7it [00:00, 117.02it/s]
Running evaluation: 7it [00:00, 116.81it/s]
Running evaluation: 7it [00:00, 117.71it/s]
Running evaluation: 7it [00:00, 110.81it/s]
Running evaluation: 7it [00:00, 110.88it/s]
Running evaluation: 7it [00:00, 119.42it/s]
Running evaluation: 7it [00:00, 117.16it/s]
Running evaluation: 7it [00:00, 116.88it/s]
Running evaluation: 7it [00:00, 116.94it/s]
Running evaluation: 7it [00:00, 117.95it/s]
Running evaluation: 7it [00:00, 118.53it/s]
Running evaluation: 7it [00:00, 118.25it/s]
Running evaluation: 7it [00:00, 117.25it/s]
Running evaluation: 7it [00:00, 117.24it/s]
Running evaluation: 7it [00:00, 111.61it/s]
Running evaluation: 7it [00:00, 117.66it/s]
Running evaluation: 7it [00:00, 116.48it/s]
Running evaluation: 7it [00:00, 118.91it/s]
Running evaluation: 7it [00:00, 116.96it/s]
Running evaluation: 7it [00:00, 117.45it/s]
Running evaluation: 7it [00:00, 117.72it/s]
Running evaluation: 7it [00:00, 117.11it/s]
Running evaluation: 7it [00:00, 119.03it/s]
Running evaluation: 7it [00:00, 117.27it/s]
Running evaluation: 7it [00:00, 115.85it/s]
Running evaluation: 7it [00:00, 116.81it/s]
Running evaluation: 7it [00:00, 114.83it/s]
Running evaluation: 7it [00:00, 114.51it/s]
Running evaluation: 7it [00:00, 117.55it/s]
Running evaluation: 7it [00:00, 117.61it/s]
Running evaluation: 7it [00:00, 113.01it/s]
Running evaluation: 7it [00:00, 120.31it/s]
Running evaluation: 7it [00:00, 116.42it/s]
Running evaluation: 7it [00:00, 117.58it/s]
Running evaluation: 7it [00:00, 115.97it/s]
Running evaluation: 7it [00:00, 115.95it/s]
Running evaluation: 7it [00:00, 116.68it/s]
Running evaluation: 7it [00:00, 117.57it/s]
Running evaluation: 7it [00:00, 116.45it/s]
Running evaluation: 7it [00:00, 117.27it/s]
Running evaluation: 7it [00:00, 117.23it/s]
Running evaluation: 7it [00:00, 117.03it/s]
Running evaluation: 7it [00:00, 116.86it/s]
Running evaluation: 7it [00:00, 117.04it/s]
Running evaluation: 7it [00:00, 113.78it/s]
Running evaluation: 7it [00:00, 118.45it/s]
Running evaluation: 7it [00:00, 117.12it/s]
Running evaluation: 7it [00:00, 115.88it/s]
Running evaluation: 7it [00:00, 117.78it/s]
Running evaluation: 7it [00:00, 116.61it/s]
Running evaluation: 7it [00:00, 117.86it/s]
Running evaluation: 7it [00:00, 110.49it/s]
Running evaluation: 7it [00:00, 117.58it/s]
Running evaluation: 7it [00:00, 117.49it/s]
Running evaluation: 7it [00:00, 117.21it/s]
Running evaluation: 7it [00:00, 116.86it/s]
Running evaluation: 7it [00:00, 117.37it/s]
Running evaluation: 7it [00:00, 117.20it/s]
Running evaluation: 7it [00:00, 117.37it/s]
Running evaluation: 7it [00:00, 117.49it/s]
Running evaluation: 7it [00:00, 116.85it/s]
Running evaluation: 7it [00:00, 118.58it/s]
Running evaluation: 7it [00:00, 116.89it/s]
Running evaluation: 7it [00:00, 117.42it/s]
Running evaluation: 7it [00:00, 115.11it/s]
Running evaluation: 7it [00:00, 118.88it/s]
Running evaluation: 7it [00:00, 119.35it/s]
Running evaluation: 7it [00:00, 117.78it/s]
Running evaluation: 7it [00:00, 119.83it/s]
Running evaluation: 7it [00:00, 118.00it/s]
Running evaluation: 7it [00:00, 119.65it/s]
Running evaluation: 7it [00:00, 118.36it/s]
Running evaluation: 7it [00:00, 118.52it/s]
Running evaluation: 7it [00:00, 120.59it/s]
Running evaluation: 7it [00:00, 116.53it/s]
Running evaluation: 7it [00:00, 31.71it/s]
Running evaluation: 7it [00:00, 118.50it/s]
Running evaluation: 7it [00:00, 118.19it/s]
Running evaluation: 7it [00:00, 117.17it/s]
Running evaluation: 7it [00:00, 117.32it/s]
Running evaluation: 7it [00:00, 117.35it/s]
Running evaluation: 7it [00:00, 118.49it/s]
Running evaluation: 7it [00:00, 117.33it/s]
Running evaluation: 7it [00:00, 116.93it/s]
Running evaluation: 7it [00:00, 116.40it/s]
Running evaluation: 7it [00:00, 117.52it/s]
Running evaluation: 7it [00:00, 120.22it/s]
Running evaluation: 7it [00:00, 116.98it/s]
Running evaluation: 7it [00:00, 118.68it/s]
Running evaluation: 7it [00:00, 115.84it/s]
Running evaluation: 7it [00:00, 118.47it/s]
Running evaluation: 7it [00:00, 118.44it/s]
Running evaluation: 7it [00:00, 118.64it/s]
Running evaluation: 7it [00:00, 116.94it/s]
Running evaluation: 7it [00:00, 117.16it/s]
Running evaluation: 7it [00:00, 116.65it/s]
Running evaluation: 7it [00:00, 115.48it/s]
Running evaluation: 7it [00:00, 117.40it/s]
Running evaluation: 7it [00:00, 117.23it/s]
Running evaluation: 7it [00:00, 120.67it/s]
Running evaluation: 7it [00:00, 120.90it/s]
Running evaluation: 7it [00:00, 117.75it/s]
Running evaluation: 7it [00:00, 117.40it/s]
Running evaluation: 7it [00:00, 117.04it/s]
Running evaluation: 7it [00:00, 117.14it/s]
Running evaluation: 7it [00:00, 116.18it/s]
Running evaluation: 7it [00:00, 117.74it/s]
Running evaluation: 7it [00:00, 116.45it/s]
Running evaluation: 7it [00:00, 115.34it/s]
Running evaluation: 7it [00:00, 117.22it/s]
Running evaluation: 7it [00:00, 116.89it/s]
Running evaluation: 7it [00:00, 116.88it/s]
Running evaluation: 7it [00:00, 118.24it/s]
Running evaluation: 7it [00:00, 119.06it/s]
Running evaluation: 7it [00:00, 117.20it/s]
Running evaluation: 7it [00:00, 117.75it/s]
Running evaluation: 7it [00:00, 118.55it/s]
Running evaluation: 7it [00:00, 118.29it/s]
Running evaluation: 7it [00:00, 115.44it/s]
Running evaluation: 7it [00:00, 119.40it/s]
Running evaluation: 7it [00:00, 118.46it/s]
Running evaluation: 7it [00:00, 117.51it/s]
Running evaluation: 7it [00:00, 117.25it/s]
Running evaluation: 7it [00:00, 117.18it/s]
Running evaluation: 7it [00:00, 118.72it/s]
Running evaluation: 7it [00:00, 114.29it/s]
Running evaluation: 7it [00:00, 115.78it/s]
Running evaluation: 7it [00:00, 118.55it/s]
Running evaluation: 7it [00:00, 115.66it/s]
Running evaluation: 7it [00:00, 120.82it/s]
Running evaluation: 7it [00:00, 115.64it/s]
Running evaluation: 7it [00:00, 117.12it/s]
Running evaluation: 7it [00:00, 117.03it/s]
Running evaluation: 7it [00:00, 119.17it/s]
Running evaluation: 7it [00:00, 117.26it/s]
Running evaluation: 7it [00:00, 114.82it/s]
Running evaluation: 7it [00:00, 118.27it/s]
Running evaluation: 7it [00:00, 117.11it/s]
Running evaluation: 7it [00:00, 116.81it/s]
Running evaluation: 7it [00:00, 118.46it/s]
Running evaluation: 7it [00:00, 118.31it/s]
Running evaluation: 7it [00:00, 117.35it/s]
Running evaluation: 7it [00:00, 113.01it/s]
Running evaluation: 7it [00:00, 118.19it/s]
Running evaluation: 7it [00:00, 118.30it/s]
Running evaluation: 7it [00:00, 116.63it/s]
Running evaluation: 7it [00:00, 117.83it/s]
Running evaluation: 7it [00:00, 116.38it/s]
Running evaluation: 7it [00:00, 119.78it/s]
Running evaluation: 7it [00:00, 118.91it/s]
Running evaluation: 7it [00:00, 117.37it/s]
Running evaluation: 7it [00:00, 118.89it/s]
Running evaluation: 7it [00:00, 118.34it/s]
Running evaluation: 7it [00:00, 116.58it/s]
Running evaluation: 7it [00:00, 120.75it/s]
Running evaluation: 7it [00:00, 114.82it/s]
Running evaluation: 7it [00:00, 116.61it/s]
Running evaluation: 7it [00:00, 116.16it/s]
Running evaluation: 7it [00:00, 120.27it/s]
Running evaluation: 7it [00:00, 116.51it/s]
Running evaluation: 7it [00:00, 118.83it/s]
Running evaluation: 7it [00:00, 117.04it/s]
Running evaluation: 7it [00:00, 116.09it/s]
Running evaluation: 7it [00:00, 118.68it/s]
Running evaluation: 7it [00:00, 116.25it/s]
Running evaluation: 7it [00:00, 115.67it/s]
Running evaluation: 7it [00:00, 114.16it/s]
Running evaluation: 7it [00:00, 119.47it/s]
Running evaluation: 7it [00:00, 118.00it/s]
Running evaluation: 7it [00:00, 118.71it/s]
Running evaluation: 7it [00:00, 117.06it/s]
Running evaluation: 7it [00:00, 118.40it/s]
Running evaluation: 7it [00:00, 118.64it/s]
Running evaluation: 7it [00:00, 56.98it/s]

In [21]:

print("CRPS:", agg_metric["mean_wQuantileLoss"])
print("ND:", agg_metric["ND"])
print("NRMSE:", agg_metric["NRMSE"])
print("")
print("CRPS-Sum:", agg_metric["m_sum_mean_wQuantileLoss"])
print("ND-Sum:", agg_metric["m_sum_ND"])
print("NRMSE-Sum:", agg_metric["m_sum_NRMSE"])

CRPS: 0.04990688413586434
ND: 0.06389410998642815
NRMSE: 0.5150398390716022

CRPS-Sum: 0.020686286420912865
ND-Sum: 0.02713582247461173
NRMSE-Sum: 0.038176941001114524

In [ ]:

519 KiB Raw Blame History

519 KiB

Raw Blame History