{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "ExecuteTime": { "end_time": "2022-12-23T09:43:54.678816Z", "start_time": "2022-12-23T09:43:54.672175Z" } }, "outputs": [], "source": [ "# import warnings\n", "# warnings.simplefilter(\"ignore\")\n", "\n", "# autoreload import your package\n", "%load_ext autoreload\n", "%autoreload 2" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "ExecuteTime": { "end_time": "2022-12-23T09:43:55.584475Z", "start_time": "2022-12-23T09:43:54.679524Z" } }, "outputs": [], "source": [ "%matplotlib inline\n", "import matplotlib.pyplot as plt\n", "\n", "import numpy as np\n", "import pandas as pd\n", "\n", "import torch" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "ExecuteTime": { "end_time": "2022-12-23T09:43:55.755092Z", "start_time": "2022-12-23T09:43:55.585959Z" } }, "outputs": [], "source": [ "from gluonts.dataset.multivariate_grouper import MultivariateGrouper\n", "from gluonts.dataset.repository.datasets import dataset_recipes, get_dataset\n", "from gluonts.evaluation.backtest import make_evaluation_predictions\n", "from gluonts.evaluation import MultivariateEvaluator" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "ExecuteTime": { "end_time": "2022-12-23T09:43:55.836294Z", "start_time": "2022-12-23T09:43:55.756050Z" } }, "outputs": [], "source": [ "from pts.model.tempflow import TempFlowEstimator\n", "from pts.model.time_grad2 import TimeGradEstimator2\n", "from pts.model.transformer_tempflow import TransformerTempFlowEstimator\n", "from pts import Trainer" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "ExecuteTime": { "end_time": "2022-12-23T09:43:55.875122Z", "start_time": "2022-12-23T09:43:55.837634Z" } }, "outputs": [], "source": [ "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", "device" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Data" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "ExecuteTime": { "end_time": "2022-12-23T09:43:55.890801Z", "start_time": "2022-12-23T09:43:55.875991Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Available datasets: ['constant', 'exchange_rate', 'solar-energy', 'electricity', 'traffic', 'exchange_rate_nips', 'electricity_nips', 'traffic_nips', 'solar_nips', 'wiki-rolling_nips', 'taxi_30min', 'kaggle_web_traffic_with_missing', 'kaggle_web_traffic_without_missing', 'kaggle_web_traffic_weekly', 'm1_yearly', 'm1_quarterly', 'm1_monthly', 'nn5_daily_with_missing', 'nn5_daily_without_missing', 'nn5_weekly', 'tourism_monthly', 'tourism_quarterly', 'tourism_yearly', 'cif_2016', 'london_smart_meters_without_missing', 'wind_farms_without_missing', 'car_parts_without_missing', 'dominick', 'fred_md', 'pedestrian_counts', 'hospital', 'covid_deaths', 'kdd_cup_2018_without_missing', 'weather', 'm3_monthly', 'm3_quarterly', 'm3_yearly', 'm3_other', 'm4_hourly', 'm4_daily', 'm4_weekly', 'm4_monthly', 'm4_quarterly', 'm4_yearly', 'm5', 'uber_tlc_daily', 'uber_tlc_hourly', 'airpassengers']\n" ] } ], "source": [ "print(f\"Available datasets: {list(dataset_recipes.keys())}\")" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "ExecuteTime": { "end_time": "2022-12-23T09:43:55.906924Z", "start_time": "2022-12-23T09:43:55.891681Z" }, "scrolled": true }, "outputs": [], "source": [ "# exchange_rate_nips, electricity_nips, traffic_nips, solar_nips, wiki-rolling_nips, ## taxi_30min is buggy still\n", "dataset = get_dataset(\"electricity_nips\", regenerate=False)" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "ExecuteTime": { "end_time": "2022-12-23T09:43:55.923232Z", "start_time": "2022-12-23T09:43:55.907597Z" } }, "outputs": [ { "data": { "text/plain": [ "MetaData(freq='H', target=None, feat_static_cat=[CategoricalFeatureInfo(name='feat_static_cat_0', cardinality='370')], feat_static_real=[], feat_dynamic_real=[], feat_dynamic_cat=[], prediction_length=24)" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "dataset.metadata" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "ExecuteTime": { "end_time": "2022-12-23T09:43:56.450920Z", "start_time": "2022-12-23T09:43:55.924531Z" } }, "outputs": [], "source": [ "train_grouper = MultivariateGrouper(max_target_dim=min(2000, int(dataset.metadata.feat_static_cat[0].cardinality)))\n", "\n", "test_grouper = MultivariateGrouper(\n", " num_test_dates=int(len(dataset.test)/len(dataset.train)*2),\n", "# num_test_dates=7,\n", " max_target_dim=min(2000, int(dataset.metadata.feat_static_cat[0].cardinality)))" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "ExecuteTime": { "end_time": "2022-12-23T09:43:59.956426Z", "start_time": "2022-12-23T09:43:56.451881Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/wassname/miniforge3/envs/glounts/lib/python3.9/site-packages/gluonts/dataset/multivariate_grouper.py:191: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.\n", " return {FieldName.TARGET: np.array([funcs(data) for data in dataset])}\n" ] } ], "source": [ "dataset_train = train_grouper(dataset.train)\n", "dataset_test = test_grouper(dataset.test)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Train" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "ExecuteTime": { "end_time": "2022-12-23T09:43:59.970082Z", "start_time": "2022-12-23T09:43:59.957922Z" } }, "outputs": [], "source": [ "estimator = TimeGradEstimator2(\n", " target_dim=int(dataset.metadata.feat_static_cat[0].cardinality),\n", " prediction_length=dataset.metadata.prediction_length,\n", " context_length=dataset.metadata.prediction_length,\n", " cell_type='GRU',\n", " input_size=1484,\n", " freq=dataset.metadata.freq,\n", " loss_type='l2',\n", " scaling=True,\n", " diff_steps=100,\n", " beta_end=0.1,\n", " beta_schedule=\"linear\",\n", " trainer=Trainer(device=device,\n", " epochs=20,\n", " learning_rate=1e-3,\n", " num_batches_per_epoch=100,\n", " batch_size=64,)\n", ")" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "ExecuteTime": { "end_time": "2022-12-23T11:56:39.121839Z", "start_time": "2022-12-23T09:43:59.971197Z" }, "scrolled": true }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "a16f9f9d61ef42e7beca0a83aad84d5f", "version_major": 2, "version_minor": 0 }, "text/plain": [ " 0%| | 0/99 [00:00" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "def plot(target, forecast, prediction_length, prediction_intervals=(50.0, 90.0), color='g', fname=None):\n", " label_prefix = \"\"\n", " rows = 4\n", " cols = 4\n", " fig, axs = plt.subplots(rows, cols, figsize=(24, 24))\n", " axx = axs.ravel()\n", " seq_len, target_dim = target.shape\n", " \n", " ps = [50.0] + [\n", " 50.0 + f * c / 2.0 for c in prediction_intervals for f in [-1.0, +1.0]\n", " ]\n", " \n", " percentiles_sorted = sorted(set(ps))\n", " \n", " def alpha_for_percentile(p):\n", " return (p / 100.0) ** 0.3\n", " \n", " for dim in range(0, min(rows * cols, target_dim)):\n", " ax = axx[dim]\n", "\n", " target[-2 * prediction_length :][dim].plot(ax=ax)\n", " \n", " ps_data = [forecast.quantile(p / 100.0)[:,dim] for p in percentiles_sorted]\n", " i_p50 = len(percentiles_sorted) // 2\n", " \n", " p50_data = ps_data[i_p50]\n", " p50_series = pd.Series(data=p50_data, index=forecast.index)\n", " p50_series.plot(color=color, ls=\"-\", label=f\"{label_prefix}median\", ax=ax)\n", " \n", " for i in range(len(percentiles_sorted) // 2):\n", " ptile = percentiles_sorted[i]\n", " alpha = alpha_for_percentile(ptile)\n", " ax.fill_between(\n", " forecast.index,\n", " ps_data[i],\n", " ps_data[-i - 1],\n", " facecolor=color,\n", " alpha=alpha,\n", " interpolate=True,\n", " )\n", " # Hack to create labels for the error intervals.\n", " # Doesn't actually plot anything, because we only pass a single data point\n", " pd.Series(data=p50_data[:1], index=forecast.index[:1]).plot(\n", " color=color,\n", " alpha=alpha,\n", " linewidth=10,\n", " label=f\"{label_prefix}{100 - ptile * 2}%\",\n", " ax=ax,\n", " )\n", "\n", " legend = [\"observations\", \"median prediction\"] + [f\"{k}% prediction interval\" for k in prediction_intervals][::-1] \n", " axx[0].legend(legend, loc=\"upper left\")\n", " \n", " if fname is not None:\n", " plt.savefig(fname, bbox_inches='tight', pad_inches=0.05)\n", " \n", "plot(\n", " target=targets[0],\n", " forecast=forecasts[0],\n", " prediction_length=dataset.metadata.prediction_length,\n", ")\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": 31, "metadata": { "ExecuteTime": { "end_time": "2022-12-23T13:52:10.714194Z", "start_time": "2022-12-23T13:52:10.699899Z" } }, "outputs": [], "source": [ "evaluator = MultivariateEvaluator(quantiles=(np.arange(20)/20.0)[1:], \n", " target_agg_funcs={'sum': np.sum})" ] }, { "cell_type": "code", "execution_count": 32, "metadata": { "ExecuteTime": { "end_time": "2022-12-23T13:52:28.912219Z", "start_time": "2022-12-23T13:52:10.715121Z" }, "scrolled": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Running evaluation: 7it [00:00, 151.21it/s]\n", "Running evaluation: 7it [00:00, 156.67it/s]\n", "Running evaluation: 7it [00:00, 192.62it/s]\n", "Running evaluation: 7it [00:00, 192.32it/s]\n", "Running evaluation: 7it [00:00, 188.02it/s]\n", "Running evaluation: 7it [00:00, 193.28it/s]\n", "Running evaluation: 7it [00:00, 195.61it/s]\n", "Running evaluation: 7it [00:00, 195.35it/s]\n", "Running evaluation: 7it [00:00, 193.99it/s]\n", "Running evaluation: 7it [00:00, 193.99it/s]\n", "Running evaluation: 7it [00:00, 194.13it/s]\n", "Running evaluation: 7it [00:00, 185.22it/s]\n", "Running evaluation: 7it [00:00, 192.47it/s]\n", "Running evaluation: 7it [00:00, 194.77it/s]\n", "Running evaluation: 7it [00:00, 194.23it/s]\n", "Running evaluation: 7it [00:00, 193.09it/s]\n", "Running evaluation: 7it [00:00, 192.30it/s]\n", "Running evaluation: 7it [00:00, 194.60it/s]\n", "Running evaluation: 7it [00:00, 194.02it/s]\n", "Running evaluation: 7it [00:00, 194.20it/s]\n", "Running evaluation: 7it [00:00, 166.73it/s]\n", "Running evaluation: 7it [00:00, 190.92it/s]\n", "Running evaluation: 7it [00:00, 156.11it/s]\n", "Running evaluation: 7it [00:00, 194.57it/s]\n", "Running evaluation: 7it [00:00, 194.46it/s]\n", "Running evaluation: 7it [00:00, 183.93it/s]\n", "Running evaluation: 7it [00:00, 193.86it/s]\n", "Running evaluation: 7it [00:00, 191.97it/s]\n", "Running evaluation: 7it [00:00, 192.15it/s]\n", "Running evaluation: 7it [00:00, 193.87it/s]\n", "Running evaluation: 7it [00:00, 194.17it/s]\n", "Running evaluation: 7it [00:00, 193.41it/s]\n", "Running evaluation: 7it [00:00, 192.87it/s]\n", "Running evaluation: 7it [00:00, 154.46it/s]\n", "Running evaluation: 7it [00:00, 188.58it/s]\n", "Running evaluation: 7it [00:00, 185.73it/s]\n", "Running evaluation: 7it [00:00, 193.99it/s]\n", "Running evaluation: 7it [00:00, 195.03it/s]\n", "Running evaluation: 7it [00:00, 192.94it/s]\n", "Running evaluation: 7it [00:00, 192.92it/s]\n", "Running evaluation: 7it [00:00, 194.53it/s]\n", "Running evaluation: 7it [00:00, 193.95it/s]\n", "Running evaluation: 7it [00:00, 194.04it/s]\n", "Running evaluation: 7it [00:00, 191.67it/s]\n", "Running evaluation: 7it [00:00, 190.08it/s]\n", "Running evaluation: 7it [00:00, 184.91it/s]\n", "Running evaluation: 7it [00:00, 186.52it/s]\n", "Running evaluation: 7it [00:00, 192.02it/s]\n", "Running evaluation: 7it [00:00, 193.40it/s]\n", "Running evaluation: 7it [00:00, 192.95it/s]\n", "Running evaluation: 7it [00:00, 188.36it/s]\n", "Running evaluation: 7it [00:00, 187.79it/s]\n", "Running evaluation: 7it [00:00, 184.49it/s]\n", "Running evaluation: 7it [00:00, 191.97it/s]\n", "Running evaluation: 7it [00:00, 189.43it/s]\n", "Running evaluation: 7it [00:00, 185.60it/s]\n", "Running evaluation: 7it [00:00, 190.13it/s]\n", "Running evaluation: 7it [00:00, 159.38it/s]\n", "Running evaluation: 7it [00:00, 195.38it/s]\n", "Running evaluation: 7it [00:00, 192.07it/s]\n", "Running evaluation: 7it [00:00, 195.75it/s]\n", "Running evaluation: 7it [00:00, 192.39it/s]\n", "Running evaluation: 7it [00:00, 194.80it/s]\n", "Running evaluation: 7it [00:00, 193.92it/s]\n", "Running evaluation: 7it [00:00, 193.92it/s]\n", "Running evaluation: 7it [00:00, 194.20it/s]\n", "Running evaluation: 7it [00:00, 193.85it/s]\n", "Running evaluation: 7it [00:00, 194.09it/s]\n", "Running evaluation: 7it [00:00, 186.10it/s]\n", "Running evaluation: 7it [00:00, 191.88it/s]\n", "Running evaluation: 7it [00:00, 191.76it/s]\n", "Running evaluation: 7it [00:00, 192.69it/s]\n", "Running evaluation: 7it [00:00, 189.27it/s]\n", "Running evaluation: 7it [00:00, 190.78it/s]\n", "Running evaluation: 7it [00:00, 186.90it/s]\n", "Running evaluation: 7it [00:00, 188.99it/s]\n", "Running evaluation: 7it [00:00, 184.75it/s]\n", "Running evaluation: 7it [00:00, 195.41it/s]\n", "Running evaluation: 7it [00:00, 194.39it/s]\n", "Running evaluation: 7it [00:00, 193.03it/s]\n", "Running evaluation: 7it [00:00, 189.73it/s]\n", "Running evaluation: 7it [00:00, 187.00it/s]\n", "Running evaluation: 7it [00:00, 184.57it/s]\n", "Running evaluation: 7it [00:00, 186.42it/s]\n", "Running evaluation: 7it [00:00, 183.87it/s]\n", "Running evaluation: 7it [00:00, 185.84it/s]\n", "Running evaluation: 7it [00:00, 190.44it/s]\n", "Running evaluation: 7it [00:00, 182.26it/s]\n", "Running evaluation: 7it [00:00, 189.75it/s]\n", "Running evaluation: 7it [00:00, 188.45it/s]\n", "Running evaluation: 7it [00:00, 182.41it/s]\n", "Running evaluation: 7it [00:00, 193.84it/s]\n", "Running evaluation: 7it [00:00, 195.02it/s]\n", "Running evaluation: 7it [00:00, 195.07it/s]\n", "Running evaluation: 7it [00:00, 191.71it/s]\n", "Running evaluation: 7it [00:00, 194.79it/s]\n", "Running evaluation: 7it [00:00, 193.47it/s]\n", "Running evaluation: 7it [00:00, 191.42it/s]\n", "Running evaluation: 7it [00:00, 192.02it/s]\n", "Running evaluation: 7it [00:00, 188.63it/s]\n", "Running evaluation: 7it [00:00, 188.76it/s]\n", "Running evaluation: 7it [00:00, 192.92it/s]\n", "Running evaluation: 7it [00:00, 195.18it/s]\n", "Running evaluation: 7it [00:00, 194.27it/s]\n", "Running evaluation: 7it [00:00, 194.46it/s]\n", "Running evaluation: 7it [00:00, 193.51it/s]\n", "Running evaluation: 7it [00:00, 192.42it/s]\n", "Running evaluation: 7it [00:00, 193.74it/s]\n", "Running evaluation: 7it [00:00, 191.62it/s]\n", "Running evaluation: 7it [00:00, 153.47it/s]\n", "Running evaluation: 7it [00:00, 179.22it/s]\n", "Running evaluation: 7it [00:00, 192.83it/s]\n", "Running evaluation: 7it [00:00, 189.19it/s]\n", "Running evaluation: 7it [00:00, 191.59it/s]\n", "Running evaluation: 7it [00:00, 190.83it/s]\n", "Running evaluation: 7it [00:00, 191.73it/s]\n", "Running evaluation: 7it [00:00, 191.36it/s]\n", "Running evaluation: 7it [00:00, 191.07it/s]\n", "Running evaluation: 7it [00:00, 192.73it/s]\n", "Running evaluation: 7it [00:00, 190.68it/s]\n", "Running evaluation: 7it [00:00, 190.55it/s]\n", "Running evaluation: 7it [00:00, 190.33it/s]\n", "Running evaluation: 7it [00:00, 190.45it/s]\n", "Running evaluation: 7it [00:00, 192.55it/s]\n", "Running evaluation: 7it [00:00, 192.68it/s]\n", "Running evaluation: 7it [00:00, 191.28it/s]\n", "Running evaluation: 7it [00:00, 190.88it/s]\n", "Running evaluation: 7it [00:00, 190.84it/s]\n", "Running evaluation: 7it [00:00, 191.08it/s]\n", "Running evaluation: 7it [00:00, 192.09it/s]\n", "Running evaluation: 7it [00:00, 191.97it/s]\n", "Running evaluation: 7it [00:00, 191.10it/s]\n", "Running evaluation: 7it [00:00, 190.62it/s]\n", "Running evaluation: 7it [00:00, 191.28it/s]\n", "Running evaluation: 7it [00:00, 190.71it/s]\n", "Running evaluation: 7it [00:00, 163.89it/s]\n", "Running evaluation: 7it [00:00, 192.57it/s]\n", "Running evaluation: 7it [00:00, 192.81it/s]\n", "Running evaluation: 7it [00:00, 191.10it/s]\n", "Running evaluation: 7it [00:00, 192.91it/s]\n", "Running evaluation: 7it [00:00, 186.25it/s]\n", "Running evaluation: 7it [00:00, 190.11it/s]\n", "Running evaluation: 7it [00:00, 191.45it/s]\n", "Running evaluation: 7it [00:00, 189.58it/s]\n", "Running evaluation: 7it [00:00, 193.55it/s]\n", "Running evaluation: 7it [00:00, 190.67it/s]\n", "Running evaluation: 7it [00:00, 190.86it/s]\n", "Running evaluation: 7it [00:00, 191.36it/s]\n", "Running evaluation: 7it [00:00, 190.19it/s]\n", "Running evaluation: 7it [00:00, 191.80it/s]\n", "Running evaluation: 7it [00:00, 192.14it/s]\n", "Running evaluation: 7it [00:00, 190.02it/s]\n", "Running evaluation: 7it [00:00, 186.95it/s]\n", "Running evaluation: 7it [00:00, 191.14it/s]\n", "Running evaluation: 7it [00:00, 191.97it/s]\n", "Running evaluation: 7it [00:00, 192.12it/s]\n", "Running evaluation: 7it [00:00, 192.70it/s]\n", "Running evaluation: 7it [00:00, 191.99it/s]\n", "Running evaluation: 7it [00:00, 191.42it/s]\n", "Running evaluation: 7it [00:00, 188.65it/s]\n", "Running evaluation: 7it [00:00, 191.83it/s]\n", "Running evaluation: 7it [00:00, 192.42it/s]\n", "Running evaluation: 7it [00:00, 191.82it/s]\n", "Running evaluation: 7it [00:00, 193.46it/s]\n", "Running evaluation: 7it [00:00, 192.27it/s]\n", "Running evaluation: 7it [00:00, 193.55it/s]\n", "Running evaluation: 7it [00:00, 192.96it/s]\n", "Running evaluation: 7it [00:00, 194.48it/s]\n", "Running evaluation: 7it [00:00, 193.33it/s]\n", "Running evaluation: 7it [00:00, 194.17it/s]\n", "Running evaluation: 7it [00:00, 193.64it/s]\n", "Running evaluation: 7it [00:00, 193.99it/s]\n", "Running evaluation: 7it [00:00, 194.13it/s]\n", "Running evaluation: 7it [00:00, 191.82it/s]\n", "Running evaluation: 7it [00:00, 189.29it/s]\n", "Running evaluation: 7it [00:00, 192.03it/s]\n", "Running evaluation: 7it [00:00, 191.11it/s]\n", "Running evaluation: 7it [00:00, 191.79it/s]\n", "Running evaluation: 7it [00:00, 190.93it/s]\n", "Running evaluation: 7it [00:00, 190.72it/s]\n", "Running evaluation: 7it [00:00, 190.66it/s]\n", "Running evaluation: 7it [00:00, 189.63it/s]\n", "Running evaluation: 7it [00:00, 192.11it/s]\n", "Running evaluation: 7it [00:00, 191.93it/s]\n", "Running evaluation: 7it [00:00, 191.11it/s]\n", "Running evaluation: 7it [00:00, 184.21it/s]\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Running evaluation: 7it [00:00, 193.33it/s]\n", "Running evaluation: 7it [00:00, 192.55it/s]\n", "Running evaluation: 7it [00:00, 190.83it/s]\n", "Running evaluation: 7it [00:00, 192.37it/s]\n", "Running evaluation: 7it [00:00, 192.45it/s]\n", "Running evaluation: 7it [00:00, 193.15it/s]\n", "Running evaluation: 7it [00:00, 158.96it/s]\n", "Running evaluation: 7it [00:00, 193.38it/s]\n", "Running evaluation: 7it [00:00, 192.98it/s]\n", "Running evaluation: 7it [00:00, 192.97it/s]\n", "Running evaluation: 7it [00:00, 192.20it/s]\n", "Running evaluation: 7it [00:00, 157.28it/s]\n", "Running evaluation: 7it [00:00, 159.54it/s]\n", "Running evaluation: 7it [00:00, 195.29it/s]\n", "Running evaluation: 7it [00:00, 34.25it/s]\n", "Running evaluation: 7it [00:00, 194.70it/s]\n", "Running evaluation: 7it [00:00, 193.68it/s]\n", "Running evaluation: 7it [00:00, 195.91it/s]\n", "Running evaluation: 7it [00:00, 193.03it/s]\n", "Running evaluation: 7it [00:00, 190.39it/s]\n", "Running evaluation: 7it [00:00, 193.65it/s]\n", "Running evaluation: 7it [00:00, 192.67it/s]\n", "Running evaluation: 7it [00:00, 191.10it/s]\n", "Running evaluation: 7it [00:00, 190.83it/s]\n", "Running evaluation: 7it [00:00, 189.38it/s]\n", "Running evaluation: 7it [00:00, 190.91it/s]\n", "Running evaluation: 7it [00:00, 192.85it/s]\n", "Running evaluation: 7it [00:00, 187.22it/s]\n", "Running evaluation: 7it [00:00, 192.22it/s]\n", "Running evaluation: 7it [00:00, 191.70it/s]\n", "Running evaluation: 7it [00:00, 193.26it/s]\n", "Running evaluation: 7it [00:00, 191.22it/s]\n", "Running evaluation: 7it [00:00, 190.86it/s]\n", "Running evaluation: 7it [00:00, 192.00it/s]\n", "Running evaluation: 7it [00:00, 191.83it/s]\n", "Running evaluation: 7it [00:00, 191.67it/s]\n", "Running evaluation: 7it [00:00, 191.66it/s]\n", "Running evaluation: 7it [00:00, 191.55it/s]\n", "Running evaluation: 7it [00:00, 179.74it/s]\n", "Running evaluation: 7it [00:00, 152.90it/s]\n", "Running evaluation: 7it [00:00, 186.51it/s]\n", "Running evaluation: 7it [00:00, 162.05it/s]\n", "Running evaluation: 7it [00:00, 157.74it/s]\n", "Running evaluation: 7it [00:00, 164.43it/s]\n", "Running evaluation: 7it [00:00, 165.50it/s]\n", "Running evaluation: 7it [00:00, 158.67it/s]\n", "Running evaluation: 7it [00:00, 161.64it/s]\n", "Running evaluation: 7it [00:00, 163.31it/s]\n", "Running evaluation: 7it [00:00, 159.46it/s]\n", "Running evaluation: 7it [00:00, 173.43it/s]\n", "Running evaluation: 7it [00:00, 192.58it/s]\n", "Running evaluation: 7it [00:00, 150.51it/s]\n", "Running evaluation: 7it [00:00, 193.45it/s]\n", "Running evaluation: 7it [00:00, 194.62it/s]\n", "Running evaluation: 7it [00:00, 191.16it/s]\n", "Running evaluation: 7it [00:00, 192.81it/s]\n", "Running evaluation: 7it [00:00, 180.15it/s]\n", "Running evaluation: 7it [00:00, 186.00it/s]\n", "Running evaluation: 7it [00:00, 190.21it/s]\n", "Running evaluation: 7it [00:00, 189.26it/s]\n", "Running evaluation: 7it [00:00, 183.65it/s]\n", "Running evaluation: 7it [00:00, 192.93it/s]\n", "Running evaluation: 7it [00:00, 190.50it/s]\n", "Running evaluation: 7it [00:00, 163.13it/s]\n", "Running evaluation: 7it [00:00, 192.06it/s]\n", "Running evaluation: 7it [00:00, 154.22it/s]\n", "Running evaluation: 7it [00:00, 155.54it/s]\n", "Running evaluation: 7it [00:00, 194.01it/s]\n", "Running evaluation: 7it [00:00, 192.83it/s]\n", "Running evaluation: 7it [00:00, 192.94it/s]\n", "Running evaluation: 7it [00:00, 193.57it/s]\n", "Running evaluation: 7it [00:00, 187.94it/s]\n", "Running evaluation: 7it [00:00, 193.30it/s]\n", "Running evaluation: 7it [00:00, 190.59it/s]\n", "Running evaluation: 7it [00:00, 188.12it/s]\n", "Running evaluation: 7it [00:00, 190.00it/s]\n", "Running evaluation: 7it [00:00, 194.14it/s]\n", "Running evaluation: 7it [00:00, 195.08it/s]\n", "Running evaluation: 7it [00:00, 190.26it/s]\n", "Running evaluation: 7it [00:00, 193.29it/s]\n", "Running evaluation: 7it [00:00, 193.18it/s]\n", "Running evaluation: 7it [00:00, 154.33it/s]\n", "Running evaluation: 7it [00:00, 155.53it/s]\n", "Running evaluation: 7it [00:00, 167.64it/s]\n", "Running evaluation: 7it [00:00, 195.84it/s]\n", "Running evaluation: 7it [00:00, 196.49it/s]\n", "Running evaluation: 7it [00:00, 193.01it/s]\n", "Running evaluation: 7it [00:00, 193.12it/s]\n", "Running evaluation: 7it [00:00, 194.06it/s]\n", "Running evaluation: 7it [00:00, 193.50it/s]\n", "Running evaluation: 7it [00:00, 193.28it/s]\n", "Running evaluation: 7it [00:00, 193.10it/s]\n", "Running evaluation: 7it [00:00, 192.80it/s]\n", "Running evaluation: 7it [00:00, 192.72it/s]\n", "Running evaluation: 7it [00:00, 192.57it/s]\n", "Running evaluation: 7it [00:00, 193.09it/s]\n", "Running evaluation: 7it [00:00, 190.65it/s]\n", "Running evaluation: 7it [00:00, 190.45it/s]\n", "Running evaluation: 7it [00:00, 191.09it/s]\n", "Running evaluation: 7it [00:00, 180.15it/s]\n", "Running evaluation: 7it [00:00, 160.74it/s]\n", "Running evaluation: 7it [00:00, 194.36it/s]\n", "Running evaluation: 7it [00:00, 158.67it/s]\n", "Running evaluation: 7it [00:00, 192.95it/s]\n", "Running evaluation: 7it [00:00, 195.92it/s]\n", "Running evaluation: 7it [00:00, 194.23it/s]\n", "Running evaluation: 7it [00:00, 190.50it/s]\n", "Running evaluation: 7it [00:00, 186.04it/s]\n", "Running evaluation: 7it [00:00, 181.80it/s]\n", "Running evaluation: 7it [00:00, 190.71it/s]\n", "Running evaluation: 7it [00:00, 191.39it/s]\n", "Running evaluation: 7it [00:00, 193.43it/s]\n", "Running evaluation: 7it [00:00, 191.92it/s]\n", "Running evaluation: 7it [00:00, 191.56it/s]\n", "Running evaluation: 7it [00:00, 190.04it/s]\n", "Running evaluation: 7it [00:00, 191.25it/s]\n", "Running evaluation: 7it [00:00, 193.07it/s]\n", "Running evaluation: 7it [00:00, 191.56it/s]\n", "Running evaluation: 7it [00:00, 191.05it/s]\n", "Running evaluation: 7it [00:00, 191.42it/s]\n", "Running evaluation: 7it [00:00, 191.63it/s]\n", "Running evaluation: 7it [00:00, 193.74it/s]\n", "Running evaluation: 7it [00:00, 191.62it/s]\n", "Running evaluation: 7it [00:00, 181.43it/s]\n", "Running evaluation: 7it [00:00, 155.05it/s]\n", "Running evaluation: 7it [00:00, 193.70it/s]\n", "Running evaluation: 7it [00:00, 192.15it/s]\n", "Running evaluation: 7it [00:00, 193.43it/s]\n", "Running evaluation: 7it [00:00, 194.97it/s]\n", "Running evaluation: 7it [00:00, 194.73it/s]\n", "Running evaluation: 7it [00:00, 166.38it/s]\n", "Running evaluation: 7it [00:00, 194.80it/s]\n", "Running evaluation: 7it [00:00, 194.81it/s]\n", "Running evaluation: 7it [00:00, 193.02it/s]\n", "Running evaluation: 7it [00:00, 193.42it/s]\n", "Running evaluation: 7it [00:00, 193.24it/s]\n", "Running evaluation: 7it [00:00, 193.26it/s]\n", "/home/wassname/miniforge3/envs/glounts/lib/python3.9/site-packages/pandas/core/dtypes/astype.py:170: UserWarning: Warning: converting a masked element to nan.\n", " return arr.astype(dtype, copy=True)\n", "/home/wassname/miniforge3/envs/glounts/lib/python3.9/site-packages/gluonts/evaluation/_base.py:464: RuntimeWarning: divide by zero encountered in double_scalars\n", " totals[\"NRMSE\"] = totals[\"RMSE\"] / totals[\"abs_target_mean\"]\n", "/home/wassname/miniforge3/envs/glounts/lib/python3.9/site-packages/gluonts/evaluation/_base.py:465: RuntimeWarning: divide by zero encountered in double_scalars\n", " totals[\"ND\"] = totals[\"abs_error\"] / totals[\"abs_target_sum\"]\n", "/home/wassname/miniforge3/envs/glounts/lib/python3.9/site-packages/gluonts/evaluation/_base.py:469: RuntimeWarning: divide by zero encountered in double_scalars\n", " totals[quantile.loss_name] / totals[\"abs_target_sum\"]\n", "Running evaluation: 7it [00:00, 193.96it/s]\n", "Running evaluation: 7it [00:00, 191.91it/s]\n", "Running evaluation: 7it [00:00, 191.63it/s]\n", "Running evaluation: 7it [00:00, 192.20it/s]\n", "Running evaluation: 7it [00:00, 192.59it/s]\n", "Running evaluation: 7it [00:00, 190.47it/s]\n", "Running evaluation: 7it [00:00, 193.37it/s]\n", "Running evaluation: 7it [00:00, 190.39it/s]\n", "Running evaluation: 7it [00:00, 192.48it/s]\n", "Running evaluation: 7it [00:00, 193.35it/s]\n", "Running evaluation: 7it [00:00, 190.82it/s]\n", "Running evaluation: 7it [00:00, 192.13it/s]\n", "Running evaluation: 7it [00:00, 190.25it/s]\n", "Running evaluation: 7it [00:00, 191.69it/s]\n", "Running evaluation: 7it [00:00, 194.23it/s]\n", "Running evaluation: 7it [00:00, 153.90it/s]\n", "Running evaluation: 7it [00:00, 192.72it/s]\n", "Running evaluation: 7it [00:00, 157.24it/s]\n", "Running evaluation: 7it [00:00, 192.90it/s]\n", "Running evaluation: 7it [00:00, 193.97it/s]\n", "Running evaluation: 7it [00:00, 194.41it/s]\n", "Running evaluation: 7it [00:00, 180.61it/s]\n", "Running evaluation: 7it [00:00, 182.52it/s]\n", "Running evaluation: 7it [00:00, 182.43it/s]\n", "Running evaluation: 7it [00:00, 182.24it/s]\n", "Running evaluation: 7it [00:00, 154.65it/s]\n", "Running evaluation: 7it [00:00, 158.70it/s]\n", "Running evaluation: 7it [00:00, 191.62it/s]\n", "Running evaluation: 7it [00:00, 184.40it/s]\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Running evaluation: 7it [00:00, 190.42it/s]\n", "Running evaluation: 7it [00:00, 189.33it/s]\n", "Running evaluation: 7it [00:00, 194.73it/s]\n", "Running evaluation: 7it [00:00, 194.68it/s]\n", "Running evaluation: 7it [00:00, 194.87it/s]\n", "Running evaluation: 7it [00:00, 193.06it/s]\n", "Running evaluation: 7it [00:00, 193.41it/s]\n", "Running evaluation: 7it [00:00, 192.88it/s]\n", "Running evaluation: 7it [00:00, 193.53it/s]\n", "Running evaluation: 7it [00:00, 193.23it/s]\n", "Running evaluation: 7it [00:00, 190.94it/s]\n", "Running evaluation: 7it [00:00, 189.98it/s]\n", "Running evaluation: 7it [00:00, 191.22it/s]\n", "Running evaluation: 7it [00:00, 192.33it/s]\n", "Running evaluation: 7it [00:00, 191.35it/s]\n", "Running evaluation: 7it [00:00, 192.65it/s]\n", "Running evaluation: 7it [00:00, 158.27it/s]\n", "Running evaluation: 7it [00:00, 157.50it/s]\n", "Running evaluation: 7it [00:00, 140.75it/s]\n" ] } ], "source": [ "agg_metric, item_metrics = evaluator(targets, forecasts, num_series=len(dataset_test))" ] }, { "cell_type": "code", "execution_count": 33, "metadata": { "ExecuteTime": { "end_time": "2022-12-23T13:52:28.925023Z", "start_time": "2022-12-23T13:52:28.913362Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CRPS: 1.1069321160944334\n", "ND: 1.003746678968134\n", "NRMSE: 7.097175779249079\n", "\n", "CRPS-Sum: 0.6612398866066339\n", "ND-Sum: 0.98568530431658\n", "NRMSE-Sum: 1.075262764787951\n" ] } ], "source": [ "print(\"CRPS:\", agg_metric[\"mean_wQuantileLoss\"])\n", "print(\"ND:\", agg_metric[\"ND\"]) # totals[\"abs_error\"] / totals[\"abs_target_sum\"]\n", "print(\"NRMSE:\", agg_metric[\"NRMSE\"]) # totals[\"RMSE\"] / totals[\"abs_target_mean\"]\n", "print(\"\")\n", "print(\"CRPS-Sum:\", agg_metric[\"m_sum_mean_wQuantileLoss\"])\n", "print(\"ND-Sum:\", agg_metric[\"m_sum_ND\"])\n", "print(\"NRMSE-Sum:\", agg_metric[\"m_sum_NRMSE\"])" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "glounts", "language": "python", "name": "glounts" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.15" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": true }, "vscode": { "interpreter": { "hash": "7f25a1f13147a60511cf6766827402baf95cbe50d53a241197155306ee38fe70" } } }, "nbformat": 4, "nbformat_minor": 4 }