wassname/pytorch-ts
1
0
Fork 0
mirror of https://github.com/wassname/pytorch-ts.git synced 2026-08-11 11:24:31 +08:00
Code Issues Packages Projects Releases Wiki Activity

initial gluonts dependency

Browse Source
This commit is contained in:
Dr. Kashif Rasul
2020-12-17 17:04:56 +01:00
parent ecc31f6082
commit b072ab227b
88 changed files with 498 additions and 11571 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/dataset/test_common.py
-30
View File
@@ -1,30 +0,0 @@
# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# or in the "license" file accompanying this file. This file is distributed
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language governing
# permissions and limitations under the License.
# First-party imports
from pts.dataset import FieldName
def test_dataset_fields():
assert (
"feat_static_cat" == FieldName.FEAT_STATIC_CAT
), "Error in the FieldName 'feat_static_cat'."
assert (
"feat_static_real" == FieldName.FEAT_STATIC_REAL
), "Error in the FieldName 'feat_static_real'."
assert (
"feat_dynamic_cat" == FieldName.FEAT_DYNAMIC_CAT
), "Error in the FieldName 'feat_dynamic_cat'."
assert (
"feat_dynamic_real" == FieldName.FEAT_DYNAMIC_REAL
), "Error in the FieldName 'feat_dynamic_real'."
test/dataset/test_multivariate_grouper.py
-129
View File
@@ -1,129 +0,0 @@
# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# or in the "license" file accompanying this file. This file is distributed
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language governing
# permissions and limitations under the License.
import numpy as np
# Standard library imports
import pytest
# First-party imports
from pts.dataset import ListDataset, MultivariateGrouper
UNIVARIATE_TS = [
[
{"start": "2014-09-07", "target": [1, 2, 3, 4]},
{"start": "2014-09-07", "target": [5, 6, 7, 8]},
],
[
{"start": "2014-09-07", "target": [1, 2, 3, 4]},
{"start": "2014-09-08", "target": [5, 6, 7, 8]},
],
[
{"start": "2014-09-07", "target": [1, 2, 3, 4]},
{"start": "2014-09-07", "target": [0]},
],
[
{"start": "2014-09-07", "target": [1, 2, 3, 4]},
{"start": "2014-09-01", "target": [0]},
],
[
{"start": "2014-09-07", "target": [1, 2, 3, 4]},
{"start": "2014-09-08", "target": [5, 6, 7, 8]},
],
]
MULTIVARIATE_TS = [
[{"start": "2014-09-07", "target": [[1, 2, 3, 4], [5, 6, 7, 8]]}],
[{"start": "2014-09-07", "target": [[1, 2, 3, 4, 2.5], [6.5, 5, 6, 7, 8]],}],
[{"start": "2014-09-07", "target": [[1, 2, 3, 4], [0, 0, 0, 0]]}],
[
{
"start": "2014-09-01",
"target": [
[2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 1, 2, 3, 4],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
],
}
],
[{"start": "2014-09-07", "target": [[1, 2, 3, 4, 0], [0, 5, 6, 7, 8]]}],
]
TRAIN_FILL_RULE = [np.mean, np.mean, np.mean, np.mean, lambda x: 0.0]
@pytest.mark.parametrize(
"univariate_ts, multivariate_ts, train_fill_rule",
zip(UNIVARIATE_TS, MULTIVARIATE_TS, TRAIN_FILL_RULE),
)
def test_multivariate_grouper_train(
univariate_ts, multivariate_ts, train_fill_rule
) -> None:
univariate_ds = ListDataset(univariate_ts, freq="1D")
multivariate_ds = ListDataset(multivariate_ts, freq="1D", one_dim_target=False)
grouper = MultivariateGrouper(train_fill_rule=train_fill_rule)
assert (
list(grouper(univariate_ds))[0]["target"] == list(multivariate_ds)[0]["target"]
).all()
assert list(grouper(univariate_ds))[0]["start"] == list(multivariate_ds)[0]["start"]
UNIVARIATE_TS_TEST = [
[
{"start": "2014-09-07", "target": [1, 2, 3, 4]},
{"start": "2014-09-07", "target": [5, 6, 7, 8]},
{"start": "2014-09-08", "target": [0, 1, 2, 3]},
{"start": "2014-09-08", "target": [4, 5, 6, 7]},
],
[
{"start": "2014-09-07", "target": [1, 2, 3, 4]},
{"start": "2014-09-07", "target": [5, 6, 7, 8]},
{"start": "2014-09-08", "target": [0, 1, 2, 3]},
{"start": "2014-09-08", "target": [4, 5, 6, 7]},
],
]
MULTIVARIATE_TS_TEST = [
[
{"start": "2014-09-07", "target": [[1, 2, 3, 4], [5, 6, 7, 8]]},
{"start": "2014-09-07", "target": [[0, 0, 1, 2, 3], [0, 4, 5, 6, 7]]},
],
[
{"start": "2014-09-07", "target": [[5, 6, 7, 8]]},
{"start": "2014-09-07", "target": [[0, 4, 5, 6, 7]]},
],
]
TEST_FILL_RULE = [lambda x: 0.0, lambda x: 0.0]
MAX_TARGET_DIM = [2, 1]
@pytest.mark.parametrize(
"univariate_ts, multivariate_ts, test_fill_rule, max_target_dim",
zip(UNIVARIATE_TS_TEST, MULTIVARIATE_TS_TEST, TEST_FILL_RULE, MAX_TARGET_DIM,),
)
def test_multivariate_grouper_test(
univariate_ts, multivariate_ts, test_fill_rule, max_target_dim
) -> None:
univariate_ds = ListDataset(univariate_ts, freq="1D")
multivariate_ds = ListDataset(multivariate_ts, freq="1D", one_dim_target=False)
grouper = MultivariateGrouper(
test_fill_rule=test_fill_rule, num_test_dates=2, max_target_dim=max_target_dim,
)
for grouped_data, multivariate_data in zip(grouper(univariate_ds), multivariate_ds):
assert (grouped_data["target"] == multivariate_data["target"]).all()
assert grouped_data["start"] == multivariate_data["start"]
test/dataset/test_process.py
-21
View File
@@ -1,21 +0,0 @@
import pandas as pd
import pytest
from pts.dataset import ProcessStartField
@pytest.mark.parametrize(
"freq, expected",
[
("B", "2019-11-01"),
("W", "2019-11-03"),
("M", "2019-11-30"),
("12M", "2019-11-30"),
("A-DEC", "2019-12-31"),
],
)
def test_process_start_field(freq, expected):
process = ProcessStartField.process
given = "2019-11-01 12:34:56"
assert process(given, freq) == pd.Timestamp(expected, freq)
test/dataset/test_stat.py
-340
View File
@@ -1,340 +0,0 @@
# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# or in the "license" file accompanying this file. This file is distributed
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language governing
# permissions and limitations under the License.
# Standard library imports
import unittest
from typing import cast
# Third-party imports
import numpy as np
import pandas as pd
# First-party imports
from pts.dataset import DataEntry, Dataset
from pts.dataset.stat import (
DatasetStatistics,
ScaleHistogram,
calculate_dataset_statistics,
)
def make_dummy_dynamic_feat(target, num_features) -> np.ndarray:
# gives dummy dynamic_feat constructed from the target
return np.vstack([target * (i + 1) for i in range(num_features)])
# default values for TimeSeries field
start = pd.Timestamp("1985-01-02", freq="1D")
target = np.random.randint(0, 10, 20)
fsc = [0, 1]
fsr = [0.1, 0.2]
def make_time_series(
start=start,
target=target,
feat_static_cat=fsc,
feat_static_real=fsr,
num_feat_dynamic_cat=1,
num_feat_dynamic_real=1,
) -> DataEntry:
feat_dynamic_cat = (
make_dummy_dynamic_feat(target, num_feat_dynamic_cat).astype("int64")
if num_feat_dynamic_cat > 0
else None
)
feat_dynamic_real = (
make_dummy_dynamic_feat(target, num_feat_dynamic_real).astype("float")
if num_feat_dynamic_real > 0
else None
)
data = {
"start": start,
"target": target,
"feat_static_cat": feat_static_cat,
"feat_static_real": feat_static_real,
"feat_dynamic_cat": feat_dynamic_cat,
"feat_dynamic_real": feat_dynamic_real,
}
return data
def ts(
start,
target,
feat_static_cat=None,
feat_static_real=None,
feat_dynamic_cat=None,
feat_dynamic_real=None,
) -> DataEntry:
d = {"start": start, "target": target}
if feat_static_cat is not None:
d["feat_static_cat"] = feat_static_cat
if feat_static_real is not None:
d["feat_static_real"] = feat_static_real
if feat_dynamic_cat is not None:
d["feat_dynamic_cat"] = feat_dynamic_cat
if feat_dynamic_real is not None:
d["feat_dynamic_real"] = feat_dynamic_real
return d
class DatasetStatisticsTest(unittest.TestCase):
def test_dataset_statistics(self) -> None:
n = 2
T = 10
# use integers to avoid float conversion that can fail comparison
np.random.seed(0)
targets = np.random.randint(0, 10, (n, T))
scale_histogram = ScaleHistogram()
for i in range(n):
scale_histogram.add(targets[i, :])
scale_histogram.add([])
expected = DatasetStatistics(
integer_dataset=True,
num_time_series=n + 1,
num_time_observations=targets.size,
mean_target_length=T * 2 / 3,
min_target=targets.min(),
mean_target=targets.mean(),
mean_abs_target=targets.mean(),
max_target=targets.max(),
feat_static_real=[{0.1}, {0.2, 0.3}],
feat_static_cat=[{1}, {2, 3}],
num_feat_dynamic_real=2,
num_feat_dynamic_cat=2,
num_missing_values=0,
scale_histogram=scale_histogram,
)
# FIXME: the cast below is a hack to make mypy happy
timeseries = cast(
Dataset,
[
make_time_series(
target=targets[0, :],
feat_static_cat=[1, 2],
feat_static_real=[0.1, 0.2],
num_feat_dynamic_cat=2,
num_feat_dynamic_real=2,
),
make_time_series(
target=targets[1, :],
feat_static_cat=[1, 3],
feat_static_real=[0.1, 0.3],
num_feat_dynamic_cat=2,
num_feat_dynamic_real=2,
),
make_time_series(
target=np.array([]),
feat_static_cat=[1, 3],
feat_static_real=[0.1, 0.3],
num_feat_dynamic_cat=2,
num_feat_dynamic_real=2,
),
],
)
found = calculate_dataset_statistics(timeseries)
assert expected == found
def test_dataset_histogram(self) -> None:
# generates 2 ** N - 1 timeseries with constant increasing values
N = 6
n = 2 ** N - 1
T = 5
targets = np.ones((n, T))
for i in range(0, n):
targets[i, :] = targets[i, :] * i
# FIXME: the cast below is a hack to make mypy happy
timeseries = cast(
Dataset, [make_time_series(target=targets[i, :]) for i in range(n)]
)
found = calculate_dataset_statistics(timeseries)
hist = found.scale_histogram.bin_counts
for i in range(0, N):
assert i in hist
assert hist[i] == 2 ** i
class DatasetStatisticsExceptions(unittest.TestCase):
def test_dataset_statistics_exceptions(self) -> None:
def check_error_message(expected_regex, dataset) -> None:
with self.assertRaisesRegex(Exception, expected_regex):
calculate_dataset_statistics(dataset)
check_error_message("Time series dataset is empty!", [])
check_error_message(
"Only empty time series found in the dataset!",
[make_time_series(target=np.random.randint(0, 10, 0))],
)
# infinite target
# check_error_message(
# "Target values have to be finite (e.g., not inf, -inf, "
# "or None) and cannot exceed single precision floating "
# "point range.",
# [make_time_series(target=np.full(20, np.inf))]
# )
# different number of feat_dynamic_{cat, real}
check_error_message(
"Found instances with different number of features in "
"feat_dynamic_cat, found one with 2 and another with 1.",
[
make_time_series(num_feat_dynamic_cat=2),
make_time_series(num_feat_dynamic_cat=1),
],
)
check_error_message(
"Found instances with different number of features in "
"feat_dynamic_cat, found one with 0 and another with 1.",
[
make_time_series(num_feat_dynamic_cat=0),
make_time_series(num_feat_dynamic_cat=1),
],
)
check_error_message(
"feat_dynamic_cat was found for some instances but not others.",
[
make_time_series(num_feat_dynamic_cat=1),
make_time_series(num_feat_dynamic_cat=0),
],
)
check_error_message(
"Found instances with different number of features in "
"feat_dynamic_real, found one with 2 and another with 1.",
[
make_time_series(num_feat_dynamic_real=2),
make_time_series(num_feat_dynamic_real=1),
],
)
check_error_message(
"Found instances with different number of features in "
"feat_dynamic_real, found one with 0 and another with 1.",
[
make_time_series(num_feat_dynamic_real=0),
make_time_series(num_feat_dynamic_real=1),
],
)
check_error_message(
"feat_dynamic_real was found for some instances but not others.",
[
make_time_series(num_feat_dynamic_real=1),
make_time_series(num_feat_dynamic_real=0),
],
)
# infinite feat_dynamic_{cat,real}
inf_dynamic_feat = np.full((2, len(target)), np.inf)
check_error_message(
"Features values have to be finite and cannot exceed single "
"precision floating point range.",
[
ts(
start,
target,
feat_dynamic_cat=inf_dynamic_feat,
feat_static_cat=[0, 1],
)
],
)
check_error_message(
"Features values have to be finite and cannot exceed single "
"precision floating point range.",
[
ts(
start,
target,
feat_dynamic_real=inf_dynamic_feat,
feat_static_cat=[0, 1],
)
],
)
# feat_dynamic_{cat, real} different length from target
check_error_message(
"Each feature in feat_dynamic_cat has to have the same length as the "
"target. Found an instance with feat_dynamic_cat of length 1 and a "
"target of length 20.",
[
ts(
start=start,
target=target,
feat_static_cat=[0, 1],
feat_dynamic_cat=np.ones((1, 1)),
)
],
)
check_error_message(
"Each feature in feat_dynamic_real has to have the same length as the "
"target. Found an instance with feat_dynamic_real of length 1 and a "
"target of length 20.",
[
ts(
start=start,
target=target,
feat_static_cat=[0, 1],
feat_dynamic_real=np.ones((1, 1)),
)
],
)
# feat_static_{cat, real} different length
check_error_message(
"Not all feat_static_cat vectors have the same length 2 != 1.",
[
ts(start=start, target=target, feat_static_cat=[0, 1]),
ts(start=start, target=target, feat_static_cat=[1]),
],
)
check_error_message(
"Not all feat_static_real vectors have the same length 2 != 1.",
[
ts(start=start, target=target, feat_static_real=[0, 1]),
ts(start=start, target=target, feat_static_real=[1]),
],
)
calculate_dataset_statistics(
# FIXME: the cast below is a hack to make mypy happy
cast(
Dataset,
[
make_time_series(num_feat_dynamic_cat=2),
make_time_series(num_feat_dynamic_cat=2),
],
)
)
calculate_dataset_statistics(
# FIXME: the cast below is a hack to make mypy happy
cast(
Dataset,
[
make_time_series(num_feat_dynamic_cat=0),
make_time_series(num_feat_dynamic_cat=0),
],
)
)
test/evaluation/test_evaluator.py
-649
View File
@@ -1,649 +0,0 @@
# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# or in the "license" file accompanying this file. This file is distributed
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language governing
# permissions and limitations under the License.
# Third-party imports
import numpy as np
import pandas as pd
import pytest
# First-party imports
from pts.evaluation import (
Evaluator,
MultivariateEvaluator,
)
from pts.feature import get_seasonality
from pts.model.forecast import QuantileForecast, SampleForecast
QUANTILES = [str(q / 10.0) for q in range(1, 10)]
def data_iterator(ts):
"""
:param ts: list of pd.Series or pd.DataFrame
:return:
"""
for i in range(len(ts)):
yield ts[i]
def fcst_iterator(fcst, start_dates, freq):
"""
:param fcst: list of numpy arrays with the sample paths
:return:
"""
for i in range(len(fcst)):
yield SampleForecast(samples=fcst[i], start_date=start_dates[i], freq=freq)
def iterator(it):
"""
Convenience function to toggle whether to consume dataset and forecasts as iterators or iterables.
:param it:
:return: it (as iterator)
"""
return iter(it)
def iterable(it):
"""
Convenience function to toggle whether to consume dataset and forecasts as iterators or iterables.
:param it:
:return: it (as iterable)
"""
return list(it)
def naive_forecaster(ts, prediction_length, num_samples=100, target_dim=0):
"""
:param ts: pandas.Series
:param prediction_length:
:param num_samples: number of sample paths
:param target_dim: number of axes of target (0: scalar, 1: array, ...)
:return: np.array with dimension (num_samples, prediction_length)
"""
# naive prediction: last observed value
naive_pred = ts.values[-prediction_length - 1]
assert len(naive_pred.shape) == target_dim
return np.tile(
naive_pred,
(num_samples, prediction_length) + tuple(1 for _ in range(target_dim)),
)
def naive_multivariate_forecaster(ts, prediction_length, num_samples=100):
return naive_forecaster(ts, prediction_length, num_samples, target_dim=1)
def calculate_metrics(
timeseries,
evaluator,
ts_datastructure,
has_nans=False,
forecaster=naive_forecaster,
input_type=iterator,
):
num_timeseries = timeseries.shape[0]
num_timestamps = timeseries.shape[1]
if has_nans:
timeseries[0, 1] = np.nan
timeseries[0, 7] = np.nan
num_samples = 100
prediction_length = 3
freq = "1D"
ts_start_dates = (
[]
) # starting date of each time series - can be different in general
pd_timeseries = [] # list of pandas.DataFrame
samples = [] # list of forecast samples
start_dates = [] # start date of the prediction range
for i in range(num_timeseries):
ts_start_dates.append(pd.Timestamp(year=2018, month=1, day=1, hour=1))
index = pd.date_range(ts_start_dates[i], periods=num_timestamps, freq=freq)
pd_timeseries.append(ts_datastructure(timeseries[i], index=index))
samples.append(forecaster(pd_timeseries[i], prediction_length, num_samples))
start_dates.append(
pd.date_range(ts_start_dates[i], periods=num_timestamps, freq=freq)[
-prediction_length
]
)
# data iterator
data_iter = input_type(data_iterator(pd_timeseries))
fcst_iter = input_type(fcst_iterator(samples, start_dates, freq))
# evaluate
agg_df, item_df = evaluator(data_iter, fcst_iter)
return agg_df, item_df
TIMESERIES_M4 = [
np.array(
[
[
2.943_013,
2.822_251,
4.196_222,
1.328_664,
4.947_390,
3.333_131,
1.479_800,
2.265_094,
3.413_493,
3.497_607,
],
[
-0.126_781_2,
3.057_412_2,
1.901_594_4,
2.772_549_5,
3.312_853_1,
4.411_818_0,
3.709_025_2,
4.322_028,
2.565_359,
3.074_308,
],
[
2.542_998,
2.336_757,
1.417_916,
1.335_139,
2.523_035,
3.645_589,
3.382_819,
2.075_960,
2.643_869,
2.772_456,
],
[
0.315_685_6,
1.892_312_1,
2.476_861_2,
3.511_628_6,
4.384_346_5,
2.960_685_6,
4.897_572_5,
3.280_125,
4.768_556,
4.958_616,
],
[
2.205_877_3,
0.782_759_4,
2.401_420_8,
2.385_643_4,
4.845_818_2,
3.102_322_9,
3.567_723_7,
4.878_143,
3.735_245,
2.218_113,
],
]
),
np.array(
[
[
13.11301,
13.16225,
14.70622,
12.00866,
15.79739,
14.35313,
12.66980,
13.62509,
14.94349,
15.19761,
],
[
10.04322,
13.39741,
12.41159,
13.45255,
14.16285,
15.43182,
14.89903,
15.68203,
14.09536,
14.77431,
],
[
12.71300,
12.67676,
11.92792,
12.01514,
13.37303,
14.66559,
14.57282,
13.43596,
14.17387,
14.47246,
],
[
10.48569,
12.23231,
12.98686,
14.19163,
15.23435,
13.98069,
16.08757,
14.64012,
16.29856,
16.65862,
],
[
12.37588,
11.12276,
12.91142,
13.06564,
15.69582,
14.12232,
14.75772,
16.23814,
15.26524,
13.91811,
],
]
),
]
RES_M4 = [
{
"MASE": 0.816_837_618,
"MAPE": 0.324_517_430_685_928_1,
"sMAPE": 0.326_973_268_4,
"seasonal_error": np.array(
[1.908_101, 1.258_838, 0.63018, 1.238_201, 1.287_771]
),
},
{
"MASE": 0.723_948_2,
"MAPE": 0.063_634_129_851_747_6,
"sMAPE": 0.065_310_85,
"seasonal_error": np.array(
[1.867_847, 1.315_505, 0.602_587_4, 1.351_535, 1.339_179]
),
},
]
@pytest.mark.parametrize("timeseries, res", zip(TIMESERIES_M4, RES_M4))
def test_MASE_sMAPE_M4(timeseries, res):
ts_datastructure = pd.Series
evaluator = Evaluator(quantiles=QUANTILES)
agg_df, item_df = calculate_metrics(timeseries, evaluator, ts_datastructure)
assert abs((agg_df["MASE"] - res["MASE"]) / res["MASE"]) < 0.001, (
"Scores for the metric MASE do not match: "
"\nexpected: {} \nobtained: {}".format(res["MASE"], agg_df["MASE"])
)
assert abs((agg_df["MAPE"] - res["MAPE"]) / res["MAPE"]) < 0.001, (
"Scores for the metric MAPE do not match: \nexpected: {} "
"\nobtained: {}".format(res["MAPE"], agg_df["MAPE"])
)
assert abs((agg_df["sMAPE"] - res["sMAPE"]) / res["sMAPE"]) < 0.001, (
"Scores for the metric sMAPE do not match: \nexpected: {} "
"\nobtained: {}".format(res["sMAPE"], agg_df["sMAPE"])
)
assert sum(abs(item_df["seasonal_error"].values - res["seasonal_error"])) < 0.001, (
"Scores for the metric seasonal_error do not match: \nexpected: {} "
"\nobtained: {}".format(res["seasonal_error"], item_df["seasonal_error"].values)
)
TIMESERIES = [
np.ones((5, 10), dtype=np.float64),
np.ones((5, 10), dtype=np.float64),
np.arange(0, 50, dtype=np.float64).reshape(5, 10),
np.arange(0, 50, dtype=np.float64).reshape(5, 10),
np.array([[np.nan] * 10, [1.0] * 10]),
]
RES = [
{
"MSE": 0.0,
"abs_error": 0.0,
"abs_target_sum": 15.0,
"abs_target_mean": 1.0,
"seasonal_error": 0.0,
"MASE": 0.0,
"MAPE": 0.0,
"sMAPE": 0.0,
"MSIS": 0.0,
"RMSE": 0.0,
"NRMSE": 0.0,
"ND": 0.0,
"MAE_Coverage": 0.5,
},
{
"MSE": 0.0,
"abs_error": 0.0,
"abs_target_sum": 14.0,
"abs_target_mean": 1.0,
"seasonal_error": 0.0,
"MASE": 0.0,
"MAPE": 0.0,
"sMAPE": 0.0,
"MSIS": 0.0,
"RMSE": 0.0,
"NRMSE": 0.0,
"ND": 0.0,
"MAE_Coverage": 0.5,
},
{
"MSE": 4.666_666_666_666,
"abs_error": 30.0,
"abs_target_sum": 420.0,
"abs_target_mean": 28.0,
"seasonal_error": 1.0,
"MASE": 2.0,
"MAPE": 0.103_112_211_532_524_85,
"sMAPE": 0.113_254_049_3,
"MSIS": 80.0,
"RMSE": 2.160_246_899_469_286_9,
"NRMSE": 0.077_151_674_981_045_956,
"ND": 0.071_428_571_428_571_42,
"MAE_Coverage": 0.5,
},
{
"MSE": 5.033_333_333_333_3,
"abs_error": 29.0,
"abs_target_sum": 413.0,
"abs_target_mean": 28.1,
"seasonal_error": 1.0,
"MASE": 2.1,
"MAPE": 0.113_032_846_453_159_77,
"sMAPE": 0.125_854_781_903_299_57,
"MSIS": 84.0,
"RMSE": 2.243_509_156_061_845_6,
"NRMSE": 0.079_840_183_489_745_39,
"ND": 0.070_217_917_675_544_79,
"MAE_Coverage": 0.5,
},
{
"MSE": 0.0,
"abs_error": 0.0,
"abs_target_sum": 3.0,
"abs_target_mean": 1.0,
"seasonal_error": 0.0,
"MASE": 0.0,
"MAPE": 0.0,
"sMAPE": 0.0,
"MSIS": 0.0,
"RMSE": 0.0,
"NRMSE": 0.0,
"ND": 0.0,
"MAE_Coverage": 0.5,
},
]
HAS_NANS = [False, True, False, True, True]
INPUT_TYPE = [iterable, iterable, iterator, iterator, iterable]
@pytest.mark.parametrize(
"timeseries, res, has_nans, input_type", zip(TIMESERIES, RES, HAS_NANS, INPUT_TYPE),
)
def test_metrics(timeseries, res, has_nans, input_type):
ts_datastructure = pd.Series
evaluator = Evaluator(quantiles=QUANTILES, num_workers=0)
agg_metrics, item_metrics = calculate_metrics(
timeseries,
evaluator,
ts_datastructure,
has_nans=has_nans,
input_type=input_type,
)
for metric, score in agg_metrics.items():
if metric in res.keys():
assert abs(score - res[metric]) < 0.001, (
"Scores for the metric {} do not match: \nexpected: {} "
"\nobtained: {}".format(metric, res[metric], score)
)
@pytest.mark.parametrize(
"timeseries, res, has_nans, input_type", zip(TIMESERIES, RES, HAS_NANS, INPUT_TYPE),
)
def test_metrics_mp(timeseries, res, has_nans, input_type):
ts_datastructure = pd.Series
# Default will be multiprocessing evaluator
evaluator = Evaluator(quantiles=QUANTILES, num_workers=4)
agg_metrics, item_metrics = calculate_metrics(
timeseries,
evaluator,
ts_datastructure,
has_nans=has_nans,
input_type=input_type,
)
for metric, score in agg_metrics.items():
if metric in res.keys():
assert abs(score - res[metric]) < 0.001, (
"Scores for the metric {} do not match: \nexpected: {} "
"\nobtained: {}".format(metric, res[metric], score)
)
TIMESERIES_MULTIVARIATE = [
np.ones((5, 10, 2), dtype=np.float64),
np.ones((5, 10, 2), dtype=np.float64),
np.ones((5, 10, 2), dtype=np.float64),
np.stack(
(
np.arange(0, 50, dtype=np.float64).reshape(5, 10),
np.arange(50, 100, dtype=np.float64).reshape(5, 10),
),
axis=2,
),
np.stack(
(
np.arange(0, 50, dtype=np.float64).reshape(5, 10),
np.arange(50, 100, dtype=np.float64).reshape(5, 10),
),
axis=2,
),
np.stack(
(
np.arange(0, 50, dtype=np.float64).reshape(5, 10),
np.arange(50, 100, dtype=np.float64).reshape(5, 10),
),
axis=2,
),
]
RES_MULTIVARIATE = [
{
"MSE": 0.0,
"0_MSE": 0.0,
"1_MSE": 0.0,
"abs_error": 0.0,
"abs_target_sum": 15.0,
"abs_target_mean": 1.0,
"seasonal_error": 0.0,
"MASE": 0.0,
"sMAPE": 0.0,
"MSIS": 0.0,
"RMSE": 0.0,
"NRMSE": 0.0,
"ND": 0.0,
"MAE_Coverage": 0.5,
"m_sum_MSE": 0.0,
},
{
"MSE": 0.0,
"abs_error": 0.0,
"abs_target_sum": 15.0,
"abs_target_mean": 1.0,
"seasonal_error": 0.0,
"MASE": 0.0,
"sMAPE": 0.0,
"MSIS": 0.0,
"RMSE": 0.0,
"NRMSE": 0.0,
"ND": 0.0,
"MAE_Coverage": 0.5,
"m_sum_MSE": 0.0,
},
{
"MSE": 0.0,
"abs_error": 0.0,
"abs_target_sum": 30.0,
"abs_target_mean": 1.0,
"seasonal_error": 0.0,
"MASE": 0.0,
"sMAPE": 0.0,
"MSIS": 0.0,
"RMSE": 0.0,
"NRMSE": 0.0,
"ND": 0.0,
"MAE_Coverage": 0.5,
"m_sum_MSE": 0.0,
},
{
"MSE": 4.666_666_666_666,
"abs_error": 30.0,
"abs_target_sum": 420.0,
"abs_target_mean": 28.0,
"seasonal_error": 1.0,
"MASE": 2.0,
"sMAPE": 0.113_254_049_3,
"MSIS": 80.0,
"RMSE": 2.160_246_899_469_286_9,
"NRMSE": 0.077_151_674_981_045_956,
"ND": 0.071_428_571_428_571_42,
"MAE_Coverage": 0.5,
"m_sum_MSE": 18.666_666_666_666,
},
{
"MSE": 4.666_666_666_666,
"abs_error": 30.0,
"abs_target_sum": 1170.0,
"abs_target_mean": 78.0,
"seasonal_error": 1.0,
"MASE": 2.0,
"sMAPE": 0.026_842_301_756_499_45,
"MSIS": 80.0,
"RMSE": 2.160_246_899_469_286_9,
"NRMSE": 0.027_695_473_070_119_065,
"ND": 0.025_641_025_641_025_64,
"MAE_Coverage": 0.5,
"m_sum_MSE": 18.666_666_666_666,
},
{
"MSE": 4.666_666_666_666,
"abs_error": 60.0,
"abs_target_sum": 1590.0,
"abs_target_mean": 53.0,
"seasonal_error": 1.0,
"MASE": 2.0,
"sMAPE": 0.070_048_175_528_249_73,
"MSIS": 80.0,
"RMSE": 2.160_246_899_469_286_9,
"NRMSE": 0.040_759_375_461_684_65,
"ND": 0.037_735_849_056_603_77,
"MAE_Coverage": 0.5,
"m_sum_MSE": 18.666_666_666_666,
},
]
HAS_NANS_MULTIVARIATE = [False, False, False, False, False, False]
EVAL_DIMS = [[0], [1], [0, 1], [0], [1], None]
INPUT_TYPE = [iterable, iterable, iterator, iterator, iterable, iterator]
@pytest.mark.parametrize(
"timeseries, res, has_nans, eval_dims, input_type",
zip(
TIMESERIES_MULTIVARIATE,
RES_MULTIVARIATE,
HAS_NANS_MULTIVARIATE,
EVAL_DIMS,
INPUT_TYPE,
),
)
def test_metrics_multivariate(timeseries, res, has_nans, eval_dims, input_type):
ts_datastructure = pd.DataFrame
evaluator = MultivariateEvaluator(
quantiles=QUANTILES, eval_dims=eval_dims, target_agg_funcs={"sum": np.sum},
)
agg_metrics, item_metrics = calculate_metrics(
timeseries,
evaluator,
ts_datastructure,
has_nans=has_nans,
forecaster=naive_multivariate_forecaster,
input_type=input_type,
)
for metric, score in agg_metrics.items():
if metric in res.keys():
assert abs(score - res[metric]) < 0.001, (
"Scores for the metric {} do not match: \nexpected: {} "
"\nobtained: {}".format(metric, res[metric], score)
)
def test_evaluation_with_QuantileForecast():
start = "2012-01-11"
target = [2.4, 1.0, 3.0, 4.4, 5.5, 4.9] * 11
index = pd.date_range(start=start, freq="1D", periods=len(target))
ts = pd.Series(index=index, data=target)
ev = Evaluator(quantiles=("0.1", "0.2", "0.5"))
fcst = [
QuantileForecast(
start_date=pd.Timestamp("2012-01-11"),
freq="D",
forecast_arrays=np.array([[2.4, 9.0, 3.0, 2.4, 5.5, 4.9] * 10]),
forecast_keys=["0.5"],
)
]
agg_metric, _ = ev(iter([ts]), iter(fcst))
assert np.isfinite(agg_metric["wQuantileLoss[0.5]"])
@pytest.mark.parametrize(
"freq, expected_seasonality",
[
("1H", 24),
("H", 24),
("2H", 12),
("3H", 8),
("4H", 6),
("15H", 1),
("5B", 1),
("1B", 5),
("2W", 1),
("3M", 4),
("1D", 1),
("7D", 1),
("8D", 1),
],
)
def test_get_seasonality(freq, expected_seasonality):
assert get_seasonality(freq) == expected_seasonality
test/feature/test_lag.py
-311
View File
@@ -1,311 +0,0 @@
# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# or in the "license" file accompanying this file. This file is distributed
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language governing
# permissions and limitations under the License.
from pts.feature import get_lags_for_frequency
# These are the expected lags for common frequencies and corner cases.
# By default all frequencies have the following lags: [1, 2, 3, 4, 5, 6, 7].
# Remaining lags correspond to the same `season` (+/- `delta`) in previous `k` cycles.
expected_lags = {
# (apart from the default lags) centered around each of the last 3 hours (delta = 2)
"min": [
1,
2,
3,
4,
5,
6,
7,
58,
59,
60,
61,
62,
118,
119,
120,
121,
122,
178,
179,
180,
181,
182,
],
# centered around each of the last 3 hours (delta = 2) + last 7 days (delta = 1)
"15min": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
+ [
95,
96,
97,
191,
192,
193,
287,
288,
289,
383,
384,
385,
479,
480,
481,
575,
576,
577,
671,
672,
673,
],
# centered around each of the last 3 hours (delta = 2) + last 7 days (delta = 1) + 3 weeks (delta = 1)
"30min": [1, 2, 3, 4, 5, 6, 7, 8]
+ [
47,
48,
49,
95,
96,
97,
143,
144,
145,
191,
192,
193,
239,
240,
241,
287,
288,
289,
335,
336,
337,
]
+ [671, 672, 673, 1007, 1008, 1009],
# centered around each of the last 3 hours (delta = 2) + last 7 days (delta = 1) + last 6 weeks (delta = 1)
"59min": [1, 2, 3, 4, 5, 6, 7]
+ [
23,
24,
25,
47,
48,
49,
72,
73,
74,
96,
97,
98,
121,
122,
123,
145,
146,
147,
169,
170,
171,
]
+ [340, 341, 342, 511, 512, 513, 682, 683, 684, 731, 732, 733],
# centered around each of the last 3 hours (delta = 2) + last 7 days (delta = 1) + last 6 weeks (delta = 1)
"61min": [1, 2, 3, 4, 5, 6, 7]
+ [
22,
23,
24,
46,
47,
48,
69,
70,
71,
93,
94,
95,
117,
118,
119,
140,
141,
142,
164,
165,
166,
]
+ [329, 330, 331, 494, 495, 496, 659, 660, 661, 707, 708, 709],
# centered around each of the last 3 hours (delta = 2) + last 7 days (delta = 1) + last 6 weeks (delta = 1)
"H": [1, 2, 3, 4, 5, 6, 7]
+ [
23,
24,
25,
47,
48,
49,
71,
72,
73,
95,
96,
97,
119,
120,
121,
143,
144,
145,
167,
168,
169,
]
+ [335, 336, 337, 503, 504, 505, 671, 672, 673, 719, 720, 721],
# centered around each of the last 7 days (delta = 1) + last 4 weeks (delta = 1) + last 1 month (delta = 1) +
# last 8th and 12th weeks (delta = 0)
"6H": [
1,
2,
3,
4,
5,
6,
7,
8,
9,
11,
12,
13,
15,
16,
17,
19,
20,
21,
23,
24,
25,
27,
28,
29,
]
+ [55, 56, 57, 83, 84, 85, 111, 112, 113]
+ [119, 120, 121]
+ [224, 336],
# centered around each of the last 7 days (delta = 1) + last 4 weeks (delta = 1) + last 1 month (delta = 1) +
# last 8th and 12th weeks (delta = 0) + last year (delta = 1)
"12H": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
+ [27, 28, 29, 41, 42, 43, 55, 56, 57]
+ [59, 60, 61]
+ [112, 168]
+ [727, 728, 729],
# centered around each of the last 7 days (delta = 1) + last 4 weeks (delta = 1) + last 1 month (delta = 1) +
# last 8th and 12th weeks (delta = 0) + last 3 years (delta = 1)
"23H": [1, 2, 3, 4, 5, 6, 7, 8]
+ [13, 14, 15, 20, 21, 22, 28, 29]
+ [30, 31, 32]
+ [58, 87]
+ [378, 379, 380, 758, 759, 760, 1138, 1139, 1140],
# centered around each of the last 7 days (delta = 1) + last 4 weeks (delta = 1) + last 1 month (delta = 1) +
# last 8th and 12th weeks (delta = 0) + last 3 years (delta = 1)
"25H": [1, 2, 3, 4, 5, 6, 7]
+ [12, 13, 14, 19, 20, 21, 25, 26, 27]
+ [28, 29]
+ [53, 80]
+ [348, 349, 350, 697, 698, 699, 1047, 1048, 1049],
# centered around each of the last 7 days (delta = 1) + last 4 weeks (delta = 1) + last 1 month (delta = 1) +
# last 8th and 12th weeks (delta = 0) + last 3 years (delta = 1)
"D": [1, 2, 3, 4, 5, 6, 7, 8]
+ [13, 14, 15, 20, 21, 22, 27, 28, 29]
+ [30, 31]
+ [56, 84]
+ [363, 364, 365, 727, 728, 729, 1091, 1092, 1093],
# centered around each of the last 7 days (delta = 1) + last 4 weeks (delta = 1) + last 1 month (delta = 1) +
# last 8th and 12th weeks (delta = 0) + last 3 years (delta = 1)
"2D": [1, 2, 3, 4, 5]
+ [6, 7, 8, 9, 10, 11, 13, 14, 15]
+ [16]
+ [28, 42]
+ [181, 182, 183, 363, 364, 365, 545, 546, 547],
# centered around each of the last 3 months (delta = 0) + last 3 years (delta = 1) (assuming 52 weeks per year)
"6D": [1, 2, 3, 4, 5, 6, 7, 9, 14] + [59, 60, 61, 120, 121, 122, 181, 182, 183],
# centered around each of the last 3 months (delta = 0) + last 3 years (delta = 1) (assuming 52 weeks per year)
"W": [1, 2, 3, 4, 5, 6, 7, 8, 12] + [51, 52, 53, 103, 104, 105, 155, 156, 157],
# centered around each of the last 3 months (delta = 0) + last 3 years (delta = 1) (assuming 52 weeks per year)
"8D": [1, 2, 3, 4, 5, 6, 7, 10] + [44, 45, 46, 90, 91, 92, 135, 136, 137],
# centered around each of the last 3 years (delta = 1)
"4W": [1, 2, 3, 4, 5, 6, 7, 12, 13, 14, 25, 26, 27, 38, 39, 40],
# centered around each of the last 3 years (delta = 1)
"3W": [1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 33, 34, 35, 51, 52, 53],
# centered around each of the last 3 years (delta = 1)
"5W": [1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 19, 20, 21, 30, 31, 32],
# centered around each of the last 3 years (delta = 1)
"M": [1, 2, 3, 4, 5, 6, 7, 11, 12, 13, 23, 24, 25, 35, 36, 37],
# default
"6M": [1, 2, 3, 4, 5, 6, 7],
# default
"12M": [1, 2, 3, 4, 5, 6, 7],
}
# For the default multiple (1)
for freq in ["min", "H", "D", "W", "M"]:
expected_lags["1" + freq] = expected_lags[freq]
# For frequencies that do not have unique form
expected_lags["60min"] = expected_lags["1H"]
expected_lags["24H"] = expected_lags["1D"]
expected_lags["7D"] = expected_lags["1W"]
def test_lags():
freq_strs = [
"min",
"1min",
"15min",
"30min",
"59min",
"60min",
"61min",
"H",
"1H",
"6H",
"12H",
"23H",
"24H",
"25H",
"D",
"1D",
"2D",
"6D",
"7D",
"8D",
"W",
"1W",
"3W",
"4W",
"5W",
"M",
"6M",
"12M",
]
for freq_str in freq_strs:
lags = get_lags_for_frequency(freq_str)
assert (
lags == expected_lags[freq_str]
), "lags do not match for the frequency '{}':\nexpected: {},\nprovided: {}".format(
freq_str, expected_lags[freq_str], lags
)
test/modules/test_distribution_output.py
+1 -1
View File
@@ -17,8 +17,8 @@ from torch.nn.utils import clip_grad_norm_
from torch.optim import SGD
from torch.utils.data import TensorDataset, DataLoader
from gluonts.torch.modules.distribution_output import DistributionOutput
from pts.modules import (
DistributionOutput,
StudentTOutput,
BetaOutput,
NegativeBinomialOutput,
test/modules/test_implicit_quantile_distr_output.py
+6 -5
View File
@@ -11,13 +11,14 @@ from torch.nn.utils import clip_grad_norm_
from torch.optim import SGD
from torch.utils.data import TensorDataset, DataLoader
from gluonts.dataset.repository.datasets import get_dataset
from gluonts.evaluation import Evaluator
from gluonts.evaluation.backtest import make_evaluation_predictions
from gluonts.torch.modules.distribution_output import DistributionOutput
from pts import Trainer
from pts.dataset.repository import get_dataset
from pts.evaluation import make_evaluation_predictions, Evaluator
from pts.model.deepar import DeepAREstimator
from pts.model.simple_feedforward import SimpleFeedForwardEstimator
from pts.modules import (
DistributionOutput,
ImplicitQuantileOutput
)
@@ -172,7 +173,7 @@ def test_training_with_implicit_quantile_output():
)
forecasts = list(forecast_it)
tss = list(ts_it)
evaluator = Evaluator()
evaluator = Evaluator(num_workers=0)
agg_metrics, item_metrics = evaluator(iter(tss), iter(forecasts), num_series=len(dataset.test))
assert agg_metrics["MSE"] > 0
@@ -220,7 +221,7 @@ def test_instanciation_of_args_proj():
)
forecasts = list(forecast_it)
tss = list(ts_it)
evaluator = Evaluator()
evaluator = Evaluator(num_workers=0)
agg_metrics, item_metrics = evaluator(iter(tss), iter(forecasts), num_series=len(dataset.test))
assert distr_output.method_calls == 2
test/test_transform.py
-808
View File
@@ -1,808 +0,0 @@
# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# or in the "license" file accompanying this file. This file is distributed
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language governing
# permissions and limitations under the License.
# Standard library imports
from typing import Tuple
# Third-party imports
import numpy as np
import pandas as pd
import pytest
import torch
from pts import transform
# First-party imports
from pts.dataset import (
ProcessStartField,
FieldName,
ListDataset,
DataEntry,
calculate_dataset_statistics,
ScaleHistogram,
)
from pts.feature import time_feature
FREQ = "1D"
TEST_VALUES = {
"is_train": [True, False],
"target": [np.zeros(0), np.random.rand(13), np.random.rand(100)],
"start": [
ProcessStartField.process("2012-01-02", freq="1D"),
ProcessStartField.process("1994-02-19 20:01:02", freq="3D"),
],
"use_prediction_features": [True, False],
"allow_target_padding": [True, False],
}
def test_align_timestamp():
def aligned_with(date_str, freq):
return str(ProcessStartField.process(date_str, freq=freq))
for _ in range(2):
assert aligned_with("2012-03-05 09:13:12", "min") == "2012-03-05 09:13:00"
assert aligned_with("2012-03-05 09:13:12", "2min") == "2012-03-05 09:12:00"
assert aligned_with("2012-03-05 09:13:12", "H") == "2012-03-05 09:00:00"
assert aligned_with("2012-03-05 09:13:12", "D") == "2012-03-05 00:00:00"
assert aligned_with("2012-03-05 09:13:12", "W") == "2012-03-11 00:00:00"
assert aligned_with("2012-03-05 09:13:12", "4W") == "2012-03-11 00:00:00"
assert aligned_with("2012-03-05 09:13:12", "M") == "2012-03-31 00:00:00"
assert aligned_with("2012-03-05 09:13:12", "3M") == "2012-03-31 00:00:00"
assert aligned_with("2012-03-05 09:13:12", "Y") == "2012-12-31 00:00:00"
assert aligned_with("2012-03-05 09:14:11", "min") == "2012-03-05 09:14:00"
assert aligned_with("2012-03-05 09:14:11", "2min") == "2012-03-05 09:14:00"
assert aligned_with("2012-03-05 09:14:11", "H") == "2012-03-05 09:00:00"
assert aligned_with("2012-03-05 09:14:11", "D") == "2012-03-05 00:00:00"
assert aligned_with("2012-03-05 09:14:11", "W") == "2012-03-11 00:00:00"
assert aligned_with("2012-03-05 09:14:11", "4W") == "2012-03-11 00:00:00"
assert aligned_with("2012-03-05 09:14:11", "M") == "2012-03-31 00:00:00"
assert aligned_with("2012-03-05 09:14:11", "3M") == "2012-03-31 00:00:00"
@pytest.mark.parametrize("is_train", TEST_VALUES["is_train"])
@pytest.mark.parametrize("target", TEST_VALUES["target"])
@pytest.mark.parametrize("start", TEST_VALUES["start"])
def test_AddTimeFeatures(start, target, is_train: bool):
pred_length = 13
t = transform.AddTimeFeatures(
start_field=FieldName.START,
target_field=FieldName.TARGET,
output_field="myout",
pred_length=pred_length,
time_features=[time_feature.DayOfWeek(), time_feature.DayOfMonth()],
)
data = {"start": start, "target": target}
res = t.map_transform(data, is_train=is_train)
mat = res["myout"]
expected_length = len(target) + (0 if is_train else pred_length)
assert mat.shape == (2, expected_length)
tmp_idx = pd.date_range(start=start, freq=start.freq, periods=expected_length)
assert np.alltrue(mat[0] == time_feature.DayOfWeek()(tmp_idx))
assert np.alltrue(mat[1] == time_feature.DayOfMonth()(tmp_idx))
@pytest.mark.parametrize("is_train", TEST_VALUES["is_train"])
@pytest.mark.parametrize("target", TEST_VALUES["target"])
@pytest.mark.parametrize("start", TEST_VALUES["start"])
def test_AddTimeFeatures_empty_time_features(start, target, is_train: bool):
pred_length = 13
t = transform.AddTimeFeatures(
start_field=FieldName.START,
target_field=FieldName.TARGET,
output_field="myout",
pred_length=pred_length,
time_features=[],
)
data = {"start": start, "target": target}
res = t.map_transform(data, is_train=is_train)
assert res["myout"] is None
@pytest.mark.parametrize("is_train", TEST_VALUES["is_train"])
@pytest.mark.parametrize("target", TEST_VALUES["target"])
@pytest.mark.parametrize("start", TEST_VALUES["start"])
def test_AddAgeFeatures(start, target, is_train: bool):
pred_length = 13
t = transform.AddAgeFeature(
pred_length=pred_length,
target_field=FieldName.TARGET,
output_field="age",
log_scale=True,
)
data = {"start": start, "target": target}
out = t.map_transform(data, is_train=is_train)
expected_length = len(target) + (0 if is_train else pred_length)
assert out["age"].shape[-1] == expected_length
assert np.allclose(
out["age"],
np.log10(2.0 + np.arange(expected_length)).reshape((1, expected_length)),
)
@pytest.mark.parametrize("pick_incomplete", TEST_VALUES["allow_target_padding"])
@pytest.mark.parametrize("is_train", TEST_VALUES["is_train"])
@pytest.mark.parametrize("target", TEST_VALUES["target"])
@pytest.mark.parametrize("start", TEST_VALUES["start"])
def test_InstanceSplitter(start, target, is_train: bool, pick_incomplete: bool):
train_length = 100
pred_length = 13
t = transform.InstanceSplitter(
target_field=FieldName.TARGET,
is_pad_field=FieldName.IS_PAD,
start_field=FieldName.START,
forecast_start_field=FieldName.FORECAST_START,
train_sampler=transform.UniformSplitSampler(p=1.0),
past_length=train_length,
future_length=pred_length,
time_series_fields=["some_time_feature"],
pick_incomplete=pick_incomplete,
)
other_feat = np.arange(len(target) + 100)
data = {
"start": start,
"target": target,
"some_time_feature": other_feat,
"some_other_col": "ABC",
}
if not is_train and not pick_incomplete and len(target) < train_length:
with pytest.raises(AssertionError):
out = list(t.flatmap_transform(data, is_train=is_train))
return
else:
out = list(t.flatmap_transform(data, is_train=is_train))
if is_train:
assert len(out) == max(
0, len(target) - pred_length + 1 - (0 if pick_incomplete else train_length),
)
else:
assert len(out) == 1
for o in out:
assert "target" not in o
assert "some_time_feature" not in o
assert "some_other_col" in o
assert len(o["past_some_time_feature"]) == train_length
assert len(o["past_target"]) == train_length
if is_train:
assert len(o["future_target"]) == pred_length
assert len(o["future_some_time_feature"]) == pred_length
else:
assert len(o["future_target"]) == 0
assert len(o["future_some_time_feature"]) == pred_length
# expected_length = len(target) + (0 if is_train else pred_length)
# assert len(out['age']) == expected_length
# assert np.alltrue(out['age'] == np.log10(2.0 + np.arange(expected_length)))
@pytest.mark.parametrize("is_train", TEST_VALUES["is_train"])
@pytest.mark.parametrize("target", TEST_VALUES["target"])
@pytest.mark.parametrize("start", TEST_VALUES["start"])
@pytest.mark.parametrize(
"use_prediction_features", TEST_VALUES["use_prediction_features"]
)
@pytest.mark.parametrize("allow_target_padding", TEST_VALUES["allow_target_padding"])
def test_CanonicalInstanceSplitter(
start,
target,
is_train: bool,
use_prediction_features: bool,
allow_target_padding: bool,
):
train_length = 100
pred_length = 13
t = transform.CanonicalInstanceSplitter(
target_field=FieldName.TARGET,
is_pad_field=FieldName.IS_PAD,
start_field=FieldName.START,
forecast_start_field=FieldName.FORECAST_START,
instance_sampler=transform.UniformSplitSampler(p=1.0),
instance_length=train_length,
prediction_length=pred_length,
time_series_fields=["some_time_feature"],
allow_target_padding=allow_target_padding,
use_prediction_features=use_prediction_features,
)
other_feat = np.arange(len(target) + 100)
data = {
"start": start,
"target": target,
"some_time_feature": other_feat,
"some_other_col": "ABC",
}
out = list(t.flatmap_transform(data, is_train=is_train))
min_num_instances = 1 if allow_target_padding else 0
if is_train:
assert len(out) == max(min_num_instances, len(target) - train_length + 1)
else:
assert len(out) == 1
for o in out:
assert "target" not in o
assert "future_target" not in o
assert "some_time_feature" not in o
assert "some_other_col" in o
assert len(o["past_some_time_feature"]) == train_length
assert len(o["past_target"]) == train_length
if use_prediction_features and not is_train:
assert len(o["future_some_time_feature"]) == pred_length
def test_Transformation():
train_length = 100
ds = ListDataset(
[{"start": "2012-01-01", "target": [0.2] * train_length}], freq="1D"
)
pred_length = 10
t = transform.Chain(
trans=[
transform.AddTimeFeatures(
start_field=FieldName.START,
target_field=FieldName.TARGET,
output_field="time_feat",
time_features=[
time_feature.DayOfWeek(),
time_feature.DayOfMonth(),
time_feature.MonthOfYear(),
],
pred_length=pred_length,
),
transform.AddAgeFeature(
target_field=FieldName.TARGET,
output_field="age",
pred_length=pred_length,
log_scale=True,
),
transform.AddObservedValuesIndicator(
target_field=FieldName.TARGET, output_field="observed_values"
),
transform.VstackFeatures(
output_field="dynamic_feat",
input_fields=["age", "time_feat"],
drop_inputs=True,
),
transform.InstanceSplitter(
target_field=FieldName.TARGET,
is_pad_field=FieldName.IS_PAD,
start_field=FieldName.START,
forecast_start_field=FieldName.FORECAST_START,
train_sampler=transform.ExpectedNumInstanceSampler(num_instances=4),
past_length=train_length,
future_length=pred_length,
time_series_fields=["dynamic_feat", "observed_values"],
),
]
)
for u in t(iter(ds), is_train=True):
print(u)
@pytest.mark.parametrize("is_train", TEST_VALUES["is_train"])
def test_multi_dim_transformation(is_train):
train_length = 10
first_dim: list = list(np.arange(1, 11, 1))
first_dim[-1] = "NaN"
second_dim: list = list(np.arange(11, 21, 1))
second_dim[0] = "NaN"
ds = ListDataset(
data_iter=[{"start": "2012-01-01", "target": [first_dim, second_dim]}],
freq="1D",
one_dim_target=False,
)
pred_length = 2
# Looks weird - but this is necessary to assert the nan entries correctly.
first_dim[-1] = np.nan
second_dim[0] = np.nan
t = transform.Chain(
trans=[
transform.AddTimeFeatures(
start_field=FieldName.START,
target_field=FieldName.TARGET,
output_field="time_feat",
time_features=[
time_feature.DayOfWeek(),
time_feature.DayOfMonth(),
time_feature.MonthOfYear(),
],
pred_length=pred_length,
),
transform.AddAgeFeature(
target_field=FieldName.TARGET,
output_field="age",
pred_length=pred_length,
log_scale=True,
),
transform.AddObservedValuesIndicator(
target_field=FieldName.TARGET,
output_field="observed_values",
convert_nans=False,
),
transform.VstackFeatures(
output_field="dynamic_feat",
input_fields=["age", "time_feat"],
drop_inputs=True,
),
transform.InstanceSplitter(
target_field=FieldName.TARGET,
is_pad_field=FieldName.IS_PAD,
start_field=FieldName.START,
forecast_start_field=FieldName.FORECAST_START,
train_sampler=transform.ExpectedNumInstanceSampler(num_instances=4),
past_length=train_length,
future_length=pred_length,
time_series_fields=["dynamic_feat", "observed_values"],
time_first=False,
),
]
)
if is_train:
for u in t(iter(ds), is_train=True):
assert_shape(u["past_target"], (2, 10))
assert_shape(u["past_dynamic_feat"], (4, 10))
assert_shape(u["past_observed_values"], (2, 10))
assert_shape(u["future_target"], (2, 2))
assert_padded_array(
u["past_observed_values"],
np.array([[1.0] * 9 + [0.0], [0.0] + [1.0] * 9]),
u["past_is_pad"],
)
assert_padded_array(
u["past_target"], np.array([first_dim, second_dim]), u["past_is_pad"],
)
else:
for u in t(iter(ds), is_train=False):
assert_shape(u["past_target"], (2, 10))
assert_shape(u["past_dynamic_feat"], (4, 10))
assert_shape(u["past_observed_values"], (2, 10))
assert_shape(u["future_target"], (2, 0))
assert_padded_array(
u["past_observed_values"],
np.array([[1.0] * 9 + [0.0], [0.0] + [1.0] * 9]),
u["past_is_pad"],
)
assert_padded_array(
u["past_target"], np.array([first_dim, second_dim]), u["past_is_pad"],
)
def test_ExpectedNumInstanceSampler():
N = 6
train_length = 2
pred_length = 1
ds = make_dataset(N, train_length)
t = transform.Chain(
trans=[
transform.InstanceSplitter(
target_field=FieldName.TARGET,
is_pad_field=FieldName.IS_PAD,
start_field=FieldName.START,
forecast_start_field=FieldName.FORECAST_START,
train_sampler=transform.ExpectedNumInstanceSampler(num_instances=4),
past_length=train_length,
future_length=pred_length,
pick_incomplete=True,
)
]
)
scale_hist = ScaleHistogram()
repetition = 2
for i in range(repetition):
for data in t(iter(ds), is_train=True):
target_values = data["past_target"]
# for simplicity, discard values that are zeros to avoid confusion with padding
target_values = target_values[target_values > 0]
scale_hist.add(target_values)
expected_values = {i: 2 ** i * repetition for i in range(1, N)}
assert expected_values == scale_hist.bin_counts
def test_BucketInstanceSampler():
N = 6
train_length = 2
pred_length = 1
ds = make_dataset(N, train_length)
dataset_stats = calculate_dataset_statistics(ds)
t = transform.Chain(
trans=[
transform.InstanceSplitter(
target_field=FieldName.TARGET,
is_pad_field=FieldName.IS_PAD,
start_field=FieldName.START,
forecast_start_field=FieldName.FORECAST_START,
train_sampler=transform.BucketInstanceSampler(
dataset_stats.scale_histogram
),
past_length=train_length,
future_length=pred_length,
pick_incomplete=True,
)
]
)
scale_hist = ScaleHistogram()
repetition = 200
for i in range(repetition):
for data in t(iter(ds), is_train=True):
target_values = data["past_target"]
# for simplicity, discard values that are zeros to avoid confusion with padding
target_values = target_values[target_values > 0]
scale_hist.add(target_values)
expected_values = {i: repetition for i in range(1, N)}
found_values = scale_hist.bin_counts
for i in range(1, N):
assert abs(expected_values[i] - found_values[i] < expected_values[i] * 0.3)
def test_cdf_to_gaussian_transformation():
def make_test_data():
target = np.array(
[0, 0, 0, 0, 10, 10, 20, 20, 30, 30, 40, 50, 59, 60, 60, 70, 80, 90, 100,]
).tolist()
np.random.shuffle(target)
multi_dim_target = np.array([target, target]).transpose()
past_is_pad = np.array([[0] * len(target)]).transpose()
past_observed_target = np.array(
[[1] * len(target), [1] * len(target)]
).transpose()
ds = ListDataset(
# Mimic output from InstanceSplitter
data_iter=[
{
"start": "2012-01-01",
"target": multi_dim_target,
"past_target": multi_dim_target,
"future_target": multi_dim_target,
"past_is_pad": past_is_pad,
f"past_{FieldName.OBSERVED_VALUES}": past_observed_target,
}
],
freq="1D",
one_dim_target=False,
)
return ds
def make_fake_output(u: DataEntry):
fake_output = np.expand_dims(
np.expand_dims(u["past_target_cdf"], axis=0), axis=0
)
return fake_output
ds = make_test_data()
t = transform.Chain(
trans=[
transform.CDFtoGaussianTransform(
target_field=FieldName.TARGET,
observed_values_field=FieldName.OBSERVED_VALUES,
max_context_length=20,
target_dim=2,
)
]
)
for u in t(iter(ds), is_train=False):
fake_output = make_fake_output(u)
# Fake transformation chain output
u["past_target_sorted"] = torch.tensor(
np.expand_dims(u["past_target_sorted"], axis=0)
)
u["slopes"] = torch.tensor(np.expand_dims(u["slopes"], axis=0))
u["intercepts"] = torch.tensor(np.expand_dims(u["intercepts"], axis=0))
back_transformed = transform.cdf_to_gaussian_forward_transform(u, fake_output)
# Get any sample/batch (slopes[i][:, d]they are all the same)
back_transformed = back_transformed[0][0]
original_target = u["target"]
# Original target and back-transformed target should be the same
assert np.allclose(original_target, back_transformed)
def test_gaussian_cdf():
try:
from scipy.stats import norm
except:
pytest.skip("scipy not installed skipping test for erf")
x = np.array(
[-1000, -100, -10] + np.linspace(-2, 2, 1001).tolist() + [10, 100, 1000]
)
y_gluonts = transform.CDFtoGaussianTransform.standard_gaussian_cdf(x)
y_scipy = norm.cdf(x)
assert np.allclose(y_gluonts, y_scipy, atol=1e-7)
def test_gaussian_ppf():
try:
from scipy.stats import norm
except:
pytest.skip("scipy not installed skipping test for erf")
x = np.linspace(0.0001, 0.9999, 1001)
y_gluonts = transform.CDFtoGaussianTransform.standard_gaussian_ppf(x)
y_scipy = norm.ppf(x)
assert np.allclose(y_gluonts, y_scipy, atol=1e-7)
def test_target_dim_indicator():
target = np.array([0, 2, 3, 10]).tolist()
multi_dim_target = np.array([target, target, target, target])
dataset = ListDataset(
data_iter=[{"start": "2012-01-01", "target": multi_dim_target}],
freq="1D",
one_dim_target=False,
)
t = transform.Chain(
trans=[
transform.TargetDimIndicator(
target_field=FieldName.TARGET, field_name="target_dimensions"
)
]
)
for data_entry in t(dataset, is_train=True):
assert (data_entry["target_dimensions"] == np.array([0, 1, 2, 3])).all()
@pytest.fixture
def point_process_dataset():
ia_times = np.array([0.2, 0.7, 0.2, 0.5, 0.3, 0.3, 0.2, 0.1])
marks = np.array([0, 1, 2, 0, 1, 2, 2, 2])
lds = ListDataset(
[
{
"target": np.c_[ia_times, marks].T,
"start": pd.Timestamp("2011-01-01 00:00:00", freq="H"),
"end": pd.Timestamp("2011-01-01 03:00:00", freq="H"),
}
],
freq="H",
one_dim_target=False,
)
return lds
class MockContinuousTimeSampler(transform.ContinuousTimePointSampler):
# noinspection PyMissingConstructor,PyUnusedLocal
def __init__(self, ret_values, *args, **kwargs):
self._ret_values = ret_values
def __call__(self, *args, **kwargs):
return np.array(self._ret_values)
def test_ctsplitter_mask_sorted(point_process_dataset):
d = next(iter(point_process_dataset))
ia_times = d["target"][0, :]
ts = np.cumsum(ia_times)
splitter = transform.ContinuousTimeInstanceSplitter(
2, 1, train_sampler=transform.ContinuousTimeUniformSampler(num_instances=10),
)
# no boundary conditions
res = splitter._mask_sorted(ts, 1, 2)
assert all([a == b for a, b in zip([2, 3, 4], res)])
# lower bound equal, exclusive of upper bound
res = splitter._mask_sorted(np.array([1, 2, 3, 4, 5, 6]), 1, 2)
assert all([a == b for a, b in zip([0], res)])
def test_ctsplitter_no_train_last_point(point_process_dataset):
splitter = transform.ContinuousTimeInstanceSplitter(
2, 1, train_sampler=transform.ContinuousTimeUniformSampler(num_instances=10),
)
iter_de = splitter(point_process_dataset, is_train=False)
d_out = next(iter(iter_de))
assert "future_target" not in d_out
assert "future_valid_length" not in d_out
assert "past_target" in d_out
assert "past_valid_length" in d_out
assert d_out["past_valid_length"] == 6
assert np.allclose(
[0.1, 0.5, 0.3, 0.3, 0.2, 0.1], d_out["past_target"][..., 0], atol=0.01
)
def test_ctsplitter_train_correct(point_process_dataset):
splitter = transform.ContinuousTimeInstanceSplitter(
1,
1,
train_sampler=MockContinuousTimeSampler(
ret_values=[1.01, 1.5, 1.99], num_instances=3
),
)
iter_de = splitter(point_process_dataset, is_train=True)
outputs = list(iter_de)
assert outputs[0]["past_valid_length"] == 2
assert outputs[0]["future_valid_length"] == 3
assert np.allclose(outputs[0]["past_target"], np.array([[0.19, 0.7], [0, 1]]).T)
assert np.allclose(
outputs[0]["future_target"], np.array([[0.09, 0.5, 0.3], [2, 0, 1]]).T
)
assert outputs[1]["past_valid_length"] == 2
assert outputs[1]["future_valid_length"] == 4
assert outputs[2]["past_valid_length"] == 3
assert outputs[2]["future_valid_length"] == 3
def test_ctsplitter_train_correct_out_count(point_process_dataset):
# produce new TPP data by shuffling existing TS instance
def shuffle_iterator(num_duplications=5):
for entry in point_process_dataset:
for i in range(num_duplications):
d = dict.copy(entry)
d["target"] = np.random.permutation(d["target"].T).T
yield d
splitter = transform.ContinuousTimeInstanceSplitter(
1,
1,
train_sampler=MockContinuousTimeSampler(
ret_values=[1.01, 1.5, 1.99], num_instances=3
),
)
iter_de = splitter(shuffle_iterator(), is_train=True)
outputs = list(iter_de)
assert len(outputs) == 5 * 3
def test_ctsplitter_train_samples_correct_times(point_process_dataset):
splitter = transform.ContinuousTimeInstanceSplitter(
1.25, 1.25, train_sampler=transform.ContinuousTimeUniformSampler(20)
)
iter_de = splitter(point_process_dataset, is_train=True)
assert all(
[
(
pd.Timestamp("2011-01-01 01:15:00")
<= d["forecast_start"]
<= pd.Timestamp("2011-01-01 01:45:00")
)
for d in iter_de
]
)
def test_ctsplitter_train_short_intervals(point_process_dataset):
splitter = transform.ContinuousTimeInstanceSplitter(
0.01,
0.01,
train_sampler=MockContinuousTimeSampler(
ret_values=[1.01, 1.5, 1.99], num_instances=3
),
)
iter_de = splitter(point_process_dataset, is_train=True)
for d in iter_de:
assert d["future_valid_length"] == d["past_valid_length"] == 0
assert np.prod(np.shape(d["past_target"])) == 0
assert np.prod(np.shape(d["future_target"])) == 0
def make_dataset(N, train_length):
# generates 2 ** N - 1 timeseries with constant increasing values
n = 2 ** N - 1
targets = np.ones((n, train_length))
for i in range(0, n):
targets[i, :] = targets[i, :] * i
ds = ListDataset(
data_iter=[{"start": "2012-01-01", "target": targets[i, :]} for i in range(n)],
freq="1D",
)
return ds
def assert_shape(array: np.array, reference_shape: Tuple[int, int]):
assert (
array.shape == reference_shape
), f"Shape should be {reference_shape} but found {array.shape}."
def assert_padded_array(
sampled_array: np.array, reference_array: np.array, padding_array: np.array
):
num_padded = int(np.sum(padding_array))
sampled_no_padding = sampled_array[:, num_padded:]
reference_array = np.roll(reference_array, num_padded, axis=1)
reference_no_padding = reference_array[:, num_padded:]
# Convert nans to dummy value for assertion because
# np.nan == np.nan -> False.
reference_no_padding[np.isnan(reference_no_padding)] = 9999.0
sampled_no_padding[np.isnan(sampled_no_padding)] = 9999.0
reference_no_padding = np.array(reference_no_padding, dtype=np.float32)
assert (sampled_no_padding == reference_no_padding).all(), (
f"Sampled and reference arrays do not match. '"
f"Got {sampled_no_padding} but should be {reference_no_padding}."
)