From 48f5cbbe09154b86af34fb9955c83ceb33803f2a Mon Sep 17 00:00:00 2001 From: "Dr. Kashif Rasul" Date: Fri, 27 Dec 2019 20:54:09 +0100 Subject: [PATCH] added dataset rerpository --- pts/dataset/__init__.py | 9 +- pts/dataset/file_dataset.py | 2 +- pts/dataset/repository/__init__.py | 14 ++ pts/dataset/repository/_gp_copula_2019.py | 160 ++++++++++++++++++ pts/dataset/repository/_lstnet.py | 195 ++++++++++++++++++++++ pts/dataset/repository/_m4.py | 80 +++++++++ pts/dataset/repository/_util.py | 70 ++++++++ pts/dataset/repository/datasets.py | 171 +++++++++++++++++++ pts/dataset/utils.py | 106 ++++++++++++ 9 files changed, 805 insertions(+), 2 deletions(-) create mode 100644 pts/dataset/repository/__init__.py create mode 100644 pts/dataset/repository/_gp_copula_2019.py create mode 100644 pts/dataset/repository/_lstnet.py create mode 100644 pts/dataset/repository/_m4.py create mode 100644 pts/dataset/repository/_util.py create mode 100644 pts/dataset/repository/datasets.py diff --git a/pts/dataset/__init__.py b/pts/dataset/__init__.py index a9737a4..6a78a86 100644 --- a/pts/dataset/__init__.py +++ b/pts/dataset/__init__.py @@ -3,7 +3,14 @@ from .list_dataset import ListDataset from .file_dataset import FileDataset from .loader import TrainDataLoader, InferenceDataLoader from .process import ProcessStartField, ProcessDataEntry -from .utils import to_pandas +from .utils import ( + to_pandas, + load_datasets, + save_datasets, + serialize_data_entry, + frequency_add, + forecast_start, +) from .stat import DatasetStatistics, ScaleHistogram, calculate_dataset_statistics from .artificial import ( ArtificialDataset, diff --git a/pts/dataset/file_dataset.py b/pts/dataset/file_dataset.py index c0edbb0..f221ea5 100644 --- a/pts/dataset/file_dataset.py +++ b/pts/dataset/file_dataset.py @@ -101,4 +101,4 @@ class FileDataset(Dataset): List[Path] List of the paths of all files composing the dataset. """ - return glob.glob(self.path) + return glob.glob(str(self.path)) diff --git a/pts/dataset/repository/__init__.py b/pts/dataset/repository/__init__.py new file mode 100644 index 0000000..0bd39eb --- /dev/null +++ b/pts/dataset/repository/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + +from .datasets import get_dataset, dataset_recipes \ No newline at end of file diff --git a/pts/dataset/repository/_gp_copula_2019.py b/pts/dataset/repository/_gp_copula_2019.py new file mode 100644 index 0000000..3e5d822 --- /dev/null +++ b/pts/dataset/repository/_gp_copula_2019.py @@ -0,0 +1,160 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + +""" +Loads the datasets used in Salinas et al. 2019 (https://tinyurl.com/woyhhqy). +This wrapper downloads and unpacks them so they don'thave to be attached as +large files in GluonTS master. +""" +import json +import os +import tarfile +import shutil +from pathlib import Path +from typing import NamedTuple, Optional +from urllib import request + +from pts.dataset import FileDataset, FieldName + +from ._util import save_to_file, to_dict, metadata + + +class GPCopulaDataset(NamedTuple): + name: str + url: str + num_series: int + prediction_length: int + freq: str + rolling_evaluations: int + max_target_dim: Optional[int] = None + + +root = ( + "https://raw.githubusercontent.com/mbohlkeschneider/gluon-ts/mv_release/datasets/" +) + +datasets_info = { + "exchange_rate_nips": GPCopulaDataset( + name="exchange_rate_nips", + url=root + "exchange_rate_nips.tar.gz", + num_series=8, + prediction_length=30, + freq="B", + rolling_evaluations=5, + max_target_dim=None, + ), + "electricity_nips": GPCopulaDataset( + name="electricity_nips", + url=root + "electricity_nips.tar.gz", + # original dataset can be found at https://archive.ics.uci.edu/ml/datasets/ElectricityLoadDiagrams20112014# + num_series=370, + prediction_length=24, + freq="H", + rolling_evaluations=7, + max_target_dim=None, + ), + "traffic_nips": GPCopulaDataset( + name="traffic_nips", + url=root + "traffic_nips.tar.gz", + # note there are 963 in the original dataset from https://archive.ics.uci.edu/ml/datasets/PEMS-SF + num_series=963, + prediction_length=24, + freq="H", + rolling_evaluations=7, + max_target_dim=None, + ), + "solar_nips": GPCopulaDataset( + name="solar-energy", + url=root + "solar_nips.tar.gz", + num_series=137, + prediction_length=24, + freq="H", + rolling_evaluations=7, + max_target_dim=None, + ), + "wiki-rolling_nips": GPCopulaDataset( + name="wiki-rolling_nips", + # That file lives on GitHub Large file storage (lfs). We need to use + # the exact link, otherwise it will only open the lfs pointer file. + url="https://github.com/mbohlkeschneider/gluon-ts/raw/650ad5ffe92d20e89d491966b6d8b4459e219be8/datasets/wiki-rolling_nips.tar.gz", + num_series=9535, + prediction_length=30, + freq="D", + rolling_evaluations=5, + max_target_dim=2000, + ), + "taxi_30min": GPCopulaDataset( + name="taxi_30min", + url=root + "taxi_30min.tar.gz", + num_series=130, + prediction_length=24, + freq="30min", + rolling_evaluations=57, + max_target_dim=None, + ), +} + + +def generate_gp_copula_dataset(dataset_path: Path, dataset_name: str): + ds_info = datasets_info[dataset_name] + os.makedirs(dataset_path, exist_ok=True) + + download_dataset(dataset_path.parent, ds_info) + save_metadata(dataset_path, ds_info) + save_dataset(dataset_path / "train", ds_info) + save_dataset(dataset_path / "test", ds_info) + clean_up_dataset(dataset_path, ds_info) + + +def download_dataset(dataset_path: Path, ds_info: GPCopulaDataset): + request.urlretrieve(ds_info.url, dataset_path / f"{ds_info.name}.tar.gz") + + with tarfile.open(dataset_path / f"{ds_info.name}.tar.gz") as tar: + tar.extractall(path=dataset_path) + + +def save_metadata(dataset_path: Path, ds_info: GPCopulaDataset): + with open(dataset_path / "metadata.json", "w") as f: + f.write( + json.dumps( + metadata( + cardinality=ds_info.num_series, + freq=ds_info.freq, + prediction_length=ds_info.prediction_length, + ) + ) + ) + + +def save_dataset(dataset_path: Path, ds_info: GPCopulaDataset): + dataset = list(FileDataset(dataset_path, freq=ds_info.freq)) + shutil.rmtree(dataset_path) + train_file = dataset_path / "data.json" + save_to_file( + train_file, + [ + to_dict( + target_values=data_entry[FieldName.TARGET], + start=data_entry[FieldName.START], + # Handles adding categorical features of rolling + # evaluation dates + cat=[cat - ds_info.num_series * (cat // ds_info.num_series)], + ) + for cat, data_entry in enumerate(dataset) + ], + ) + + +def clean_up_dataset(dataset_path: Path, ds_info: GPCopulaDataset): + os.remove(dataset_path.parent / f"{ds_info.name}.tar.gz") + shutil.rmtree(dataset_path / "metadata") diff --git a/pts/dataset/repository/_lstnet.py b/pts/dataset/repository/_lstnet.py new file mode 100644 index 0000000..638a321 --- /dev/null +++ b/pts/dataset/repository/_lstnet.py @@ -0,0 +1,195 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + +""" +Here we reuse the datasets used by LSTNet as the processed url of the datasets +are available on GitHub. +""" +import json +import os +from pathlib import Path +from typing import List, NamedTuple, Optional + +import pandas as pd + +from pts.dataset import frequency_add + +from ._util import save_to_file, to_dict, metadata + + +def load_from_pandas( + df: pd.DataFrame, time_index: pd.DatetimeIndex, agg_freq: Optional[str] = None, +) -> List[pd.Series]: + df = df.set_index(time_index) + + pivot_df = df.transpose() + pivot_df.head() + + timeseries = [] + for row in pivot_df.iterrows(): + ts = pd.Series(row[1].values, index=time_index) + if agg_freq is not None: + ts = ts.resample(agg_freq).sum() + first_valid = ts[ts.notnull()].index[0] + last_valid = ts[ts.notnull()].index[-1] + ts = ts[first_valid:last_valid] + + timeseries.append(ts) + + return timeseries + + +class LstnetDataset(NamedTuple): + name: str + url: str + num_series: int + num_time_steps: int + prediction_length: int + rolling_evaluations: int + freq: str + start_date: str + agg_freq: Optional[str] = None + + +root = ( + "https://raw.githubusercontent.com/laiguokun/multivariate-time-series-data/master/" +) + +datasets_info = { + "exchange_rate": LstnetDataset( + name="exchange_rate", + url=root + "exchange_rate/exchange_rate.txt.gz", + num_series=8, + num_time_steps=7588, + prediction_length=30, + rolling_evaluations=5, + start_date="1990-01-01", + freq="1B", + agg_freq=None, + ), + "electricity": LstnetDataset( + name="electricity", + url=root + "electricity/electricity.txt.gz", + # original dataset can be found at https://archive.ics.uci.edu/ml/datasets/ElectricityLoadDiagrams20112014# + # the aggregated ones that is used from LSTNet filters out from the initial 370 series the one with no data + # in 2011 + num_series=321, + num_time_steps=26304, + prediction_length=24, + rolling_evaluations=7, + start_date="2012-01-01", + freq="1H", + agg_freq=None, + ), + "traffic": LstnetDataset( + name="traffic", + url=root + "traffic/traffic.txt.gz", + # note there are 963 in the original dataset from https://archive.ics.uci.edu/ml/datasets/PEMS-SF + # but only 862 in LSTNet + num_series=862, + num_time_steps=17544, + prediction_length=24, + rolling_evaluations=7, + start_date="2015-01-01", + freq="H", + agg_freq=None, + ), + "solar-energy": LstnetDataset( + name="solar-energy", + url=root + "solar-energy/solar_AL.txt.gz", + num_series=137, + num_time_steps=52560, + prediction_length=24, + rolling_evaluations=7, + start_date="2006-01-01", + freq="10min", + agg_freq="1H", + ), +} + + +def generate_lstnet_dataset(dataset_path: Path, dataset_name: str): + ds_info = datasets_info[dataset_name] + + os.makedirs(dataset_path, exist_ok=True) + + with open(dataset_path / "metadata.json", "w") as f: + f.write( + json.dumps( + metadata( + cardinality=ds_info.num_series, + freq=ds_info.freq, + prediction_length=ds_info.prediction_length, + ) + ) + ) + + train_file = dataset_path / "train" / "data.json" + test_file = dataset_path / "test" / "data.json" + + time_index = pd.date_range( + start=ds_info.start_date, freq=ds_info.freq, periods=ds_info.num_time_steps, + ) + + df = pd.read_csv(ds_info.url, header=None) + + assert df.shape == ( + ds_info.num_time_steps, + ds_info.num_series, + ), f"expected num_time_steps/num_series {(ds_info.num_time_steps, ds_info.num_series)} but got {df.shape}" + + timeseries = load_from_pandas( + df=df, time_index=time_index, agg_freq=ds_info.agg_freq + ) + + # the last date seen during training + ts_index = timeseries[0].index + training_end = ts_index[int(len(ts_index) * (8 / 10))] + + train_ts = [] + for cat, ts in enumerate(timeseries): + sliced_ts = ts[:training_end] + if len(sliced_ts) > 0: + train_ts.append( + to_dict( + target_values=sliced_ts.values, start=sliced_ts.index[0], cat=[cat], + ) + ) + + assert len(train_ts) == ds_info.num_series + + save_to_file(train_file, train_ts) + + # time of the first prediction + prediction_dates = [ + frequency_add(training_end, i * ds_info.prediction_length) + for i in range(ds_info.rolling_evaluations) + ] + + test_ts = [] + for prediction_start_date in prediction_dates: + for cat, ts in enumerate(timeseries): + # print(prediction_start_date) + prediction_end_date = frequency_add( + prediction_start_date, ds_info.prediction_length + ) + sliced_ts = ts[:prediction_end_date] + test_ts.append( + to_dict( + target_values=sliced_ts.values, start=sliced_ts.index[0], cat=[cat], + ) + ) + + assert len(test_ts) == ds_info.num_series * ds_info.rolling_evaluations + + save_to_file(test_file, test_ts) diff --git a/pts/dataset/repository/_m4.py b/pts/dataset/repository/_m4.py new file mode 100644 index 0000000..d24dd94 --- /dev/null +++ b/pts/dataset/repository/_m4.py @@ -0,0 +1,80 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + +from pathlib import Path +import os +import json + +import pandas as pd +import numpy as np + +from ._util import save_to_file, to_dict, metadata + + +def generate_m4_dataset( + dataset_path: Path, m4_freq: str, pandas_freq: str, prediction_length: int +): + m4_dataset_url = "https://github.com/M4Competition/M4-methods/raw/master/Dataset" + train_df = pd.read_csv(f"{m4_dataset_url}/Train/{m4_freq}-train.csv", index_col=0) + test_df = pd.read_csv(f"{m4_dataset_url}/Test/{m4_freq}-test.csv", index_col=0) + + os.makedirs(dataset_path, exist_ok=True) + + with open(dataset_path / "metadata.json", "w") as f: + f.write( + json.dumps( + metadata( + cardinality=len(train_df), + freq=pandas_freq, + prediction_length=prediction_length, + ) + ) + ) + + train_file = dataset_path / "train" / "data.json" + test_file = dataset_path / "test" / "data.json" + + train_target_values = [ts[~np.isnan(ts)] for ts in train_df.values] + + test_target_values = [ + np.hstack([train_ts, test_ts]) + for train_ts, test_ts in zip(train_target_values, test_df.values) + ] + + if m4_freq == "Yearly": + # some time series have more than 300 years which can not be represented in pandas, + # this is probably due to a misclassification of those time series as Yearly + # we simply use only the last 300 years for training + # note this does not affect test time as prediction length is less than 300 years + train_target_values = [ts[-300:] for ts in train_target_values] + test_target_values = [ts[-300:] for ts in test_target_values] + + # the original dataset did not include time stamps, so we use a mock start date for each time series + # we use the earliest point available in pandas + mock_start_dataset = "1750-01-01 00:00:00" + + save_to_file( + train_file, + [ + to_dict(target_values=target, start=mock_start_dataset, cat=[cat]) + for cat, target in enumerate(train_target_values) + ], + ) + + save_to_file( + test_file, + [ + to_dict(target_values=target, start=mock_start_dataset, cat=[cat]) + for cat, target in enumerate(test_target_values) + ], + ) diff --git a/pts/dataset/repository/_util.py b/pts/dataset/repository/_util.py new file mode 100644 index 0000000..46f45a3 --- /dev/null +++ b/pts/dataset/repository/_util.py @@ -0,0 +1,70 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + +import json +import os +from pathlib import Path +from typing import Dict, List, Optional + +import numpy as np + + +def to_dict(target_values: np.ndarray, start: str, cat: Optional[List[int]] = None): + def serialize(x): + if np.isnan(x): + return "NaN" + else: + # return x + return float("{0:.6f}".format(float(x))) + + res = { + "start": str(start), + "target": [serialize(x) for x in target_values], + } + + if cat is not None: + res["feat_static_cat"] = cat + + return res + + +def save_to_file(path: Path, data: List[Dict]): + print(f"saving time-series into {path}") + path_dir = os.path.dirname(path) + os.makedirs(path_dir, exist_ok=True) + with open(path, "wb") as fp: + for d in data: + fp.write(json.dumps(d).encode("utf-8")) + fp.write("\n".encode("utf-8")) + + +def get_download_path() -> Path: + """ + + Returns + ------- + Path + default path to download datasets + /home/username/.pytorch/pytorch-ts/ + """ + return Path(str(Path.home() / ".pytorch" / "pytorch-ts")) + + +def metadata(cardinality: int, freq: str, prediction_length: int): + return { + "freq": freq, + "prediction_length": prediction_length, + "feat_static_cat": [ + {"name": "feat_static_cat", "cardinality": str(cardinality)} + ], + } diff --git a/pts/dataset/repository/datasets.py b/pts/dataset/repository/datasets.py new file mode 100644 index 0000000..b7b3afb --- /dev/null +++ b/pts/dataset/repository/datasets.py @@ -0,0 +1,171 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + +import logging +from collections import OrderedDict +from functools import partial +from pathlib import Path + +from pts.dataset import ConstantDataset, TrainDatasets, load_datasets + +from ._lstnet import generate_lstnet_dataset +from ._m4 import generate_m4_dataset +from ._gp_copula_2019 import generate_gp_copula_dataset +from ._util import get_download_path + +m4_freq = "Hourly" +pandas_freq = "H" +dataset_path = Path(f"m4-{m4_freq}") +prediction_length = 48 + +dataset_recipes = OrderedDict( + { + "exchange_rate": partial(generate_lstnet_dataset, dataset_name="exchange_rate"), + "solar-energy": partial(generate_lstnet_dataset, dataset_name="solar-energy"), + "electricity": partial(generate_lstnet_dataset, dataset_name="electricity"), + "traffic": partial(generate_lstnet_dataset, dataset_name="traffic"), + "exchange_rate_nips": partial( + generate_gp_copula_dataset, dataset_name="exchange_rate_nips" + ), + "electricity_nips": partial( + generate_gp_copula_dataset, dataset_name="electricity_nips" + ), + "traffic_nips": partial( + generate_gp_copula_dataset, dataset_name="traffic_nips" + ), + "solar_nips": partial(generate_gp_copula_dataset, dataset_name="solar_nips"), + "wiki-rolling_nips": partial( + generate_gp_copula_dataset, dataset_name="wiki-rolling_nips" + ), + "taxi_30min": partial(generate_gp_copula_dataset, dataset_name="taxi_30min"), + "m4_hourly": partial( + generate_m4_dataset, + m4_freq="Hourly", + pandas_freq="H", + prediction_length=48, + ), + "m4_daily": partial( + generate_m4_dataset, m4_freq="Daily", pandas_freq="D", prediction_length=14, + ), + "m4_weekly": partial( + generate_m4_dataset, + m4_freq="Weekly", + pandas_freq="W", + prediction_length=13, + ), + "m4_monthly": partial( + generate_m4_dataset, + m4_freq="Monthly", + pandas_freq="M", + prediction_length=18, + ), + "m4_quarterly": partial( + generate_m4_dataset, + m4_freq="Quarterly", + pandas_freq="3M", + prediction_length=8, + ), + "m4_yearly": partial( + generate_m4_dataset, + m4_freq="Yearly", + pandas_freq="12M", + prediction_length=6, + ), + } +) + +dataset_names = list(dataset_recipes.keys()) + +default_dataset_path = get_download_path() / "datasets" + + +def materialize_dataset( + dataset_name: str, path: Path = default_dataset_path, regenerate: bool = False, +) -> Path: + """ + Ensures that the dataset is materialized under the `path / dataset_name` + path. + + Parameters + ---------- + dataset_name + name of the dataset, for instance "m4_hourly" + regenerate + whether to regenerate the dataset even if a local file is present. + If this flag is False and the file is present, the dataset will not + be downloaded again. + path + where the dataset should be saved + Returns + ------- + the path where the dataset is materialized + """ + assert dataset_name in dataset_recipes.keys(), ( + f"{dataset_name} is not present, please choose one from " + f"{dataset_recipes.keys()}." + ) + + path.mkdir(parents=True, exist_ok=True) + dataset_path = path / dataset_name + + dataset_recipe = dataset_recipes[dataset_name] + + if not dataset_path.exists() or regenerate: + logging.info(f"downloading and processing {dataset_name}") + dataset_recipe(dataset_path=dataset_path) + else: + logging.info(f"using dataset already processed in path {dataset_path}.") + + return dataset_path + + +def get_dataset( + dataset_name: str, path: Path = default_dataset_path, regenerate: bool = False, +) -> TrainDatasets: + """ + Get a repository dataset. + + The datasets that can be obtained through this function have been used + with different processing over time by several papers (e.g., [SFG17]_, + [LCY+18]_, and [YRD15]_). + + Parameters + ---------- + dataset_name + name of the dataset, for instance "m4_hourly" + regenerate + whether to regenerate the dataset even if a local file is present. + If this flag is False and the file is present, the dataset will not + be downloaded again. + path + where the dataset should be saved + Returns + ------- + dataset obtained by either downloading or reloading from local file. + """ + dataset_path = materialize_dataset(dataset_name, path, regenerate) + + return load_datasets( + metadata=dataset_path / "metadata.json", + train=dataset_path / "train" / "*.json", + test=dataset_path / "test" / "*.json", + ) + + +if __name__ == "__main__": + + for dataset in dataset_names: + print(f"generate {dataset}") + ds = get_dataset(dataset, regenerate=True) + print(ds.metadata) + print(sum(1 for _ in list(iter(ds.train)))) diff --git a/pts/dataset/utils.py b/pts/dataset/utils.py index 79e9949..9cf9c05 100644 --- a/pts/dataset/utils.py +++ b/pts/dataset/utils.py @@ -1,4 +1,20 @@ +from pathlib import Path +import shutil + +import numpy as np import pandas as pd +import rapidjson as json + +from .common import TrainDatasets, MetaData +from .file_dataset import FileDataset + + +def frequency_add(ts: pd.Timestamp, amount: int) -> pd.Timestamp: + return ts + ts.freq * amount + + +def forecast_start(entry): + return frequency_add(entry["start"], len(entry["target"])) def to_pandas(instance: dict, freq: str = None) -> pd.Series: @@ -24,3 +40,93 @@ def to_pandas(instance: dict, freq: str = None) -> pd.Series: freq = start.freqstr index = pd.date_range(start=start, periods=len(target), freq=freq) return pd.Series(target, index=index) + + +def load_datasets(metadata, train, test) -> TrainDatasets: + """ + Loads a dataset given metadata, train and test path. + Parameters + ---------- + metadata + Path to the metadata file + train + Path to the training dataset files. + test + Path to the test dataset files. + Returns + ------- + TrainDatasets + An object collecting metadata, training data, test data. + """ + meta = MetaData.parse_file(metadata) + train_ds = FileDataset(train, meta.freq) + test_ds = FileDataset(test, meta.freq) if test else None + + return TrainDatasets(metadata=meta, train=train_ds, test=test_ds) + + +def save_datasets(dataset: TrainDatasets, path_str: str, overwrite=True) -> None: + """ + Saves an TrainDatasets object to a JSON Lines file. + + Parameters + ---------- + dataset + The training datasets. + path_str + Where to save the dataset. + overwrite + Whether to delete previous version in this folder. + """ + path = Path(path_str) + + if overwrite: + shutil.rmtree(path, ignore_errors=True) + + def dump_line(f, line): + f.write(json.dumps(line).encode("utf-8")) + f.write("\n".encode("utf-8")) + + (path / "metadata").mkdir(parents=True) + with open(path / "metadata/metadata.json", "wb") as f: + dump_line(f, dataset.metadata.dict()) + + (path / "train").mkdir(parents=True) + with open(path / "train/data.json", "wb") as f: + for entry in dataset.train: + dump_line(f, serialize_data_entry(entry)) + + if dataset.test is not None: + (path / "test").mkdir(parents=True) + with open(path / "test/data.json", "wb") as f: + for entry in dataset.test: + dump_line(f, serialize_data_entry(entry)) + + +def serialize_data_entry(data): + """ + Encode the numpy values in the a DataEntry dictionary into lists so the + dictionary can be JSON serialized. + + Parameters + ---------- + data + The dictionary to be transformed. + + Returns + ------- + Dict + The transformed dictionary, where all fields where transformed into + strings. + """ + + def serialize_field(field): + if isinstance(field, np.ndarray): + # circumvent https://github.com/micropython/micropython/issues/3511 + nan_ix = np.isnan(field) + field = field.astype(np.object_) + field[nan_ix] = "NaN" + return field.tolist() + return str(field) + + return {k: serialize_field(v) for k, v in data.items() if v is not None}