diff --git a/pts/dataset/__init__.py b/pts/dataset/__init__.py index 6a78a86..abbc308 100644 --- a/pts/dataset/__init__.py +++ b/pts/dataset/__init__.py @@ -19,4 +19,5 @@ from .artificial import ( RecipeDataset, constant_dataset, default_synthetic, + generate_sf2, ) diff --git a/pts/dataset/artificial.py b/pts/dataset/artificial.py index 6167ac4..fdc3d4f 100644 --- a/pts/dataset/artificial.py +++ b/pts/dataset/artificial.py @@ -1,9 +1,11 @@ +import os import math import random from typing import Callable, List, NamedTuple, Optional, Tuple, Union import numpy as np import pandas as pd +import rapidjson as json from .common import ( MetaData, @@ -785,3 +787,34 @@ def constant_dataset() -> Tuple[DatasetInfo, Dataset, Dataset]: ) return info, train_ds, test_ds + + +def generate_sf2( + filename: str, time_series: List, is_missing: bool, num_missing: int +) -> None: + # This function generates the test and train json files which will be converted to csv format + if not os.path.exists(os.path.dirname(filename)): + os.makedirs(os.path.dirname(filename)) + with open(filename, "w") as json_file: + for ts in time_series: + if is_missing: + target = [] # type: List + # For Forecast don't output feat_static_cat and feat_static_real + for j, val in enumerate(ts[FieldName.TARGET]): + # only add ones that are not missing + if j != 0 and j % num_missing == 0: + target.append(None) + else: + target.append(val) + ts[FieldName.TARGET] = target + ts.pop(FieldName.FEAT_STATIC_CAT, None) + ts.pop(FieldName.FEAT_STATIC_REAL, None) + # Chop features in training set + if FieldName.FEAT_DYNAMIC_REAL in ts.keys() and "train" in filename: + # TODO: Fix for missing values + for i, feat_dynamic_real in enumerate(ts[FieldName.FEAT_DYNAMIC_REAL]): + ts[FieldName.FEAT_DYNAMIC_REAL][i] = feat_dynamic_real[ + : len(ts[FieldName.TARGET]) + ] + json.dump(ts, json_file) + json_file.write("\n") diff --git a/pts/dataset/repository/_artificial.py b/pts/dataset/repository/_artificial.py new file mode 100644 index 0000000..8b5fcda --- /dev/null +++ b/pts/dataset/repository/_artificial.py @@ -0,0 +1,48 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + +# Standard library imports +import json +from pathlib import Path + +# First-party imports +from pts.dataset import ArtificialDataset, generate_sf2, serialize_data_entry + + +def generate_artificial_dataset(dataset_path: Path, dataset: ArtificialDataset) -> None: + dataset_path_train = dataset_path / "train" + dataset_path_test = dataset_path / "test" + + dataset_path.mkdir(exist_ok=True) + dataset_path_train.mkdir(exist_ok=False) + dataset_path_test.mkdir(exist_ok=False) + + ds = dataset.generate() + assert ds.test is not None + + with (dataset_path / "metadata.json").open("w") as fp: + json.dump(ds.metadata.dict(), fp, indent=2, sort_keys=True) + + generate_sf2( + filename=str(dataset_path_train / "train.json"), + time_series=list(map(serialize_data_entry, ds.train)), + is_missing=False, + num_missing=0, + ) + + generate_sf2( + filename=str(dataset_path_test / "test.json"), + time_series=list(map(serialize_data_entry, ds.test)), + is_missing=False, + num_missing=0, + ) diff --git a/pts/dataset/repository/datasets.py b/pts/dataset/repository/datasets.py index b7b3afb..aee5190 100644 --- a/pts/dataset/repository/datasets.py +++ b/pts/dataset/repository/datasets.py @@ -18,6 +18,7 @@ from pathlib import Path from pts.dataset import ConstantDataset, TrainDatasets, load_datasets +from ._artificial import generate_artificial_dataset from ._lstnet import generate_lstnet_dataset from ._m4 import generate_m4_dataset from ._gp_copula_2019 import generate_gp_copula_dataset @@ -30,6 +31,8 @@ prediction_length = 48 dataset_recipes = OrderedDict( { + # each recipe generates a dataset given a path + "constant": partial(generate_artificial_dataset, dataset=ConstantDataset()), "exchange_rate": partial(generate_lstnet_dataset, dataset_name="exchange_rate"), "solar-energy": partial(generate_lstnet_dataset, dataset_name="solar-energy"), "electricity": partial(generate_lstnet_dataset, dataset_name="electricity"),