mirror of
https://github.com/wassname/pytorch-ts.git
synced 2026-07-12 12:35:06 +08:00
Dr. Kashif Rasul
821 lines
30 KiB
Python
821 lines
30 KiB
Python
import os
|
|
import math
|
|
import random
|
|
from typing import Callable, List, NamedTuple, Optional, Tuple, Union
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
import rapidjson as json
|
|
|
|
from .common import (
|
|
MetaData,
|
|
CategoricalFeatureInfo,
|
|
BasicFeatureInfo,
|
|
FieldName,
|
|
Dataset,
|
|
TrainDatasets,
|
|
DataEntry,
|
|
)
|
|
from .list_dataset import ListDataset
|
|
from .stat import DatasetStatistics, calculate_dataset_statistics
|
|
from .recipe import (
|
|
BinaryHolidays,
|
|
BinaryMarkovChain,
|
|
Constant,
|
|
ForEachCat,
|
|
Lag,
|
|
LinearTrend,
|
|
RandomCat,
|
|
RandomGaussian,
|
|
Stack,
|
|
generate,
|
|
take_as_list,
|
|
)
|
|
|
|
|
|
class DatasetInfo(NamedTuple):
|
|
"""
|
|
Information stored on a dataset. When downloading from the repository, the
|
|
dataset repository checks that the obtained version matches the one
|
|
declared in dataset_info/dataset_name.json.
|
|
"""
|
|
|
|
name: str
|
|
metadata: MetaData
|
|
prediction_length: int
|
|
train_statistics: DatasetStatistics
|
|
test_statistics: DatasetStatistics
|
|
|
|
|
|
class ArtificialDataset:
|
|
"""
|
|
Parent class of a dataset that can be generated from code.
|
|
"""
|
|
|
|
def __init__(self, freq) -> None:
|
|
self.freq = freq
|
|
|
|
@property
|
|
def metadata(self) -> MetaData:
|
|
pass
|
|
|
|
@property
|
|
def train(self) -> List[DataEntry]:
|
|
pass
|
|
|
|
@property
|
|
def test(self) -> List[DataEntry]:
|
|
pass
|
|
|
|
# todo return the same type as dataset repo for better usability
|
|
def generate(self) -> TrainDatasets:
|
|
return TrainDatasets(
|
|
metadata=self.metadata,
|
|
train=ListDataset(self.train, self.freq),
|
|
test=ListDataset(self.test, self.freq),
|
|
)
|
|
|
|
|
|
class ConstantDataset(ArtificialDataset):
|
|
def __init__(
|
|
self,
|
|
num_timeseries: int = 10,
|
|
num_steps: int = 30,
|
|
freq: str = "1H",
|
|
start: str = "2000-01-01 00:00:00",
|
|
is_nan: bool = False, # Generates constant dataset of 0s with explicit NaN missing values
|
|
is_random_constant: bool = False, # Inserts random constant value for each time series
|
|
is_different_scales: bool = False, # Generates constants on various scales
|
|
is_piecewise: bool = False, # Determines whether the time series in the test
|
|
# and train set should have different constant values
|
|
is_noise: bool = False, # Determines whether to add Gaussian noise to the constant dataset
|
|
is_long: bool = False, # Determines whether some time series will have very long lengths
|
|
is_short: bool = False, # Determines whether some time series will have very short lengths
|
|
is_trend: bool = False, # Determines whether to add linear trends
|
|
num_missing_middle: int = 0, # Number of missing values in the middle of the time series
|
|
is_promotions: bool = False, # Determines whether to add promotions to the target time series
|
|
# and to store in metadata
|
|
holidays: Optional[
|
|
List[pd.Timestamp]
|
|
] = None, # Determines whether to add holidays to the target time series
|
|
# and to store in metadata
|
|
) -> None:
|
|
super(ConstantDataset, self).__init__(freq)
|
|
self.num_timeseries = num_timeseries
|
|
self.num_steps = num_steps
|
|
self.num_training_steps = self.num_steps // 10 * 8
|
|
self.prediction_length = self.num_steps - self.num_training_steps
|
|
self.start = start
|
|
self.is_nan = is_nan
|
|
self.is_random_constant = is_random_constant
|
|
self.is_different_scales = is_different_scales
|
|
self.is_piecewise = is_piecewise
|
|
self.is_noise = is_noise
|
|
self.is_long = is_long
|
|
self.is_short = is_short
|
|
self.is_trend = is_trend
|
|
self.num_missing_middle = num_missing_middle
|
|
self.is_promotions = is_promotions
|
|
self.holidays = holidays
|
|
|
|
@property
|
|
def metadata(self) -> MetaData:
|
|
metadata = MetaData(
|
|
freq=self.freq,
|
|
feat_static_cat=[
|
|
{
|
|
"name": "feat_static_cat_000",
|
|
"cardinality": str(self.num_timeseries),
|
|
}
|
|
],
|
|
feat_static_real=[{"name": "feat_static_real_000"}],
|
|
prediction_length=self.prediction_length,
|
|
)
|
|
if self.is_promotions or self.holidays:
|
|
metadata = MetaData(
|
|
freq=self.freq,
|
|
feat_static_cat=[
|
|
{
|
|
"name": "feat_static_cat_000",
|
|
"cardinality": str(self.num_timeseries),
|
|
}
|
|
],
|
|
feat_static_real=[{"name": "feat_static_real_000"}],
|
|
feat_dynamic_real=[BasicFeatureInfo(name=FieldName.FEAT_DYNAMIC_REAL)],
|
|
prediction_length=self.prediction_length,
|
|
)
|
|
return metadata
|
|
|
|
def determine_constant(
|
|
self, index: int, constant: Optional[float] = None, seed: int = 1
|
|
) -> Optional[float]:
|
|
if self.is_random_constant:
|
|
my_random = random.Random(seed)
|
|
constant = (index + 1) * my_random.random()
|
|
elif self.is_different_scales:
|
|
if index == 0:
|
|
constant = 1e-8
|
|
elif constant is not None:
|
|
constant *= 100
|
|
else:
|
|
constant = float(index)
|
|
return constant
|
|
|
|
def compute_data_from_recipe(
|
|
self,
|
|
num_steps: int,
|
|
constant: Optional[float] = None,
|
|
one_to_zero: float = 0.1,
|
|
zero_to_one: float = 0.1,
|
|
scale_features: float = 200,
|
|
) -> TrainDatasets:
|
|
recipe = []
|
|
recipe_type = Constant(constant)
|
|
if self.is_noise:
|
|
recipe_type += RandomGaussian() # Use default stddev = 1.0
|
|
if self.is_trend:
|
|
recipe_type += LinearTrend()
|
|
if self.is_promotions:
|
|
recipe.append(
|
|
("binary_causal", BinaryMarkovChain(one_to_zero, zero_to_one))
|
|
)
|
|
recipe.append((FieldName.FEAT_DYNAMIC_REAL, Stack(["binary_causal"])))
|
|
recipe_type += scale_features * Lag("binary_causal", lag=0)
|
|
if self.holidays:
|
|
timestamp = self.init_date()
|
|
# Compute dates array
|
|
dates = []
|
|
for i in range(num_steps):
|
|
dates.append(timestamp)
|
|
timestamp += 1
|
|
recipe.append(("binary_holidays", BinaryHolidays(dates, self.holidays)))
|
|
recipe.append((FieldName.FEAT_DYNAMIC_REAL, Stack(["binary_holidays"])))
|
|
recipe_type += scale_features * Lag("binary_holidays", lag=0)
|
|
recipe.append((FieldName.TARGET, recipe_type))
|
|
max_train_length = num_steps - self.prediction_length
|
|
data = RecipeDataset(
|
|
recipe=recipe,
|
|
metadata=self.metadata,
|
|
max_train_length=max_train_length,
|
|
prediction_length=self.prediction_length,
|
|
num_timeseries=1, # Add 1 time series at a time in the loop for different constant valus per time series
|
|
)
|
|
generated = data.generate()
|
|
return generated
|
|
|
|
def piecewise_constant(self, index: int, num_steps: int) -> List:
|
|
target = []
|
|
for j in range(num_steps):
|
|
if j < self.num_training_steps:
|
|
constant = self.determine_constant(index=index)
|
|
else:
|
|
constant = self.determine_constant(index=index, seed=2)
|
|
target.append(constant)
|
|
return target
|
|
|
|
def get_num_steps(
|
|
self,
|
|
index: int,
|
|
num_steps_max: int = 10000,
|
|
long_freq: int = 4,
|
|
num_steps_min: int = 2,
|
|
short_freq: int = 4,
|
|
) -> int:
|
|
num_steps = self.num_steps
|
|
if self.is_long and index % long_freq == 0:
|
|
num_steps = num_steps_max
|
|
elif self.is_short and index % short_freq == 0:
|
|
num_steps = num_steps_min
|
|
return num_steps
|
|
|
|
def init_date(self) -> pd.Timestamp:
|
|
week_dict = {
|
|
0: "MON",
|
|
1: "TUE",
|
|
2: "WED",
|
|
3: "THU",
|
|
4: "FRI",
|
|
5: "SAT",
|
|
6: "SUN",
|
|
}
|
|
timestamp = pd.Timestamp(self.start)
|
|
freq_week_start = self.freq
|
|
if freq_week_start == "W":
|
|
freq_week_start = f"W-{week_dict[timestamp.weekday()]}"
|
|
return pd.Timestamp(self.start, freq=freq_week_start)
|
|
|
|
@staticmethod
|
|
def insert_nans_and_zeros(ts_len: int) -> List:
|
|
target = []
|
|
for j in range(ts_len):
|
|
# Place NaNs at even indices. Use convention no NaNs before start date.
|
|
if j != 0 and j % 2 == 0:
|
|
target.append(np.nan)
|
|
# Place zeros at odd indices
|
|
else:
|
|
target.append(0.0)
|
|
return target
|
|
|
|
def insert_missing_vals_middle(
|
|
self, ts_len: int, constant: Optional[float]
|
|
) -> List:
|
|
target = []
|
|
lower_bound = (self.num_training_steps - self.num_missing_middle) // 2
|
|
upper_bound = (self.num_training_steps + self.num_missing_middle) // 2
|
|
num_missing_endpts = math.floor(0.1 * self.num_missing_middle)
|
|
for j in range(ts_len):
|
|
if (
|
|
(0 < j < lower_bound and j % (2 * num_missing_endpts) == 0)
|
|
or (lower_bound <= j < upper_bound)
|
|
or (j >= upper_bound and j % (2 * num_missing_endpts) == 0)
|
|
):
|
|
val = np.nan
|
|
else:
|
|
val = constant
|
|
target.append(val)
|
|
return target
|
|
|
|
def generate_ts(self, num_ts_steps: int, is_train: bool = False) -> List[DataEntry]:
|
|
res = []
|
|
constant = None
|
|
for i in range(self.num_timeseries):
|
|
if self.is_nan:
|
|
target = self.insert_nans_and_zeros(num_ts_steps)
|
|
elif self.is_piecewise:
|
|
target = self.piecewise_constant(i, num_ts_steps)
|
|
else:
|
|
constant = self.determine_constant(i, constant)
|
|
if self.num_missing_middle > 0:
|
|
target = self.insert_missing_vals_middle(num_ts_steps, constant)
|
|
elif (
|
|
self.is_noise
|
|
or self.is_trend
|
|
or self.is_promotions
|
|
or self.holidays
|
|
):
|
|
|
|
num_steps = self.get_num_steps(i)
|
|
generated = self.compute_data_from_recipe(num_steps, constant)
|
|
if is_train:
|
|
time_series = generated.train
|
|
else:
|
|
assert generated.test is not None
|
|
time_series = generated.test
|
|
# returns np array convert to list for consistency
|
|
target = list(time_series)[0][FieldName.TARGET].tolist()
|
|
else:
|
|
target = [constant] * num_ts_steps
|
|
ts_data = dict(
|
|
start=self.start,
|
|
target=target,
|
|
item_id=str(i),
|
|
feat_static_cat=[i],
|
|
feat_static_real=[i],
|
|
)
|
|
if self.is_promotions or self.holidays:
|
|
ts_data[FieldName.FEAT_DYNAMIC_REAL] = list(time_series)[0][
|
|
FieldName.FEAT_DYNAMIC_REAL
|
|
].tolist()
|
|
res.append(ts_data)
|
|
return res
|
|
|
|
@property
|
|
def train(self) -> List[DataEntry]:
|
|
return self.generate_ts(num_ts_steps=self.num_training_steps, is_train=True)
|
|
|
|
@property
|
|
def test(self) -> List[DataEntry]:
|
|
return self.generate_ts(num_ts_steps=self.num_steps)
|
|
|
|
|
|
class ComplexSeasonalTimeSeries(ArtificialDataset):
|
|
"""
|
|
Generate sinus time series that ramp up and reach a certain amplitude, and
|
|
level and have additional spikes on each sunday.
|
|
|
|
|
|
TODO: This could be converted to a RecipeDataset to avoid code duplication.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
num_series: int = 100,
|
|
prediction_length: int = 20,
|
|
freq_str: str = "D",
|
|
length_low: int = 30,
|
|
length_high: int = 200,
|
|
min_val: float = -10000,
|
|
max_val: float = 10000,
|
|
is_integer: bool = False,
|
|
proportion_missing_values: float = 0,
|
|
is_noise: bool = True,
|
|
is_scale: bool = True,
|
|
percentage_unique_timestamps: float = 0.07,
|
|
is_out_of_bounds_date: bool = False,
|
|
seasonality: Optional[int] = None,
|
|
clip_values: bool = False,
|
|
) -> None:
|
|
"""
|
|
:param num_series: number of time series generated in the train and
|
|
test set
|
|
:param prediction_length:
|
|
:param freq_str:
|
|
:param length_low: minimum length of a time-series, must be larger than
|
|
prediction_length
|
|
:param length_high: maximum length of a time-series
|
|
:param min_val: min value of a time-series
|
|
:param max_val: max value of a time-series
|
|
:param is_integer: whether the dataset has integers or not
|
|
:param proportion_missing_values:
|
|
:param is_noise: whether to add noise
|
|
:param is_scale: whether to add scale
|
|
:param percentage_unique_timestamps: percentage of random start dates bounded between 0 and 1
|
|
:param is_out_of_bounds_date: determines whether to use very old start dates and start dates far in the future
|
|
:param seasonality: Seasonality of the generated data. If not given uses default seasonality for frequency
|
|
:param clip_values: if True the values will be clipped to [min_val, max_val], otherwise linearly scales them
|
|
"""
|
|
assert length_low > prediction_length
|
|
super(ComplexSeasonalTimeSeries, self).__init__(freq_str)
|
|
self.num_series = num_series
|
|
self.prediction_length = prediction_length
|
|
self.length_low = length_low
|
|
self.length_high = length_high
|
|
self.freq_str = freq_str
|
|
self.min_val = min_val
|
|
self.max_val = max_val
|
|
self.is_integer = is_integer
|
|
self.proportion_missing_values = proportion_missing_values
|
|
self.is_noise = is_noise
|
|
self.is_scale = is_scale
|
|
self.percentage_unique_timestamps = percentage_unique_timestamps
|
|
self.is_out_of_bounds_date = is_out_of_bounds_date
|
|
self.seasonality = seasonality
|
|
self.clip_values = clip_values
|
|
|
|
@property
|
|
def metadata(self) -> MetaData:
|
|
return MetaData(freq=self.freq, prediction_length=self.prediction_length)
|
|
|
|
def _get_period(self) -> int:
|
|
if self.seasonality is not None:
|
|
return self.seasonality
|
|
if self.freq_str == "M":
|
|
return 24
|
|
elif self.freq_str == "W":
|
|
return 52
|
|
elif self.freq_str == "D":
|
|
return 14
|
|
elif self.freq_str == "H":
|
|
return 24
|
|
elif self.freq_str == "min":
|
|
return 60
|
|
else:
|
|
raise RuntimeError()
|
|
|
|
def _get_start(self, index: int, my_random: random.Random) -> str:
|
|
if (
|
|
self.is_out_of_bounds_date and index == 0
|
|
): # Add edge case of dates out of normal bounds past date
|
|
start_y, start_m, start_d = (
|
|
1690,
|
|
2,
|
|
7,
|
|
) # Pandas doesn't allot before 1650
|
|
start_h, start_min = 18, 36
|
|
elif (
|
|
self.is_out_of_bounds_date and index == self.num_series - 1
|
|
): # Add edge case of dates out of normal bounds future date
|
|
start_y, start_m, start_d = (
|
|
2030,
|
|
6,
|
|
3,
|
|
) # Pandas doesn't allot before 1650
|
|
start_h, start_min = 18, 36
|
|
# assume that only 100 * percentage_unique_timestamps of timestamps are unique
|
|
elif my_random.random() < self.percentage_unique_timestamps:
|
|
start_y = my_random.randint(2000, 2018)
|
|
start_m = my_random.randint(1, 12)
|
|
start_d = my_random.randint(1, 28)
|
|
start_h = my_random.randint(0, 23)
|
|
start_min = my_random.randint(0, 59)
|
|
else:
|
|
start_y, start_m, start_d = 2013, 11, 28
|
|
start_h, start_min = 18, 36
|
|
|
|
if self.freq_str == "M":
|
|
return "%04.d-%02.d" % (start_y, start_m)
|
|
elif self.freq_str in ["W", "D"]:
|
|
return "%04.d-%02.d-%02.d" % (start_y, start_m, start_d)
|
|
elif self.freq_str == "H":
|
|
return "%04.d-%02.d-%02.d %02.d:00:00" % (
|
|
start_y,
|
|
start_m,
|
|
start_d,
|
|
start_h,
|
|
)
|
|
else:
|
|
return "%04.d-%02.d-%02.d %02.d:%02.d:00" % (
|
|
start_y,
|
|
start_m,
|
|
start_d,
|
|
start_h,
|
|
start_min,
|
|
)
|
|
|
|
def _special_time_point_indicator(self, index) -> bool:
|
|
if self.freq_str == "M":
|
|
return index.month == 1
|
|
elif self.freq_str == "W":
|
|
return index.month % 2 == 0
|
|
elif self.freq_str == "D":
|
|
return index.dayofweek == 0
|
|
elif self.freq_str == "H":
|
|
return index.hour == 0
|
|
elif self.freq_str == "min":
|
|
return index.minute % 30 == 0
|
|
else:
|
|
raise RuntimeError(f'Bad freq_str value "{index}"')
|
|
|
|
@property
|
|
def train(self) -> List[DataEntry]:
|
|
return [
|
|
dict(
|
|
start=ts[FieldName.START],
|
|
target=ts[FieldName.TARGET][: -self.prediction_length],
|
|
item_id=ts[FieldName.ITEM_ID],
|
|
)
|
|
for ts in self.make_timeseries()
|
|
]
|
|
|
|
@property
|
|
def test(self) -> List[DataEntry]:
|
|
return self.make_timeseries()
|
|
|
|
def make_timeseries(self, seed: int = 1) -> List[DataEntry]:
|
|
res = []
|
|
# Fix seed so that the training set is the same
|
|
# as the test set from 0:self.prediction_length for the two independent calls
|
|
|
|
def sigmoid(x: np.ndarray) -> np.ndarray:
|
|
return 1.0 / (1.0 + np.exp(-x))
|
|
|
|
# Ensure same start dates in test and training set
|
|
my_random = random.Random(seed)
|
|
state = np.random.RandomState(seed)
|
|
for i in range(self.num_series):
|
|
val_range = self.max_val - self.min_val
|
|
length = state.randint(low=self.length_low, high=self.length_high)
|
|
start = self._get_start(i, my_random)
|
|
envelope = sigmoid((np.arange(length) - 20.0) / 10.0)
|
|
level = 0.3 * val_range * (state.random_sample() - 0.5)
|
|
phi = 2 * np.pi * state.random_sample()
|
|
period = self._get_period()
|
|
w = 2 * np.pi / period
|
|
t = np.arange(length)
|
|
idx = pd.date_range(start=start, freq=self.freq_str, periods=length)
|
|
special_tp_indicator = self._special_time_point_indicator(idx)
|
|
sunday_effect = state.random_sample() * special_tp_indicator
|
|
v = np.sin(w * t + phi) + sunday_effect
|
|
|
|
if self.is_scale:
|
|
scale = 0.1 * val_range * state.random_sample()
|
|
v *= scale
|
|
v += level
|
|
if self.is_noise:
|
|
noise_range = 0.02 * val_range * state.random_sample()
|
|
noise = noise_range * state.normal(size=length)
|
|
v += noise
|
|
v = envelope * v
|
|
if self.clip_values:
|
|
np.clip(v, a_min=self.min_val, a_max=self.max_val, out=v)
|
|
else:
|
|
"""
|
|
Rather than mapping [v_min, v_max] to [self.min_val, self.max_val] which would lead to
|
|
all the time series having the same min and max, we want to keep the same interval length
|
|
(v_max - v_min). We thus shift the interval [v_min, v_max] in [self.min_val, self.max_val]
|
|
and clip it if needed.
|
|
"""
|
|
v_min, v_max = v.min(), v.max()
|
|
p_min, p_max = (
|
|
max(self.min_val, v_min),
|
|
min(self.max_val, v_max),
|
|
)
|
|
shifted_min = np.clip(
|
|
p_min + (p_max - v_max), a_min=self.min_val, a_max=self.max_val,
|
|
)
|
|
shifted_max = np.clip(
|
|
p_max + (p_min - v_min), a_min=self.min_val, a_max=self.max_val,
|
|
)
|
|
v = shifted_min + (shifted_max - shifted_min) * (v - v_min) / (
|
|
v_max - v_min
|
|
)
|
|
|
|
if self.is_integer:
|
|
np.clip(
|
|
v, a_min=np.ceil(self.min_val), a_max=np.floor(self.max_val), out=v,
|
|
)
|
|
v = np.round(v).astype(int)
|
|
v = list(v.tolist())
|
|
if self.proportion_missing_values > 0:
|
|
assert (
|
|
self.proportion_missing_values < 1.0
|
|
), "Please chose a number 0 < x < 1.0"
|
|
idx = np.arange(len(v))
|
|
state.shuffle(idx)
|
|
num_missing_values = (
|
|
int(len(v) * self.proportion_missing_values) + 1
|
|
) # Add one in case this gets zero
|
|
missing_idx = idx[:num_missing_values]
|
|
for j in missing_idx:
|
|
# Using convention that there are no missing values before the start date.
|
|
if j != 0:
|
|
v[j] = None if state.rand() < 0.5 else "NaN"
|
|
res.append(
|
|
dict(
|
|
start=pd.Timestamp(start, freq=self.freq_str),
|
|
target=np.array(v),
|
|
item_id=i,
|
|
)
|
|
)
|
|
return res
|
|
|
|
|
|
class RecipeDataset(ArtificialDataset):
|
|
"""Synthetic data set generated by providing a recipe.
|
|
|
|
A recipe is either a (non-deterministic) function
|
|
|
|
f(length: int, global_state: dict) -> dict
|
|
|
|
or list of (field, function) tuples of the form
|
|
|
|
(field: str, f(data: dict, length: int, global_state: dict) -> dict)
|
|
|
|
which is processed sequentially, with data initially set to {},
|
|
and each entry updating data[field] to the output of the function
|
|
call.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
recipe: Union[Callable, List[Tuple[str, Callable]]],
|
|
metadata: MetaData,
|
|
max_train_length: int,
|
|
prediction_length: int,
|
|
num_timeseries: int,
|
|
trim_length_fun=lambda x, **kwargs: 0,
|
|
data_start=pd.Timestamp("2014-01-01"),
|
|
) -> None:
|
|
"""
|
|
|
|
:param recipe: The recipe to generate from (see class docstring)
|
|
:param metadata: The metadata to be included in the dataset
|
|
:param max_train_length: The maximum length of a training time series.
|
|
:param prediction_length: The length of the prediction range
|
|
:param num_timeseries: Number of time series to generate
|
|
:param trim_length_fun: Callable f(x: int) -> int returning the
|
|
(shortened) training length
|
|
:param data_start: Start date for the data set
|
|
"""
|
|
super().__init__(freq=metadata.freq)
|
|
|
|
self.recipe = recipe
|
|
self._metadata = metadata
|
|
self.max_train_length = max_train_length
|
|
self.prediction_length = prediction_length
|
|
self.trim_length_fun = trim_length_fun
|
|
self.num_timeseries = num_timeseries
|
|
self.data_start = pd.Timestamp(data_start, freq=self._metadata.freq)
|
|
|
|
@property
|
|
def metadata(self) -> MetaData:
|
|
return self._metadata
|
|
|
|
def dataset_info(self, train_ds: Dataset, test_ds: Dataset) -> DatasetInfo:
|
|
return DatasetInfo(
|
|
name=f"RecipeDataset({repr(self.recipe)})",
|
|
metadata=self.metadata,
|
|
prediction_length=self.prediction_length,
|
|
train_statistics=calculate_dataset_statistics(train_ds),
|
|
test_statistics=calculate_dataset_statistics(test_ds),
|
|
)
|
|
|
|
@staticmethod
|
|
def trim_ts_item_end(x: DataEntry, length: int) -> DataEntry:
|
|
"""Trim a TimeSeriesItem into a training range, by removing
|
|
the last prediction_length time points from the target and dynamic
|
|
features."""
|
|
y = dict(
|
|
item_id=x[FieldName.ITEM_ID],
|
|
start=x[FieldName.START],
|
|
target=x[FieldName.TARGET][:-length],
|
|
)
|
|
|
|
if FieldName.FEAT_DYNAMIC_CAT in x:
|
|
y[FieldName.FEAT_DYNAMIC_CAT] = x[FieldName.FEAT_DYNAMIC_CAT][:, :-length]
|
|
if FieldName.FEAT_DYNAMIC_REAL in x:
|
|
y[FieldName.FEAT_DYNAMIC_REAL] = x[FieldName.FEAT_DYNAMIC_REAL][:, :-length]
|
|
return y
|
|
|
|
@staticmethod
|
|
def trim_ts_item_front(x: DataEntry, length: int) -> DataEntry:
|
|
"""Trim a TimeSeriesItem into a training range, by removing
|
|
the first offset_front time points from the target and dynamic
|
|
features."""
|
|
assert length <= len(x[FieldName.TARGET])
|
|
|
|
y = dict(
|
|
item_id=x[FieldName.ITEM_ID],
|
|
start=x[FieldName.START] + length * x[FieldName.START].freq,
|
|
target=x[FieldName.TARGET][length:],
|
|
)
|
|
|
|
if FieldName.FEAT_DYNAMIC_CAT in x:
|
|
y[FieldName.FEAT_DYNAMIC_CAT] = x[FieldName.FEAT_DYNAMIC_CAT][:, length:]
|
|
if FieldName.FEAT_DYNAMIC_REAL in x:
|
|
y[FieldName.FEAT_DYNAMIC_REAL] = x[FieldName.FEAT_DYNAMIC_REAL][:, length:]
|
|
return y
|
|
|
|
def generate(self) -> TrainDatasets:
|
|
metadata = self.metadata
|
|
data_it = generate(
|
|
length=self.max_train_length + self.prediction_length,
|
|
recipe=self.recipe,
|
|
start=self.data_start,
|
|
)
|
|
full_length_data = take_as_list(data_it, self.num_timeseries)
|
|
|
|
test_data = [
|
|
RecipeDataset.trim_ts_item_front(
|
|
x, self.trim_length_fun(x, train_length=self.max_train_length)
|
|
)
|
|
for x in full_length_data
|
|
]
|
|
train_data = [
|
|
RecipeDataset.trim_ts_item_end(x, self.prediction_length) for x in test_data
|
|
]
|
|
return TrainDatasets(
|
|
metadata=metadata,
|
|
train=ListDataset(train_data, metadata.freq),
|
|
test=ListDataset(test_data, metadata.freq),
|
|
)
|
|
|
|
|
|
def default_synthetic() -> Tuple[DatasetInfo, Dataset, Dataset]:
|
|
|
|
recipe = [
|
|
(FieldName.TARGET, LinearTrend() + RandomGaussian()),
|
|
(FieldName.FEAT_STATIC_CAT, RandomCat([10])),
|
|
(
|
|
FieldName.FEAT_STATIC_REAL,
|
|
ForEachCat(RandomGaussian(1, (10,)), FieldName.FEAT_STATIC_CAT)
|
|
+ RandomGaussian(0.1, (10,)),
|
|
),
|
|
]
|
|
|
|
data = RecipeDataset(
|
|
recipe=recipe,
|
|
metadata=MetaData(
|
|
freq="D",
|
|
feat_static_real=[BasicFeatureInfo(name=FieldName.FEAT_STATIC_REAL)],
|
|
feat_static_cat=[
|
|
CategoricalFeatureInfo(name=FieldName.FEAT_STATIC_CAT, cardinality=10)
|
|
],
|
|
feat_dynamic_real=[BasicFeatureInfo(name=FieldName.FEAT_DYNAMIC_REAL)],
|
|
),
|
|
max_train_length=20,
|
|
prediction_length=10,
|
|
num_timeseries=10,
|
|
trim_length_fun=lambda x, **kwargs: np.minimum(
|
|
int(np.random.geometric(1 / (kwargs["train_length"] / 2))),
|
|
kwargs["train_length"],
|
|
),
|
|
)
|
|
|
|
generated = data.generate()
|
|
assert generated.test is not None
|
|
info = data.dataset_info(generated.train, generated.test)
|
|
|
|
return info, generated.train, generated.test
|
|
|
|
|
|
def constant_dataset() -> Tuple[DatasetInfo, Dataset, Dataset]:
|
|
metadata = MetaData(
|
|
freq="1H",
|
|
feat_static_cat=[
|
|
CategoricalFeatureInfo(name="feat_static_cat_000", cardinality="10")
|
|
],
|
|
feat_static_real=[BasicFeatureInfo(name="feat_static_real_000")],
|
|
)
|
|
|
|
start_date = "2000-01-01 00:00:00"
|
|
|
|
train_ds = ListDataset(
|
|
data_iter=[
|
|
{
|
|
FieldName.ITEM_ID: str(i),
|
|
FieldName.START: start_date,
|
|
FieldName.TARGET: [float(i)] * 24,
|
|
FieldName.FEAT_STATIC_CAT: [i],
|
|
FieldName.FEAT_STATIC_REAL: [float(i)],
|
|
}
|
|
for i in range(10)
|
|
],
|
|
freq=metadata.freq,
|
|
)
|
|
|
|
test_ds = ListDataset(
|
|
data_iter=[
|
|
{
|
|
FieldName.ITEM_ID: str(i),
|
|
FieldName.START: start_date,
|
|
FieldName.TARGET: [float(i)] * 30,
|
|
FieldName.FEAT_STATIC_CAT: [i],
|
|
FieldName.FEAT_STATIC_REAL: [float(i)],
|
|
}
|
|
for i in range(10)
|
|
],
|
|
freq=metadata.freq,
|
|
)
|
|
|
|
info = DatasetInfo(
|
|
name="constant_dataset",
|
|
metadata=metadata,
|
|
prediction_length=6,
|
|
train_statistics=calculate_dataset_statistics(train_ds),
|
|
test_statistics=calculate_dataset_statistics(test_ds),
|
|
)
|
|
|
|
return info, train_ds, test_ds
|
|
|
|
|
|
def generate_sf2(
|
|
filename: str, time_series: List, is_missing: bool, num_missing: int
|
|
) -> None:
|
|
# This function generates the test and train json files which will be converted to csv format
|
|
if not os.path.exists(os.path.dirname(filename)):
|
|
os.makedirs(os.path.dirname(filename))
|
|
with open(filename, "w") as json_file:
|
|
for ts in time_series:
|
|
if is_missing:
|
|
target = [] # type: List
|
|
# For Forecast don't output feat_static_cat and feat_static_real
|
|
for j, val in enumerate(ts[FieldName.TARGET]):
|
|
# only add ones that are not missing
|
|
if j != 0 and j % num_missing == 0:
|
|
target.append(None)
|
|
else:
|
|
target.append(val)
|
|
ts[FieldName.TARGET] = target
|
|
ts.pop(FieldName.FEAT_STATIC_CAT, None)
|
|
ts.pop(FieldName.FEAT_STATIC_REAL, None)
|
|
# Chop features in training set
|
|
if FieldName.FEAT_DYNAMIC_REAL in ts.keys() and "train" in filename:
|
|
# TODO: Fix for missing values
|
|
for i, feat_dynamic_real in enumerate(ts[FieldName.FEAT_DYNAMIC_REAL]):
|
|
ts[FieldName.FEAT_DYNAMIC_REAL][i] = feat_dynamic_real[
|
|
: len(ts[FieldName.TARGET])
|
|
]
|
|
json.dump(ts, json_file)
|
|
json_file.write("\n")
|