From ddeca6793a381b88769f1aa97158eb23d7a541fc Mon Sep 17 00:00:00 2001 From: Kashif Rasul Date: Sat, 14 Dec 2019 16:14:02 +0100 Subject: [PATCH] formatting --- pts/transform/convert.py | 62 ++++++++---------------------------- pts/transform/feature.py | 15 +++------ pts/transform/sampler.py | 4 +-- pts/transform/split.py | 69 ++++++++++++---------------------------- 4 files changed, 40 insertions(+), 110 deletions(-) diff --git a/pts/transform/convert.py b/pts/transform/convert.py index a95cbc4..6df4ac4 100644 --- a/pts/transform/convert.py +++ b/pts/transform/convert.py @@ -114,27 +114,18 @@ class VstackFeatures(SimpleTransformation): """ def __init__( - self, - output_field: str, - input_fields: List[str], - drop_inputs: bool = True, + self, output_field: str, input_fields: List[str], drop_inputs: bool = True, ) -> None: self.output_field = output_field self.input_fields = input_fields self.cols_to_drop = ( [] if not drop_inputs - else [ - fname for fname in self.input_fields if fname != output_field - ] + else [fname for fname in self.input_fields if fname != output_field] ) def transform(self, data: DataEntry) -> DataEntry: - r = [ - data[fname] - for fname in self.input_fields - if data[fname] is not None - ] + r = [data[fname] for fname in self.input_fields if data[fname] is not None] output = np.vstack(r) data[self.output_field] = output for fname in self.cols_to_drop: @@ -159,27 +150,18 @@ class ConcatFeatures(SimpleTransformation): """ def __init__( - self, - output_field: str, - input_fields: List[str], - drop_inputs: bool = True, + self, output_field: str, input_fields: List[str], drop_inputs: bool = True, ) -> None: self.output_field = output_field self.input_fields = input_fields self.cols_to_drop = ( [] if not drop_inputs - else [ - fname for fname in self.input_fields if fname != output_field - ] + else [fname for fname in self.input_fields if fname != output_field] ) def transform(self, data: DataEntry) -> DataEntry: - r = [ - data[fname] - for fname in self.input_fields - if data[fname] is not None - ] + r = [data[fname] for fname in self.input_fields if data[fname] is not None] output = np.concatenate(r) data[self.output_field] = output for fname in self.cols_to_drop: @@ -235,19 +217,14 @@ class ListFeatures(SimpleTransformation): """ def __init__( - self, - output_field: str, - input_fields: List[str], - drop_inputs: bool = True, + self, output_field: str, input_fields: List[str], drop_inputs: bool = True, ) -> None: self.output_field = output_field self.input_fields = input_fields self.cols_to_drop = ( [] if not drop_inputs - else [ - fname for fname in self.input_fields if fname != output_field - ] + else [fname for fname in self.input_fields if fname != output_field] ) def transform(self, data: DataEntry) -> DataEntry: @@ -462,17 +439,14 @@ class CDFtoGaussianTransform(MapTransformation): sorted_target_length, target_dim = sorted_target.shape quantiles = np.stack( - [np.arange(sorted_target_length) for _ in range(target_dim)], - axis=1, + [np.arange(sorted_target_length) for _ in range(target_dim)], axis=1, ) / float(sorted_target_length) x_diff = np.diff(sorted_target, axis=0) y_diff = np.diff(quantiles, axis=0) # Calculate slopes of the pw-linear pieces. - slopes = np.where( - x_diff == 0.0, np.zeros_like(x_diff), y_diff / x_diff - ) + slopes = np.where(x_diff == 0.0, np.zeros_like(x_diff), y_diff / x_diff) zeroes = np.zeros_like(np.expand_dims(slopes[0, :], axis=0)) slopes = np.append(slopes, zeroes, axis=0) @@ -513,9 +487,7 @@ class CDFtoGaussianTransform(MapTransformation): """ m = sorted_values.shape[0] - quantiles = self._forward_transform( - sorted_values, values, slopes, intercepts - ) + quantiles = self._forward_transform(sorted_values, values, slopes, intercepts) quantiles = np.clip( quantiles, self.winsorized_cutoff(m), 1 - self.winsorized_cutoff(m) @@ -526,9 +498,7 @@ class CDFtoGaussianTransform(MapTransformation): def _add_noise(x: np.array) -> np.array: scale_noise = 0.2 std = np.sqrt( - (np.square(x - x.mean(axis=1, keepdims=True))).mean( - axis=1, keepdims=True - ) + (np.square(x - x.mean(axis=1, keepdims=True))).mean(axis=1, keepdims=True) ) noise = np.random.normal( loc=np.zeros_like(x), scale=np.ones_like(x) * std * scale_noise @@ -537,9 +507,7 @@ class CDFtoGaussianTransform(MapTransformation): return x @staticmethod - def _search_sorted( - sorted_vec: np.array, to_insert_vec: np.array - ) -> np.array: + def _search_sorted(sorted_vec: np.array, to_insert_vec: np.array) -> np.array: """ Finds the indices of the active piece-wise linear function. @@ -557,9 +525,7 @@ class CDFtoGaussianTransform(MapTransformation): Indices mapping to the active linear function. """ indices_left = np.searchsorted(sorted_vec, to_insert_vec, side="left") - indices_right = np.searchsorted( - sorted_vec, to_insert_vec, side="right" - ) + indices_right = np.searchsorted(sorted_vec, to_insert_vec, side="right") indices = indices_left + (indices_right - indices_left) // 2 indices = indices - 1 diff --git a/pts/transform/feature.py b/pts/transform/feature.py index 4defd5d..5f6d3b9 100644 --- a/pts/transform/feature.py +++ b/pts/transform/feature.py @@ -173,25 +173,18 @@ class AddTimeFeatures(MapTransformation): if self._min_time_point is None: self._min_time_point = start self._max_time_point = end - self._min_time_point = min( - shift_timestamp(start, -50), self._min_time_point - ) - self._max_time_point = max( - shift_timestamp(end, 50), self._max_time_point - ) + self._min_time_point = min(shift_timestamp(start, -50), self._min_time_point) + self._max_time_point = max(shift_timestamp(end, 50), self._max_time_point) self.full_date_range = pd.date_range( self._min_time_point, self._max_time_point, freq=start.freq ) self._full_range_date_features = ( - np.vstack( - [feat(self.full_date_range) for feat in self.date_features] - ) + np.vstack([feat(self.full_date_range) for feat in self.date_features]) if self.date_features else None ) self._date_index = pd.Series( - index=self.full_date_range, - data=np.arange(len(self.full_date_range)), + index=self.full_date_range, data=np.arange(len(self.full_date_range)), ) def map_transform(self, data: DataEntry, is_train: bool) -> DataEntry: diff --git a/pts/transform/sampler.py b/pts/transform/sampler.py index a46b6c1..8fa7bd3 100644 --- a/pts/transform/sampler.py +++ b/pts/transform/sampler.py @@ -60,9 +60,7 @@ class UniformSplitSampler(InstanceSampler): self.lookup = np.arange(2 ** 13) def __call__(self, ts: np.ndarray, a: int, b: int) -> np.ndarray: - assert ( - a <= b - ), "First index must be less than or equal to the last index." + assert a <= b, "First index must be less than or equal to the last index." while ts.shape[-1] >= len(self.lookup): self.lookup = np.arange(2 * len(self.lookup)) mask = np.random.uniform(low=0.0, high=1.0, size=b - a + 1) < self.p diff --git a/pts/transform/split.py b/pts/transform/split.py index 38bf86c..431a629 100644 --- a/pts/transform/split.py +++ b/pts/transform/split.py @@ -34,9 +34,7 @@ def shift_timestamp(ts: pd.Timestamp, offset: int) -> pd.Timestamp: @lru_cache(maxsize=10000) -def _shift_timestamp_helper( - ts: pd.Timestamp, freq: str, offset: int -) -> pd.Timestamp: +def _shift_timestamp_helper(ts: pd.Timestamp, freq: str, offset: int) -> pd.Timestamp: """ We are using this helper function which explicitly uses the frequency as a parameter, because the frequency is not included in the hash of a time @@ -128,9 +126,7 @@ class InstanceSplitter(FlatMapTransformation): self.past_length = past_length self.future_length = future_length self.output_NTC = output_NTC - self.ts_fields = ( - time_series_fields if time_series_fields is not None else [] - ) + self.ts_fields = time_series_fields if time_series_fields is not None else [] self.target_field = target_field self.is_pad_field = is_pad_field self.start_field = start_field @@ -143,9 +139,7 @@ class InstanceSplitter(FlatMapTransformation): def _future(self, col_name): return f"future_{col_name}" - def flatmap_transform( - self, data: DataEntry, is_train: bool - ) -> Iterator[DataEntry]: + def flatmap_transform(self, data: DataEntry, is_train: bool) -> Iterator[DataEntry]: pl = self.future_length slice_cols = self.ts_fields + [self.target_field] target = data[self.target_field] @@ -166,9 +160,7 @@ class InstanceSplitter(FlatMapTransformation): ) else: sampling_indices = self.train_sampler( - target, - self.past_length, - len_target - self.future_length, + target, self.past_length, len_target - self.future_length, ) else: sampling_indices = [len_target] @@ -183,8 +175,7 @@ class InstanceSplitter(FlatMapTransformation): past_piece = d[ts_field][..., i - self.past_length : i] elif i < self.past_length: pad_block = np.zeros( - d[ts_field].shape[:-1] + (pad_length,), - dtype=d[ts_field].dtype, + d[ts_field].shape[:-1] + (pad_length,), dtype=d[ts_field].dtype, ) past_piece = np.concatenate( [pad_block, d[ts_field][..., :i]], axis=-1 @@ -200,17 +191,11 @@ class InstanceSplitter(FlatMapTransformation): if self.output_NTC: for ts_field in slice_cols: - d[self._past(ts_field)] = d[ - self._past(ts_field) - ].transpose() - d[self._future(ts_field)] = d[ - self._future(ts_field) - ].transpose() + d[self._past(ts_field)] = d[self._past(ts_field)].transpose() + d[self._future(ts_field)] = d[self._future(ts_field)].transpose() d[self._past(self.is_pad_field)] = pad_indicator - d[self.forecast_start_field] = shift_timestamp( - d[self.start_field], i - ) + d[self.forecast_start_field] = shift_timestamp(d[self.start_field], i) yield d @@ -308,9 +293,7 @@ class CanonicalInstanceSplitter(FlatMapTransformation): def _future(self, col_name): return f"future_{col_name}" - def flatmap_transform( - self, data: DataEntry, is_train: bool - ) -> Iterator[DataEntry]: + def flatmap_transform(self, data: DataEntry, is_train: bool) -> Iterator[DataEntry]: ts_fields = self.dynamic_feature_fields + [self.target_field] ts_target = data[self.target_field] @@ -354,9 +337,7 @@ class CanonicalInstanceSplitter(FlatMapTransformation): pad_pre = self.pad_value * np.ones( shape=full_ts.shape[:-1] + (pad_length,) ) - past_ts = np.concatenate( - [pad_pre, full_ts[..., :i]], axis=-1 - ) + past_ts = np.concatenate([pad_pre, full_ts[..., :i]], axis=-1) else: past_ts = full_ts[..., (i - self.instance_length) : i] @@ -365,13 +346,9 @@ class CanonicalInstanceSplitter(FlatMapTransformation): if self.use_prediction_features and not is_train: if not ts_field == self.target_field: - future_ts = full_ts[ - ..., i : i + self.prediction_length - ] + future_ts = full_ts[..., i : i + self.prediction_length] future_ts = ( - future_ts.transpose() - if self.output_NTC - else future_ts + future_ts.transpose() if self.output_NTC else future_ts ) d[self._future(ts_field)] = future_ts @@ -386,7 +363,7 @@ class CanonicalInstanceSplitter(FlatMapTransformation): class ContinuousTimeInstanceSplitter(FlatMapTransformation): """ - Selects training instances by slicing "intervals" from a continous-time + Selects training instances by slicing "intervals" from a continuos-time process instantiation. Concretely, the input data is expected to describe an instantiation from a point (or jump) process, with the "target" identifying inter-arrival times and other features (marks), as described @@ -461,15 +438,13 @@ class ContinuousTimeInstanceSplitter(FlatMapTransformation): end = np.searchsorted(a, ub) return np.arange(start, end) - def flatmap_transform( - self, data: DataEntry, is_train: bool - ) -> Iterator[DataEntry]: + def flatmap_transform(self, data: DataEntry, is_train: bool) -> Iterator[DataEntry]: assert data[self.start_field].freq == data[self.end_field].freq - total_interval_length = ( - data[self.end_field] - data[self.start_field] - ) / data[self.start_field].freq.delta + total_interval_length = (data[self.end_field] - data[self.start_field]) / data[ + self.start_field + ].freq.delta # sample forecast start times in continuous time if is_train: @@ -512,9 +487,7 @@ class ContinuousTimeInstanceSplitter(FlatMapTransformation): past_mask = self._mask_sorted(ts, past_start, future_start) - past_ia_times = np.diff(np.r_[0, ts[past_mask] - past_start])[ - np.newaxis - ] + past_ia_times = np.diff(np.r_[0, ts[past_mask] - past_start])[np.newaxis] r[f"past_{self.target_field}"] = np.concatenate( [past_ia_times, marks[:, past_mask]], axis=0 @@ -532,9 +505,9 @@ class ContinuousTimeInstanceSplitter(FlatMapTransformation): future_mask = self._mask_sorted(ts, future_start, future_end) - future_ia_times = np.diff( - np.r_[0, ts[future_mask] - future_start] - )[np.newaxis] + future_ia_times = np.diff(np.r_[0, ts[future_mask] - future_start])[ + np.newaxis + ] r[f"future_{self.target_field}"] = np.concatenate( [future_ia_times, marks[:, future_mask]], axis=0