From aa9aa2ed81700e66402094676f844ce8622b3d6d Mon Sep 17 00:00:00 2001 From: Kashif Rasul Date: Sun, 17 Nov 2019 11:03:26 +0100 Subject: [PATCH] added inital scaler --- pts/modules/__init__.py | 3 +- pts/modules/scaler.py | 109 +++++++++++++++++++ test/modules/test_scaler.py | 202 ++++++++++++++++++++++++++++++++++++ 3 files changed, 313 insertions(+), 1 deletion(-) create mode 100644 pts/modules/scaler.py create mode 100644 test/modules/test_scaler.py diff --git a/pts/modules/__init__.py b/pts/modules/__init__.py index 92a406c..e34b7d8 100644 --- a/pts/modules/__init__.py +++ b/pts/modules/__init__.py @@ -1,3 +1,4 @@ from .distribution_output import ArgProj, Output, DistributionOutput, StudentTOutput from .lambda_layer import LambdaLayer -from .feature import FeatureEmbedder, FeatureAssembler \ No newline at end of file +from .feature import FeatureEmbedder, FeatureAssembler +from .scaler import MeanScaler, NOPScaler \ No newline at end of file diff --git a/pts/modules/scaler.py b/pts/modules/scaler.py new file mode 100644 index 0000000..fb554bd --- /dev/null +++ b/pts/modules/scaler.py @@ -0,0 +1,109 @@ +from typing import Tuple +from abc import ABC, abstractmethod + +import torch +import torch.nn as nn + + +class Scaler(ABC, nn.Module): + def __init__(self, keepdim: bool = False): + super().__init__() + self.keepdim = keepdim + + @abstractmethod + def compute_scale( + self, data: torch.Tensor, observed_indicator: torch.Tensor + ) -> torch.Tensor: + pass + + def forward( + self, data: torch.Tensor, observed_indicator: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Parameters + ---------- + data + tensor of shape (N, T, C) containing the data to be scaled + + observed_indicator + observed_indicator: binary tensor with the same shape as + ``data``, that has 1 in correspondence of observed data points, + and 0 in correspondence of missing data points. + + Returns + ------- + Tensor + Tensor containing the "scaled" data, shape: (N, T, C). + Tensor + Tensor containing the scale, of shape (N, C) if ``keepdim == False``, and shape + (N, 1, C) if ``keepdim == True``. + """ + + scale = self.compute_scale(data, observed_indicator) + + if self.keepdim: + scale = scale.unsqueeze(1) + return data / scale, scale + else: + return data / scale.unsqueeze(1), scale + + +class MeanScaler(Scaler): + """ + The ``MeanScaler`` computes a per-item scale according to the average + absolute value over time of each item. The average is computed only among + the observed values in the data tensor, as indicated by the second + argument. Items with no observed data are assigned a scale based on the + global average. + + Parameters + ---------- + minimum_scale + default scale that is used if the time series has only zeros. + """ + + def __init__(self, minimum_scale: float = 1e-10, *args, **kwargs): + super().__init__(*args, **kwargs) + self.minimum_scale = minimum_scale + + def compute_scale( + self, data: torch.Tensor, observed_indicator: torch.Tensor + ) -> torch.Tensor: + # these will have shape (N, C) + num_observed = observed_indicator.sum(dim=1) + sum_observed = (data.abs() * observed_indicator).sum(dim=1) + + # first compute a global scale per-dimension + total_observed = num_observed.sum(dim=0) + denominator = torch.max(total_observed, torch.tensor(1.0)) + default_scale = sum_observed.sum(dim=0) / denominator + + # then compute a per-item, per-dimension scale + denominator = torch.max(num_observed, torch.tensor(1.0)) + scale = sum_observed / denominator + + # use per-batch scale when no element is observed + # or when the sequence contains only zeros + scale = torch.where( + sum_observed > torch.zeros_like(sum_observed), + scale, + default_scale * torch.ones_like(num_observed), + ) + + return torch.max(scale, torch.tensor(self.minimum_scale)) + + +class NOPScaler(Scaler): + """ + The ``NOPScaler`` assigns a scale equals to 1 to each input item, i.e., + no scaling is applied upon calling the ``NOPScaler``. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def compute_scale( + self, data: torch.Tensor, observed_indicator: torch.Tensor + ) -> torch.Tensor: + return torch.ones_like(data).mean(dim=1) + diff --git a/test/modules/test_scaler.py b/test/modules/test_scaler.py new file mode 100644 index 0000000..42a63af --- /dev/null +++ b/test/modules/test_scaler.py @@ -0,0 +1,202 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + +import pytest +import numpy as np + +import torch + +from pts.modules import MeanScaler, NOPScaler + + +test_cases = [ + ( + MeanScaler(), + torch.tensor( + [ + [1.0] * 50, + [0.0] * 25 + [3.0] * 25, + [2.0] * 49 + [1.5] * 1, + [0.0] * 50, + [1.0] * 50, + ] + ), + torch.tensor( + [ + [1.0] * 50, + [0.0] * 25 + [1.0] * 25, + [0.0] * 49 + [1.0] * 1, + [1.0] * 50, + [0.0] * 50, + ] + ), + torch.tensor([1.0, 3.0, 1.5, 1.00396824, 1.00396824]), + ), + ( + MeanScaler(keepdim=True), + torch.tensor( + [ + [1.0] * 50, + [0.0] * 25 + [3.0] * 25, + [2.0] * 49 + [1.5] * 1, + [0.0] * 50, + [1.0] * 50, + ] + ), + torch.tensor( + [ + [1.0] * 50, + [0.0] * 25 + [1.0] * 25, + [0.0] * 49 + [1.0] * 1, + [1.0] * 50, + [0.0] * 50, + ] + ), + torch.tensor([1.0, 3.0, 1.5, 1.00396824, 1.00396824]).unsqueeze(1), + ), + ( + MeanScaler(), + torch.tensor( + [ + [[1.0]] * 50, + [[0.0]] * 25 + [[3.0]] * 25, + [[2.0]] * 49 + [[1.5]] * 1, + [[0.0]] * 50, + [[1.0]] * 50, + ] + ), + torch.tensor( + [ + [[1.0]] * 50, + [[0.0]] * 25 + [[1.0]] * 25, + [[0.0]] * 49 + [[1.0]] * 1, + [[1.0]] * 50, + [[0.0]] * 50, + ] + ), + torch.tensor([1.0, 3.0, 1.5, 1.00396824, 1.00396824]).unsqueeze(1), + ), + ( + MeanScaler(minimum_scale=1e-8), + torch.tensor( + [ + [[1.0, 2.0]] * 50, + [[0.0, 0.0]] * 25 + [[3.0, 6.0]] * 25, + [[2.0, 4.0]] * 49 + [[1.5, 3.0]] * 1, + [[0.0, 0.0]] * 50, + [[1.0, 2.0]] * 50, + ] + ), + torch.tensor( + [ + [[1.0, 1.0]] * 50, + [[0.0, 1.0]] * 25 + [[1.0, 0.0]] * 25, + [[1.0, 0.0]] * 49 + [[0.0, 1.0]] * 1, + [[1.0, 0.0]] * 50, + [[0.0, 1.0]] * 50, + ] + ), + torch.tensor( + [ + [1.0, 2.0], + [3.0, 1.61111116], + [2.0, 3.0], + [1.28160918, 1.61111116], + [1.28160918, 2.0], + ] + ), + ), + ( + MeanScaler(), + torch.tensor( + [ + [120.0] * 25 + [150.0] * 25, + [0.0] * 10 + [3.0] * 20 + [61.0] * 20, + [0.0] * 50, + [2e-2] * 10 + [0.0] * 30 + [3e-2] * 10, + ] + ), + torch.tensor( + [ + [1.0] * 25 + [1.0] * 25, + [0.0] * 10 + [1.0] * 20 + [1.0] * 20, + [0.0] * 50, + [1.0] * 10 + [0.0] * 30 + [1.0] * 10, + ] + ), + torch.tensor([135.0, 32.0, 73.00454712, 2.5e-2]), + ), + ( + MeanScaler(), + torch.randn((5, 30)), + torch.zeros((5, 30)), + 1e-10 * torch.ones((5,)), + ), + ( + MeanScaler(minimum_scale=1e-6), + torch.randn((5, 30, 1)), + torch.zeros((5, 30, 1)), + 1e-6 * torch.ones((5, 1)), + ), + ( + MeanScaler(minimum_scale=1e-12), + torch.randn((5, 30, 3)), + torch.zeros((5, 30, 3)), + 1e-12 * torch.ones((5, 3)), + ), + ( + NOPScaler(), + torch.randn((10, 20, 30)), + torch.randn((10, 20, 30)) > 0, + torch.ones((10, 30)), + ), + ( + NOPScaler(), + torch.randn((10, 20, 30)), + torch.ones((10, 20, 30)), + torch.ones((10, 30)), + ), + ( + NOPScaler(), + torch.randn((10, 20, 30)), + torch.zeros((10, 20, 30)), + torch.ones((10, 30)), + ), +] + + +@pytest.mark.parametrize("s, target, observed, expected_scale", test_cases) +def test_scaler(s, target, observed, expected_scale): + target_scaled, scale = s(target, observed) + + assert np.allclose( + expected_scale.numpy(), scale.numpy() + ), "mismatch in the scale computation" + + if s.keepdim: + expected_target_scaled = target / expected_scale + else: + expected_target_scaled = target / expected_scale.unsqueeze(1) + + assert np.allclose( + expected_target_scaled.numpy(), target_scaled.numpy() + ), "mismatch in the scaled target computation" + + +@pytest.mark.parametrize("target, observed", []) +def test_nopscaler(target, observed): + s = NOPScaler() + target_scaled, scale = s(target, observed) + + assert torch.norm(target - target_scaled) == 0 + assert torch.norm(torch.ones_like(target).mean(dim=1) - scale) == 0