From cb012405bf236062f28aa175fd724daecad54d78 Mon Sep 17 00:00:00 2001 From: "Dr. Kashif Rasul" Date: Mon, 30 Mar 2020 13:01:08 +0200 Subject: [PATCH] optimized the imports --- pts/__init__.py | 7 ++-- pts/core/component.py | 12 +++--- pts/core/serde.py | 17 ++++---- pts/dataset/__init__.py | 42 +++++++++---------- pts/dataset/artificial.py | 4 +- pts/dataset/common.py | 3 +- pts/dataset/file_dataset.py | 6 +-- pts/dataset/list_dataset.py | 2 +- pts/dataset/loader.py | 3 +- pts/dataset/multivariate_grouper.py | 3 +- pts/dataset/repository/_gp_copula_2019.py | 3 +- pts/dataset/repository/_lstnet.py | 1 - pts/dataset/repository/_m4.py | 6 +-- pts/dataset/repository/datasets.py | 3 +- pts/dataset/stat.py | 2 +- pts/dataset/transformed_iterable_dataset.py | 4 +- pts/dataset/utils.py | 2 +- pts/evaluation/__init__.py | 2 +- pts/evaluation/backtest.py | 7 ++-- pts/evaluation/evaluator.py | 3 +- pts/feature/__init__.py | 2 +- pts/feature/lag.py | 2 +- pts/feature/time_feature.py | 1 + pts/feature/utils.py | 2 +- pts/model/deepar/deepar_estimator.py | 8 ++-- pts/model/deepar/deepar_network.py | 4 +- pts/model/deepvar/deepvar_estimator.py | 16 +++---- pts/model/deepvar/deepvar_network.py | 4 +- pts/model/estimator.py | 3 +- pts/model/forecast.py | 2 +- pts/model/forecast_generator.py | 2 +- pts/model/n_beats/__init__.py | 2 +- pts/model/n_beats/n_beats_ensemble.py | 9 ++-- pts/model/n_beats/n_beats_estimator.py | 7 +--- pts/model/n_beats/n_beats_network.py | 1 - pts/model/predictor.py | 15 +++---- .../simple_feedforward_estimator.py | 3 +- .../simple_feedforward_network.py | 2 +- pts/model/tempflow/tempflow_estimator.py | 19 +++------ pts/model/tempflow/tempflow_network.py | 3 +- .../transformer/transformer_estimator.py | 38 +++++++---------- pts/model/transformer/transformer_network.py | 5 +-- .../transformer_tempflow_estimator.py | 19 +++------ .../transformer_tempflow_network.py | 4 +- pts/model/utils.py | 2 +- pts/modules/__init__.py | 4 +- pts/modules/distribution_output.py | 4 +- pts/modules/feature.py | 2 +- pts/modules/flows.py | 1 - pts/modules/lambda_layer.py | 1 - pts/modules/scaler.py | 2 +- pts/trainer.py | 4 +- pts/transform/__init__.py | 27 +++++------- pts/transform/convert.py | 6 +-- pts/transform/feature.py | 4 +- pts/transform/field.py | 2 +- pts/transform/sampler.py | 2 +- pts/transform/split.py | 7 ++-- pts/transform/transform.py | 5 +-- setup.py | 1 - test/dataset/test_multivariate_grouper.py | 2 +- test/dataset/test_process.py | 2 +- test/model/deepar/test_auxillary_outputs.py | 9 ++-- test/model/test_deepvar.py | 10 ++--- test/model/test_forecast.py | 3 +- test/modules/test_distribution_output.py | 11 +++-- test/modules/test_feature.py | 4 +- test/modules/test_scaler.py | 4 +- test/test_transform.py | 4 +- 69 files changed, 180 insertions(+), 248 deletions(-) diff --git a/pts/__init__.py b/pts/__init__.py index 63a6af7..0a48be3 100644 --- a/pts/__init__.py +++ b/pts/__init__.py @@ -1,8 +1,9 @@ from pkgutil import extend_path -from pkg_resources import get_distribution, DistributionNotFound -from .trainer import Trainer -from .exception import assert_pts +from pkg_resources import get_distribution, DistributionNotFound + +from .exception import assert_pts +from .trainer import Trainer __path__ = extend_path(__path__, __name__) # type: ignore diff --git a/pts/core/component.py b/pts/core/component.py index 6ad703e..9888596 100644 --- a/pts/core/component.py +++ b/pts/core/component.py @@ -1,10 +1,12 @@ -import inspect -from pydantic import BaseConfig, BaseModel, create_model -from typing import Any -from collections import OrderedDict -from pts.core.serde import dump_code import functools +import inspect +from collections import OrderedDict +from typing import Any + import torch +from pydantic import BaseConfig, BaseModel, create_model + +from pts.core.serde import dump_code class BaseValidatedInitializerModel(BaseModel): diff --git a/pts/core/serde.py b/pts/core/serde.py index 4df49ce..83d7b0c 100644 --- a/pts/core/serde.py +++ b/pts/core/serde.py @@ -1,13 +1,14 @@ -from typing import Any, Optional, cast, NamedTuple -import json -from functools import singledispatch -from pts.core import fqname_for -import numpy as np -import textwrap -from pydoc import locate -import math import itertools +import json +import math +import textwrap +from functools import singledispatch +from pydoc import locate +from typing import Any, Optional, cast, NamedTuple +import numpy as np + +from pts.core import fqname_for bad_type_msg = textwrap.dedent( """ diff --git a/pts/dataset/__init__.py b/pts/dataset/__init__.py index 0f37e34..699a841 100644 --- a/pts/dataset/__init__.py +++ b/pts/dataset/__init__.py @@ -1,24 +1,3 @@ -from .common import ( - DataEntry, - FieldName, - Dataset, - MetaData, - TrainDatasets, - DateConstants, -) -from .list_dataset import ListDataset -from .file_dataset import FileDataset -from .loader import TrainDataLoader, InferenceDataLoader -from .process import ProcessStartField, ProcessDataEntry -from .utils import ( - to_pandas, - load_datasets, - save_datasets, - serialize_data_entry, - frequency_add, - forecast_start, -) -from .stat import DatasetStatistics, ScaleHistogram, calculate_dataset_statistics from .artificial import ( ArtificialDataset, ConstantDataset, @@ -28,5 +7,26 @@ from .artificial import ( default_synthetic, generate_sf2, ) +from .common import ( + DataEntry, + FieldName, + Dataset, + MetaData, + TrainDatasets, + DateConstants, +) +from .file_dataset import FileDataset +from .list_dataset import ListDataset +from .loader import TrainDataLoader, InferenceDataLoader from .multivariate_grouper import MultivariateGrouper +from .process import ProcessStartField, ProcessDataEntry +from .stat import DatasetStatistics, ScaleHistogram, calculate_dataset_statistics from .transformed_iterable_dataset import TransformedIterableDataset +from .utils import ( + to_pandas, + load_datasets, + save_datasets, + serialize_data_entry, + frequency_add, + forecast_start, +) diff --git a/pts/dataset/artificial.py b/pts/dataset/artificial.py index fdc3d4f..a7f7783 100644 --- a/pts/dataset/artificial.py +++ b/pts/dataset/artificial.py @@ -1,5 +1,5 @@ -import os import math +import os import random from typing import Callable, List, NamedTuple, Optional, Tuple, Union @@ -17,7 +17,6 @@ from .common import ( DataEntry, ) from .list_dataset import ListDataset -from .stat import DatasetStatistics, calculate_dataset_statistics from .recipe import ( BinaryHolidays, BinaryMarkovChain, @@ -31,6 +30,7 @@ from .recipe import ( generate, take_as_list, ) +from .stat import DatasetStatistics, calculate_dataset_statistics class DatasetInfo(NamedTuple): diff --git a/pts/dataset/common.py b/pts/dataset/common.py index f16ecf5..1221e4d 100644 --- a/pts/dataset/common.py +++ b/pts/dataset/common.py @@ -1,5 +1,4 @@ -from abc import ABC, abstractmethod -from typing import Any, Dict, Iterable, NamedTuple, Sized, List, Optional, Iterator +from typing import Any, Dict, Iterable, NamedTuple, List, Optional import pandas as pd from pydantic import BaseModel diff --git a/pts/dataset/file_dataset.py b/pts/dataset/file_dataset.py index 6d1f5b4..9d298cb 100644 --- a/pts/dataset/file_dataset.py +++ b/pts/dataset/file_dataset.py @@ -1,9 +1,9 @@ import functools -from pathlib import Path -from typing import NamedTuple -from typing import Iterator, List import glob import random +from pathlib import Path +from typing import Iterator, List +from typing import NamedTuple import rapidjson as json diff --git a/pts/dataset/list_dataset.py b/pts/dataset/list_dataset.py index a2d2170..3648f05 100644 --- a/pts/dataset/list_dataset.py +++ b/pts/dataset/list_dataset.py @@ -1,5 +1,5 @@ -from typing import Iterable import random +from typing import Iterable from .common import DataEntry, Dataset, SourceContext from .process import ProcessDataEntry diff --git a/pts/dataset/loader.py b/pts/dataset/loader.py index 17e806e..fa33585 100644 --- a/pts/dataset/loader.py +++ b/pts/dataset/loader.py @@ -3,13 +3,12 @@ from collections import defaultdict from typing import Any, Dict, Iterable, Iterator, List, Optional # noqa: F401 import numpy as np - # Third-party imports import torch +from pts.transform.transform import Transformation # First-party imports from .common import DataEntry, Dataset -from pts.transform import Transformation DataBatch = Dict[str, Any] diff --git a/pts/dataset/multivariate_grouper.py b/pts/dataset/multivariate_grouper.py index 765ba8e..2f12b48 100644 --- a/pts/dataset/multivariate_grouper.py +++ b/pts/dataset/multivariate_grouper.py @@ -13,9 +13,10 @@ # Standard library imports import logging +from typing import Callable, Optional + import numpy as np import pandas as pd -from typing import Callable, Optional # First-party imports from .common import DataEntry, Dataset, FieldName, DateConstants diff --git a/pts/dataset/repository/_gp_copula_2019.py b/pts/dataset/repository/_gp_copula_2019.py index 2debcbf..591f3b9 100644 --- a/pts/dataset/repository/_gp_copula_2019.py +++ b/pts/dataset/repository/_gp_copula_2019.py @@ -18,14 +18,13 @@ large files in GluonTS master. """ import json import os -import tarfile import shutil +import tarfile from pathlib import Path from typing import NamedTuple, Optional from urllib import request from pts.dataset import FileDataset, FieldName - from ._util import save_to_file, to_dict, metadata diff --git a/pts/dataset/repository/_lstnet.py b/pts/dataset/repository/_lstnet.py index 311385c..ade1708 100644 --- a/pts/dataset/repository/_lstnet.py +++ b/pts/dataset/repository/_lstnet.py @@ -23,7 +23,6 @@ from typing import List, NamedTuple, Optional import pandas as pd from pts.dataset import frequency_add - from ._util import save_to_file, to_dict, metadata diff --git a/pts/dataset/repository/_m4.py b/pts/dataset/repository/_m4.py index ffb1641..6fce8ad 100644 --- a/pts/dataset/repository/_m4.py +++ b/pts/dataset/repository/_m4.py @@ -11,12 +11,12 @@ # express or implied. See the License for the specific language governing # permissions and limitations under the License. -from pathlib import Path -import os import json +import os +from pathlib import Path -import pandas as pd import numpy as np +import pandas as pd from ._util import save_to_file, to_dict, metadata diff --git a/pts/dataset/repository/datasets.py b/pts/dataset/repository/datasets.py index aee5190..e173e6f 100644 --- a/pts/dataset/repository/datasets.py +++ b/pts/dataset/repository/datasets.py @@ -17,11 +17,10 @@ from functools import partial from pathlib import Path from pts.dataset import ConstantDataset, TrainDatasets, load_datasets - from ._artificial import generate_artificial_dataset +from ._gp_copula_2019 import generate_gp_copula_dataset from ._lstnet import generate_lstnet_dataset from ._m4 import generate_m4_dataset -from ._gp_copula_2019 import generate_gp_copula_dataset from ._util import get_download_path m4_freq = "Hourly" diff --git a/pts/dataset/stat.py b/pts/dataset/stat.py index 9f81566..5568710 100644 --- a/pts/dataset/stat.py +++ b/pts/dataset/stat.py @@ -5,8 +5,8 @@ from typing import Any, List, NamedTuple, Optional, Set import numpy as np from tqdm import tqdm -from .common import FieldName from pts.exception import assert_pts +from .common import FieldName class ScaleHistogram: diff --git a/pts/dataset/transformed_iterable_dataset.py b/pts/dataset/transformed_iterable_dataset.py index 6027edc..3be87c6 100644 --- a/pts/dataset/transformed_iterable_dataset.py +++ b/pts/dataset/transformed_iterable_dataset.py @@ -1,12 +1,10 @@ import itertools from typing import Dict, Iterable, Iterator, Optional -import random import numpy as np import torch -from pts.transform import Transformation - +from pts.transform.transform import Transformation from .common import DataEntry, Dataset diff --git a/pts/dataset/utils.py b/pts/dataset/utils.py index e571cde..4bd3002 100644 --- a/pts/dataset/utils.py +++ b/pts/dataset/utils.py @@ -1,5 +1,5 @@ -from pathlib import Path import shutil +from pathlib import Path import numpy as np import pandas as pd diff --git a/pts/evaluation/__init__.py b/pts/evaluation/__init__.py index 42273d1..95aefb3 100644 --- a/pts/evaluation/__init__.py +++ b/pts/evaluation/__init__.py @@ -1,2 +1,2 @@ -from .evaluator import Evaluator, MultivariateEvaluator from .backtest import make_evaluation_predictions, backtest_metrics +from .evaluator import Evaluator, MultivariateEvaluator diff --git a/pts/evaluation/backtest.py b/pts/evaluation/backtest.py index e714296..04210cf 100644 --- a/pts/evaluation/backtest.py +++ b/pts/evaluation/backtest.py @@ -5,16 +5,15 @@ from typing import Dict, Iterator, NamedTuple, Optional, Tuple, Union # Third-party imports import pandas as pd -# First-party imports -from pts.transform import AdhocTransform, TransformedDataset from pts.dataset import ( DataEntry, Dataset, - InferenceDataLoader, DatasetStatistics, calculate_dataset_statistics, ) -from pts.model import Estimator, PTSEstimator, PTSPredictor, Predictor, Forecast +from pts.model import Estimator, Predictor, Forecast +# First-party imports +from pts.transform import AdhocTransform, TransformedDataset from .evaluator import Evaluator diff --git a/pts/evaluation/evaluator.py b/pts/evaluation/evaluator.py index 712d36d..766c7af 100644 --- a/pts/evaluation/evaluator.py +++ b/pts/evaluation/evaluator.py @@ -10,15 +10,14 @@ from typing import ( Union, Callable, ) -from tqdm import tqdm # Third-party imports import numpy as np import pandas as pd from tqdm import tqdm -from pts.model import Quantile, Forecast from pts.feature import get_seasonality +from pts.model import Quantile, Forecast class Evaluator: diff --git a/pts/feature/__init__.py b/pts/feature/__init__.py index ac7bc20..c85f7f7 100644 --- a/pts/feature/__init__.py +++ b/pts/feature/__init__.py @@ -1,3 +1,4 @@ +from .holiday import SPECIAL_DATE_FEATURES, SpecialDateFeatureSet from .lag import get_lags_for_frequency, get_fourier_lags_for_frequency from .time_feature import ( DayOfMonth, @@ -12,5 +13,4 @@ from .time_feature import ( time_features_from_frequency_str, fourier_time_features_from_frequency_str, ) -from .holiday import SPECIAL_DATE_FEATURES, SpecialDateFeatureSet from .utils import get_granularity, get_seasonality diff --git a/pts/feature/lag.py b/pts/feature/lag.py index bf16c55..c4c0982 100644 --- a/pts/feature/lag.py +++ b/pts/feature/lag.py @@ -12,7 +12,7 @@ # permissions and limitations under the License. # Standard library imports -from typing import List, Optional, Tuple +from typing import List, Optional # Third-party imports import numpy as np diff --git a/pts/feature/time_feature.py b/pts/feature/time_feature.py index 816a8a0..30d22dd 100644 --- a/pts/feature/time_feature.py +++ b/pts/feature/time_feature.py @@ -4,6 +4,7 @@ from typing import List import numpy as np import pandas as pd from pandas.tseries.frequencies import to_offset + from pts.core.component import validated from .utils import get_granularity diff --git a/pts/feature/utils.py b/pts/feature/utils.py index 71ca135..c0e23d0 100644 --- a/pts/feature/utils.py +++ b/pts/feature/utils.py @@ -1,6 +1,6 @@ import re -from typing import Tuple from functools import lru_cache +from typing import Tuple def get_granularity(freq_str: str) -> Tuple[int, str]: diff --git a/pts/model/deepar/deepar_estimator.py b/pts/model/deepar/deepar_estimator.py index a83d76a..965f5b9 100644 --- a/pts/model/deepar/deepar_estimator.py +++ b/pts/model/deepar/deepar_estimator.py @@ -1,16 +1,18 @@ from typing import List, Optional import numpy as np - import torch import torch.nn as nn from pts import Trainer +from pts.dataset import FieldName from pts.feature import ( TimeFeature, get_lags_for_frequency, time_features_from_frequency_str, ) +from pts.model import PTSEstimator, Predictor, PTSPredictor, copy_parameters +from pts.modules import DistributionOutput, StudentTOutput from pts.transform import ( Transformation, Chain, @@ -24,10 +26,6 @@ from pts.transform import ( InstanceSplitter, ExpectedNumInstanceSampler, ) -from pts.dataset import FieldName -from pts.model import PTSEstimator, Predictor, PTSPredictor, copy_parameters -from pts.modules import DistributionOutput, StudentTOutput - from .deepar_network import DeepARTrainingNetwork, DeepARPredictionNetwork diff --git a/pts/model/deepar/deepar_network.py b/pts/model/deepar/deepar_network.py index 81cb319..1a68742 100644 --- a/pts/model/deepar/deepar_network.py +++ b/pts/model/deepar/deepar_network.py @@ -1,13 +1,13 @@ from typing import List, Optional, Tuple, Union +import numpy as np import torch import torch.nn as nn from torch.distributions import Distribution -import numpy as np from pts.core.component import validated -from pts.modules import DistributionOutput, MeanScaler, NOPScaler, FeatureEmbedder from pts.model import weighted_average +from pts.modules import DistributionOutput, MeanScaler, NOPScaler, FeatureEmbedder def prod(xs): diff --git a/pts/model/deepvar/deepvar_estimator.py b/pts/model/deepvar/deepvar_estimator.py index d5fa798..9b09fc6 100644 --- a/pts/model/deepvar/deepvar_estimator.py +++ b/pts/model/deepvar/deepvar_estimator.py @@ -1,14 +1,16 @@ from typing import List, Optional -import numpy as np -import pandas as pd import torch -import torch.nn as nn from pts import Trainer +from pts.dataset import FieldName +from pts.feature import ( + TimeFeature, + fourier_time_features_from_frequency_str, + get_fourier_lags_for_frequency, +) from pts.model import PTSEstimator, PTSPredictor, copy_parameters from pts.modules import DistributionOutput, LowRankMultivariateNormalOutput -from pts.dataset import FieldName from pts.transform import ( Transformation, Chain, @@ -25,12 +27,6 @@ from pts.transform import ( SetFieldIfNotPresent, TargetDimIndicator, ) -from pts.feature import ( - TimeFeature, - fourier_time_features_from_frequency_str, - get_fourier_lags_for_frequency, -) - from .deepvar_network import DeepVARTrainingNetwork, DeepVARPredictionNetwork diff --git a/pts/model/deepvar/deepvar_network.py b/pts/model/deepvar/deepvar_network.py index 754f265..7d2e21c 100644 --- a/pts/model/deepvar/deepvar_network.py +++ b/pts/model/deepvar/deepvar_network.py @@ -2,12 +2,10 @@ from typing import List, Optional, Tuple, Union import torch import torch.nn as nn -from torch.distributions import Distribution -import numpy as np from pts.core.component import validated -from pts.modules import DistributionOutput, MeanScaler, NOPScaler from pts.model import weighted_average +from pts.modules import DistributionOutput, MeanScaler, NOPScaler class DeepVARTrainingNetwork(nn.Module): diff --git a/pts/model/estimator.py b/pts/model/estimator.py index 07b047b..2e47b8c 100644 --- a/pts/model/estimator.py +++ b/pts/model/estimator.py @@ -6,10 +6,9 @@ import torch import torch.nn as nn from torch.utils.data import DataLoader +from pts import Trainer from pts.dataset import Dataset, TransformedIterableDataset from pts.transform import Transformation -from pts import Trainer - from .predictor import Predictor from .utils import get_module_forward_input_names diff --git a/pts/model/forecast.py b/pts/model/forecast.py index 35e8784..54d602f 100644 --- a/pts/model/forecast.py +++ b/pts/model/forecast.py @@ -2,10 +2,10 @@ from abc import ABC, abstractmethod from enum import Enum from typing import Dict, List, Optional, Set, Union, Callable -from pydantic import BaseModel, Field import numpy as np import pandas as pd import torch +from pydantic import BaseModel, Field from torch.distributions import Distribution from .quantile import Quantile diff --git a/pts/model/forecast_generator.py b/pts/model/forecast_generator.py index 93e4cab..5323eb6 100644 --- a/pts/model/forecast_generator.py +++ b/pts/model/forecast_generator.py @@ -5,10 +5,10 @@ import numpy as np import torch import torch.nn as nn +from pts.core.component import validated from pts.dataset import InferenceDataLoader, DataEntry, FieldName from pts.modules import DistributionOutput from .forecast import Forecast, DistributionForecast, QuantileForecast, SampleForecast -from pts.core.component import validated OutputTransform = Callable[[DataEntry, np.ndarray], np.ndarray] diff --git a/pts/model/n_beats/__init__.py b/pts/model/n_beats/__init__.py index c18d7f0..4273e54 100644 --- a/pts/model/n_beats/__init__.py +++ b/pts/model/n_beats/__init__.py @@ -1,2 +1,2 @@ -from .n_beats_estimator import NBEATSEstimator from .n_beats_ensemble import NBEATSEnsembleEstimator +from .n_beats_estimator import NBEATSEstimator diff --git a/pts/model/n_beats/n_beats_ensemble.py b/pts/model/n_beats/n_beats_ensemble.py index 9fbdd71..861519c 100644 --- a/pts/model/n_beats/n_beats_ensemble.py +++ b/pts/model/n_beats/n_beats_ensemble.py @@ -1,16 +1,15 @@ -from typing import List, Optional, Iterator -from itertools import product import copy import logging +from itertools import product +from typing import List, Optional, Iterator import numpy as np -from pts.model import Predictor, SampleForecast, Forecast, Estimator from pts import Trainer from pts.dataset import Dataset, FieldName - -from .n_beats_network import VALID_LOSS_FUNCTIONS +from pts.model import Predictor, SampleForecast, Forecast, Estimator from .n_beats_estimator import NBEATSEstimator +from .n_beats_network import VALID_LOSS_FUNCTIONS AGGREGATION_METHODS = "median", "mean", "none" diff --git a/pts/model/n_beats/n_beats_estimator.py b/pts/model/n_beats/n_beats_estimator.py index 5dcd121..38adb0c 100644 --- a/pts/model/n_beats/n_beats_estimator.py +++ b/pts/model/n_beats/n_beats_estimator.py @@ -1,11 +1,10 @@ from typing import List, Optional -import numpy as np - import torch import torch.nn as nn from pts import Trainer +from pts.dataset import FieldName from pts.model import PTSEstimator, Predictor, PTSPredictor, copy_parameters from pts.transform import ( InstanceSplitter, @@ -14,15 +13,13 @@ from pts.transform import ( RemoveFields, ExpectedNumInstanceSampler, ) -from pts.dataset import FieldName - from .n_beats_network import ( NBEATSPredictionNetwork, NBEATSTrainingNetwork, VALID_N_BEATS_STACK_TYPES, - VALID_LOSS_FUNCTIONS, ) + class NBEATSEstimator(PTSEstimator): def __init__( self, diff --git a/pts/model/n_beats/n_beats_network.py b/pts/model/n_beats/n_beats_network.py index 42f928f..af5fcea 100644 --- a/pts/model/n_beats/n_beats_network.py +++ b/pts/model/n_beats/n_beats_network.py @@ -1,7 +1,6 @@ from typing import List, Tuple import numpy as np - import torch import torch.nn as nn import torch.nn.functional as F diff --git a/pts/model/predictor.py b/pts/model/predictor.py index e567059..d1099f8 100644 --- a/pts/model/predictor.py +++ b/pts/model/predictor.py @@ -1,19 +1,20 @@ -import pts import json +from abc import ABC, abstractmethod +from pathlib import Path +from pydoc import locate +from typing import Iterator, Callable, Optional + import numpy as np import torch import torch.nn as nn -from abc import ABC, abstractmethod -from pydoc import locate -from typing import Iterator, Callable, Optional + +import pts +from pts.core.serde import dump_json, fqname_for, load_json from pts.dataset import Dataset, DataEntry, InferenceDataLoader from pts.transform import Transformation -from pathlib import Path from .forecast import Forecast from .forecast_generator import ForecastGenerator, SampleForecastGenerator from .utils import get_module_forward_input_names -from pts.core.serde import dump_json, fqname_for, load_json - OutputTransform = Callable[[DataEntry, np.ndarray], np.ndarray] diff --git a/pts/model/simple_feedforward/simple_feedforward_estimator.py b/pts/model/simple_feedforward/simple_feedforward_estimator.py index 87c6b22..722f338 100644 --- a/pts/model/simple_feedforward/simple_feedforward_estimator.py +++ b/pts/model/simple_feedforward/simple_feedforward_estimator.py @@ -4,16 +4,15 @@ import torch import torch.nn as nn from pts import Trainer +from pts.dataset import FieldName from pts.model import PTSEstimator, PTSPredictor, copy_parameters from pts.modules import DistributionOutput, StudentTOutput -from pts.dataset import FieldName from pts.transform import ( Transformation, Chain, InstanceSplitter, ExpectedNumInstanceSampler, ) - from .simple_feedforward_network import ( SimpleFeedForwardTrainingNetwork, SimpleFeedForwardPredictionNetwork, diff --git a/pts/model/simple_feedforward/simple_feedforward_network.py b/pts/model/simple_feedforward/simple_feedforward_network.py index 6749f59..4fe418d 100644 --- a/pts/model/simple_feedforward/simple_feedforward_network.py +++ b/pts/model/simple_feedforward/simple_feedforward_network.py @@ -3,8 +3,8 @@ from typing import List import torch import torch.nn as nn from torch.distributions import Distribution -from pts.core.component import validated +from pts.core.component import validated from pts.modules import MeanScaler, NOPScaler, DistributionOutput, LambdaLayer diff --git a/pts/model/tempflow/tempflow_estimator.py b/pts/model/tempflow/tempflow_estimator.py index addf955..7429770 100644 --- a/pts/model/tempflow/tempflow_estimator.py +++ b/pts/model/tempflow/tempflow_estimator.py @@ -1,21 +1,20 @@ from typing import List, Optional -import numpy as np -import pandas as pd import torch -import torch.nn as nn from pts import Trainer -from pts.model import PTSEstimator, PTSPredictor, copy_parameters -from pts.modules import RealNVP from pts.dataset import FieldName +from pts.feature import ( + TimeFeature, + fourier_time_features_from_frequency_str, + get_fourier_lags_for_frequency, +) +from pts.model import PTSEstimator, PTSPredictor, copy_parameters from pts.transform import ( Transformation, Chain, InstanceSplitter, ExpectedNumInstanceSampler, - CDFtoGaussianTransform, - cdf_to_gaussian_forward_transform, RenameFields, AsNumpyArray, ExpandDimArray, @@ -25,12 +24,6 @@ from pts.transform import ( SetFieldIfNotPresent, TargetDimIndicator, ) -from pts.feature import ( - TimeFeature, - fourier_time_features_from_frequency_str, - get_fourier_lags_for_frequency, -) - from .tempflow_network import TempFlowTrainingNetwork, TempFlowPredictionNetwork diff --git a/pts/model/tempflow/tempflow_network.py b/pts/model/tempflow/tempflow_network.py index 94cf29e..e4ee471 100644 --- a/pts/model/tempflow/tempflow_network.py +++ b/pts/model/tempflow/tempflow_network.py @@ -3,10 +3,9 @@ from typing import List, Optional, Tuple, Union import torch import torch.nn as nn -import numpy as np from pts.core.component import validated -from pts.modules import RealNVP, MAF, FlowOutput, MeanScaler, NOPScaler from pts.model import weighted_average +from pts.modules import RealNVP, MAF, FlowOutput, MeanScaler, NOPScaler class TempFlowTrainingNetwork(nn.Module): diff --git a/pts/model/transformer/transformer_estimator.py b/pts/model/transformer/transformer_estimator.py index 4e43df6..da285bb 100644 --- a/pts/model/transformer/transformer_estimator.py +++ b/pts/model/transformer/transformer_estimator.py @@ -1,39 +1,31 @@ from typing import List, Optional import numpy as np -import pandas as pd import torch import torch.nn as nn from pts import Trainer -from pts.model import PTSEstimator, Predictor, PTSPredictor, copy_parameters -from pts.modules import DistributionOutput, StudentTOutput from pts.dataset import FieldName -from pts.transform import ( - Transformation, - Chain, - InstanceSplitter, - ExpectedNumInstanceSampler, - CDFtoGaussianTransform, - cdf_to_gaussian_forward_transform, - RenameFields, - RemoveFields, - AddAgeFeature, - AsNumpyArray, - ExpandDimArray, - AddObservedValuesIndicator, - AddTimeFeatures, - VstackFeatures, - SetFieldIfNotPresent, - TargetDimIndicator, - SetField, -) from pts.feature import ( TimeFeature, fourier_time_features_from_frequency_str, get_fourier_lags_for_frequency, ) - +from pts.model import PTSEstimator, Predictor, PTSPredictor, copy_parameters +from pts.modules import DistributionOutput, StudentTOutput +from pts.transform import ( + Transformation, + Chain, + InstanceSplitter, + ExpectedNumInstanceSampler, + RemoveFields, + AddAgeFeature, + AsNumpyArray, + AddObservedValuesIndicator, + AddTimeFeatures, + VstackFeatures, + SetField, +) from .transformer_network import ( TransformerTrainingNetwork, TransformerPredictionNetwork, diff --git a/pts/model/transformer/transformer_network.py b/pts/model/transformer/transformer_network.py index 12f6256..38e5ca5 100644 --- a/pts/model/transformer/transformer_network.py +++ b/pts/model/transformer/transformer_network.py @@ -1,13 +1,10 @@ -from typing import List, Optional, Tuple, Union +from typing import List, Optional, Tuple import torch import torch.nn as nn -from torch.distributions import Distribution -import numpy as np from pts.core.component import validated from pts.modules import DistributionOutput, MeanScaler, NOPScaler, FeatureEmbedder -from pts.model import weighted_average def prod(xs): diff --git a/pts/model/transformer_tempflow/transformer_tempflow_estimator.py b/pts/model/transformer_tempflow/transformer_tempflow_estimator.py index aabef6b..3f93183 100644 --- a/pts/model/transformer_tempflow/transformer_tempflow_estimator.py +++ b/pts/model/transformer_tempflow/transformer_tempflow_estimator.py @@ -1,21 +1,20 @@ from typing import List, Optional -import numpy as np -import pandas as pd import torch -import torch.nn as nn from pts import Trainer -from pts.model import PTSEstimator, PTSPredictor, copy_parameters -from pts.modules import RealNVP from pts.dataset import FieldName +from pts.feature import ( + TimeFeature, + fourier_time_features_from_frequency_str, + get_fourier_lags_for_frequency, +) +from pts.model import PTSEstimator, PTSPredictor, copy_parameters from pts.transform import ( Transformation, Chain, InstanceSplitter, ExpectedNumInstanceSampler, - CDFtoGaussianTransform, - cdf_to_gaussian_forward_transform, RenameFields, AsNumpyArray, ExpandDimArray, @@ -25,12 +24,6 @@ from pts.transform import ( SetFieldIfNotPresent, TargetDimIndicator, ) -from pts.feature import ( - TimeFeature, - fourier_time_features_from_frequency_str, - get_fourier_lags_for_frequency, -) - from .transformer_tempflow_network import TransformerTempFlowTrainingNetwork, TransformerTempFlowPredictionNetwork diff --git a/pts/model/transformer_tempflow/transformer_tempflow_network.py b/pts/model/transformer_tempflow/transformer_tempflow_network.py index 37c1b87..3a064e0 100644 --- a/pts/model/transformer_tempflow/transformer_tempflow_network.py +++ b/pts/model/transformer_tempflow/transformer_tempflow_network.py @@ -1,12 +1,10 @@ -from typing import List, Optional, Tuple, Union +from typing import List, Optional, Tuple import torch import torch.nn as nn -import numpy as np from pts.core.component import validated from pts.modules import RealNVP, MAF, FlowOutput, MeanScaler, NOPScaler -from pts.model import weighted_average class TransformerTempFlowTrainingNetwork(nn.Module): diff --git a/pts/model/utils.py b/pts/model/utils.py index 08fe7d2..356bd08 100644 --- a/pts/model/utils.py +++ b/pts/model/utils.py @@ -1,5 +1,5 @@ -from typing import Optional import inspect +from typing import Optional import torch import torch.nn as nn diff --git a/pts/modules/__init__.py b/pts/modules/__init__.py index 73090f8..df0beba 100644 --- a/pts/modules/__init__.py +++ b/pts/modules/__init__.py @@ -11,7 +11,7 @@ from .distribution_output import ( MultivariateNormalOutput, FlowOutput, ) -from .lambda_layer import LambdaLayer from .feature import FeatureEmbedder, FeatureAssembler -from .scaler import MeanScaler, NOPScaler from .flows import RealNVP, MAF +from .lambda_layer import LambdaLayer +from .scaler import MeanScaler, NOPScaler diff --git a/pts/modules/distribution_output.py b/pts/modules/distribution_output.py index 05cb53c..ad08783 100644 --- a/pts/modules/distribution_output.py +++ b/pts/modules/distribution_output.py @@ -5,10 +5,8 @@ import numpy as np import torch import torch.nn as nn import torch.nn.functional as F -from pts.core.component import validated from torch.distributions import ( Distribution, - Normal, Beta, NegativeBinomial, StudentT, @@ -20,8 +18,8 @@ from torch.distributions import ( AffineTransform, ) +from pts.core.component import validated from .lambda_layer import LambdaLayer -from .flows import RealNVP class ArgProj(nn.Module): diff --git a/pts/modules/feature.py b/pts/modules/feature.py index a4aae50..db794e2 100644 --- a/pts/modules/feature.py +++ b/pts/modules/feature.py @@ -1,4 +1,4 @@ -from typing import Callable, List, Optional +from typing import List, Optional import torch import torch.nn as nn diff --git a/pts/modules/flows.py b/pts/modules/flows.py index 5f90f27..8ef12dd 100644 --- a/pts/modules/flows.py +++ b/pts/modules/flows.py @@ -1,6 +1,5 @@ import copy import math -from abc import ABC, abstractmethod import torch import torch.nn as nn diff --git a/pts/modules/lambda_layer.py b/pts/modules/lambda_layer.py index 65df82f..732bafc 100644 --- a/pts/modules/lambda_layer.py +++ b/pts/modules/lambda_layer.py @@ -1,4 +1,3 @@ -import torch import torch.nn as nn diff --git a/pts/modules/scaler.py b/pts/modules/scaler.py index 35d5beb..f04824f 100644 --- a/pts/modules/scaler.py +++ b/pts/modules/scaler.py @@ -1,5 +1,5 @@ -from typing import Tuple from abc import ABC, abstractmethod +from typing import Tuple import torch import torch.nn as nn diff --git a/pts/trainer.py b/pts/trainer.py index 00728d8..8c61fc3 100644 --- a/pts/trainer.py +++ b/pts/trainer.py @@ -1,12 +1,12 @@ import time -from typing import Any, List, NamedTuple, Optional, Union +from typing import List, Optional import torch import torch.nn as nn +from torch.utils.data import DataLoader from torch.utils.tensorboard import SummaryWriter from tqdm import tqdm -from torch.utils.data import DataLoader class Trainer: def __init__( diff --git a/pts/transform/__init__.py b/pts/transform/__init__.py index 4d1f56b..12cc028 100644 --- a/pts/transform/__init__.py +++ b/pts/transform/__init__.py @@ -1,14 +1,3 @@ -from .transform import ( - Transformation, - Chain, - Identity, - MapTransformation, - SimpleTransformation, - AdhocTransform, - FlatMapTransformation, - FilterTransformation, -) - from .convert import ( AsNumpyArray, ExpandDimArray, @@ -21,9 +10,7 @@ from .convert import ( CDFtoGaussianTransform, cdf_to_gaussian_forward_transform, ) - from .dataset import TransformedDataset - from .feature import ( target_transformation_length, AddObservedValuesIndicator, @@ -31,7 +18,6 @@ from .feature import ( AddTimeFeatures, AddAgeFeature, ) - from .field import ( RemoveFields, RenameFields, @@ -39,8 +25,6 @@ from .field import ( SetFieldIfNotPresent, SelectFields, ) - - from .sampler import ( InstanceSampler, UniformSplitSampler, @@ -50,10 +34,19 @@ from .sampler import ( ContinuousTimePointSampler, ContinuousTimeUniformSampler, ) - from .split import ( shift_timestamp, InstanceSplitter, CanonicalInstanceSplitter, ContinuousTimeInstanceSplitter, ) +from .transform import ( + Transformation, + Chain, + Identity, + MapTransformation, + SimpleTransformation, + AdhocTransform, + FlatMapTransformation, + FilterTransformation, +) diff --git a/pts/transform/convert.py b/pts/transform/convert.py index 2803600..253b5dd 100644 --- a/pts/transform/convert.py +++ b/pts/transform/convert.py @@ -14,14 +14,12 @@ from typing import Iterator, List, Tuple, Optional import numpy as np +import torch from scipy.special import erf, erfinv -import torch - -from pts.exception import assert_pts from pts.core.component import validated from pts.dataset import DataEntry - +from pts.exception import assert_pts from .transform import ( SimpleTransformation, MapTransformation, diff --git a/pts/transform/feature.py b/pts/transform/feature.py index 34d2b3d..84062a6 100644 --- a/pts/transform/feature.py +++ b/pts/transform/feature.py @@ -16,11 +16,11 @@ from typing import List import numpy as np import pandas as pd +from pts.core.component import validated from pts.dataset import DataEntry from pts.feature import TimeFeature -from pts.core.component import validated -from .transform import SimpleTransformation, MapTransformation from .split import shift_timestamp +from .transform import SimpleTransformation, MapTransformation def target_transformation_length( diff --git a/pts/transform/field.py b/pts/transform/field.py index 502afe1..ccbfa19 100644 --- a/pts/transform/field.py +++ b/pts/transform/field.py @@ -14,8 +14,8 @@ from collections import Counter from typing import Any, Dict, List -from pts.dataset import DataEntry from pts.core.component import validated +from pts.dataset import DataEntry from .transform import SimpleTransformation, MapTransformation diff --git a/pts/transform/sampler.py b/pts/transform/sampler.py index a4c2a95..b61a610 100644 --- a/pts/transform/sampler.py +++ b/pts/transform/sampler.py @@ -15,8 +15,8 @@ from abc import ABC, abstractmethod import numpy as np -from pts.dataset.stat import ScaleHistogram from pts.core.component import validated +from pts.dataset.stat import ScaleHistogram class InstanceSampler(ABC): diff --git a/pts/transform/split.py b/pts/transform/split.py index d2ad55e..6395321 100644 --- a/pts/transform/split.py +++ b/pts/transform/split.py @@ -17,11 +17,10 @@ from typing import Iterator, List, Optional import numpy as np import pandas as pd -from pts.dataset import DataEntry, FieldName - -from .transform import FlatMapTransformation -from .sampler import InstanceSampler, ContinuousTimePointSampler from pts.core.component import validated +from pts.dataset import DataEntry, FieldName +from .sampler import InstanceSampler, ContinuousTimePointSampler +from .transform import FlatMapTransformation def shift_timestamp(ts: pd.Timestamp, offset: int) -> pd.Timestamp: diff --git a/pts/transform/transform.py b/pts/transform/transform.py index 0fe680f..5826c91 100644 --- a/pts/transform/transform.py +++ b/pts/transform/transform.py @@ -1,11 +1,10 @@ from abc import ABC, abstractmethod -from typing import Callable, Iterator, Iterable, List from functools import reduce +from typing import Callable, Iterator, Iterable, List + from pts.core.component import validated - from pts.dataset import DataEntry - MAX_IDLE_TRANSFORMS = 100 diff --git a/setup.py b/setup.py index f0f8818..2e69c81 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,6 @@ setup( 'pandas', 'scipy', 'tqdm', - 'ujson', 'pydantic', 'matplotlib', 'python-rapidjson', diff --git a/test/dataset/test_multivariate_grouper.py b/test/dataset/test_multivariate_grouper.py index 730ea72..429fb20 100644 --- a/test/dataset/test_multivariate_grouper.py +++ b/test/dataset/test_multivariate_grouper.py @@ -11,9 +11,9 @@ # express or implied. See the License for the specific language governing # permissions and limitations under the License. +import numpy as np # Standard library imports import pytest -import numpy as np # First-party imports from pts.dataset import ListDataset, MultivariateGrouper diff --git a/test/dataset/test_process.py b/test/dataset/test_process.py index cf341d4..21996d9 100644 --- a/test/dataset/test_process.py +++ b/test/dataset/test_process.py @@ -1,5 +1,5 @@ -import pytest import pandas as pd +import pytest from pts.dataset import ProcessStartField diff --git a/test/model/deepar/test_auxillary_outputs.py b/test/model/deepar/test_auxillary_outputs.py index e0ef100..14c36c4 100644 --- a/test/model/deepar/test_auxillary_outputs.py +++ b/test/model/deepar/test_auxillary_outputs.py @@ -15,12 +15,11 @@ from itertools import islice import torch -from pts.modules import StudentTOutput -from pts.dataset import constant_dataset, TrainDataLoader -from pts.model.deepar import DeepAREstimator -from pts.model import get_module_forward_input_names from pts import Trainer - +from pts.dataset import constant_dataset, TrainDataLoader +from pts.model import get_module_forward_input_names +from pts.model.deepar import DeepAREstimator +from pts.modules import StudentTOutput ds_info, train_ds, test_ds = constant_dataset() freq = ds_info.metadata.freq diff --git a/test/model/test_deepvar.py b/test/model/test_deepvar.py index f4ab350..5f0a9ce 100644 --- a/test/model/test_deepvar.py +++ b/test/model/test_deepvar.py @@ -14,17 +14,17 @@ # First-party imports import pytest +from pts import Trainer +from pts.dataset import TrainDatasets, MultivariateGrouper from pts.dataset.artificial import constant_dataset +from pts.evaluation import MultivariateEvaluator +from pts.evaluation import backtest_metrics +from pts.model.deepvar import DeepVAREstimator from pts.modules import ( IndependentNormalOutput, LowRankMultivariateNormalOutput, MultivariateNormalOutput, ) -from pts.evaluation import backtest_metrics -from pts.model.deepvar import DeepVAREstimator -from pts.dataset import TrainDatasets, MultivariateGrouper -from pts import Trainer -from pts.evaluation import MultivariateEvaluator def load_multivariate_constant_dataset(): diff --git a/test/model/test_forecast.py b/test/model/test_forecast.py index 2c21bb0..1163750 100644 --- a/test/model/test_forecast.py +++ b/test/model/test_forecast.py @@ -16,6 +16,7 @@ import numpy as np import pandas as pd import pytest import torch +from torch.distributions import Uniform # First-party imports from pts.model import ( @@ -24,8 +25,6 @@ from pts.model import ( DistributionForecast, ) -from torch.distributions import Uniform - QUANTILES = np.arange(1, 100) / 100 SAMPLES = np.arange(101).reshape(101, 1) / 100 START_DATE = pd.Timestamp(2017, 1, 1, 12) diff --git a/test/modules/test_distribution_output.py b/test/modules/test_distribution_output.py index 262ea73..9a23f63 100644 --- a/test/modules/test_distribution_output.py +++ b/test/modules/test_distribution_output.py @@ -1,13 +1,9 @@ -import pytest -from typing import Iterable, List, Tuple +from typing import List, Tuple import numpy as np - +import pytest import torch import torch.nn as nn -from torch.nn.utils import clip_grad_norm_ -from torch.utils.data import TensorDataset, DataLoader -from torch.optim import SGD from torch.distributions import ( StudentT, Beta, @@ -17,6 +13,9 @@ from torch.distributions import ( Independent, Normal, ) +from torch.nn.utils import clip_grad_norm_ +from torch.optim import SGD +from torch.utils.data import TensorDataset, DataLoader from pts.modules import ( DistributionOutput, diff --git a/test/modules/test_feature.py b/test/modules/test_feature.py index e1e9ecb..47ca954 100644 --- a/test/modules/test_feature.py +++ b/test/modules/test_feature.py @@ -1,10 +1,8 @@ -import pytest from itertools import chain, combinations - +import pytest import torch import torch.nn as nn -from torch.distributions import Uniform from pts.modules import FeatureEmbedder, FeatureAssembler diff --git a/test/modules/test_scaler.py b/test/modules/test_scaler.py index 42a63af..8475e1a 100644 --- a/test/modules/test_scaler.py +++ b/test/modules/test_scaler.py @@ -11,14 +11,12 @@ # express or implied. See the License for the specific language governing # permissions and limitations under the License. -import pytest import numpy as np - +import pytest import torch from pts.modules import MeanScaler, NOPScaler - test_cases = [ ( MeanScaler(), diff --git a/test/test_transform.py b/test/test_transform.py index d5cdae0..48c6ab9 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -17,9 +17,10 @@ from typing import Tuple # Third-party imports import numpy as np import pandas as pd -import torch import pytest +import torch +from pts import transform # First-party imports from pts.dataset import ( ProcessStartField, @@ -29,7 +30,6 @@ from pts.dataset import ( calculate_dataset_statistics, ScaleHistogram, ) -from pts import transform from pts.feature import time_feature FREQ = "1D"