wassname/pytorch-transformer-ts
1
0 Fork 0
mirror of https://github.com/wassname/pytorch-transformer-ts.git synced 2026-10-04 13:00:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

added informer

Browse Source
This commit is contained in:
Kashif Rasul committed 2022-04-13 14:51:21 +02:00
1 parent 07834941c2
commit a8a046ab0d
5 files changed
+1651
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

informer/__init__.py
+9
View File
@@ -0,0 +1,9 @@
from .estimator import InformerEstimator
from .lightning_module import InformerLightningModule
from .module import InformerModel
__all__ = [
"InformerModel",
"InformerLightningModule",
"InformerEstimator",
]
informer/estimator.py
+319
View File
@@ -0,0 +1,319 @@
from typing import Any, Dict, Iterable, List, Optional
import torch
from gluonts.core.component import validated
from gluonts.dataset.common import Dataset
from gluonts.dataset.field_names import FieldName
from gluonts.itertools import Cyclic, IterableSlice, PseudoShuffled
from gluonts.time_feature import TimeFeature, time_features_from_frequency_str
from gluonts.torch.model.estimator import PyTorchLightningEstimator
from gluonts.torch.model.predictor import PyTorchPredictor
from gluonts.torch.modules.distribution_output import DistributionOutput, StudentTOutput
from gluonts.torch.modules.loss import DistributionLoss, NegativeLogLikelihood
from gluonts.torch.util import IterableDataset
from gluonts.transform import (
AddAgeFeature,
AddObservedValuesIndicator,
AddTimeFeatures,
AsNumpyArray,
Chain,
ExpectedNumInstanceSampler,
InstanceSplitter,
RemoveFields,
SelectFields,
SetField,
TestSplitSampler,
Transformation,
ValidationSplitSampler,
VstackFeatures,
)
from gluonts.transform.sampler import InstanceSampler
from torch.utils.data import DataLoader
from lightning_module import InformerLightningModule
from module import InformerModel
PREDICTION_INPUT_NAMES = [
"feat_static_cat",
"feat_static_real",
"past_time_feat",
"past_target",
"past_observed_values",
"future_time_feat",
]
TRAINING_INPUT_NAMES = PREDICTION_INPUT_NAMES + [
"future_target",
"future_observed_values",
]
class InformerEstimator(PyTorchLightningEstimator):
@validated()
def __init__(
self,
freq: str,
prediction_length: int,
# Informer arguments
nhead: int,
num_encoder_layers: int,
num_decoder_layers: int,
dim_feedforward: int,
input_size: int = 1,
activation: str = "gelu",
dropout: float = 0.1,
attn: str = "prob",
factor: int = 5,
distil: bool = True,
context_length: Optional[int] = None,
num_feat_dynamic_real: int = 0,
num_feat_static_cat: int = 0,
num_feat_static_real: int = 0,
cardinality: Optional[List[int]] = None,
embedding_dimension: Optional[List[int]] = None,
distr_output: DistributionOutput = StudentTOutput(),
loss: DistributionLoss = NegativeLogLikelihood(),
scaling: bool = True,
lags_seq: Optional[List[int]] = None,
time_features: Optional[List[TimeFeature]] = None,
num_parallel_samples: int = 100,
batch_size: int = 32,
num_batches_per_epoch: int = 50,
trainer_kwargs: Optional[Dict[str, Any]] = dict(),
train_sampler: Optional[InstanceSampler] = None,
validation_sampler: Optional[InstanceSampler] = None,
) -> None:
trainer_kwargs = {
"max_epochs": 100,
**trainer_kwargs,
}
super().__init__(trainer_kwargs=trainer_kwargs)
self.freq = freq
self.context_length = (
context_length if context_length is not None else prediction_length
)
self.prediction_length = prediction_length
self.distr_output = distr_output
self.loss = loss
self.input_size = input_size
self.nhead = nhead
self.num_encoder_layers = num_encoder_layers
self.num_decoder_layers = num_decoder_layers
self.activation = activation
self.dim_feedforward = dim_feedforward
self.dropout = dropout
self.attn = attn
self.factor = factor
self.distil = distil
self.num_feat_dynamic_real = num_feat_dynamic_real
self.num_feat_static_cat = num_feat_static_cat
self.num_feat_static_real = num_feat_static_real
self.cardinality = (
cardinality if cardinality and num_feat_static_cat > 0 else [1]
)
self.embedding_dimension = embedding_dimension
self.scaling = scaling
self.lags_seq = lags_seq
self.time_features = (
time_features
if time_features is not None
else time_features_from_frequency_str(self.freq)
)
self.num_parallel_samples = num_parallel_samples
self.batch_size = batch_size
self.num_batches_per_epoch = num_batches_per_epoch
self.train_sampler = train_sampler or ExpectedNumInstanceSampler(
num_instances=1.0, min_future=prediction_length
)
self.validation_sampler = validation_sampler or ValidationSplitSampler(
min_future=prediction_length
)
def create_transformation(self) -> Transformation:
remove_field_names = []
if self.num_feat_static_real == 0:
remove_field_names.append(FieldName.FEAT_STATIC_REAL)
if self.num_feat_dynamic_real == 0:
remove_field_names.append(FieldName.FEAT_DYNAMIC_REAL)
return Chain(
[RemoveFields(field_names=remove_field_names)]
+ (
[SetField(output_field=FieldName.FEAT_STATIC_CAT, value=[0])]
if not self.num_feat_static_cat > 0
else []
)
+ (
[SetField(output_field=FieldName.FEAT_STATIC_REAL, value=[0.0])]
if not self.num_feat_static_real > 0
else []
)
+ [
AsNumpyArray(
field=FieldName.FEAT_STATIC_CAT,
expected_ndim=1,
dtype=int,
),
AsNumpyArray(
field=FieldName.FEAT_STATIC_REAL,
expected_ndim=1,
),
AsNumpyArray(
field=FieldName.TARGET,
# in the following line, we add 1 for the time dimension
expected_ndim=1 + len(self.distr_output.event_shape),
),
AddObservedValuesIndicator(
target_field=FieldName.TARGET,
output_field=FieldName.OBSERVED_VALUES,
),
AddTimeFeatures(
start_field=FieldName.START,
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_TIME,
time_features=self.time_features,
pred_length=self.prediction_length,
),
AddAgeFeature(
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_AGE,
pred_length=self.prediction_length,
log_scale=True,
),
VstackFeatures(
output_field=FieldName.FEAT_TIME,
input_fields=[FieldName.FEAT_TIME, FieldName.FEAT_AGE]
+ (
[FieldName.FEAT_DYNAMIC_REAL]
if self.num_feat_dynamic_real > 0
else []
),
),
]
)
def _create_instance_splitter(self, module: InformerLightningModule, mode: str):
assert mode in ["training", "validation", "test"]
instance_sampler = {
"training": self.train_sampler,
"validation": self.validation_sampler,
"test": TestSplitSampler(),
}[mode]
return InstanceSplitter(
target_field=FieldName.TARGET,
is_pad_field=FieldName.IS_PAD,
start_field=FieldName.START,
forecast_start_field=FieldName.FORECAST_START,
instance_sampler=instance_sampler,
past_length=module.model._past_length,
future_length=self.prediction_length,
time_series_fields=[
FieldName.FEAT_TIME,
FieldName.OBSERVED_VALUES,
],
dummy_value=self.distr_output.value_in_support,
)
def create_training_data_loader(
self,
data: Dataset,
module: InformerLightningModule,
shuffle_buffer_length: Optional[int] = None,
**kwargs,
) -> Iterable:
transformation = self._create_instance_splitter(
module, "training"
) + SelectFields(TRAINING_INPUT_NAMES)
training_instances = transformation.apply(
Cyclic(data)
if shuffle_buffer_length is None
else PseudoShuffled(
Cyclic(data), shuffle_buffer_length=shuffle_buffer_length
)
)
return IterableSlice(
iter(
DataLoader(
IterableDataset(training_instances),
batch_size=self.batch_size,
**kwargs,
)
),
self.num_batches_per_epoch,
)
def create_validation_data_loader(
self,
data: Dataset,
module: InformerLightningModule,
**kwargs,
) -> Iterable:
transformation = self._create_instance_splitter(
module, "validation"
) + SelectFields(TRAINING_INPUT_NAMES)
validation_instances = transformation.apply(data)
return DataLoader(
IterableDataset(validation_instances),
batch_size=self.batch_size,
**kwargs,
)
def create_predictor(
self,
transformation: Transformation,
module: InformerLightningModule,
) -> PyTorchPredictor:
prediction_splitter = self._create_instance_splitter(module, "test")
return PyTorchPredictor(
input_transform=transformation + prediction_splitter,
input_names=PREDICTION_INPUT_NAMES,
prediction_net=module.model,
batch_size=self.batch_size,
freq=self.freq,
prediction_length=self.prediction_length,
device=torch.device("cuda" if torch.cuda.is_available() else "cpu"),
)
def create_lightning_module(self) -> InformerLightningModule:
model = InformerModel(
freq=self.freq,
context_length=self.context_length,
prediction_length=self.prediction_length,
num_feat_dynamic_real=1
+ self.num_feat_dynamic_real
+ len(self.time_features),
num_feat_static_real=max(1, self.num_feat_static_real),
num_feat_static_cat=max(1, self.num_feat_static_cat),
cardinality=self.cardinality,
embedding_dimension=self.embedding_dimension,
# Informer arguments
nhead=self.nhead,
num_encoder_layers=self.num_encoder_layers,
num_decoder_layers=self.num_decoder_layers,
activation=self.activation,
dropout=self.dropout,
dim_feedforward=self.dim_feedforward,
attn=self.attn,
factor=self.factor,
distil=self.distil,
# univariate input
input_size=self.input_size,
distr_output=self.distr_output,
lags_seq=self.lags_seq,
scaling=self.scaling,
num_parallel_samples=self.num_parallel_samples,
)
return InformerLightningModule(model=model, loss=self.loss)
informer/informer.ipynb
+522
View File
@@ -0,0 +1,522 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "420561b7",
"metadata": {},
"outputs": [],
"source": [
"%matplotlib inline\n",
"from matplotlib import pyplot as plt\n",
"import matplotlib.dates as mdates\n",
"\n",
"from itertools import islice"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "b10c3dd3",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/kashif/.env/pytorch/lib/python3.8/site-packages/xgboost/compat.py:36: FutureWarning: pandas.Int64Index is deprecated and will be removed from pandas in a future version. Use pandas.Index with the appropriate dtype instead.\n",
" from pandas import MultiIndex, Int64Index\n"
]
}
],
"source": [
"from gluonts.evaluation import make_evaluation_predictions, Evaluator\n",
"from gluonts.dataset.repository.datasets import get_dataset\n",
"\n",
"from estimator import InformerEstimator"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "d018b7fb",
"metadata": {},
"outputs": [],
"source": [
"dataset = get_dataset(\"electricity\")"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "e772234f",
"metadata": {},
"outputs": [],
"source": [
"estimator = InformerEstimator(\n",
" freq=dataset.metadata.freq,\n",
" prediction_length=dataset.metadata.prediction_length,\n",
" context_length=dataset.metadata.prediction_length*2,\n",
" \n",
" # \n",
" num_feat_static_cat=1,\n",
" cardinality=[321],\n",
" embedding_dimension=[3],\n",
" \n",
" # attention hyper-params\n",
" dim_feedforward=32,\n",
" num_encoder_layers=2,\n",
" num_decoder_layers=2,\n",
" nhead=2,\n",
" activation=\"relu\",\n",
" \n",
" # training params\n",
" batch_size=128,\n",
" num_batches_per_epoch=100,\n",
" trainer_kwargs=dict(max_epochs=50, accelerator='gpu', gpus=1),\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "22d804e4",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/kashif/.env/pytorch/lib/python3.8/site-packages/pytorch_lightning/utilities/parsing.py:244: UserWarning: Attribute 'model' is an instance of `nn.Module` and is already saved during checkpointing. It is recommended to ignore them using `self.save_hyperparameters(ignore=['model'])`.\n",
" rank_zero_warn(\n",
"/home/kashif/.env/pytorch/lib/python3.8/site-packages/pytorch_lightning/utilities/parsing.py:244: UserWarning: Attribute 'loss' is an instance of `nn.Module` and is already saved during checkpointing. It is recommended to ignore them using `self.save_hyperparameters(ignore=['loss'])`.\n",
" rank_zero_warn(\n",
"/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:324: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version.\n",
" timestamp = pd.Timestamp(timestamp_input, freq=freq)\n",
"/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:324: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version.\n",
" timestamp = pd.Timestamp(timestamp_input, freq=freq)\n",
"/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:327: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" if isinstance(timestamp.freq, Tick):\n",
"GPU available: True, used: True\n",
"/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:327: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" if isinstance(timestamp.freq, Tick):\n",
"TPU available: False, using: 0 TPU cores\n",
"/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:329: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" timestamp.floor(timestamp.freq), timestamp.freq\n",
"/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:329: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" timestamp.floor(timestamp.freq), timestamp.freq\n",
"IPU available: False, using: 0 IPUs\n",
"/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:324: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version.\n",
" timestamp = pd.Timestamp(timestamp_input, freq=freq)\n",
"/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:324: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version.\n",
" timestamp = pd.Timestamp(timestamp_input, freq=freq)\n",
"HPU available: False, using: 0 HPUs\n",
"/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:328: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version.\n",
" return pd.Timestamp(\n",
"/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:324: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version.\n",
" timestamp = pd.Timestamp(timestamp_input, freq=freq)\n",
"/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:324: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version.\n",
" timestamp = pd.Timestamp(timestamp_input, freq=freq)\n",
"/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:328: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version.\n",
" return pd.Timestamp(\n",
"/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:327: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" if isinstance(timestamp.freq, Tick):\n",
"/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:327: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" if isinstance(timestamp.freq, Tick):\n",
"/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:327: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" if isinstance(timestamp.freq, Tick):\n",
"/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:327: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" if isinstance(timestamp.freq, Tick):\n",
"/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:329: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" timestamp.floor(timestamp.freq), timestamp.freq\n",
"/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" self._freq_base = start.freq.base\n",
"/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:329: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" timestamp.floor(timestamp.freq), timestamp.freq\n",
"/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:329: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" timestamp.floor(timestamp.freq), timestamp.freq\n",
"/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:328: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version.\n",
" return pd.Timestamp(\n",
"/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:329: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" timestamp.floor(timestamp.freq), timestamp.freq\n",
"/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" self._freq_base = start.freq.base\n",
"/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:328: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version.\n",
" return pd.Timestamp(\n",
"/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" return _shift_timestamp_helper(ts, ts.freq, offset)\n",
"/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:328: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version.\n",
" return pd.Timestamp(\n",
"/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:324: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version.\n",
" timestamp = pd.Timestamp(timestamp_input, freq=freq)\n",
"/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:324: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version.\n",
" timestamp = pd.Timestamp(timestamp_input, freq=freq)\n",
"/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:328: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version.\n",
" return pd.Timestamp(\n",
"/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:327: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" if isinstance(timestamp.freq, Tick):\n",
"/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:327: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" if isinstance(timestamp.freq, Tick):\n",
"/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" self._freq_base = start.freq.base\n",
"/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" return _shift_timestamp_helper(ts, ts.freq, offset)\n",
"/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" self._freq_base = start.freq.base\n",
"/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" self._freq_base = start.freq.base\n",
"/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" return _shift_timestamp_helper(ts, ts.freq, offset)\n",
"/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:329: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" timestamp.floor(timestamp.freq), timestamp.freq\n",
"/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" return _shift_timestamp_helper(ts, ts.freq, offset)\n",
"/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" return _shift_timestamp_helper(ts, ts.freq, offset)\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" self._freq_base = start.freq.base\n",
"/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:329: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" timestamp.floor(timestamp.freq), timestamp.freq\n",
"/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:328: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version.\n",
" return pd.Timestamp(\n",
"/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" return _shift_timestamp_helper(ts, ts.freq, offset)\n",
"/home/kashif/gluon-ts-PR/src/gluonts/dataset/common.py:328: FutureWarning: The 'freq' argument in Timestamp is deprecated and will be removed in a future version.\n",
" return pd.Timestamp(\n",
"/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" self._freq_base = start.freq.base\n",
"/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" self._freq_base = start.freq.base\n",
"/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" ..., i0 : i0 + length * start.freq.n : start.freq.n\n",
"/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" ..., i0 : i0 + length * start.freq.n : start.freq.n\n",
"/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" ..., i0 : i0 + length * start.freq.n : start.freq.n\n",
"/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" return _shift_timestamp_helper(ts, ts.freq, offset)\n",
"/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" ..., i0 : i0 + length * start.freq.n : start.freq.n\n",
"/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" ..., i0 : i0 + length * start.freq.n : start.freq.n\n",
"/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" return _shift_timestamp_helper(ts, ts.freq, offset)\n",
"/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" ..., i0 : i0 + length * start.freq.n : start.freq.n\n",
"/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" self._freq_base is None or self._freq_base == start.freq.base\n",
"/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" self._freq_base is None or self._freq_base == start.freq.base\n",
"/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" self._freq_base is None or self._freq_base == start.freq.base\n",
"/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" self._freq_base is None or self._freq_base == start.freq.base\n",
"/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" self._freq_base is None or self._freq_base == start.freq.base\n",
"/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" self._freq_base is None or self._freq_base == start.freq.base\n",
"/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" ..., i0 : i0 + length * start.freq.n : start.freq.n\n",
"/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" ..., i0 : i0 + length * start.freq.n : start.freq.n\n",
"/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" self._freq_base is None or self._freq_base == start.freq.base\n",
"/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" self._freq_base is None or self._freq_base == start.freq.base\n",
"/home/kashif/.env/pytorch/lib/python3.8/site-packages/pytorch_lightning/trainer/configuration_validator.py:133: UserWarning: You defined a `validation_step` but have no `val_dataloader`. Skipping val loop.\n",
" rank_zero_warn(\"You defined a `validation_step` but have no `val_dataloader`. Skipping val loop.\")\n",
"LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
"\n",
" | Name | Type | Params\n",
"------------------------------------------------\n",
"0 | model | InformerModel | 84.3 K\n",
"1 | loss | NegativeLogLikelihood | 0 \n",
"------------------------------------------------\n",
"84.3 K Trainable params\n",
"0 Non-trainable params\n",
"84.3 K Total params\n",
"0.337 Total estimated model params size (MB)\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "f0fa05e7e28e4f46b71ae4fe592c57c5",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Training: 0it [00:00, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Epoch 0, global step 100: 'train_loss' reached 6.72319 (best 6.72319), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/informer/lightning_logs/version_21/checkpoints/epoch=0-step=100.ckpt' as top 1\n",
"Epoch 1, global step 200: 'train_loss' reached 6.08895 (best 6.08895), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/informer/lightning_logs/version_21/checkpoints/epoch=1-step=200.ckpt' as top 1\n",
"Epoch 2, global step 300: 'train_loss' reached 5.89698 (best 5.89698), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/informer/lightning_logs/version_21/checkpoints/epoch=2-step=300.ckpt' as top 1\n",
"Epoch 3, global step 400: 'train_loss' reached 5.73449 (best 5.73449), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/informer/lightning_logs/version_21/checkpoints/epoch=3-step=400.ckpt' as top 1\n",
"Epoch 4, global step 500: 'train_loss' reached 5.72003 (best 5.72003), saving model to '/mnt/scratch/kashif/pytorch-transformer-ts/informer/lightning_logs/version_21/checkpoints/epoch=4-step=500.ckpt' as top 1\n"
]
}
],
"source": [
"predictor = estimator.train(\n",
" training_data=dataset.train,\n",
" num_workers=8,\n",
" shuffle_buffer_length=1024\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "11a47d5a",
"metadata": {},
"outputs": [],
"source": [
"forecast_it, ts_it = make_evaluation_predictions(\n",
" dataset=dataset.test, \n",
" predictor=predictor\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "e4e94932",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:343: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" self._freq_base = start.freq.base\n",
"/home/kashif/gluon-ts-PR/src/gluonts/transform/split.py:36: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" return _shift_timestamp_helper(ts, ts.freq, offset)\n",
"/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:384: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" ..., i0 : i0 + length * start.freq.n : start.freq.n\n",
"/home/kashif/gluon-ts-PR/src/gluonts/transform/feature.py:340: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" self._freq_base is None or self._freq_base == start.freq.base\n"
]
}
],
"source": [
"forecasts = list(forecast_it)"
]
},
{
"cell_type": "code",
"execution_count": 17,
"id": "17c5e570",
"metadata": {},
"outputs": [],
"source": [
"tss = list(ts_it)"
]
},
{
"cell_type": "code",
"execution_count": 18,
"id": "9985be71",
"metadata": {},
"outputs": [],
"source": [
"evaluator = Evaluator()"
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "cca60b1e",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Running evaluation: 2247it [00:00, 4312.72it/s]/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" date_before_forecast = forecast.index[0] - forecast.index[0].freq\n",
"/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" date_before_forecast = forecast.index[0] - forecast.index[0].freq\n",
"/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" date_before_forecast = forecast.index[0] - forecast.index[0].freq\n",
"/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" date_before_forecast = forecast.index[0] - forecast.index[0].freq\n",
"/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" date_before_forecast = forecast.index[0] - forecast.index[0].freq\n",
"/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" date_before_forecast = forecast.index[0] - forecast.index[0].freq\n",
"/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" date_before_forecast = forecast.index[0] - forecast.index[0].freq\n",
"/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" date_before_forecast = forecast.index[0] - forecast.index[0].freq\n",
"/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" date_before_forecast = forecast.index[0] - forecast.index[0].freq\n",
"\n",
"/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" date_before_forecast = forecast.index[0] - forecast.index[0].freq\n",
"/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" date_before_forecast = forecast.index[0] - forecast.index[0].freq\n",
"/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" date_before_forecast = forecast.index[0] - forecast.index[0].freq\n",
"/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" date_before_forecast = forecast.index[0] - forecast.index[0].freq\n",
"/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" date_before_forecast = forecast.index[0] - forecast.index[0].freq\n",
"/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" date_before_forecast = forecast.index[0] - forecast.index[0].freq\n",
"/home/kashif/gluon-ts-PR/src/gluonts/evaluation/_base.py:306: FutureWarning: Timestamp.freq is deprecated and will be removed in a future version.\n",
" date_before_forecast = forecast.index[0] - forecast.index[0].freq\n",
"/home/kashif/.env/pytorch/lib/python3.8/site-packages/pandas/core/construction.py:781: UserWarning: Warning: converting a masked element to nan.\n",
" subarr = np.array(arr, dtype=dtype, copy=copy)\n"
]
}
],
"source": [
"agg_metrics, ts_metrics = evaluator(iter(tss), iter(forecasts))"
]
},
{
"cell_type": "code",
"execution_count": 20,
"id": "92389256",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'MSE': 1912270.1601813699,\n",
" 'abs_error': 9499909.78665924,\n",
" 'abs_target_sum': 128632956.0,\n",
" 'abs_target_mean': 2385.272140631954,\n",
" 'seasonal_error': 189.49338196116761,\n",
" 'MASE': 0.9230363743475488,\n",
" 'MAPE': 0.13762913075625383,\n",
" 'sMAPE': 0.1225657482189261,\n",
" 'MSIS': 6.837486088818375,\n",
" 'QuantileLoss[0.1]': 4183971.882330881,\n",
" 'Coverage[0.1]': 0.13139741878059635,\n",
" 'QuantileLoss[0.2]': 6507467.195659928,\n",
" 'Coverage[0.2]': 0.24473371903278449,\n",
" 'QuantileLoss[0.3]': 8081948.245087599,\n",
" 'Coverage[0.3]': 0.34848316273549923,\n",
" 'QuantileLoss[0.4]': 9055948.958718367,\n",
" 'Coverage[0.4]': 0.4462987687286753,\n",
" 'QuantileLoss[0.5]': 9499909.849783681,\n",
" 'Coverage[0.5]': 0.5364560154279779,\n",
" 'QuantileLoss[0.6]': 9417650.74509728,\n",
" 'Coverage[0.6]': 0.6173972704346535,\n",
" 'QuantileLoss[0.7]': 8747053.378532637,\n",
" 'Coverage[0.7]': 0.7036418928942294,\n",
" 'QuantileLoss[0.8]': 7436005.54045527,\n",
" 'Coverage[0.8]': 0.7901090342679128,\n",
" 'QuantileLoss[0.9]': 5067467.9464677805,\n",
" 'Coverage[0.9]': 0.8804146269099541,\n",
" 'RMSE': 1382.8485673353282,\n",
" 'NRMSE': 0.5797445682524747,\n",
" 'ND': 0.07385284519667915,\n",
" 'wQuantileLoss[0.1]': 0.03252643811070377,\n",
" 'wQuantileLoss[0.2]': 0.05058942434363343,\n",
" 'wQuantileLoss[0.3]': 0.06282953059935589,\n",
" 'wQuantileLoss[0.4]': 0.0704014681798836,\n",
" 'wQuantileLoss[0.5]': 0.07385284568741218,\n",
" 'wQuantileLoss[0.6]': 0.0732133586753404,\n",
" 'wQuantileLoss[0.7]': 0.06800009616923237,\n",
" 'wQuantileLoss[0.8]': 0.057807934853454424,\n",
" 'wQuantileLoss[0.9]': 0.03939478733947294,\n",
" 'mean_absolute_QuantileLoss': 7555269.304681491,\n",
" 'mean_wQuantileLoss': 0.05873509821760989,\n",
" 'MAE_Coverage': 0.028653842984061054,\n",
" 'OWA': nan}"
]
},
"execution_count": 20,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"agg_metrics"
]
},
{
"cell_type": "code",
"execution_count": 21,
"id": "23878611",
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAABY4AAAQlCAYAAAARPll0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAEAAElEQVR4nOzdd3xkeXnn+8+vcpBKWZ3z9Mz0DDADM0wPDsAADsAFdo1Z53A3sN41jru+xva1767Xfhlf2+sAXtvsXe+aNSY4gAETzASSh+me0JM7t7qVc+V06pzzu3+cqpJKqpJKUkXpeb9e/ZrpqlPSr9UtnTrPeX7fR2mtEUIIIYQQQgghhBBCCCFKXO1egBBCCCGEEEIIIYQQQojOIoVjIYQQQgghhBBCCCGEEBWkcCyEEEIIIYQQQgghhBCighSOhRBCCCGEEEIIIYQQQlSQwrEQQgghhBBCCCGEEEKIClI4FkIIIYQQQgghhBBCCFFBCsdCCCGEEEIIIYQQQgghKkjhWIg9QCnlUkr9nFLqklIqp5SaUEr9nlIq3O61CSGEELuJUuqXlFJ/rZS6oZTSSqmb7V6TEEIIsdsopW5XSv26UuoJpdSCUiqplHpWKfUrcp0rROMorXW71yCEaDKl1B8CPw18CvgCcAb4KeDrwFu01nYblyeEEELsGkopDSwDzwD3AQmt9fG2LkoIIYTYZZRSHwB+EvgM8ARQAB4C/gXwPPCg1jrbvhUKsTtI4ViIXU4pdTfwAvAprfW7Vz3+U8AfAT+ktf6rdq1PCCGE2E2UUie11jeK//8i0COFYyGEEKKxlFL3A1e11vE1j/8G8CvAT2mtP9SWxQmxi0hUhRC73w8ACviDNY//dyAD/HCrFySEEELsVqWisRBCCCGaR2v91NqicdEniv99RSvXI8RuJYVjIXa/1wI2cH71g1rrHPBs8XkhhBBCCCGEEKLbHS7+d66tqxBil5DCsRC730FgUWudr/LcFDCslPK1eE1CCCGEEEIIIUTDKKXcwK8CJiBxjEI0gBSOhdj9QkC1ojFAbtUxQgghhBBCCCFEt/oD4HXAr2mtL7d5LULsClI4FmL3ywD+Gs8FVh0jhBBCCCGEEEJ0HaXUfwHeB3xYa/1b7V6PELuFFI6F2P2mceIoqhWPD+HEWBgtXpMQQgghhBBCCLFjSqn/BPzfwP8EfqK9qxFid5HCsRC735M43+sPrH5QKRUA7gWeasOahBBCCCGEEEKIHSkWjf8f4C+Af6211u1dkRC7ixSOhdj9PgFo4GfXPP5vcLKNP9rqBQkhhBBCCCGEEDuhlPo1nKLx/wb+pdbabvOShNh1lNyMEWL3U0p9ECfv6VPA54EzwE8D/wS8SU6wQgghRGMopX4EOFb87U8BPuD3ir+/pbX+321ZmBBCCLGLKKV+EvgQMA78KrD2mnZOa/3lli9MiF1GCsdC7AFKKTdOx/F7gePAIk4n8q9prVPtW5kQQgixuyilvgK8ocbTX9Vav7F1qxFCCCF2J6XU/wJ+bIND5JwrRANI4VgIIYQQQgghhBBCCCFEBck4FkIIIYQQQgghhBBCCFFBCsdCCCGEEEIIIYQQQgghKkjhWAghhBBCCCGEEEIIIUQFKRwLIYQQQgghhBBCCCGEqOBp9wK2a3h4WB8/frzdyxBCCLGHPP3004ta65F2r6PV5JwrhBCi1fbiOVfOt0IIIdpho3Nu1xaOjx8/zlNPPdXuZQghhNhDlFK32r2GdpBzrhBCiFbbi+dcOd8KIYRoh43OuRJVIYQQQgghhBBCCCGEEKKCFI6FEEIIIYQQQgghhBBCVNhy4VgpdZtS6s+UUs8rpSyl1FeqHKOUUr+slJpQSmWVUl9TSt1b5bi7lFKPKKUySqlppdSvK6Xc2/ujCCGEEEIIIYQQQgghhGiE7XQc3w28DbgMXKlxzPuBXwV+G3gHkAIeVkrtLx2glBoAHgY08C7g14H/APznbaxJCCGEEEIIIYQQQgghRINsp3D8Wa31Ea31e4CX1j6plArgFI5/S2v9Ia31w8B7cArE71t16E8AQeB7tNZf1lr/KU7R+OeVUpFtrEsIIYQQQgghhBBCCCFEA2y5cKy1tjc55FuACPDJVa9JA58F3rrquLcCX9JaJ1Y99nGcYvIbtrouIYQQQgghhBCiWeqJbVxz/O8rpbRS6ndbtEQhhBCioZoxHO9OwAKurnn8YvG51cddWn2A1nocyKw5TgghhBBCCCGEaLd6YhsBZ54P8K+AxEbHCSGEEJ2sGYXjASCltbbWPB4FQkop36rjYlVeHy0+t45S6r1KqaeUUk8tLCw0ar1CCCGEEEIIIcRmNoxtXOODwB/iXN8KIYQQXakZheOm0Vp/WGt9v9b6/pGRkXYvRwghhBBCCCHEHlFHbCMASqnvxdlF+4HmrkgIIYRormYUjqNAj1LKvebxASCjtTZWHddX5fUDyF1ZIYToGJa9dgOJEEKI3eq3Pn+RH/3z8+1ehhBdSykVBH4PeH9x1o/YoZ/430/zK596od3LEEKIPakZheNLgBu4bc3jazONL7Emy1gpdQQIrTlOCCFEG00mJknmk+1ehhBCiBa4sZjmyqz8zBdiB34JmAH+sp6DJY5xc1fmk1ybT7V7GUIIsSc1o3D8OM4AgPeUHlBKhYB3AF9YddwXgO9SSvWueuz7gCzw1SasSwghxDakC2lenH+x3csQQgjRAoZpk8gV2r0MIbqSUuoE8B+Bn9Fa63peI3GMm0tkTTKG7IATQoh22HLhWCkVUkp9bzG36RAwUvq9Uiqktc7hZDn9slLqJ5VSbwb+uvi5PrjqQ/0pkAf+Tin1FqXUe4H/BPxXrbVMnhVCiA6RLWS5tHgJ0zbbvRQhhBBNZpg2GcPCtOqKchVCVPoAToPUZaVUv1KqH+c62F/8vWrr6rpUIlcgbcj7UCGEaAfPNl4zilMIXq30+xPATZwTpgtnm84Q8BTwHVrrudILtNbRYlH5Q8BngRjw+zjFYyGEEB0ia2bJW3muLl3lzMiZdc9fWrzEncN3VnmlEEKIbmMUC8bJnMlA2Nfm1QjRde4A7gG+Z83j7yv+OgJMtnpR3SxXsDBMm3ReCsdCCNEOWy4ca61vAhveKS1uy/nN4q+NjnsZeNNW1yCEEKJ1MoUMAC/Ov7iucDweH+f5ueelcCyEELuEYTqF40SuIIVjIbbuXwM9ax77OE4U458AEmK8RcmcUzDO5CWqQggh2mE7HcdCCCH2kGwhC8BSdomZ5AwHeg8AULAKfO3W17BseSMvhBC7RblwnJXuPiHWKs7ueVvxt4eASDHCEeDzWuunqrwmB0xorb/SmlXuLsli5nraMNFaI2kfQgjRWlI4FkIIsaFSxzE4XcelwvE3J79JykjhUs2YsyqEEKIdSlEVMiBPiKrqiW0UDZQodhzbGnIFm6DP3eYVCSHE3iKFYyGEEBvKmtny/4/FxkgbaWK5GC8vvAyArW0My8Dnli3NQgjR7VY6jqVw3A62tuWGbAerJ7axymuON2Uxe0Ry1U2sVN6UwrEQQrSYFI6FEELUZNkWhmWUf29rm+fnnmcsNlZxXM7MSeFYCCF2gbzpxA9Jx3F7LGeXGQ4Nt3sZQnSM1bE5GcME/O1bjBBC7EFyO1sIIURNq7uNS56be45EPlHxWM7MtWpJQgghmigvGcdtNZ+eb/cShOgoqzuO0zIgTwghWk4Kx0IIIWpanW+8ESkcCyHE7lCOqpCO47ZYSC+gtW73MoToGKt/FqUNuaElhBCtJoVjIYQQNWUL6zuOq5HCsRBCdD+t9cpwPMk4boucmVu3q0eIvSyZWykWp/NSOBZCiFaTwrEQQoiaqkVVVJM3801eiRBCiGYzbU2p2XV1sUa0Rjxb4M+/UmAittjupQjRMVbfxJKoCiFEJ7G13e4ltIQUjoUQQtQkURVCCLF3lGIqQKIq2uHZiRjnrimeGJtr91KE6BiJnInbpQCJqhBCdA7LtljM7I0bvVI4FkIIUZNEVQghxN5RUTiW4Xgtlys43ZQz8VSbVyJE50jmCuzr9QMSVSGE6ByxXGzPREtJ4VgIIURN0nEshBB7RynfGKTjuB3yxcL9XLK+c68QtXz2uWl+5H+cw7K7f9BiImuyvy8AQMaQqAohRGeI5WIk88l2L6MlpHAshBCipnozjqVwLIQQ3a/Ucex2KRmO1waljuOlVA6tu7/gJ9pnNp7j61cXyeyCaIdErsBQjx+vW5GSjmMhRIeI5qIkDSkcCyGE2OMkqkIIIfaOUsfrYNhHQobjtVzp65/Kqz2z/VU0R8jvBnbHMLlkziQS8BLyechI4VgI0SGi2ah0HAshhBASVSGEEHtHqeN4uMdPKm9iWntjWninyBc7jjN5N9FctM2rEd2sx+8BdscwuUSuQG/AQ9jnJi1RFUKIDiEZx0IIIfY8W9vkrXxdx9Z7nBBCiM5Vyjge7vEByLbwFssUnK93Nu9iObvc5tWIbhbyFQvHXf49bNuaVN4kEvQS9nu6/s8jhNgdtNbEcjFSxt4YZiuFYyGEEFXVG1MBYNompi1v5oUQopuVOo5HevyAM5RKtE7OcHKlM3k30ax0HIvtC++SqIqUYaI1RAIeQn6PdBwLITpC0khiaQtLW6SNdLuX03RSOBZCCFFVvTEVJRJXIYQQ3a1UOB4qdhwncjIgr5VKg8wyeZdEVYgdCRc7jrt9OF5pSGck4KXH75aOYyFER1h9c3cvDMiTwrEQQoiqsmb9HccghWMhhOh2huV08w2XOo6lcNxS2WJUhWG6WEhF0Vq3eUWiW5U6jrs9bqa066E34CHkk6gKIURnWH1zdy8MyJPCsRBCiKqk41gIIfaWfGFlOB5IVEWrlQrHAMkce2bojmi8sL/Ucdzd0Q7J4s2rSNBLj9+zK4b9CSG6XywXK/+/dBzvgFLq+5VSzyilUkqpKaXUR5RSB9cco5RSv6yUmlBKZZVSX1NK3dusNQkLine truncated
"text/plain": [
"<Figure size 1440x1080 with 9 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"plt.figure(figsize=(20, 15))\n",
"date_formater = mdates.DateFormatter('%b, %d')\n",
"plt.rcParams.update({'font.size': 15})\n",
"\n",
"for idx, (forecast, ts) in islice(enumerate(zip(forecasts, tss)), 9):\n",
" ax = plt.subplot(3, 3, idx+1)\n",
"\n",
" plt.plot(ts[-4 * dataset.metadata.prediction_length:], label=\"target\", )\n",
" forecast.plot( color='g')\n",
" plt.xticks(rotation=60)\n",
" plt.title(forecast.item_id)\n",
" ax.xaxis.set_major_formatter(date_formater)\n",
"\n",
"plt.gcf().tight_layout()\n",
"plt.legend()\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f34e7aa7",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.10"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
informer/lightning_module.py
+80
View File
@@ -0,0 +1,80 @@
import pytorch_lightning as pl
import torch
from gluonts.torch.modules.loss import DistributionLoss, NegativeLogLikelihood
from gluonts.torch.util import weighted_average
from module import InformerModel
class InformerLightningModule(pl.LightningModule):
def __init__(
self,
model: InformerModel,
loss: DistributionLoss = NegativeLogLikelihood(),
lr: float = 1e-3,
weight_decay: float = 1e-8,
) -> None:
super().__init__()
self.save_hyperparameters()
self.model = model
self.loss = loss
self.lr = lr
self.weight_decay = weight_decay
def training_step(self, batch, batch_idx: int):
"""Execute training step"""
train_loss = self(batch)
self.log(
"train_loss",
train_loss,
on_epoch=True,
on_step=False,
prog_bar=True,
)
return train_loss
def validation_step(self, batch, batch_idx: int):
"""Execute validation step"""
with torch.inference_mode():
val_loss = self(batch)
self.log("val_loss", val_loss, on_epoch=True, on_step=False, prog_bar=True)
return val_loss
def configure_optimizers(self):
"""Returns the optimizer to use"""
return torch.optim.Adam(
self.model.parameters(),
lr=self.lr,
weight_decay=self.weight_decay,
)
def forward(self, batch):
feat_static_cat = batch["feat_static_cat"]
feat_static_real = batch["feat_static_real"]
past_time_feat = batch["past_time_feat"]
past_target = batch["past_target"]
future_time_feat = batch["future_time_feat"]
future_target = batch["future_target"]
past_observed_values = batch["past_observed_values"]
future_observed_values = batch["future_observed_values"]
transformer_inputs, scale, _ = self.model.create_network_inputs(
feat_static_cat,
feat_static_real,
past_time_feat,
past_target,
past_observed_values,
future_time_feat,
future_target,
)
params = self.model.output_params(transformer_inputs)
distr = self.model.output_distribution(params, scale)
loss_values = self.loss(distr, future_target)
if len(self.model.target_shape) == 0:
loss_weights = future_observed_values
else:
loss_weights = future_observed_values.min(dim=-1, keepdim=False)
return weighted_average(loss_values, weights=loss_weights)
informer/module.py
+721
View File
@@ -0,0 +1,721 @@
from math import sqrt
from typing import List, Optional
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from gluonts.core.component import validated
from gluonts.time_feature import get_lags_for_frequency
from gluonts.torch.modules.distribution_output import DistributionOutput, StudentTOutput
from gluonts.torch.modules.feature import FeatureEmbedder
from gluonts.torch.modules.scaler import MeanScaler, NOPScaler
class TriangularCausalMask:
def __init__(self, B, L, device="cpu"):
mask_shape = [B, 1, L, L]
with torch.no_grad():
self._mask = torch.triu(
torch.ones(mask_shape, dtype=torch.bool), diagonal=1
).to(device)
@property
def mask(self):
return self._mask
class ProbMask:
def __init__(self, B, H, L, index, scores, device="cpu"):
_mask = torch.ones(L, scores.shape[-1], dtype=torch.bool).to(device).triu(1)
_mask_ex = _mask[None, None, :].expand(B, H, L, scores.shape[-1])
indicator = _mask_ex[
torch.arange(B)[:, None, None], torch.arange(H)[None, :, None], index, :
].to(device)
self._mask = indicator.view(scores.shape).to(device)
@property
def mask(self):
return self._mask
class FullAttention(nn.Module):
def __init__(
self,
mask_flag=True,
factor=5,
scale=None,
attention_dropout=0.1,
output_attention=False,
):
super(FullAttention, self).__init__()
self.scale = scale
self.mask_flag = mask_flag
self.output_attention = output_attention
self.dropout = nn.Dropout(attention_dropout)
def forward(self, queries, keys, values, attn_mask):
B, L, H, E = queries.shape
_, S, _, D = values.shape
scale = self.scale or 1.0 / sqrt(E)
scores = torch.einsum("blhe,bshe->bhls", queries, keys)
if self.mask_flag:
if attn_mask is None:
attn_mask = TriangularCausalMask(B, L, device=queries.device)
scores.masked_fill_(attn_mask.mask, -np.inf)
A = self.dropout(torch.softmax(scale * scores, dim=-1))
V = torch.einsum("bhls,bshd->blhd", A, values)
if self.output_attention:
return (V.contiguous(), A)
else:
return (V.contiguous(), None)
class ProbAttention(nn.Module):
def __init__(
self,
mask_flag=True,
factor=5,
scale=None,
attention_dropout=0.1,
output_attention=False,
):
super(ProbAttention, self).__init__()
self.factor = factor
self.scale = scale
self.mask_flag = mask_flag
self.output_attention = output_attention
self.dropout = nn.Dropout(attention_dropout)
def _prob_QK(self, Q, K, sample_k, n_top): # n_top: c*ln(L_q)
# Q [B, H, L, D]
B, H, L_K, E = K.shape
_, _, L_Q, _ = Q.shape
# calculate the sampled Q_K
K_expand = K.unsqueeze(-3).expand(B, H, L_Q, L_K, E)
index_sample = torch.randint(
L_K, (L_Q, sample_k)
) # real U = U_part(factor*ln(L_k))*L_q
K_sample = K_expand[:, :, torch.arange(L_Q).unsqueeze(1), index_sample, :]
Q_K_sample = torch.matmul(Q.unsqueeze(-2), K_sample.transpose(-2, -1)).squeeze(
-2
)
# find the Top_k query with sparisty measurement
M = Q_K_sample.max(-1)[0] - torch.div(Q_K_sample.sum(-1), L_K)
M_top = M.topk(n_top, sorted=False)[1]
# use the reduced Q to calculate Q_K
Q_reduce = Q[
torch.arange(B)[:, None, None], torch.arange(H)[None, :, None], M_top, :
] # factor*ln(L_q)
Q_K = torch.matmul(Q_reduce, K.transpose(-2, -1)) # factor*ln(L_q)*L_k
return Q_K, M_top
def _get_initial_context(self, V, L_Q):
B, H, L_V, D = V.shape
if not self.mask_flag:
# V_sum = V.sum(dim=-2)
V_sum = V.mean(dim=-2)
contex = V_sum.unsqueeze(-2).expand(B, H, L_Q, V_sum.shape[-1]).clone()
else: # use mask
assert L_Q == L_V # requires that L_Q == L_V, i.e. for self-attention only
contex = V.cumsum(dim=-2)
return contex
def _update_context(self, context_in, V, scores, index, L_Q, attn_mask):
B, H, L_V, D = V.shape
if self.mask_flag:
attn_mask = ProbMask(B, H, L_Q, index, scores, device=V.device)
scores.masked_fill_(attn_mask.mask, -np.inf)
attn = torch.softmax(scores, dim=-1) # nn.Softmax(dim=-1)(scores)
context_in[
torch.arange(B)[:, None, None], torch.arange(H)[None, :, None], index, :
] = torch.matmul(attn, V).type_as(context_in)
if self.output_attention:
attns = (torch.ones([B, H, L_V, L_V]) / L_V).type_as(attn).to(attn.device)
attns[
torch.arange(B)[:, None, None], torch.arange(H)[None, :, None], index, :
] = attn
return (context_in, attns)
else:
return (context_in, None)
def forward(self, queries, keys, values, attn_mask):
B, L_Q, H, D = queries.shape
_, L_K, _, _ = keys.shape
queries = queries.transpose(2, 1)
keys = keys.transpose(2, 1)
values = values.transpose(2, 1)
U_part = self.factor * np.ceil(np.log(L_K)).astype("int").item() # c*ln(L_k)
u = self.factor * np.ceil(np.log(L_Q)).astype("int").item() # c*ln(L_q)
U_part = U_part if U_part < L_K else L_K
u = u if u < L_Q else L_Q
scores_top, index = self._prob_QK(queries, keys, sample_k=U_part, n_top=u)
# add scale factor
scale = self.scale or 1.0 / sqrt(D)
if scale is not None:
scores_top = scores_top * scale
# get the context
context = self._get_initial_context(values, L_Q)
# update the context with selected top_k queries
context, attn = self._update_context(
context, values, scores_top, index, L_Q, attn_mask
)
return context.transpose(2, 1).contiguous(), attn
class AttentionLayer(nn.Module):
def __init__(
self, attention, d_model, n_heads, d_keys=None, d_values=None, mix=False
):
super(AttentionLayer, self).__init__()
d_keys = d_keys or (d_model // n_heads)
d_values = d_values or (d_model // n_heads)
self.inner_attention = attention
self.query_projection = nn.Linear(d_model, d_keys * n_heads)
self.key_projection = nn.Linear(d_model, d_keys * n_heads)
self.value_projection = nn.Linear(d_model, d_values * n_heads)
self.out_projection = nn.Linear(d_values * n_heads, d_model)
self.n_heads = n_heads
self.mix = mix
def forward(self, queries, keys, values, attn_mask):
B, L, _ = queries.shape
_, S, _ = keys.shape
H = self.n_heads
queries = self.query_projection(queries).view(B, L, H, -1)
keys = self.key_projection(keys).view(B, S, H, -1)
values = self.value_projection(values).view(B, S, H, -1)
out, attn = self.inner_attention(queries, keys, values, attn_mask)
if self.mix:
out = out.transpose(2, 1).contiguous()
out = out.view(B, L, -1)
return self.out_projection(out), attn
class ConvLayer(nn.Module):
def __init__(self, c_in):
super(ConvLayer, self).__init__()
self.downConv = nn.Conv1d(
in_channels=c_in,
out_channels=c_in,
kernel_size=3,
padding=1,
padding_mode="circular",
)
self.norm = nn.BatchNorm1d(c_in)
self.activation = nn.ELU()
self.maxPool = nn.MaxPool1d(kernel_size=3, stride=2, padding=1)
def forward(self, x):
x = self.downConv(x.permute(0, 2, 1))
x = self.norm(x)
x = self.activation(x)
x = self.maxPool(x)
x = x.transpose(1, 2)
return x
class EncoderLayer(nn.Module):
def __init__(self, attention, d_model, d_ff=None, dropout=0.1, activation="relu"):
super(EncoderLayer, self).__init__()
d_ff = d_ff or 4 * d_model
self.attention = attention
self.conv1 = nn.Conv1d(in_channels=d_model, out_channels=d_ff, kernel_size=1)
self.conv2 = nn.Conv1d(in_channels=d_ff, out_channels=d_model, kernel_size=1)
self.norm1 = nn.LayerNorm(d_model)
self.norm2 = nn.LayerNorm(d_model)
self.dropout = nn.Dropout(dropout)
self.activation = F.relu if activation == "relu" else F.gelu
def forward(self, x, attn_mask=None):
# x [B, L, D]
# x = x + self.dropout(self.attention(
# x, x, x,
# attn_mask = attn_mask
# ))
new_x, attn = self.attention(x, x, x, attn_mask=attn_mask)
x = x + self.dropout(new_x)
y = x = self.norm1(x)
y = self.dropout(self.activation(self.conv1(y.transpose(-1, 1))))
y = self.dropout(self.conv2(y).transpose(-1, 1))
return self.norm2(x + y), attn
class Encoder(nn.Module):
def __init__(self, attn_layers, conv_layers=None, norm_layer=None):
super(Encoder, self).__init__()
self.attn_layers = nn.ModuleList(attn_layers)
self.conv_layers = (
nn.ModuleList(conv_layers) if conv_layers is not None else None
)
self.norm = norm_layer
def forward(self, x, attn_mask=None):
# x [B, L, D]
attns = []
if self.conv_layers is not None:
for attn_layer, conv_layer in zip(self.attn_layers, self.conv_layers):
x, attn = attn_layer(x, attn_mask=attn_mask)
x = conv_layer(x)
attns.append(attn)
x, attn = self.attn_layers[-1](x, attn_mask=attn_mask)
attns.append(attn)
else:
for attn_layer in self.attn_layers:
x, attn = attn_layer(x, attn_mask=attn_mask)
attns.append(attn)
if self.norm is not None:
x = self.norm(x)
return x, attns
class DecoderLayer(nn.Module):
def __init__(
self,
self_attention,
cross_attention,
d_model,
d_ff=None,
dropout=0.1,
activation="relu",
):
super(DecoderLayer, self).__init__()
d_ff = d_ff or 4 * d_model
self.self_attention = self_attention
self.cross_attention = cross_attention
self.conv1 = nn.Conv1d(in_channels=d_model, out_channels=d_ff, kernel_size=1)
self.conv2 = nn.Conv1d(in_channels=d_ff, out_channels=d_model, kernel_size=1)
self.norm1 = nn.LayerNorm(d_model)
self.norm2 = nn.LayerNorm(d_model)
self.norm3 = nn.LayerNorm(d_model)
self.dropout = nn.Dropout(dropout)
self.activation = F.relu if activation == "relu" else F.gelu
def forward(self, x, cross, x_mask=None, cross_mask=None):
x = x + self.dropout(self.self_attention(x, x, x, attn_mask=x_mask)[0])
x = self.norm1(x)
x = x + self.dropout(
self.cross_attention(x, cross, cross, attn_mask=cross_mask)[0]
)
y = x = self.norm2(x)
y = self.dropout(self.activation(self.conv1(y.transpose(-1, 1))))
y = self.dropout(self.conv2(y).transpose(-1, 1))
return self.norm3(x + y)
class Decoder(nn.Module):
def __init__(self, layers, norm_layer=None):
super(Decoder, self).__init__()
self.layers = nn.ModuleList(layers)
self.norm = norm_layer
def forward(self, x, cross, x_mask=None, cross_mask=None):
for layer in self.layers:
x = layer(x, cross, x_mask=x_mask, cross_mask=cross_mask)
if self.norm is not None:
x = self.norm(x)
return x
class InformerModel(nn.Module):
@validated()
def __init__(
self,
freq: str,
context_length: int,
prediction_length: int,
num_feat_dynamic_real: int,
num_feat_static_real: int,
num_feat_static_cat: int,
cardinality: List[int],
# Informer arguments
nhead: int,
num_encoder_layers: int,
num_decoder_layers: int,
dim_feedforward: int,
activation: str = "gelu",
dropout: float = 0.1,
attn: str = "prob",
factor: int = 5,
distil: bool = True,
# univariate input
input_size: int = 1,
embedding_dimension: Optional[List[int]] = None,
distr_output: DistributionOutput = StudentTOutput(),
lags_seq: Optional[List[int]] = None,
scaling: bool = True,
num_parallel_samples: int = 100,
) -> None:
super().__init__()
self.input_size = input_size
self.target_shape = distr_output.event_shape
self.num_feat_dynamic_real = num_feat_dynamic_real
self.num_feat_static_cat = num_feat_static_cat
self.num_feat_static_real = num_feat_static_real
self.embedding_dimension = (
embedding_dimension
if embedding_dimension is not None or cardinality is None
else [min(50, (cat + 1) // 2) for cat in cardinality]
)
self.lags_seq = lags_seq or get_lags_for_frequency(freq_str=freq)
self.num_parallel_samples = num_parallel_samples
self.history_length = context_length + max(self.lags_seq)
self.embedder = FeatureEmbedder(
cardinalities=cardinality,
embedding_dims=self.embedding_dimension,
)
if scaling:
self.scaler = MeanScaler(dim=1, keepdim=True)
else:
self.scaler = NOPScaler(dim=1, keepdim=True)
# total feature size
d_model = self.input_size * len(self.lags_seq) + self._number_of_features
self.context_length = context_length
self.prediction_length = prediction_length
self.distr_output = distr_output
self.param_proj = distr_output.get_args_proj(d_model)
# Informer enc-decoder
Attn = ProbAttention if attn == "prob" else FullAttention
# Encoder
self.encoder = Encoder(
[
EncoderLayer(
AttentionLayer(
Attn(
mask_flag=False,
factor=factor,
attention_dropout=dropout,
output_attention=False,
),
d_model,
nhead,
mix=False,
),
d_model,
d_ff=dim_feedforward,
dropout=dropout,
activation=activation,
)
for l in range(num_encoder_layers)
],
[ConvLayer(d_model) for l in range(num_encoder_layers - 1)]
if distil
else None,
norm_layer=torch.nn.LayerNorm(d_model),
)
# Masked Decoder
self.decoder = Decoder(
[
DecoderLayer(
AttentionLayer(
Attn(
mask_flag=True,
factor=factor,
attention_dropout=dropout,
output_attention=False,
),
d_model,
nhead,
mix=True,
),
AttentionLayer(
FullAttention(
mask_flag=False,
factor=factor,
attention_dropout=dropout,
output_attention=False,
),
d_model,
nhead,
mix=False,
),
d_model,
d_ff=dim_feedforward,
dropout=dropout,
activation=activation,
)
for l in range(num_decoder_layers)
],
norm_layer=torch.nn.LayerNorm(d_model),
)
@property
def _number_of_features(self) -> int:
return (
sum(self.embedding_dimension)
+ self.num_feat_dynamic_real
+ self.num_feat_static_real
+ 1 # the log(scale)
)
@property
def _past_length(self) -> int:
return self.context_length + max(self.lags_seq)
def get_lagged_subsequences(
self, sequence: torch.Tensor, subsequences_length: int, shift: int = 0
) -> torch.Tensor:
"""
Returns lagged subsequences of a given sequence.
Parameters
----------
sequence : Tensor
the sequence from which lagged subsequences should be extracted.
Shape: (N, T, C).
subsequences_length : int
length of the subsequences to be extracted.
shift: int
shift the lags by this amount back.
Returns
--------
lagged : Tensor
a tensor of shape (N, S, C, I), where S = subsequences_length and
I = len(indices), containing lagged subsequences. Specifically,
lagged[i, j, :, k] = sequence[i, -indices[k]-S+j, :].
"""
sequence_length = sequence.shape[1]
indices = [lag - shift for lag in self.lags_seq]
assert max(indices) + subsequences_length <= sequence_length, (
f"lags cannot go further than history length, found lag {max(indices)} "
f"while history length is only {sequence_length}"
)
lagged_values = []
for lag_index in indices:
begin_index = -lag_index - subsequences_length
end_index = -lag_index if lag_index > 0 else None
lagged_values.append(sequence[:, begin_index:end_index, ...])
return torch.stack(lagged_values, dim=-1)
def _check_shapes(
self,
prior_input: torch.Tensor,
inputs: torch.Tensor,
features: Optional[torch.Tensor],
) -> None:
assert len(prior_input.shape) == len(inputs.shape)
assert (
len(prior_input.shape) == 2 and self.input_size == 1
) or prior_input.shape[2] == self.input_size
assert (len(inputs.shape) == 2 and self.input_size == 1) or inputs.shape[
-1
] == self.input_size
assert (
features is None or features.shape[2] == self._number_of_features
), f"{features.shape[2]}, expected {self._number_of_features}"
def create_network_inputs(
self,
feat_static_cat: torch.Tensor,
feat_static_real: torch.Tensor,
past_time_feat: torch.Tensor,
past_target: torch.Tensor,
past_observed_values: torch.Tensor,
future_time_feat: Optional[torch.Tensor] = None,
future_target: Optional[torch.Tensor] = None,
):
# time feature
time_feat = (
torch.cat(
(
past_time_feat[:, self._past_length - self.context_length :, ...],
future_time_feat,
),
dim=1,
)
if future_target is not None
else past_time_feat[:, self._past_length - self.context_length :, ...]
)
# target
context = past_target[:, -self.context_length :]
observed_context = past_observed_values[:, -self.context_length :]
_, scale = self.scaler(context, observed_context)
inputs = (
torch.cat((past_target, future_target), dim=1) / scale
if future_target is not None
else past_target / scale
)
inputs_length = (
self._past_length + self.prediction_length
if future_target is not None
else self._past_length
)
assert inputs.shape[1] == inputs_length
subsequences_length = (
self.context_length + self.prediction_length
if future_target is not None
else self.context_length
)
# embeddings
embedded_cat = self.embedder(feat_static_cat)
static_feat = torch.cat(
(embedded_cat, feat_static_real, scale.log()),
dim=1,
)
expanded_static_feat = static_feat.unsqueeze(1).expand(
-1, time_feat.shape[1], -1
)
features = torch.cat((expanded_static_feat, time_feat), dim=-1)
# self._check_shapes(prior_input, inputs, features)
# sequence = torch.cat((prior_input, inputs), dim=1)
lagged_sequence = self.get_lagged_subsequences(
sequence=inputs,
subsequences_length=subsequences_length,
)
lags_shape = lagged_sequence.shape
reshaped_lagged_sequence = lagged_sequence.reshape(
lags_shape[0], lags_shape[1], -1
)
transformer_inputs = torch.cat((reshaped_lagged_sequence, features), dim=-1)
return transformer_inputs, scale, static_feat
def output_params(self, transformer_inputs):
enc_input = transformer_inputs[:, : self.context_length, ...]
dec_input = transformer_inputs[:, self.context_length :, ...]
enc_out, _ = self.encoder(enc_input)
dec_output = self.decoder(dec_input, enc_out)
return self.param_proj(dec_output)
@torch.jit.ignore
def output_distribution(
self, params, scale=None, trailing_n=None
) -> torch.distributions.Distribution:
sliced_params = params
if trailing_n is not None:
sliced_params = [p[:, -trailing_n:] for p in params]
return self.distr_output.distribution(sliced_params, scale=scale)
# for prediction
def forward(
self,
feat_static_cat: torch.Tensor,
feat_static_real: torch.Tensor,
past_time_feat: torch.Tensor,
past_target: torch.Tensor,
past_observed_values: torch.Tensor,
future_time_feat: torch.Tensor,
num_parallel_samples: Optional[int] = None,
) -> torch.Tensor:
if num_parallel_samples is None:
num_parallel_samples = self.num_parallel_samples
encoder_inputs, scale, static_feat = self.create_network_inputs(
feat_static_cat,
feat_static_real,
past_time_feat,
past_target,
past_observed_values,
)
enc_out, _ = self.encoder(encoder_inputs)
repeated_scale = scale.repeat_interleave(
repeats=self.num_parallel_samples, dim=0
)
repeated_past_target = (
past_target.repeat_interleave(repeats=self.num_parallel_samples, dim=0)
/ repeated_scale
)
expanded_static_feat = static_feat.unsqueeze(1).expand(
-1, future_time_feat.shape[1], -1
)
features = torch.cat((expanded_static_feat, future_time_feat), dim=-1)
repeated_features = features.repeat_interleave(
repeats=self.num_parallel_samples, dim=0
)
repeated_enc_out = enc_out.repeat_interleave(
repeats=self.num_parallel_samples, dim=0
)
future_samples = []
# greedy decoding
for k in range(self.prediction_length):
# self._check_shapes(repeated_past_target, next_sample, next_features)
# sequence = torch.cat((repeated_past_target, next_sample), dim=1)
lagged_sequence = self.get_lagged_subsequences(
sequence=repeated_past_target,
subsequences_length=1 + k,
shift=1,
)
lags_shape = lagged_sequence.shape
reshaped_lagged_sequence = lagged_sequence.reshape(
lags_shape[0], lags_shape[1], -1
)
decoder_input = torch.cat(
(reshaped_lagged_sequence, repeated_features[:, : k + 1]), dim=-1
)
output = self.decoder(decoder_input, repeated_enc_out)
params = self.param_proj(output[:, -1:])
distr = self.output_distribution(params, scale=repeated_scale)
next_sample = distr.sample()
repeated_past_target = torch.cat(
(repeated_past_target, next_sample / repeated_scale), dim=1
)
future_samples.append(next_sample)
concat_future_samples = torch.cat(future_samples, dim=1)
return concat_future_samples.reshape(
(-1, self.num_parallel_samples, self.prediction_length) + self.target_shape,
)