pytorch-transformer-ts/fedformer/FEDformer.ipynb

In [1]:

import torch 

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print('Using device:', device)
print('CUDA version: ', torch.version.cuda)
print('Default current GPU used: ', torch.cuda.current_device())
print('Device count: ', torch.cuda.device_count())
for i in range(torch.cuda.device_count()):
    print('Device name:', torch.cuda.get_device_name(i))


if device.type == 'cuda':
    print('Allocated:', round(torch.cuda.memory_allocated(0)/1024**3,1), 'GB')
    print('Cached:   ', round(torch.cuda.memory_reserved(0)/1024**3,1), 'GB')

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html
  from .autonotebook import tqdm as notebook_tqdm

Using device: cuda
CUDA version:  10.2
Default current GPU used:  0
Device count:  1
Device name: Tesla V100-PCIE-16GB
Allocated: 0.0 GB
Cached:    0.0 GB

In [2]:

import logging

In [8]:

logging.basicConfig(filename='output.log',level = logging.INFO)

In [1]:

%matplotlib inline
from matplotlib import pyplot as plt
import matplotlib.dates as mdates

from itertools import islice

---------------------------------------------------------------------------
ImportError                               Traceback (most recent call last)
/tmp/ipykernel_268/789959619.py in <module>
----> 1 get_ipython().run_line_magic('matplotlib', 'inline')
      2 from matplotlib import pyplot as plt
      3 import matplotlib.dates as mdates
      4 
      5 from itertools import islice

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/IPython/core/interactiveshell.py in run_line_magic(self, magic_name, line, _stack_depth)
   2362                 kwargs['local_ns'] = self.get_local_scope(stack_depth)
   2363             with self.builtin_trap:
-> 2364                 result = fn(*args, **kwargs)
   2365             return result
   2366 

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/decorator.py in fun(*args, **kw)
    230             if not kwsyntax:
    231                 args, kw = fix(args, kw, sig)
--> 232             return caller(func, *(extras + args), **kw)
    233     fun.__name__ = func.__name__
    234     fun.__doc__ = func.__doc__

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/IPython/core/magic.py in <lambda>(f, *a, **k)
    185     # but it's overkill for just that one bit of state.
    186     def magic_deco(arg):
--> 187         call = lambda f, *a, **k: f(*a, **k)
    188 
    189         if callable(arg):

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/IPython/core/magics/pylab.py in matplotlib(self, line)
     97             print("Available matplotlib backends: %s" % backends_list)
     98         else:
---> 99             gui, backend = self.shell.enable_matplotlib(args.gui.lower() if isinstance(args.gui, str) else args.gui)
    100             self._show_matplotlib_backend(args.gui, backend)
    101 

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/IPython/core/interactiveshell.py in enable_matplotlib(self, gui)
   3531         """
   3532         from IPython.core import pylabtools as pt
-> 3533         from matplotlib_inline.backend_inline import configure_inline_support
   3534         gui, backend = pt.find_gui_and_backend(gui, self.pylab_gui_select)
   3535 

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/matplotlib_inline/__init__.py in <module>
----> 1 from . import backend_inline, config  # noqa
      2 __version__ = "0.1.6"  # noqa

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/matplotlib_inline/backend_inline.py in <module>
      4 # Distributed under the terms of the BSD 3-Clause License.
      5 
----> 6 import matplotlib
      7 from matplotlib import colors
      8 from matplotlib.backends import backend_agg

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/matplotlib/__init__.py in <module>
    107 # cbook must import matplotlib only within function
    108 # definitions, so it is safe to import from it here.
--> 109 from . import _api, _version, cbook, docstring, rcsetup
    110 from matplotlib.cbook import MatplotlibDeprecationWarning, sanitize_sequence
    111 from matplotlib.cbook import mplDeprecation  # deprecated

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/matplotlib/rcsetup.py in <module>
     25 from matplotlib import _api, cbook
     26 from matplotlib.cbook import ls_mapper
---> 27 from matplotlib.colors import Colormap, is_color_like
     28 from matplotlib.fontconfig_pattern import parse_fontconfig_pattern
     29 from matplotlib._enums import JoinStyle, CapStyle

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/matplotlib/colors.py in <module>
     49 from numbers import Number
     50 import re
---> 51 from PIL import Image
     52 from PIL.PngImagePlugin import PngInfo
     53 

~/.local/lib/python3.7/site-packages/PIL/Image.py in <module>
     98     # Also note that Image.core is not a publicly documented interface,
     99     # and should be considered private and subject to change.
--> 100     from . import _imaging as core
    101 
    102     if __version__ != getattr(core, "PILLOW_VERSION", None):

ImportError: cannot import name '_imaging' from 'PIL' (/ccs/home/hstellar/.local/lib/python3.7/site-packages/PIL/__init__.py)

In [1]:

from gluonts.evaluation import make_evaluation_predictions, Evaluator
from gluonts.dataset.repository.datasets import get_dataset

from estimator import FEDformerEstimator

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/gluonts/json.py:102: UserWarning: Using `json`-module for json-handling. Consider installing one of `orjson`, `ujson` to speed up serialization and deserialization.
  "Using `json`-module for json-handling. "
/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html
  from .autonotebook import tqdm as notebook_tqdm

In [2]:

dataset = get_dataset("electricity")

In [17]:

dataset

Out [17]:

TrainDatasets(metadata=MetaData(freq='1H', target=None, feat_static_cat=[CategoricalFeatureInfo(name='feat_static_cat_0', cardinality='321')], feat_static_real=[], feat_dynamic_real=[], feat_dynamic_cat=[], prediction_length=24), train=DatasetCollection(datasets=[Map(data=JsonLinesFile(path=PosixPath('/ccs/home/hstellar/.mxnet/gluon-ts/datasets/electricity/train/data.json.gz')))], interleave=False), test=DatasetCollection(datasets=[Map(data=JsonLinesFile(path=PosixPath('/ccs/home/hstellar/.mxnet/gluon-ts/datasets/electricity/test/data.json.gz')))], interleave=False))

In [5]:

estimator = FEDformerEstimator(
    freq='h',
    prediction_length=dataset.metadata.prediction_length,
    context_length=dataset.metadata.prediction_length*7,
    dim_feedforward=16,
    num_feat_static_cat=1,
    cardinality=[321],
    embedding_dimension=[3],
    # attention hyper-params
    num_encoder_layers=2,
    num_decoder_layers=1,
    nhead=2,
    activation="relu",
    moving_avg=[24],
    # training params
    batch_size=128,
    num_batches_per_epoch=50,
    trainer_kwargs=dict(max_epochs=1, accelerator='gpu', gpus=1),
)

In [6]:

predictor = estimator.train(
    training_data=dataset.train,
    num_workers=8,
    # shuffle_buffer_length=1024
)

fourier enhanced block used!
modes=64, index=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 30, 31, 32, 34, 36, 37, 40, 41, 44, 45, 46, 48, 49, 50, 52, 53, 54, 55, 56, 58, 59, 62, 63, 64, 65, 67, 68, 69, 70, 71, 73, 74, 76, 79, 80, 83]
fourier enhanced block used!
modes=64, index=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53]
 fourier enhanced cross attention used!
dim_feedforward 16
enc_modes: 64, dec_modes: 54
encoder_self_att FourierBlock()

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

  | Name  | Type           | Params
-----------------------------------------
0 | model | FEDformerModel | 263 K 
-----------------------------------------
263 K     Trainable params
0         Non-trainable params
263 K     Total params
1.053     Total estimated model params size (MB)

Epoch 0: : 0it [00:00, ?it/s]transformer_inputs torch.Size([128, 192, 50])
enc_input torch.Size([128, 168, 50])
torch.Size([128, 168, 50])
query proj torch.Size([128, 168, 50])
x_ft size torch.Size([128, 2, 25, 85])
weight size torch.Size([2, 25, 25, 64])
index [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 30, 31, 32, 34, 36, 37, 40, 41, 44, 45, 46, 48, 49, 50, 52, 53, 54, 55, 56, 58, 59, 62, 63, 64, 65, 67, 68, 69, 70, 71, 73, 74, 76, 79, 80, 83]
torch.Size([128, 168, 50])
query proj torch.Size([128, 168, 50])
x_ft size torch.Size([128, 2, 25, 85])
weight size torch.Size([2, 25, 25, 64])
index [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 30, 31, 32, 34, 36, 37, 40, 41, 44, 45, 46, 48, 49, 50, 52, 53, 54, 55, 56, 58, 59, 62, 63, 64, 65, 67, 68, 69, 70, 71, 73, 74, 76, 79, 80, 83]
torch.Size([128, 24, 50])
query proj torch.Size([128, 24, 50])
x_ft size torch.Size([128, 2, 25, 13])
weight size torch.Size([2, 25, 25, 54])
index [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53]

---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
/tmp/ipykernel_5824/608781419.py in <module>
      1 predictor = estimator.train(
      2     training_data=dataset.train,
----> 3     num_workers=8,
      4     # shuffle_buffer_length=1024
      5 )

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/gluonts/torch/model/estimator.py in train(self, training_data, validation_data, num_workers, shuffle_buffer_length, cache_data, ckpt_path, **kwargs)
    235             cache_data=cache_data,
    236             ckpt_path=ckpt_path,
--> 237             **kwargs,
    238         ).predictor

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/gluonts/torch/model/estimator.py in train_model(self, training_data, validation_data, num_workers, shuffle_buffer_length, cache_data, ckpt_path, **kwargs)
    199             train_dataloaders=training_data_loader,
    200             val_dataloaders=validation_data_loader,
--> 201             ckpt_path=ckpt_path,
    202         )
    203 

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py in fit(self, model, train_dataloaders, val_dataloaders, datamodule, ckpt_path)
    695         self.strategy.model = model
    696         self._call_and_handle_interrupt(
--> 697             self._fit_impl, model, train_dataloaders, val_dataloaders, datamodule, ckpt_path
    698         )
    699 

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py in _call_and_handle_interrupt(self, trainer_fn, *args, **kwargs)
    648                 return self.strategy.launcher.launch(trainer_fn, *args, trainer=self, **kwargs)
    649             else:
--> 650                 return trainer_fn(*args, **kwargs)
    651         # TODO(awaelchli): Unify both exceptions below, where `KeyboardError` doesn't re-raise
    652         except KeyboardInterrupt as exception:

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py in _fit_impl(self, model, train_dataloaders, val_dataloaders, datamodule, ckpt_path)
    733             ckpt_path, model_provided=True, model_connected=self.lightning_module is not None
    734         )
--> 735         results = self._run(model, ckpt_path=self.ckpt_path)
    736 
    737         assert self.state.stopped

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py in _run(self, model, ckpt_path)
   1164         self._checkpoint_connector.resume_end()
   1165 
-> 1166         results = self._run_stage()
   1167 
   1168         log.detail(f"{self.__class__.__name__}: trainer tearing down")

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py in _run_stage(self)
   1250         if self.predicting:
   1251             return self._run_predict()
-> 1252         return self._run_train()
   1253 
   1254     def _pre_training_routine(self):

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py in _run_train(self)
   1281 
   1282         with torch.autograd.set_detect_anomaly(self._detect_anomaly):
-> 1283             self.fit_loop.run()
   1284 
   1285     def _run_evaluate(self) -> _EVALUATE_OUTPUT:

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/pytorch_lightning/loops/loop.py in run(self, *args, **kwargs)
    198             try:
    199                 self.on_advance_start(*args, **kwargs)
--> 200                 self.advance(*args, **kwargs)
    201                 self.on_advance_end()
    202                 self._restarting = False

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/pytorch_lightning/loops/fit_loop.py in advance(self)
    269         )
    270         with self.trainer.profiler.profile("run_training_epoch"):
--> 271             self._outputs = self.epoch_loop.run(self._data_fetcher)
    272 
    273     def on_advance_end(self) -> None:

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/pytorch_lightning/loops/loop.py in run(self, *args, **kwargs)
    198             try:
    199                 self.on_advance_start(*args, **kwargs)
--> 200                 self.advance(*args, **kwargs)
    201                 self.on_advance_end()
    202                 self._restarting = False

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/pytorch_lightning/loops/epoch/training_epoch_loop.py in advance(self, data_fetcher)
    201 
    202             with self.trainer.profiler.profile("run_training_batch"):
--> 203                 batch_output = self.batch_loop.run(kwargs)
    204 
    205         self.batch_progress.increment_processed()

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/pytorch_lightning/loops/loop.py in run(self, *args, **kwargs)
    198             try:
    199                 self.on_advance_start(*args, **kwargs)
--> 200                 self.advance(*args, **kwargs)
    201                 self.on_advance_end()
    202                 self._restarting = False

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/pytorch_lightning/loops/batch/training_batch_loop.py in advance(self, kwargs)
     85                 self.trainer.optimizers, self.trainer.optimizer_frequencies, kwargs.get("batch_idx", 0)
     86             )
---> 87             outputs = self.optimizer_loop.run(optimizers, kwargs)
     88         else:
     89             outputs = self.manual_loop.run(kwargs)

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/pytorch_lightning/loops/loop.py in run(self, *args, **kwargs)
    198             try:
    199                 self.on_advance_start(*args, **kwargs)
--> 200                 self.advance(*args, **kwargs)
    201                 self.on_advance_end()
    202                 self._restarting = False

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py in advance(self, optimizers, kwargs)
    199         kwargs = self._build_kwargs(kwargs, self.optimizer_idx, self._hiddens)
    200 
--> 201         result = self._run_optimization(kwargs, self._optimizers[self.optim_progress.optimizer_position])
    202         if result.loss is not None:
    203             # automatic optimization assumes a loss needs to be returned for extras to be considered as the batch

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py in _run_optimization(self, kwargs, optimizer)
    246         else:
    247             # the `batch_idx` is optional with inter-batch parallelism
--> 248             self._optimizer_step(optimizer, opt_idx, kwargs.get("batch_idx", 0), closure)
    249 
    250         result = closure.consume_result()

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py in _optimizer_step(self, optimizer, opt_idx, batch_idx, train_step_and_backward_closure)
    365             on_tpu=isinstance(self.trainer.accelerator, TPUAccelerator),
    366             using_native_amp=(self.trainer.amp_backend == AMPType.NATIVE),
--> 367             using_lbfgs=is_lbfgs,
    368         )
    369 

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py in _call_lightning_module_hook(self, hook_name, pl_module, *args, **kwargs)
   1548 
   1549         with self.profiler.profile(f"[LightningModule]{pl_module.__class__.__name__}.{hook_name}"):
-> 1550             output = fn(*args, **kwargs)
   1551 
   1552         # restore current_fx when nested context

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/pytorch_lightning/core/module.py in optimizer_step(self, epoch, batch_idx, optimizer, optimizer_idx, optimizer_closure, on_tpu, using_native_amp, using_lbfgs)
   1703 
   1704         """
-> 1705         optimizer.step(closure=optimizer_closure)
   1706 
   1707     def optimizer_zero_grad(self, epoch: int, batch_idx: int, optimizer: Optimizer, optimizer_idx: int):

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/pytorch_lightning/core/optimizer.py in step(self, closure, **kwargs)
    166 
    167         assert self._strategy is not None
--> 168         step_output = self._strategy.optimizer_step(self._optimizer, self._optimizer_idx, closure, **kwargs)
    169 
    170         self._on_after_step()

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/pytorch_lightning/strategies/strategy.py in optimizer_step(self, optimizer, opt_idx, closure, model, **kwargs)
    214         """
    215         model = model or self.lightning_module
--> 216         return self.precision_plugin.optimizer_step(model, optimizer, opt_idx, closure, **kwargs)
    217 
    218     def _setup_model_and_optimizers(self, model: Module, optimizers: List[Optimizer]) -> Tuple[Module, List[Optimizer]]:

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py in optimizer_step(self, model, optimizer, optimizer_idx, closure, **kwargs)
    151         if isinstance(model, pl.LightningModule):
    152             closure = partial(self._wrap_closure, model, optimizer, optimizer_idx, closure)
--> 153         return optimizer.step(closure=closure, **kwargs)
    154 
    155     def _track_grad_norm(self, trainer: "pl.Trainer") -> None:

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/torch/optim/optimizer.py in wrapper(*args, **kwargs)
    111                 profile_name = "Optimizer.step#{}.step".format(obj.__class__.__name__)
    112                 with torch.autograd.profiler.record_function(profile_name):
--> 113                     return func(*args, **kwargs)
    114             return wrapper
    115 

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/torch/autograd/grad_mode.py in decorate_context(*args, **kwargs)
     25         def decorate_context(*args, **kwargs):
     26             with self.clone():
---> 27                 return func(*args, **kwargs)
     28         return cast(F, decorate_context)
     29 

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/torch/optim/adam.py in step(self, closure)
    116         if closure is not None:
    117             with torch.enable_grad():
--> 118                 loss = closure()
    119 
    120         for group in self.param_groups:

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py in _wrap_closure(self, model, optimizer, optimizer_idx, closure)
    136         consistent with the ``PrecisionPlugin`` subclasses that cannot pass ``optimizer.step(closure)`` directly.
    137         """
--> 138         closure_result = closure()
    139         self._after_closure(model, optimizer, optimizer_idx)
    140         return closure_result

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py in __call__(self, *args, **kwargs)
    144 
    145     def __call__(self, *args: Any, **kwargs: Any) -> Optional[Tensor]:
--> 146         self._result = self.closure(*args, **kwargs)
    147         return self._result.loss
    148 

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py in closure(self, *args, **kwargs)
    130 
    131     def closure(self, *args: Any, **kwargs: Any) -> ClosureResult:
--> 132         step_output = self._step_fn()
    133 
    134         if step_output.closure_loss is None:

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py in _training_step(self, kwargs)
    405         """
    406         # manually capture logged metrics
--> 407         training_step_output = self.trainer._call_strategy_hook("training_step", *kwargs.values())
    408         self.trainer.strategy.post_training_step()
    409 

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py in _call_strategy_hook(self, hook_name, *args, **kwargs)
   1702 
   1703         with self.profiler.profile(f"[Strategy]{self.strategy.__class__.__name__}.{hook_name}"):
-> 1704             output = fn(*args, **kwargs)
   1705 
   1706         # restore current_fx when nested context

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/pytorch_lightning/strategies/strategy.py in training_step(self, *args, **kwargs)
    356         with self.precision_plugin.train_step_context():
    357             assert isinstance(self.model, TrainingStep)
--> 358             return self.model.training_step(*args, **kwargs)
    359 
    360     def post_training_step(self) -> None:

/autofs/nccs-svm1_home1/hstellar/pytorch-transformer-ts/fedformer/lightning_module.py in training_step(self, batch, batch_idx)
     24     def training_step(self, batch, batch_idx: int):
     25         """Execute training step"""
---> 26         train_loss = self(batch)
     27         self.log(
     28             "train_loss",

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
   1128         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1129                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1130             return forward_call(*input, **kwargs)
   1131         # Do not call functions when jit is used
   1132         full_backward_hooks, non_full_backward_hooks = [], []

/autofs/nccs-svm1_home1/hstellar/pytorch-transformer-ts/fedformer/lightning_module.py in forward(self, batch)
     69         )
     70         print('transformer_inputs', transformer_inputs.size())
---> 71         params = self.model.output_params(transformer_inputs)
     72         distr = self.model.output_distribution(params, scale)
     73 

/autofs/nccs-svm1_home1/hstellar/pytorch-transformer-ts/fedformer/module.py in output_params(self, transformer_inputs)
   1969         print('enc_input',enc_input.shape)
   1970         enc_out, _ = self.encoder(enc_input)
-> 1971         dec_output = self.decoder(dec_input, enc_out)
   1972 
   1973         return self.param_proj(dec_output)

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
   1128         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1129                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1130             return forward_call(*input, **kwargs)
   1131         # Do not call functions when jit is used
   1132         full_backward_hooks, non_full_backward_hooks = [], []

/autofs/nccs-svm1_home1/hstellar/pytorch-transformer-ts/fedformer/module.py in forward(self, x, cross, x_mask, cross_mask, trend)
    846     def forward(self, x, cross, x_mask=None, cross_mask=None, trend=None):
    847         for layer in self.layers:
--> 848             x, residual_trend = layer(x, cross, x_mask=x_mask, cross_mask=cross_mask)
    849             trend = trend + residual_trend
    850 

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
   1128         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1129                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1130             return forward_call(*input, **kwargs)
   1131         # Do not call functions when jit is used
   1132         full_backward_hooks, non_full_backward_hooks = [], []

/autofs/nccs-svm1_home1/hstellar/pytorch-transformer-ts/fedformer/module.py in forward(self, x, cross, x_mask, cross_mask)
    813 
    814     def forward(self, x, cross, x_mask=None, cross_mask=None):
--> 815         x = x + self.dropout(self.self_attention(x, x, x, attn_mask=x_mask)[0])
    816 
    817         x, trend1 = self.decomp1(x)

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
   1128         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1129                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1130             return forward_call(*input, **kwargs)
   1131         # Do not call functions when jit is used
   1132         full_backward_hooks, non_full_backward_hooks = [], []

/autofs/nccs-svm1_home1/hstellar/pytorch-transformer-ts/fedformer/module.py in forward(self, queries, keys, values, attn_mask)
    596         values = self.value_projection(values).view(B, S, H, -1)
    597 
--> 598         out, attn = self.inner_correlation(queries, keys, values, attn_mask)
    599 
    600         out = out.view(B, L, -1)

/ccs/proj/csc499/hstellar/rapids/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
   1128         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1129                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1130             return forward_call(*input, **kwargs)
   1131         # Do not call functions when jit is used
   1132         full_backward_hooks, non_full_backward_hooks = [], []

/autofs/nccs-svm1_home1/hstellar/pytorch-transformer-ts/fedformer/module.py in forward(self, q, k, v, mask)
   1095         for wi, i in enumerate(self.index):
   1096             out_ft[:, :, :, wi] = self.compl_mul1d(
-> 1097                 x_ft[:, :, :, i], self.weights1[:, :, :, wi]
   1098             )
   1099         # Return to time domain

IndexError: index 13 is out of bounds for dimension 3 with size 13

In [ ]:

forecast_it, ts_it = make_evaluation_predictions(
    dataset=dataset.test, 
    predictor=predictor
)

In [ ]:

forecasts = list(forecast_it)

In [ ]:

tss = list(ts_it)

In [ ]:

evaluator = Evaluator()

In [ ]:

agg_metrics, ts_metrics = evaluator(iter(tss), iter(forecasts))

In [ ]:

agg_metrics

In [ ]:

plt.figure(figsize=(20, 15))
date_formater = mdates.DateFormatter('%b, %d')
plt.rcParams.update({'font.size': 15})

for idx, (forecast, ts) in islice(enumerate(zip(forecasts, tss)), 9):
    ax = plt.subplot(3, 3, idx+1)

    plt.plot(ts[-4 * dataset.metadata.prediction_length:], label="target", )
    forecast.plot( color='g')
    plt.xticks(rotation=60)
    plt.title(forecast.item_id)
    ax.xaxis.set_major_formatter(date_formater)

plt.gcf().tight_layout()
plt.legend()
plt.show()

In [ ]: