wip

data/datasets.py

@@ -131,6 +131,20 @@ class ForecastDataset(Dataset):
         if self.cross_learn:
             return self.n_time_samples * self.n_dims
         return self.n_time_samples
+    
+    def get_inds(self, idx):
+        cx_start = idx
+        cx_end = cx_start + self.lookback_len
+        c_start = cx_end + self.gap
+        c_end = c_start + self.horizon_len
+        
+        q_start = c_end + self.gap
+        q_end = q_start + self.horizon_len
+        
+        qx_end = q_start
+        qx_start = qx_end - self.lookback_len
+        
+        return cx_start, cx_end, c_start, c_end, qx_start, qx_end, q_start, q_end
 
     def __getitem__(self, idx: int) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
         if self.cross_learn:
@@ -139,16 +153,8 @@ class ForecastDataset(Dataset):
             idx = idx % self.n_time_samples
         else:
             dim_slice = slice(None)
-            
-        cx_start = idx
-        cx_end = cx_start + self.lookback_len
-        c_start = cx_end + self.gap
-        c_end = c_start + self.horizon_len
         
-        qx_start = cx_end + self.gap
-        qx_end = qx_start + self.lookback_len
-        q_start = qx_end + self.gap
-        q_end = q_start + self.horizon_len
+        cx_start, cx_end, c_start, c_end, qx_start, qx_end, q_start, q_end = self.get_inds(idx)
 
         context_past_x = self.data_x[cx_start:cx_end, dim_slice]
         context_y = self.data_y[c_start:c_end, dim_slice]

experiments/configs/ECL/192M.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'ECL/192M'
-build.module = 'experiments.forecast'
-build.repeat = 1
-build.variables_dict = {
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'electricity/electricity.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True
-ForecastDataset.features = 'M'
-ForecastDataset.horizon_len = 192
-ForecastDataset.lookback_mult = 5

experiments/configs/ECL/336M.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'ECL/336M'
-build.module = 'experiments.forecast'
-build.repeat = 1
-build.variables_dict = {
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'electricity/electricity.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True
-ForecastDataset.features = 'M'
-ForecastDataset.horizon_len = 336
-ForecastDataset.lookback_mult = 3

experiments/configs/ECL/720M.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'ECL/720M'
-build.module = 'experiments.forecast'
-build.repeat = 1
-build.variables_dict = {
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'electricity/electricity.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True
-ForecastDataset.features = 'M'
-ForecastDataset.horizon_len = 720
-ForecastDataset.lookback_mult = 1

experiments/configs/ECL/96M.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'ECL/96M'
-build.module = 'experiments.forecast'
-build.repeat = 1
-build.variables_dict = {
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'electricity/electricity.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True
-ForecastDataset.features = 'M'
-ForecastDataset.horizon_len = 96
-ForecastDataset.lookback_mult = 9

experiments/configs/ETTm2/192M.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'ETTm2/192M'
-build.module = 'experiments.forecast'
-build.repeat = 1
-build.variables_dict = {
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'ETT-small/ETTm2.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True
-ForecastDataset.features = 'M'
-ForecastDataset.horizon_len = 192
-ForecastDataset.lookback_mult = 5

experiments/configs/ETTm2/192S.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'ETTm2/192S'
-build.module = 'experiments.forecast'
-build.repeat = 1
-build.variables_dict = {
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'ETT-small/ETTm2.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True
-ForecastDataset.features = 'M'
-ForecastDataset.horizon_len = 192
-ForecastDataset.lookback_mult = 3

experiments/configs/ETTm2/336M.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'ETTm2/336M'
-build.module = 'experiments.forecast'
-build.repeat = 1
-build.variables_dict = {
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'ETT-small/ETTm2.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True
-ForecastDataset.features = 'M'
-ForecastDataset.horizon_len = 336
-ForecastDataset.lookback_mult = 3

experiments/configs/ETTm2/336S.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'ETTm2/336S'
-build.module = 'experiments.forecast'
-build.repeat = 1
-build.variables_dict = {
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'ETT-small/ETTm2.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True
-ForecastDataset.features = 'S'
-ForecastDataset.horizon_len = 336
-ForecastDataset.lookback_mult = 7

experiments/configs/ETTm2/720M.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'ETTm2/720M'
-build.module = 'experiments.forecast'
-build.repeat = 1
-build.variables_dict = {
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'ETT-small/ETTm2.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True
-ForecastDataset.features = 'M'
-ForecastDataset.horizon_len = 720
-ForecastDataset.lookback_mult = 1

experiments/configs/ETTm2/720S.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'ETTm2/720S'
-build.module = 'experiments.forecast'
-build.repeat = 1
-build.variables_dict = {
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'ETT-small/ETTm2.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True
-ForecastDataset.features = 'S'
-ForecastDataset.horizon_len = 720
-ForecastDataset.lookback_mult = 1

experiments/configs/ETTm2/96M.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'ETTm2/96M'
-build.module = 'experiments.forecast'
-build.repeat = 1
-build.variables_dict = {
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'ETT-small/ETTm2.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True
-ForecastDataset.features = 'M'
-ForecastDataset.horizon_len = 96
-ForecastDataset.lookback_mult = 7

experiments/configs/ETTm2/96S.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'ETTm2/96S'
-build.module = 'experiments.forecast'
-build.repeat = 1
-build.variables_dict = {
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'ETT-small/ETTm2.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True
-ForecastDataset.features = 'S'
-ForecastDataset.horizon_len = 96
-ForecastDataset.lookback_mult = 5

experiments/configs/Exchange/192M.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'Exchange/192M'
-build.module = 'experiments.forecast'
-build.repeat = 1
-build.variables_dict = {
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'exchange_rate/exchange_rate.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True
-ForecastDataset.features = 'M'
-ForecastDataset.horizon_len = 192
-ForecastDataset.lookback_mult = 5

experiments/configs/Exchange/192S.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'Exchange/192S'
-build.module = 'experiments.forecast'
-build.repeat = 1
-build.variables_dict = {
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'exchange_rate/exchange_rate.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True
-ForecastDataset.features = 'S'
-ForecastDataset.horizon_len = 192
-ForecastDataset.lookback_mult = 1

experiments/configs/Exchange/336M.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'Exchange/336M'
-build.module = 'experiments.forecast'
-build.repeat = 1
-build.variables_dict = {
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'exchange_rate/exchange_rate.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True
-ForecastDataset.features = 'M'
-ForecastDataset.horizon_len = 336
-ForecastDataset.lookback_mult = 7

experiments/configs/Exchange/336S.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'Exchange/336S'
-build.module = 'experiments.forecast'
-build.repeat = 1
-build.variables_dict = {
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'exchange_rate/exchange_rate.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True
-ForecastDataset.features = 'S'
-ForecastDataset.horizon_len = 336
-ForecastDataset.lookback_mult = 7

experiments/configs/Exchange/720M.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'Exchange/720M'
-build.module = 'experiments.forecast'
-build.repeat = 1
-build.variables_dict = {
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'exchange_rate/exchange_rate.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True
-ForecastDataset.features = 'M'
-ForecastDataset.horizon_len = 720
-ForecastDataset.lookback_mult = 3

experiments/configs/Exchange/720S.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'Exchange/720S'
-build.module = 'experiments.forecast'
-build.repeat = 1
-build.variables_dict = {
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'exchange_rate/exchange_rate.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True
-ForecastDataset.features = 'S'
-ForecastDataset.horizon_len = 720
-ForecastDataset.lookback_mult = 3

experiments/configs/Exchange/96M.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'Exchange/96M'
-build.module = 'experiments.forecast'
-build.repeat = 1
-build.variables_dict = {
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'exchange_rate/exchange_rate.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True
-ForecastDataset.features = 'M'
-ForecastDataset.horizon_len = 96
-ForecastDataset.lookback_mult = 1

experiments/configs/Exchange/96S.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'Exchange/96S'
-build.module = 'experiments.forecast'
-build.repeat = 1
-build.variables_dict = {
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'exchange_rate/exchange_rate.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True
-ForecastDataset.features = 'S'
-ForecastDataset.horizon_len = 96
-ForecastDataset.lookback_mult = 3

experiments/configs/ILI/24M.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'ILI/96M'
-build.module = 'experiments.forecast'
-build.repeat = 1
-build.variables_dict = {
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'illness/national_illness.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True
-ForecastDataset.features = 'M'
-ForecastDataset.horizon_len = 24
-ForecastDataset.lookback_mult = 9

experiments/configs/ILI/36M.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'ILI/192M'
-build.module = 'experiments.forecast'
-build.repeat = 1
-build.variables_dict = {
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'illness/national_illness.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True
-ForecastDataset.features = 'M'
-ForecastDataset.horizon_len = 36
-ForecastDataset.lookback_mult = 7

experiments/configs/ILI/48M.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'ILI/336M'
-build.module = 'experiments.forecast'
-build.repeat = 1
-build.variables_dict = {
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'illness/national_illness.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True
-ForecastDataset.features = 'M'
-ForecastDataset.horizon_len = 48
-ForecastDataset.lookback_mult = 5

experiments/configs/ILI/60M.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'ILI/720M'
-build.module = 'experiments.forecast'
-build.repeat = 1
-build.variables_dict = {
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'illness/national_illness.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True
-ForecastDataset.features = 'M'
-ForecastDataset.horizon_len = 60
-ForecastDataset.lookback_mult = 3

experiments/configs/Stocks/96M2S.gin

@@ -23,15 +23,17 @@ Checkpoint.patience = 7
 
 deeptime3.layer_size = 32
 deeptime3.inr_layers = 5
-deeptime3.n_fourier_feats = 4096
+deeptime3.dropout = 0.3
+deeptime3.base_learner = 'Ridge'
+deeptime3.n_fourier_feats = 2048
 deeptime3.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
 
 ForecastDataset.data_path = 'stocks/OXY_2019.csv.gz'
 ForecastDataset.target = 'RSMKs_18_144_72'
 ForecastDataset.scale = True
 ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
+ForecastDataset.time_features = 'h'
 ForecastDataset.normalise_time_features = True
 ForecastDataset.features = 'M2S'
-ForecastDataset.horizon_len = 96
-ForecastDataset.lookback_mult = 1
+ForecastDataset.horizon_len = 46
+ForecastDataset.lookback_mult = 2

experiments/configs/Traffic/192M.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'Traffic/192M'
-build.module = 'experiments.forecast'
-build.repeat = 1
-build.variables_dict = {
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'traffic/traffic.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True
-ForecastDataset.features = 'M'
-ForecastDataset.horizon_len = 192
-ForecastDataset.lookback_mult = 5

experiments/configs/Traffic/336M.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'Traffic/336M'
-build.module = 'experiments.forecast'
-build.repeat = 1
-build.variables_dict = {
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'traffic/traffic.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True
-ForecastDataset.features = 'M'
-ForecastDataset.horizon_len = 336
-ForecastDataset.lookback_mult = 5

experiments/configs/Traffic/720M.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'Traffic/720M'
-build.module = 'experiments.forecast'
-build.repeat = 1
-build.variables_dict = {
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'traffic/traffic.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True
-ForecastDataset.features = 'M'
-ForecastDataset.horizon_len = 720
-ForecastDataset.lookback_mult = 3

experiments/configs/Traffic/96M.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'Traffic/96M'
-build.module = 'experiments.forecast'
-build.repeat = 1
-build.variables_dict = {
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'traffic/traffic.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True
-ForecastDataset.features = 'M'
-ForecastDataset.horizon_len = 96
-ForecastDataset.lookback_mult = 9

experiments/configs/Weather/192M.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'Weather/192M'
-build.module = 'experiments.forecast'
-build.repeat = 1
-build.variables_dict = {
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'weather/weather.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True
-ForecastDataset.features = 'M'
-ForecastDataset.horizon_len = 192
-ForecastDataset.lookback_mult = 7

experiments/configs/Weather/336M.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'Weather/336M'
-build.module = 'experiments.forecast'
-build.repeat = 1
-build.variables_dict = {
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'weather/weather.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True
-ForecastDataset.features = 'M'
-ForecastDataset.horizon_len = 192
-ForecastDataset.lookback_mult = 3

experiments/configs/Weather/720M.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'Weather/720M'
-build.module = 'experiments.forecast'
-build.repeat = 1
-build.variables_dict = {
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'weather/weather.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True
-ForecastDataset.features = 'M'
-ForecastDataset.horizon_len = 720
-ForecastDataset.lookback_mult = 5

experiments/configs/Weather/96M.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'Weather/96M'
-build.module = 'experiments.forecast'
-build.repeat = 1
-build.variables_dict = {
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'weather/weather.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True
-ForecastDataset.features = 'M'
-ForecastDataset.horizon_len = 96
-ForecastDataset.lookback_mult = 9

experiments/configs/hp_search/ECL.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'hp_search/ECL'
-build.module = 'experiments.forecast'
-build.repeat = 3
-build.variables_dict = {
-    'ForecastDataset.lookback_mult': [1, 3, 5, 7, 9],
-    'ForecastDataset.horizon_len': [96, 192, 336, 720],
-    'ForecastDataset.features': ['M'],
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'electricity/electricity.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True

experiments/configs/hp_search/ETTm2.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'hp_search/ETTm2'
-build.module = 'experiments.forecast'
-build.repeat = 3
-build.variables_dict = {
-    'ForecastDataset.lookback_mult': [1, 3, 5, 7, 9],
-    'ForecastDataset.horizon_len': [96, 192, 336, 720],
-    'ForecastDataset.features': ['M', 'S'],
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'ETT-small/ETTm2.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True

experiments/configs/hp_search/Exchange.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'hp_search/Exchange'
-build.module = 'experiments.forecast'
-build.repeat = 3
-build.variables_dict = {
-    'ForecastDataset.lookback_mult': [1, 3, 5, 7, 9],
-    'ForecastDataset.horizon_len': [96, 192, 336, 720],
-    'ForecastDataset.features': ['M', 'S'],
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'exchange_rate/exchange_rate.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True

experiments/configs/hp_search/ILI.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'hp_search/ILI'
-build.module = 'experiments.forecast'
-build.repeat = 3
-build.variables_dict = {
-    'ForecastDataset.lookback_mult': [1, 3, 5, 7, 9],
-    'ForecastDataset.horizon_len': [24, 36, 48, 60],
-    'ForecastDataset.features': ['M'],
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'illness/national_illness.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True

experiments/configs/hp_search/Stocks.gin

@@ -0,0 +1,47 @@
+build.experiment_name = 'Stocks/96M2S'
+build.module = 'experiments.forecast'
+build.repeat = 1
+build.variables_dict = {
+    # 'ForecastDataset.lookback_mult': [1, 3, 5, 7, 9],
+    # 'ForecastDataset.horizon_len': [6, 12, 24, 48, 96, 192, 336, 720],
+    # 'ForecastDataset.features': ['m', 'h', 'd'],
+    'deeptime3.base_learner': ['Ridge', 'None', 'Transformer'],
+    'deeptime3.inr': ['INR', 'INRPlus2'],
+    'deeptime3.encoder': ['inception', ],
+    # 'deeptime3.dropout': [0.0, 0.1, 0.3, 0.5,],
+}
+
+instance.model_type = 'deeptime3'
+instance.save_vals = False
+
+get_optimizer.lr = 1e-3
+get_optimizer.lambda_lr = 1.
+get_optimizer.weight_decay = 0.
+
+get_scheduler.warmup_epochs = 5
+
+get_data.batch_size = 256
+
+train.loss_name = 'mse'
+train.epochs = 50
+train.clip = 10.
+
+Checkpoint.patience = 7
+
+deeptime3.layer_size = 32
+deeptime3.inr_layers = 5
+deeptime3.dropout = 0.3
+deeptime3.base_learner = 'Ridge'
+deeptime3.n_fourier_feats = 2048
+deeptime3.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
+
+ForecastDataset.data_path = 'stocks/OXY_2019.csv.gz'
+ForecastDataset.target = 'RSMKs_18_144_72'
+ForecastDataset.scale = True
+ForecastDataset.cross_learn = False
+ForecastDataset.time_features = []
+# ForecastDataset.time_features = 'h'
+ForecastDataset.normalise_time_features = True
+ForecastDataset.features = 'M2S'
+ForecastDataset.horizon_len = 46
+ForecastDataset.lookback_mult = 2

experiments/configs/hp_search/Traffic.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'hp_search/Traffic'
-build.module = 'experiments.forecast'
-build.repeat = 3
-build.variables_dict = {
-    'ForecastDataset.lookback_mult': [1, 3, 5, 7, 9],
-    'ForecastDataset.horizon_len': [96, 192, 336, 720],
-    'ForecastDataset.features': ['M'],
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'traffic/traffic.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True

experiments/configs/hp_search/Weather.gin

@@ -1,37 +0,0 @@
-build.experiment_name = 'hp_search/Weather'
-build.module = 'experiments.forecast'
-build.repeat = 3
-build.variables_dict = {
-    'ForecastDataset.lookback_mult': [1, 3, 5, 7, 9],
-    'ForecastDataset.horizon_len': [96, 192, 336, 720],
-    'ForecastDataset.features': ['M'],
-}
-
-instance.model_type = 'deeptime'
-instance.save_vals = False
-
-get_optimizer.lr = 1e-3
-get_optimizer.lambda_lr = 1.
-get_optimizer.weight_decay = 0.
-
-get_scheduler.warmup_epochs = 5
-
-get_data.batch_size = 256
-
-train.loss_name = 'mse'
-train.epochs = 50
-train.clip = 10.
-
-Checkpoint.patience = 7
-
-deeptime.layer_size = 256
-deeptime.inr_layers = 5
-deeptime.n_fourier_feats = 4096
-deeptime.scales = [0.01, 0.1, 1, 5, 10, 20, 50, 100]
-
-ForecastDataset.data_path = 'weather/weather.csv'
-ForecastDataset.target = 'OT'
-ForecastDataset.scale = True
-ForecastDataset.cross_learn = False
-ForecastDataset.time_features = []
-ForecastDataset.normalise_time_features = True

mjc_notes.md

@@ -60,6 +60,20 @@ My mods (I added past other variables):
 Where INR is one of [mlp, lstm, lstm2, transformer, transforme2, inceptioncausal]
 
 TODO:
-- M2S mode
-- add other INR's
-- add None as learner
+
+- [x] try just one predictor
+- [x] compare multi
+- losses:
+    - try logp? nah
+    - mae?
+- [x] make my own csv with 5m data (maybe 10k rows)
+- [ ] backtest?
+
+- [x] M2S mode
+- [ ] add other INR's
+- [ ] add None as learner
+
+```
+python -m experiments.forecast --config_path=experiments/configs/hp_search/Stocks.gin build_experiment
+./run.sh
+```

models/DeepTIMe3.py

@@ -11,40 +11,54 @@ import torch.nn as nn
 from torch import Tensor
 from einops import rearrange, repeat, reduce
 
-from models.modules.causalinception import CausalInceptionTimePlus, CausalConv1d
+from models.modules.metareghead import RegressionHead
+from models.modules.causalinception import CausalInceptionTimePlus
 from models.modules.inrplus2 import INRPlus2
-from models.modules.regressors import RidgeRegressor
+from models.modules.inr import INR
+from models.modules.encoders import LSTMEncoder, TransformerEncoder2, TransformerEncoder, InceptionEncoder, LSTMEncoder2, MLPEncoder
 
+# from models.modules.regressors import RidgeRegressor
 
 @gin.configurable()
-def deeptime3(dim_size:int, datetime_feats: int, layer_size: int, inr_layers: int, n_fourier_feats: int, scales: float):
-    return DeepTIMe3(dim_size, datetime_feats, layer_size, inr_layers, n_fourier_feats, scales)
+def deeptime3(dim_size:int, datetime_feats: int, layer_size: int, inr_layers: int, n_fourier_feats: int, scales: float, dropout: float, base_learner: str, encoder:str, inr: str):
+    return DeepTIMe3(dim_size, datetime_feats, layer_size, inr_layers, n_fourier_feats, scales, dropout, base_learner, encoder, inr)
 
 
 class DeepTIMe3(nn.Module):
-    def __init__(self, dim_size: int, datetime_feats: int, layer_size: int, inr_layers: int, n_fourier_feats: int, scales: float, dropout: float=0.3):
+    def __init__(self, dim_size: int, datetime_feats: int, layer_size: int, inr_layers: int, n_fourier_feats: int, scales: float, dropout: float=0.3, base_learner:str='Ridge', encoder:str='inception', inr:str='INR'):
         super().__init__()
         
         # encode the past
-        encoded_size = layer_size//2
-        self.encoder = CausalInceptionTimePlus(
-            c_in=dim_size, c_out=encoded_size, 
-            # nf=32, depth=6,
-            nf=17, depth=3,            
-            bn=True,
-            dilation=6,
-            ks=[39, 19, 3],
-            coord=True, fc_dropout=dropout,
-        )
+        if encoder == 'inception':
+            encoded_size = layer_size
+            self.encoder = CausalInceptionTimePlus(
+                c_in=dim_size, c_out=encoded_size, 
+                # nf=24, depth=4,
+                nf=17, depth=3,    
+                bn=True,
+                dilation=2,
+                ks=[39, 19, 3],
+                coord=True, fc_dropout=dropout,
+            )
+        elif encoder == 'lstm':
+            self.encoder = LSTMEncoder()
+        else:
+            raise NotADirectoryError(encoder)
         
         # translate coords to a representation, given a summary of the past
         coord_size = 1
         in_feats=datetime_feats+encoded_size+coord_size
-        self.inr = INRPlus2(in_feats=in_feats, layers=inr_layers, layer_size=layer_size,
+        if inr=='INRPlus2':
+            self.inr = INRPlus2(in_feats=in_feats, layers=inr_layers, layer_size=layer_size,
                        n_fourier_feats=n_fourier_feats, scales=scales, dropout=dropout)
+        elif inr=="INR":
+            self.inr = INR(in_feats=in_feats, layers=inr_layers, layer_size=layer_size,
+                       n_fourier_feats=n_fourier_feats, scales=scales, dropout=dropout)
+        else:
+            raise NotImplementedError(inr)
         
         # meta learn y given a representation
-        self.adaptive_weights = RidgeRegressor()
+        self.regressionhead = RegressionHead(base_learner=base_learner, d=layer_size, dropout=dropout)
 
         self.datetime_feats = datetime_feats
         self.inr_layers = inr_layers
@@ -78,8 +92,7 @@ class DeepTIMe3(nn.Module):
         context_reprs = self.encode_and_decode(context_past_x, context_time)
         query_reprs = self.encode_and_decode(query_past_x, query_time, offset=context_reprs.shape[1])
 
-        w, b = self.adaptive_weights(context_reprs, context_y)
-        preds = self.forecast(query_reprs, w, b)
+        preds = self.regressionhead(query_reprs, context_reprs, context_y)
         return preds
 
     def forecast(self, inp: Tensor, w: Tensor, b: Tensor) -> Tensor:

models/modules/encoders.py

@@ -0,0 +1,272 @@
+
+# import gin
+import torch
+import torch.nn as nn
+from torch import Tensor
+from einops import rearrange, repeat, reduce
+
+from tsai.models.InceptionTimePlus import (
+    Conv,
+    noop,
+    nn,
+    LinBnDrop,
+    GAP1d,
+    torch,
+    AddCoords1d, BatchNorm
+)
+from tsai.models.TSTPlus import TSTPlus
+from tsai.models.TSPerceiver import TSPerceiver
+from tsai.models.TSSequencerPlus import TSSequencerPlus
+from torch.nn.utils import weight_norm, spectral_norm
+from .causalinception import CausalInceptionTimePlus
+from .inr import INR
+
+def custom_head(head_nf, c_out, seq_len):
+    return nn.Sequential(
+        # AddCoords1d(),
+        # Conv(head_nf+1, head_nf, 2, bias=True, norm='Spectral'),
+        # nn.BatchNorm1d(head_nf),
+        # # nn.Dropout(0.15),
+        # nn.ReLU(),
+        AddCoords1d(),
+        Conv(head_nf + 1, c_out, 1, bias=False, norm="Spectral"),
+    )
+    
+class LinBnDropSN(nn.Sequential):
+    "Module grouping `BatchNorm1d`, `Dropout` and `Linear` layers"
+    def __init__(self, n_in, n_out, bn=True, p=0., act=None, lin_first=False, norm=None):
+        layers = [BatchNorm(n_out if lin_first else n_in, ndim=1)] if bn else []
+        if p != 0: layers.append(nn.Dropout(p))
+        lin = [spectral_norm(nn.Linear(n_in, n_out, bias=not bn))]
+        if act is not None: lin.append(act)
+        layers = lin+layers if lin_first else layers+lin
+        super().__init__(*layers)
+
+
+class InceptionEncoder(nn.Module):
+    def __init__(self, c_in, c_out, *args, **kwargs):
+        super().__init__()
+        self.net = CausalInceptionTimePlus(
+            c_in=c_in, c_out=c_out, custom_head=custom_head, *args, **kwargs
+        )
+        bn = kwargs.get("bn", True)
+        fc_dropout = kwargs.get("fc_dropout", 0.15)
+        self.pool = nn.Sequential(
+            # GACP1d(1),
+            # LinBnDrop(c_out*2, c_out, bn=bn, p=dropout)
+            GAP1d(1),
+            LinBnDropSN(c_out, c_out, bn=bn, p=fc_dropout),
+        )
+        self.head = nn.Sequential(
+            # just to make sure we get a spectral norm final layer (after cat)
+            LinBnDropSN(c_out*2, c_out*2, bn=bn, p=fc_dropout),
+        )
+
+    def forward(self, x):
+        """
+        Takes in a sequence of shape (batch, sequence, features)
+        and outputs a representation of shape (batch, features)
+        """
+        outs = self.net(x.permute(0, 2, 1))  # .permute(0, 2, 1)
+        last = outs[:, :, -1]  # take last
+        max = self.pool(outs)
+        return self.head(torch.cat([max, last], 1))
+
+
+class TransformerEncoder(nn.Module):
+    def __init__(
+        self,
+        c_in,
+        c_out,
+        seq_len,
+        layers=3,
+        layer_size=512,
+        dropout=0.1,
+        n_heads=4,
+        conv_dropout=0,
+        *args,
+        **kwargs,
+    ):
+        super().__init__()
+        # d_model (82) must be divisible by n_heads (4)
+        layer_size = layer_size // n_heads * n_heads
+        d_model = layer_size // 2
+        self.net = TSTPlus(
+            c_in=c_in,
+            c_out=c_out,
+            seq_len=seq_len,
+            d_model=d_model,
+            n_heads=n_heads,
+            d_k=d_model // n_heads,
+            d_v=d_model // n_heads,
+            d_ff=layer_size,
+            n_layers=layers,
+            dropout=conv_dropout,
+            fc_dropout=dropout,
+            flatten=False,
+            # *args, **kwargs
+        )
+
+    def forward(self, x):
+        """
+        Takes in a sequence of shape (batch, sequence, features)
+        and outputs a representation of shape (batch, features)
+        """
+        outs = self.net(x.permute(0, 2, 1))
+        return outs
+    
+
+class TransformerEncoder2(nn.Module):
+    def __init__(
+        self,
+        c_in,
+        c_out,
+        seq_len,
+        layers=3,
+        layer_size=512,
+        dropout=0.1,
+        n_heads=4,
+        conv_dropout=0,
+        *args,
+        **kwargs,
+    ):
+        super().__init__()
+        # d_model (82) must be divisible by n_heads (4)
+        layer_size = layer_size // n_heads * n_heads
+        d_model = layer_size // 2 
+        self.net = TSPerceiver(
+            c_in=c_in,
+            c_out=c_out,
+            seq_len=seq_len,
+            
+            # cat_szs=0, n_cont=0, 
+            n_latents=layer_size, d_latent=layer_size//4, 
+            # d_context=None, 
+            self_per_cross_attn=1,
+                #  share_weights=True, cross_n_heads=1, d_head=None, 
+                 
+                 
+            # d_model=d_model,
+            # d_k=d_model // n_heads,
+            # d_v=d_model // n_heads,
+            # d_ff=layer_size,
+            
+            self_n_heads=n_heads,
+            n_layers=layers,
+            attn_dropout=conv_dropout,
+            fc_dropout=dropout,
+        )
+
+    def forward(self, x):
+        """
+        Takes in a sequence of shape (batch, sequence, features)
+        and outputs a representation of shape (batch, features)
+        """
+        outs = self.net(x.permute(0, 2, 1))
+        return outs
+
+
+class LSTMEncoder(nn.Module):
+    def __init__(
+        self,
+        c_in,
+        c_out,
+        dropout=0.1,
+        conv_dropout=0,
+        layers=1,
+        layer_size=100,
+        *args,
+        **kwargs,
+    ):
+        super().__init__()
+        self.rnn = nn.LSTM(
+            c_in,
+            layer_size,
+            num_layers=layers,
+            bias=True,
+            batch_first=True,
+            dropout=conv_dropout,
+            bidirectional=False,
+        )
+        self.dropout = nn.Dropout(dropout) if dropout else noop
+        self.fc = nn.Linear(layer_size * (1 + 0), c_out)
+
+    def forward(self, x):
+        """
+        Takes in a sequence of shape (batch, sequence, features)
+        and outputs a representation of shape (batch, features)
+        """
+        # x = x.transpose(2,1)    # [batch_size x n_vars x seq_len] --> [batch_size x seq_len x n_vars]
+        output, _ = self.rnn(
+            x
+        )  # output from all sequence steps: [batch_size x seq_len x hidden_size * (1 + bidirectional)]
+        output = output[
+            :, -1
+        ]  # output from last sequence step : [batch_size x hidden_size * (1 + bidirectional)]
+        return self.fc(self.dropout(output))
+
+
+
+class LSTMEncoder2(nn.Module):
+    def __init__(
+        self,
+        c_in,
+        c_out,
+        seq_len,
+        dropout=0.1,
+        conv_dropout=0,
+        layers=1,
+        layer_size=100,
+        *args,
+        **kwargs,
+    ):
+        super().__init__()
+        self.rnn = TSSequencerPlus(
+            c_in=c_in,
+            c_out=c_out,
+            seq_len=seq_len,
+            d_model=layer_size,
+            depth=layers,
+            lstm_dropout=conv_dropout,
+            fc_dropout=dropout,
+            pre_norm=False, use_token=True, use_pe=True, 
+            use_bn=False, 
+        )
+
+    def forward(self, x):
+        """
+        Takes in a sequence of shape (batch, sequence, features)
+        and outputs a representation of shape (batch, features)
+        """
+        return self.rnn(
+            x.transpose(2, 1)
+        )
+
+
+class MLPEncoder(nn.Module):
+    def __init__(
+        self,
+        c_in,
+        c_out,
+        scales=[0.01, 0.1, 1, 5, 10, 20, 50, 100],
+        n_fourier_feats=4096,
+        layers=2,
+        layer_size=32,
+        *args,
+        **kwargs,
+    ):
+        super().__init__()
+        self.net = INR(
+            in_feats=c_in,
+            scales=scales,
+            n_fourier_feats=n_fourier_feats,
+            layers=layers,
+            layer_size=layer_size,
+        )
+
+    def forward(self, x):
+        """
+        Takes in a sequence of shape (batch, sequence, features)
+        and outputs a representation of shape (batch, features)
+        """
+        return self.net(x)[:, -1]

models/modules/metareghead.py

@@ -0,0 +1,79 @@
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from models.modules.regressors import RidgeRegressor
+from models.modules.inr import INR, INRLayer
+
+class SumHead(nn.Module):
+    def __init__(self, d, c_out=1, ):
+        super().__init__()
+        self.l = nn.Linear(d, c_out) # init a random transform
+
+    def forward(self, query, support, support_labels):
+        return self.l(query)
+
+
+class TransformerHead(nn.Module):
+    def __init__(self, d, c_out=1, dropout=0.1, num_heads=16):
+        super().__init__()
+        if d<64:
+            num_heads = 4
+        d = d//num_heads*num_heads # make sure it's divisable
+        hidden_dim = d//4
+        # the value is just one class, so let's embed it first
+        self.value_encoder = nn.Sequential(
+            INRLayer(c_out, hidden_dim, dropout=0),
+            nn.Linear(hidden_dim, hidden_dim)
+        )
+        self.l = nn.MultiheadAttention(embed_dim=d, num_heads=num_heads, batch_first=True, kdim=d, vdim=hidden_dim, add_bias_kv=True, bias=True)
+        # after using attention let's decode it
+        self.decoder = nn.Sequential(
+            INRLayer(d, d, dropout=dropout),
+            nn.Linear(d, c_out)
+        )
+
+    def forward(self, query, support, support_labels, *args, **kwargs):
+        """
+        Fits the support set with ridge regression and 
+        returns the classification score on the query set.
+
+        Parameters:
+        query:  a (tasks_per_batch, n_query, d) Tensor.
+        support:  a (tasks_per_batch, n_support, d) Tensor.
+        support_labels: a (tasks_per_batch, n_support) Tensor.
+        n_way: a scalar. Represents the number of classes in a few-shot classification task.
+        n_shot: a scalar. Represents the number of support examples given per class.
+        lambda_reg: a scalar. Represents the strength of L2 regularization.
+        Returns: a (tasks_per_batch, n_query, n_way) Tensor.
+        """
+        # should be (batch, seq, feature)
+        value = self.value_encoder(support_labels)
+        attn_output, _ = self.l(query=query, key=support, value=value)
+        o = self.decoder(attn_output)
+        return o
+
+
+class RegressionHead(nn.Module):
+    def __init__(self, base_learner='Ridge', d=512, enable_scale=True, dropout=0.1, num_heads=16):
+        super().__init__()
+        if ('Ridge' in base_learner):
+            # the regular DeepTime one
+            self.head = RidgeRegressor()
+        elif ("None" in base_learner):
+            self.head = SumHead(d=d)
+        elif ("Transformer" in base_learner):
+            self.head = TransformerHead(d=d, dropout=dropout, num_heads=num_heads)
+        else:
+            raise NotImplementedError(base_learner)
+        
+        # Add a learnable scale
+        self.enable_scale = enable_scale
+        self.scale = nn.Parameter(torch.FloatTensor([1.0]))
+        
+    def forward(self, query, support, support_labels, **kwargs):
+        if self.enable_scale:
+            return self.scale * self.head(query, support, support_labels, **kwargs)
+        else:
+            return self.head(query, support, support_labels, **kwargs)

models/modules/regressors.py

@@ -9,6 +9,7 @@ import torch
 from torch import Tensor
 from torch import nn
 import torch.nn.functional as F
+from einops import rearrange, repeat, reduce
 
 
 class RidgeRegressor(nn.Module):
@@ -16,11 +17,15 @@ class RidgeRegressor(nn.Module):
         super().__init__()
         self._lambda = nn.Parameter(torch.as_tensor(lambda_init, dtype=torch.float))
 
-    def forward(self, reprs: Tensor, x: Tensor, reg_coeff: Optional[float] = None) -> Tensor:
+    def forward(self, query_reprs:Tensor, context_reprs: Tensor, context_y: Tensor, reg_coeff: Optional[float] = None) -> Tensor:
         if reg_coeff is None:
             reg_coeff = self.reg_coeff()
-        w, b = self.get_weights(reprs, x, reg_coeff)
-        return w, b
+        w, b = self.get_weights(context_reprs, context_y, reg_coeff)
+        preds = self.forecast(query_reprs, w, b)
+        return preds
+    
+    def forecast(self, inp: Tensor, w: Tensor, b: Tensor) -> Tensor:
+        return torch.einsum('... d o, ... t d -> ... t o', [w, inp]) + b
 
     def get_weights(self, X: Tensor, Y: Tensor, reg_coeff: float) -> Tensor:
         batch_size, n_samples, n_dim = X.shape

run.sh

@@ -1,4 +1,5 @@
-for dataset in ECL ETTm2 Exchange ILI Traffic Weather Stocks; do
+set -x
+for dataset in Stocks; do
   for instance in `/bin/ls -d storage/experiments/$dataset/*/*`; do
       echo $instance
       make run command=${instance}/command