add feature for future

src/data/smart_meter.py

@@ -23,23 +23,32 @@ def collate_fns(max_num_context, max_num_extra_target, sample, sort=True):
         x = torch.from_numpy(x).float()
         y = torch.from_numpy(y).float()
 
+        x[:,:max_num_context, -1] = 0  # Feature to let the model know this is past data
+        n=x[:, max_num_context:, -1].shape[1]
+        x[:, max_num_context:, -1] = torch.arange(1, n+1)/1.0/n  # Feature to let the model know this is past data
         x_context = x[:, :max_num_context]
         y_context = y[:, :max_num_context]
-
+    
         if sample:
             x_target_extra = x[:, max_num_context:]
             y_target_extra = y[:, max_num_context:]
 
-            # This is slightly differen't than normal, we are ensuring that out target point are in the future, to mimic deployment
+            # This is slightly differen't than normal, we are ensuring that our target point are in the future, to mimic deployment
             x_context, y_context = npsample_batch(
                 x_context, y_context, size=num_context, sort=sort
             )
+
             x_target_extra, y_target_extra = npsample_batch(
                 x_target_extra, y_target_extra, size=num_extra_target, sort=sort
             )
 
             x = torch.cat([x_context, x_target_extra], 1)
-            y = torch.cat([y_context, y_target_extra], 1)
+            y = torch.cat([y_context, y_target_extra], 1)            
+
+        assert (x_context[:, :, -1]==0).all()
+        assert (x[:, -1, -1] > 0).all()
+        assert (x[:, 0, -1] == 0).all()
+        
         return x_context, y_context, x, y
 
     return collate_fn
@@ -61,6 +70,9 @@ class SmartMeterDataSet(torch.utils.data.Dataset):
         columns = ['tstp'] + list(set(rows.columns) - set(['tstp']))
         rows = rows[columns]
 
+        # This will be the last row, and will change it upon sample to let the model know some points are in the future
+        rows['future']=1
+
         x = rows.drop(columns=self.label_names)
         y = rows[self.label_names]
         return x, y

src/models/lstm.py

@@ -2,6 +2,7 @@ import os
 import numpy as np
 import pandas as pd
 import torch
+from tqdm.auto import tqdm
 from torch import nn
 from torch.nn import functional as F
 from torch.utils.data import DataLoader
@@ -140,6 +141,13 @@ class LSTM_PL(pl.LightningModule):
         assert torch.isfinite(avg_loss)
         return {"avg_val_loss": avg_loss, "log": tensorboard_logs}
 
+
+    def test_step(self, *args, **kwargs):
+        return self.validation_step(*args, **kwargs)
+
+    def test_end(self, *args, **kwargs):
+        return self.validation_end(*args, **kwargs)
+
     def configure_optimizers(self):
         optim = torch.optim.Adam(self.parameters(), lr=self.hparams["learning_rate"])
         scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
@@ -219,13 +227,13 @@ class LSTM_PL(pl.LightningModule):
         return parser
 
 
-def plot_from_loader(loader, model, vis_i=670):
+def plot_from_loader(loader, model, vis_i=670, n=100):
     dset_test = loader.dataset
     label_names = dset_test.label_names
     y_trues = []
     y_preds = []
     vis_i = min(vis_i, len(dset_test))
-    for i in range(vis_i, vis_i + 100):
+    for i in tqdm(range(vis_i, vis_i + n)):
         x_rows, y_rows = dset_test.iloc(i)
         x, y = dset_test[i]
         device = next(model.parameters()).device