update

readme.md

@@ -1 +1,42 @@
+Using attentive neural process for forecasting smartmeter data in pytorch
+
+Changes for stability:
+- in eval mode, take mean of latent space, and mean output, don't sample
+- use log_variance where possible
+  - and add a minimum bound to std (in log domain) to avoid mode collapse
+- use pytorch attention (which has dropout)
+- use batchnorm and dropout on channel dimensions
+- added log_prob loss
+- check and skip nonfinite values because for extreme inputs we can still get nan's
+
+
+# Usage:
+
+
+- see requirements.txt for requirements
 - Download data from https://www.kaggle.com/jeanmidev/smart-meters-in-london/version/11
+  - Unzip smart-meters-in-london.zip
+  - unzip data/smart-meters-in-london/halfhourly_dataset.zip
+  - You should now have the file: `./data/smart-meters-in-london/halfhourly_dataset/block_0.csv`
+- Start and run the notebook [smartmeters.ipynb](https://github.com/wassname/attentive-neural-processes/blob/master/smartmeters.ipynb)
+
+# Example outputs:
+
+Here the black dots are input data, the dotted line is the true data. The blue line is the prediction, and the blue shadow is the uncertainty.
+
+![](docs/1.png)
+
+![](docs/4.png)
+
+![](docs/5.png)
+
+![](docs/train.png)
+
+
+# See also:
+
+- Original code in tensorflow: https://github.com/deepmind/neural-processes/blob/master/attentive_neural_process.ipynb
+- First pytorch implementation: https://github.com/soobinseo/Attentive-Neural-Process/blob/master/network.py
+- Second pytorch implementation (has some major bugs) https://github.com/KurochkinAlexey/Attentive-neural-processes/blob/master/anp_1d_regression.ipynb
+- If you want to try vanilla neural processes: https://github.com/EmilienDupont/neural-processes/blob/master/example-1d.ipynb
+

requirements.txt

@@ -0,0 +1,4 @@
+torch==1.2.0
+tqdm
+pandas
+numpy

src/data/smart_meter.py

@@ -54,8 +54,13 @@ class SmartMeterDataSet(torch.utils.data.Dataset):
 
     def __getitem__(self, i):
         rows = self.df.iloc[i : i + (self.num_context + self.num_extra_target)].copy()
-# (df['tstp'] -  df['tstp'].iloc[0]).dt.total_seconds()
-        rows['tstp'] = (rows['tstp'] - rows['tstp'].iloc[0]).dt.total_seconds() /  86400.0
+        rows['tstp'] = (rows['tstp'] - rows['tstp'].iloc[0]).dt.total_seconds() / 86400.0
+        rows = rows.sort_values('tstp')
+
+        # make sure tstp, which is our x axis, is the first value
+        columns = ['tstp'] + list(set(rows.columns) - set(['tstp']))
+        rows = rows[columns]
+
         x = rows.drop(columns=self.label_names).values
         y = rows[self.label_names].values
         return x, y

src/models/model.py

@@ -51,7 +51,7 @@ class LatentModel(nn.Module):
         dropout=0,
     ):
 
-        super(LatentModel, self).__init__()
+        super().__init__()
 
         self._latent_encoder = LatentEncoder(
             x_dim + y_dim,
@@ -78,6 +78,7 @@ class LatentModel(nn.Module):
             x_dim,
             y_dim,
             hidden_dim=hidden_dim,
+            latent_dim=latent_dim,
             n_decoder_layers=n_decoder_layers,
             dropout=dropout,
         )
@@ -87,9 +88,12 @@ class LatentModel(nn.Module):
 
         dist_prior, log_var_prior = self._latent_encoder(context_x, context_y)
 
-        if target_y is not None:
+        if (target_y is not None):
             dist_post, log_var_post = self._latent_encoder(target_x, target_y)
-            z = dist_post.rsample()
+            if self.training:
+                z = dist_post.rsample()
+            else:
+                z = dist_post.loc
         else:
             z = (
                 dist_prior.loc
@@ -115,5 +119,6 @@ class LatentModel(nn.Module):
             kl_loss = None
             loss = None
 
-        return dist.rsample(), kl_loss, loss, dist.scale
+        y_pred = dist.rsample() if self.training else dist.loc
+        return y_pred, kl_loss, loss, dist.scale
 

src/models/modules.py

@@ -7,7 +7,7 @@ class NPBlockRelu2d(nn.Module):
     """Block for Neural Processes."""
 
     def __init__(self, in_channels, out_channels, dropout=0, norm=True):
-        super(NPBlockRelu2d, self).__init__()
+        super().__init__()
         self.linear = nn.Linear(in_channels, out_channels)
         self.act = nn.ReLU()
         self.dropout = nn.Dropout2d(dropout)
@@ -101,7 +101,7 @@ class LatentEncoder(nn.Module):
         min_std=0.1,
         dropout=0,
     ):
-        super(LatentEncoder, self).__init__()
+        super().__init__()
         self._input_layer = NPBlockRelu2d(input_dim, hidden_dim, dropout)
         self._encoder = nn.Sequential(
             *[
@@ -119,7 +119,6 @@ class LatentEncoder(nn.Module):
 
     def forward(self, x, y):
         encoder_input = torch.cat([x, y], dim=-1)
-
         encoded = self._input_layer(encoder_input)
 
         encoded = self._encoder(encoded)
@@ -153,7 +152,7 @@ class DeterministicEncoder(nn.Module):
         dropout=0,
         n_heads=8,
     ):
-        super(DeterministicEncoder, self).__init__()
+        super().__init__()
         self._input_layer = NPBlockRelu2d(input_dim, hidden_dim, dropout)
         self._d_encoder = nn.Sequential(
             *[
@@ -194,22 +193,21 @@ class Decoder(nn.Module):
         min_std=0.1,
         dropout=0,
     ):
-        super(Decoder, self).__init__()
+        super().__init__()
         self._target_transform = NPBlockRelu2d(x_dim, hidden_dim, dropout)
-        hidden_dim_2 = 2 * hidden_dim + latent_dim
+        hidden_dim_2 = 2* hidden_dim + latent_dim
         self._decoder = nn.Sequential(
             *[
                 NPBlockRelu2d(hidden_dim_2, hidden_dim_2, dropout)
                 for _ in range(n_decoder_layers)
             ]
         )
-        self._mean = nn.Linear(2 * hidden_dim + latent_dim, y_dim)
-        self._std = nn.Linear(2 * hidden_dim + latent_dim, y_dim)
+        self._mean = nn.Linear(hidden_dim_2, y_dim)
+        self._std = nn.Linear(hidden_dim_2, y_dim)
         self.min_std = min_std
 
     def forward(self, r, z, target_x):
         x = self._target_transform(target_x)
-
         representation = torch.cat([torch.cat([r, z], dim=-1), x], dim=-1)
         representation = self._decoder(representation)