fixes

neural_processes/data/smart_meter.py

@@ -2,7 +2,7 @@ from pathlib import Path
 import pandas as pd
 import numpy as np
 import torch
-from tqdm.auto import tqdm
+
 
 from diskcache import Cache
 
@@ -28,7 +28,7 @@ def collate_fns(max_num_context, max_num_extra_target, sample, sort=True, contex
         x = torch.from_numpy(x).float()
         y = torch.from_numpy(y).float()
 
-        # Last feature will show how far in time a point is from our last context
+        # Last feature will show how far in time a point is from out last context
         assert (np.diff(x[:, :, 0], 1)>=0).all(), 'first features should be ordered e.g. seconds'
         assert (x[:, max_num_context, -1]==0.).all(), 'last features should be empty'
         time = x[:, :, 0]
@@ -53,7 +53,7 @@ def collate_fns(max_num_context, max_num_extra_target, sample, sort=True, contex
             )
 
         # do we want to compute loss over context+target_extra, or focus in on only target_extra?
-        if context_in_target is True:
+        if context_in_target:
             x_target = torch.cat([x_context, x_target_extra], 1)
             y_target = torch.cat([y_context, y_target_extra], 1)
         else:
@@ -81,7 +81,7 @@ class SmartMeterDataSet(torch.utils.data.Dataset):
         rows = rows.sort_values('tstp')
 
         # make sure tstp, which is our x axis, is the first value
-        columns = ['tstp'] + list(set(rows.columns) - set(['tstp', 'block'])) + ['future']
+        columns = ['tstp'] + list(set(rows.columns) - set(['tstp'])) + ['future']
         rows['future'] = 0.
         rows = rows[columns]
 
@@ -99,14 +99,23 @@ class SmartMeterDataSet(torch.utils.data.Dataset):
     def __len__(self):
         return len(self.df) - (self.num_context + self.num_extra_target)
 
+@cache.memoize()
+def get_smartmeter_df(indir=Path('./data/smart-meters-in-london'), use_logy=False):
+    csv_files = sorted((indir/'halfhourly_dataset').glob('*.csv'))[:1]
+    # print(csv_files)
+    df = pd.concat([pd.read_csv(f, parse_dates=[1], na_values=['Null']) for f in csv_files])
+#     print(df.info())
+
+    df = df.groupby('tstp').mean()
+    df['tstp'] = df.index
+    df.index.name = ''
 
-def load_weather_csv(infile):
-           
     # Load weather data
-    df_weather = pd.read_csv(infile, parse_dates=[3])
+    df_weather = pd.read_csv(indir/'weather_hourly_darksky.csv', parse_dates=[3])
+
     use_cols = ['visibility', 'windBearing', 'temperature', 'time', 'dewPoint',
-        'pressure', 'apparentTemperature', 'windSpeed', 
-        'humidity']
+           'pressure', 'apparentTemperature', 'windSpeed', 
+           'humidity']
     df_weather = df_weather[use_cols].set_index('time')
 
     # Resample to match energy data    
@@ -114,91 +123,54 @@ def load_weather_csv(infile):
 
     # Normalise
     weather_norms=dict(mean={'visibility': 11.2,
-    'windBearing': 195.7,
-    'temperature': 10.5,
-    'dewPoint': 6.5,
-    'pressure': 1014.1,
-    'apparentTemperature': 9.2,
-    'windSpeed': 3.9,
-    'humidity': 0.8},
+     'windBearing': 195.7,
+     'temperature': 10.5,
+     'dewPoint': 6.5,
+     'pressure': 1014.1,
+     'apparentTemperature': 9.2,
+     'windSpeed': 3.9,
+     'humidity': 0.8},
     std={'visibility': 3.1,
-    'windBearing': 90.6,
-    'temperature': 5.8,
-    'dewPoint': 5.0,
-    'pressure': 11.4,
-    'apparentTemperature': 6.9,
-    'windSpeed': 2.0,
-    'humidity': 0.1})
+     'windBearing': 90.6,
+     'temperature': 5.8,
+     'dewPoint': 5.0,
+     'pressure': 11.4,
+     'apparentTemperature': 6.9,
+     'windSpeed': 2.0,
+     'humidity': 0.1})
 
     for col in df_weather.columns:
         df_weather[col] -= weather_norms['mean'][col]
         df_weather[col] /= weather_norms['std'][col]
-    return df_weather
 
-def f2i(f: Path) -> int:
-    """block_2.csv->2"""
-    return int(f.stem.split('_')[-1])
-
-def is_test(f):
-    return f2i(f) % 8 == 1
-
-def is_val(f):
-    return f2i(f) % 7==1
-
-@cache.memoize()
-def get_smartmeter_df(indir=Path('./data/smart-meters-in-london'), max_files=60, use_logy=False):
+    df = pd.concat([df, df_weather], 1).dropna()
+    
     
-    df_weather = load_weather_csv(indir/'weather_hourly_darksky.csv')    
-
     # Also find bank holidays
     df_hols = pd.read_csv(indir/'uk_bank_holidays.csv', parse_dates=[0])
     holidays = set(df_hols['Bank holidays'].dt.round('D'))  
 
-    def load_csv(f):
-        df = pd.read_csv(f, parse_dates=[1], na_values=['Null'])
+    df['holiday'] = df.tstp.apply(lambda dt:dt.floor('D') in holidays).astype(int)
 
-        # Do a whole block as one series
-        df = df.groupby('tstp').mean()
-        df = df.sort_values('tstp')
+    # Add time features
+    time = df.tstp
+    df["month"] = time.dt.month / 12.0
+    df['day'] = time.dt.day / 310.0
+    df['week'] = time.dt.week / 52.0
+    df['hour'] = time.dt.hour / 24.0
+    df['minute'] = time.dt.minute / 24.0
+    df['dayofweek'] = time.dt.dayofweek / 7.0
 
-        df['block'] = f2i(f)
+    # Drop nan and 0's
+    df = df[df['energy(kWh/hh)']!=0]
+    df = df.dropna()
 
-        # Drop nan and 0's
-        df = df[df['energy(kWh/hh)'] != 0]
-        df = df.dropna()
-        # df.index.name = 'tstp'
-        df['tstp'] = df.index
-
-        # join weather and holidays
-        df = pd.concat([df, df_weather], 1).dropna()
-        df['holiday'] = df.tstp.apply(lambda dt: dt.floor('D') in holidays).astype(int)
-
-        # Add time features
-        time = df.tstp
-        df["month"] = time.dt.month / 12.0
-        df['day'] = time.dt.day / 310.0
-        df['week'] = time.dt.week / 52.0
-        df['hour'] = time.dt.hour / 24.0
-        df['minute'] = time.dt.minute / 24.0
-        df['dayofweek'] = time.dt.dayofweek / 7.0
-
-        if use_logy:
-            df['energy(kWh/hh)'] = np.log(df['energy(kWh/hh)']+1e-4)
-        return df
+    if use_logy:
+        df['energy(kWh/hh)'] = np.log(df['energy(kWh/hh)']+1e-4)
+    df = df.sort_values('tstp')
     
-    csv_files = list((indir / 'halfhourly_dataset').glob('*.csv'))
-    csv_files.sort(key=f2i)
-    csv_files = csv_files[:max_files]
-    
-    test_files = [f for f in csv_files if is_test(f)]
-    val_files = [f for f in csv_files if is_val(f) and (not is_test(f))]
-    train_files = [f for f in csv_files if (not is_val(f)) and (not is_test(f))]
-    # print(len(train_files), len(val_files), len(test_files))
-    # print(train_files, val_files, test_files)
-    assert not set(train_files).intersection(set(test_files), set(val_files))
-    assert not set(test_files).intersection(set(val_files))
-
-    df_test = pd.concat([load_csv(f) for f in tqdm(test_files, desc='test csv')], 0)
-    df_val = pd.concat([load_csv(f) for f in tqdm(val_files, desc='val csv')], 0)
-    df_train = pd.concat([load_csv(f) for f in tqdm(train_files, desc='train csv')], 0)
-    return df_train, df_val, df_test
+    # split data
+    n_split = -int(len(df)*0.1)
+    df_train = df[:n_split]
+    df_test = df[n_split:]
+    return df_train, df_test

neural_processes/lightning.py

@@ -95,8 +95,8 @@ class PL_Seq2Seq(pl.LightningModule):
 
     def _get_cache_dfs(self):
         if self._dfs is None:
-            df_train, df_val, df_test = get_smartmeter_df()
-            self._dfs = dict(df_train=df_train, df_val=df_val, df_test=df_test)
+            df_train, df_test = get_smartmeter_df()
+            self._dfs = dict(df_train=df_train, df_test=df_test)
         return self._dfs
 
     @pl.data_loader
@@ -126,7 +126,7 @@ class PL_Seq2Seq(pl.LightningModule):
 
     @pl.data_loader
     def val_dataloader(self):
-        df_test = self._get_cache_dfs()['df_val']
+        df_test = self._get_cache_dfs()['df_test']
         data_test = SmartMeterDataSet(
             df_test, self.hparams["num_context"], self.hparams["num_extra_target"]
         )

neural_processes/models/transformer_autor.py

@@ -67,8 +67,8 @@ class TransformerAutoRNet(nn.Module):
             attention_dropout=self.hparams.attention_dropout,
             dropout=self.hparams.dropout,
         ).get()
-        self.mean = NPBlockRelu2d(hidden_out_size*n_heads, self.hparams.output_size)
-        self.std = NPBlockRelu2d(hidden_out_size*n_heads, self.hparams.output_size)
+        self.mean = nn.Linear(hidden_out_size*n_heads, self.hparams.output_size)
+        self.std = nn.Linear(hidden_out_size*n_heads, self.hparams.output_size)
 
     def forward(self, context_x, context_y, target_x, target_y=None, mask_context=True, mask_target=True):
         device = next(self.parameters()).device

neural_processes/models/transformer_seq2seq.py

@@ -80,8 +80,8 @@ class TransformerSeq2SeqNet(nn.Module):
         self.decoder = nn.TransformerDecoder(
             layer_dec, num_layers=self.hparams.nlayers, norm=decoder_norm
         )
-        self.mean = NPBlockRelu2d(hidden_out_size, self.hparams.output_size)
-        self.std = NPBlockRelu2d(hidden_out_size, self.hparams.output_size)
+        self.mean = nn.Linear(hidden_out_size, self.hparams.output_size)
+        self.std = nn.Linear(hidden_out_size, self.hparams.output_size)
         self._use_lvar = False
         # self._reset_parameters()
 

neural_processes/models/transformer_seq2seq_autor.py

@@ -88,8 +88,8 @@ class TransformerSeq2SeqAutoRNet(nn.Module):
             attention_dropout=self.hparams.attention_dropout,
             dropout=self.hparams.dropout,
         ).get()
-        self.mean = NPBlockRelu2d(hidden_out_size*n_heads, self.hparams.output_size)
-        self.std = NPBlockRelu2d(hidden_out_size*n_heads, self.hparams.output_size)
+        self.mean = nn.Linear(hidden_out_size*n_heads, self.hparams.output_size)
+        self.std = nn.Linear(hidden_out_size*n_heads, self.hparams.output_size)
 
     def forward(self, context_x, context_y, target_x, target_y=None, mask_context=True, mask_target=True):
         device = next(self.parameters()).device