misc

seq2seq_time/data/data.py

@@ -7,9 +7,11 @@ from sklearn_pandas import DataFrameMapper
 import xarray as xr
 import pandas as pd
 import numpy as np
+import zipfile
 
 from .dataset import Seq2SeqDataSet
 from .util import normalize_encode_dataframe, timeseries_split
+from ..util import dset_to_nc
 from .tidal import generate_tidal_periods
 
 
@@ -77,20 +79,18 @@ class GasSensor(RegressionForecastData):
         url = 'http://archive.ics.uci.edu/ml/machine-learning-databases/00487/gas-sensor-array-temperature-modulation.zip'
         
         # download if needed
-        extract_path = self.datasets_root/'GasSensor'
-        files = sorted(extract_path.glob('*.csv'))
-        if len(files)<13:
-            print('download_and_extract_archive')
-            download_and_extract_archive(url, self.datasets_root, extract_path)
+        # extract_path = self.datasets_root/'gas-sensor-array-temperature-modulation.zip'
+        download_url(url, self.datasets_root)
         
-        # Load csv's
-        files = sorted(extract_path.glob('*.csv'))
-        dfs = []
-        for f in files:
-            now = pd.to_datetime(f.stem, format='%Y%m%d_%H%M%S')
-            df = pd.read_csv(f)
-            df.index = pd.to_timedelta(df['Time (s)'], unit='s') + now
-            dfs.append(df)
+        # Load csv's from inside zip
+        zf = zipfile.ZipFile(self.datasets_root / 'gas-sensor-array-temperature-modulation.zip')
+        dfs=[]
+        for f in zf.namelist():
+            if f.endswith('.csv'):
+                now = pd.to_datetime(Pdset_to_ncath(f).stem, format='%Y%m%d_%H%M%S')
+                df = pd.read_csv(zf.open(f))
+                df.index = pd.to_timedelta(df['Time (s)'], unit='s') + now
+                dfs.append(df)
         self.df = pd.concat(dfs).dropna(subset=self.columns_target)
 
         df = df[[ 'CO (ppm)', 'Humidity (%r.h.)', 'Temperature (C)',
@@ -272,11 +272,7 @@ def get_current_timeseries(
         # Add tidal freqs
         xd = xd.merge(df_eta)
 
-        # Cache to nc
-        xd.to_netcdf(outfile)
-        print(
-            f'wrote "{outfile}" with size {outfile.stat().st_size*1e-6:2.2f} MB'
-        )
+        dset_to_nc(xd, outfile)
     return outfile
 
 

seq2seq_time/data/dataset.py

@@ -31,7 +31,7 @@ class Seq2SeqDataSet(torch.utils.data.Dataset):
         assert df.index.freq is not None, 'should have freq'
         assert_no_objects(df)
 
-        self.df = df.dropna(subset=columns_target)
+        self.df = df.dropna(subset=columns_target).ffill()
 
         self.window_past = window_past
         self.window_future = window_future
@@ -100,7 +100,7 @@ class Seq2SeqDataSet(torch.utils.data.Dataset):
     
     def __repr__(self):
         t = self.df.index
-        return f'<{type(self).__name__}(shape={self.df.shape}, times={t[0]} to {t[1]} at {t.freq.freqstr})>'
+        return f'<{type(self).__name__}(shape={self.df.shape}, times={t[0]} to {t[1]})>'
 
 
 class Seq2SeqDataSets(torch.utils.data.Dataset):

seq2seq_time/data/util.py

@@ -16,4 +16,4 @@ def normalize_encode_dataframe(df, encoder=OrdinalEncoder):
 def timeseries_split(df, test_fraction=0.2):
     """Split timeseries data with test in the future"""
     i = int(len(df)*test_fraction)
-    return df.iloc[:i], df.iloc[i:]
+    return df.iloc[:-i], df.iloc[-i:]