datasets

notebooks/01.0-mc-datasets.py

@@ -33,6 +33,7 @@ import matplotlib.pyplot as plt
 
 from pathlib import Path
 from tqdm.auto import tqdm
+from IPython.display import display, HTML
 # -
 import warnings
 warnings.simplefilter('once')
@@ -62,8 +63,8 @@ hv.renderer('bokeh').theme = ggplot_theme
 # print(f'using {device}')
 
 window_past = 48*2
-window_future = 48*2
-batch_size = 128
+window_future = 48
+batch_size = 4
 datasets_root = Path('../data/processed/')
 # -
 
@@ -77,11 +78,24 @@ from seq2seq_time.data.data import IMOSCurrentsVel, AppliancesEnergyPrediction,
 datasets = [IMOSCurrentsVel, BejingPM25, GasSensor, AppliancesEnergyPrediction, MetroInterstateTraffic, ]
 datasets
 # -
+
+
+for dataset in datasets:
+    d = dataset(datasets_root)
+    display(HTML(f"<h3>{dataset.__name__}</h3>"))
+    print(d.__doc__)
+    print('columns_forecast', d.columns_forecast)
+    print('columns_past', d.columns_past)
+    print('columns_target', d.columns_target)
+    print
+    display(d.df)
+
 # View train, test, val splits
 l = hv.Layout()
 for dataset in datasets:
     d = dataset(datasets_root)
     
+    
     p = dynspread(
         datashade(hv.Scatter(d.df_train[d.columns_target[0]]),
                   cmap='red'))
@@ -91,10 +105,11 @@ for dataset in datasets:
     p *= dynspread(
         datashade(hv.Scatter(d.df_test[d.columns_target[0]]),
                   cmap='blue'))
-    p = p.opts(title=f"{dataset}")
+    p = p.opts(title=f"{dataset.__name__}, n={len(d)}, freq={d.df.index.freq.freqstr}")
     display(p)
 
 
+
 # +
 # plot a batch
 def plot_batch_y(ds, i):
@@ -106,7 +121,7 @@ def plot_batch_y(ds, i):
     p *= hv.VLine(now).relabel('now').opts(color='red')
     return p
 
-def plot_batches_y(dataset):
+def plot_batches_y(dataset, window_past=window_past, window_future=window_future):
     ds_name = type(dataset).__name__
     opts=dict(width=200, height=100, xaxis=None, yaxis=None)
     ds_train, ds_val, ds_test = d.to_datasets(window_past=window_past,
@@ -119,22 +134,23 @@ def plot_batches_y(dataset):
         l += plot_batch_y(ds_train, i).opts(title=f'train {i}', **opts) 
         l += plot_batch_y(ds_val, i).opts(title=f'val {i}', **opts)
         l += plot_batch_y(ds_test, i).opts(title=f'test {i}', **opts)
-    return l.opts(shared_axes=False, toolbar='right', title=ds_name).cols(3)
+    return l.opts(shared_axes=False, toolbar='right', title=f"{ds_name} freq={d.df.index.freq.freqstr}").cols(3)
 
-# +
 
-    
+# -
 
 # View train, test, val splits
 for dataset in datasets:
+    ds_name = type(dataset).__name__
     d = dataset(datasets_root)
+    print(d)
     display(plot_batches_y(d))
 
 
 # +
 def plot_batch_x(ds, i):
     """Plot input features"""
-    x_past, y_past, x_future, y_future = ds.get_rows(10)
+    x_past, y_past, x_future, y_future = ds.get_rows(i)
     x = pd.concat([x_past, x_future])
     p = hv.NdOverlay({
         col: hv.Curve(x[col]) for col in x.columns
@@ -154,15 +170,10 @@ def plot_batches_x(d):
 
 # -
 
-# View train, test, val splits
+ds_train, ds_val, ds_test = d.to_datasets(window_past=window_past,
+                                          window_future=window_future)
+
+# View input columns
 for dataset in datasets:
     d = dataset(datasets_root)
     display(plot_batches_x(d))
-
-
-
-
-
-
-
-