more models

seq2seq_time/models/inceptiontime.py

@@ -0,0 +1,98 @@
+# from https://mohcinemadkour.github.io/posts/2019/10/Machine%20Learning,%20timeseriesAI,%20Time%20Series%20Classification,%20fastai_timeseries,%20TSC%20bechmark/
+
+
+# This is an unofficial PyTorch implementation by Ignacio Oguiza - oguiza@gmail.com based on:
+
+# Fawaz, H. I., Lucas, B., Forestier, G., Pelletier, C., Schmidt, D. F., Weber, J., ... & Petitjean, F. (2019). InceptionTime: Finding AlexNet for Time Series Classification. arXiv preprint arXiv:1909.04939.
+# Official InceptionTime tensorflow implementation: https://github.com/hfawaz/InceptionTime
+
+import torch
+import torch.nn as nn
+
+def noop(x):
+    return x
+
+def shortcut(c_in, c_out):
+    return nn.Sequential(*[nn.Conv1d(c_in, c_out, kernel_size=1), 
+                           nn.BatchNorm1d(c_out)])
+    
+class Inception(nn.Module):
+    def __init__(self, c_in, bottleneck=32, ks=40, nb_filters=32):
+
+        super().__init__()
+        self.bottleneck = nn.Conv1d(c_in, bottleneck, 1) if bottleneck and c_in > 1 else noop
+        mts_feat = bottleneck or c_in
+        conv_layers = []
+        kss = [ks // (2**i) for i in range(3)]
+        # ensure odd kss until nn.Conv1d with padding='same' is available in pytorch 1.3
+        kss = [ksi if ksi % 2 != 0 else ksi - 1 for ksi in kss]  
+        for i in range(len(kss)):
+            conv_layers.append(
+                nn.Conv1d(mts_feat, nb_filters, kernel_size=kss[i], padding=kss[i] // 2))
+        self.conv_layers = nn.ModuleList(conv_layers)
+        self.maxpool = nn.MaxPool1d(3, stride=1, padding=1)
+        self.conv = nn.Conv1d(c_in, nb_filters, kernel_size=1)
+        self.bn = nn.BatchNorm1d(nb_filters * 4)
+        self.act = nn.ReLU()
+
+    def forward(self, x):
+        input_tensor = x
+        x = self.bottleneck(input_tensor)
+        for i in range(3):
+            out_ = self.conv_layers[i](x)
+            if i == 0: out = out_
+            else: out = torch.cat((out, out_), 1)
+        mp = self.conv(self.maxpool(input_tensor))
+        inc_out = torch.cat((out, mp), 1)
+        return self.act(self.bn(inc_out))
+
+
+class InceptionBlock(nn.Module):
+    def __init__(self,c_in,bottleneck=32,ks=40,nb_filters=32,residual=True,depth=6):
+
+        super().__init__()
+
+        self.residual = residual
+        self.depth = depth
+
+        #inception & residual layers
+        inc_mods = []
+        res_layers = []
+        res = 0
+        for d in range(depth):
+            inc_mods.append(
+                Inception(c_in if d == 0 else nb_filters * 4, bottleneck=bottleneck if d > 0 else 0,ks=ks,
+                          nb_filters=nb_filters))
+            if self.residual and d % 3 == 2:
+                res_layers.append(shortcut(c_in if res == 0 else nb_filters * 4, nb_filters * 4))
+                res += 1
+            else: res_layer = res_layers.append(None)
+        self.inc_mods = nn.ModuleList(inc_mods)
+        self.res_layers = nn.ModuleList(res_layers)
+        self.act = nn.ReLU()
+        
+    def forward(self, x):
+        res = x
+        for d, l in enumerate(range(self.depth)):
+            x = self.inc_mods[d](x)
+            if self.residual and d % 3 == 2:
+                res = self.res_layers[d](res)
+                x += res
+                res = x
+                x = self.act(x)
+        return x
+    
+class InceptionTime(nn.Module):
+    def __init__(self,c_in,c_out,bottleneck=32,ks=40,nb_filters=32,residual=True,depth=6):
+        super().__init__()
+        self.block = InceptionBlock(c_in,bottleneck=bottleneck,ks=ks,nb_filters=nb_filters,
+                                    residual=residual,depth=depth)
+        self.gap = nn.AdaptiveAvgPool1d(1)
+        self.fc = nn.Linear(nb_filters * 4, c_out)
+
+    def forward(self, x):
+        x = self.block(x)
+        x = self.gap(x).squeeze(-1)
+        x = self.fc(x)
+        return x
+

seq2seq_time/models/xattention.py

@@ -0,0 +1,66 @@
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+from ..util import mask_upper_triangular
+
+class CrossAttention(nn.Module):
+    """
+    A single transformer, masking nan or 0
+    """
+    def __init__(self, x_dim, y_dim, attention_dropout=0, nhead=8, nlayers=8, hidden_size=32, nan_value=0, min_std=0.01):
+        super().__init__()
+        self._min_std = min_std
+        self.nan_value = nan_value
+        enc_x_dim = x_dim + y_dim
+
+        self.enc_emb = nn.Linear(enc_x_dim, hidden_size)
+        encoder_norm = nn.LayerNorm(hidden_size)
+        layer_enc = nn.TransformerEncoderLayer(
+            d_model=hidden_size,
+            dim_feedforward=hidden_size*8,
+            dropout=attention_dropout,
+            nhead=nhead,
+            # activation
+        )
+        self.encoder = nn.TransformerEncoder(
+            layer_enc, num_layers=nlayers, norm=encoder_norm
+        )
+        self.mean = nn.Linear(hidden_size, y_dim)
+        self.std = nn.Linear(hidden_size, y_dim)
+
+    def forward(self, past_x, past_y, future_x, future_y=None):
+        device = next(self.parameters()).device
+        B, S, _ = future_x.shape
+        future_y_fake = past_y[:, -1:, :].repeat(1, S, 1).to(device)
+        # future_y_fake = (
+        #     torch.ones(past_y.shape[0], future_x.shape[1], past_y.shape[2]).float().to(device) * past_y[:, -1].repeat(B, S, 1)
+        # )
+        context = torch.cat([past_x, past_y], -1).detach()
+        target = torch.cat([future_x, future_y_fake], -1).detach()
+        x = torch.cat([context, target * 1], 1).detach()
+
+        # Masks
+        x_mask = torch.isfinite(x) & (x != self.nan_value)
+        x[~x_mask] = 0
+        x = x.detach()
+        x_key_padding_mask = ~x_mask.any(-1)
+
+        x = self.enc_emb(x).permute(1, 0, 2)
+
+        B, S, _ = x.shape
+        mask = mask_upper_triangular(S, device)
+        
+        outputs = self.encoder(x, mask=mask#, src_key_padding_mask=x_key_padding_mask
+        ).permute(
+            1, 0, 2
+        )
+
+        # Seems to help a little, especially with extrapolating out of bounds
+        steps = past_y.shape[1]
+        mean = self.mean(outputs)[:, steps:, :]
+        log_sigma = self.std(outputs)[:, steps:, :]
+        
+        sigma = self._min_std + (1 - self._min_std) * F.softplus(log_sigma)
+        return torch.distributions.Normal(mean, sigma), {}
+