From b40d311e0bcc09a77cbb8d8030a3a678b16d3967 Mon Sep 17 00:00:00 2001 From: wassname Date: Mon, 26 Oct 2020 15:01:42 +0800 Subject: [PATCH] more models --- seq2seq_time/models/inceptiontime.py | 98 ++++++++++++++++++++++++++++ seq2seq_time/models/xattention.py | 66 +++++++++++++++++++ 2 files changed, 164 insertions(+) create mode 100644 seq2seq_time/models/inceptiontime.py create mode 100644 seq2seq_time/models/xattention.py diff --git a/seq2seq_time/models/inceptiontime.py b/seq2seq_time/models/inceptiontime.py new file mode 100644 index 0000000..0d33f9c --- /dev/null +++ b/seq2seq_time/models/inceptiontime.py @@ -0,0 +1,98 @@ +# from https://mohcinemadkour.github.io/posts/2019/10/Machine%20Learning,%20timeseriesAI,%20Time%20Series%20Classification,%20fastai_timeseries,%20TSC%20bechmark/ + + +# This is an unofficial PyTorch implementation by Ignacio Oguiza - oguiza@gmail.com based on: + +# Fawaz, H. I., Lucas, B., Forestier, G., Pelletier, C., Schmidt, D. F., Weber, J., ... & Petitjean, F. (2019). InceptionTime: Finding AlexNet for Time Series Classification. arXiv preprint arXiv:1909.04939. +# Official InceptionTime tensorflow implementation: https://github.com/hfawaz/InceptionTime + +import torch +import torch.nn as nn + +def noop(x): + return x + +def shortcut(c_in, c_out): + return nn.Sequential(*[nn.Conv1d(c_in, c_out, kernel_size=1), + nn.BatchNorm1d(c_out)]) + +class Inception(nn.Module): + def __init__(self, c_in, bottleneck=32, ks=40, nb_filters=32): + + super().__init__() + self.bottleneck = nn.Conv1d(c_in, bottleneck, 1) if bottleneck and c_in > 1 else noop + mts_feat = bottleneck or c_in + conv_layers = [] + kss = [ks // (2**i) for i in range(3)] + # ensure odd kss until nn.Conv1d with padding='same' is available in pytorch 1.3 + kss = [ksi if ksi % 2 != 0 else ksi - 1 for ksi in kss] + for i in range(len(kss)): + conv_layers.append( + nn.Conv1d(mts_feat, nb_filters, kernel_size=kss[i], padding=kss[i] // 2)) + self.conv_layers = nn.ModuleList(conv_layers) + self.maxpool = nn.MaxPool1d(3, stride=1, padding=1) + self.conv = nn.Conv1d(c_in, nb_filters, kernel_size=1) + self.bn = nn.BatchNorm1d(nb_filters * 4) + self.act = nn.ReLU() + + def forward(self, x): + input_tensor = x + x = self.bottleneck(input_tensor) + for i in range(3): + out_ = self.conv_layers[i](x) + if i == 0: out = out_ + else: out = torch.cat((out, out_), 1) + mp = self.conv(self.maxpool(input_tensor)) + inc_out = torch.cat((out, mp), 1) + return self.act(self.bn(inc_out)) + + +class InceptionBlock(nn.Module): + def __init__(self,c_in,bottleneck=32,ks=40,nb_filters=32,residual=True,depth=6): + + super().__init__() + + self.residual = residual + self.depth = depth + + #inception & residual layers + inc_mods = [] + res_layers = [] + res = 0 + for d in range(depth): + inc_mods.append( + Inception(c_in if d == 0 else nb_filters * 4, bottleneck=bottleneck if d > 0 else 0,ks=ks, + nb_filters=nb_filters)) + if self.residual and d % 3 == 2: + res_layers.append(shortcut(c_in if res == 0 else nb_filters * 4, nb_filters * 4)) + res += 1 + else: res_layer = res_layers.append(None) + self.inc_mods = nn.ModuleList(inc_mods) + self.res_layers = nn.ModuleList(res_layers) + self.act = nn.ReLU() + + def forward(self, x): + res = x + for d, l in enumerate(range(self.depth)): + x = self.inc_mods[d](x) + if self.residual and d % 3 == 2: + res = self.res_layers[d](res) + x += res + res = x + x = self.act(x) + return x + +class InceptionTime(nn.Module): + def __init__(self,c_in,c_out,bottleneck=32,ks=40,nb_filters=32,residual=True,depth=6): + super().__init__() + self.block = InceptionBlock(c_in,bottleneck=bottleneck,ks=ks,nb_filters=nb_filters, + residual=residual,depth=depth) + self.gap = nn.AdaptiveAvgPool1d(1) + self.fc = nn.Linear(nb_filters * 4, c_out) + + def forward(self, x): + x = self.block(x) + x = self.gap(x).squeeze(-1) + x = self.fc(x) + return x + diff --git a/seq2seq_time/models/xattention.py b/seq2seq_time/models/xattention.py new file mode 100644 index 0000000..b6f302c --- /dev/null +++ b/seq2seq_time/models/xattention.py @@ -0,0 +1,66 @@ +import torch +from torch import nn +from torch.nn import functional as F + +from ..util import mask_upper_triangular + +class CrossAttention(nn.Module): + """ + A single transformer, masking nan or 0 + """ + def __init__(self, x_dim, y_dim, attention_dropout=0, nhead=8, nlayers=8, hidden_size=32, nan_value=0, min_std=0.01): + super().__init__() + self._min_std = min_std + self.nan_value = nan_value + enc_x_dim = x_dim + y_dim + + self.enc_emb = nn.Linear(enc_x_dim, hidden_size) + encoder_norm = nn.LayerNorm(hidden_size) + layer_enc = nn.TransformerEncoderLayer( + d_model=hidden_size, + dim_feedforward=hidden_size*8, + dropout=attention_dropout, + nhead=nhead, + # activation + ) + self.encoder = nn.TransformerEncoder( + layer_enc, num_layers=nlayers, norm=encoder_norm + ) + self.mean = nn.Linear(hidden_size, y_dim) + self.std = nn.Linear(hidden_size, y_dim) + + def forward(self, past_x, past_y, future_x, future_y=None): + device = next(self.parameters()).device + B, S, _ = future_x.shape + future_y_fake = past_y[:, -1:, :].repeat(1, S, 1).to(device) + # future_y_fake = ( + # torch.ones(past_y.shape[0], future_x.shape[1], past_y.shape[2]).float().to(device) * past_y[:, -1].repeat(B, S, 1) + # ) + context = torch.cat([past_x, past_y], -1).detach() + target = torch.cat([future_x, future_y_fake], -1).detach() + x = torch.cat([context, target * 1], 1).detach() + + # Masks + x_mask = torch.isfinite(x) & (x != self.nan_value) + x[~x_mask] = 0 + x = x.detach() + x_key_padding_mask = ~x_mask.any(-1) + + x = self.enc_emb(x).permute(1, 0, 2) + + B, S, _ = x.shape + mask = mask_upper_triangular(S, device) + + outputs = self.encoder(x, mask=mask#, src_key_padding_mask=x_key_padding_mask + ).permute( + 1, 0, 2 + ) + + # Seems to help a little, especially with extrapolating out of bounds + steps = past_y.shape[1] + mean = self.mean(outputs)[:, steps:, :] + log_sigma = self.std(outputs)[:, steps:, :] + + sigma = self._min_std + (1 - self._min_std) * F.softplus(log_sigma) + return torch.distributions.Normal(mean, sigma), {} +