From ff30fb5642fb54a6d8e519f7eb682e0e72274dcb Mon Sep 17 00:00:00 2001
From: Piotr Czapla <github@piotrczapla.com>
Date: Thu, 27 Dec 2018 14:57:15 +0100
Subject: [PATCH] Fastai upgrade

---
 fastai_contrib/data.py   | 26 +++++++++++++-------------
 fastai_contrib/models.py |  2 +-
 ulmfit/train_clas.py     | 10 +++-------
 3 files changed, 17 insertions(+), 21 deletions(-)

diff --git a/fastai_contrib/data.py b/fastai_contrib/data.py
index d7be826..34a205e 100644
--- a/fastai_contrib/data.py
+++ b/fastai_contrib/data.py
@@ -13,24 +13,24 @@ class LanguageModelLoader(): # copy of the original LanguageModelLoader
     "Create a dataloader with bptt slightly changing."
     def __init__(self, dataset:LabelList, bs:int=64, bptt:int=70,
                  lm_type:LanguageModelType=LanguageModelType.FwdLM, shuffle:bool=False,
-                 max_len:int=25):
-        self.dataset,self.bs,self.bptt,self.lm_type,self.shuffle = dataset,bs,bptt,lm_type,shuffle
+                 max_len:int=25, p_bptt:int=0.95):
+        self.dataset,self.bs,self.bptt,self.lm_type,self.shuffle, self.p_bptt = dataset,bs,bptt,lm_type,shuffle,p_bptt
         self.first,self.i,self.iter = True,0,0
         self.n = len(np.concatenate(dataset.x.items)) // self.bs if len(dataset.x.items) > 0 else 0
         self.max_len,self.num_workers = max_len,0
-        self.init_kwargs = dict(bs=bs, bptt=bptt, lm_type=lm_type, shuffle=shuffle, max_len=max_len)
+        self.init_kwargs = dict(bs=bs, bptt=bptt, lm_type=lm_type, shuffle=shuffle, max_len=max_len, p_bptt=p_bptt)
 
     def __iter__(self):
         if getattr(self.dataset, 'item', None) is not None:
-            yield LongTensor(getattr(self.dataset, 'item')).unsqueeze(1),LongTensor([0])
+            yield LongTensor(getattr(self.dataset, 'item'))[None],LongTensor([0])
         idx = np.random.permutation(len(self.dataset)) if self.shuffle else range(len(self.dataset))
-        data = self.batchify(np.concatenate([np.array(self.dataset.x.items[i], dtype=np.int) for i in idx]))
+        data = self.batchify(np.concatenate([self.dataset.x.items[i] for i in idx]))
 
         pos, itr = 0,0
         while pos < self.n-1 and itr<len(self):
             if self.first and pos == 0: self.first,seq_len = False,self.bptt + self.max_len
             else:
-                bptt = self.bptt if np.random.random() < 0.95 else self.bptt / 2.
+                bptt = self.bptt if np.random.random() < self.p_bptt else self.bptt / 2.
                 seq_len = max(5, int(np.random.normal(bptt, 5)))
                 seq_len = min(seq_len, self.bptt + self.max_len)
             res = self.get_batch(data, pos, seq_len)
@@ -49,17 +49,17 @@ class LanguageModelLoader(): # copy of the original LanguageModelLoader
     def batchify(self, data:np.ndarray) -> LongTensor:
         "Split the corpus `data` in batches."
         nb = data.shape[0] // self.bs
-        data = np.array(data[:nb*self.bs]).reshape(self.bs, -1).T
-        if self.lm_type == LanguageModelType.BwdLM: data=data[::-1].copy()
-        elif self.lm_type == LanguageModelType.BiLM: data = np.stack([data, data[::-1].copy()], axis=2)
+        data = np.array(data[:nb*self.bs]).reshape(self.bs, -1)
+        if self.lm_type == LanguageModelType.BwdLM: data = data[:,::-1].copy()
+        elif self.lm_type == LanguageModelType.BiLM: data = np.stack([data, data[:,::-1].copy()], axis=2)
         return LongTensor(data)
 
     def get_batch(self, data:LongTensor,  i:int, seq_len:int) -> Tuple[LongTensor, LongTensor]:
         "Create a batch at `i` of a given `seq_len`."
-        seq_len = min(seq_len, len(data) - 1 - i)
-        x = data[i:i+seq_len]
-        y = data[i+1:i+1+seq_len].contiguous() # x & y has 2 elements on the last dimension
-        y = y.view(-1, 2) if self.lm_type == LanguageModelType.BiLM else y.view(-1)
+        seq_len = min(seq_len, data.shape[1] - 1 - i)
+        x = data[:,i:i+seq_len]
+        y = data[:,i+1:i+1+seq_len]
+        #y = y.view(-1, 2) if self.lm_type == LanguageModelType.BiLM else y.view(-1)
         return x,y
 
 #endregion
diff --git a/fastai_contrib/models.py b/fastai_contrib/models.py
index b5de84c..03f406b 100644
--- a/fastai_contrib/models.py
+++ b/fastai_contrib/models.py
@@ -29,7 +29,7 @@ class BiLMModel(nn.Module):
             b = input[..., 1]
         elif len(input.shape) == 2: # sl, bs - support during classification mode
             f = input
-            b = torch.flip(input, [0])
+            b = torch.flip(input, [1]) # todo test if we are duplicating the backward pass correctly
         else:
             raise AttributeError(f"Inorrect size of input, {input.shape}")
         fwd_o = self.fwd_lm(f)
diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py
index 627cb09..264be7c 100644
--- a/ulmfit/train_clas.py
+++ b/ulmfit/train_clas.py
@@ -5,16 +5,12 @@ Optionally fine-tune LM before.
 from sacremoses import MosesTokenizer
 
 import fastai
-import numpy as np
-import pickle
+import torch
 
 from fastai import *
 from fastai.callbacks import CSVLogger, SaveModelCallback
 from fastai.text import *
 
-import torch
-from fastai.text import TextLMDataBunch, TextClasDataBunch, language_model_learner, text_classifier_learner
-from fastai import fit_one_cycle, accuracy
 from fastai_contrib.data import LanguageModelType
 from fastai_contrib.learner import bilm_text_classifier_learner, bilm_learner, accuracy_fwd, accuracy_bwd
 from fastai_contrib.utils import PAD, UNK, read_clas_data, PAD_TOKEN_ID, DATASETS, TRN, VAL, TST, ensure_paths_exists, \
@@ -91,8 +87,8 @@ class CLSHyperParams(LMHyperParams):
                 learn.fit_one_cycle(2, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7), wd=1e-7)
         print(f"Saving models at {learn.path / learn.model_dir}")
         learn.save('cls_last', with_opt=False)
-        self.validate_cls('cls_last')
-        self.validate_cls('cls_best')
+        self.validate_cls('cls_last', bs=bs)
+        self.validate_cls('cls_best', bs=bs)
         return learn
 
     def validate_cls(self, save_name='cls_last', bs=40):