Fastai upgrade

fastai_contrib/data.py

@@ -13,24 +13,24 @@ class LanguageModelLoader(): # copy of the original LanguageModelLoader
     "Create a dataloader with bptt slightly changing."
     def __init__(self, dataset:LabelList, bs:int=64, bptt:int=70,
                  lm_type:LanguageModelType=LanguageModelType.FwdLM, shuffle:bool=False,
-                 max_len:int=25):
-        self.dataset,self.bs,self.bptt,self.lm_type,self.shuffle = dataset,bs,bptt,lm_type,shuffle
+                 max_len:int=25, p_bptt:int=0.95):
+        self.dataset,self.bs,self.bptt,self.lm_type,self.shuffle, self.p_bptt = dataset,bs,bptt,lm_type,shuffle,p_bptt
         self.first,self.i,self.iter = True,0,0
         self.n = len(np.concatenate(dataset.x.items)) // self.bs if len(dataset.x.items) > 0 else 0
         self.max_len,self.num_workers = max_len,0
-        self.init_kwargs = dict(bs=bs, bptt=bptt, lm_type=lm_type, shuffle=shuffle, max_len=max_len)
+        self.init_kwargs = dict(bs=bs, bptt=bptt, lm_type=lm_type, shuffle=shuffle, max_len=max_len, p_bptt=p_bptt)
 
     def __iter__(self):
         if getattr(self.dataset, 'item', None) is not None:
-            yield LongTensor(getattr(self.dataset, 'item')).unsqueeze(1),LongTensor([0])
+            yield LongTensor(getattr(self.dataset, 'item'))[None],LongTensor([0])
         idx = np.random.permutation(len(self.dataset)) if self.shuffle else range(len(self.dataset))
-        data = self.batchify(np.concatenate([np.array(self.dataset.x.items[i], dtype=np.int) for i in idx]))
+        data = self.batchify(np.concatenate([self.dataset.x.items[i] for i in idx]))
 
         pos, itr = 0,0
         while pos < self.n-1 and itr<len(self):
             if self.first and pos == 0: self.first,seq_len = False,self.bptt + self.max_len
             else:
-                bptt = self.bptt if np.random.random() < 0.95 else self.bptt / 2.
+                bptt = self.bptt if np.random.random() < self.p_bptt else self.bptt / 2.
                 seq_len = max(5, int(np.random.normal(bptt, 5)))
                 seq_len = min(seq_len, self.bptt + self.max_len)
             res = self.get_batch(data, pos, seq_len)
@@ -49,17 +49,17 @@ class LanguageModelLoader(): # copy of the original LanguageModelLoader
     def batchify(self, data:np.ndarray) -> LongTensor:
         "Split the corpus `data` in batches."
         nb = data.shape[0] // self.bs
-        data = np.array(data[:nb*self.bs]).reshape(self.bs, -1).T
-        if self.lm_type == LanguageModelType.BwdLM: data=data[::-1].copy()
-        elif self.lm_type == LanguageModelType.BiLM: data = np.stack([data, data[::-1].copy()], axis=2)
+        data = np.array(data[:nb*self.bs]).reshape(self.bs, -1)
+        if self.lm_type == LanguageModelType.BwdLM: data = data[:,::-1].copy()
+        elif self.lm_type == LanguageModelType.BiLM: data = np.stack([data, data[:,::-1].copy()], axis=2)
         return LongTensor(data)
 
     def get_batch(self, data:LongTensor,  i:int, seq_len:int) -> Tuple[LongTensor, LongTensor]:
         "Create a batch at `i` of a given `seq_len`."
-        seq_len = min(seq_len, len(data) - 1 - i)
-        x = data[i:i+seq_len]
-        y = data[i+1:i+1+seq_len].contiguous() # x & y has 2 elements on the last dimension
-        y = y.view(-1, 2) if self.lm_type == LanguageModelType.BiLM else y.view(-1)
+        seq_len = min(seq_len, data.shape[1] - 1 - i)
+        x = data[:,i:i+seq_len]
+        y = data[:,i+1:i+1+seq_len]
+        #y = y.view(-1, 2) if self.lm_type == LanguageModelType.BiLM else y.view(-1)
         return x,y
 
 #endregion