Make the end to end test run correctly

fastai_contrib/data.py

@@ -37,9 +37,17 @@ class LanguageModelLoader(): # copy of the original LanguageModelLoader
             itr += 1
             yield res
 
-    def __len__(self) -> int: return (self.n-1) // self.bptt
+    def __len__(self) -> int: return int(math.ceil((self.n-1) / self.bptt)) # so that it is always at least 1
     def __getattr__(self,k:str)->Any: return getattr(self.dataset, k)
 
+    @property
+    def batch_size(self):
+        return self.bs
+
+    @batch_size.setter
+    def batch_size(self, v):
+        self.bs = v
+
     def batchify(self, data:np.ndarray) -> LongTensor:
         "Split the corpus `data` in batches."
         nb = data.shape[0] // self.bs

tests/test_end_to_end.py

@@ -11,10 +11,8 @@ from fastai_contrib.utils import *
 It is a mixture of a pytest unit test and woven together to compose an end to end functional test. 
 """
 
-
-def delete_test_models():
-    data = get_data_folder()
-
+import fastai.core
+fastai.core.turn_off_parallel_execution=True
 
 def copy_head(src_fn, dst_fn, n=1000):
     with src_fn.open("r") as s, dst_fn.open("w") as d:
@@ -35,7 +33,7 @@ def get_test_data():
     test_wt.mkdir(exist_ok=True, parents=True)
     test_imdb.mkdir(exist_ok=True, parents=True)
 
-    sz=10
+    sz=1
     # we use the same text to see if models can overfit
     copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.train.tokens', n=10*sz)
     copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.valid.tokens', n=6*sz)
@@ -71,11 +69,12 @@ def test_ulmfit_default_end_to_end():
         cuda_id=cuda_id,
         fine_tune=True,
         max_vocab=1000,
-        bs=2, bptt=70, name=lm_name + '-imdb-clas',
+        num_lm_epochs=0,
+        bs=4, # minimum size is 4 otherwise it somewhere becomes 1 and fit stops working
+        bptt=70,
+        name=lm_name + '-imdb-clas',
         dataset='imdb')
 
-    delete_test_models()
-
 
 def test_ulmfit_sentencepiece_end_to_end():
     """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset.
@@ -89,8 +88,8 @@ def test_ulmfit_sentencepiece_end_to_end():
         cuda_id=cuda_id,
         qrnn=True,
         subword=True,
-        max_vocab=1000,
-        bs=80,
+        max_vocab=100,
+        bs=2,
         num_epochs=1,
         name=lm_name,
     )
@@ -100,8 +99,7 @@ def test_ulmfit_sentencepiece_end_to_end():
     # NOTE: ds_pct is not available for sentencepiece -- tests are on the complete dataset
     #       sentencepiece for finetuning/classification is currently not implemented
 
-    delete_test_models()
-
 
 if __name__ == "__main__":
     fire.Fire()  # allows using all functions via CLI e.g. python utils.py prepare_imdb aclImdb.tgz
+

ulmfit/pretrain_lm.py

@@ -74,7 +74,9 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo
 
         sp = get_sentencepiece(dir_path, trn_path, name, vocab_size=max_vocab)
 
-        data_lm = TextLMDataBunch.from_csv(dir_path, 'train.csv', **sp)
+        lm_type = contrib_data.LanguageModelType.BiLM if bidir else  contrib_data.LanguageModelType.FwdLM
+
+        data_lm = TextLMDataBunch.from_csv(dir_path, 'train.csv', **sp, bs=bs, bptt=bptt, lm_type=lm_type)
         itos = data_lm.train_ds.vocab.itos
         stoi = data_lm.train_ds.vocab.stoi
     else:

ulmfit/train_clas.py

@@ -19,7 +19,7 @@ from pathlib import Path
 
 
 def new_train_clas(data_dir, lang='en', cuda_id=0, pretrain_name='wt103', model_dir='models',
-                   qrnn=False,
+                   qrnn=False, num_lm_epochs=10,
                    fine_tune=True, max_vocab=60000, bs=20, bptt=70, name='imdb-clas',
                    dataset='imdb', bidir=False, ds_pct=1.0, train=True):
     """
@@ -101,7 +101,7 @@ def new_train_clas(data_dir, lang='en', cuda_id=0, pretrain_name='wt103', model_
 
         learn.fit_one_cycle(1, 1e-2, moms=(0.8, 0.7))
         learn.unfreeze()
-        learn.fit_one_cycle(10, 1e-3, moms=(0.8, 0.7))
+        if num_lm_epochs > 0: learn.fit_one_cycle(num_lm_epochs, 1e-3, moms=(0.8, 0.7))
 
         # save encoder
         learn.save_encoder(lm_enc_finetuned)