Merge branch 'pr/29'

README.md

@@ -88,3 +88,13 @@ $ git push --set-upstream n-waves ulmfit_multilingual  # to automatically push u
     - `bilm` -- scripts to train biLM ELMo style, Bert style
     - `class`  -- scripts to test classifiers on multiple languages
     - `xnli` -- scripts to test nli 
+
+
+## Running tests
+
+To run the tests, the following data is necessary:
+
+- wikitext-2 (prepared by `./prepare_wiki-en.sh`, along with wikitext-103)
+- imdb (prepared by `./prepare_imdb.sh`)
+
+then simply run tests, e.g. `pytest .`

experiments/cls_test_wt103_1_f.ipynb

@@ -714,9 +714,9 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python [conda env:fastaiv1]",
+   "display_name": "fastai-dev",
    "language": "python",
-   "name": "conda-env-fastaiv1-py"
+   "name": "fastai-dev"
   },
   "language_info": {
    "codemirror_mode": {
@@ -728,7 +728,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.0"
+   "version": "3.6.8"
   }
  },
  "nbformat": 4,

requirements.txt

@@ -1,4 +1,5 @@
 fire>=0.1.3
 cupy>=5.0.0
 scikit-learn>=0.20
-sacremoses>=0.0.5
+sacremoses>=0.0.5
+sentencepiece

tests/test_end_to_end.py

@@ -25,7 +25,8 @@ def get_test_data():
     imdb = data / "imdb"
 
     test_data = data / "test"
-    shutil.rmtree(test_data)
+    if test_data.exists():
+        shutil.rmtree(test_data)
 
     test_wt = test_data / 'wikitext-s'
     test_imdb = test_data / 'imdb'

tests/test_text_data.py

@@ -22,38 +22,35 @@ def test_should_load_backwards_lm():
     df = text_df(['neg','pos'])
 
     data = TextLMDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=0, text_cols=["text"], bs=2,
-                                   lm_type=contrib_data.LanguageModelType.BwdLM,
-                                   ld_cls=contrib_data.LanguageModelLoader)
+                                   lm_type=contrib_data.LanguageModelType.BwdLM)
     lml = data.train_dl.dl
     lml.data = lml.batchify(np.concatenate([lml.dataset.x.items[i] for i in range(len(lml.dataset))]))
     batch = lml.get_batch(lml.data, 0, 70)
 
-    assert batch[0].shape == (70, lml.bs)
+    assert batch[0].shape == (lml.bs, 70)
     assert batch[1].shape == (70*lml.bs,)
 
-
-    as_text = [lml.dataset.vocab.itos[x] for x in batch[0][:,0]]
-    np.testing.assert_array_equal(as_text[:5], ["world", "hello", '1', 'xxfld', 'project',])
+    as_text = [lml.dataset.vocab.itos[x] for x in batch[0][0]]
+    np.testing.assert_array_equal(as_text[:5], ["world", "hello", 'xxbos', 'project', 'cool'])
 
 def test_should_load_bi_lm():
     path = untar_data(URLs.IMDB_SAMPLE)
     df = text_df(['neg', 'pos'])
 
     data = TextLMDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=0, text_cols=["text"], bs=2,
-                                   lm_type=contrib_data.LanguageModelType.BiLM,
-                                   ld_cls=contrib_data.LanguageModelLoader)
+                                   lm_type=contrib_data.LanguageModelType.BiLM)
     lml = data.train_dl.dl
     lml.data = lml.batchify(np.concatenate([lml.dataset.x.items[i] for i in range(len(lml.dataset))]))
     batch = lml.get_batch(lml.data, 0, 70)
 
-    assert batch[0].shape == (70, lml.bs, 2)
+    assert batch[0].shape == (lml.bs, 70, 2)
     assert batch[1].shape == (70*lml.bs, 2)
 
-    as_text = [lml.dataset.vocab.itos[x] for x in batch[0][:, 0, 0]]
-    np.testing.assert_array_equal(as_text[:7], "xxfld 1 fast ai is a cool".split())
+    as_text = [lml.dataset.vocab.itos[x] for x in batch[0][0, :, 0]]
+    np.testing.assert_array_equal(as_text[:7], "xxbos fast ai is a cool project".split())
 
-    as_text = [lml.dataset.vocab.itos[x] for x in batch[0][:,0,1]]
-    np.testing.assert_array_equal(as_text[:5], ["world", "hello", '1', 'xxfld', 'project',])
+    as_text = [lml.dataset.vocab.itos[x] for x in batch[0][0, :, 1]]
+    np.testing.assert_array_equal(as_text[:5], ["world", "hello", 'xxbos', 'project', 'cool'])
 
 ###################### NEW CODE
 

tests/test_text_train.py

@@ -42,7 +42,7 @@ def learn():
 
 def text_df(n_labels):
     data = []
-    texts = ["fast ai is a cool project", "hello world"]
+    texts = ["fast ai is a cool project", "hello world"] * 20
     for ind, text in enumerate(texts):
         sample = {}
         for label in range(n_labels): sample[label] = ind%2
@@ -58,19 +58,21 @@ def test_val_loss(learn):
 
 
 def test_bilm_classifier_loads_encoder():
-    n_labels=2
+    n_labels=1
+    nl = 1
+    emb_sz = 100
     path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data', 'tmp')
     os.makedirs(path)
     try:
-        df = text_df(n_labels=1)
+        df = text_df(n_labels=n_labels)
         lmdf = df#[["text"]]
         print(lmdf.head())
         lmdata = TextLMDataBunch.from_df(path, lmdf, lmdf, tokenizer=Tokenizer(BaseTokenizer),
                                        lm_type=contrib_data.LanguageModelType.BiLM)
-        learn = bilm_learner(lmdata, emb_sz=100, nl=1, drop_mult=0.1, qrnn=False)
+        learn = bilm_learner(lmdata, emb_sz=emb_sz, nl=nl, drop_mult=0.1, qrnn=False)
         learn.save_encoder("enc")
-        data = TextClasDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=list(range(n_labels)), text_cols=["text"])
-        classifier = bilm_text_classifier_learner(data, emb_sz=100, nl=1, drop_mult=0.1, qrnn=False)
+        data = TextClasDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=list(range(n_labels)), text_cols=["text"], bs=8)
+        classifier = bilm_text_classifier_learner(data, emb_sz=emb_sz, nl=nl, drop_mult=0.1, qrnn=False)
         print(last_layer(classifier.model), )
         classifier.load_encoder("enc")
         classifier.fit(1)

ulmfit/postprocess_wikitext.py

@@ -53,7 +53,7 @@ def limit_vocab(unk_path, vocab):
             tokens = [''] + tokens
             line = ' '.join(tokens)
             f_out.write(line)
-    print(f'{unk_path.name}. # of tokens: {total_num_tokens}')
+    print(f'{unk_path.name}. # of tokens: {total_num_tokens}')
     temp_file_path.replace(unk_path)
 
 
@@ -101,5 +101,6 @@ def postprocess_wikitext(path, lang):
         unk_path = dest_path / f'{lang}.wiki.{split}.tokens'
         limit_vocab(unk_path, vocab)
 
+
 if __name__ == '__main__':
-    fire.Fire(postprocess_wikitext)
+    fire.Fire(postprocess_wikitext)

ulmfit/pretrain_lm.py

@@ -56,6 +56,7 @@ class LMHyperParams:
     dataset_path: str # data_dir
 
     base_lm_path: str = None
+    backwards: str = False
     bidir: bool =False
     qrnn: bool = True
     max_vocab: int = 60000
@@ -71,12 +72,17 @@ class LMHyperParams:
     dps = (0.25, 0.1, 0.2, 0.02, 0.15) # consider removing dps & clip from the default hyperparams and put them to train
     clip: float = 0.12
     bptt: int = 70
+    # alpha and beta - defaults like in fastai/text/learner.py:RNNLearner()
+    rnn_alpha: float = 2  # activation regularization (AR)
+    rnn_beta: float = 1  # temporal activation regularization (TAR)
 
     lang: str = 'en'
     name: str = None
     cuda_id: InitVar[int] = 0
 
     def __post_init__(self, cuda_id):
+        if self.bidir and self.backwards:
+            raise ValueError('Both "backwards" and "bidir" options cannot be enabled at the same time')
         if not torch.cuda.is_available():
             print('CUDA not available. Setting device=-1.')
             cuda_id = -1
@@ -100,7 +106,16 @@ class LMHyperParams:
     def tokenizer_prefix(self): return f"{self.tokenizer.value}{self.max_vocab // 1000}k"
 
     @property
-    def model_prefix(self): return ('bi' if self.bidir else '') + ('qrnn' if self.qrnn else 'lstm')
+    def model_direction(self):
+        if self.bidir:
+            return 'bi'
+        if self.backwards:
+            return 'bwd'
+        else:
+            return ''
+
+    @property
+    def model_prefix(self): return self.model_direction + ('qrnn' if self.qrnn else 'lstm')
 
     @property
     def model_name(self): return f"{self.model_prefix}_{self.name}.m"
@@ -110,9 +125,14 @@ class LMHyperParams:
 
     @property
     def lm_type(self):
-        return contrib_data.LanguageModelType.BiLM if self.bidir else contrib_data.LanguageModelType.FwdLM
+        if self.bidir:
+            return contrib_data.LanguageModelType.BiLM
+        if self.backwards:
+            return contrib_data.LanguageModelType.BwdLM
+        else:
+            return contrib_data.LanguageModelType.FwdLM
 
-    def tokenzier_to_fastai_args(self, sp_data_func, use_moses):
+    def tokenizer_to_fastai_args(self, sp_data_func, use_moses):
         tok_func = MosesTokenizerFunc if use_moses else BaseTokenizer
         if self.tokenizer is Tokenizers.SUBWORD:
             if self.base_lm_path and not(self.cache_dir/"spm.model").exists(): # ensure we are using the same sentence piece model
@@ -169,7 +189,7 @@ class LMHyperParams:
                 learn.unfreeze()
                 if not learn.true_wd: learn.fit_one_cycle(num_epochs, lr, (0.8, 0.7), wd=1e-7)
                 else:                 learn.fit_one_cycle(num_epochs, lr, (0.8, 0.7)) # TODO find proper values
-        learn.save("lm_best_with_opt", with_opt=False)
+        learn.save("lm_best_with_opt", with_opt=True)
         learn.save_encoder(ENC_BEST)
         learn.save(LM_BEST, with_opt=False)
         print(learn.path)
@@ -183,7 +203,8 @@ class LMHyperParams:
 
         trn_args = dict(tie_weights=True, clip=self.clip, bptt=self.bptt,
                         pretrained_fnames=self.pretrained_fnames,
-                        pretrained_model=self.pretrained_model)
+                        pretrained_model=self.pretrained_model,
+                        alpha=self.rnn_alpha, beta=self.rnn_beta)
         trn_args.update(kwargs)
         print ("Training args: ", trn_args, "dps: ", dps or self.dps)
         learn = lm_learner(data_lm, emb_sz=self.emb_sz, nh=self.nh, nl=self.nl, pad_token=PAD_TOKEN_ID,
@@ -208,7 +229,7 @@ class LMHyperParams:
         for path_ in [trn_path, val_path, tst_path]:
             assert path_.exists(), f'Error: {path_} does not exist.'
 
-        args = self.tokenzier_to_fastai_args(sp_data_func=self.load_train_text, use_moses=False)
+        args = self.tokenizer_to_fastai_args(sp_data_func=self.load_train_text, use_moses=False)
         try:
             data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type, bs=bs)
             print("Tokenized data loaded")

ulmfit/train_clas.py

@@ -152,7 +152,7 @@ class CLSHyperParams(LMHyperParams):
         lm_trn_df = lm_trn_df[val_len:]
         lm_val_df = lm_trn_df[:val_len]
 
-        args = self.tokenzier_to_fastai_args(sp_data_func=lambda: trn_df[1], use_moses=use_moses)
+        args = self.tokenizer_to_fastai_args(sp_data_func=lambda: trn_df[1], use_moses=use_moses)
         try:
             if force: raise FileNotFoundError("Forcing reloading of caches")
             data_lm = TextLMDataBunch.load(self.cache_dir, 'lm', lm_type=self.lm_type, bs=bs)