Merge branch 'master' into backwards-lm

ulmfit/postprocess_wikitext.py

@@ -53,7 +53,7 @@ def limit_vocab(unk_path, vocab):
             tokens = [''] + tokens
             line = ' '.join(tokens)
             f_out.write(line)
-    print(f'{unk_path.name}. # of tokens: {total_num_tokens}')
+    print(f'{unk_path.name}. # of tokens: {total_num_tokens}')
     temp_file_path.replace(unk_path)
 
 
@@ -101,5 +101,6 @@ def postprocess_wikitext(path, lang):
         unk_path = dest_path / f'{lang}.wiki.{split}.tokens'
         limit_vocab(unk_path, vocab)
 
+
 if __name__ == '__main__':
-    fire.Fire(postprocess_wikitext)
+    fire.Fire(postprocess_wikitext)

ulmfit/pretrain_lm.py

@@ -72,6 +72,9 @@ class LMHyperParams:
     dps = (0.25, 0.1, 0.2, 0.02, 0.15) # consider removing dps & clip from the default hyperparams and put them to train
     clip: float = 0.12
     bptt: int = 70
+    # alpha and beta - defaults like in fastai/text/learner.py:RNNLearner()
+    rnn_alpha: float = 2  # activation regularization (AR)
+    rnn_beta: float = 1  # temporal activation regularization (TAR)
 
     lang: str = 'en'
     name: str = None
@@ -130,7 +133,7 @@ class LMHyperParams:
         else:
             return contrib_data.LanguageModelType.FwdLM
 
-    def tokenzier_to_fastai_args(self, trn_data_loading_func, add_moses):
+    def tokenizer_to_fastai_args(self, trn_data_loading_func, add_moses):
         tok_func = MosesTokenizerFunc if add_moses else BaseTokenizer
         if self.tokenizer is Tokenizers.SUBWORD:
             if self.base_lm_path: # ensure we are using the same sentence piece model
@@ -186,7 +189,7 @@ class LMHyperParams:
                 learn.unfreeze()
                 if not learn.true_wd: learn.fit_one_cycle(num_epochs, lr, (0.8, 0.7), wd=1e-7)
                 else:                 learn.fit_one_cycle(num_epochs, lr, (0.8, 0.7)) # TODO find proper values
-        learn.save("lm_best_with_opt", with_opt=False)
+        learn.save("lm_best_with_opt", with_opt=True)
         learn.save_encoder(ENC_BEST)
         learn.save(LM_BEST, with_opt=False)
         print(learn.path)
@@ -200,7 +203,8 @@ class LMHyperParams:
 
         trn_args = dict(tie_weights=True, clip=self.clip, bptt=self.bptt,
                         pretrained_fnames=self.pretrained_fnames,
-                        pretrained_model=self.pretrained_model)
+                        pretrained_model=self.pretrained_model,
+                        alpha=self.rnn_alpha, beta=self.rnn_beta)
         trn_args.update(kwargs)
         print ("Training args: ", trn_args, "dps: ", dps or self.dps)
         learn = lm_learner(data_lm, emb_sz=self.emb_sz, nh=self.nh, nl=self.nl, pad_token=PAD_TOKEN_ID,
@@ -224,7 +228,7 @@ class LMHyperParams:
         for path_ in [trn_path, val_path, tst_path]:
             assert path_.exists(), f'Error: {path_} does not exist.'
 
-        args = self.tokenzier_to_fastai_args(trn_data_loading_func=self.load_train_text, add_moses=False)
+        args = self.tokenizer_to_fastai_args(trn_data_loading_func=self.load_train_text, add_moses=False)
         try:
             data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type, bs=bs)
             print("Tokenized data loaded")

ulmfit/train_clas.py

@@ -125,7 +125,7 @@ class CLSHyperParams(LMHyperParams):
             trn_df, val_df = trn_df[:trn_len], trn_df[trn_len:]
             cls_cache = '.'
 
-        args = self.tokenzier_to_fastai_args(trn_data_loading_func=lambda: trn_df[1], add_moses=True)
+        args = self.tokenizer_to_fastai_args(trn_data_loading_func=lambda: trn_df[1], add_moses=True)
 
         try:
             if force: raise FileNotFoundError("Forcing reloading of caches")