Make decoder bias optional

These changes were made during poleval'19 competition to try to
reproduce results from fastai v0.7 in v1.0.

fastai_contrib/learner.py

@@ -86,7 +86,13 @@ def convert_weights_with_prefix(wgts:Weights, stoi_wgts:Dict[str,int], itos_new:
     if 'model' in wgts:
         wgts['model'] = convert_weights_with_prefix(wgts['model'], stoi_wgts, itos_new, prefix)
     else:
-        dec_bias, enc_wgts = wgts[prefix+'1.decoder.bias'], wgts[prefix+'0.encoder.weight']
+        #dec_bias, enc_wgts = wgts[prefix+'1.decoder.bias'], wgts[prefix+'0.encoder.weight']
+        has_bias = prefix+'1.decoder.bias' in wgts
+        enc_wgts = wgts[prefix+'0.encoder.weight']
+        if has_bias:
+          dec_bias = wgts[prefix+'1.decoder.bias']
+        else:
+          dec_bias = enc_wgts.new_zeros((len(stoi_wgts),))
         bias_m, wgts_m = dec_bias.mean(0), enc_wgts.mean(0)
         new_w = enc_wgts.new_zeros((len(itos_new),enc_wgts.size(1))).zero_()
         new_b = dec_bias.new_zeros((len(itos_new),)).zero_()
@@ -101,7 +107,8 @@ def convert_weights_with_prefix(wgts:Weights, stoi_wgts:Dict[str,int], itos_new:
         wgts[prefix+'0.encoder.weight'] = new_w
         wgts[prefix+'0.encoder_dp.emb.weight'] = new_w.clone()
         wgts[prefix+'1.decoder.weight'] = new_w.clone()
-        wgts[prefix+'1.decoder.bias'] = new_b
+        if has_bias:
+          wgts[prefix+'1.decoder.bias'] = new_b
     return wgts
 
 #endregion
@@ -110,4 +117,4 @@ def convert_weights_with_prefix(wgts:Weights, stoi_wgts:Dict[str,int], itos_new:
 import fastai.text.learner
 fastai.text.learner.convert_weights = convert_weights
 
-#endregion
+#endregion

fastai_contrib/models.py

@@ -1,7 +1,7 @@
 from fastai.torch_core import *
 from fastai.layers import *
 from fastai.text.models import *
-
+from fastai.text.learner import *
 #region New code
 
 class BiLMModel(nn.Module):
@@ -163,4 +163,4 @@ def get_birnn_classifier(bptt:int, max_seq:int, n_class:int, vocab_sz:int, emb_s
     model.reset()
     return model
 
-#endregion
+#endregion

fastai_contrib/utils.py

@@ -80,7 +80,7 @@ def get_sentencepiece(cache_dir:PathOrStr, load_text, pre_rules: ListRules=None,
     pre_rules = pre_rules if pre_rules is not None else defaults.text_pre_rules
     post_rules = post_rules if post_rules is not None else defaults.text_post_rules
 
-    special_cases = defaults.text_spec_tok
+    special_cases = defaults.text_spec_tok + ['<link>', '<user>', '<number>', '<emoji>', '</emoji>']
     if not os.path.isfile(cache_dir / 'spm.model') or not os.path.isfile(cache_dir / f'itos.pkl'):
         # load the text from the train tokens file
         text = load_text()
@@ -100,7 +100,7 @@ def get_sentencepiece(cache_dir:PathOrStr, load_text, pre_rules: ListRules=None,
         sp_params = [
             f"--input={raw_text_path}",
             f"--character_coverage={char_coverage}",
-            f"--unk_id={len(defaults.text_spec_tok)}",
+            f"--unk_id={len(special_cases)}",
             f"--pad_id=-1",
             f"--bos_id=-1",
             f"--eos_id=-1",

ulmfit/pretrain_lm.py

@@ -39,7 +39,9 @@ def istitle(line):
     return len(re.findall(r'^ ?= [^=]* = ?$', line)) != 0
 
 def read_wiki_articles(filename):
+    return pd.read_csv(filename, sep="\t", header=None, names=["texts"])
     articles = []
+
     with open(filename, encoding='utf8') as f:
         lines = f.readlines()
     current_article = []
@@ -70,7 +72,8 @@ class LMHyperParams:
 
     # these hyperparameters are for training on ~100M tokens (e.g. WikiText-103)
     # for training on smaller datasets, more dropout is necessary
-    dps = dict(output_p=0.25, hidden_p=0.1, input_p=0.2, embed_p=0.02, weight_p=0.15) # consider removing dps & clip from the default hyperparams and put them to train
+    # buggy dps = dict(output_p=0.25, hidden_p=0.1, input_p=0.2, embed_p=0.02, weight_p=0.15) # consider removing dps & clip from the default hyperparams and put them to train
+    dps = dict(input_p=0.25, output_p=0.1, weight_p=0.2, embed_p=0.02, hidden_p=0.15) # consider removing dps & clip from the default hyperparams and put them to train
     clip: float = 0.12
     bptt: int = 70
     # alpha and beta - defaults like in fastai/text/learner.py:RNNLearner()
@@ -171,10 +174,10 @@ class LMHyperParams:
         with (self.model_dir / 'info.json').open("w") as fp: json.dump(vals, fp)
         print("Saving info", self.model_dir / 'info.json')
 
-    def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, lr=5e-3, label_smoothing_eps=0.0):
+    def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, lr=5e-3, label_smoothing_eps=0.0, out_bias=True):
         self.model_dir.mkdir(exist_ok=True, parents=True)
         data_lm = self.load_wiki_data(bs=bs) if data_lm is None else data_lm
-        learn = self.create_lm_learner(data_lm, drop_mult=drop_mult, label_smoothing_eps=label_smoothing_eps)
+        learn = self.create_lm_learner(data_lm, drop_mult=drop_mult, label_smoothing_eps=label_smoothing_eps, out_bias=out_bias)
         print("Bptt", data_lm.bptt)
         learn.true_wd = true_wd
         if num_epochs > 0:
@@ -204,10 +207,10 @@ class LMHyperParams:
         # do we need to return `learn'? it adds noise to Fire output
         #return learn
 
-    def create_lm_learner(self, data_lm, dps=None, label_smoothing_eps=0.0, **kwargs):
+    def create_lm_learner(self, data_lm, dps=None, label_smoothing_eps=0.0, out_bias=True, **kwargs):
         assert self.bidir == False, "bidirectional model is not yet supported"
         config = dict(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, pad_token=PAD_TOKEN_ID, qrnn=self.qrnn,
-                          tie_weights=True, out_bias=True)
+                          tie_weights=True, out_bias=out_bias)
         config.update(dps or self.dps)
         trn_args = dict(clip=self.clip, alpha=self.rnn_alpha, beta=self.rnn_beta)
         trn_args.update(kwargs)

ulmfit/train_clas.py

@@ -12,6 +12,12 @@ import fire
 
 from ulmfit.pretrain_lm import LMHyperParams, ENC_BEST
 
+from sklearn.metrics import f1_score as f1s, precision_score, recall_score
+
+def f1_score(preds, targs):
+    preds = torch.max(preds, dim=1)[1].cpu().numpy()
+    targs = targs.cpu().numpy()
+    return torch.tensor(f1s(targs, preds))
 
 class CLSHyperParams(LMHyperParams):
     # dir_path -> data/imdb/
@@ -107,9 +113,12 @@ class CLSHyperParams(LMHyperParams):
         if data_tst is None:
             _, _, data_tst = self.load_cls_data(bs)
         if learn is None:
-            learn = self.create_cls_learner(data_tst, drop_mult=0.3)
+            learn = self.create_cls_learner(data_tst, drop_mult=0.3, metrics=[f1_score, accuracy])
             learn.unfreeze()
         learn.load(save_name)
+        print(data_tst.one_batch()[1])
+        print(data_tst.one_batch()[1])
+        print(data_tst.one_batch()[1])
         results = learn.validate(data_tst.valid_dl)
         print(f"Loss and accuracy using ({save_name}):", results)
         return list(map(float, results))
@@ -141,7 +150,7 @@ class CLSHyperParams(LMHyperParams):
         self.model_dir.mkdir(exist_ok=True, parents=True)
         add_trn_to_lm = True
         lang = self.lang
-        use_moses = True
+        use_moses = False #True
         if 'xnli' in str(self.dataset_dir):
             NotImplementedError("Support for Xnli is not implemented yet")
         if 'imdb' in self.dataset_dir.name: