Fix seeds

fastai_contrib/utils.py

@@ -80,7 +80,8 @@ def get_sentencepiece(cache_dir:PathOrStr, load_text, pre_rules: ListRules=None,
     pre_rules = pre_rules if pre_rules is not None else defaults.text_pre_rules
     post_rules = post_rules if post_rules is not None else defaults.text_post_rules
 
-    special_cases = defaults.text_spec_tok + ['<link>', '<user>', '<number>', '<emoji>', '</emoji>']
+    #special_cases = defaults.text_spec_tok + ['<link>', '<user>', '<number>', '<emoji>', '</emoji>']
+    special_cases = defaults.text_spec_tok + ['xxlink', 'xxuser', 'xxnumber', 'xxemoji', 'yyemoji']
     if not os.path.isfile(cache_dir / 'spm.model') or not os.path.isfile(cache_dir / f'itos.pkl'):
         # load the text from the train tokens file
         text = load_text()

ulmfit/create_wikitext.py

@@ -7,6 +7,7 @@ Articles are tokenized using the Moses tokenizer. Articles with least than
 import argparse
 from pathlib import Path
 import json
+import csv
 
 from shutil import copyfile
 
@@ -24,7 +25,7 @@ def get_texts(root):
                     if text.strip() == title:
                         # print('No content continuing...')
                         continue
-                    yield (f"={title}=\n"+text)
+                    yield text
 
 
 def write_wikitext(file_path, text_iter, mt, num_tokens, mode='w'):
@@ -65,7 +66,43 @@ def write_wikitext(file_path, text_iter, mt, num_tokens, mode='w'):
         file_path, i, total_num_tokens))
 
 
-def main(args):
+def wiki2csv(file_path, text_iter, num_tokens):
+    total_num_tokens = 0
+    print(f'Writing to {file_path}...')
+    i = 0
+    
+    with open(file_path, 'w', encoding='utf-8') as csvfile:
+        f_out = csv.writer(csvfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
+        for i, text in enumerate(text_iter):
+            num_tokens_article = 0  # count the number of tokens in an article
+            tokenized_paragraphs = []
+            paragraphs = text.split('\n')
+
+            for paragraph in paragraphs:
+                tokenized = paragraph.strip()
+                tokenized_paragraphs.append(tokenized)
+
+                tokens = tokenized.split(' ')  # split on whitespace to keep newlines
+                # don't count empty lines
+                tokens = [token for token in tokens if token]
+
+                # calculate length based on tokens; add 1 for newline
+                num_tokens_article += len(tokens) + 1
+
+            if num_tokens_article < 100:
+                # only use articles that have at least 100 tokens
+                continue
+
+            f_out.writerow(['\n'.join(tokenized_paragraphs)])
+
+            total_num_tokens += num_tokens_article + 1
+            if num_tokens is not None and total_num_tokens > num_tokens:
+                break
+            if i % 10000 == 0 and i > 0:
+                print('Processed {:,} documents. Total # tokens: {:,}.'.format(i, total_num_tokens))
+
+
+def main2(args):
 
     input_path = Path(args.input)
     output = Path(args.output)
@@ -102,6 +139,20 @@ def main(args):
     copyfile(lrg_wiki_train, all_wiki_train)
     write_wikitext(all_wiki_train, text_iter, mt,  None, mode='a')
 
+def main(args):
+
+    input_path = Path(args.input)
+    output = Path(args.output)
+    assert input_path.exists(), f'Error: {input_path} does not exist.'
+    output.mkdir(exist_ok=True)
+
+    lrg_wiki = output / f'{args.lang}-100'
+    lrg_wiki.mkdir(exist_ok=True)
+
+    text_iter = get_texts(input_path)
+
+    wiki2csv(lrg_wiki / "rawtexts.csv", text_iter, int(2e7))
+
 if __name__ == '__main__':
 
     parser = argparse.ArgumentParser()

ulmfit/pretrain_lm.py

@@ -11,6 +11,7 @@ import fire
 
 from fastai import *
 from fastai.callbacks import CSVLogger, SaveModelCallback
+import fastai.text
 from fastai.text import *
 import torch
 from fastai_contrib.utils import read_file, read_whitespace_file, \
@@ -39,6 +40,7 @@ def istitle(line):
     return len(re.findall(r'^ ?= [^=]* = ?$', line)) != 0
 
 def read_wiki_articles(filename):
+    return pd.read_csv(filename, header=None, names=["texts"]).fillna("")
     return pd.read_csv(filename, sep="\t", header=None, names=["texts"])
     articles = []
 
@@ -174,7 +176,14 @@ class LMHyperParams:
         with (self.model_dir / 'info.json').open("w") as fp: json.dump(vals, fp)
         print("Saving info", self.model_dir / 'info.json')
 
-    def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, lr=5e-3, label_smoothing_eps=0.0, out_bias=True):
+    def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, lr=5e-3, label_smoothing_eps=0.0, out_bias=True, seed=None):
+        if seed is not None:
+            print(f"Setting seed to {seed}")
+            torch.manual_seed(seed)
+            torch.backends.cudnn.deterministic = True
+            torch.backends.cudnn.benchmark = False
+            np.random.seed(seed)
+
         self.model_dir.mkdir(exist_ok=True, parents=True)
         data_lm = self.load_wiki_data(bs=bs) if data_lm is None else data_lm
         learn = self.create_lm_learner(data_lm, drop_mult=drop_mult, label_smoothing_eps=label_smoothing_eps, out_bias=out_bias)

ulmfit/train_clas.py

@@ -3,7 +3,7 @@ Train a classifier on top of a language model trained with `pretrain_lm.py`.
 Optionally fine-tune LM before.
 """
 
-from fastai.callbacks import CSVLogger
+from fastai.callbacks import CSVLogger, SaveModelCallback
 from fastai.text import *
 
 from fastai_contrib.utils import PAD_TOKEN_ID
@@ -71,7 +71,7 @@ class CLSHyperParams(LMHyperParams):
 
     def train_cls(self, num_lm_epochs, unfreeze=True, num_cls_frozen_epochs=1, bs=40, drop_mul_lm=0.3, drop_mul_cls=0.5,
                   use_test_for_validation=False, num_cls_epochs=2, limit=None, noise=0.0, cls_max_len=20*70, lr_sched='layered',
-                  label_smoothing_eps=0.0, random_init=False):
+                  label_smoothing_eps=0.0, random_init=False, seed=None, dump_preds=None):
         assert use_test_for_validation == False, "use_test_for_validation=True is not supported"
         self.model_dir.mkdir(exist_ok=True, parents=True)
 
@@ -82,11 +82,15 @@ class CLSHyperParams(LMHyperParams):
 
         if self.need_fine_tune_lm and not random_init:
             if not (self.model_dir/(ENC_BEST+".pth")).exists():
-                self.train_lm(num_lm_epochs, data_lm=data_lm, drop_mult=drop_mul_lm, label_smoothing_eps=label_smoothing_eps)
+                self.train_lm(num_lm_epochs, data_lm=data_lm, drop_mult=drop_mul_lm, label_smoothing_eps=label_smoothing_eps, seed=seed)
             else:
                 print("Language model already exist, skipping finetuning")
+        loss_func = CrossEntropyFlat(weight=torch.FloatTensor([0.5,30]).cuda())
         learn = self.create_cls_learner(data_clas, drop_mult=drop_mul_cls, max_len=cls_max_len,
-                                        label_smoothing_eps=label_smoothing_eps, random_init=random_init)
+                                        label_smoothing_eps=label_smoothing_eps, random_init=random_init,
+                                        metrics=[FBeta(beta=1.0), f1_score, accuracy],
+                                        loss_func=loss_func)
+
         if not random_init:
             try:
                 learn.load('cls_best')
@@ -96,6 +100,12 @@ class CLSHyperParams(LMHyperParams):
         else:
             print("Starting classifier from random weights")
 
+        if seed is not None:
+            print(f"Setting seed to {seed}")
+            torch.manual_seed(seed)
+            torch.backends.cudnn.deterministic = True
+            torch.backends.cudnn.benchmark = False
+            np.random.seed(seed)
 
         if hasattr(self, 'lr_schedule_'+lr_sched):
             learn.true_wd = True
@@ -105,22 +115,27 @@ class CLSHyperParams(LMHyperParams):
 
         print(f"Saving models at {learn.path / learn.model_dir}")
         learn.save('cls_last', with_opt=False)
-        learn.save('cls_best', with_opt=False) # we don't use early stopping for the time being
+        #learn.save('cls_best', with_opt=False) # we don't use early stopping for the time being
         del learn
         return self.validate_cls('cls_best', bs=bs, data_tst=data_tst, learn=None)
 
-    def validate_cls(self, save_name='cls_best', bs=40, data_tst=None, learn=None):
+    def validate_cls(self, save_name='cls_best', bs=40, data_tst=None, learn=None, dump_preds=None):
         if data_tst is None:
             _, _, data_tst = self.load_cls_data(bs)
         if learn is None:
-            learn = self.create_cls_learner(data_tst, drop_mult=0.3, metrics=[f1_score, accuracy])
+            fbeta = FBeta(beta=1.0)
+            fbeta.on_train_begin()
+            learn = self.create_cls_learner(data_tst, drop_mult=0.3, metrics=[fbeta, f1_score, accuracy])
             learn.unfreeze()
         learn.load(save_name)
-        print(data_tst.one_batch()[1])
-        print(data_tst.one_batch()[1])
-        print(data_tst.one_batch()[1])
+        probs, targets = learn.get_preds(ordered=True)
+        preds = np.argmax(probs.cpu().numpy(), axis=1)
+        if dump_preds:
+            with open(dump_preds, 'w') as f:
+                f.write('\n'.join([str(x) for x in preds]))
         results = learn.validate(data_tst.valid_dl)
-        print(f"Loss and accuracy using ({save_name}):", results)
+        print(f"F1 score bin: {results[1].item()}")
+        print(f"Loss, f1_score, almost f1_score and accuracy using ({save_name}):", results)
         return list(map(float, results))
 
     def create_cls_learner(self, data_clas, dps=None, label_smoothing_eps=0.0, random_init=False, **kwargs):
@@ -140,7 +155,7 @@ class CLSHyperParams(LMHyperParams):
             learn.freeze()
 
         learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/cls-history"),
-                               #partial(SaveModelCallback, every='improvement', name='cls_best') disabled due to memory issues
+                               partial(SaveModelCallback, every='improvement', name='cls_best', monitor="f_beta")
                                ]
         if label_smoothing_eps > 0.0:
             learn.loss_func = FlattenedLoss(LabelSmoothingCrossEntropy, eps=label_smoothing_eps)