From 3125ce7d8b42464d2d8bde4dca6f927dc6cc6e1c Mon Sep 17 00:00:00 2001 From: Marcin Date: Sat, 11 May 2019 16:59:59 +0200 Subject: [PATCH] Fix seeds --- fastai_contrib/utils.py | 3 ++- ulmfit/create_wikitext.py | 55 +++++++++++++++++++++++++++++++++++++-- ulmfit/pretrain_lm.py | 11 +++++++- ulmfit/train_clas.py | 39 ++++++++++++++++++--------- 4 files changed, 92 insertions(+), 16 deletions(-) diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py index f527596..c7eb129 100644 --- a/fastai_contrib/utils.py +++ b/fastai_contrib/utils.py @@ -80,7 +80,8 @@ def get_sentencepiece(cache_dir:PathOrStr, load_text, pre_rules: ListRules=None, pre_rules = pre_rules if pre_rules is not None else defaults.text_pre_rules post_rules = post_rules if post_rules is not None else defaults.text_post_rules - special_cases = defaults.text_spec_tok + ['', '', '', '', ''] + #special_cases = defaults.text_spec_tok + ['', '', '', '', ''] + special_cases = defaults.text_spec_tok + ['xxlink', 'xxuser', 'xxnumber', 'xxemoji', 'yyemoji'] if not os.path.isfile(cache_dir / 'spm.model') or not os.path.isfile(cache_dir / f'itos.pkl'): # load the text from the train tokens file text = load_text() diff --git a/ulmfit/create_wikitext.py b/ulmfit/create_wikitext.py index 30073d4..6ba156c 100644 --- a/ulmfit/create_wikitext.py +++ b/ulmfit/create_wikitext.py @@ -7,6 +7,7 @@ Articles are tokenized using the Moses tokenizer. Articles with least than import argparse from pathlib import Path import json +import csv from shutil import copyfile @@ -24,7 +25,7 @@ def get_texts(root): if text.strip() == title: # print('No content continuing...') continue - yield (f"={title}=\n"+text) + yield text def write_wikitext(file_path, text_iter, mt, num_tokens, mode='w'): @@ -65,7 +66,43 @@ def write_wikitext(file_path, text_iter, mt, num_tokens, mode='w'): file_path, i, total_num_tokens)) -def main(args): +def wiki2csv(file_path, text_iter, num_tokens): + total_num_tokens = 0 + print(f'Writing to {file_path}...') + i = 0 + + with open(file_path, 'w', encoding='utf-8') as csvfile: + f_out = csv.writer(csvfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) + for i, text in enumerate(text_iter): + num_tokens_article = 0 # count the number of tokens in an article + tokenized_paragraphs = [] + paragraphs = text.split('\n') + + for paragraph in paragraphs: + tokenized = paragraph.strip() + tokenized_paragraphs.append(tokenized) + + tokens = tokenized.split(' ') # split on whitespace to keep newlines + # don't count empty lines + tokens = [token for token in tokens if token] + + # calculate length based on tokens; add 1 for newline + num_tokens_article += len(tokens) + 1 + + if num_tokens_article < 100: + # only use articles that have at least 100 tokens + continue + + f_out.writerow(['\n'.join(tokenized_paragraphs)]) + + total_num_tokens += num_tokens_article + 1 + if num_tokens is not None and total_num_tokens > num_tokens: + break + if i % 10000 == 0 and i > 0: + print('Processed {:,} documents. Total # tokens: {:,}.'.format(i, total_num_tokens)) + + +def main2(args): input_path = Path(args.input) output = Path(args.output) @@ -102,6 +139,20 @@ def main(args): copyfile(lrg_wiki_train, all_wiki_train) write_wikitext(all_wiki_train, text_iter, mt, None, mode='a') +def main(args): + + input_path = Path(args.input) + output = Path(args.output) + assert input_path.exists(), f'Error: {input_path} does not exist.' + output.mkdir(exist_ok=True) + + lrg_wiki = output / f'{args.lang}-100' + lrg_wiki.mkdir(exist_ok=True) + + text_iter = get_texts(input_path) + + wiki2csv(lrg_wiki / "rawtexts.csv", text_iter, int(2e7)) + if __name__ == '__main__': parser = argparse.ArgumentParser() diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index e7f8c5c..00f3bc6 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -11,6 +11,7 @@ import fire from fastai import * from fastai.callbacks import CSVLogger, SaveModelCallback +import fastai.text from fastai.text import * import torch from fastai_contrib.utils import read_file, read_whitespace_file, \ @@ -39,6 +40,7 @@ def istitle(line): return len(re.findall(r'^ ?= [^=]* = ?$', line)) != 0 def read_wiki_articles(filename): + return pd.read_csv(filename, header=None, names=["texts"]).fillna("") return pd.read_csv(filename, sep="\t", header=None, names=["texts"]) articles = [] @@ -174,7 +176,14 @@ class LMHyperParams: with (self.model_dir / 'info.json').open("w") as fp: json.dump(vals, fp) print("Saving info", self.model_dir / 'info.json') - def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, lr=5e-3, label_smoothing_eps=0.0, out_bias=True): + def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, lr=5e-3, label_smoothing_eps=0.0, out_bias=True, seed=None): + if seed is not None: + print(f"Setting seed to {seed}") + torch.manual_seed(seed) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + np.random.seed(seed) + self.model_dir.mkdir(exist_ok=True, parents=True) data_lm = self.load_wiki_data(bs=bs) if data_lm is None else data_lm learn = self.create_lm_learner(data_lm, drop_mult=drop_mult, label_smoothing_eps=label_smoothing_eps, out_bias=out_bias) diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 4396136..1db1106 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -3,7 +3,7 @@ Train a classifier on top of a language model trained with `pretrain_lm.py`. Optionally fine-tune LM before. """ -from fastai.callbacks import CSVLogger +from fastai.callbacks import CSVLogger, SaveModelCallback from fastai.text import * from fastai_contrib.utils import PAD_TOKEN_ID @@ -71,7 +71,7 @@ class CLSHyperParams(LMHyperParams): def train_cls(self, num_lm_epochs, unfreeze=True, num_cls_frozen_epochs=1, bs=40, drop_mul_lm=0.3, drop_mul_cls=0.5, use_test_for_validation=False, num_cls_epochs=2, limit=None, noise=0.0, cls_max_len=20*70, lr_sched='layered', - label_smoothing_eps=0.0, random_init=False): + label_smoothing_eps=0.0, random_init=False, seed=None, dump_preds=None): assert use_test_for_validation == False, "use_test_for_validation=True is not supported" self.model_dir.mkdir(exist_ok=True, parents=True) @@ -82,11 +82,15 @@ class CLSHyperParams(LMHyperParams): if self.need_fine_tune_lm and not random_init: if not (self.model_dir/(ENC_BEST+".pth")).exists(): - self.train_lm(num_lm_epochs, data_lm=data_lm, drop_mult=drop_mul_lm, label_smoothing_eps=label_smoothing_eps) + self.train_lm(num_lm_epochs, data_lm=data_lm, drop_mult=drop_mul_lm, label_smoothing_eps=label_smoothing_eps, seed=seed) else: print("Language model already exist, skipping finetuning") + loss_func = CrossEntropyFlat(weight=torch.FloatTensor([0.5,30]).cuda()) learn = self.create_cls_learner(data_clas, drop_mult=drop_mul_cls, max_len=cls_max_len, - label_smoothing_eps=label_smoothing_eps, random_init=random_init) + label_smoothing_eps=label_smoothing_eps, random_init=random_init, + metrics=[FBeta(beta=1.0), f1_score, accuracy], + loss_func=loss_func) + if not random_init: try: learn.load('cls_best') @@ -96,6 +100,12 @@ class CLSHyperParams(LMHyperParams): else: print("Starting classifier from random weights") + if seed is not None: + print(f"Setting seed to {seed}") + torch.manual_seed(seed) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + np.random.seed(seed) if hasattr(self, 'lr_schedule_'+lr_sched): learn.true_wd = True @@ -105,22 +115,27 @@ class CLSHyperParams(LMHyperParams): print(f"Saving models at {learn.path / learn.model_dir}") learn.save('cls_last', with_opt=False) - learn.save('cls_best', with_opt=False) # we don't use early stopping for the time being + #learn.save('cls_best', with_opt=False) # we don't use early stopping for the time being del learn return self.validate_cls('cls_best', bs=bs, data_tst=data_tst, learn=None) - def validate_cls(self, save_name='cls_best', bs=40, data_tst=None, learn=None): + def validate_cls(self, save_name='cls_best', bs=40, data_tst=None, learn=None, dump_preds=None): if data_tst is None: _, _, data_tst = self.load_cls_data(bs) if learn is None: - learn = self.create_cls_learner(data_tst, drop_mult=0.3, metrics=[f1_score, accuracy]) + fbeta = FBeta(beta=1.0) + fbeta.on_train_begin() + learn = self.create_cls_learner(data_tst, drop_mult=0.3, metrics=[fbeta, f1_score, accuracy]) learn.unfreeze() learn.load(save_name) - print(data_tst.one_batch()[1]) - print(data_tst.one_batch()[1]) - print(data_tst.one_batch()[1]) + probs, targets = learn.get_preds(ordered=True) + preds = np.argmax(probs.cpu().numpy(), axis=1) + if dump_preds: + with open(dump_preds, 'w') as f: + f.write('\n'.join([str(x) for x in preds])) results = learn.validate(data_tst.valid_dl) - print(f"Loss and accuracy using ({save_name}):", results) + print(f"F1 score bin: {results[1].item()}") + print(f"Loss, f1_score, almost f1_score and accuracy using ({save_name}):", results) return list(map(float, results)) def create_cls_learner(self, data_clas, dps=None, label_smoothing_eps=0.0, random_init=False, **kwargs): @@ -140,7 +155,7 @@ class CLSHyperParams(LMHyperParams): learn.freeze() learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/cls-history"), - #partial(SaveModelCallback, every='improvement', name='cls_best') disabled due to memory issues + partial(SaveModelCallback, every='improvement', name='cls_best', monitor="f_beta") ] if label_smoothing_eps > 0.0: learn.loss_func = FlattenedLoss(LabelSmoothingCrossEntropy, eps=label_smoothing_eps)