From c4e0b7b3d652e8f788343b80f5f9df41a5afe0b0 Mon Sep 17 00:00:00 2001 From: "NAUSICAA\\Julian" Date: Sun, 25 Nov 2018 23:47:24 -0300 Subject: [PATCH 01/15] Lower memory footprint --- fastai_contrib/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py index 2dd21f9..8ae01cd 100644 --- a/fastai_contrib/utils.py +++ b/fastai_contrib/utils.py @@ -70,11 +70,11 @@ def get_sentencepiece(path:PathOrStr, trn_path:Path, name:str, pre_rules:ListRul pre_rules = pre_rules if pre_rules is not None else [] post_rules = post_rules if post_rules is not None else [] - # load the text frmo the train tokens file - text = [line.rstrip('\n') for line in open(trn_path)] - text = list(filter(None, text)) if not os.path.isfile(path / 'models' / 'spm.model') or not os.path.isfile(path / 'models' / f'itos_{name}.pkl'): + # load the text frmo the train tokens file + text = [line.rstrip('\n') for line in open(trn_path)] + text = list(filter(None, text)) raw_text = reduce(lambda t, rule: rule(t), pre_rules, '\n'.join(text)) raw_text_path = path / cache_name / 'all_text.txt' with open(raw_text_path, 'w') as f: @@ -337,7 +337,7 @@ def read_file(file_path, outname): with open(file_path, encoding='utf8') as f: text = f.readlines() df = pd.DataFrame( - {'text': np.array(text), 'labels': np.zeros(len(text))}, + {'text': text, 'labels': np.zeros(len(text))}, columns=['labels', 'text']) df.to_csv(file_path.parent / f'{outname}.csv', header=False, index=False) From 699093725f743b89f26055cd1f24d4dd89ce5432 Mon Sep 17 00:00:00 2001 From: "NAUSICAA\\Julian" Date: Mon, 26 Nov 2018 14:49:38 -0300 Subject: [PATCH 02/15] Script to merge two language token files --- ulmfit/merge_langs.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 ulmfit/merge_langs.py diff --git a/ulmfit/merge_langs.py b/ulmfit/merge_langs.py new file mode 100644 index 0000000..7fee7ed --- /dev/null +++ b/ulmfit/merge_langs.py @@ -0,0 +1,35 @@ +""" +Script to merge WikiText files created with `create_wikitext.py`. +""" +import fire +from pathlib import Path +from contextlib import ExitStack + +def merge_wikitext(paths, langs, dest_path, num_sentences): + wiki_paths = [Path(path) for path in paths] + for wiki_path in wiki_paths: + assert wiki_path.exists(), f'Error: {wiki_path} does not exist.' + dest_path = Path(dest_path) + dest_path.mkdir(exist_ok=True) + splits = ['train', 'valid', 'test'] + concat_langs = '-'.join(langs) + for split in splits: + with ExitStack() as stack: + files = [stack.enter_context(open( + wiki_path / f'{lang}.wiki.{split}.tokens', 'r', encoding='utf-8')) + for lang, wiki_path in zip(langs, wiki_paths)] + + output = stack.enter_context(open(dest_path / f'{concat_langs}.wiki.{split}.tokens', 'w', encoding='utf-8')) + done = False + while not done: + for file in files: + lines = [file.readline() for x in range(num_sentences)] + size = len(lines) + lines = [line for line in lines if line] + if len(lines) < size: + done = True + for line in lines: + output.write(line) + +if __name__ == '__main__': + fire.Fire(merge_wikitext) \ No newline at end of file From 807404196520054634f219910a991871c6e7b69a Mon Sep 17 00:00:00 2001 From: "NAUSICAA\\Julian" Date: Mon, 26 Nov 2018 21:24:19 -0300 Subject: [PATCH 03/15] Cache intermediate results --- ulmfit/pretrain_lm.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index aaca9e7..4d6ab9f 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -9,6 +9,7 @@ import fire from fastai import * from fastai.text import * +from fastai.callbacks.tracker import SaveModelCallback import torch from fastai_contrib.utils import read_file, read_whitespace_file, \ validate, PAD, UNK, get_sentencepiece @@ -69,14 +70,16 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo trn_path = dir_path / f'{lang}.wiki.train.tokens' val_path = dir_path / f'{lang}.wiki.valid.tokens' - read_file(trn_path, 'train') - read_file(val_path, 'valid') - - sp = get_sentencepiece(dir_path, trn_path, name, vocab_size=max_vocab) - lm_type = contrib_data.LanguageModelType.BiLM if bidir else contrib_data.LanguageModelType.FwdLM - - data_lm = TextLMDataBunch.from_csv(dir_path, 'train.csv', **sp, bs=bs, bptt=bptt, lm_type=lm_type) + try: + data_lm = TextLMDataBunch.load(dir_path, bs=bs, bptt=bptt, lm_type=lm_type) + print("Saved DataBunch loaded") + except FileNotFoundError: + read_file(trn_path, 'train') + read_file(val_path, 'valid') + sp = get_sentencepiece(dir_path, trn_path, name, vocab_size=max_vocab) + data_lm = TextLMDataBunch.from_csv(dir_path, 'train.csv', **sp, bs=bs, bptt=bptt, lm_type=lm_type) + data_lm.save(); itos = data_lm.train_ds.vocab.itos stoi = data_lm.train_ds.vocab.stoi else: @@ -139,7 +142,8 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo lm_learner = bilm_learner if bidir else language_model_learner learn = lm_learner(data_lm, bptt=bptt, emb_sz=emb_sz, nh=nh, nl=nl, pad_token=1, drop_mult=drop_mult, tie_weights=True, model_dir=model_dir.name, - bias=True, qrnn=qrnn, clip=0.12) + bias=True, qrnn=qrnn, clip=0.12, + callbacks=[SaveModelCallback(every='epoch')]) # compared to standard Adam, we set beta_1 to 0.8 learn.opt_fn = partial(optim.Adam, betas=(0.8, 0.99)) From 4d4ef1ae95e3dae2e126ca12931037f26cf11f38 Mon Sep 17 00:00:00 2001 From: "NAUSICAA\\Julian" Date: Mon, 26 Nov 2018 22:43:39 -0300 Subject: [PATCH 04/15] Change callback_fns --- ulmfit/pretrain_lm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 4d6ab9f..2396eaf 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -79,7 +79,7 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo read_file(val_path, 'valid') sp = get_sentencepiece(dir_path, trn_path, name, vocab_size=max_vocab) data_lm = TextLMDataBunch.from_csv(dir_path, 'train.csv', **sp, bs=bs, bptt=bptt, lm_type=lm_type) - data_lm.save(); + data_lm.save() itos = data_lm.train_ds.vocab.itos stoi = data_lm.train_ds.vocab.stoi else: @@ -143,7 +143,7 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo learn = lm_learner(data_lm, bptt=bptt, emb_sz=emb_sz, nh=nh, nl=nl, pad_token=1, drop_mult=drop_mult, tie_weights=True, model_dir=model_dir.name, bias=True, qrnn=qrnn, clip=0.12, - callbacks=[SaveModelCallback(every='epoch')]) + callbacks=[lambda lrn: SaveModelCallback(lrn, every='epoch')]) # compared to standard Adam, we set beta_1 to 0.8 learn.opt_fn = partial(optim.Adam, betas=(0.8, 0.99)) From bf33f222daaa2ae73a6aea08a1e4f614e34e1401 Mon Sep 17 00:00:00 2001 From: "NAUSICAA\\Julian" Date: Mon, 26 Nov 2018 22:48:42 -0300 Subject: [PATCH 05/15] callback > callback_fns --- ulmfit/pretrain_lm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 2396eaf..44dfa70 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -143,7 +143,7 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo learn = lm_learner(data_lm, bptt=bptt, emb_sz=emb_sz, nh=nh, nl=nl, pad_token=1, drop_mult=drop_mult, tie_weights=True, model_dir=model_dir.name, bias=True, qrnn=qrnn, clip=0.12, - callbacks=[lambda lrn: SaveModelCallback(lrn, every='epoch')]) + callback_fns=[lambda lrn: SaveModelCallback(lrn, every='epoch')]) # compared to standard Adam, we set beta_1 to 0.8 learn.opt_fn = partial(optim.Adam, betas=(0.8, 0.99)) From 3ef3be1b93d1d2f93349e8cb429111d0c440fbfb Mon Sep 17 00:00:00 2001 From: Julian Eisenschlos Date: Mon, 24 Dec 2018 17:02:37 -0300 Subject: [PATCH 06/15] Create train_xlingual_cls.py --- ulmfit/train_xlingual_cls.py | 79 ++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 ulmfit/train_xlingual_cls.py diff --git a/ulmfit/train_xlingual_cls.py b/ulmfit/train_xlingual_cls.py new file mode 100644 index 0000000..f383411 --- /dev/null +++ b/ulmfit/train_xlingual_cls.py @@ -0,0 +1,79 @@ +from dataclasses import dataclass +from ulmfit.train_clas import CLSHyperParams, MosesTokenizerFunc +from ulmfit.pretrain_lm import LMHyperParams, Tokenizers, ENC_BEST +from fastai.text import TextLMDataBunch, TextClasDataBunch, language_model_learner, text_classifier_learner +from fastai_contrib.utils import PAD, UNK, read_clas_data, PAD_TOKEN_ID, DATASETS, TRN, VAL, TST, ensure_paths_exists, \ + get_sentencepiece + +from typing import List +from pathlib import Path +import pandas as pd +import fire + +@dataclass +class XLingualCLSHyperParams(CLSHyperParams): + csv_name: str='train.csv' + target_paths: List[str] = None + + def __post_init__(self, *args, **kwargs): + super().__post_init__(*args, **kwargs) + self.target_paths = [] if self.target_paths is None else self.target_paths + + def load_cls_data(self, bs, force=False, use_test_for_validation=False, **kwargs): + if self.tokenizer is Tokenizers.SUBWORD: + args = get_sentencepiece(self.base_lm_path.parent, self.base_lm_path.parent / 'train.csv', + self.name, vocab_size=self.max_vocab, pre_rules=[], post_rules=[]) + elif self.tokenizer is Tokenizers.MOSES: + args = dict(tokenizer=Tokenizer(tok_func=MosesTokenizerFunc, lang='en', pre_rules=[], post_rules=[])) + elif self.tokenizer is Tokenizers.MOSES_FA: + args = dict(tokenizer=Tokenizer(tok_func=MosesTokenizerFunc, lang='en')) # use default pre/post rules + elif self.tokenizer is Tokenizers.FASTAI: + args = dict() + else: + raise ValueError( + f"self.tokenizer has wrong value {self.tokenizer}, Allowed values are taken from {Tokenizers}") + + src_path = self.dataset_path + csv_name = self.csv_name + tgt_paths = [Path(tgt_path) for tgt_path in self.target_paths] + mixed_csv = pd.read_csv(src_path / csv_name, header=None) + for tgt_path in tgt_paths: + mixed_csv = pd.concat([mixed_csv, pd.read_csv(tgt_path / csv_name, header=None)]) + + xcvs_name = ('x_' + csv_name) + mixed_csv.to_csv(src_path / xcvs_name, header=None, index=False) + + data_eval = [ + TextClasDataBunch.from_csv(path=tgt_path, csv_name=csv_name, **kwargs) + for tgt_path in tgt_paths + ] + + try: + if force: raise FileNotFoundError("Forcing reloading of caches") + data_lm = TextLMDataBunch.load(src_path, 'xlm', lm_type=self.lm_type, bs=bs) + print(f"Tokenized data loaded, xlm.trn {len(data_lm.train_ds)}, lm.val {len(data_lm.valid_ds)}") + except FileNotFoundError: + print(f"Running tokenization...") + data_lm = TextLMDataBunch.from_csv(path=src_path, csv_name=xcvs_name, bs=bs, lm_type=self.lm_type, **kwargs, **args) + print(f"Saving tokenized: cls.trn {len(data_lm.train_ds)}, cls.val {len(data_lm.valid_ds)}") + data_lm.save('xlm') + + try: + if force: raise FileNotFoundError("Forcing reloading of caches") + data_cls = TextClasDataBunch.load(src_path, 'cls', bs=bs) + print(f"Tokenized data loaded, cls.trn {len(data_cls.train_ds)}, cls.val {len(data_cls.valid_ds)}") + except FileNotFoundError: + args['vocab'] = data_lm.vocab # make sure we use the same vocab for classifcation + print(f"Running tokenization...") + data_cls = TextClasDataBunch.from_csv(path=src_path, csv_name=csv_name, bs=bs, **kwargs, **args) + + print(f"Saving tokenized: cls.trn {len(data_cls.train_ds)}, cls.val {len(data_cls.valid_ds)}") + data_cls.save('cls') + + print('Size of vocabulary:', len(data_lm.vocab.itos)) + print('First 20 words in vocab:', data_lm.vocab.itos[:20]) + return data_cls, data_lm # , data_eval + + +if __name__ == '__main__': + fire.Fire(XLingualCLSHyperParams) From 0436ba7ca1aee296ea4555435c0d644bc578c925 Mon Sep 17 00:00:00 2001 From: Julian Eisenschlos Date: Mon, 24 Dec 2018 17:09:22 -0300 Subject: [PATCH 07/15] Rename train_xlingual_cls.py to train_xlingual_clas.py --- ulmfit/{train_xlingual_cls.py => train_xlingual_clas.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename ulmfit/{train_xlingual_cls.py => train_xlingual_clas.py} (100%) diff --git a/ulmfit/train_xlingual_cls.py b/ulmfit/train_xlingual_clas.py similarity index 100% rename from ulmfit/train_xlingual_cls.py rename to ulmfit/train_xlingual_clas.py From d439f814d38fbd3c99e01bf8951a16db18c4f076 Mon Sep 17 00:00:00 2001 From: "NAUSICAA\\Julian" Date: Sat, 29 Dec 2018 17:02:14 -0300 Subject: [PATCH 08/15] Validate other languages --- ulmfit/train_xlingual_clas.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/ulmfit/train_xlingual_clas.py b/ulmfit/train_xlingual_clas.py index f383411..6d94ddf 100644 --- a/ulmfit/train_xlingual_clas.py +++ b/ulmfit/train_xlingual_clas.py @@ -19,7 +19,7 @@ class XLingualCLSHyperParams(CLSHyperParams): super().__post_init__(*args, **kwargs) self.target_paths = [] if self.target_paths is None else self.target_paths - def load_cls_data(self, bs, force=False, use_test_for_validation=False, **kwargs): + def get_tokenizer_args(self): if self.tokenizer is Tokenizers.SUBWORD: args = get_sentencepiece(self.base_lm_path.parent, self.base_lm_path.parent / 'train.csv', self.name, vocab_size=self.max_vocab, pre_rules=[], post_rules=[]) @@ -33,6 +33,10 @@ class XLingualCLSHyperParams(CLSHyperParams): raise ValueError( f"self.tokenizer has wrong value {self.tokenizer}, Allowed values are taken from {Tokenizers}") + return args + + def load_cls_data(self, bs, force=False, use_test_for_validation=False, **kwargs): + args = self.get_tokenizer_args() src_path = self.dataset_path csv_name = self.csv_name tgt_paths = [Path(tgt_path) for tgt_path in self.target_paths] @@ -42,11 +46,6 @@ class XLingualCLSHyperParams(CLSHyperParams): xcvs_name = ('x_' + csv_name) mixed_csv.to_csv(src_path / xcvs_name, header=None, index=False) - - data_eval = [ - TextClasDataBunch.from_csv(path=tgt_path, csv_name=csv_name, **kwargs) - for tgt_path in tgt_paths - ] try: if force: raise FileNotFoundError("Forcing reloading of caches") @@ -72,7 +71,20 @@ class XLingualCLSHyperParams(CLSHyperParams): print('Size of vocabulary:', len(data_lm.vocab.itos)) print('First 20 words in vocab:', data_lm.vocab.itos[:20]) - return data_cls, data_lm # , data_eval + return data_cls, data_lm + + def validate_cls(self, save_name='cls_last', bs=40): + args = self.get_tokenizer_args() + data_clas, data_lm = self.load_cls_data_full(bs, use_test_for_validation=True) + data_eval = [ + TextClasDataBunch.from_csv(path=Path(tgt_path), csv_name=self.csv_name, **args) + for tgt_path in self.target_paths + ] + + for data in [data_clas] + data_eval: + learn = self.create_cls_learner(data, drop_mult=0.1) + learn.load(save_name) + print(f"Loss and accuracy using ({save_name}) for dataset at {data.path}:", learn.validate()) if __name__ == '__main__': From 30565a2148b4cc0e3563e795d2588dea393ff544 Mon Sep 17 00:00:00 2001 From: Julian Eisenschlos Date: Sat, 29 Dec 2018 20:09:46 +0000 Subject: [PATCH 09/15] Remove bad imports --- fastai_contrib/learner.py | 4 ++-- fastai_contrib/utils.py | 1 - ulmfit/train_clas.py | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/fastai_contrib/learner.py b/fastai_contrib/learner.py index 9699c41..6775e00 100644 --- a/fastai_contrib/learner.py +++ b/fastai_contrib/learner.py @@ -1,6 +1,6 @@ from torch.nn import CrossEntropyLoss -from fastai import GradientClipping, accuracy +from fastai.metrics import accuracy from fastai.callbacks import * from fastai.basic_data import * from fastai.datasets import untar_data @@ -108,4 +108,4 @@ def convert_weights_with_prefix(wgts:Weights, stoi_wgts:Dict[str,int], itos_new: import fastai.text.learner fastai.text.learner.convert_weights = convert_weights -#endregion \ No newline at end of file +#endregion diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py index ce5cb31..62cc97e 100644 --- a/fastai_contrib/utils.py +++ b/fastai_contrib/utils.py @@ -4,7 +4,6 @@ Utility methods for data processing. import pandas as pd import numpy as np import fire -from fastai import F, to_device import torch from tqdm import tqdm import re diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 627cb09..fafb573 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -14,7 +14,6 @@ from fastai.text import * import torch from fastai.text import TextLMDataBunch, TextClasDataBunch, language_model_learner, text_classifier_learner -from fastai import fit_one_cycle, accuracy from fastai_contrib.data import LanguageModelType from fastai_contrib.learner import bilm_text_classifier_learner, bilm_learner, accuracy_fwd, accuracy_bwd from fastai_contrib.utils import PAD, UNK, read_clas_data, PAD_TOKEN_ID, DATASETS, TRN, VAL, TST, ensure_paths_exists, \ From b78a2ebaada287d5191c71df89bcedd2cac90428 Mon Sep 17 00:00:00 2001 From: "NAUSICAA\\Julian" Date: Sat, 29 Dec 2018 17:12:38 -0300 Subject: [PATCH 10/15] Fixing typo --- ulmfit/train_xlingual_clas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ulmfit/train_xlingual_clas.py b/ulmfit/train_xlingual_clas.py index 6d94ddf..12a7ea7 100644 --- a/ulmfit/train_xlingual_clas.py +++ b/ulmfit/train_xlingual_clas.py @@ -75,7 +75,7 @@ class XLingualCLSHyperParams(CLSHyperParams): def validate_cls(self, save_name='cls_last', bs=40): args = self.get_tokenizer_args() - data_clas, data_lm = self.load_cls_data_full(bs, use_test_for_validation=True) + data_clas, data_lm = self.load_cls_data(bs, use_test_for_validation=True) data_eval = [ TextClasDataBunch.from_csv(path=Path(tgt_path), csv_name=self.csv_name, **args) for tgt_path in self.target_paths From 6780559b46b76aa8eac19658309f0b1d9e2ed332 Mon Sep 17 00:00:00 2001 From: "NAUSICAA\\Julian" Date: Sat, 29 Dec 2018 17:13:38 -0300 Subject: [PATCH 11/15] Dataset name --- ulmfit/train_xlingual_clas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ulmfit/train_xlingual_clas.py b/ulmfit/train_xlingual_clas.py index 12a7ea7..b32a715 100644 --- a/ulmfit/train_xlingual_clas.py +++ b/ulmfit/train_xlingual_clas.py @@ -50,7 +50,7 @@ class XLingualCLSHyperParams(CLSHyperParams): try: if force: raise FileNotFoundError("Forcing reloading of caches") data_lm = TextLMDataBunch.load(src_path, 'xlm', lm_type=self.lm_type, bs=bs) - print(f"Tokenized data loaded, xlm.trn {len(data_lm.train_ds)}, lm.val {len(data_lm.valid_ds)}") + print(f"Tokenized data loaded, xlm.trn {len(data_lm.train_ds)}, xlm.val {len(data_lm.valid_ds)}") except FileNotFoundError: print(f"Running tokenization...") data_lm = TextLMDataBunch.from_csv(path=src_path, csv_name=xcvs_name, bs=bs, lm_type=self.lm_type, **kwargs, **args) From 0856d33fa854fa1b15738e0175758931eaebc16a Mon Sep 17 00:00:00 2001 From: "NAUSICAA\\Julian" Date: Thu, 3 Jan 2019 09:35:17 -0300 Subject: [PATCH 12/15] Merge issues --- ulmfit/pretrain_lm.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 6495155..c9d3b47 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -12,7 +12,6 @@ import fire from fastai import * from fastai.callbacks import CSVLogger, SaveModelCallback from fastai.text import * -from fastai.callbacks.tracker import SaveModelCallback import torch from fastai_contrib.utils import read_file, read_whitespace_file, \ validate, PAD, UNK, get_sentencepiece, read_clas_data, TRN, VAL, TST, PAD_TOKEN_ID, MosesTokenizerFunc, \ From 39d38f0f46070726482ee7edbda71b9abd212f17 Mon Sep 17 00:00:00 2001 From: "NAUSICAA\\Julian" Date: Thu, 3 Jan 2019 09:40:31 -0300 Subject: [PATCH 13/15] Restore newline --- fastai_contrib/learner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastai_contrib/learner.py b/fastai_contrib/learner.py index 93d51b1..67e79cb 100644 --- a/fastai_contrib/learner.py +++ b/fastai_contrib/learner.py @@ -110,4 +110,4 @@ def convert_weights_with_prefix(wgts:Weights, stoi_wgts:Dict[str,int], itos_new: import fastai.text.learner fastai.text.learner.convert_weights = convert_weights -#endregion +#endregion \ No newline at end of file From 0e2e058b827b23c91999a48dfe105c8ffe2a57bf Mon Sep 17 00:00:00 2001 From: "NAUSICAA\\Julian" Date: Thu, 3 Jan 2019 17:02:56 -0300 Subject: [PATCH 14/15] Simplify finetuning using new helper methods --- ulmfit/train_xlingual_clas.py | 27 ++++----------------------- 1 file changed, 4 insertions(+), 23 deletions(-) diff --git a/ulmfit/train_xlingual_clas.py b/ulmfit/train_xlingual_clas.py index b32a715..0381973 100644 --- a/ulmfit/train_xlingual_clas.py +++ b/ulmfit/train_xlingual_clas.py @@ -1,9 +1,6 @@ from dataclasses import dataclass -from ulmfit.train_clas import CLSHyperParams, MosesTokenizerFunc -from ulmfit.pretrain_lm import LMHyperParams, Tokenizers, ENC_BEST -from fastai.text import TextLMDataBunch, TextClasDataBunch, language_model_learner, text_classifier_learner -from fastai_contrib.utils import PAD, UNK, read_clas_data, PAD_TOKEN_ID, DATASETS, TRN, VAL, TST, ensure_paths_exists, \ - get_sentencepiece +from ulmfit.train_clas import CLSHyperParams +from fastai.text import TextLMDataBunch, TextClasDataBunch from typing import List from pathlib import Path @@ -19,24 +16,8 @@ class XLingualCLSHyperParams(CLSHyperParams): super().__post_init__(*args, **kwargs) self.target_paths = [] if self.target_paths is None else self.target_paths - def get_tokenizer_args(self): - if self.tokenizer is Tokenizers.SUBWORD: - args = get_sentencepiece(self.base_lm_path.parent, self.base_lm_path.parent / 'train.csv', - self.name, vocab_size=self.max_vocab, pre_rules=[], post_rules=[]) - elif self.tokenizer is Tokenizers.MOSES: - args = dict(tokenizer=Tokenizer(tok_func=MosesTokenizerFunc, lang='en', pre_rules=[], post_rules=[])) - elif self.tokenizer is Tokenizers.MOSES_FA: - args = dict(tokenizer=Tokenizer(tok_func=MosesTokenizerFunc, lang='en')) # use default pre/post rules - elif self.tokenizer is Tokenizers.FASTAI: - args = dict() - else: - raise ValueError( - f"self.tokenizer has wrong value {self.tokenizer}, Allowed values are taken from {Tokenizers}") - - return args - def load_cls_data(self, bs, force=False, use_test_for_validation=False, **kwargs): - args = self.get_tokenizer_args() + args = self.tokenzier_to_fastai_args(trn_data_loading_func=lambda: trn_df[1], add_moses=True) src_path = self.dataset_path csv_name = self.csv_name tgt_paths = [Path(tgt_path) for tgt_path in self.target_paths] @@ -74,7 +55,7 @@ class XLingualCLSHyperParams(CLSHyperParams): return data_cls, data_lm def validate_cls(self, save_name='cls_last', bs=40): - args = self.get_tokenizer_args() + args = self.tokenzier_to_fastai_args(trn_data_loading_func=lambda: trn_df[1], add_moses=True) data_clas, data_lm = self.load_cls_data(bs, use_test_for_validation=True) data_eval = [ TextClasDataBunch.from_csv(path=Path(tgt_path), csv_name=self.csv_name, **args) From 866d0c0bed8bb4ef82f6f84d7a3f14f4a7802fdf Mon Sep 17 00:00:00 2001 From: "NAUSICAA\\Julian" Date: Tue, 12 Feb 2019 12:52:12 -0300 Subject: [PATCH 15/15] Addiing XLingual LM --- ulmfit/pretrain_xlingual_lm.py | 109 +++++++++++++++++++++++++++++++++ ulmfit/train_xlingual_clas.py | 3 +- 2 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 ulmfit/pretrain_xlingual_lm.py diff --git a/ulmfit/pretrain_xlingual_lm.py b/ulmfit/pretrain_xlingual_lm.py new file mode 100644 index 0000000..803989b --- /dev/null +++ b/ulmfit/pretrain_xlingual_lm.py @@ -0,0 +1,109 @@ +from dataclasses import dataclass +from ulmfit.train_clas import LMHyperParams +from fastai.text import TextLMDataBunch, TextClasDataBunch +from fastai.basic_train import LearnerCallback +from fastai.torch_core import PBar, Rank0Tensor +from torch import nn, Tensor + +from typing import List, Collection, Any +from pathlib import Path +import pandas as pd +import fire +import random + +@dataclass +class ParallelAlignmentCallback(LearnerCallback): + "A `LearnerCallback` that adds parallel alignment between sentences." + + data_src:TextClasDataBunch + data_tgt:TextClasDataBunch + alpha:float=0.1 + + def __post_init__(self): + self.bs = self.data_src.bs + self.loss = nn.CosineEmbeddingLoss(margin=0.5) + self.ones = torch.cat((torch.ones(self.bs), -torch.ones(self.bs))) + + def pool(self, x:Tensor, bs:int, is_max:bool): + "Pool the tensor along the seq_len dimension." + f = F.adaptive_max_pool1d if is_max else F.adaptive_avg_pool1d + return f(x.transpose(1,2), (1,)).view(bs,-1) + + def get_representation(batch): + last_output = self.learn.model(batch) + output = last_output[1][-1] + bs,sl,_ = output.size() + avgpool = self.pool(output, bs, False) + mxpool = self.pool(output, bs, True) + return torch.cat([output[:,-1], mxpool, avgpool], 1) + + def on_train_begin(self, pbar:PBar, metrics_names:Collection[str], **kwargs:Any)->None: + self.counter = 0 + + def on_backward_begin(self, last_loss:Rank0Tensor, last_input:Tensor, **kwargs): + "Adjust the loss by adding similarity of parallel sentences" + src_rep = self.get_representation(data_src.train_ds[self.counter]) + tgt_rep = self.get_representation(data_tgt.train_ds[self.counter]) + + offset = -random.randrange(1, self.bs) + + src_rep = torch.cat((src_rep, src_rep)) + tgt_rep = torch.cat((tgt_rep, tgt_rep[range(offset, self.bs + offset)])) + + parallel_loss = self.alpha * self.loss(src_rep, tgt_rep, self.y) + + self.counter += 1 + self.counter %= len(data_src.train_ds) + return last_loss + parallel_loss + + +@dataclass +class XLingualLMHyperParams(LMHyperParams): + + parallel_data_path: str=None + parallel_data_bs: int=32 + src_lang: str=None + tgt_lang: str=None + + def create_lm_learner(self, data_lm, dps=None, **kwargs): + learner = super().create_lm_learner(data_lm, dps, **kwargs) + if self.parallel_data_path is not None: + src_trn_df = pd.read_csv(self.parallel_data_path / self.src_lang / 'train.csv', header=None) + tgt_trn_df = pd.read_csv(self.parallel_data_path / self.tgt_lang / 'train.csv', header=None) + bs = self.parallel_data_bs + data_src = TextClasDataBunch.from_df(path=self.cache_dir, train_df=src_trn_df, lm_type=self.lm_type, bs=bs) + data_tgt = TextClasDataBunch.from_df(path=self.cache_dir, train_df=tgt_trn_df, lm_type=self.lm_type, bs=bs) + learner.callback_fns = [ + partial(ParallelAlignmentCallback, data_src=data_src, data_tgt=data_tgt) + ] + learner.callback_fns + + def load_wiki_data(self, bs=70): + trn_path = self.dataset_path / f'{self.lang}.wiki.train.tokens' + val_path = self.dataset_path / f'{self.lang}.wiki.valid.tokens' + tst_path = self.dataset_path / f'{self.lang}.wiki.test.tokens' + for path_ in [trn_path, val_path, tst_path]: + assert path_.exists(), f'Error: {path_} does not exist.' + + args = self.tokenzier_to_fastai_args(trn_data_loading_func=self.load_train_text, add_moses=False) + try: + data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type, bs=bs) + print("Tokenized data loaded") + except FileNotFoundError: + print("Running tokenization") + data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=read_wiki_articles(trn_path), + valid_df=read_wiki_articles(val_path), + classes=None, lm_type=self.lm_type, max_vocab=self.max_vocab, + bs=bs, text_cols='texts', **args) + data_lm.save('.') + + itos, stoi, trn_path = data_lm.vocab.itos, data_lm.vocab.stoi, data_lm.path + print('Size of vocabulary:', len(itos)) + print('First 20 words in vocab:', data_lm.vocab.itos[:20]) + return data_lm + +if __name__ == '__main__': + fire.Fire(XLingualLMHyperParams) + + +# python -m ulmfit.XLingualLMHyperParams --dataset-path data/wiki/wikitext-103 --bidir=True --qrnn=True --nl=4 --tokenizer=sp --name 'nl4' --bs 120 --cuda-id 0 - train 10 --drop-mult=0 --bs 40 + diff --git a/ulmfit/train_xlingual_clas.py b/ulmfit/train_xlingual_clas.py index 0381973..d30f89e 100644 --- a/ulmfit/train_xlingual_clas.py +++ b/ulmfit/train_xlingual_clas.py @@ -10,7 +10,8 @@ import fire @dataclass class XLingualCLSHyperParams(CLSHyperParams): csv_name: str='train.csv' - target_paths: List[str] = None + target_paths: List[str]=None + parallel_data_path: str=None def __post_init__(self, *args, **kwargs): super().__post_init__(*args, **kwargs)