From e6407fe0c914a9626a012a3c863a4dd78875b457 Mon Sep 17 00:00:00 2001 From: Tomasz Pietruszka Date: Thu, 10 Jan 2019 00:45:22 +0100 Subject: [PATCH 01/13] Added the param and model type for BwdLM --- ulmfit/pretrain_lm.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index c9d3b47..3f70319 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -56,6 +56,7 @@ class LMHyperParams: dataset_path: str # data_dir base_lm_path: str = None + backwards: str = False bidir: bool =False qrnn: bool = True max_vocab: int = 60000 @@ -77,6 +78,8 @@ class LMHyperParams: cuda_id: InitVar[int] = 0 def __post_init__(self, cuda_id): + if self.bidir and self.backwards: + raise ValueError('Both "backwards" and "bidir" options cannot be enabled at the same time') if not torch.cuda.is_available(): print('CUDA not available. Setting device=-1.') cuda_id = -1 @@ -101,7 +104,16 @@ class LMHyperParams: def tokenizer_prefix(self): return f"{self.tokenizer.value}{self.max_vocab // 1000}k" @property - def model_prefix(self): return ('bi' if self.bidir else '') + ('qrnn' if self.qrnn else 'lstm') + def model_direction(self): + if self.bidir: + return 'bi' + if self.backwards: + return 'bwd' + else: + return '' + + @property + def model_prefix(self): return self.model_direction + ('qrnn' if self.qrnn else 'lstm') @property def model_name(self): return f"{self.model_prefix}_{self.name}.m" @@ -111,7 +123,12 @@ class LMHyperParams: @property def lm_type(self): - return contrib_data.LanguageModelType.BiLM if self.bidir else contrib_data.LanguageModelType.FwdLM + if self.bidir: + return contrib_data.LanguageModelType.BiLM + if self.backwards: + return contrib_data.LanguageModelType.BwdLM + else: + return contrib_data.LanguageModelType.FwdLM def tokenzier_to_fastai_args(self, trn_data_loading_func, add_moses): tok_func = MosesTokenizerFunc if add_moses else BaseTokenizer From 593c5661bf0347f8aa65cf6286c5101c8eb7fe19 Mon Sep 17 00:00:00 2001 From: Tomasz Pietruszka Date: Sun, 13 Jan 2019 17:26:16 +0100 Subject: [PATCH 02/13] Added alpha and beta params for RNNTrainer --- ulmfit/pretrain_lm.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index c9d3b47..90636a8 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -71,6 +71,9 @@ class LMHyperParams: dps = (0.25, 0.1, 0.2, 0.02, 0.15) # consider removing dps & clip from the default hyperparams and put them to train clip: float = 0.12 bptt: int = 70 + # alpha and beta - defaults like in fastai/text/learner.py:RNNLearner() + rnn_alpha: float = 2 # activation regularization (AR) + rnn_beta: float = 1 # temporal activation regularization (TAR) lang: str = 'en' name: str = None @@ -183,7 +186,8 @@ class LMHyperParams: trn_args = dict(tie_weights=True, clip=self.clip, bptt=self.bptt, pretrained_fnames=self.pretrained_fnames, - pretrained_model=self.pretrained_model) + pretrained_model=self.pretrained_model, + alpha=self.rnn_alpha, beta=self.rnn_beta) trn_args.update(kwargs) print ("Training args: ", trn_args, "dps: ", dps or self.dps) learn = lm_learner(data_lm, emb_sz=self.emb_sz, nh=self.nh, nl=self.nl, pad_token=PAD_TOKEN_ID, From a1e66c39d4b1e27db776fa934b737f4b1c75062a Mon Sep 17 00:00:00 2001 From: Tomasz Pietruszka Date: Sun, 13 Jan 2019 17:27:29 +0100 Subject: [PATCH 03/13] tokenzier->tokenizer typo --- ulmfit/pretrain_lm.py | 4 ++-- ulmfit/train_clas.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 90636a8..a4b68e9 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -116,7 +116,7 @@ class LMHyperParams: def lm_type(self): return contrib_data.LanguageModelType.BiLM if self.bidir else contrib_data.LanguageModelType.FwdLM - def tokenzier_to_fastai_args(self, trn_data_loading_func, add_moses): + def tokenizer_to_fastai_args(self, trn_data_loading_func, add_moses): tok_func = MosesTokenizerFunc if add_moses else BaseTokenizer if self.tokenizer is Tokenizers.SUBWORD: if self.base_lm_path: # ensure we are using the same sentence piece model @@ -211,7 +211,7 @@ class LMHyperParams: for path_ in [trn_path, val_path, tst_path]: assert path_.exists(), f'Error: {path_} does not exist.' - args = self.tokenzier_to_fastai_args(trn_data_loading_func=self.load_train_text, add_moses=False) + args = self.tokenizer_to_fastai_args(trn_data_loading_func=self.load_train_text, add_moses=False) try: data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type, bs=bs) print("Tokenized data loaded") diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 0f9221f..825d1f0 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -125,7 +125,7 @@ class CLSHyperParams(LMHyperParams): trn_df, val_df = trn_df[:trn_len], trn_df[trn_len:] cls_cache = '.' - args = self.tokenzier_to_fastai_args(trn_data_loading_func=lambda: trn_df[1], add_moses=True) + args = self.tokenizer_to_fastai_args(trn_data_loading_func=lambda: trn_df[1], add_moses=True) try: if force: raise FileNotFoundError("Forcing reloading of caches") From b00410e0cb91a152506ebfc9eedbae007634c4f6 Mon Sep 17 00:00:00 2001 From: Tomasz Pietruszka Date: Sun, 13 Jan 2019 17:28:39 +0100 Subject: [PATCH 04/13] LM save with_opt fix --- ulmfit/pretrain_lm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index a4b68e9..7909e3b 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -172,7 +172,7 @@ class LMHyperParams: learn.unfreeze() if not learn.true_wd: learn.fit_one_cycle(num_epochs, lr, (0.8, 0.7), wd=1e-7) else: learn.fit_one_cycle(num_epochs, lr, (0.8, 0.7)) # TODO find proper values - learn.save("lm_best_with_opt", with_opt=False) + learn.save("lm_best_with_opt", with_opt=True) learn.save_encoder(ENC_BEST) learn.save(LM_BEST, with_opt=False) print(learn.path) From 5ba8ae4c59a72813ce4338c983cd087cacb25b02 Mon Sep 17 00:00:00 2001 From: Tomasz Pietruszka Date: Sun, 13 Jan 2019 17:29:55 +0100 Subject: [PATCH 05/13] non-ascii char removed from code. Caused display bugs --- ulmfit/postprocess_wikitext.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ulmfit/postprocess_wikitext.py b/ulmfit/postprocess_wikitext.py index 7ae8774..481631c 100644 --- a/ulmfit/postprocess_wikitext.py +++ b/ulmfit/postprocess_wikitext.py @@ -53,7 +53,7 @@ def limit_vocab(unk_path, vocab): tokens = [''] + tokens line = ' '.join(tokens) f_out.write(line) - print(f'{unk_path.name}. # of tokens: {total_num_tokens}') + print(f'{unk_path.name}. # of tokens: {total_num_tokens}') temp_file_path.replace(unk_path) @@ -101,5 +101,6 @@ def postprocess_wikitext(path, lang): unk_path = dest_path / f'{lang}.wiki.{split}.tokens' limit_vocab(unk_path, vocab) + if __name__ == '__main__': - fire.Fire(postprocess_wikitext) \ No newline at end of file + fire.Fire(postprocess_wikitext) From dd296f30883b09c566239e7bf63a8c46069fa123 Mon Sep 17 00:00:00 2001 From: Tomasz Pietruszka Date: Sun, 13 Jan 2019 17:30:35 +0100 Subject: [PATCH 06/13] prepare_wiki.sh made executable --- prepare_wiki.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 prepare_wiki.sh diff --git a/prepare_wiki.sh b/prepare_wiki.sh old mode 100644 new mode 100755 From 0e864fb29140a765ee20bbe7ea8afaec9df25d88 Mon Sep 17 00:00:00 2001 From: Tomasz Pietruszka Date: Tue, 22 Jan 2019 20:10:38 +0100 Subject: [PATCH 07/13] test_end_to_end now working, added instructions to README, sentencepiece dependency --- README.md | 10 ++++++++++ prepare_imdb.sh | 0 requirements.txt | 3 ++- tests/test_end_to_end.py | 3 ++- 4 files changed, 14 insertions(+), 2 deletions(-) mode change 100644 => 100755 prepare_imdb.sh diff --git a/README.md b/README.md index a04fca8..41cd8b6 100644 --- a/README.md +++ b/README.md @@ -88,3 +88,13 @@ $ git push --set-upstream n-waves ulmfit_multilingual # to automatically push u - `bilm` -- scripts to train biLM ELMo style, Bert style - `class` -- scripts to test classifiers on multiple languages - `xnli` -- scripts to test nli + + +## Running tests + +To run the tests, the following data is necessary: + +- wikitext-2 (prepared by `./prepare_wiki-en.sh`, along with wikitext-103) +- imdb (prepared by `./prepare_imdb.sh`) + +then simply run tests, e.g. `pytest .` diff --git a/prepare_imdb.sh b/prepare_imdb.sh old mode 100644 new mode 100755 diff --git a/requirements.txt b/requirements.txt index eb003e9..b211a96 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ fire>=0.1.3 cupy>=5.0.0 scikit-learn>=0.20 -sacremoses>=0.0.5 \ No newline at end of file +sacremoses>=0.0.5 +sentencepiece diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index 3d4d112..266bc3f 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -25,7 +25,8 @@ def get_test_data(): imdb = data / "imdb" test_data = data / "test" - shutil.rmtree(test_data) + if test_data.exists(): + shutil.rmtree(test_data) test_wt = test_data / 'wikitext-s' test_imdb = test_data / 'imdb' From 386fc49431162993c1e0aade2856cf6ed2ff1321 Mon Sep 17 00:00:00 2001 From: Tomasz Pietruszka Date: Tue, 22 Jan 2019 20:31:30 +0100 Subject: [PATCH 08/13] Adapted test_text_data to batch dimension being the first, and only xxbos token at the start --- experiments/cls_test_wt103_1_f.ipynb | 6 +++--- tests/test_text_data.py | 23 ++++++++++------------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/experiments/cls_test_wt103_1_f.ipynb b/experiments/cls_test_wt103_1_f.ipynb index 26a6e2e..ee3fd22 100644 --- a/experiments/cls_test_wt103_1_f.ipynb +++ b/experiments/cls_test_wt103_1_f.ipynb @@ -714,9 +714,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python [conda env:fastaiv1]", + "display_name": "fastai-dev", "language": "python", - "name": "conda-env-fastaiv1-py" + "name": "fastai-dev" }, "language_info": { "codemirror_mode": { @@ -728,7 +728,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.0" + "version": "3.6.8" } }, "nbformat": 4, diff --git a/tests/test_text_data.py b/tests/test_text_data.py index 6b3dabf..1b96079 100644 --- a/tests/test_text_data.py +++ b/tests/test_text_data.py @@ -22,38 +22,35 @@ def test_should_load_backwards_lm(): df = text_df(['neg','pos']) data = TextLMDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=0, text_cols=["text"], bs=2, - lm_type=contrib_data.LanguageModelType.BwdLM, - ld_cls=contrib_data.LanguageModelLoader) + lm_type=contrib_data.LanguageModelType.BwdLM) lml = data.train_dl.dl lml.data = lml.batchify(np.concatenate([lml.dataset.x.items[i] for i in range(len(lml.dataset))])) batch = lml.get_batch(lml.data, 0, 70) - assert batch[0].shape == (70, lml.bs) + assert batch[0].shape == (lml.bs, 70) assert batch[1].shape == (70*lml.bs,) - - as_text = [lml.dataset.vocab.itos[x] for x in batch[0][:,0]] - np.testing.assert_array_equal(as_text[:5], ["world", "hello", '1', 'xxfld', 'project',]) + as_text = [lml.dataset.vocab.itos[x] for x in batch[0][0]] + np.testing.assert_array_equal(as_text[:5], ["world", "hello", 'xxbos', 'project', 'cool']) def test_should_load_bi_lm(): path = untar_data(URLs.IMDB_SAMPLE) df = text_df(['neg', 'pos']) data = TextLMDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=0, text_cols=["text"], bs=2, - lm_type=contrib_data.LanguageModelType.BiLM, - ld_cls=contrib_data.LanguageModelLoader) + lm_type=contrib_data.LanguageModelType.BiLM) lml = data.train_dl.dl lml.data = lml.batchify(np.concatenate([lml.dataset.x.items[i] for i in range(len(lml.dataset))])) batch = lml.get_batch(lml.data, 0, 70) - assert batch[0].shape == (70, lml.bs, 2) + assert batch[0].shape == (lml.bs, 70, 2) assert batch[1].shape == (70*lml.bs, 2) - as_text = [lml.dataset.vocab.itos[x] for x in batch[0][:, 0, 0]] - np.testing.assert_array_equal(as_text[:7], "xxfld 1 fast ai is a cool".split()) + as_text = [lml.dataset.vocab.itos[x] for x in batch[0][0, :, 0]] + np.testing.assert_array_equal(as_text[:7], "xxbos fast ai is a cool project".split()) - as_text = [lml.dataset.vocab.itos[x] for x in batch[0][:,0,1]] - np.testing.assert_array_equal(as_text[:5], ["world", "hello", '1', 'xxfld', 'project',]) + as_text = [lml.dataset.vocab.itos[x] for x in batch[0][0, :, 1]] + np.testing.assert_array_equal(as_text[:5], ["world", "hello", 'xxbos', 'project', 'cool']) ###################### NEW CODE From c6cf44a7b9154b54e46eabe677d09ed03c0e12df Mon Sep 17 00:00:00 2001 From: Tomasz Pietruszka Date: Wed, 23 Jan 2019 00:05:38 +0100 Subject: [PATCH 09/13] Fixed test_bilm_classifier_loads_encoder - inconsistent settings, problematic input (too short) --- tests/test_text_train.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/test_text_train.py b/tests/test_text_train.py index 0103939..7c3d894 100644 --- a/tests/test_text_train.py +++ b/tests/test_text_train.py @@ -42,7 +42,7 @@ def learn(): def text_df(n_labels): data = [] - texts = ["fast ai is a cool project", "hello world"] + texts = ["fast ai is a cool project", "hello world"] * 20 for ind, text in enumerate(texts): sample = {} for label in range(n_labels): sample[label] = ind%2 @@ -58,19 +58,21 @@ def test_val_loss(learn): def test_bilm_classifier_loads_encoder(): - n_labels=2 + n_labels=1 + nl = 1 + emb_sz = 100 path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data', 'tmp') os.makedirs(path) try: - df = text_df(n_labels=1) + df = text_df(n_labels=n_labels) lmdf = df#[["text"]] print(lmdf.head()) lmdata = TextLMDataBunch.from_df(path, lmdf, lmdf, tokenizer=Tokenizer(BaseTokenizer), lm_type=contrib_data.LanguageModelType.BiLM) - learn = bilm_learner(lmdata, emb_sz=100, nl=1, drop_mult=0.1, qrnn=False) + learn = bilm_learner(lmdata, emb_sz=emb_sz, nl=nl, drop_mult=0.1, qrnn=False) learn.save_encoder("enc") - data = TextClasDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=list(range(n_labels)), text_cols=["text"]) - classifier = bilm_text_classifier_learner(data, emb_sz=100, nl=1, drop_mult=0.1, qrnn=False) + data = TextClasDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=list(range(n_labels)), text_cols=["text"], bs=8) + classifier = bilm_text_classifier_learner(data, emb_sz=emb_sz, nl=nl, drop_mult=0.1, qrnn=False) print(last_layer(classifier.model), ) classifier.load_encoder("enc") classifier.fit(1) From a1e7a79b5718869fff97a834560b11ac2b46dc0a Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Tue, 12 Feb 2019 15:00:29 +0100 Subject: [PATCH 10/13] Add result logs --- results/logs/de.md | 30 +++++++++++++++++ results/logs/es.md | 81 ++++++++++++++++++++++++++++++++++++++++++++++ results/logs/ja.md | 61 ++++++++++++++++++++++++++++++++++ results/logs/ru.md | 78 ++++++++++++++++++++++++++++++++++++++++++++ results/logs/zh.md | 6 ++++ 5 files changed, 256 insertions(+) create mode 100644 results/logs/es.md create mode 100644 results/logs/ru.md create mode 100644 results/logs/zh.md diff --git a/results/logs/de.md b/results/logs/de.md index 89b88b2..dc44577 100644 --- a/results/logs/de.md +++ b/results/logs/de.md @@ -483,3 +483,33 @@ epoch train_loss valid_loss accuracy Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-noise0.4.m Loss and accuracy using (cls_last): [0.62477165, tensor(0.7717)] ``` +#### 15% +``` +python -m ulmfit cls --dataset-path data/mldoc/de-1 --base-lm-path data/mldoc/de-1/models/sp30k/lstm_nl4.m --lang=de --name 'nl4-noise0.15' --cuda-id=1 - train 0 --bs 40 --noise=0.15 --num-cls-epochs=2 --drop-mult-cls=0.2 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-noise0.15.m +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/de.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Added noise to 150 examples, only 0.85 have correct labels +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-noise0.15.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.836104 0.584330 0.897000 +epoch train_loss valid_loss accuracy +1 0.692108 0.303470 0.930000 +epoch train_loss valid_loss accuracy +1 0.653277 0.330520 0.924000 +epoch train_loss valid_loss accuracy +1 0.541086 0.331944 0.922000 +2 0.523274 0.335986 0.922000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-noise0.15.m +Loss and accuracy using (cls_last): [0.28749043, tensor(0.9355)] +``` \ No newline at end of file diff --git a/results/logs/es.md b/results/logs/es.md new file mode 100644 index 0000000..d11db90 --- /dev/null +++ b/results/logs/es.md @@ -0,0 +1,81 @@ +```` + + +python -m ulmfit lm --dataset-path data/wiki/es-100 --cuda-id=0 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 30000 --lang es --qrnn=False - train 10 --bs=50 --drop_mult=0 +Running tokenization +Wiki text was split to 96224 articles +Wiki text was split to 105 articles +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': None, 'pretrained_model': None, 'drop_mult': 0} dps: [0.25 0.1 0.2 0.02 0.15] +Training lm from random weights +epoch train_loss valid_loss accuracy +1 3.269541 3.451855 0.387471 +2 3.161740 3.423016 0.386158 +3 3.187431 3.419638 0.388626 +4 3.115763 3.357066 0.393877 +5 2.996527 3.291787 0.402488 +6 3.021759 3.202183 0.410873 +7 2.998267 3.104373 0.422624 +8 2.827225 3.006537 0.436010 +9 2.784576 2.937735 0.446654 +10 2.789913 2.918509 0.450055 +data/wiki/es-100/models/sp30k +Saving info data/wiki/es-100/models/sp30k/lstm_nl4.m/info.json +```` + +### MLDoc + +``` +python -m ulmfit cls --dataset-path data/mldoc/es-1 --base-lm-path data/wiki/es-100/models/sp30k/lstm_nl4.m --lang=es --name 'nl4' --cuda-id=1 - train 20 --bs 40 --num-cls-epochs=2 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv +Running tokenization... +Saving tokenized: cls.trn 13013, cls.val 1445 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/es-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/es-100/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/es-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/es-100/models/sp30k/lstm_nl4.m/../itos')] +epoch train_loss valid_loss accuracy +1 2.805415 2.188974 0.537779 +epoch train_loss valid_loss accuracy +1 2.429727 1.989691 0.569048 +2 2.218828 1.794969 0.603721 +3 2.015097 1.644815 0.629609 +4 1.877210 1.537773 0.646898 +5 1.775648 1.450283 0.660861 +6 1.749334 1.377085 0.672146 +7 1.601073 1.311101 0.684400 +8 1.564420 1.251074 0.694900 +9 1.532728 1.197607 0.704779 +10 1.391921 1.145408 0.716044 +11 1.379958 1.093550 0.726937 +12 1.324111 1.048308 0.735890 +13 1.344113 1.007926 0.745691 +14 1.243085 0.969521 0.754591 +15 1.230809 0.937330 0.762675 +16 1.162501 0.913408 0.768044 +17 1.170092 0.894892 0.773239 +18 1.110860 0.884449 0.775603 +19 1.115907 0.880448 0.776671 +20 1.083033 0.878421 0.776931 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.621574 0.391042 0.856000 +epoch train_loss valid_loss accuracy +1 0.411668 0.215625 0.935000 +epoch train_loss valid_loss accuracy +1 0.340519 0.222422 0.935000 +epoch train_loss valid_loss accuracy +1 0.281729 0.192193 0.949000 +2 0.262074 0.202975 0.945000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m +Loss and accuracy using (cls_best): [0.1749019, tensor(0.9515)] +``` \ No newline at end of file diff --git a/results/logs/ja.md b/results/logs/ja.md index 6e0872f..183943e 100644 --- a/results/logs/ja.md +++ b/results/logs/ja.md @@ -87,4 +87,65 @@ epoch train_loss valid_loss accuracy 8 0.278896 0.358145 0.877000 Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m Loss and accuracy using (cls_best): [0.29789856, tensor(0.8920)] +``` + +### JA on 100 elements +``` +python -m ulmfit cls --dataset-path data/mldoc/ja-1 --base-lm-path data/wiki/ja-100/models/sp30k/lstm_nl4.m --lang=ja --name 'nl4-100' --cuda-id=1 - train 20 --bs 40 --num-cls-epochs=8 --limit=100 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-100.m +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Limiting data set to: 100 +Running tokenization... +Saving tokenized: cls.trn 100, cls.val 100 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp30k/lstm_nl4.m/../itos')] +epoch train_loss valid_loss accuracy +1 2.837937 2.387255 0.518590 +epoch train_loss valid_loss accuracy +1 2.466900 2.193583 0.549492 +2 2.232762 1.983981 0.586658 +3 2.026505 1.810167 0.615649 +4 1.918111 1.679784 0.636613 +5 1.748909 1.577095 0.653108 +6 1.708709 1.491436 0.667657 +7 1.640415 1.420449 0.679619 +8 1.577434 1.359511 0.690194 +9 1.551961 1.302819 0.700306 +10 1.475623 1.252393 0.710039 +11 1.435565 1.208159 0.718740 +12 1.354910 1.161781 0.727927 +13 1.351157 1.123244 0.736009 +14 1.299070 1.086383 0.743896 +15 1.258739 1.055745 0.750383 +16 1.210775 1.035209 0.754965 +17 1.228421 1.018373 0.758963 +18 1.179444 1.007714 0.761158 +19 1.197443 1.003041 0.762068 +20 1.163223 1.001939 0.762211 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-100.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 1.269222 1.360420 0.340000 +epoch train_loss valid_loss accuracy +1 0.969350 1.314497 0.400000 +epoch train_loss valid_loss accuracy +1 0.832396 1.263416 0.550000 +epoch train_loss valid_loss accuracy +1 0.780991 1.225439 0.600000 +2 0.765755 1.183010 0.600000 +3 0.749420 1.139053 0.600000 +4 0.731800 1.093319 0.610000 +5 0.711152 1.054695 0.610000 +6 0.694611 1.029465 0.580000 +7 0.680276 1.004366 0.580000 +8 0.668421 0.984848 0.590000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-100.m +Loss and accuracy using (cls_best): [0.81621724, tensor(0.7437)] ``` \ No newline at end of file diff --git a/results/logs/ru.md b/results/logs/ru.md new file mode 100644 index 0000000..04e5bdd --- /dev/null +++ b/results/logs/ru.md @@ -0,0 +1,78 @@ +# RU +## SP30k nl4 +### LM +``` +python -m ulmfit lm --dataset-path data/wiki/ru-100 --cuda-id=0 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 30000 --lang ru --qrnn=False - train 10 --bs=50 --drop_mult=0 +Size of vocabulary: 30000 [39/805] +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', '▁и', 'е', 'и', 'й', '▁на', 'х'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': None, 'pretrained_model': None, 'drop_mult': 0} dps: [0.25 0.1 0.2 0.02 0.15] +Training lm from random weights +epoch train_loss valid_loss accuracy +1 3.200520 3.295865 0.436852 +2 3.027569 3.168700 0.445551 +3 3.007320 3.132495 0.450450 +4 2.940000 3.041745 0.459344 +5 2.876227 2.952338 0.469182 +6 2.742553 2.860888 0.480943 +7 2.684717 2.769994 0.492934 +8 2.569419 2.669971 0.507300 +9 2.525698 2.604086 0.516840 +10 2.495174 2.591011 0.519415 +data/wiki/ru-100/models/sp30k +Saving info data/wiki/ru-100/models/sp30k/lstm_nl4.m/info.json +``` +### MLDoc +MultiCCA: 85.65% ulmfit: 87.27% +``` +python -m ulmfit cls --dataset-path data/mldoc/ru-1 --base-lm-path data/wiki/ru-100/models/sp30k/lstm_nl4.m --lang=ru --name 'nl4-100' --cuda-id=1 - train 20 --bs 40 --num-cls-epochs=2 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv +Running tokenization... +Saving tokenized: cls.trn 9195, cls.val 1021 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', '▁и', 'е', 'и', 'й', '▁на', 'х'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp30k/lstm_nl4.m/../itos')] +epoch train_loss valid_loss accuracy +1 2.764138 2.289755 0.552181 +epoch train_loss valid_loss accuracy +1 2.414295 2.161708 0.572407 +2 2.310551 2.013092 0.596075 +3 2.124479 1.864450 0.620103 +4 1.970015 1.723395 0.642392 +5 1.883664 1.623308 0.658949 +6 1.793856 1.513542 0.677954 +7 1.625767 1.424582 0.693092 +8 1.677054 1.335406 0.709802 +9 1.578936 1.264322 0.723626 +10 1.523383 1.194463 0.737942 +11 1.436643 1.129712 0.750586 +12 1.351507 1.072792 0.762524 +13 1.357552 1.020739 0.773266 +14 1.310516 0.975852 0.783653 +15 1.216484 0.940323 0.791262 +16 1.187942 0.909915 0.797675 +17 1.141316 0.885367 0.803305 +18 1.114629 0.871992 0.805929 +19 1.075366 0.867010 0.807009 +20 1.166387 0.865594 0.807241 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.831180 0.610087 0.787000 +epoch train_loss valid_loss accuracy +1 0.678307 0.435860 0.856000 +epoch train_loss valid_loss accuracy +1 0.547668 0.399889 0.870000 +epoch train_loss valid_loss accuracy +1 0.445839 0.396535 0.869000 +2 0.417901 0.369961 0.882000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m +Loss and accuracy using (cls_best): [0.38499942, tensor(0.8727)] +``` \ No newline at end of file diff --git a/results/logs/zh.md b/results/logs/zh.md new file mode 100644 index 0000000..c341d56 --- /dev/null +++ b/results/logs/zh.md @@ -0,0 +1,6 @@ + + + +``` +python -m ulmfit lm --dataset-path data/wiki/zh-100 --cuda-id=0 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 60000 --lang zh --qrnn=False - train 10 --bs=50 --drop_mult=0 +``` \ No newline at end of file From e7271f2a293e05f2ec853565b728144c7733f711 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Tue, 12 Feb 2019 15:01:01 +0100 Subject: [PATCH 11/13] Add ability to evalulate multiple models at once --- calc_100.sh | 4 ++++ ulmfit/__main__.py | 30 ++++++++++++++++++++++++++++++ ulmfit/pretrain_lm.py | 3 ++- ulmfit/train_clas.py | 19 +++++++++++-------- 4 files changed, 47 insertions(+), 9 deletions(-) create mode 100755 calc_100.sh diff --git a/calc_100.sh b/calc_100.sh new file mode 100755 index 0000000..158de54 --- /dev/null +++ b/calc_100.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +LANGS +for \ No newline at end of file diff --git a/ulmfit/__main__.py b/ulmfit/__main__.py index 41c00ce..fb56560 100644 --- a/ulmfit/__main__.py +++ b/ulmfit/__main__.py @@ -1,14 +1,26 @@ +import gc +import shutil from functools import wraps import fire from .pretrain_lm import LMHyperParams from .train_clas import CLSHyperParams +from pathlib import Path class FireView: def __init__(self, **kwargs): for k,v in kwargs.items(): setattr(self, k, v) +def get_dataset_path(p): + return [x for x in p.parents if x.name == "models"][0].parent + +def get_lang_from_dataset_path(ds): + lang,*_ = ds.name.split("-") + if len(lang) == 2: + return lang + return "en" + class ULMFiT: @wraps(LMHyperParams) def lm(self, dataset_path, **changes): @@ -22,5 +34,23 @@ class ULMFiT: params = CLSHyperParams.from_lm(dataset_path, base_lm_path, **changes) return FireView(train=params.train_cls, validate_cls=params.validate_cls) + def eval(self, glob="mldoc/*-1/models/sp30k/lstm_nl4.m", name="tmp-100", cuda_id=0, **trn_params): + results={} + for base_model in Path("data").glob(glob): + dataset_path = get_dataset_path(base_model) + lang = get_lang_from_dataset_path(dataset_path) + params = CLSHyperParams.from_lm(dataset_path, base_model, lang=lang, name=name, cuda_id=cuda_id) + key = str(params.model_dir.relative_to(Path.cwd())) + if params.model_dir.exists(): + print("Evaluating previously trained model") + results[key] = params.validate_cls()[1] + else: + print("Training") + results[key] = params.train_cls(num_lm_epochs=0, **trn_params)[1] + params = None + gc.collect() + + print(list(sorted(results.items()))) + if __name__ == '__main__': fire.Fire(ULMFiT()) \ No newline at end of file diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 08530f2..fef99bf 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -89,7 +89,6 @@ class LMHyperParams: self.cache_dir = self.dataset_path / 'models' / self.tokenizer_prefix self.model_dir = self.cache_dir / self.model_name - self.model_dir.mkdir(exist_ok=True, parents=True) print('Max vocab:', self.max_vocab) print('Cache dir:', self.cache_dir) print('Model dir:', self.model_dir) @@ -147,6 +146,7 @@ class LMHyperParams: print("Saving info", self.model_dir / 'info.json') def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, lr=5e-3): + self.model_dir.mkdir(exist_ok=True, parents=True) data_lm = self.load_wiki_data(bs=bs) if data_lm is None else data_lm learn = self.create_lm_learner(data_lm, drop_mult=drop_mult) @@ -201,6 +201,7 @@ class LMHyperParams: return [line.rstrip('\n') for line in f] def load_wiki_data(self, bs=70): + self.model_dir.mkdir(exist_ok=True, parents=True) trn_path = self.dataset_path / f'{self.lang}.wiki.train.tokens' val_path = self.dataset_path / f'{self.lang}.wiki.valid.tokens' tst_path = self.dataset_path / f'{self.lang}.wiki.test.tokens' diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 3de435d..ae5e7c3 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -37,9 +37,10 @@ class CLSHyperParams(LMHyperParams): @property def need_fine_tune_lm(self): return not (self.model_dir/f"enc_best.pth").exists() - def train_cls(self, num_lm_epochs, unfreeze=True, bs=40, true_wd=True, drop_mul_lm=0.3, drop_mul_cls=0.5, + def train_cls(self, num_lm_epochs, unfreeze=True, num_cls_frozen_epochs=1, bs=40, true_wd=True, drop_mul_lm=0.3, drop_mul_cls=0.5, use_test_for_validation=False, num_cls_epochs=2, limit=None, noise=0.0): assert use_test_for_validation == False, "use_test_for_validation=True is not supported" + self.model_dir.mkdir(exist_ok=True, parents=True) data_clas, data_lm, data_tst = self.load_cls_data(bs, limit=limit, noise=noise) @@ -54,7 +55,7 @@ class CLSHyperParams(LMHyperParams): learn.true_wd = True print("Starting classifier training") learn.freeze_to(-1) - learn.fit_one_cycle(1, 2e-2, moms=(0.8, 0.7)) + learn.fit_one_cycle(num_cls_frozen_epochs, 2e-2, moms=(0.8, 0.7)) if unfreeze: learn.freeze_to(-2) learn.fit_one_cycle(1, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7)) @@ -65,7 +66,7 @@ class CLSHyperParams(LMHyperParams): else: learn.true_wd = False print("Starting classifier training") - learn.fit_one_cycle(1, 5e-2, moms=(0.8, 0.7), wd=1e-7) + learn.fit_one_cycle(num_cls_frozen_epochs, 5e-2, moms=(0.8, 0.7), wd=1e-7) if unfreeze: learn.freeze_to(-2) learn.fit_one_cycle(1, slice(5e-2 / (2.6 ** 4), 5e-2), moms=(0.8, 0.7), wd=1e-7) @@ -76,17 +77,18 @@ class CLSHyperParams(LMHyperParams): print(f"Saving models at {learn.path / learn.model_dir}") learn.save('cls_last', with_opt=False) - self.validate_cls('cls_best', bs=bs, limit=limit, data_tst=data_tst, learn=learn) - return None + return self.validate_cls('cls_best', bs=bs, data_tst=data_tst, learn=learn) - def validate_cls(self, save_name='cls_last', limit=None, bs=40, data_tst=None, learn=None): + def validate_cls(self, save_name='cls_last', bs=40, data_tst=None, learn=None): if data_tst is None: - _, _, data_tst = self.load_cls_data(bs, limit=limit) + _, _, data_tst = self.load_cls_data(bs) if learn is None: learn = self.create_cls_learner(data_tst, drop_mult=0.3) learn.unfreeze() learn.load(save_name) - print(f"Loss and accuracy using ({save_name}):", learn.validate(data_tst.valid_dl)) + results = learn.validate(data_tst.valid_dl) + print(f"Loss and accuracy using ({save_name}):", results) + return list(map(float, results)) def create_cls_learner(self, data_clas, dps=None, **kwargs): fastai.text.learner.default_dropout['language'] = dps or self.dps @@ -104,6 +106,7 @@ class CLSHyperParams(LMHyperParams): return learn def load_cls_data(self, bs, **kwargs): + self.model_dir.mkdir(exist_ok=True, parents=True) add_trn_to_lm = True lang = self.lang use_moses = True From e72cdfb6dbb386353f97a27afbe461a0a391b5b3 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Tue, 12 Feb 2019 15:01:22 +0100 Subject: [PATCH 12/13] Add result logs (it) --- results/logs/it.md | 82 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 results/logs/it.md diff --git a/results/logs/it.md b/results/logs/it.md new file mode 100644 index 0000000..472e123 --- /dev/null +++ b/results/logs/it.md @@ -0,0 +1,82 @@ +# FR +## SP30k LSTM nl 4 +### LM +``` +python -m ulmfit lm --dataset-path data/wiki/it-100 --lang=it --bidir=False --qrnn=False --max-vocab 30000 --nl 4 --tokenizer=sp --name 'nl4bs100' - train 10 --bs 100 --dropout-mult=0 +Wiki text was split to 164583 articles +Wiki text was split to 98 articles +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', "▁&'", "'", '▁e', '▁il', '▁la', 'e', '▁in'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': None, 'pretrained_model': None, 'drop_mult': 0.0} dps: [0.25 0.1 0.2 0.02 0.15] +Training lm from random weights +epoch train_loss valid_loss accuracy +1 3.306743 3.717148 0.353641 +2 3.126413 3.606443 0.360839 +3 3.062586 3.545493 0.365721 +4 3.055600 3.474823 0.373451 +5 2.927211 3.406635 0.380311 +6 2.924096 3.321370 0.389487 +7 2.779998 3.233350 0.399968 +8 2.722100 3.147745 0.410365 +9 2.615910 3.087420 0.419097 +10 2.565747 3.075364 0.420906 +data/wiki/it-100/models/sp30k +Saving info data/wiki/it-100/models/sp30k/lstm_nl4bs100.m/info.json +``` + + +### MLDoc +MultiCCA: 85.55%, ULMFiT 88.42% +``` +python -m ulmfit cls --dataset-path data/mldoc/it-1 --base-lm-path data/wiki/it-100/models/sp30k/lstm_nl4bs100.m --lang=it --name 'nl4bs100' --cuda-id=1 - train 20 --bs 40 --num-cls-epochs=2 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4bs100.m +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/it.dev.csv +Running tokenization... +Saving tokenized: cls.trn 13500, cls.val 1500 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', "▁&'", "'", '▁e', '▁il', '▁la', 'e', '▁in'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/it-100/models/sp30k/lstm_nl4bs100.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/it-100/models/sp30k/lstm_nl4bs100.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/it-100/models/sp30k/lstm_nl4bs100.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/it-100/models/sp30k/lstm_nl4bs100.m/../itos')] +epoch train_loss valid_loss accuracy +1 2.826957 2.518636 0.492175 +epoch train_loss valid_loss accuracy +1 2.606302 2.397623 0.509596 +2 2.470586 2.260363 0.531301 +3 2.334087 2.113640 0.554089 +4 2.176830 1.988222 0.572687 +5 2.123101 1.869944 0.591537 +6 2.011187 1.770606 0.606682 +7 1.934953 1.676852 0.622504 +8 1.889363 1.592609 0.637525 +9 1.774590 1.517665 0.652233 +10 1.725905 1.435543 0.666759 +11 1.670903 1.365167 0.681168 +12 1.610080 1.302561 0.694462 +13 1.522876 1.242124 0.708201 +14 1.478528 1.193259 0.718366 +15 1.423993 1.150854 0.728324 +16 1.389901 1.115550 0.735836 +17 1.365959 1.094267 0.740730 +18 1.347579 1.079465 0.744019 +19 1.321906 1.074090 0.745281 +20 1.332676 1.073143 0.745453 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4bs100.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.632703 0.463210 0.831000 +epoch train_loss valid_loss accuracy +1 0.527650 0.390041 0.858000 +epoch train_loss valid_loss accuracy +1 0.436223 0.326409 0.871000 +epoch train_loss valid_loss accuracy +1 0.361738 0.321380 0.875000 +2 0.340658 0.315946 0.877000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4bs100.m +Loss and accuracy using (cls_best): [0.32998973, tensor(0.8842)] +``` \ No newline at end of file From 4e1b76feeec302781508ce9c6bfd2395b57521b0 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Tue, 12 Feb 2019 15:01:42 +0100 Subject: [PATCH 13/13] Add MLDoc summary & zeroshot logs --- results/MLDoc.md | 27 ++++ results/logs/common.md | 294 +++++++++++++++++++++++++++++++++++++++ results/logs/zeroshot.md | 265 +++++++++++++++++++++++++++++++++++ 3 files changed, 586 insertions(+) create mode 100644 results/MLDoc.md create mode 100644 results/logs/common.md create mode 100644 results/logs/zeroshot.md diff --git a/results/MLDoc.md b/results/MLDoc.md new file mode 100644 index 0000000..11c5006 --- /dev/null +++ b/results/MLDoc.md @@ -0,0 +1,27 @@ + +# non-zeroshot +| Model | en | de | es | fr | it | ja | ru | zh | +|----------------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|------------| +|LASER | 90.73 | 92.70 | 88.75 | 90.80 | 85.93 | 85.15 | 84.65 | 88.98 | +|MultiCCA | 92.2 | 93.70 | 94.45 | 92.05 | 85.55 | 85.35 | 85.65 | 87.30 | +|ULMFiT | | **95.4** | **95.15** | **93.67** | **88.42** | **89.20** | **87.27** | | +|ULMFiT 100 | | 91.35 | 83.32 | 88.77 | 77.99 | 71.12 | 72.20 | | + +# Zero shot approaches + +| Model | en | de | es | fr | it | ja | ru | zh | +|----------------------|------------|-----------|-----------|-----------|-----------|-----------|-----------|------------| +|LASER 0 shot | 80.75 (en) | 87.03 (fr)| 82.60 (it)| 82.83 (de)| 73.25 (de)| 60.95 (en)| 68.83 (it)| 72.90 (de) | +|LASER base 0 shot | | 86.48 | 79.23 | 76.73 | +|ULMFiT 0 shot | | **91.97**| **85.35** | 85.54 | +|ULMFiT 100 for comp. | | 91.35 | 83.32 | 88.77 | 77.99 | 71.12 | 72.20 | | + + +To simulate ulmfit zero shot we add noise to the training labels to simulate training from Laser labels + +| Model | en | de | es | fr | it | ja | ru | zh | +|----------------------|------------|-----------|-----------|-----------|-----------|-----------|-----------|------------| +|LASER 0 shot | 80.75 (en) | 87.03 (fr)| 82.60 (it)| 82.83 (de)| 73.25 (de)| 60.95 (en)| 68.83 (it)| 72.90 (de) | +|ULMFiT | | **95.4** | **95.15** | **93.67** | **88.42** | **89.20** | **87.27** | | +| Noise | 20% | 13% | 18% | 18% | 27% | 40% | 32% | 28% | +|ULMFiT noise ~ 0 shot | | 94.49 | 93.12 | 90.49 | 83.72 | 74.72 | 75.67 | | diff --git a/results/logs/common.md b/results/logs/common.md new file mode 100644 index 0000000..9142b2b --- /dev/null +++ b/results/logs/common.md @@ -0,0 +1,294 @@ +# MLDoc +## Limiit to 100 examples +``` +python -m ulmfit eval --glob="mldoc/*-1/models/sp30k/lstm_nl4.m" --name nl4-100e8 --cuda-id=1 --limit=100 --num-cls-epochs=8 +{ + 'data/mldoc/it-1/models/sp30k/lstm_nl4-100e8.m': 0.7799999713897705, + 'data/mldoc/de-1/models/sp30k/lstm_nl4-100e8.m': 0.9135000109672546, + 'data/mldoc/ja-1/models/sp30k/lstm_nl4-100e8.m': 0.7112500071525574, + 'data/mldoc/fr-1/models/sp30k/lstm_nl4-100e8.m': 0.8877500295639038, + 'data/mldoc/ru-1/models/sp30k/lstm_nl4-100e8.m': 0.722000002861023, + 'data/mldoc/es-1/models/sp30k/lstm_nl4-100e8.m': 0.8169999718666077 +} +``` + + +## Noise + +``` +noise=0.13 +lang=de +python -m ulmfit eval --glob="mldoc/${lang}-1/models/sp30k/lstm_nl4.m" --name nl4-noise --cuda-id=1 --num-cls-epochs=2 --noise=${noise} +{'data/mldoc/de-1/models/sp30k/lstm_nl4-noise.m': 0.9449999928474426} + +noise=0.18 +lang=es +python -m ulmfit eval --glob="mldoc/${lang}-1/models/sp30k/lstm_nl4.m" --name nl4-noise --cuda-id=1 --num-cls-epochs=2 --noise=${noise} +{'data/mldoc/es-1/models/sp30k/lstm_nl4-noise.m': 0.9312499761581421} + +noise=0.18 +lang=fr +python -m ulmfit eval --glob="mldoc/${lang}-1/models/sp30k/lstm_nl4.m" --name nl4-noise --cuda-id=1 --num-cls-epochs=2 --noise=${noise} +{'data/mldoc/fr-1/models/sp30k/lstm_nl4-noise.m': 0.9049999713897705} + +noise=0.27 +lang=it +python -m ulmfit eval --glob="mldoc/${lang}-1/models/sp30k/lstm_nl4.m" --name nl4-noise --cuda-id=1 --num-cls-epochs=2 --noise=${noise} +{'data/mldoc/it-1/models/sp30k/lstm_nl4-noise.m': 0.8372499942779541} + +noise=0.4 +lang=ja +python -m ulmfit eval --glob="mldoc/${lang}-1/models/sp30k/lstm_nl4.m" --name nl4-noise --cuda-id=1 --num-cls-epochs=2 --noise=${noise} +{'data/mldoc/ja-1/models/sp30k/lstm_nl4-noise.m': 0.7472500205039978 + +noise=0.32 +lang=ru +python -m ulmfit eval --glob="mldoc/${lang}-1/models/sp30k/lstm_nl4.m" --name nl4-noise --cuda-id=1 --num-cls-epochs=2 --noise=${noise} +{'data/mldoc/ru-1/models/sp30k/lstm_nl4-noise.m': 0.7567499876022339} + +noise=0.28 +lang=zh +python -m ulmfit eval --glob="mldoc/${lang}-1/models/sp30k/lstm_nl4.m" --name nl4-noise --cuda-id=1 --num-cls-epochs=2 --noise=${noise} +``` + + + + + +### LIMIT LOgs +``` +python -m ulmfit eval --name nl4-100e8 --cuda-id=1 --limit=100 --num-cls-epochs=8 ✘ 130 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4-100e8.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/it.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Limiting data set to: 100 +Tokenized data loaded, cls.trn 100, cls.val 100 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', "▁&'", "'", '▁e', '▁il', '▁la', 'e', '▁in'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4-100e8.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 1.214805 1.382632 0.280000 +epoch train_loss valid_loss accuracy +1 0.977314 1.269534 0.450000 +epoch train_loss valid_loss accuracy +1 0.856274 1.223441 0.530000 +epoch train_loss valid_loss accuracy +1 0.718223 1.188048 0.620000 +2 0.735718 1.130525 0.730000 +3 0.730894 1.069027 0.710000 +4 0.715334 1.015253 0.710000 +5 0.716080 0.965223 0.720000 +6 0.695554 0.918456 0.730000 +7 0.689949 0.892840 0.730000 +8 0.675208 0.876222 0.720000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4-100e8.m +Loss and accuracy using (cls_best): [0.7090041, tensor(0.7800)] +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-100e8.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/de.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Limiting data set to: 100 +Running tokenization... +Saving tokenized: cls.trn 100, cls.val 100 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-100e8.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 1.141527 1.328262 0.280000 +epoch train_loss valid_loss accuracy +1 0.703434 1.170250 0.510000 +epoch train_loss valid_loss accuracy +1 0.568693 1.051980 0.780000 +epoch train_loss valid_loss accuracy +1 0.455238 0.990438 0.800000 +2 0.475659 0.928943 0.850000 +3 0.477652 0.848537 0.920000 +4 0.455583 0.769415 0.930000 +5 0.450824 0.690618 0.930000 +6 0.443699 0.633900 0.940000 +7 0.430881 0.563667 0.950000 +8 0.419999 0.524655 0.950000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-100e8.m +Loss and accuracy using (cls_best): [0.45835665, tensor(0.9135)] +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-100e8.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Limiting data set to: 100 +Tokenized data loaded, cls.trn 100, cls.val 100 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-100e8.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 1.342269 1.399389 0.230000 +epoch train_loss valid_loss accuracy +1 0.957341 1.344665 0.280000 +epoch train_loss valid_loss accuracy +1 0.881869 1.301798 0.450000 +epoch train_loss valid_loss accuracy +1 0.887575 1.280226 0.440000 +2 0.835731 1.257639 0.450000 +3 0.813987 1.219512 0.510000 +4 0.792665 1.181309 0.520000 +5 0.785690 1.151372 0.510000 +6 0.784095 1.152232 0.500000 +7 0.768115 1.133895 0.520000 +8 0.769684 1.124231 0.530000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-100e8.m +Loss and accuracy using (cls_best): [0.8863698, tensor(0.7113)] +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4-100e8.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/fr.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Limiting data set to: 100 +Running tokenization... +Saving tokenized: cls.trn 100, cls.val 100 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4-100e8.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 1.220506 1.413276 0.200000 +epoch train_loss valid_loss accuracy +1 0.791777 1.306999 0.290000 +epoch train_loss valid_loss accuracy +1 0.572241 1.190053 0.580000 +epoch train_loss valid_loss accuracy +1 0.502800 1.130456 0.710000 +2 0.515115 1.056434 0.770000 +3 0.522720 0.974482 0.780000 +4 0.518296 0.881002 0.840000 +5 0.496588 0.825646 0.880000 +6 0.490416 0.771587 0.860000 +7 0.497172 0.722874 0.850000 +8 0.491894 0.682278 0.850000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4-100e8.m +Loss and accuracy using (cls_best): [0.5428351, tensor(0.8878)] +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4-100e8.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv +Tokenized data loaded, lm.trn 9195, lm.val 1021 +Limiting data set to: 100 +Running tokenization... +Saving tokenized: cls.trn 100, cls.val 100 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', '▁и', 'е', 'и', 'й', '▁на', 'х'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4-100e8.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 1.367201 1.409767 0.240000 +epoch train_loss valid_loss accuracy +1 1.099071 1.320811 0.330000 +epoch train_loss valid_loss accuracy +1 0.875845 1.253172 0.410000 +epoch train_loss valid_loss accuracy +1 0.775657 1.215067 0.580000 +2 0.774420 1.171324 0.660000 +3 0.766028 1.118901 0.680000 +4 0.744478 1.074021 0.680000 +5 0.738797 1.033736 0.660000 +6 0.733380 0.997304 0.660000 +7 0.723470 0.977280 0.670000 +8 0.710699 0.953586 0.640000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4-100e8.m +Loss and accuracy using (cls_best): [0.8535175, tensor(0.7220)] +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-100e8.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv +Tokenized data loaded, lm.trn 13013, lm.val 1445 +Limiting data set to: 100 +Running tokenization... +Saving tokenized: cls.trn 100, cls.val 100 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-100e8.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 1.142170 1.330161 0.300000 +epoch train_loss valid_loss accuracy +1 0.767807 1.212253 0.420000 +epoch train_loss valid_loss accuracy +1 0.636803 1.099303 0.540000 +epoch train_loss valid_loss accuracy +1 0.584241 0.997207 0.610000 +2 0.578480 0.907674 0.710000 +3 0.548451 0.830268 0.730000 +4 0.535560 0.762040 0.750000 +5 0.522172 0.746566 0.740000 +6 0.506584 0.676038 0.770000 +7 0.493665 0.651112 0.770000 +8 0.493031 0.621689 0.770000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-100e8.m +Loss and accuracy using (cls_best): [0.54911107, tensor(0.8170)] +{'data/mldoc/it-1/models/sp30k/lstm_nl4-100e8.m': 0.7799999713897705, 'data/mldoc/de-1/models/sp30k/lstm_nl4-100e8.m': 0.9135000109672546, 'data/mldoc/ja-1/models/sp30k/lstm_nl4-100e8.m': 0.7112500071525574, 'data/mldoc/fr-1/models/sp30k/lstm_nl4-100e8.m': 0.8877500295639038, 'data/mldoc/ru-1/models/sp30k/lstm_nl4-100e8.m': 0.722000002861023, 'data/mldoc/es-1/models/sp30k/lstm_nl4-100e8.m': 0.8169999718666077} + +python -m ulmfit eval --glob="mldoc/es-1/models/sp30k/lstm_nl4.m" --name nl4-100-2nd --cuda-id=1 --num-cls-epochs=8 --limit=100 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-100-2nd.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv +Tokenized data loaded, lm.trn 13013, lm.val 1445 +Limiting data set to: 100 +Tokenized data loaded, cls.trn 100, cls.val 100 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-100-2nd.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 1.243127 1.354496 0.290000 +epoch train_loss valid_loss accuracy +1 0.840900 1.213333 0.460000 +epoch train_loss valid_loss accuracy +1 0.656407 1.055138 0.750000 +epoch train_loss valid_loss accuracy +1 0.558013 0.983957 0.780000 +2 0.554590 0.915244 0.750000 +3 0.536740 0.840074 0.770000 +4 0.521179 0.759908 0.790000 +5 0.515218 0.692961 0.810000 +6 0.500587 0.639504 0.810000 +7 0.486596 0.593410 0.840000 +8 0.472318 0.550126 0.830000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-100-2nd.m +Loss and accuracy using (cls_best): [0.5382241, tensor(0.8332)] +{'data/mldoc/es-1/models/sp30k/lstm_nl4-100-2nd.m': 0.8332499861717224} + +``` \ No newline at end of file diff --git a/results/logs/zeroshot.md b/results/logs/zeroshot.md new file mode 100644 index 0000000..e199336 --- /dev/null +++ b/results/logs/zeroshot.md @@ -0,0 +1,265 @@ +# Laser Performance +Accuracy matrix: + +| Train | en | de | es | fr | it | ru | zh | +|-------|-------|-------|-------|-------|-------|-------|-------| +| en: | 90.88 | 86.48 | 67.62 | 61.98 | 69.95 | 22.95 | 11.65 | +| de: | 73.23 | 92.90 | 77.23 | 74.05 | 72.30 | 24.80 | 9.93 | +| es: | 65.62 | 80.58 | 92.03 | 73.28 | 69.03 | 34.10 | 12.58 | +| fr: | 78.35 | 85.45 | 78.20 | 89.68 | 69.85 | 33.88 | 9.68 | +| it: | 73.93 | 84.58 | 79.23 | 76.73 | 84.03 | 34.48 | 11.83 | +| ru: | 57.33 | 63.78 | 45.80 | 52.78 | 51.15 | 66.08 | 36.28 | +| zh: | 26.15 | 28.13 | 21.88 | 29.33 | 30.58 | 34.38 | 75.62 | + +# DE +Laser 0shot: 86.48, ULMFiT 0shot: 91.97 +``` +python ../../source/classify.py embed-2019-02-12/mldoc.en-en.h5 ~/workspace/ulmfit-multilingual/data/mldoc/de-1 + | Test: 86.48% | classes: 24.30 22.77 28.90 24.02 + Making train set + | Train: 85.70% | classes: 27.00 21.40 27.60 24.00 +Accuracy 0.857 + 0 1 +0 3 Tokio (Reuter) - Der Dollar ist am Donnerstag ... +1 3 Kairo (Reuter) - Die ägyptische Zentralbank se... +2 2 Bonn (Reuter) - Wegen einer Bombendrohung ist ... +3 0 Berlin (Reuter) - Die Bahn AG will mit Hilfe p... +4 3 08.15 Uhr MEZ - Deutsche Aktien nach den Rekor... + + Making dev set + | Train: 85.60% | classes: 23.70 22.30 30.60 23.40 +Accuracy 0.856 + 0 1 +0 1 New York (Reuter) - Das Vertrauen der US-Verbr... +1 2 Tokio (Reuter) - Russische Patrouillenboote ha... +2 2 Paris (Reuter) - Bei der Volksabstimmung in Al... +3 2 Belgrad (Reuter) - Die serbische Polizei hat n... +4 0 München (Reuter) - Der Stuttgarter Bosch-Konze... +``` +``` +python -m ulmfit cls --dataset-path data/mldoc/de-1-laser --base-lm-path data/mldoc/de-1/models/sp30k/lstm_nl4.m --lang=de --name 'nl4' --cuda-id=1 - train 0 --bs 40 --num-cls-epochs=2 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser/models/sp30k/lstm_nl4.m +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser/de.dev.csv +Running tokenization... +Saving tokenized: cls.trn 13500, cls.val 1500 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser/models/sp30k/lstm_nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.671869 0.466408 0.863000 +epoch train_loss valid_loss accuracy +1 0.518045 0.388151 0.887000 +epoch train_loss valid_loss accuracy +1 0.375156 0.370652 0.893000 +epoch train_loss valid_loss accuracy +1 0.339284 0.367223 0.891000 +2 0.314325 0.369492 0.891000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser/models/sp30k/lstm_nl4.m +Loss and accuracy using (cls_best): [0.25416428, tensor(0.9197)] +0.25416427850723267 +0.9197499752044678 +``` + + + + + +# ES from IT +``` +python ../../source/classify.py embed-2019-02-12/mldoc.it-it.h5 ~/workspace/ulmfit-multilingual/data/mldoc/es-1 ✘ 130 + | Test: 79.23% | classes: 25.48 16.45 24.18 33.90 + Making train set + | Train: 80.30% | classes: 27.10 19.20 22.60 31.10 +Accuracy 0.803 + 0 1 +0 3 LONDRES, 5 sep (Reuter) - El dólar se mantenía... +1 1 MADRID, 30 dic (Reuter) - La Generalitat de Va... +2 3 PARIS, 30 jun (Reuter) - La Bolsa de París neg... +3 0 MADRID, 23 dic (Reuter) - La agencia de valore... +4 0 MADRID, 4 Feb (Reuter) - El Banco Bilbao Vizca... + + Making dev set + | Train: 79.70% | classes: 25.40 17.50 26.20 30.90 +Accuracy 0.797 + 0 1 +0 0 NUEVA YORK, 11 abr (Reuter) - MCI Communicatio... +1 3 FRANCFORT, 17 jun (Reuter) - La Bolsa de Franc... +2 1 BONN, 3 jun (Reuter) - Un destacado miembro de... +3 2 LONDRES, 3 sep (Reuter) - El secretario de Def... +4 2 MADRID, 3 oct (Reuter) - Las acciones de Pryca... +``` + +``` +python -m ulmfit cls --dataset-path data/mldoc/es-1-laser-it --base-lm-path data/mldoc/es-1/models/sp30k/lstm_nl4.m --lang=es --name 'nl4' --cuda-id=1 - train 0 --bs 40 --num-cls-epochs=2 +``` + +# FR from IT +``` +python ../../source/classify.py embed-2019-02-12/mldoc.it-it.h5 ~/workspace/ulmfit-multilingual/data/mldoc/fr-1 + | Test: 76.73% | classes: 21.65 21.98 31.77 24.60 + Making train set + | Train: 79.20% | classes: 22.20 22.40 31.40 24.00 +Accuracy 0.792 + 0 1 +0 2 WASHINGTON, 13 septembre, Reuter - Les Etats-U... +1 1 PARIS, 10 juillet, Reuter - L'audit des financ... +2 2 MOSCOU, 29 mai, Reuter - Après l'accord interv... +3 2 PARIS, 1er octobre, Reuter - Le groupe communi... +4 0 LONDRES, 3 juin, Reuter - National Grid Group ... + + Making dev set + | Train: 76.60% | classes: 23.30 20.10 33.00 23.60 +Accuracy 0.766 + 0 1 +0 0 PARIS, 30 décembre, Reuter - Zodiac . Chiffre ... +1 0 AJACCIO, 11 décembre, Reuter - Une charge de 7... +2 0 BRUXELLES, 26 décembre, Reuter - 1997 s'annonc... +3 0 PARIS, 26 septembre, Reuter - Alcatel Alsthom ... +4 0 NEW YORK, 25 octobre, Reuter - La hausse plus ... +``` + +``` +python -m ulmfit cls --dataset-path data/mldoc/fr-1-laser-it --base-lm-path data/mldoc/fr-1/models/sp30k/lstm_nl4.m --lang=fr --name 'nl4' --cuda-id=1 - train 0 --bs 40 --num-cls-epochs=2 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-it/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-it/models/sp30k/lstm_nl4.m +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-it/fr.dev.csv +Running tokenization... +Saving tokenized: cls.trn 13500, cls.val 1500 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-it/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-it/models/sp30k/lstm_nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.737947 0.627607 0.793000 +epoch train_loss valid_loss accuracy +1 0.603060 0.513449 0.831000 +epoch train_loss valid_loss accuracy +1 0.481312 0.499689 0.828000 +epoch train_loss valid_loss accuracy +1 0.422958 0.508330 0.825000 +2 0.408061 0.493875 0.839000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-it/models/sp30k/lstm_nl4.m +Loss and accuracy using (cls_best): [0.4295174, tensor(0.8555)] +0.42951738834381104 +0.8554999828338623 +``` +# FR From EN +``` +python ../../source/classify.py embed-2019-02-12/mldoc.en-en.h5 ~/workspace/ulmfit-multilingual/data/mldoc/fr-1 + | Test: 61.98% | classes: 11.85 41.10 40.05 7.00 + Making train set + | Train: 63.70% | classes: 11.70 43.80 38.40 6.10 +Accuracy 0.637 + 0 1 +0 2 WASHINGTON, 13 septembre, Reuter - Les Etats-U... +1 1 PARIS, 10 juillet, Reuter - L'audit des financ... +2 2 MOSCOU, 29 mai, Reuter - Après l'accord interv... +3 2 PARIS, 1er octobre, Reuter - Le groupe communi... +4 0 LONDRES, 3 juin, Reuter - National Grid Group ... + + Making dev set + | Train: 61.60% | classes: 11.90 40.90 39.70 7.50 +Accuracy 0.616 + 0 1 +0 1 PARIS, 30 décembre, Reuter - Zodiac . Chiffre ... +1 0 AJACCIO, 11 décembre, Reuter - Une charge de 7... +2 1 BRUXELLES, 26 décembre, Reuter - 1997 s'annonc... +3 1 PARIS, 26 septembre, Reuter - Alcatel Alsthom ... +4 1 NEW YORK, 25 octobre, Reuter - La hausse plus ... +``` +``` + +python -m ulmfit cls --dataset-path data/mldoc/fr-1-laser --base-lm-path data/mldoc/fr-1/models/sp30k/lstm_nl4.m --lang=fr --name 'nl4-laser' --cuda-id=1 - train 0 --bs 40 --num-cls-epochs=2 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-laser.m +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/fr.dev.csv +Running tokenization... +Saving tokenized: cls.trn 13500, cls.val 1500 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-laser.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.797327 0.697984 0.730000 +epoch train_loss valid_loss accuracy +1 0.639780 0.582377 0.763000 +epoch train_loss valid_loss accuracy +1 0.585295 0.582596 0.762000 +epoch train_loss valid_loss accuracy +1 0.482629 0.582803 0.765000 +2 0.470849 0.582416 0.771000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-laser.m +Loss and accuracy using (cls_best): [0.80327946, tensor(0.6920)] +``` + + + +### No Unfreeze +#### one epoch +``` +python -m ulmfit cls --dataset-path data/mldoc/fr-1-laser --base-lm-path data/mldoc/fr-1/models/sp30k/lstm_nl4.m --lang=fr --name 'nl4-no_unfreeze' --cuda-id=1 - train 0 --bs 40 --num-cls-epochs=2 --unfreeze=False +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-no_unfreeze.m +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/fr.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Tokenized data loaded, cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-no_unfreeze.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.800256 0.783174 0.701000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-no_unfreeze.m +Loss and accuracy using (cls_best): [1.1735736, tensor(0.5077)] +1.173573613166809 +0.5077499747276306 +``` +#### 4 epochs +ulmfit: 63.67% +``` +python -m ulmfit cls --dataset-path data/mldoc/fr-1-laser --base-lm-path data/mldoc/fr-1/models/sp30k/lstm_nl4.m --lang=fr --name 'nl4-no_unfreeze2' --cuda-id=1 - train 0 --bs 40 --num-cls-epochs=2 --unfreeze=False --num-cls-frozen-epochs=4 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-no_unfreeze2.m +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/fr.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Tokenized data loaded, cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-no_unfreeze2.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.832118 0.750073 0.717000 +2 0.729266 0.617375 0.749000 +3 0.645946 0.623189 0.751000 +4 0.566385 0.608672 0.760000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-no_unfreeze2.m +Loss and accuracy using (cls_best): [0.97152597, tensor(0.6367)] +``` \ No newline at end of file