From 8497cc1e5c787eb0efa19ddcdd0b80fe17b1b528 Mon Sep 17 00:00:00 2001 From: Marcin Date: Mon, 11 Feb 2019 15:43:43 +0100 Subject: [PATCH 01/18] Move LM and classifier parameters to configs --- prepare_wiki.sh | 0 ulmfit/__main__.py | 2 +- ulmfit/pretrain_lm.py | 37 ++++++++++++++++++++++++------------- ulmfit/train_clas.py | 27 ++++++++++++++++----------- 4 files changed, 41 insertions(+), 25 deletions(-) mode change 100644 => 100755 prepare_wiki.sh diff --git a/prepare_wiki.sh b/prepare_wiki.sh old mode 100644 new mode 100755 diff --git a/ulmfit/__main__.py b/ulmfit/__main__.py index 41c00ce..cc4a48b 100644 --- a/ulmfit/__main__.py +++ b/ulmfit/__main__.py @@ -23,4 +23,4 @@ class ULMFiT: return FireView(train=params.train_cls, validate_cls=params.validate_cls) if __name__ == '__main__': - fire.Fire(ULMFiT()) \ No newline at end of file + fire.Fire(ULMFiT()) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 08530f2..7543243 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -68,7 +68,7 @@ class LMHyperParams: # these hyperparameters are for training on ~100M tokens (e.g. WikiText-103) # for training on smaller datasets, more dropout is necessary - dps = (0.25, 0.1, 0.2, 0.02, 0.15) # consider removing dps & clip from the default hyperparams and put them to train + dps = dict(output_p=0.25, hidden_p=0.1, input_p=0.2, embed_p=0.02, weight_p=0.15) # consider removing dps & clip from the default hyperparams and put them to train clip: float = 0.12 bptt: int = 70 @@ -93,7 +93,6 @@ class LMHyperParams: print('Max vocab:', self.max_vocab) print('Cache dir:', self.cache_dir) print('Model dir:', self.model_dir) - self.dps = np.array(self.dps) if self.nh is None: self.nh = 1550 if self.qrnn else 1150 if self.name is None: self.name = self.lang @@ -175,19 +174,29 @@ class LMHyperParams: print(learn.path) self.save_info() - return learn + # do we need to return `learn'? it adds noise to Fire output + #return learn def create_lm_learner(self, data_lm, dps=None, **kwargs): - fastai.text.learner.default_dropout['language'] = dps or self.dps - lm_learner = bilm_learner if self.bidir else language_model_learner - - trn_args = dict(tie_weights=True, clip=self.clip, bptt=self.bptt, - pretrained_fnames=self.pretrained_fnames, - pretrained_model=self.pretrained_model) + assert self.bidir == False, "bidirectional model is not yet supported" + config = dict(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, pad_token=PAD_TOKEN_ID, qrnn=self.qrnn, bidir=self.bidir, + tie_weights=True, out_bias=True) + config.update(dps or self.dps) + trn_args = dict(clip=self.clip) trn_args.update(kwargs) print ("Training args: ", trn_args, "dps: ", dps or self.dps) - learn = lm_learner(data_lm, emb_sz=self.emb_sz, nh=self.nh, nl=self.nl, pad_token=PAD_TOKEN_ID, - bias=True, qrnn=self.qrnn, model_dir=self.model_dir.relative_to(data_lm.path), **trn_args) + learn = language_model_learner(data_lm, AWD_LSTM, config=config, model_dir=self.model_dir.relative_to(data_lm.path), pretrained=False, **trn_args) + if self.pretrained_model is not None: + print("Loading pretrained model") + model_path = untar_data(self.pretrained_model, data=False) + fnames = [list(model_path.glob(f'*.{ext}'))[0] for ext in ['pth', 'pkl']] + learn.load_pretrained(*fnames) + learn.freeze() + if self.pretrained_fnames is not None: + print("Loading pretrained model") + fnames = [learn.path/learn.model_dir/f'{fn}.{ext}' for fn,ext in zip(self.pretrained_fnames, ['pth', 'pkl'])] + learn.load_pretrained(*fnames) + learn.freeze() # compared to standard Adam, we set beta_1 to 0.8 learn.opt_fn = partial(optim.Adam, betas=(0.8, 0.99)) learn.metrics = [accuracy_fwd, accuracy_bwd] if self.bidir else [accuracy] @@ -209,13 +218,15 @@ class LMHyperParams: args = self.tokenzier_to_fastai_args(sp_data_func=self.load_train_text, use_moses=False) try: - data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type, bs=bs) + data_lm = TextLMDataBunch.load(self.cache_dir, '.', + bs=bs) print("Tokenized data loaded") except FileNotFoundError: print("Running tokenization") data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=read_wiki_articles(trn_path), valid_df=read_wiki_articles(val_path), - classes=None, lm_type=self.lm_type, max_vocab=self.max_vocab, + classes=None, + max_vocab=self.max_vocab, bs=bs, text_cols='texts', **args) data_lm.save('.') diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 3de435d..4645ba8 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -89,16 +89,21 @@ class CLSHyperParams(LMHyperParams): print(f"Loss and accuracy using ({save_name}):", learn.validate(data_tst.valid_dl)) def create_cls_learner(self, data_clas, dps=None, **kwargs): - fastai.text.learner.default_dropout['language'] = dps or self.dps - trn_args=dict(bptt=self.bptt, clip=self.clip,) + assert self.bidir == False, "bidirectional model is not yet supported" + config = dict(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, pad_token=PAD_TOKEN_ID, qrnn=self.qrnn, bidir=self.bidir) + config.update(dps or self.dps) + trn_args=dict(bptt=self.bptt, clip=self.clip) trn_args.update(kwargs) - classifier_learner = text_classifier_learner - if self.bidir: - classifier_learner = bilm_text_classifier_learner - trn_args['bicls_head'] = self.bicls_head - learn = classifier_learner(data_clas, pad_token=PAD_TOKEN_ID, - path=self.model_dir.parent, model_dir=self.model_dir.name, - qrnn=self.qrnn, emb_sz=self.emb_sz, nh=self.nh, nl=self.nl, **trn_args) + learn = text_classifier_learner(data_clas, AWD_LSTM, config=config, + pretrained=False, path=self.model_dir.parent, model_dir=self.model_dir.name, **trn_args) + + if self.pretrained_model is not None: + print("Loading pretrained model") + model_path = untar_data(self.pretrained_model, data=False) + fnames = [list(model_path.glob(f'*.{ext}'))[0] for ext in ['pth', 'pkl']] + learn.load_pretrained(*fnames, strict=False) + learn.freeze() + learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/cls-history"), partial(SaveModelCallback, every='improvement', name='cls_best')] return learn @@ -152,12 +157,12 @@ class CLSHyperParams(LMHyperParams): args = self.tokenzier_to_fastai_args(sp_data_func=lambda: trn_df[1], use_moses=use_moses) try: if force: raise FileNotFoundError("Forcing reloading of caches") - data_lm = TextLMDataBunch.load(self.cache_dir, 'lm', lm_type=self.lm_type, bs=bs) + data_lm = TextLMDataBunch.load(self.cache_dir, 'lm', bs=bs) print(f"Tokenized data loaded, lm.trn {len(data_lm.train_ds)}, lm.val {len(data_lm.valid_ds)}") except FileNotFoundError: print(f"Running tokenization...") data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=lm_trn_df, valid_df=lm_val_df, - max_vocab=self.max_vocab, bs=bs, lm_type=self.lm_type, **args) + max_vocab=self.max_vocab, bs=bs, **args) print(f"Saving tokenized: cls.trn {len(data_lm.train_ds)}, cls.val {len(data_lm.valid_ds)}") data_lm.save('lm') From 2eee051c67aaa80878b0c2669e3ff45789591462 Mon Sep 17 00:00:00 2001 From: Marcin Date: Wed, 13 Feb 2019 13:40:05 +0100 Subject: [PATCH 02/18] Expose max length parameter --- ulmfit/train_clas.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 4645ba8..054d85a 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -24,6 +24,7 @@ from pathlib import Path from ulmfit.pretrain_lm import LMHyperParams, Tokenizers, ENC_BEST + class CLSHyperParams(LMHyperParams): # dir_path -> data/imdb/ use_test_for_validation=False @@ -38,13 +39,13 @@ class CLSHyperParams(LMHyperParams): def need_fine_tune_lm(self): return not (self.model_dir/f"enc_best.pth").exists() def train_cls(self, num_lm_epochs, unfreeze=True, bs=40, true_wd=True, drop_mul_lm=0.3, drop_mul_cls=0.5, - use_test_for_validation=False, num_cls_epochs=2, limit=None, noise=0.0): + use_test_for_validation=False, num_cls_epochs=2, limit=None, noise=0.0, cls_max_len=20*70): assert use_test_for_validation == False, "use_test_for_validation=True is not supported" data_clas, data_lm, data_tst = self.load_cls_data(bs, limit=limit, noise=noise) if self.need_fine_tune_lm: self.train_lm(num_lm_epochs, data_lm=data_lm, true_wd=true_wd, drop_mult=drop_mul_lm) - learn = self.create_cls_learner(data_clas, drop_mult=drop_mul_cls) + learn = self.create_cls_learner(data_clas, drop_mult=drop_mul_cls, max_len=cls_max_len) try: learn.load('cls_last') print("Loading last classifier") From b14a393671450d20712b6480497aa8a5c8e8d23b Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Wed, 13 Feb 2019 15:29:16 +0100 Subject: [PATCH 03/18] Add more results including full zeroshot results --- results/MLDoc.md | 46 +- results/logs/en.md | 6 + results/logs/es.md | 6 +- results/logs/fr.md | 6 +- results/logs/ja.md | 6 +- results/logs/zeroshot.md | 882 +++++++++++++++++++++++++++++++-------- results/logs/zh.md | 159 ++++++- 7 files changed, 916 insertions(+), 195 deletions(-) create mode 100644 results/logs/en.md diff --git a/results/MLDoc.md b/results/MLDoc.md index 11c5006..58e9009 100644 --- a/results/MLDoc.md +++ b/results/MLDoc.md @@ -1,27 +1,39 @@ -# non-zeroshot +## Supervised classification results on MLDoc | Model | en | de | es | fr | it | ja | ru | zh | |----------------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|------------| |LASER | 90.73 | 92.70 | 88.75 | 90.80 | 85.93 | 85.15 | 84.65 | 88.98 | |MultiCCA | 92.2 | 93.70 | 94.45 | 92.05 | 85.55 | 85.35 | 85.65 | 87.30 | -|ULMFiT | | **95.4** | **95.15** | **93.67** | **88.42** | **89.20** | **87.27** | | -|ULMFiT 100 | | 91.35 | 83.32 | 88.77 | 77.99 | 71.12 | 72.20 | | +|ULMFiT | | **95.4** | **95.15** | **93.67** | **88.42** | **89.20** | **87.27** | **90.20** | +|ULMFiT 100 | | 91.35 | 83.32 | 88.77 | 77.99 | 71.12 | 72.20 | | -# Zero shot approaches +^ - sp60k lstm nl 4 -| Model | en | de | es | fr | it | ja | ru | zh | -|----------------------|------------|-----------|-----------|-----------|-----------|-----------|-----------|------------| -|LASER 0 shot | 80.75 (en) | 87.03 (fr)| 82.60 (it)| 82.83 (de)| 73.25 (de)| 60.95 (en)| 68.83 (it)| 72.90 (de) | -|LASER base 0 shot | | 86.48 | 79.23 | 76.73 | -|ULMFiT 0 shot | | **91.97**| **85.35** | 85.54 | -|ULMFiT 100 for comp. | | 91.35 | 83.32 | 88.77 | 77.99 | 71.12 | 72.20 | | +## Zero shot approaches +| Model | de | es | fr | it | ru | zh | +|----------------------|------------|------------|-----------|-----------|-----------|-----------| +| LASER-de | | 81.40 | 81.50 | 74.53 | 64.58 | 73.20 | +| LASER-fr | 88.75 | 80.12 | | 72.58 | 67.35 | 79.40 | +| LASER-en | 87.65 | 75.48 | 84.00 | 71.18 | 66.58 | 76.65 | +| | | | | | | | +| ULMFiT on LASER-de | | **85.50** | 87.37 | **78.75** | 66.95 | 72.32 | +| ULMFiT on LASER-fr | 92.22 | 81.00 | | 76.88 | 68.33 | **84.65** | +| ULMFiT on LASER-en | **92.95** | 80.50 | **88.78** | 76.20 | **70.05** | 80.45 | +| | | | | | | | +| % impr over LASER-de | | 22% | 32% | 17% | 7% | *-3%* | +| % impr over LASER-fr | 31% | 4% | | 16% | 3% | 25% | +| % impr over LASER-en | 43% | 20% | 30% | 17% | 10% | 16% | +| ULMFiT 100 for comp. | 91.35 | 83.32 | 88.77 | 77.99 | 71.12 | | -To simulate ulmfit zero shot we add noise to the training labels to simulate training from Laser labels +All ULMFiT examples above were trained on 1k training data generated by a LASER classification model + +## Noise resistance + +| Model | en | de | es | fr | it | ja | ru | zh | +|---------------------------------|------------|-----------|-----------|-----------|-----------|-----------|-----------|------------| +|LASER 0 shot | 80.75 (en) | 87.03 (fr)| 82.60 (it)| 82.83 (de)| 73.25 (de)| 60.95 (en)| 68.83 (it)| 72.90 (de) | +|ULMFiT | | **95.4** | **95.15** | **93.67** | **88.42** | **89.20** | **87.27** | | +| % of noise | 20% | 13% | 18% | 18% | 27% | 40% | 32% | 28% | +|ULMFiT trained on 1k noisy exmp. | | 94.49 | 93.12 | 90.49 | 83.72 | 74.72 | 75.67 | | -| Model | en | de | es | fr | it | ja | ru | zh | -|----------------------|------------|-----------|-----------|-----------|-----------|-----------|-----------|------------| -|LASER 0 shot | 80.75 (en) | 87.03 (fr)| 82.60 (it)| 82.83 (de)| 73.25 (de)| 60.95 (en)| 68.83 (it)| 72.90 (de) | -|ULMFiT | | **95.4** | **95.15** | **93.67** | **88.42** | **89.20** | **87.27** | | -| Noise | 20% | 13% | 18% | 18% | 27% | 40% | 32% | 28% | -|ULMFiT noise ~ 0 shot | | 94.49 | 93.12 | 90.49 | 83.72 | 74.72 | 75.67 | | diff --git a/results/logs/en.md b/results/logs/en.md new file mode 100644 index 0000000..8f68020 --- /dev/null +++ b/results/logs/en.md @@ -0,0 +1,6 @@ +# EN +## SP30k LSTM nl 4 +### LM + +### MLDoc + diff --git a/results/logs/es.md b/results/logs/es.md index d11db90..833014e 100644 --- a/results/logs/es.md +++ b/results/logs/es.md @@ -1,6 +1,8 @@ +# ES + +## SP30k LSTM nl 4 +### LM ```` - - python -m ulmfit lm --dataset-path data/wiki/es-100 --cuda-id=0 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 30000 --lang es --qrnn=False - train 10 --bs=50 --drop_mult=0 Running tokenization Wiki text was split to 96224 articles diff --git a/results/logs/fr.md b/results/logs/fr.md index 7feda12..ff87353 100644 --- a/results/logs/fr.md +++ b/results/logs/fr.md @@ -29,8 +29,8 @@ data/wiki/fr-100/models/sp30k Saving info data/wiki/fr-100/models/sp30k/lstm_nl4.m/info.json ``` -## MLDocs -### First run +### MLDocs +#### First run MultiCCA 92.05, ulmfit 93.90 ``` python -m ulmfit cls --dataset-path data/mldoc/fr-1 --base-lm-path data/wiki/fr-100/models/sp30k/lstm_nl4.m --lang=fr --name 'nl4' --cuda-id=1 - train 20 --bs 40 @@ -86,7 +86,7 @@ Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/mod Loss and accuracy using (cls_best): [0.18914989, tensor(0.9390)] ``` -## Second run +#### Second run MultiCCA 92.05, ulmfit 93.67 ``` python -m ulmfit cls --dataset-path data/mldoc/fr-1 --base-lm-path data/wiki/fr-100/models/sp30k/lstm_nl4.m --lang=fr --name 'nl4-2nd' --cuda-id=1 - train 20 --bs 40 --num-cls-epochs=8 diff --git a/results/logs/ja.md b/results/logs/ja.md index 183943e..8d8dd4a 100644 --- a/results/logs/ja.md +++ b/results/logs/ja.md @@ -1,4 +1,6 @@ -## +# JA +## SP30k LSTM nl 4 +### LM ``` python -m ulmfit lm --dataset-path data/wiki/ja-100 --cuda-id=0 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 30000 \ --lang ja --qrnn=False - train 10 --bs=50 --drop_mult=0 @@ -27,7 +29,7 @@ data/wiki/ja-100/models/sp30k Saving info data/wiki/ja-100/models/sp30k/lstm_nl4.m/info.json ``` -## MLDoc +### MLDoc MultiCCA 85.35%, ULMFiT 89.20% ``` python -m ulmfit cls --dataset-path data/mldoc/ja-1 --base-lm-path data/wiki/ja-100/models/sp30k/lstm_nl4.m --lang=ja --name 'nl4' --cuda-id=1 - train 20 --bs 40 --num-cls-epochs=8 diff --git a/results/logs/zeroshot.md b/results/logs/zeroshot.md index e199336..0eeb0f1 100644 --- a/results/logs/zeroshot.md +++ b/results/logs/zeroshot.md @@ -1,215 +1,761 @@ -# Laser Performance +## Laser Perforamnce + Accuracy matrix: -| Train | en | de | es | fr | it | ru | zh | +| Train | en | de | es | fr | it | ru | zh | |-------|-------|-------|-------|-------|-------|-------|-------| -| en: | 90.88 | 86.48 | 67.62 | 61.98 | 69.95 | 22.95 | 11.65 | -| de: | 73.23 | 92.90 | 77.23 | 74.05 | 72.30 | 24.80 | 9.93 | -| es: | 65.62 | 80.58 | 92.03 | 73.28 | 69.03 | 34.10 | 12.58 | -| fr: | 78.35 | 85.45 | 78.20 | 89.68 | 69.85 | 33.88 | 9.68 | -| it: | 73.93 | 84.58 | 79.23 | 76.73 | 84.03 | 34.48 | 11.83 | -| ru: | 57.33 | 63.78 | 45.80 | 52.78 | 51.15 | 66.08 | 36.28 | -| zh: | 26.15 | 28.13 | 21.88 | 29.33 | 30.58 | 34.38 | 75.62 | - -# DE -Laser 0shot: 86.48, ULMFiT 0shot: 91.97 -``` -python ../../source/classify.py embed-2019-02-12/mldoc.en-en.h5 ~/workspace/ulmfit-multilingual/data/mldoc/de-1 - | Test: 86.48% | classes: 24.30 22.77 28.90 24.02 - Making train set - | Train: 85.70% | classes: 27.00 21.40 27.60 24.00 -Accuracy 0.857 - 0 1 -0 3 Tokio (Reuter) - Der Dollar ist am Donnerstag ... -1 3 Kairo (Reuter) - Die ägyptische Zentralbank se... -2 2 Bonn (Reuter) - Wegen einer Bombendrohung ist ... -3 0 Berlin (Reuter) - Die Bahn AG will mit Hilfe p... -4 3 08.15 Uhr MEZ - Deutsche Aktien nach den Rekor... +| en: | 91.48 | 87.65 | 75.48 | 84.00 | 71.18 | 66.58 | 76.65 | +| de: | 78.23 | 93.50 | 81.40 | 81.50 | 74.53 | 64.58 | 73.20 | +| es: | 71.62 | 84.00 | 93.73 | 78.90 | 73.38 | 53.33 | 55.83 | +| fr: | 81.30 | 88.75 | 80.12 | 90.85 | 72.58 | 67.35 | 79.40 | +| it: | 74.33 | 83.53 | 80.58 | 79.78 | 84.48 | 66.45 | 63.35 | +| ru: | 72.38 | 81.65 | 65.73 | 71.30 | 63.33 | 85.45 | 59.58 | +| zh: | 74.98 | 81.35 | 72.20 | 73.28 | 70.08 | 66.23 | 88.30 | - Making dev set - | Train: 85.60% | classes: 23.70 22.30 30.60 23.40 -Accuracy 0.856 - 0 1 -0 1 New York (Reuter) - Das Vertrauen der US-Verbr... -1 2 Tokio (Reuter) - Russische Patrouillenboote ha... -2 2 Paris (Reuter) - Bei der Volksabstimmung in Al... -3 2 Belgrad (Reuter) - Die serbische Polizei hat n... -4 0 München (Reuter) - Der Stuttgarter Bosch-Konze... + +## Evaluation of Laser Performance ``` -``` -python -m ulmfit cls --dataset-path data/mldoc/de-1-laser --base-lm-path data/mldoc/de-1/models/sp30k/lstm_nl4.m --lang=de --name 'nl4' --cuda-id=1 - train 0 --bs 40 --num-cls-epochs=2 +python -m ulmfit eval --glob="mldoc/*-1/models/sp60k/lstm_nl4.m" --dataset_template="{}-laser-*" --name nl4 --cuda-id=0 ✘ 130 +Max vocab: 60000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp60k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp60k/lstm_nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/zh.dev.csv +Running tokenization... +Saving tokenized: cls.trn 13500, cls.val 1500 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 60000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁是', '▁人'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp60k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp60k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp60k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp60k/lstm_nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.789124 0.620514 0.781000 +epoch train_loss valid_loss accuracy +1 0.621348 0.524669 0.828000 +epoch train_loss valid_loss accuracy +1 0.497774 0.467979 0.842000 +epoch train_loss valid_loss accuracy +1 0.445851 0.479755 0.833000 +2 0.424097 0.468968 0.826000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp60k/lstm_nl4.m +Loss and accuracy using (cls_best): [0.53502685, tensor(0.8235)] +[('data/mldoc/zh-1-laser-fr/models/sp60k/lstm_nl4.m', 0.8234999775886536)] +python -m ulmfit eval --glob="mldoc/*-1/models/sp60k/lstm_nl4.m" --dataset_template="{}-laser-*" --name nl4 --cuda-id=0 ✘ 130 +Max vocab: 60000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp60k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp60k/lstm_nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/zh.dev.csv +Running tokenization... +Saving tokenized: cls.trn 13500, cls.val 1500 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 60000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁是', '▁人'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp60k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp60k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp60k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp60k/lstm_nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.789124 0.620514 0.781000 +epoch train_loss valid_loss accuracy +1 0.621348 0.524669 0.828000 +epoch train_loss valid_loss accuracy +1 0.497774 0.467979 0.842000 +epoch train_loss valid_loss accuracy +1 0.445851 0.479755 0.833000 +2 0.424097 0.468968 0.826000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp60k/lstm_nl4.m +Loss and accuracy using (cls_best): [0.53502685, tensor(0.8235)] +[('data/mldoc/zh-1-laser-fr/models/sp60k/lstm_nl4.m', 0.8234999775886536)] +(fastaiv1) pczapla@galatea ~/w/ulmfit-multilingual ❯❯❯ python -m ulmfit eval --glob="mldoc/*-1/models/sp30k/lstm_nl4.m" --dataset_template="{}-laser-*" --name nl4 --cuda-id=0 Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser/models/sp30k/lstm_nl4.m -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser/de.dev.csv +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/models/sp30k/lstm_nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/it.dev.csv +Running tokenization... +Saving tokenized: cls.trn 13500, cls.val 1500 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', "▁&'", "'", '▁e', '▁il', '▁la', 'e', '▁in'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/models/sp30k/lstm_nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.823176 0.588192 0.802000 +epoch train_loss valid_loss accuracy +1 0.654395 0.465622 0.846000 +epoch train_loss valid_loss accuracy +1 0.536948 0.453061 0.847000 +epoch train_loss valid_loss accuracy +1 0.488410 0.454361 0.845000 +2 0.450684 0.448873 0.849000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/models/sp30k/lstm_nl4.m +Loss and accuracy using (cls_best): [0.6332891, tensor(0.7875)] +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/models/sp30k/lstm_nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/de.dev.csv Running tokenization... Saving tokenized: cls.trn 13500, cls.val 1500 Running tokenization... Saving tokenized: cls.trn 1000, cls.val 1000 Size of vocabulary: 30000 First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser/models/sp30k/lstm_nl4.m/info.json +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/models/sp30k/lstm_nl4.m/info.json Starting classifier training epoch train_loss valid_loss accuracy -1 0.671869 0.466408 0.863000 +1 0.566941 0.389549 0.882000 epoch train_loss valid_loss accuracy -1 0.518045 0.388151 0.887000 +1 0.399470 0.302616 0.898000 epoch train_loss valid_loss accuracy -1 0.375156 0.370652 0.893000 +1 0.349054 0.336955 0.900000 epoch train_loss valid_loss accuracy -1 0.339284 0.367223 0.891000 -2 0.314325 0.369492 0.891000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.25416428, tensor(0.9197)] -0.25416427850723267 -0.9197499752044678 -``` - - - - - -# ES from IT -``` -python ../../source/classify.py embed-2019-02-12/mldoc.it-it.h5 ~/workspace/ulmfit-multilingual/data/mldoc/es-1 ✘ 130 - | Test: 79.23% | classes: 25.48 16.45 24.18 33.90 - Making train set - | Train: 80.30% | classes: 27.10 19.20 22.60 31.10 -Accuracy 0.803 - 0 1 -0 3 LONDRES, 5 sep (Reuter) - El dólar se mantenía... -1 1 MADRID, 30 dic (Reuter) - La Generalitat de Va... -2 3 PARIS, 30 jun (Reuter) - La Bolsa de París neg... -3 0 MADRID, 23 dic (Reuter) - La agencia de valore... -4 0 MADRID, 4 Feb (Reuter) - El Banco Bilbao Vizca... - - Making dev set - | Train: 79.70% | classes: 25.40 17.50 26.20 30.90 -Accuracy 0.797 - 0 1 -0 0 NUEVA YORK, 11 abr (Reuter) - MCI Communicatio... -1 3 FRANCFORT, 17 jun (Reuter) - La Bolsa de Franc... -2 1 BONN, 3 jun (Reuter) - Un destacado miembro de... -3 2 LONDRES, 3 sep (Reuter) - El secretario de Def... -4 2 MADRID, 3 oct (Reuter) - Las acciones de Pryca... -``` - -``` -python -m ulmfit cls --dataset-path data/mldoc/es-1-laser-it --base-lm-path data/mldoc/es-1/models/sp30k/lstm_nl4.m --lang=es --name 'nl4' --cuda-id=1 - train 0 --bs 40 --num-cls-epochs=2 -``` - -# FR from IT -``` -python ../../source/classify.py embed-2019-02-12/mldoc.it-it.h5 ~/workspace/ulmfit-multilingual/data/mldoc/fr-1 - | Test: 76.73% | classes: 21.65 21.98 31.77 24.60 - Making train set - | Train: 79.20% | classes: 22.20 22.40 31.40 24.00 -Accuracy 0.792 - 0 1 -0 2 WASHINGTON, 13 septembre, Reuter - Les Etats-U... -1 1 PARIS, 10 juillet, Reuter - L'audit des financ... -2 2 MOSCOU, 29 mai, Reuter - Après l'accord interv... -3 2 PARIS, 1er octobre, Reuter - Le groupe communi... -4 0 LONDRES, 3 juin, Reuter - National Grid Group ... - - Making dev set - | Train: 76.60% | classes: 23.30 20.10 33.00 23.60 -Accuracy 0.766 - 0 1 -0 0 PARIS, 30 décembre, Reuter - Zodiac . Chiffre ... -1 0 AJACCIO, 11 décembre, Reuter - Une charge de 7... -2 0 BRUXELLES, 26 décembre, Reuter - 1997 s'annonc... -3 0 PARIS, 26 septembre, Reuter - Alcatel Alsthom ... -4 0 NEW YORK, 25 octobre, Reuter - La hausse plus ... -``` - -``` -python -m ulmfit cls --dataset-path data/mldoc/fr-1-laser-it --base-lm-path data/mldoc/fr-1/models/sp30k/lstm_nl4.m --lang=fr --name 'nl4' --cuda-id=1 - train 0 --bs 40 --num-cls-epochs=2 +1 0.278230 0.333488 0.896000 +2 0.275510 0.343370 0.899000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/models/sp30k/lstm_nl4.m +Loss and accuracy using (cls_best): [0.26227093, tensor(0.9222)] +Traceback (most recent call last): + File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/runpy.py", line 193, in _run_module_as_main + "__main__", mod_spec) + File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/runpy.py", line 85, in _run_code + exec(code, run_globals) + File "/home/pczapla/workspace/ulmfit-multilingual/ulmfit/__main__.py", line 58, in + fire.Fire(ULMFiT()) + File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/fire/core.py", line 127, in Fire + component_trace = _Fire(component, args, context, name) + File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/fire/core.py", line 366, in _Fire + component, remaining_args) + File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/fire/core.py", line 542, in _CallCallable + result = fn(*varargs, **kwargs) + File "/home/pczapla/workspace/ulmfit-multilingual/ulmfit/__main__.py", line 41, in eval + dataset_path = get_dataset_path(base_model, dataset_template) + File "/home/pczapla/workspace/ulmfit-multilingual/ulmfit/__main__.py", line 17, in get_dataset_path + return list(ds.parent.glob(dataset_template.format(ds.name)))[0] +IndexError: list index out of range +(fastaiv1) pczapla@galatea ~/w/ulmfit-multilingual ❯❯❯ python -m ulmfit eval --glob="mldoc/*-1/models/sp30k/lstm_nl4.m" --dataset_template="{}-laser-*" --name nl4 --cuda-id=0 ✘ 1 Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-it/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-it/models/sp30k/lstm_nl4.m -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-it/fr.dev.csv +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/models/sp30k/lstm_nl4.m +Evaluating previously trained model +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/it.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Tokenized data loaded, cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', "▁&'", "'", '▁e', '▁il', '▁la', 'e', '▁in'] +Loss and accuracy using (cls_last): [0.6332891, tensor(0.7875)] +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/models/sp30k/lstm_nl4.m +Evaluating previously trained model +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/de.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Tokenized data loaded, cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] +Loss and accuracy using (cls_last): [0.26227093, tensor(0.9222)] +Skipping data/mldoc/ja-1/models/sp30k/lstm_nl4.m as template {}-laser-* was not found +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp30k/lstm_nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/zh.dev.csv +Running tokenization... +Saving tokenized: cls.trn 13500, cls.val 1500 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp30k/lstm_nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.745636 0.601781 0.812000 +epoch train_loss valid_loss accuracy +1 0.564749 0.435314 0.851000 +epoch train_loss valid_loss accuracy +1 0.485875 0.428803 0.850000 +epoch train_loss valid_loss accuracy +1 0.405431 0.439304 0.847000 +2 0.418333 0.442639 0.845000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp30k/lstm_nl4.m +Loss and accuracy using (cls_best): [0.5289812, tensor(0.8465)] +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en/models/sp30k/lstm_nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en/fr.dev.csv Running tokenization... Saving tokenized: cls.trn 13500, cls.val 1500 Running tokenization... Saving tokenized: cls.trn 1000, cls.val 1000 Size of vocabulary: 30000 First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-it/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-it/models/sp30k/lstm_nl4.m/info.json +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en/models/sp30k/lstm_nl4.m/info.json Starting classifier training epoch train_loss valid_loss accuracy -1 0.737947 0.627607 0.793000 +1 0.669493 0.510190 0.852000 epoch train_loss valid_loss accuracy -1 0.603060 0.513449 0.831000 +1 0.464863 0.349456 0.888000 epoch train_loss valid_loss accuracy -1 0.481312 0.499689 0.828000 +1 0.396977 0.335358 0.879000 epoch train_loss valid_loss accuracy -1 0.422958 0.508330 0.825000 -2 0.408061 0.493875 0.839000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-it/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.4295174, tensor(0.8555)] -0.42951738834381104 -0.8554999828338623 -``` -# FR From EN -``` -python ../../source/classify.py embed-2019-02-12/mldoc.en-en.h5 ~/workspace/ulmfit-multilingual/data/mldoc/fr-1 - | Test: 61.98% | classes: 11.85 41.10 40.05 7.00 - Making train set - | Train: 63.70% | classes: 11.70 43.80 38.40 6.10 -Accuracy 0.637 - 0 1 -0 2 WASHINGTON, 13 septembre, Reuter - Les Etats-U... -1 1 PARIS, 10 juillet, Reuter - L'audit des financ... -2 2 MOSCOU, 29 mai, Reuter - Après l'accord interv... -3 2 PARIS, 1er octobre, Reuter - Le groupe communi... -4 0 LONDRES, 3 juin, Reuter - National Grid Group ... - - Making dev set - | Train: 61.60% | classes: 11.90 40.90 39.70 7.50 -Accuracy 0.616 - 0 1 -0 1 PARIS, 30 décembre, Reuter - Zodiac . Chiffre ... -1 0 AJACCIO, 11 décembre, Reuter - Une charge de 7... -2 1 BRUXELLES, 26 décembre, Reuter - 1997 s'annonc... -3 1 PARIS, 26 septembre, Reuter - Alcatel Alsthom ... -4 1 NEW YORK, 25 octobre, Reuter - La hausse plus ... -``` -``` - -python -m ulmfit cls --dataset-path data/mldoc/fr-1-laser --base-lm-path data/mldoc/fr-1/models/sp30k/lstm_nl4.m --lang=fr --name 'nl4-laser' --cuda-id=1 - train 0 --bs 40 --num-cls-epochs=2 +1 0.316100 0.326822 0.882000 +2 0.292052 0.326660 0.874000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en/models/sp30k/lstm_nl4.m +Loss and accuracy using (cls_best): [0.3416499, tensor(0.8878)] Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-laser.m -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/fr.dev.csv +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-fr/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-fr/models/sp30k/lstm_nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-fr/ru.dev.csv +Running tokenization... +Saving tokenized: cls.trn 9195, cls.val 1021 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', '▁и', 'е', 'и', 'й', '▁на', 'х'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-fr/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-fr/models/sp30k/lstm_nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.906124 0.592495 0.797000 +epoch train_loss valid_loss accuracy +1 0.751562 0.440800 0.842000 +epoch train_loss valid_loss accuracy +1 0.631221 0.393381 0.860000 +epoch train_loss valid_loss accuracy +1 0.582251 0.376320 0.867000 +2 0.543821 0.374095 0.860000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-fr/models/sp30k/lstm_nl4.m +Loss and accuracy using (cls_best): [1.0429544, tensor(0.6833)] +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-de/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-de/models/sp30k/lstm_nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-de/es.dev.csv +Running tokenization... +Saving tokenized: cls.trn 13013, cls.val 1445 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-de/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-de/models/sp30k/lstm_nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.667080 0.471187 0.884000 +epoch train_loss valid_loss accuracy +1 0.553853 0.329840 0.904000 +epoch train_loss valid_loss accuracy +1 0.463647 0.309136 0.907000 +epoch train_loss valid_loss accuracy +1 0.396284 0.282263 0.911000 +2 0.368159 0.287222 0.916000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-de/models/sp30k/lstm_nl4.m +Loss and accuracy using (cls_best): [0.5038375, tensor(0.8550)] +[('data/mldoc/de-1-laser-fr/models/sp30k/lstm_nl4.m', 0.922249972820282), ('data/mldoc/es-1-laser-de/models/sp30k/lstm_nl4.m', 0.8550000190734863), ('data/mldoc/fr-1-laser-en/models/sp30k/lstm_nl4.m', 0.8877500295639038), ('data/mldoc/it-1-laser-de/models/sp30k/lstm_nl4.m', 0.7875000238418579), ('data/mldoc/ru-1-laser-fr/models/sp30k/lstm_nl4.m', 0.6832500100135803), ('data/mldoc/zh-1-laser-fr/models/sp30k/lstm_nl4.m', 0.8464999794960022)] +``` +second run +``` +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-de/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-de/models/sp30k/lstm_nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-de/de.dev.csv +Running tokenization... +Saving tokenized: cls.trn 13500, cls.val 1500 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-de/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-de/models/sp30k/lstm_nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.467292 0.243158 0.919000 +epoch train_loss valid_loss accuracy +1 0.270090 0.207252 0.941000 +epoch train_loss valid_loss accuracy +1 0.201597 0.219442 0.934000 +epoch train_loss valid_loss accuracy +1 0.193163 0.199092 0.943000 +2 0.169631 0.199501 0.940000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-de/models/sp30k/lstm_nl4.m +Loss and accuracy using (cls_best): [0.16265252, tensor(0.9545)] +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-en/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-en/models/sp30k/lstm_nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-en/de.dev.csv +Running tokenization... +Saving tokenized: cls.trn 13500, cls.val 1500 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-en/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-en/models/sp30k/lstm_nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.575276 0.419917 0.879000 +epoch train_loss valid_loss accuracy +1 0.475003 0.263138 0.909000 +epoch train_loss valid_loss accuracy +1 0.345987 0.260215 0.911000 +epoch train_loss valid_loss accuracy +1 0.305776 0.268171 0.906000 +2 0.289134 0.267642 0.911000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-en/models/sp30k/lstm_nl4.m +Loss and accuracy using (cls_best): [0.23464507, tensor(0.9295)] +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/models/sp30k/lstm_nl4.m +Evaluating previously trained model +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/de.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Tokenized data loaded, cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] +Loss and accuracy using (cls_last): [0.26227093, tensor(0.9222)] +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-de/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-de/models/sp30k/lstm_nl4.m +Evaluating previously trained model +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-de/es.dev.csv +Tokenized data loaded, lm.trn 13013, lm.val 1445 +Tokenized data loaded, cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] +Loss and accuracy using (cls_last): [0.5038375, tensor(0.8550)] +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en/models/sp30k/lstm_nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en/es.dev.csv +Running tokenization... +Saving tokenized: cls.trn 13013, cls.val 1445 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en/models/sp30k/lstm_nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.784271 0.667321 0.741000 +epoch train_loss valid_loss accuracy +1 0.601108 0.471457 0.854000 +epoch train_loss valid_loss accuracy +1 0.489287 0.428631 0.854000 +epoch train_loss valid_loss accuracy +1 0.434144 0.413409 0.864000 +2 0.443724 0.385349 0.869000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en/models/sp30k/lstm_nl4.m +Loss and accuracy using (cls_best): [0.82167965, tensor(0.8050)] +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-fr/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-fr/models/sp30k/lstm_nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-fr/es.dev.csv +Running tokenization... +Saving tokenized: cls.trn 13013, cls.val 1445 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-fr/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-fr/models/sp30k/lstm_nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.752788 0.615142 0.786000 +epoch train_loss valid_loss accuracy +1 0.566108 0.403893 0.870000 +epoch train_loss valid_loss accuracy +1 0.503008 0.468810 0.865000 +epoch train_loss valid_loss accuracy +1 0.413641 0.448900 0.873000 +2 0.381155 0.413034 0.879000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-fr/models/sp30k/lstm_nl4.m +Loss and accuracy using (cls_best): [0.7937071, tensor(0.8100)] +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-de/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-de/models/sp30k/lstm_nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-de/fr.dev.csv Running tokenization... Saving tokenized: cls.trn 13500, cls.val 1500 Running tokenization... Saving tokenized: cls.trn 1000, cls.val 1000 Size of vocabulary: 30000 First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-laser.m/info.json +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-de/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-de/models/sp30k/lstm_nl4.m/info.json Starting classifier training epoch train_loss valid_loss accuracy -1 0.797327 0.697984 0.730000 +1 0.674638 0.524605 0.796000 epoch train_loss valid_loss accuracy -1 0.639780 0.582377 0.763000 +1 0.493693 0.401442 0.851000 epoch train_loss valid_loss accuracy -1 0.585295 0.582596 0.762000 +1 0.418525 0.394886 0.859000 epoch train_loss valid_loss accuracy -1 0.482629 0.582803 0.765000 -2 0.470849 0.582416 0.771000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-laser.m -Loss and accuracy using (cls_best): [0.80327946, tensor(0.6920)] +1 0.343561 0.402565 0.862000 +2 0.335855 0.418237 0.851000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-de/models/sp30k/lstm_nl4.m +Loss and accuracy using (cls_best): [0.44778627, tensor(0.8737)] +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en/models/sp30k/lstm_nl4.m +Evaluating previously trained model +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en/fr.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Tokenized data loaded, cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] +Loss and accuracy using (cls_last): [0.3416499, tensor(0.8878)] +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-fr/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-fr/models/sp30k/lstm_nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-fr/fr.dev.csv +Running tokenization... +Saving tokenized: cls.trn 13500, cls.val 1500 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-fr/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-fr/models/sp30k/lstm_nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.477812 0.332947 0.894000 +epoch train_loss valid_loss accuracy +1 0.305868 0.201659 0.937000 +epoch train_loss valid_loss accuracy +1 0.208116 0.224481 0.931000 +epoch train_loss valid_loss accuracy +1 0.146847 0.214640 0.941000 +2 0.129603 0.227498 0.929000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-fr/models/sp30k/lstm_nl4.m +Loss and accuracy using (cls_best): [0.19940722, tensor(0.9358)] +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/models/sp30k/lstm_nl4.m +Evaluating previously trained model +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/it.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Tokenized data loaded, cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', "▁&'", "'", '▁e', '▁il', '▁la', 'e', '▁in'] +Loss and accuracy using (cls_last): [0.6332891, tensor(0.7875)] +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-en/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-en/models/sp30k/lstm_nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-en/it.dev.csv +Running tokenization... +Saving tokenized: cls.trn 13500, cls.val 1500 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', "▁&'", "'", '▁e', '▁il', '▁la', 'e', '▁in'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-en/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-en/models/sp30k/lstm_nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.845312 0.660645 0.769000 +epoch train_loss valid_loss accuracy +1 0.699314 0.584146 0.786000 +epoch train_loss valid_loss accuracy +1 0.556744 0.531658 0.801000 +epoch train_loss valid_loss accuracy +1 0.503091 0.529716 0.805000 +2 0.474142 0.520058 0.806000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-en/models/sp30k/lstm_nl4.m +Loss and accuracy using (cls_best): [0.7639212, tensor(0.7620)] +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-fr/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-fr/models/sp30k/lstm_nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-fr/it.dev.csv +Running tokenization... +Saving tokenized: cls.trn 13500, cls.val 1500 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', "▁&'", "'", '▁e', '▁il', '▁la', 'e', '▁in'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-fr/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-fr/models/sp30k/lstm_nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.773207 0.568426 0.803000 +epoch train_loss valid_loss accuracy +1 0.570457 0.516704 0.821000 +epoch train_loss valid_loss accuracy +1 0.527280 0.460192 0.840000 +epoch train_loss valid_loss accuracy +1 0.469201 0.461563 0.841000 +2 0.458892 0.443310 0.836000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-fr/models/sp30k/lstm_nl4.m +Loss and accuracy using (cls_best): [0.80693215, tensor(0.7688)] +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-de/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-de/models/sp30k/lstm_nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-de/ru.dev.csv +Running tokenization... +Saving tokenized: cls.trn 9195, cls.val 1021 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', '▁и', 'е', 'и', 'й', '▁на', 'х'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-de/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-de/models/sp30k/lstm_nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.878202 0.549530 0.815000 +epoch train_loss valid_loss accuracy +1 0.747663 0.439798 0.860000 +epoch train_loss valid_loss accuracy +1 0.610381 0.391122 0.878000 +epoch train_loss valid_loss accuracy +1 0.563902 0.393633 0.880000 +2 0.515117 0.403987 0.878000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-de/models/sp30k/lstm_nl4.m +Loss and accuracy using (cls_best): [1.3181443, tensor(0.6695)] +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-en/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-en/models/sp30k/lstm_nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-en/ru.dev.csv +Running tokenization... +Saving tokenized: cls.trn 9195, cls.val 1021 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', '▁и', 'е', 'и', 'й', '▁на', 'х'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-en/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-en/models/sp30k/lstm_nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.897468 0.570228 0.801000 +epoch train_loss valid_loss accuracy +1 0.704874 0.560132 0.812000 +epoch train_loss valid_loss accuracy +1 0.595008 0.507041 0.816000 +epoch train_loss valid_loss accuracy +1 0.484754 0.479213 0.825000 +2 0.454896 0.501114 0.824000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-en/models/sp30k/lstm_nl4.m +Loss and accuracy using (cls_best): [1.1765001, tensor(0.7005)] +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-fr/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-fr/models/sp30k/lstm_nl4.m +Evaluating previously trained model +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-fr/ru.dev.csv +Tokenized data loaded, lm.trn 9195, lm.val 1021 +Tokenized data loaded, cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', '▁и', 'е', 'и', 'й', '▁на', 'х'] +Loss and accuracy using (cls_last): [1.0429544, tensor(0.6833)] +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-de/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-de/models/sp30k/lstm_nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-de/zh.dev.csv +Running tokenization... +Saving tokenized: cls.trn 13500, cls.val 1500 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-de/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-de/models/sp30k/lstm_nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.759435 0.762386 0.707000 +epoch train_loss valid_loss accuracy +1 0.631534 0.591862 0.786000 +epoch train_loss valid_loss accuracy +1 0.534237 0.589429 0.801000 +epoch train_loss valid_loss accuracy +1 0.454291 0.589220 0.799000 +2 0.446990 0.586956 0.804000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-de/models/sp30k/lstm_nl4.m +Loss and accuracy using (cls_best): [0.8401224, tensor(0.7232)] +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-en/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-en/models/sp30k/lstm_nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-en/zh.dev.csv +Running tokenization... +Saving tokenized: cls.trn 13500, cls.val 1500 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-en/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-en/models/sp30k/lstm_nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.827517 0.821250 0.712000 +epoch train_loss valid_loss accuracy +1 0.636761 0.656195 0.772000 +epoch train_loss valid_loss accuracy +1 0.582199 0.675501 0.769000 +epoch train_loss valid_loss accuracy +1 0.511542 0.634232 0.764000 +2 0.508244 0.647197 0.771000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-en/models/sp30k/lstm_nl4.m +Loss and accuracy using (cls_best): [0.5421255, tensor(0.8045)] +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp30k/lstm_nl4.m +Evaluating previously trained model +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/zh.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Tokenized data loaded, cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是'] +Loss and accuracy using (cls_last): [0.5289812, tensor(0.8465)] +OrderedDict([('data/mldoc/de-1-laser-de/models/sp30k/lstm_nl4.m', + 0.9545000195503235), + ('data/mldoc/de-1-laser-en/models/sp30k/lstm_nl4.m', + 0.9294999837875366), + ('data/mldoc/de-1-laser-fr/models/sp30k/lstm_nl4.m', + 0.922249972820282), + ('data/mldoc/es-1-laser-de/models/sp30k/lstm_nl4.m', + 0.8550000190734863), + ('data/mldoc/es-1-laser-en/models/sp30k/lstm_nl4.m', + 0.8050000071525574), + ('data/mldoc/es-1-laser-fr/models/sp30k/lstm_nl4.m', + 0.8100000023841858), + ('data/mldoc/fr-1-laser-de/models/sp30k/lstm_nl4.m', + 0.8737499713897705), + ('data/mldoc/fr-1-laser-en/models/sp30k/lstm_nl4.m', + 0.8877500295639038), + ('data/mldoc/fr-1-laser-fr/models/sp30k/lstm_nl4.m', + 0.9357500076293945), + ('data/mldoc/it-1-laser-de/models/sp30k/lstm_nl4.m', + 0.7875000238418579), + ('data/mldoc/it-1-laser-en/models/sp30k/lstm_nl4.m', + 0.7620000243186951), + ('data/mldoc/it-1-laser-fr/models/sp30k/lstm_nl4.m', + 0.768750011920929), + ('data/mldoc/ru-1-laser-de/models/sp30k/lstm_nl4.m', + 0.6694999933242798), + ('data/mldoc/ru-1-laser-en/models/sp30k/lstm_nl4.m', + 0.7005000114440918), + ('data/mldoc/ru-1-laser-fr/models/sp30k/lstm_nl4.m', + 0.6832500100135803), + ('data/mldoc/zh-1-laser-de/models/sp30k/lstm_nl4.m', + 0.7232499718666077), + ('data/mldoc/zh-1-laser-en/models/sp30k/lstm_nl4.m', + 0.8044999837875366), + ('data/mldoc/zh-1-laser-fr/models/sp30k/lstm_nl4.m', + 0.8464999794960022)]) +``` + + + +### Building dataset + +``` +for SRC_LANG in en de fr; do ✘ 130 + for LANG in en de es fr it ru zh; do + echo $LANG from $SRC_LANG + python ../../source/classify.py embed/mldoc.${SRC_LANG}-${SRC_LANG}.h5 ~/workspace/ulmfit-multilingual/data/mldoc/${LANG}-1 | grep Test: + done +done + +en from en + | Test: 91.48% | classes: 23.77 24.90 26.25 25.07 +de from en + | Test: 87.65% | classes: 21.98 24.45 27.65 25.93 +es from en + | Test: 75.48% | classes: 21.60 15.82 22.10 40.48 +fr from en + | Test: 84.00% | classes: 23.18 29.12 27.90 19.80 +it from en + | Test: 71.18% | classes: 23.65 22.88 25.68 27.80 +ru from en + | Test: 66.58% | classes: 29.48 13.78 34.52 22.23 +zh from en + | Test: 76.65% | classes: 30.25 31.30 13.93 24.52 +en from de + | Test: 78.23% | classes: 31.80 17.73 30.15 20.32 +de from de + | Test: 93.50% | classes: 24.45 25.45 26.00 24.10 +es from de + | Test: 81.40% | classes: 24.15 25.77 20.12 29.95 +fr from de + | Test: 81.50% | classes: 25.52 29.45 27.45 17.57 +it from de + | Test: 74.53% | classes: 24.70 27.25 22.43 25.62 +ru from de + | Test: 64.58% | classes: 45.62 9.12 26.73 18.52 +zh from de + | Test: 73.20% | classes: 31.20 43.38 7.60 17.82 +en from fr + | Test: 81.30% | classes: 28.95 18.02 24.98 28.05 +de from fr + | Test: 88.75% | classes: 24.00 23.75 24.85 27.40 +es from fr + | Test: 80.12% | classes: 24.50 14.82 18.40 42.27 +fr from fr + | Test: 90.85% | classes: 24.50 24.75 24.68 26.07 +it from fr + | Test: 72.58% | classes: 25.45 24.10 17.50 32.95 +ru from fr + | Test: 67.35% | classes: 47.15 13.62 16.68 22.55 +zh from fr + | Test: 79.40% | classes: 33.60 31.12 9.07 26.20 ``` diff --git a/results/logs/zh.md b/results/logs/zh.md index c341d56..c3760e2 100644 --- a/results/logs/zh.md +++ b/results/logs/zh.md @@ -1,6 +1,159 @@ +# ZH - - +## SP30k LSTM nl 4 +### LM ``` -python -m ulmfit lm --dataset-path data/wiki/zh-100 --cuda-id=0 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 60000 --lang zh --qrnn=False - train 10 --bs=50 --drop_mult=0 + python -m ulmfit lm --dataset-path data/wiki/zh-100 --cuda-id=0 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 30000 --lang zh --qrnn=False - train 10 --bs=50 --drop_mult=0 +Max vocab: 30000 +Cache dir: data/wiki/zh-100/models/sp30k +Model dir: data/wiki/zh-100/models/sp30k/lstm_nl4.m +Tokenized data loaded +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': None, 'pretrained_model': None, 'drop_mult': 0} dps: [0.25 0.1 0.2 0.02 0.15] +Training lm from random weights +epoch train_loss valid_loss accuracy +1 2.736679 3.050473 0.428462 +2 2.664505 3.011505 0.432414 +3 2.607435 2.942389 0.439985 +4 2.561503 2.851523 0.451965 +5 2.499060 2.798222 0.459438 +6 2.387191 2.720054 0.471021 +7 2.356725 2.648299 0.479029 +8 2.301895 2.553860 0.493597 +9 2.275601 2.481724 0.505979 +10 2.187606 2.465159 0.509590 +``` +### MLDoc +``` +python -m ulmfit cls --dataset-path data/mldoc/zh-1 --base-lm-path data/wiki/zh-100/models/sp30k/lstm_nl4.m --lang=zh --name 'nl4' --cuda-id=0 - train 20 --bs 40 --num-cls-epochs=2 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4.m +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Tokenized data loaded, cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/zh-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/zh-100/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/zh-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/zh-100/models/sp30k/lstm_nl4.m/../itos')] +epoch train_loss valid_loss accuracy +1 2.604460 2.225315 0.546099 +epoch train_loss valid_loss accuracy +1 2.240892 2.020697 0.578796 +2 2.025043 1.816424 0.613192 +3 1.832658 1.646025 0.640532 +4 1.746628 1.530125 0.659058 +5 1.621672 1.425179 0.675305 +6 1.544814 1.345650 0.689195 +7 1.464704 1.271710 0.702200 +8 1.412583 1.204830 0.714764 +9 1.332440 1.147108 0.725389 +10 1.327941 1.092910 0.736447 +11 1.227284 1.039441 0.747662 +12 1.200814 0.991910 0.758105 +13 1.161579 0.947898 0.768121 +14 1.100010 0.908599 0.776732 +15 1.059006 0.872309 0.785161 +16 1.045412 0.844972 0.791998 +17 1.026688 0.824872 0.796891 +18 1.013831 0.812786 0.799699 +19 0.978586 0.807678 0.800954 +20 0.982473 0.805671 0.801201 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.637427 0.505143 0.836000 +epoch train_loss valid_loss accuracy +1 0.471189 0.317678 0.887000 +epoch train_loss valid_loss accuracy +1 0.384985 0.288901 0.904000 +epoch train_loss valid_loss accuracy +1 0.316358 0.275456 0.906000 +2 0.295534 0.278589 0.907000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4.m +Loss and accuracy using (cls_best): [0.28411642, tensor(0.9020)] +0.2841164171695709 +0.9020000100135803 +``` + +## SP60k LSTM nl 4 +### LM +``` +Wiki text was split to 153503 articles +Wiki text was split to 145 articles +Size of vocabulary: 60000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁是', '▁人'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': None, 'pretrained_model': None, 'drop_mult': 0} dps: [0.25 0.1 0.2 0.02 0.15] +Training lm from random weights +epoch train_loss valid_loss accuracy +1 3.312704 3.701317 0.334740 +2 3.212988 3.648671 0.336709 +3 3.060103 3.584413 0.344427 +4 3.108131 3.477978 0.356738 +5 2.952951 3.410785 0.365901 +6 2.919397 3.325265 0.376316 +7 2.839392 3.224750 0.391707 +8 2.750095 3.132644 0.404416 +9 2.805704 3.066595 0.415245 +10 2.653435 3.055314 0.417736 +data/wiki/zh-100/models/sp60k +Saving info data/wiki/zh-100/models/sp60k/lstm_nl4.m/info.json +``` +### MLDoc +``` +python -m ulmfit cls --dataset-path data/mldoc/zh-1 --base-lm-path data/wiki/zh-100/models/sp60k/lstm_nl4.m --lang=zh --name 'nl4' --cu +da-id=0 - train 20 --bs 40 --num-cls-epochs=2 +Max vocab: 60000 +Cache dir: /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp60k +Model dir: /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp60k/lstm_nl4.m +Loading validation /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Tokenized data loaded, cls.trn 1000, cls.val 1000 +Size of vocabulary: 60000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁是', '▁人'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/zh-100/models/sp60k/lstm_nl4.m/lm_best'), Po +sixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/zh-100/models/sp60k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +Training lm from: [PosixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/zh-100/models/sp60k/lstm_nl4.m/lm_best'), PosixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/zh- +100/models/sp60k/lstm_nl4.m/../itos')] +epoch train_loss valid_loss accuracy +1 3.055914 2.690310 0.467917 +epoch train_loss valid_loss accuracy +1 2.713421 2.464873 0.503386 +2 2.429520 2.215309 0.543961 +3 2.247576 2.010849 0.578106 +4 2.083628 1.853473 0.602419 +5 1.969939 1.734762 0.621440 +6 1.904438 1.624005 0.640240 +7 1.783416 1.526202 0.656981 +8 1.719215 1.445780 0.671753 +9 1.621891 1.366912 0.687187 +10 1.589463 1.295759 0.701207 +11 1.510032 1.223578 0.716387 +12 1.404720 1.160607 0.729603 +13 1.414636 1.107378 0.741273 +14 1.364716 1.056422 0.753112 +15 1.327804 1.011525 0.763934 +16 1.255990 0.976447 0.771864 +17 1.181438 0.951213 0.778309 +18 1.192709 0.936060 0.781858 +19 1.190164 0.928613 0.783513 +20 1.172130 0.927612 0.783722 +/home/n-waves/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp60k +Saving info /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp60k/lstm_nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.646537 0.516221 0.836000 +epoch train_loss valid_loss accuracy +1 0.441884 0.361802 0.873000 +epoch train_loss valid_loss accuracy +1 0.376583 0.318426 0.893000 +epoch train_loss valid_loss accuracy +1 0.280910 0.314279 0.889000 +2 0.308887 0.309718 0.903000 +Saving models at /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp60k/lstm_nl4.m +Loss and accuracy using (cls_last): [0.30276635, tensor(0.8978)] ``` \ No newline at end of file From 1340f4235ca3d08b82b9b1527f52cf6338670fe5 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Wed, 13 Feb 2019 15:29:55 +0100 Subject: [PATCH 04/18] Improve ulmfit eval to allow for zeroshot laser evaluation --- ulmfit/__main__.py | 42 +++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/ulmfit/__main__.py b/ulmfit/__main__.py index fb56560..bbf69cb 100644 --- a/ulmfit/__main__.py +++ b/ulmfit/__main__.py @@ -1,5 +1,7 @@ import gc +import pprint import shutil +from collections import OrderedDict from functools import wraps import fire @@ -12,15 +14,16 @@ class FireView: for k,v in kwargs.items(): setattr(self, k, v) -def get_dataset_path(p): - return [x for x in p.parents if x.name == "models"][0].parent - def get_lang_from_dataset_path(ds): lang,*_ = ds.name.split("-") if len(lang) == 2: return lang return "en" +def get_dataset_path(p, dataset_template): + ds = [x for x in p.parents if x.name == "models"][0].parent + return ds.parent.glob(dataset_template.format(ds.name)) + class ULMFiT: @wraps(LMHyperParams) def lm(self, dataset_path, **changes): @@ -34,23 +37,24 @@ class ULMFiT: params = CLSHyperParams.from_lm(dataset_path, base_lm_path, **changes) return FireView(train=params.train_cls, validate_cls=params.validate_cls) - def eval(self, glob="mldoc/*-1/models/sp30k/lstm_nl4.m", name="tmp-100", cuda_id=0, **trn_params): - results={} - for base_model in Path("data").glob(glob): - dataset_path = get_dataset_path(base_model) - lang = get_lang_from_dataset_path(dataset_path) - params = CLSHyperParams.from_lm(dataset_path, base_model, lang=lang, name=name, cuda_id=cuda_id) - key = str(params.model_dir.relative_to(Path.cwd())) - if params.model_dir.exists(): - print("Evaluating previously trained model") - results[key] = params.validate_cls()[1] - else: - print("Training") - results[key] = params.train_cls(num_lm_epochs=0, **trn_params)[1] - params = None - gc.collect() + def eval(self, glob="mldoc/*-1/models/sp30k/lstm_nl4.m", dataset_template='{}', name="tmp-100", cuda_id=0, **trn_params): + results = OrderedDict() + for base_model in sorted(Path("data").glob(glob)): + for dataset_path in sorted(get_dataset_path(base_model, dataset_template)): + lang = get_lang_from_dataset_path(dataset_path) + params = CLSHyperParams.from_lm(dataset_path, base_model, lang=lang, name=name, cuda_id=cuda_id) + key = str(params.model_dir.relative_to(Path.cwd())) + if params.model_dir.exists(): + print("Evaluating previously trained model") + results[key] = params.validate_cls()[1] + else: + print("Training") + results[key] = params.train_cls(num_lm_epochs=0, **trn_params)[1] + del params + gc.collect() - print(list(sorted(results.items()))) + pprint.pprint(results) +# python -m ulmfit cls --dataset-path data/mldoc/de-1-laser --base-lm-path data/mldoc/de-1/models/sp30k/lstm_nl4.m --lang=de --name 'nl4' --cuda-id=1 - train 0 --bs 40 --num-cls-epochs=2 if __name__ == '__main__': fire.Fire(ULMFiT()) \ No newline at end of file From 9d893e393be739752a9a4f16252bdda040b969c4 Mon Sep 17 00:00:00 2001 From: Marcin Date: Thu, 14 Feb 2019 00:00:43 +0100 Subject: [PATCH 05/18] Add QRNN results --- results/logs/qrnn-de.md | 22 +++++++++++++++++++++ results/logs/qrnn-en.md | 44 +++++++++++++++++++++++++++++++++++++++++ results/logs/qrnn-es.md | 25 +++++++++++++++++++++++ results/logs/qrnn-ru.md | 24 ++++++++++++++++++++++ 4 files changed, 115 insertions(+) create mode 100644 results/logs/qrnn-de.md create mode 100644 results/logs/qrnn-en.md create mode 100644 results/logs/qrnn-es.md create mode 100644 results/logs/qrnn-ru.md diff --git a/results/logs/qrnn-de.md b/results/logs/qrnn-de.md new file mode 100644 index 0000000..9addcd1 --- /dev/null +++ b/results/logs/qrnn-de.md @@ -0,0 +1,22 @@ +# QRNN DE +## SP30k nl +### LM +``` +python -m ulmfit lm --dataset-path data/wiki/de-100 --bidir=False --qrnn=True --nl 4 --tokenizer='sp' --max-vocab 30000 --lang de --name 'nl4' --cuda-id=0 - train 10 --drop-mult=0 --bs=50 + +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', "▁&'", 'en', 's', '-'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Training lm from random weights +epoch train_loss valid_loss accuracy +1 2.790653 2.867094 0.511392 +2 2.742032 2.843288 0.510885 +3 2.696114 2.833874 0.512062 +4 2.671780 2.786312 0.516448 +5 2.611292 2.725993 0.522723 +6 2.542737 2.655713 0.530968 +7 2.572076 2.582141 0.539928 +8 2.465960 2.509654 0.549987 +9 2.405682 2.448580 0.558674 +10 2.339395 2.428111 0.562502 +``` diff --git a/results/logs/qrnn-en.md b/results/logs/qrnn-en.md new file mode 100644 index 0000000..06d1247 --- /dev/null +++ b/results/logs/qrnn-en.md @@ -0,0 +1,44 @@ +# QRNN EN +## SP30k nl 4 +### LM + +``` +python -m ulmfit lm --dataset-path data/wiki/wikitext-103 --bidir=False --qrnn=True --nl 4 --tokenizer='sp' --max-vocab 30000 --lang en --name 'nl4' --cuda-id=1 - train 10 --drop-mult=0 --bs=50 + +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁the', '▁,', '▁.', 's', '▁of', '▁and', '▁in', '▁to', '▁a', 'ed'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.5} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Training lm from random weights +epoch train_loss valid_loss accuracy +1 3.184221 3.256314 0.438527 +2 3.084555 3.241498 0.435628 +3 3.099060 3.258447 0.435060 +4 3.119621 3.220939 0.437597 +5 3.073662 3.165012 0.445108 +6 2.938047 3.086962 0.452921 +7 2.920506 2.998151 0.462940 +8 2.920506 2.899240 0.474378 +9 2.862836 2.835098 0.485305 +10 2.891070 2.810929 0.489867 +``` + +### LM, BS=128, drop-mult=0.5 +``` +python -m ulmfit lm --dataset-path data/wiki/wikitext-103 --bidir=False --qrnn=True --nl 4 --tokenizer='sp' --max-vocab 30000 --lang en --name 'nl4-bs128' --cuda-id=1 - train 10 --drop-mult=0.5 --bs=128 + +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁the', '▁,', '▁.', 's', '▁of', '▁and', '▁in', '▁to', '▁a', 'ed'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.5} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Training lm from random weights +epoch train_loss valid_loss accuracy +1 3.413345 3.280860 0.433011 +2 3.219606 3.129479 0.444172 +3 3.136091 3.094905 0.448493 +4 3.145281 3.033001 0.452830 +5 3.100366 2.980189 0.458984 +6 3.062894 2.923044 0.464841 +7 3.001627 2.834753 0.475316 +8 2.979051 2.792044 0.480915 +9 2.933140 2.733279 0.488346 +10 2.964397 2.720861 0.490423 +``` diff --git a/results/logs/qrnn-es.md b/results/logs/qrnn-es.md new file mode 100644 index 0000000..c103458 --- /dev/null +++ b/results/logs/qrnn-es.md @@ -0,0 +1,25 @@ +# QRNN ES + +## SP30k nl 4 +### LM +``` +python -m ulmfit lm --dataset-path data/wiki/es-100 --bidir=False --qrnn=True --nl 4 --tokenizer='sp' --max-vocab 30000 --lang es --name 'nl4' --cuda-id=0 - train 10 --drop-mult=0 --bs=50 + +Wiki text was split to 161509 articles +Wiki text was split to 78 articles +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', '▁la', '▁el', '▁en', '▁y', 's', '▁a', "▁&'"] +Training args: {'clip': 0.12, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Training lm from random weights +epoch train_loss valid_loss accuracy +1 3.067289 3.640350 0.357276 +2 2.958243 3.619773 0.358111 +3 3.033412 3.587700 0.359495 +4 2.933573 3.525202 0.367685 +5 2.904549 3.467990 0.372583 +6 2.798806 3.409506 0.380045 +7 2.733132 3.303108 0.391922 +8 2.675272 3.224150 0.401143 +9 2.635299 3.166430 0.410160 +10 2.656724 3.145599 0.413176 +``` diff --git a/results/logs/qrnn-ru.md b/results/logs/qrnn-ru.md new file mode 100644 index 0000000..36ae8d4 --- /dev/null +++ b/results/logs/qrnn-ru.md @@ -0,0 +1,24 @@ +# QRNN RU +## SP30k nl4 +### LM +``` +python -m ulmfit lm --dataset-path data/wiki/ru-100 --bidir=False --qrnn=True --nl 4 --tokenizer='sp' --max-vocab 30000 --lang ru --name 'nl4' --cuda-id=0 - train 10 --drop-mult=0 --bs=50 + +Wiki text was split to 193047 articles +Wiki text was split to 460 articles +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', '▁и', 'е', 'и', 'й', '▁на', '▁с'] +Training args: {'clip': 0.12, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Training lm from random weights +epoch train_loss valid_loss accuracy +1 3.273207 3.350111 0.429702 +2 3.169897 3.274238 0.433682 +3 3.162197 3.247077 0.435900 +4 3.131630 3.168798 0.445252 +5 3.042942 3.096774 0.453532 +6 2.950550 3.002989 0.465113 +7 2.833593 2.902871 0.478954 +8 2.829737 2.805592 0.492138 +9 2.746991 2.733609 0.503711 +10 2.687201 2.708546 0.508050 +``` From b609951561979bf1e6344336d4e4776e43c4217d Mon Sep 17 00:00:00 2001 From: Marcin Date: Thu, 14 Feb 2019 00:00:56 +0100 Subject: [PATCH 06/18] Fix path of pretrained model --- ulmfit/pretrain_lm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 61a81ad..b89a994 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -214,7 +214,7 @@ class LMHyperParams: learn.freeze() if self.pretrained_fnames is not None: print("Loading pretrained model") - fnames = [learn.path/learn.model_dir/f'{fn}.{ext}' for fn,ext in zip(self.pretrained_fnames, ['pth', 'pkl'])] + fnames = [f'{fn}.{ext}' for fn,ext in zip(self.pretrained_fnames, ['pth', 'pkl'])] learn.load_pretrained(*fnames) learn.freeze() # compared to standard Adam, we set beta_1 to 0.8 From 0535ef169a659967a10d58d0c8824e62c5308a97 Mon Sep 17 00:00:00 2001 From: Marcin Date: Thu, 14 Feb 2019 11:05:02 +0100 Subject: [PATCH 07/18] QRNN mldoc results for de, en and es --- results/logs/qrnn-de.md | 64 +++++++++++++++++++++++++++++++++++++++++ results/logs/qrnn-en.md | 64 +++++++++++++++++++++++++++++++++++++++++ results/logs/qrnn-es.md | 63 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 191 insertions(+) diff --git a/results/logs/qrnn-de.md b/results/logs/qrnn-de.md index 9addcd1..f033ce1 100644 --- a/results/logs/qrnn-de.md +++ b/results/logs/qrnn-de.md @@ -20,3 +20,67 @@ epoch train_loss valid_loss accuracy 9 2.405682 2.448580 0.558674 10 2.339395 2.428111 0.562502 ``` + +### MLDocs +``` +python -m ulmfit cls --dataset-path data/mldoc/de-1 --cuda-id=0 --base-lm-path data-filtered/data/wiki/de-100/models/sp30k/qrnn_nl4.m --lang=de --name 'nl4' - train 20 --bs 40 --cls-max-len 700 + +Max vocab: 30000 +Cache dir: /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/de-1/models/sp30k +Model dir: /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/qrnn_nl4.m +Loading validation /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/de-1/de.dev.csv +Running tokenization... +Saving tokenized: cls.trn 13500, cls.val 1500 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', "▁&'", 'en', 's', '-'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +Training lm from: [PosixPath('/home/marcin/github/n-waves/ulmfit-multilingual/data-filtered/data/wiki/de-100/models/sp30k/qrnn_nl4.m/lm_best'), PosixPath('/home/marcin/github/n-waves/ulmfit-multilingual/data-filtered/data/wiki/de-100/models/sp30k/qrnn_nl4.m/../itos')] +epoch train_loss valid_loss accuracy +1 3.450698 2.601732 0.527671 +epoch train_loss valid_loss accuracy +1 2.888087 2.477170 0.542949 +2 2.621279 2.300024 0.568743 +3 2.313220 2.120824 0.592728 +4 2.176746 1.973596 0.613343 +5 2.114441 1.857317 0.628628 +6 2.022593 1.765069 0.642017 +7 1.936942 1.696150 0.651549 +8 1.860200 1.622848 0.661923 +9 1.795039 1.549579 0.673416 +10 1.740739 1.500053 0.681305 +11 1.695835 1.448141 0.689201 +12 1.605702 1.402924 0.697096 +13 1.582328 1.354327 0.706123 +14 1.548034 1.316290 0.712870 +15 1.496170 1.282155 0.719413 +16 1.514243 1.255556 0.724801 +17 1.482411 1.236461 0.728380 +18 1.458308 1.223498 0.730708 +19 1.422691 1.218288 0.731713 +20 1.380592 1.217068 0.731893 +/home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/de-1/models/sp30k +Saving info /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/qrnn_nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.529402 0.376163 0.900000 +Better model found at epoch 1 with val_loss value: 0.3761630356311798. +epoch train_loss valid_loss accuracy +1 0.290838 0.252989 0.916000 +Better model found at epoch 1 with val_loss value: 0.25298893451690674. +epoch train_loss valid_loss accuracy +1 0.184352 0.204892 0.941000 +Better model found at epoch 1 with val_loss value: 0.20489171147346497. +epoch train_loss valid_loss accuracy +1 0.113328 0.204136 0.947000 +Better model found at epoch 1 with val_loss value: 0.20413607358932495. +2 0.106220 0.200674 0.949000 +Better model found at epoch 2 with val_loss value: 0.20067360997200012. +Saving models at /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/qrnn_nl4.m +Loss and accuracy using (cls_best): [0.15208693, tensor(0.9532)] +0.15208692848682404 +0.953249990940094 +``` diff --git a/results/logs/qrnn-en.md b/results/logs/qrnn-en.md index 06d1247..9c76223 100644 --- a/results/logs/qrnn-en.md +++ b/results/logs/qrnn-en.md @@ -42,3 +42,67 @@ epoch train_loss valid_loss accuracy 9 2.933140 2.733279 0.488346 10 2.964397 2.720861 0.490423 ``` + +### MLDocs +``` +python -m ulmfit cls --dataset-path data/mldoc/en-1 --cuda-id=0 --base-lm-path data-filtered/data/wiki/wikitext-103/models/sp30k/qrnn_nl4.m --lang=en --name 'nl4' - train 20 --bs 40 --cls-max-len 700 + +Max vocab: 30000 +Cache dir: /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/en-1/models/sp30k +Model dir: /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/en-1/models/sp30k/qrnn_nl4.m +Loading validation /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/en-1/en.dev.csv +Running tokenization... +Saving tokenized: cls.trn 13500, cls.val 1500 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁the', '▁,', '▁.', 's', '▁of', '▁and', '▁in', '▁to', '▁a', 'ed'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +Training lm from: [PosixPath('/home/marcin/github/n-waves/ulmfit-multilingual/data-filtered/data/wiki/wikitext-103/models/sp30k/qrnn_nl4.m/lm_best'), PosixPath('/home/marcin/github/n-waves/ulmfit-multilingual/data-filtered/data/wiki/wikitext-103/models/sp30k/qrnn_nl4.m/. +./itos')] +epoch train_loss valid_loss accuracy +1 4.459886 3.692770 0.364677 +epoch train_loss valid_loss accuracy +1 3.962907 3.560222 0.379027 +2 3.673292 3.378484 0.402066 +3 3.460093 3.191662 0.424295 +4 3.296515 3.030681 0.442995 +5 3.161650 2.891829 0.459052 +6 3.022674 2.776469 0.473280 +7 2.974365 2.686321 0.484403 +8 2.869587 2.593854 0.496297 +9 2.785321 2.509093 0.506853 +10 2.677728 2.440328 0.516178 +11 2.641243 2.371950 0.525810 +12 2.652385 2.320008 0.533105 +13 2.547195 2.261057 0.542046 +14 2.491570 2.216933 0.548810 +15 2.454437 2.179364 0.555077 +16 2.414449 2.147612 0.559972 +17 2.358593 2.125351 0.563405 +18 2.362696 2.111580 0.565614 +19 2.341626 2.104268 0.566749 +20 2.342680 2.102918 0.566966 +/home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/en-1/models/sp30k +Saving info /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/en-1/models/sp30k/qrnn_nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.622379 0.422002 0.882000 +Better model found at epoch 1 with val_loss value: 0.42200201749801636. +epoch train_loss valid_loss accuracy +1 0.313018 0.275563 0.908000 +Better model found at epoch 1 with val_loss value: 0.27556276321411133. +epoch train_loss valid_loss accuracy +1 0.241521 0.174606 0.933000 +Better model found at epoch 1 with val_loss value: 0.1746061146259308. +epoch train_loss valid_loss accuracy +1 0.125556 0.170286 0.940000 +Better model found at epoch 1 with val_loss value: 0.17028628289699554. +2 0.107322 0.181366 0.939000 +Saving models at /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/en-1/models/sp30k/qrnn_nl4.m +Loss and accuracy using (cls_best): [0.18917121, tensor(0.9388)] +0.1891712099313736 +0.9387500286102295 +``` diff --git a/results/logs/qrnn-es.md b/results/logs/qrnn-es.md index c103458..835446c 100644 --- a/results/logs/qrnn-es.md +++ b/results/logs/qrnn-es.md @@ -23,3 +23,66 @@ epoch train_loss valid_loss accuracy 9 2.635299 3.166430 0.410160 10 2.656724 3.145599 0.413176 ``` + +### MLDocs +``` +python -m ulmfit cls --dataset-path data/mldoc/es-1 --cuda-id=0 --base-lm-path data-filtered/data/wiki/es-100/models/sp30k/qrnn_nl4.m --lang=es --name 'nl4' - train 20 --bs 40 --cls-max-len 700 + +Max vocab: 30000 +Cache dir: /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/es-1/models/sp30k +Model dir: /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/qrnn_nl4.m +Loading validation /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv +Running tokenization... +Saving tokenized: cls.trn 13013, cls.val 1445 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', '▁la', '▁el', '▁en', '▁y', 's', '▁a', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +Training lm from: [PosixPath('/home/marcin/github/n-waves/ulmfit-multilingual/data-filtered/data/wiki/es-100/models/sp30k/qrnn_nl4.m/lm_best'), PosixPath('/home/marcin/github/n-waves/ulmfit-multilingual/data-filtered/data/wiki/es-100/models/sp30k/qrnn_nl4.m/../itos')] +epoch train_loss valid_loss accuracy +1 3.352874 2.367255 0.514858 +epoch train_loss valid_loss accuracy +1 2.796090 2.203233 0.536513 +2 2.515840 1.970145 0.576640 +3 2.198857 1.774013 0.610990 +4 2.035614 1.633484 0.633450 +5 1.944539 1.535505 0.649110 +6 1.848854 1.451618 0.661764 +7 1.788579 1.382675 0.673166 +8 1.675414 1.320675 0.683617 +9 1.614536 1.264944 0.694086 +10 1.618723 1.215493 0.702936 +11 1.504875 1.164356 0.712921 +12 1.411316 1.126858 0.721374 +13 1.421174 1.079897 0.731196 +14 1.352116 1.044965 0.738148 +15 1.318876 1.013755 0.745312 +16 1.268569 0.986391 0.751383 +17 1.273424 0.971129 0.754643 +18 1.256196 0.960661 0.757439 +19 1.233202 0.955790 0.758405 +20 1.230536 0.955070 0.758496 +/home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/es-1/models/sp30k +Saving info /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/qrnn_nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.739119 0.438338 0.867000 +Better model found at epoch 1 with val_loss value: 0.438338041305542. +epoch train_loss valid_loss accuracy +1 0.425376 0.207067 0.950000 +Better model found at epoch 1 with val_loss value: 0.20706671476364136. +epoch train_loss valid_loss accuracy +1 0.311269 0.172416 0.956000 +Better model found at epoch 1 with val_loss value: 0.17241604626178741. +epoch train_loss valid_loss accuracy +1 0.226164 0.166543 0.958000 +Better model found at epoch 1 with val_loss value: 0.1665433794260025. +2 0.199775 0.167683 0.956000 +Saving models at /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/qrnn_nl4.m +Loss and accuracy using (cls_best): [0.18184493, tensor(0.9448)] +0.18184493482112885 +0.9447500109672546 +``` From 72e86cefc9f8a11588c429d6fc7953a067cb007a Mon Sep 17 00:00:00 2001 From: Marcin Date: Thu, 14 Feb 2019 12:48:04 +0100 Subject: [PATCH 08/18] Add QRNN LM results for Italian --- results/logs/qrnn-it.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 results/logs/qrnn-it.md diff --git a/results/logs/qrnn-it.md b/results/logs/qrnn-it.md new file mode 100644 index 0000000..43e0a2f --- /dev/null +++ b/results/logs/qrnn-it.md @@ -0,0 +1,30 @@ +# IT +## SP30k QRNN nl 4 +### LM +``` +python -m ulmfit lm --dataset-path data/wiki/it-100/ --cuda-id=0 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 30000 --lang it --qrnn=True - train 10 --bs=50 --drop_mult=0 +Max vocab: 30000 +Cache dir: data/wiki/it-100/models/sp30k +Model dir: data/wiki/it-100/models/sp30k/qrnn_nl4.m +Tokenized data loaded +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', "▁&'", "'", '▁e', '▁il', '▁la', 'e', '▁in'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Training lm from random weights +epoch train_loss valid_loss accuracy +1 3.354224 3.749085 0.350236 +2 3.274838 3.697026 0.351104 +3 3.222462 3.680071 0.352152 +4 3.217652 3.628976 0.357922 +5 3.117965 3.563592 0.364370 +6 3.075397 3.483997 0.372794 +7 3.002098 3.394749 0.383217 +8 2.936974 3.316284 0.393616 +9 2.843549 3.258448 0.401605 +10 2.818070 3.240303 0.404684 +Total time: 10:49:44 +data/wiki/it-100/models/sp30k +Saving info data/wiki/it-100/models/sp30k/qrnn_nl4.m/info.json +``` + +### MLDoc From fdac9f7ccd222c6d5d5831b2047399e095338353 Mon Sep 17 00:00:00 2001 From: Marcin Date: Thu, 14 Feb 2019 14:01:33 +0100 Subject: [PATCH 09/18] Save only the best LM model --- ulmfit/pretrain_lm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index b89a994..775822a 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -221,7 +221,7 @@ class LMHyperParams: learn.opt_fn = partial(optim.Adam, betas=(0.8, 0.99)) learn.metrics = [accuracy_fwd, accuracy_bwd] if self.bidir else [accuracy] learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/lm-history"), - partial(SaveModelCallback, every='epoch', name='lm')] + partial(SaveModelCallback, every='improvement', name='lm')] return learn def load_train_text(self): From 6fda7f2cdaad9677a4ccb44539dc3c86f6cfe8bf Mon Sep 17 00:00:00 2001 From: Marcin Date: Thu, 14 Feb 2019 16:45:51 +0100 Subject: [PATCH 10/18] Download preprocessed wikis --- get_preprocessed_wikis.sh | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100755 get_preprocessed_wikis.sh diff --git a/get_preprocessed_wikis.sh b/get_preprocessed_wikis.sh new file mode 100755 index 0000000..b5c1339 --- /dev/null +++ b/get_preprocessed_wikis.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -x +ZIPNAME="preprocessed_wiki_8langs.zip" +OUTDIR="data/wiki" +wget -nc 'https://www.dropbox.com/sh/srfwvur6orq0cre/AAAQc36bcD17C1KM1mneXN7fa/data/wiki?dl=1' -O "${ZIPNAME}" +mkdir -p "${OUTDIR}" +unzip "${ZIPNAME}" -d "${OUTDIR}" + +for archive in "${OUTDIR}"/??-100.tar.gz; do tar xvf "${archive}" -C "${OUTDIR}" && rm "${archive}"; done From c28c0fde16ef4113b844c239359e7a6cddc32e76 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Thu, 14 Feb 2019 22:28:25 +0100 Subject: [PATCH 11/18] Make ulmfit eval more secure and give more flexibility in dataset_template The dataset_template can use lang as additional token to construct globs patterns. --- ulmfit/__main__.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/ulmfit/__main__.py b/ulmfit/__main__.py index bbf69cb..a36fd84 100644 --- a/ulmfit/__main__.py +++ b/ulmfit/__main__.py @@ -1,4 +1,5 @@ import gc +import os import pprint import shutil from collections import OrderedDict @@ -8,6 +9,7 @@ import fire from .pretrain_lm import LMHyperParams from .train_clas import CLSHyperParams from pathlib import Path +from string import Template class FireView: def __init__(self, **kwargs): @@ -22,7 +24,9 @@ def get_lang_from_dataset_path(ds): def get_dataset_path(p, dataset_template): ds = [x for x in p.parents if x.name == "models"][0].parent - return ds.parent.glob(dataset_template.format(ds.name)) + lang = get_lang_from_dataset_path(ds) + for ds_path in ds.parent.glob(Template(dataset_template).substitute(lang=lang, ds_name=ds.name)): + yield lang, ds_path class ULMFiT: @wraps(LMHyperParams) @@ -37,11 +41,10 @@ class ULMFiT: params = CLSHyperParams.from_lm(dataset_path, base_lm_path, **changes) return FireView(train=params.train_cls, validate_cls=params.validate_cls) - def eval(self, glob="mldoc/*-1/models/sp30k/lstm_nl4.m", dataset_template='{}', name="tmp-100", cuda_id=0, **trn_params): + def eval(self, glob="mldoc/*-1/models/sp30k/lstm_nl4.m", dataset_template='${lang}-1', name="tmp-100", cuda_id=0, **trn_params): results = OrderedDict() for base_model in sorted(Path("data").glob(glob)): - for dataset_path in sorted(get_dataset_path(base_model, dataset_template)): - lang = get_lang_from_dataset_path(dataset_path) + for lang, dataset_path in sorted(get_dataset_path(base_model, dataset_template)): params = CLSHyperParams.from_lm(dataset_path, base_model, lang=lang, name=name, cuda_id=cuda_id) key = str(params.model_dir.relative_to(Path.cwd())) if params.model_dir.exists(): @@ -54,6 +57,16 @@ class ULMFiT: gc.collect() pprint.pprint(results) + + def remove_lm_saves(self): + for lm_save in Path("data").glob("**/lm_*.pth"): + num = lm_save.stem.split("_")[-1] + if not num.isdigit(): + continue + if int(num) not in [5, 10, 15]: + print("rm ", lm_save) + os.remove(lm_save) + # python -m ulmfit cls --dataset-path data/mldoc/de-1-laser --base-lm-path data/mldoc/de-1/models/sp30k/lstm_nl4.m --lang=de --name 'nl4' --cuda-id=1 - train 0 --bs 40 --num-cls-epochs=2 if __name__ == '__main__': From cd47b3b5dc78091109c13512c1f5c2b6c9c471bc Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Thu, 14 Feb 2019 22:35:20 +0100 Subject: [PATCH 12/18] Fix use_moses=True for mldoc so that it is identical to wiki with uses_moses=False The issue was that Moses was executed after pre_rules when use_moses = True, But when data set was pre tokenized with Moses (use_moses=False) the pre_rules were executed after. So our wikipedia had the following processing: - raw text - Moses - pre_rules - split(' ') # fastai BaseTokenizer - post_rules - sentence piece While mldoc had the following tokenziation - raw text - pre_rules - Moses - post_rules - sentence piece After fix I've retrained the classfiers (without finetuning) and I haven't notice huge changes in the performance. 4 languages received slight improvment 4 got a slight decrease in performance. --- fastai_contrib/utils.py | 41 ++++++++++++++--------------------------- results/MLDoc.md | 3 ++- ulmfit/pretrain_lm.py | 27 +++++++++++++++++---------- ulmfit/train_clas.py | 2 +- 4 files changed, 34 insertions(+), 39 deletions(-) diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py index 1cde194..dce29d3 100644 --- a/fastai_contrib/utils.py +++ b/fastai_contrib/utils.py @@ -35,33 +35,28 @@ CLASSES = ['neg', 'pos', 'unsup'] number_match_re = re.compile(r'^([0-9]+[,.]?)+$') number_split_re = re.compile(r'([,.])') -class MosesTokenizerFunc(BaseTokenizer): - "Wrapper around a MosesTokenizer to make it a `BaseTokenizer`." - def __init__(self, lang:str): - super().__init__(lang=lang) - self.tok = MosesTokenizer(lang) +class MosesPreprocessingFunc(): - def tokenizer(self, t:str) -> List[str]: - return self.tok.tokenize(t, return_str=False, escape=False) + def __init__(self, lang: str): + self.mt = MosesTokenizer(lang) - def add_special_cases(self, toks:Collection[str]): - for w in toks: - assert len(self.tokenizer(w))==1, f"Tokenizer is unable to keep {w} as one token!" + def __call__(self, t: str) -> str: + return self.mt.tokenize(t, return_str=True, escape=True) class SentencePieceTokenizer(Tokenizer): "Put together rules and a tokenizer function to tokenize text with multiprocessing." def __init__(self, spm_model, lang:str='en', pre_rules:ListRules=None, - post_rules:ListRules=None, special_cases:Collection[str]=None, n_cpus:int=None, use_moses=False): + post_rules:ListRules=None, special_cases:Collection[str]=None, n_cpus:int=None): + # moses is added to preprocessing functions super().__init__(self.tok_fun_with_sp, lang, pre_rules, post_rules, special_cases, n_cpus) self.spm_model = spm_model - self.use_moses = use_moses def tok_fun_with_sp(self, lang): try: import sentencepiece as spm except ImportError: raise Exception('sentencepiece module is missing: run `pip install sentencepiece`') - tok = MosesTokenizerFunc(lang) if self.use_moses else BaseTokenizer(lang) + tok = BaseTokenizer(lang) tok.sp = spm.SentencePieceProcessor() tok.sp.Load(str(self.spm_model)) return tok @@ -72,9 +67,8 @@ class SentencePieceTokenizer(Tokenizer): toks = tok.sp.EncodeAsPieces(" ".join(toks)) return toks -def get_sentencepiece(cache_dir:PathOrStr, load_text,pre_rules:ListRules=None, post_rules:ListRules=None, - vocab_size:int=30000, model_type:str='unigram', input_sentence_size:int=1E7, - use_moses=False, lang='en'): +def get_sentencepiece(cache_dir:PathOrStr, load_text, pre_rules: ListRules=None, post_rules:ListRules=None, + vocab_size:int=30000, model_type:str='unigram', input_sentence_size:int=1E7, lang='en'): try: import sentencepiece as spm except ImportError: @@ -85,19 +79,13 @@ def get_sentencepiece(cache_dir:PathOrStr, load_text,pre_rules:ListRules=None, p post_rules = post_rules if post_rules is not None else defaults.text_post_rules special_cases = defaults.text_spec_tok - if not os.path.isfile(cache_dir / 'spm.model') or not os.path.isfile(cache_dir / f'itos.pkl'): # load the text from the train tokens file text = load_text() text = filter(lambda x: len(x.rstrip(" ")), text) text = (reduce(lambda t, rule: rule(t), pre_rules, line) for line in text) - if use_moses: - mt = MosesTokenizer(lang) - splitter = lambda t: mt.tokenize(t, return_str=False, escape=False) - else: - splitter = lambda t: t.split() def cleanup_n_postprocess(t): - t = splitter(t) + t = t.split() for r in post_rules: t = r(t) return ' '.join(t) @@ -128,10 +116,9 @@ def get_sentencepiece(cache_dir:PathOrStr, load_text,pre_rules:ListRules=None, p # We cannot use lambdas or local methods here, since `tok_func` needs to be # pickle-able in order to be called in subprocesses when multithread tokenizing tokenizer = SentencePieceTokenizer(cache_dir/'spm.model', - use_moses=use_moses, - lang=lang, - pre_rules=pre_rules, - post_rules=post_rules) + lang=lang, + pre_rules=pre_rules, + post_rules=post_rules) return {'tokenizer': tokenizer, 'vocab': vocab} diff --git a/results/MLDoc.md b/results/MLDoc.md index 58e9009..7dd706c 100644 --- a/results/MLDoc.md +++ b/results/MLDoc.md @@ -4,7 +4,8 @@ |----------------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|------------| |LASER | 90.73 | 92.70 | 88.75 | 90.80 | 85.93 | 85.15 | 84.65 | 88.98 | |MultiCCA | 92.2 | 93.70 | 94.45 | 92.05 | 85.55 | 85.35 | 85.65 | 87.30 | -|ULMFiT | | **95.4** | **95.15** | **93.67** | **88.42** | **89.20** | **87.27** | **90.20** | +|ULMFiT | | **95.4** | **95.15** | **93.67** | **88.42** | **89.20** | **87.27** | **90.20** | +|ULMFiT sp-fixed | | 95.6 | 94.80 | 94.20 | 88.52 | 88.72 | 86.85 | 90.47 | |ULMFiT 100 | | 91.35 | 83.32 | 88.77 | 77.99 | 71.12 | 72.20 | | ^ - sp60k lstm nl 4 diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 3df9213..5959f7e 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -14,8 +14,8 @@ from fastai.callbacks import CSVLogger, SaveModelCallback from fastai.text import * import torch from fastai_contrib.utils import read_file, read_whitespace_file, \ - validate, PAD, UNK, get_sentencepiece, read_clas_data, TRN, VAL, TST, PAD_TOKEN_ID, MosesTokenizerFunc, \ - replace_std_toks + validate, PAD, UNK, get_sentencepiece, read_clas_data, TRN, VAL, TST, PAD_TOKEN_ID, \ + replace_std_toks, MosesPreprocessingFunc from fastai_contrib.learner import bilm_learner, accuracy_fwd, accuracy_bwd, bilm_text_classifier_learner import pickle @@ -30,6 +30,7 @@ ENC_BEST = "enc_best" class Tokenizers(Enum): SUBWORD='sp' + BROKENSUBWORD = 'bsp' MOSES='v' MOSES_FA='vf' FASTAI='f' @@ -121,7 +122,7 @@ class LMHyperParams: def model_name(self): return f"{self.model_prefix}_{self.name}.m" @property - def pretrained_fnames(self): return [self.base_lm_path / 'lm_best', self.base_lm_path / '../itos'] if self.base_lm_path else None + def pretrained_fnames(self): return [self.base_lm_path / LM_BEST, self.base_lm_path / '../itos'] if self.base_lm_path else None @property def lm_type(self): @@ -133,8 +134,8 @@ class LMHyperParams: return contrib_data.LanguageModelType.FwdLM def tokenizer_to_fastai_args(self, sp_data_func, use_moses): - tok_func = MosesTokenizerFunc if use_moses else BaseTokenizer - if self.tokenizer is Tokenizers.SUBWORD: + moses_preproc = [MosesPreprocessingFunc(self.lang)] if use_moses else [] + if self.tokenizer is Tokenizers.SUBWORD or self.tokenizer is Tokenizers.BROKENSUBWORD: if self.base_lm_path and not(self.cache_dir/"spm.model").exists(): # ensure we are using the same sentence piece model shutil.copy(self.base_lm_path / '..' / 'itos.pkl', self.cache_dir) shutil.copy(self.base_lm_path / '..' / 'spm.model', self.cache_dir) @@ -142,13 +143,19 @@ class LMHyperParams: args = get_sentencepiece(self.cache_dir, sp_data_func, vocab_size=self.max_vocab, - use_moses=use_moses, - lang=self.lang) - + lang=self.lang, + pre_rules=moses_preproc + defaults.text_pre_rules, + post_rules=defaults.text_post_rules) elif self.tokenizer is Tokenizers.MOSES: - args = dict(tokenizer=Tokenizer(tok_func=tok_func, lang=self.lang, pre_rules=[replace_std_toks], post_rules=[])) + args = dict(tokenizer=Tokenizer(tok_func=BaseTokenizer, + lang=self.lang, + pre_rules=moses_preproc + [replace_std_toks], + post_rules=[])) elif self.tokenizer is Tokenizers.MOSES_FA: - args = dict(tokenizer=Tokenizer(tok_func=tok_func, lang=self.lang)) # use default pre/post rules + args = dict(tokenizer=Tokenizer(tok_func=BaseTokenizer, + lang=self.lang, + pre_rules=moses_preproc + defaults.text_pre_rules, + post_rules=defaults.text_post_rules)) elif self.tokenizer is Tokenizers.FASTAI: args = dict() else: diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 5cff5d3..0641790 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -15,7 +15,7 @@ from fastai_contrib import utils from fastai_contrib.data import LanguageModelType from fastai_contrib.learner import bilm_text_classifier_learner, bilm_learner, accuracy_fwd, accuracy_bwd from fastai_contrib.utils import PAD, UNK, read_clas_data, PAD_TOKEN_ID, DATASETS, TRN, VAL, TST, ensure_paths_exists, \ - get_sentencepiece, MosesTokenizerFunc + get_sentencepiece from fastai.text.transform import Vocab import fire From 22cb8b1660eed807d311a269ec9df3b216edfd71 Mon Sep 17 00:00:00 2001 From: Marcin Date: Fri, 15 Feb 2019 00:00:20 +0100 Subject: [PATCH 13/18] Download pretrained models --- get_preprocessed_wikis.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/get_preprocessed_wikis.sh b/get_preprocessed_wikis.sh index b5c1339..ac4a7ec 100755 --- a/get_preprocessed_wikis.sh +++ b/get_preprocessed_wikis.sh @@ -7,3 +7,12 @@ mkdir -p "${OUTDIR}" unzip "${ZIPNAME}" -d "${OUTDIR}" for archive in "${OUTDIR}"/??-100.tar.gz; do tar xvf "${archive}" -C "${OUTDIR}" && rm "${archive}"; done + +#optionally download models +MODELS="pretrained_lm_models.zip" +read -r -p "Download pretrained lm models? [y/N] " response +if [[ "$response" =~ ^[yY]$ ]] +then + wget -nc 'https://www.dropbox.com/sh/srfwvur6orq0cre/AAABRFdrCNHmbpf4nNcMiJwJa/models/data/wiki?dl=1' -O "${MODELS}" + unzip "${MODELS}" -d "${OUTDIR}" +fi From 0e6534ad7b77f44ea6b3fbf40364fd6a03874021 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Fri, 15 Feb 2019 01:11:39 +0100 Subject: [PATCH 14/18] Expose num_lm_epochs in ulmfit eval --- ulmfit/__main__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ulmfit/__main__.py b/ulmfit/__main__.py index 7233877..65ce3fb 100644 --- a/ulmfit/__main__.py +++ b/ulmfit/__main__.py @@ -41,7 +41,7 @@ class ULMFiT: params = CLSHyperParams.from_lm(dataset_path, base_lm_path, **changes) return FireView(train=params.train_cls, validate_cls=params.validate_cls) - def eval(self, glob="mldoc/*-1/models/sp30k/lstm_nl4.m", dataset_template='${lang}-1', name="tmp-100", cuda_id=0, **trn_params): + def eval(self, glob="mldoc/*-1/models/sp30k/lstm_nl4.m", dataset_template='${lang}-1', name="tmp-100", num_lm_epochs=0, cuda_id=0, **trn_params): results = OrderedDict() for base_model in sorted(Path("data").glob(glob)): for lang, dataset_path in sorted(get_dataset_path(base_model, dataset_template)): @@ -52,7 +52,7 @@ class ULMFiT: results[key] = params.validate_cls()[1] else: print("Training") - results[key] = params.train_cls(num_lm_epochs=0, **trn_params)[1] + results[key] = params.train_cls(num_lm_epochs=num_lm_epochs, **trn_params)[1] del params gc.collect() From 5dced1e488ede986cb3970b6c23309c1f5eb401f Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Fri, 15 Feb 2019 01:16:37 +0100 Subject: [PATCH 15/18] Remove bidir --- ulmfit/pretrain_lm.py | 2 +- ulmfit/train_clas.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 1491bc1..676f79a 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -206,7 +206,7 @@ class LMHyperParams: def create_lm_learner(self, data_lm, dps=None, **kwargs): assert self.bidir == False, "bidirectional model is not yet supported" - config = dict(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, pad_token=PAD_TOKEN_ID, qrnn=self.qrnn, bidir=self.bidir, + config = dict(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, pad_token=PAD_TOKEN_ID, qrnn=self.qrnn, tie_weights=True, out_bias=True) config.update(dps or self.dps) trn_args = dict(clip=self.clip, alpha=self.rnn_alpha, beta=self.rnn_beta) diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 7d72ec7..57531c2 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -93,7 +93,7 @@ class CLSHyperParams(LMHyperParams): def create_cls_learner(self, data_clas, dps=None, **kwargs): assert self.bidir == False, "bidirectional model is not yet supported" - config = dict(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, pad_token=PAD_TOKEN_ID, qrnn=self.qrnn, bidir=self.bidir) + config = dict(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, pad_token=PAD_TOKEN_ID, qrnn=self.qrnn) config.update(dps or self.dps) trn_args=dict(bptt=self.bptt, clip=self.clip) trn_args.update(kwargs) From 8733487d556b530f0782e9c7a89f2d50ade1236b Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Fri, 15 Feb 2019 01:17:53 +0100 Subject: [PATCH 16/18] Make the validate vs train decision based on the existance of cls_last.pth istead of a model directory --- ulmfit/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ulmfit/__main__.py b/ulmfit/__main__.py index 65ce3fb..72eb527 100644 --- a/ulmfit/__main__.py +++ b/ulmfit/__main__.py @@ -47,7 +47,7 @@ class ULMFiT: for lang, dataset_path in sorted(get_dataset_path(base_model, dataset_template)): params = CLSHyperParams.from_lm(dataset_path, base_model, lang=lang, name=name, cuda_id=cuda_id) key = str(params.model_dir.relative_to(Path.cwd())) - if params.model_dir.exists(): + if (params.model_dir/"cls_last.pth").exists(): print("Evaluating previously trained model") results[key] = params.validate_cls()[1] else: From be0511e42bec051c2d4811da4f03ebcc0ba2032a Mon Sep 17 00:00:00 2001 From: Julian Eisenschlos Date: Fri, 15 Feb 2019 15:51:35 -0300 Subject: [PATCH 17/18] Adding Bert results --- results/MLDoc.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/results/MLDoc.md b/results/MLDoc.md index 7dd706c..db55d92 100644 --- a/results/MLDoc.md +++ b/results/MLDoc.md @@ -4,9 +4,10 @@ |----------------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|------------| |LASER | 90.73 | 92.70 | 88.75 | 90.80 | 85.93 | 85.15 | 84.65 | 88.98 | |MultiCCA | 92.2 | 93.70 | 94.45 | 92.05 | 85.55 | 85.35 | 85.65 | 87.30 | -|ULMFiT | | **95.4** | **95.15** | **93.67** | **88.42** | **89.20** | **87.27** | **90.20** | +|ULMFiT | | **95.4** | 95.15 | **93.67** | **88.42** | **89.20** | **87.27** | **90.20** | |ULMFiT sp-fixed | | 95.6 | 94.80 | 94.20 | 88.52 | 88.72 | 86.85 | 90.47 | |ULMFiT 100 | | 91.35 | 83.32 | 88.77 | 77.99 | 71.12 | 72.20 | | +|BERT Multi | 94.1 | 93.8 | **95.4** | 91.1 | 87.5 | 86.4 | 86.8 | 90.8 | ^ - sp60k lstm nl 4 From b7e3a5e7a46c774ed00cbc94bb159bec86bf8fa6 Mon Sep 17 00:00:00 2001 From: Julian Eisenschlos Date: Sat, 16 Feb 2019 14:42:40 -0300 Subject: [PATCH 18/18] Update BERT Zero-shot results --- results/MLDoc.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/results/MLDoc.md b/results/MLDoc.md index db55d92..84bf1e8 100644 --- a/results/MLDoc.md +++ b/results/MLDoc.md @@ -4,10 +4,10 @@ |----------------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|------------| |LASER | 90.73 | 92.70 | 88.75 | 90.80 | 85.93 | 85.15 | 84.65 | 88.98 | |MultiCCA | 92.2 | 93.70 | 94.45 | 92.05 | 85.55 | 85.35 | 85.65 | 87.30 | -|ULMFiT | | **95.4** | 95.15 | **93.67** | **88.42** | **89.20** | **87.27** | **90.20** | +|ULMFiT | | **95.4** | **95.15** | **93.67** | **88.42** | **89.20** | **87.27** | 90.20 | |ULMFiT sp-fixed | | 95.6 | 94.80 | 94.20 | 88.52 | 88.72 | 86.85 | 90.47 | -|ULMFiT 100 | | 91.35 | 83.32 | 88.77 | 77.99 | 71.12 | 72.20 | | -|BERT Multi | 94.1 | 93.8 | **95.4** | 91.1 | 87.5 | 86.4 | 86.8 | 90.8 | +|ULMFiT 100 | | 91.35 | 83.32 | 88.77 | 77.99 | 71.12 | 72.20 | | +|Bert Multi | 93.23% | 94.0% | **95.15** | 93.20 | 85.82 | 87.48 | 86.85 | **90.72** | ^ - sp60k lstm nl 4 @@ -27,6 +27,8 @@ | % impr over LASER-fr | 31% | 4% | | 16% | 3% | 25% | | % impr over LASER-en | 43% | 20% | 30% | 17% | 10% | 16% | | ULMFiT 100 for comp. | 91.35 | 83.32 | 88.77 | 77.99 | 71.12 | | +| | | | | | | | +| Bert Multilingual-EN | 74.50 | 61.85 | 69.77 | 57.73 | 51.10 | 64.08 | All ULMFiT examples above were trained on 1k training data generated by a LASER classification model