From 260faa703cf4f5d1eb8584bc5e3a5b1b10ae40fc Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Fri, 22 Feb 2019 16:55:35 +0100 Subject: [PATCH] Correct the label smoothing implementation --- results/MLDoc.md | 4 +- results/logs/label_smoothing.md | 223 ++++++++++++++++++++++++++++++++ results/logs/qrnn-ru.md | 12 ++ tests/test_end_to_end.py | 4 +- ulmfit/__main__.py | 2 +- ulmfit/pretrain_lm.py | 2 +- ulmfit/train_clas.py | 6 +- 7 files changed, 246 insertions(+), 7 deletions(-) create mode 100644 results/logs/label_smoothing.md diff --git a/results/MLDoc.md b/results/MLDoc.md index 67cdbf0..5448079 100644 --- a/results/MLDoc.md +++ b/results/MLDoc.md @@ -10,7 +10,9 @@ |ULMFiT L30k | | 95.4 | 95.15 | 93.67 | 88.42 | 89.20 | **87.27** | 90.20 | |ULMFiT sp-fixed | | **95.6** | 94.80 | 94.20 | 88.52 | 88.72 | 86.85 | 90.47 | |ULMFIT Q15k 1cyc| **94.62** | **95.65** | 95.15 | **94.42** | **89.92** | 89.60 | | 90.78/89.82 | -|ULMFIT L30k 1cyc| | **95.85** | **96.32** | **94.82** | 89.87 | **90.45** | **87.94** | **92.02/91.64** | +|ULMFIT Q15k 1c l| | | | | | | | **92.22** | +|ULMFIT L30k 1cyc| | **95.85** | **96.32** | **94.82** | 89.87 | **90.45** | **87.94** | 92.02/91.64 | + - L30k - LSTM sp30k trained using gradual unfreezing - L30k-100 - --||-- **on 100 samples** - ULMFiT sp-fixed - --||-- with fixed tokenization diff --git a/results/logs/label_smoothing.md b/results/logs/label_smoothing.md new file mode 100644 index 0000000..c52df1e --- /dev/null +++ b/results/logs/label_smoothing.md @@ -0,0 +1,223 @@ +# MLDoc + +## QRNN 15k + +Exec 1 +``` +python -m ulmfit eval --glob="mldoc/zh-1/models/sp15k/qrnn_nl4.m" --name nl4-1cyc-sl --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.1 +Processing data/mldoc/zh-1/models/sp15k/qrnn_nl4.m +zh-1 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.859153 0.767961 0.864000 +2 0.768888 0.775161 0.904000 +3 0.658956 0.685653 0.902000 +4 0.589073 0.618438 0.923000 +5 0.540008 0.622157 0.915000 +6 0.508080 0.606979 0.914000 +7 0.487228 0.599491 0.918000 +8 0.477516 0.602196 0.923000 +Total time: 02:18 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl.m +Loss and accuracy using (cls_best): [0.2829206, tensor(0.9205)] +OrderedDict([('data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl.m', + 0.9204999804496765)]) +data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl.m: 0.9204999804496765 +``` +Exec 2 +````python -m ulmfit eval --glob="mldoc/zh-1/models/sp15k/qrnn_nl4.m" --name nl4-1cyc-sl1 --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.1 +Processing data/mldoc/zh-1/models/sp15k/qrnn_nl4.m +zh-1 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl1.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl1.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.881513 0.712176 0.865000 +2 0.743687 0.665091 0.906000 +3 0.677436 0.687689 0.873000 +4 0.595139 0.626483 0.920000 +5 0.542732 0.600652 0.914000 +6 0.512080 0.597546 0.916000 +7 0.487021 0.597065 0.912000 +8 0.476598 0.596792 0.914000 +Total time: 02:20 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl1.m +Loss and accuracy using (cls_best): [0.29172945, tensor(0.9178)] +OrderedDict([('data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl1.m', + 0.9177500009536743)]) +data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl1.m: 0.9177500009536743 +```` +Exec 4 +```bash +python -m ulmfit eval --glob="mldoc/zh-1/models/sp15k/qrnn_nl4.m" --name nl4-1cyc-sl-e4 --num-cls-epochs=4 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.1 ✘ 130 +Processing data/mldoc/zh-1/models/sp15k/qrnn_nl4.m +zh-1 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.880415 0.677291 0.901000 +2 0.729670 0.659975 0.911000 +3 0.624817 0.603056 0.921000 +4 0.542027 0.601961 0.921000 +Total time: 01:08 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m +Loss and accuracy using (cls_best): [0.28558904, tensor(0.9222)] +OrderedDict([('data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m', + 0.922249972820282)]) +data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m: 0.922249972820282 +``` +## LSTM sp30k +### 0.1 +```bash + python -m ulmfit eval --glob="mldoc/zh-1/models/sp30k/lstm_nl4.m" --name nl4-1cyc-sl --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.1 +Processing data/mldoc/zh-1/models/sp30k/lstm_nl4.m +zh-1 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.870432 0.670671 0.882000 +2 0.754248 0.824157 0.895000 +3 0.654601 0.727428 0.885000 +4 0.602772 0.668668 0.901000 +5 0.542110 0.625137 0.903000 +6 0.506150 0.617842 0.913000 +7 0.480944 0.616885 0.912000 +8 0.472876 0.614381 0.911000 +Total time: 06:38 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl.m +Loss and accuracy using (cls_best): [0.2977172, tensor(0.9233)] +OrderedDict([('data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl.m', + 0.9232500195503235)]) +data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl.m: 0.9232500195503235 +``` +### 0.2 +```bash +python -m ulmfit eval --glob="mldoc/zh-1/models/sp30k/lstm_nl4.m" --name nl4-1cyc-sl2 --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.2 +Processing data/mldoc/zh-1/models/sp30k/lstm_nl4.m +zh-1 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl2.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl2.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.045619 0.908213 0.874000 +2 0.957379 0.857977 0.921000 +3 0.891791 0.852157 0.905000 +4 0.845289 0.849923 0.914000 +5 0.818228 0.848613 0.921000 +6 0.787021 0.840483 0.920000 +7 0.776123 0.844006 0.919000 +8 0.762384 0.857240 0.916000 +Total time: 06:33 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl2.m +Loss and accuracy using (cls_best): [0.40299156, tensor(0.9170)] +OrderedDict([('data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl2.m', + 0.9169999957084656)]) +data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl2.m: 0.9169999957084656 +``` +### 0.4 +```bash + python -m ulmfit eval --glob="mldoc/zh-1/models/sp30k/lstm_nl4.m" --name nl4-1cyc-sl4 --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.4 +Processing data/mldoc/zh-1/models/sp30k/lstm_nl4.m +zh-1 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl4.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.251581 1.183341 0.898000 +2 1.214358 1.201266 0.834000 +3 1.190343 1.165525 0.919000 +4 1.168018 1.172510 0.903000 +5 1.149965 1.161660 0.914000 +6 1.140140 1.161689 0.915000 +7 1.135877 1.159853 0.912000 +8 1.134425 1.160039 0.911000 +Total time: 06:34 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl4.m +Loss and accuracy using (cls_best): [0.64041936, tensor(0.9195)] +OrderedDict([('data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl4.m', + 0.9194999933242798)]) +data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl4.m: 0.9194999933242798 +``` \ No newline at end of file diff --git a/results/logs/qrnn-ru.md b/results/logs/qrnn-ru.md index 36ae8d4..a213dd6 100644 --- a/results/logs/qrnn-ru.md +++ b/results/logs/qrnn-ru.md @@ -1,4 +1,16 @@ # QRNN RU +## SP15k nl8 +data/wiki/ru-100/models/sp15k/qrnn_nl8.m + +export CUDA_VISIBLE_DEVICES=0 +LANG=ru +python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/sp15k/qrnn_nl8.m --lang=${LANG} --name 'nl8' - train 20 --bs 20 --num-cls-epochs=8 --lr_sched=1cycle + +export CUDA_VISIBLE_DEVICES=0 +LANG=de +python -m ulmfit cls --dataset-path data/mldoc-m/${LANG}-1 --base-lm-path data/wiki-m/${LANG}-100/models/sp15k/lstm_nl4.m --lang=${LANG} --name 'nl4' - train 20 --bs 20 --num-cls-epochs=8 --lr_sched=1cycle + + ## SP30k nl4 ### LM ``` diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index 73ded8a..127c8fa 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -39,7 +39,8 @@ def get_test_data(): copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.valid.tokens', n=600*sz) copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.test.tokens', n=600*sz) copy_head(imdb / 'train.csv', test_imdb / 'train.csv', n=10*sz) - copy_head(imdb / 'train.csv', test_imdb / 'test.csv', n=6*sz) + copy_head(imdb / 'train.csv', test_imdb / 'test.csv', n=6 * sz) + copy_head(imdb / 'train.csv', test_imdb / 'dev.csv', n=6 * sz) copy_head(imdb / 'train.csv', test_imdb / 'unsup.csv', n=1*sz) return test_data, test_wt @@ -109,6 +110,7 @@ def test_ulmfit_fastai_end_to_end(): qrnn=False, tokenizer='f', max_vocab=100, + nl=1, name=lm_name, ) exp.train_lm(num_epochs=1, bs=2) diff --git a/ulmfit/__main__.py b/ulmfit/__main__.py index 13a3922..ffeabc1 100644 --- a/ulmfit/__main__.py +++ b/ulmfit/__main__.py @@ -90,7 +90,7 @@ class ULMFiT: try: params = CLSHyperParams.from_lm(dataset_path, base_model, lang=lang, name=name, cuda_id=cuda_id) key = str(params.model_dir.relative_to(Path.cwd())) - if (params.model_dir/"cls_last.pth").exists(): + if (params.model_dir/"cls_best.pth").exists(): print("Evaluating previously trained model") results[key] = params.validate_cls()[1] else: diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 5b70ef6..0dfecaa 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -231,7 +231,7 @@ class LMHyperParams: # partial(SaveModelCallback, every='improvement', name='lm') disabled due to Memory issues ] if label_smoothing_eps > 0.0: - learn.loss_func = LabelSmoothingCrossEntropy(eps=label_smoothing_eps) + learn.loss_func = FlattenedLoss(LabelSmoothingCrossEntropy, eps=label_smoothing_eps) return learn def load_train_text(self): diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 278a629..cda378d 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -77,7 +77,7 @@ class CLSHyperParams(LMHyperParams): if self.need_fine_tune_lm: self.train_lm(num_lm_epochs, data_lm=data_lm, drop_mult=drop_mul_lm, label_smoothing_eps=label_smoothing_eps) learn = self.create_cls_learner(data_clas, drop_mult=drop_mul_cls, max_len=cls_max_len, label_smoothing_eps=label_smoothing_eps) try: - learn.load('cls_last') + learn.load('cls_best') print("Loading last classifier") except FileNotFoundError: learn.load_encoder(ENC_BEST) @@ -94,7 +94,7 @@ class CLSHyperParams(LMHyperParams): del learn return self.validate_cls('cls_best', bs=bs, data_tst=data_tst, learn=None) - def validate_cls(self, save_name='cls_last', bs=40, data_tst=None, learn=None): + def validate_cls(self, save_name='cls_best', bs=40, data_tst=None, learn=None): if data_tst is None: _, _, data_tst = self.load_cls_data(bs) if learn is None: @@ -125,7 +125,7 @@ class CLSHyperParams(LMHyperParams): #partial(SaveModelCallback, every='improvement', name='cls_best') disabled due to memory issues ] if label_smoothing_eps > 0.0: - learn.loss_func = LabelSmoothingCrossEntropy(eps=label_smoothing_eps) + learn.loss_func = FlattenedLoss(LabelSmoothingCrossEntropy, eps=label_smoothing_eps) return learn def load_cls_data(self, bs, **kwargs):