From c69d31c420bb607b30e0789392ce5a22420e34c5 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Fri, 22 Feb 2019 12:01:25 +0100 Subject: [PATCH 1/3] New results lstm 30k 1cyc --- results/MLDoc.md | 4 +- results/logs/cls.md | 50 +- results/logs/common.md | 236 ++++++ results/logs/de.md | 24 + results/logs/noise/de10k-noise.md | 1158 ++++++++++++++++++++++++++++ results/logs/noise/es10k-noise.md | 1159 +++++++++++++++++++++++++++++ 6 files changed, 2628 insertions(+), 3 deletions(-) create mode 100644 results/logs/noise/de10k-noise.md create mode 100644 results/logs/noise/es10k-noise.md diff --git a/results/MLDoc.md b/results/MLDoc.md index 83c1a81..67cdbf0 100644 --- a/results/MLDoc.md +++ b/results/MLDoc.md @@ -9,8 +9,8 @@ |ULMFiT L30k-100 | | 91.35 | 83.32 | 88.77 | 77.99 | 71.12 | 72.20 | | |ULMFiT L30k | | 95.4 | 95.15 | 93.67 | 88.42 | 89.20 | **87.27** | 90.20 | |ULMFiT sp-fixed | | **95.6** | 94.80 | 94.20 | 88.52 | 88.72 | 86.85 | 90.47 | -|ULMFIT Q15k 1cyc| **94.62** | **95.65** | 95.15 | **94.42** | **89.92** | **89.60** | | 90.78/89.82 | -|ULMFIT L30k 1cyc| | | **95.95** | | | | | **92.02** | +|ULMFIT Q15k 1cyc| **94.62** | **95.65** | 95.15 | **94.42** | **89.92** | 89.60 | | 90.78/89.82 | +|ULMFIT L30k 1cyc| | **95.85** | **96.32** | **94.82** | 89.87 | **90.45** | **87.94** | **92.02/91.64** | - L30k - LSTM sp30k trained using gradual unfreezing - L30k-100 - --||-- **on 100 samples** - ULMFiT sp-fixed - --||-- with fixed tokenization diff --git a/results/logs/cls.md b/results/logs/cls.md index b29cb64..bf45ebd 100644 --- a/results/logs/cls.md +++ b/results/logs/cls.md @@ -121,4 +121,52 @@ Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/m Loss and accuracy using (cls_best): [0.5418505, tensor(0.9100)] 0.5418505072593689 0.9100000262260437 -```` \ No newline at end of file +```` + +``` +Loading pretrained model +Unknown tokens 0, first 100: [] +Training lm from: [PosixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp15k/qrnn_nl4.m/../itos')] +epoch train_loss valid_loss accuracy +1 4.037580 3.312217 0.364410 +Total time: 01:44 +epoch train_loss valid_loss accuracy +1 3.709982 3.256320 0.371825 +2 3.459413 3.150574 0.386972 +3 3.296628 3.037327 0.402039 +4 3.186458 2.914899 0.418413 +5 3.092632 2.817097 0.431216 +6 2.966957 2.726081 0.442906 +7 2.924824 2.647339 0.453871 +8 2.818279 2.561596 0.466795 +9 2.773893 2.501994 0.475877 +10 2.736084 2.438490 0.485978 +11 2.688937 2.370927 0.496899 +12 2.615245 2.314875 0.506508 +13 2.583292 2.260717 0.515725 +14 2.535631 2.220295 0.522666 +15 2.466035 2.179093 0.530148 +16 2.461427 2.151952 0.535315 +17 2.390641 2.131065 0.538749 +18 2.376235 2.116927 0.541430 +19 2.407630 2.115370 0.542039 +20 2.391378 2.112687 0.542522 +Total time: 46:33 +/home/n-waves/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp15k +Saving info /home/n-waves/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp15k/qrnn_nl4.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.466671 0.534682 0.745000 +2 0.358965 0.372612 0.875000 +3 0.251557 0.311034 0.900000 +4 0.166484 0.585425 0.865000 +5 0.101803 0.726341 0.900000 +6 0.072025 0.587875 0.885000 +7 0.045328 0.760989 0.890000 +8 0.027765 0.727203 0.890000 +Total time: 01:17 +Saving models at /home/n-waves/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp15k/qrnn_nl4.m +Loss and accuracy using (cls_best): [0.55982095, tensor(0.8970)] +0.5598209500312805 +0.8970000147819519 +``` \ No newline at end of file diff --git a/results/logs/common.md b/results/logs/common.md index c0f0fcb..1c23a0d 100644 --- a/results/logs/common.md +++ b/results/logs/common.md @@ -1,6 +1,242 @@ # MLDoc ### Different training schedules +### 1cycle -lstm +``` +(fastaiv1) pczapla@galatea ~/w/ulmfit-multilingual ❯❯❯ python -m ulmfit eval --glob="mldoc/*-1/models/sp30k/lstm_nl4.m" --name nl4-1cyc --num-cls-epochs=8 --bs=18 --lr_sched=1cycle ✘ 1 +Processing data/mldoc/de-1/models/sp30k/lstm_nl4.m +de-1 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-1cyc.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/de.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-1cyc.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.610423 0.287707 0.920000 +2 0.390499 0.266688 0.948000 +3 0.366716 0.302463 0.933000 +4 0.248321 0.305547 0.937000 +5 0.166564 0.411075 0.948000 +6 0.083940 0.406182 0.950000 +7 0.033326 0.388105 0.949000 +8 0.014658 0.397507 0.948000 +Total time: 06:42 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-1cyc.m +Loss and accuracy using (cls_best): [0.3040595, tensor(0.9585)] +Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m +es-1 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-1cyc.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv +Data lm, trn: 13013, val: 1445 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-1cyc.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.610724 0.278892 0.925000 +2 0.372022 0.348428 0.937000 +3 0.310411 0.386958 0.927000 +4 0.215536 0.273834 0.958000 +5 0.163195 0.319600 0.958000 +6 0.085268 0.313287 0.961000 +7 0.037369 0.347500 0.961000 +8 0.016851 0.338436 0.963000 +Total time: 05:36 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-1cyc.m +Loss and accuracy using (cls_best): [0.31034237, tensor(0.9632)] +Processing data/mldoc/fr-1/models/sp30k/lstm_nl4.m +fr-1 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4-1cyc.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/fr.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4-1cyc.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.642809 0.240702 0.928000 +2 0.420564 0.658542 0.852000 +3 0.443345 0.244053 0.927000 +4 0.338779 0.335634 0.914000 +5 0.224778 0.263748 0.928000 +6 0.116705 0.280655 0.944000 +7 0.072063 0.287557 0.945000 +8 0.048084 0.289200 0.946000 +Total time: 06:33 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4-1cyc.m +Loss and accuracy using (cls_best): [0.29398218, tensor(0.9482)] +Processing data/mldoc/it-1/models/sp30k/lstm_nl4.m +it-1 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4-1cyc.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/it.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', "▁&'", "'", '▁e', '▁il', '▁la', 'e', '▁in'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4-1cyc.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.736070 0.391008 0.859000 +2 0.512531 0.614638 0.860000 +3 0.343422 0.594530 0.862000 +4 0.370786 0.540225 0.884000 +5 0.234727 0.591903 0.892000 +6 0.141539 0.589971 0.906000 +7 0.073315 0.544248 0.906000 +8 0.038477 0.580940 0.904000 +Total time: 03:48 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4-1cyc.m +Loss and accuracy using (cls_best): [0.6642357, tensor(0.8988)] +Processing data/mldoc/ja-1/models/sp30k/lstm_nl4.m +ja-1 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-1cyc.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-1cyc.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.804430 0.435780 0.837000 +2 0.576932 0.452903 0.839000 +3 0.499791 0.640789 0.806000 +4 0.418024 0.610898 0.839000 +5 0.259578 0.582953 0.868000 +6 0.188161 0.719131 0.888000 +7 0.101508 0.766175 0.877000 +8 0.072221 0.795415 0.883000 +Total time: 08:19 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-1cyc.m +Loss and accuracy using (cls_best): [0.636261, tensor(0.9045)] +Processing data/mldoc/ru-1/models/sp30k/lstm_nl4.m +ru-1 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4-1cyc.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv +Data lm, trn: 9195, val: 1021 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', '▁и', 'е', 'и', 'й', '▁на', 'х'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4-1cyc.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.821141 0.532432 0.814000 +2 0.611059 0.457023 0.866000 +3 0.464408 0.484035 0.870000 +4 0.446560 0.477454 0.858000 +5 0.299095 0.906959 0.858000 +6 0.176480 0.709579 0.875000 +7 0.089683 0.781081 0.876000 +8 0.047031 0.772935 0.877000 +Total time: 08:58 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4-1cyc.m +Loss and accuracy using (cls_best): [0.8528109, tensor(0.8795)] +Processing data/mldoc/zh-1/models/sp30k/lstm_nl4.m +zh-1 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.696346 0.335083 0.895000 +2 0.505075 0.360600 0.906000 +3 0.427076 0.462661 0.883000 +4 0.351177 0.489026 0.919000 +5 0.244958 0.415151 0.918000 +6 0.156452 0.494367 0.926000 +7 0.085807 0.471611 0.927000 +8 0.046887 0.484232 0.929000 +Total time: 06:41 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc.m +Loss and accuracy using (cls_best): [0.50999963, tensor(0.9165)] +OrderedDict([('data/mldoc/de-1/models/sp30k/lstm_nl4-1cyc.m', + 0.9585000276565552), + ('data/mldoc/es-1/models/sp30k/lstm_nl4-1cyc.m', + 0.9632499814033508), + ('data/mldoc/fr-1/models/sp30k/lstm_nl4-1cyc.m', + 0.9482499957084656), + ('data/mldoc/it-1/models/sp30k/lstm_nl4-1cyc.m', + 0.8987500071525574), + ('data/mldoc/ja-1/models/sp30k/lstm_nl4-1cyc.m', + 0.9045000076293945), + ('data/mldoc/ru-1/models/sp30k/lstm_nl4-1cyc.m', + 0.8794999718666077), + ('data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc.m', + 0.9164999723434448)]) +data/mldoc/de-1/models/sp30k/lstm_nl4-1cyc.m: 0.9585000276565552 +data/mldoc/es-1/models/sp30k/lstm_nl4-1cyc.m: 0.9632499814033508 +data/mldoc/fr-1/models/sp30k/lstm_nl4-1cyc.m: 0.9482499957084656 +data/mldoc/it-1/models/sp30k/lstm_nl4-1cyc.m: 0.8987500071525574 +data/mldoc/ja-1/models/sp30k/lstm_nl4-1cyc.m: 0.9045000076293945 +data/mldoc/ru-1/models/sp30k/lstm_nl4-1cyc.m: 0.8794999718666077 +data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc.m: 0.9164999723434448 +``` + ### 2cycle ```bash diff --git a/results/logs/de.md b/results/logs/de.md index dc44577..39a6e58 100644 --- a/results/logs/de.md +++ b/results/logs/de.md @@ -1,4 +1,28 @@ # DE +## SP15k LSTM nl4 +``` +$ python -m ulmfit lm --dataset-path data/wiki/de-100 --cuda-id=0 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang de --qrnn=False - train 10 --bs=100 --drop_mult=0 + +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', +'▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Training lm from random weights +epoch train_loss valid_loss accuracy +1 2.519809 2.600072 0.529963 +2 2.436580 2.538897 0.534651 +3 2.402220 2.510569 0.537314 +4 2.305741 2.439347 0.546574 +5 2.265683 2.376482 0.553794 +6 2.210663 2.305362 0.562672 +7 2.134196 2.230041 0.572958 +8 2.085375 2.150917 0.584621 +9 2.037781 2.097170 0.593747 +10 1.986773 2.081469 0.595799 +Total time: 19:18:33 +data/wiki/de-100/models/sp15k +Saving info data/wiki/de-100/models/sp15k/lstm_nl4.m/info.jso +``` + ## VF60k LSTM nl 3 ### LM ``` diff --git a/results/logs/noise/de10k-noise.md b/results/logs/noise/de10k-noise.md new file mode 100644 index 0000000..4708277 --- /dev/null +++ b/results/logs/noise/de10k-noise.md @@ -0,0 +1,1158 @@ +# Correct Val data +``` +python -m ulmfit eval_noise_resistance --lang=de --size=10 --prefix-name="val_" +Noise: 0 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_0.m +Evaluating previously trained model +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: + + !! WARNING !! + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +Your compiler (c++) may be ABI-incompatible with PyTorch! +Please use a compiler that is ABI-compatible with GCC 4.9 and above. +See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. + +See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 +for instructions on how to install GCC 4.9 or higher. +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + !! WARNING !! + + warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) +Loss and accuracy using (cls_last): [0.3033199, tensor(0.9712)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_0.m', + 0.9712499976158142)]) +Noise: 5 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_5.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 500 examples, only 0.95 have correct labels +Added noise to 50 examples, only 0.95 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.05tv... +Data clsnoise0.05tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_5.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.436501 0.416959 0.909000 +2 0.460737 0.426361 0.904000 +3 0.360029 0.502090 0.907000 +4 0.362643 0.404165 0.916000 +5 0.283832 0.478411 0.911000 +6 0.224228 0.538951 0.915000 +7 0.159160 0.662829 0.909000 +8 0.096864 0.699768 0.910000 +Total time: 18:20 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_5.m +Loss and accuracy using (cls_best): [0.22873034, tensor(0.9565)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_5.m', + 0.9564999938011169)]) +Noise: 10 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_10.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 1000 examples, only 0.9 have correct labels +Added noise to 100 examples, only 0.9 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.1tv... +Data clsnoise0.1tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_10.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.608604 0.614120 0.851000 +2 0.563473 0.521960 0.858000 +3 0.534108 0.564487 0.867000 +4 0.473966 0.600789 0.870000 +5 0.450963 0.579453 0.869000 +6 0.365191 0.641545 0.864000 +7 0.282259 0.785328 0.850000 +8 0.227324 0.844709 0.848000 +Total time: 18:56 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_10.m +Loss and accuracy using (cls_best): [0.24939896, tensor(0.9435)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_10.m', + 0.9434999823570251)]) +Noise: 15 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_15.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 1500 examples, only 0.85 have correct labels +Added noise to 150 examples, only 0.85 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.15tv... +Data clsnoise0.15tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_15.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.679152 0.757799 0.816000 +2 0.715950 0.683697 0.810000 +3 0.694975 0.646247 0.817000 +4 0.629321 0.658788 0.818000 +5 0.588069 0.728769 0.805000 +6 0.517607 0.815471 0.796000 +7 0.402233 0.961845 0.781000 +8 0.318577 0.961705 0.779000 +Total time: 18:38 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_15.m +Loss and accuracy using (cls_best): [0.6598115, tensor(0.9300)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_15.m', + 0.9300000071525574)]) +Noise: 20 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_20.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 2000 examples, only 0.8 have correct labels +Added noise to 200 examples, only 0.8 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.2tv... +Data clsnoise0.2tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_20.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.814279 0.829818 0.755000 +2 0.824145 0.879503 0.763000 +3 0.778472 0.853143 0.766000 +4 0.782468 0.779830 0.762000 +5 0.700055 0.840979 0.766000 +6 0.600067 0.887506 0.748000 +7 0.477300 1.100056 0.734000 +8 0.372687 1.117019 0.729000 +Total time: 18:50 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_20.m +Loss and accuracy using (cls_best): [0.32372993, tensor(0.9053)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_20.m', + 0.9052500128746033)]) +Noise: 25 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_25.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 2500 examples, only 0.75 have correct labels +Added noise to 250 examples, only 0.75 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.25tv... +Data clsnoise0.25tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_25.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.930055 0.914462 0.709000 +2 0.875372 0.947374 0.723000 +3 0.903772 0.995556 0.716000 +4 0.830826 0.843742 0.727000 +5 0.761732 0.948633 0.719000 +6 0.652835 1.037753 0.698000 +7 0.522116 1.141807 0.687000 +8 0.456265 1.270369 0.675000 +Total time: 18:53 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_25.m +Loss and accuracy using (cls_best): [0.42412135, tensor(0.8655)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_25.m', + 0.8654999732971191)]) +Noise: 30 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_30.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 3000 examples, only 0.7 have correct labels +Added noise to 300 examples, only 0.7 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.3tv... +Data clsnoise0.3tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_30.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.979780 0.992838 0.647000 +2 0.964802 0.975392 0.650000 +3 0.984818 0.938703 0.664000 +4 0.947048 0.971088 0.667000 +5 0.835190 0.978763 0.651000 +6 0.779068 1.057726 0.654000 +7 0.640580 1.204921 0.617000 +8 0.581258 1.248827 0.613000 +Total time: 18:57 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_30.m +Loss and accuracy using (cls_best): [0.48816764, tensor(0.8470)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_30.m', + 0.847000002861023)]) +Noise: 35 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_35.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 3500 examples, only 0.65 have correct labels +Added noise to 350 examples, only 0.65 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.35tv... +Data clsnoise0.35tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_35.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.055006 0.998555 0.620000 +2 1.044137 0.994613 0.631000 +3 1.002715 1.070116 0.624000 +4 0.986447 1.054147 0.621000 +5 0.967057 1.016487 0.622000 +6 0.833950 1.251236 0.580000 +7 0.717729 1.236029 0.571000 +8 0.628723 1.342408 0.559000 +Total time: 19:03 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_35.m +Loss and accuracy using (cls_best): [0.6226422, tensor(0.7990)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_35.m', + 0.7990000247955322)]) +Noise: 40 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_40.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 4000 examples, only 0.6 have correct labels +Added noise to 400 examples, only 0.6 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.4tv... +Data clsnoise0.4tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_40.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.103033 1.086538 0.570000 +2 1.107361 1.056345 0.569000 +3 1.081129 1.063769 0.548000 +4 1.095896 1.061330 0.556000 +5 0.991429 1.102777 0.569000 +6 0.915104 1.178768 0.525000 +7 0.779928 1.327309 0.513000 +8 0.686467 1.426985 0.502000 +Total time: 18:50 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_40.m +Loss and accuracy using (cls_best): [0.8506142, tensor(0.7530)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_40.m', + 0.753000020980835)]) +Noise: 45 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_45.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 4500 examples, only 0.55 have correct labels +Added noise to 450 examples, only 0.55 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.45tv... +Data clsnoise0.45tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_45.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.140645 1.110783 0.512000 +2 1.136563 1.121161 0.535000 +3 1.129539 1.099606 0.509000 +4 1.108058 1.073747 0.524000 +5 1.048675 1.126527 0.504000 +6 0.956287 1.194883 0.479000 +7 0.818828 1.354711 0.461000 +8 0.758420 1.486676 0.448000 +Total time: 19:12 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_45.m +Loss and accuracy using (cls_best): [0.82442254, tensor(0.7000)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_45.m', + 0.699999988079071)]) +Noise: 50 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_50.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 5000 examples, only 0.5 have correct labels +Added noise to 500 examples, only 0.5 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.5tv... +Data clsnoise0.5tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_50.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.179953 1.145395 0.438000 +2 1.165212 1.175898 0.483000 +3 1.181145 1.127925 0.477000 +4 1.139963 1.151791 0.471000 +5 1.124215 1.130031 0.466000 +6 1.029675 1.185899 0.450000 +7 0.911210 1.320619 0.427000 +8 0.838111 1.383757 0.419000 +Total time: 18:22 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_50.m +Loss and accuracy using (cls_best): [0.9008743, tensor(0.6545)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_50.m', + 0.6545000076293945)]) +Noise: 55 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_55.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 5500 examples, only 0.45 have correct labels +Added noise to 550 examples, only 0.45 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.55tv... +Data clsnoise0.55tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_55.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.183246 1.166291 0.448000 +2 1.196840 1.163279 0.404000 +3 1.193226 1.180232 0.407000 +4 1.183507 1.153738 0.425000 +5 1.164359 1.166123 0.396000 +6 1.095628 1.207745 0.387000 +7 0.982455 1.293190 0.383000 +8 0.911601 1.383209 0.369000 +Total time: 19:04 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_55.m +Loss and accuracy using (cls_best): [1.0093645, tensor(0.6365)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_55.m', + 0.6365000009536743)]) +Noise: 60 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_60.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 6000 examples, only 0.4 have correct labels +Added noise to 600 examples, only 0.4 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.6tv... +Data clsnoise0.6tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_60.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.234308 1.216432 0.370000 +2 1.218607 1.166292 0.360000 +3 1.206626 1.179762 0.335000 +4 1.192636 1.164426 0.332000 +5 1.167782 1.226321 0.362000 +6 1.118646 1.225461 0.360000 +7 1.057706 1.246721 0.358000 +8 1.011610 1.281406 0.346000 +Total time: 18:43 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_60.m +Loss and accuracy using (cls_best): [1.0831424, tensor(0.4775)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_60.m', + 0.47749999165534973)]) +Noise: 65 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_65.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 6500 examples, only 0.35 have correct labels +Added noise to 650 examples, only 0.35 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.65tv... +Data clsnoise0.65tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_65.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.215925 1.200209 0.326000 +2 1.235970 1.173920 0.313000 +3 1.218506 1.251267 0.310000 +4 1.204741 1.182633 0.347000 +5 1.173637 1.171351 0.333000 +6 1.109750 1.226862 0.329000 +7 1.054864 1.330735 0.309000 +8 1.002966 1.325511 0.323000 +Total time: 18:20 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_65.m +Loss and accuracy using (cls_best): [1.2715616, tensor(0.3695)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_65.m', + 0.3695000112056732)]) +Noise: 70 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_70.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 7000 examples, only 0.3 have correct labels +Added noise to 700 examples, only 0.3 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.7tv... +Data clsnoise0.7tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_70.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.217860 1.219061 0.351000 +2 1.222316 1.191306 0.332000 +3 1.222368 1.254068 0.343000 +4 1.208153 1.216313 0.339000 +5 1.180702 1.166293 0.336000 +6 1.128917 1.211504 0.333000 +7 1.080312 1.271096 0.337000 +8 1.017895 1.335092 0.339000 +Total time: 18:47 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_70.m +Loss and accuracy using (cls_best): [1.4655445, tensor(0.2688)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_70.m', + 0.26875001192092896)]) +Noise: 75 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_75.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 7500 examples, only 0.25 have correct labels +Added noise to 750 examples, only 0.25 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.75tv... +Data clsnoise0.75tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_75.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.203315 1.147710 0.380000 +2 1.212273 1.190271 0.333000 +3 1.200970 1.208472 0.367000 +4 1.169665 1.165628 0.358000 +5 1.166296 1.213566 0.374000 +6 1.112285 1.462085 0.380000 +7 1.054748 1.266731 0.389000 +8 0.981918 1.692906 0.364000 +Total time: 18:34 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_75.m +Loss and accuracy using (cls_best): [1.957079, tensor(0.1252)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_75.m', + 0.12524999678134918)]) +{'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_0.m': 0.9712499976158142, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_5.m': 0.9564999938011169, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_10.m': 0.9434999823570251, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_15.m': 0.9300000071525574, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_20.m': 0.9052500128746033, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_25.m': 0.8654999732971191, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_30.m': 0.847000002861023, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_35.m': 0.7990000247955322, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_40.m': 0.753000020980835, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_45.m': 0.699999988079071, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_50.m': 0.6545000076293945, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_55.m': 0.6365000009536743, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_60.m': 0.47749999165534973, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_65.m': 0.3695000112056732, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_70.m': 0.26875001192092896, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_75.m': 0.12524999678134918} +``` +# Incorrect Val data + +``` python -m ulmfit eval_noise_resistance --size=10 +Noise: 0 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_0.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Running tokenization lm... +Data lm, trn: 13500, val: 1500 +Running tokenization cls... +Data cls, trn: 10000, val: 1000 +Running tokenization tst... +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: + + !! WARNING !! + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +Your compiler (c++) may be ABI-incompatible with PyTorch! +Please use a compiler that is ABI-compatible with GCC 4.9 and above. +See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. + +See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 +for instructions on how to install GCC 4.9 or higher. +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + !! WARNING !! + + warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_0.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.248545 0.179782 0.959000 +2 0.189914 0.262213 0.960000 +3 0.166565 0.489769 0.942000 +4 0.126547 0.252591 0.956000 +5 0.125812 0.265295 0.959000 +6 0.052031 0.339353 0.966000 +7 0.055669 0.452688 0.965000 +8 0.019922 0.439593 0.963000 +Total time: 18:52 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_0.m +Loss and accuracy using (cls_best): [0.3236656, tensor(0.9720)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_0.m', 0.972000002861023)]) +Noise: 5 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_5.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 500 examples, only 0.95 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.05... +Data clsnoise0.05, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_5.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.467763 0.204974 0.949000 +2 0.403194 0.166174 0.957000 +3 0.412454 0.176702 0.958000 +4 0.337252 0.957854 0.964000 +5 0.314917 0.179809 0.959000 +6 0.244516 0.202331 0.959000 +7 0.145377 0.225054 0.960000 +8 0.129216 0.239015 0.960000 +Total time: 19:15 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_5.m +Loss and accuracy using (cls_best): [0.18567306, tensor(0.9620)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_5.m', + 0.9620000123977661)]) +Noise: 10 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_10.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 1000 examples, only 0.9 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.1... +Data clsnoise0.1, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_10.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.611526 0.220041 0.950000 +2 0.523582 0.209251 0.959000 +3 0.566672 0.190530 0.960000 +4 0.530805 0.214880 0.949000 +5 0.470179 0.238882 0.954000 +6 0.368767 0.224209 0.950000 +7 0.276495 0.254708 0.942000 +8 0.242059 0.260761 0.940000 +Total time: 19:23 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_10.m +Loss and accuracy using (cls_best): [0.19178627, tensor(0.9557)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_10.m', + 0.9557499885559082)]) +Noise: 15 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_15.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 1500 examples, only 0.85 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.15... +Data clsnoise0.15, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_15.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.712348 0.224500 0.948000 +2 0.702935 0.188924 0.956000 +3 0.645956 0.224620 0.957000 +4 0.644018 0.248982 0.952000 +5 0.580399 0.255238 0.958000 +6 0.497618 0.297617 0.930000 +7 0.339819 0.272245 0.936000 +8 0.323421 0.297615 0.926000 +Total time: 18:50 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_15.m +Loss and accuracy using (cls_best): [0.25138617, tensor(0.9350)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_15.m', + 0.9350000023841858)]) +Noise: 20 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_20.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 2000 examples, only 0.8 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.2... +Data clsnoise0.2, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_20.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.821758 0.270898 0.954000 +2 0.831244 0.440837 0.923000 +3 0.787815 0.313425 0.962000 +4 0.730238 0.255505 0.958000 +5 0.748861 0.302285 0.959000 +6 0.603880 0.324315 0.933000 +7 0.539476 0.306479 0.931000 +8 0.431666 0.315723 0.916000 +Total time: 19:28 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_20.m +Loss and accuracy using (cls_best): [0.28679955, tensor(0.9202)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_20.m', + 0.9202499985694885)]) +Noise: 25 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_25.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 2500 examples, only 0.75 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.25... +Data clsnoise0.25, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_25.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.913877 0.325624 0.956000 +2 0.914296 0.380646 0.956000 +3 0.862887 0.400977 0.909000 +4 0.859236 0.291990 0.957000 +5 0.832637 0.447769 0.944000 +6 0.643751 0.319626 0.935000 +7 0.562953 0.444973 0.875000 +8 0.471118 0.437798 0.876000 +Total time: 19:29 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_25.m +Loss and accuracy using (cls_best): [0.53222567, tensor(0.8808)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_25.m', + 0.8807500004768372)]) +Noise: 30 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_30.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 3000 examples, only 0.7 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.3... +Data clsnoise0.3, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_30.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.967625 0.434347 0.921000 +2 1.015967 0.491977 0.927000 +3 0.943155 0.499572 0.935000 +4 0.953488 0.362500 0.951000 +5 0.859923 0.440134 0.944000 +6 0.761165 0.879942 0.894000 +7 0.662025 0.566052 0.857000 +8 0.572539 0.546405 0.826000 +Total time: 19:17 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_30.m +Loss and accuracy using (cls_best): [0.47183233, tensor(0.8540)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_30.m', + 0.8539999723434448)]) +Noise: 35 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_35.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 3500 examples, only 0.65 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.35... +Data clsnoise0.35, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_35.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.079700 0.486082 0.944000 +2 1.032614 0.470132 0.943000 +3 1.007776 0.460361 0.948000 +4 1.018879 0.476889 0.909000 +5 0.943363 1.618245 0.871000 +6 0.897375 0.604067 0.844000 +7 0.731791 10.002838 0.829000 +8 0.631117 6.527773 0.809000 +Total time: 19:26 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_35.m +Loss and accuracy using (cls_best): [0.79261243, tensor(0.8185)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_35.m', + 0.8184999823570251)]) +Noise: 40 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_40.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 4000 examples, only 0.6 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.4... +Data clsnoise0.4, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_40.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.105347 0.646775 0.915000 +2 1.121720 0.547566 0.943000 +3 1.084002 0.581386 0.949000 +4 1.058243 0.486947 0.953000 +5 1.020843 1.061957 0.945000 +6 0.944650 0.561371 0.872000 +7 0.852374 2.072736 0.815000 +8 0.745296 0.637481 0.807000 +Total time: 18:42 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_40.m +Loss and accuracy using (cls_best): [0.6664878, tensor(0.7915)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_40.m', + 0.7914999723434448)]) +Noise: 45 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_45.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 4500 examples, only 0.55 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.45... +Data clsnoise0.45, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_45.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.149571 0.715101 0.835000 +2 1.130510 0.812303 0.671000 +3 1.137401 0.566132 0.938000 +4 1.124969 0.654854 0.944000 +5 1.066340 0.624603 0.928000 +6 1.000705 0.700255 0.836000 +7 0.895356 0.726855 0.784000 +8 0.813315 0.740214 0.747000 +Total time: 19:12 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_45.m +Loss and accuracy using (cls_best): [0.74985105, tensor(0.7402)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_45.m', + 0.7402499914169312)]) +Noise: 50 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_50.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 5000 examples, only 0.5 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.5... +Data clsnoise0.5, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_50.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.176207 0.770225 0.894000 +2 1.146716 0.698875 0.916000 +3 1.159391 0.808971 0.908000 +4 1.148178 0.817309 0.715000 +5 1.142510 0.755769 0.872000 +6 1.068410 0.837932 0.839000 +7 0.985223 1.619010 0.714000 +8 0.900792 0.850288 0.672000 +Total time: 18:43 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_50.m +Loss and accuracy using (cls_best): [0.87151223, tensor(0.6888)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_50.m', + 0.6887500286102295)]) +Noise: 55 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_55.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 5500 examples, only 0.45 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.55... +Data clsnoise0.55, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_55.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.194526 0.829596 0.896000 +2 1.206947 0.800475 0.856000 +3 1.205839 0.832071 0.895000 +4 1.193005 8.321078 0.884000 +5 1.135684 3.162405 0.759000 +6 1.087508 0.866630 0.751000 +7 0.969481 0.909597 0.654000 +8 0.903410 1.040992 0.612000 +Total time: 18:41 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_55.m +Loss and accuracy using (cls_best): [0.944759, tensor(0.6118)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_55.m', + 0.6117500066757202)]) +Noise: 60 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_60.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 6000 examples, only 0.4 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.6... +Data clsnoise0.6, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_60.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.228728 1.235617 0.451000 +2 1.231521 1.103163 0.465000 +3 1.205319 1.067342 0.535000 +4 1.208439 1.171245 0.738000 +5 1.166922 1.071724 0.481000 +6 1.105134 0.913977 0.680000 +7 0.996566 1.203313 0.546000 +8 0.933625 1.092093 0.506000 +Total time: 18:30 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_60.m +Loss and accuracy using (cls_best): [1.0928471, tensor(0.5045)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_60.m', + 0.5044999718666077)]) +Noise: 65 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_65.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 6500 examples, only 0.35 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.65... +Data clsnoise0.65, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_65.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.233954 1.126451 0.312000 +2 1.228535 1.022103 0.514000 +3 1.223979 1.275958 0.040000 +4 1.201343 1.281353 0.233000 +5 1.174795 1.088655 0.714000 +6 1.137927 1.304352 0.430000 +7 1.061154 2.778949 0.450000 +8 0.992379 1.507618 0.409000 +Total time: 18:44 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_65.m +Loss and accuracy using (cls_best): [1.7004116, tensor(0.4027)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_65.m', + 0.4027499854564667)]) +Noise: 70 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_70.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 7000 examples, only 0.3 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.7... +Data clsnoise0.7, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_70.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.212556 1.284562 0.230000 +2 1.221429 1.494182 0.025000 +3 1.208761 1.601475 0.044000 +4 1.196318 1.222757 0.493000 +5 1.170390 1.235789 0.240000 +6 1.114086 1.487270 0.244000 +7 1.054812 1.470409 0.244000 +8 0.998996 1.476172 0.270000 +Total time: 19:12 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_70.m +Loss and accuracy using (cls_best): [1.4951355, tensor(0.2860)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_70.m', + 0.28600001335144043)]) +Noise: 75 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_75.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 7500 examples, only 0.25 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.75... +Data clsnoise0.75, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_75.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.206901 1.635707 0.035000 +2 1.226015 1.508531 0.222000 +3 1.203663 1.570317 0.020000 +4 1.201733 1.404657 0.054000 +5 1.161620 1.456142 0.040000 +6 1.142974 1.518606 0.037000 +7 1.067800 1.744556 0.098000 +8 0.983552 1.720158 0.117000 +Total time: 18:41 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_75.m +Loss and accuracy using (cls_best): [1.7742158, tensor(0.1138)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_75.m', + 0.11375000327825546)]) +{'data/mldoc/de-10/models/sp15k/qrnn_nl4_0.m': 0.972000002861023, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_5.m': 0.9620000123977661, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_10.m': 0.9557499885559082, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_15.m': 0.9350000023841858, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_20.m': 0.9202499985694885, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_25.m': 0.8807500004768372, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_30.m': 0.8539999723434448, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_35.m': 0.8184999823570251, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_40.m': 0.7914999723434448, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_45.m': 0.7402499914169312, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_50.m': 0.6887500286102295, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_55.m': 0.6117500066757202, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_60.m': 0.5044999718666077, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_65.m': 0.4027499854564667, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_70.m': 0.28600001335144043, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_75.m': 0.11375000327825546} +``` \ No newline at end of file diff --git a/results/logs/noise/es10k-noise.md b/results/logs/noise/es10k-noise.md new file mode 100644 index 0000000..e39e197 --- /dev/null +++ b/results/logs/noise/es10k-noise.md @@ -0,0 +1,1159 @@ +# Correct VAL +``` +python -m ulmfit eval_noise_resistance --lang=es --size=10 --prefix-name="val_" +Noise: 0 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_0.m +Evaluating previously trained model +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Data lm, trn: 13013, val: 1445 +Data cls, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: + + !! WARNING !! + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +Your compiler (c++) may be ABI-incompatible with PyTorch! +Please use a compiler that is ABI-compatible with GCC 4.9 and above. +See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. + +See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 +for instructions on how to install GCC 4.9 or higher. +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + !! WARNING !! + + warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) +Loss and accuracy using (cls_last): [0.38580656, tensor(0.9402)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_0.m', + 0.9402499794960022)]) +Noise: 5 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_5.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 472 examples, only 0.9500951575385916 have correct labels +Added noise to 50 examples, only 0.95 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.05tv... +Data clsnoise0.05tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_5.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.360209 0.664265 0.831000 +2 0.367246 0.505887 0.884000 +3 0.339675 0.646535 0.862000 +4 0.324250 0.914779 0.849000 +5 0.303626 0.592339 0.894000 +6 0.225317 0.699065 0.878000 +7 0.220272 0.726604 0.868000 +8 0.160895 0.767783 0.863000 +Total time: 11:24 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_5.m +Loss and accuracy using (cls_best): [0.36014587, tensor(0.9170)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_5.m', + 0.9169999957084656)]) +Noise: 10 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_10.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 945 examples, only 0.9000845844787482 have correct labels +Added noise to 100 examples, only 0.9 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.1tv... +Data clsnoise0.1tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_10.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.535271 0.740620 0.824000 +2 0.496433 1.023953 0.669000 +3 0.474564 0.993422 0.832000 +4 0.468398 6.249076 0.785000 +5 0.480843 0.850946 0.830000 +6 0.395878 0.789419 0.851000 +7 0.298389 2.299487 0.823000 +8 0.253688 2.279287 0.824000 +Total time: 11:21 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_10.m +Loss and accuracy using (cls_best): [0.871454, tensor(0.9075)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_10.m', + 0.9075000286102295)]) +Noise: 15 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_15.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 1418 examples, only 0.8500740114189046 have correct labels +Added noise to 150 examples, only 0.85 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.15tv... +Data clsnoise0.15tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_15.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.652501 1.015270 0.741000 +2 0.654432 1.025377 0.748000 +3 0.617786 1.639322 0.792000 +4 0.639355 1.662912 0.747000 +5 0.577110 5.244443 0.743000 +6 0.527992 3.701031 0.766000 +7 0.393420 3.620962 0.759000 +8 0.361106 3.019649 0.780000 +Total time: 11:31 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_15.m +Loss and accuracy using (cls_best): [0.35248652, tensor(0.9043)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_15.m', + 0.9042500257492065)]) +Noise: 20 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_20.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 1891 examples, only 0.8000634383590611 have correct labels +Added noise to 200 examples, only 0.8 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.2tv... +Data clsnoise0.2tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_20.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.749732 1.118057 0.710000 +2 0.785584 6.410913 0.607000 +3 0.713528 2.003799 0.754000 +4 0.752761 2.294566 0.724000 +5 0.652672 19.180836 0.708000 +6 0.596508 2.582183 0.711000 +7 0.489806 2.648639 0.712000 +8 0.467246 3.001610 0.698000 +Total time: 11:36 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_20.m +Loss and accuracy using (cls_best): [1.6113901, tensor(0.8475)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_20.m', + 0.8475000262260437)]) +Noise: 25 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_25.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 2364 examples, only 0.7500528652992176 have correct labels +Added noise to 250 examples, only 0.75 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.25tv... +Data clsnoise0.25tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_25.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.844129 1.047693 0.663000 +2 0.810037 1.353432 0.655000 +3 0.820446 3.722627 0.632000 +4 0.779205 1.445673 0.630000 +5 0.770405 1.024933 0.653000 +6 0.695584 1.400032 0.676000 +7 0.582116 1.612832 0.649000 +8 0.542117 1.422031 0.648000 +Total time: 11:48 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_25.m +Loss and accuracy using (cls_best): [0.62123287, tensor(0.8300)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_25.m', + 0.8299999833106995)]) +Noise: 30 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_30.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 2837 examples, only 0.7000422922393741 have correct labels +Added noise to 300 examples, only 0.7 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.3tv... +Data clsnoise0.3tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_30.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.946058 1.269022 0.625000 +2 0.942436 2.030650 0.636000 +3 0.920219 5.243942 0.463000 +4 0.890941 14.097425 0.605000 +5 0.835836 5.551986 0.617000 +6 0.708443 2.086009 0.624000 +7 0.639320 2.845402 0.582000 +8 0.537422 1.683678 0.597000 +Total time: 11:21 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_30.m +Loss and accuracy using (cls_best): [0.6909822, tensor(0.7915)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_30.m', + 0.7914999723434448)]) +Noise: 35 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_35.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 3310 examples, only 0.6500317191795305 have correct labels +Added noise to 350 examples, only 0.65 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.35tv... +Data clsnoise0.35tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_35.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.980721 1.266943 0.471000 +2 0.980537 1.438193 0.492000 +3 1.000082 5.081447 0.539000 +4 0.964535 1.640518 0.547000 +5 0.924568 2.078268 0.588000 +6 0.915998 1.612291 0.562000 +7 0.790721 1.804647 0.522000 +8 0.716465 2.185110 0.520000 +Total time: 11:48 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_35.m +Loss and accuracy using (cls_best): [1.8067851, tensor(0.7657)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_35.m', + 0.765749990940094)]) +Noise: 40 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_40.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 3783 examples, only 0.600021146119687 have correct labels +Added noise to 400 examples, only 0.6 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.4tv... +Data clsnoise0.4tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_40.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.031590 1.277689 0.458000 +2 1.056317 2.782609 0.349000 +3 1.044410 1.631716 0.449000 +4 1.034872 2.103323 0.478000 +5 0.966710 1.642946 0.472000 +6 0.943393 1.632070 0.490000 +7 0.856185 1.868247 0.479000 +8 0.780535 1.958063 0.483000 +Total time: 11:36 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_40.m +Loss and accuracy using (cls_best): [0.8928685, tensor(0.7508)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_40.m', + 0.7507500052452087)]) +Noise: 45 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_45.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 4256 examples, only 0.5500105730598435 have correct labels +Added noise to 450 examples, only 0.55 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.45tv... +Data clsnoise0.45tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_45.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.064314 1.302667 0.444000 +2 1.094542 2.054388 0.439000 +3 1.071187 2.209423 0.450000 +4 1.077246 6.365521 0.471000 +5 1.051128 3.242248 0.406000 +6 0.998142 2.802290 0.464000 +7 0.918943 2.492715 0.468000 +8 0.864835 8.349169 0.457000 +Total time: 11:39 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_45.m +Loss and accuracy using (cls_best): [6.738212, tensor(0.6977)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_45.m', + 0.6977499723434448)]) +Noise: 50 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_50.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 4729 examples, only 0.5 have correct labels +Added noise to 500 examples, only 0.5 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.5tv... +Data clsnoise0.5tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_50.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.101441 1.358480 0.450000 +2 1.150545 1.360492 0.392000 +3 1.120479 1.384937 0.437000 +4 1.110555 1.345856 0.401000 +5 1.089176 1.815834 0.420000 +6 1.051251 2.371094 0.422000 +7 1.004491 1.379894 0.410000 +8 0.936776 1.549806 0.424000 +Total time: 11:37 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_50.m +Loss and accuracy using (cls_best): [1.1773515, tensor(0.6180)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_50.m', + 0.6179999709129333)]) +Noise: 55 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_55.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 5201 examples, only 0.4500951575385917 have correct labels +Added noise to 550 examples, only 0.45 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.55tv... +Data clsnoise0.55tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_55.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.149922 1.401648 0.383000 +2 1.144570 9.186795 0.301000 +3 1.162765 1.583447 0.330000 +4 1.154295 1.568725 0.349000 +5 1.117640 1.624917 0.352000 +6 1.097049 2.510476 0.362000 +7 1.048231 3.798041 0.362000 +8 1.020844 3.568361 0.366000 +Total time: 11:25 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_55.m +Loss and accuracy using (cls_best): [1.6726145, tensor(0.6102)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_55.m', + 0.6102499961853027)]) +Noise: 60 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_60.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 5674 examples, only 0.4000845844787482 have correct labels +Added noise to 600 examples, only 0.4 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.6tv... +Data clsnoise0.6tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_60.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.166367 1.629363 0.358000 +2 1.175406 1.307703 0.341000 +3 1.172992 1.437346 0.342000 +4 1.163454 10.670442 0.339000 +5 1.158758 2.170272 0.325000 +6 1.122279 11.275146 0.351000 +7 1.080830 24.562853 0.359000 +8 1.055806 19.967308 0.354000 +Total time: 11:25 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_60.m +Loss and accuracy using (cls_best): [9.046943, tensor(0.4873)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_60.m', + 0.4872500002384186)]) +Noise: 65 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_65.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 6147 examples, only 0.35007401141890465 have correct labels +Added noise to 650 examples, only 0.35 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.65tv... +Data clsnoise0.65tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_65.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.188989 1.408018 0.326000 +2 1.182579 1.576379 0.279000 +3 1.180070 1.482971 0.257000 +4 1.164961 1.580750 0.283000 +5 1.166036 1.530756 0.294000 +6 1.136221 1.670132 0.297000 +7 1.102457 2.092621 0.291000 +8 1.058724 2.033105 0.287000 +Total time: 11:22 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_65.m +Loss and accuracy using (cls_best): [5.145201, tensor(0.4150)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_65.m', + 0.41499999165534973)]) +Noise: 70 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_70.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 6620 examples, only 0.30006343835906113 have correct labels +Added noise to 700 examples, only 0.3 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.7tv... +Data clsnoise0.7tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_70.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.181723 1.387360 0.342000 +2 1.179477 11.148307 0.343000 +3 1.178954 2.155812 0.293000 +4 1.177750 2.038787 0.308000 +5 1.156294 8.581575 0.319000 +6 1.122391 1.584792 0.346000 +7 1.095312 2.261911 0.346000 +8 1.048026 1.873787 0.337000 +Total time: 11:40 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_70.m +Loss and accuracy using (cls_best): [2.1144376, tensor(0.2125)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_70.m', + 0.21250000596046448)]) +Noise: 75 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_75.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 7093 examples, only 0.2500528652992176 have correct labels +Added noise to 750 examples, only 0.25 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.75tv... +Data clsnoise0.75tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_75.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.163953 1.370035 0.298000 +2 1.167424 1.883017 0.354000 +3 1.145917 1.854918 0.353000 +4 1.155910 1.328685 0.329000 +5 1.141541 1.895262 0.342000 +6 1.091663 22.743765 0.326000 +7 1.047826 50.595406 0.315000 +8 0.997792 39.199989 0.317000 +Total time: 11:30 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_75.m +Loss and accuracy using (cls_best): [38.87539, tensor(0.1922)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_75.m', + 0.19224999845027924)]) +{'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_0.m': 0.9402499794960022, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_5.m': 0.9169999957084656, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_10.m': 0.9075000286102295, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_15.m': 0.9042500257492065, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_20.m': 0.8475000262260437, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_25.m': 0.8299999833106995, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_30.m': 0.7914999723434448, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_35.m': 0.765749990940094, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_40.m': 0.7507500052452087, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_45.m': 0.6977499723434448, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_50.m': 0.6179999709129333, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_55.m': 0.6102499961853027, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_60.m': 0.4872500002384186, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_65.m': 0.41499999165534973, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_70.m': 0.21250000596046448, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_75.m': 0.19224999845027924} +``` + +# Incorrect Val data +``` +Noise: 0 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_0.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Running tokenization lm... +Data lm, trn: 13013, val: 1445 +Running tokenization cls... +Data cls, trn: 9458, val: 1000 +Running tokenization tst... +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: + + !! WARNING !! + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +Your compiler (c++) may be ABI-incompatible with PyTorch! +Please use a compiler that is ABI-compatible with GCC 4.9 and above. +See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. + +See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 +for instructions on how to install GCC 4.9 or higher. +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + !! WARNING !! + + warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_0.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.118179 0.265471 0.939000 +2 0.131938 0.823511 0.839000 +3 0.139424 0.496273 0.906000 +4 0.065336 0.315253 0.951000 +5 0.050494 0.366971 0.938000 +6 0.041825 0.296858 0.965000 +7 0.023676 0.330993 0.957000 +8 0.014899 0.337952 0.952000 +Total time: 11:14 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_0.m +Loss and accuracy using (cls_best): [0.37119418, tensor(0.9465)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_0.m', + 0.9465000033378601)]) +Noise: 5 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_5.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 472 examples, only 0.9500951575385916 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.05... +Data clsnoise0.05, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_5.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.381904 0.318090 0.893000 +2 0.366108 0.304496 0.907000 +3 0.350814 0.222635 0.954000 +4 0.305924 0.316391 0.937000 +5 0.294849 0.365258 0.938000 +6 0.226888 0.265155 0.953000 +7 0.203635 0.277920 0.944000 +8 0.180686 0.326215 0.935000 +Total time: 11:05 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_5.m +Loss and accuracy using (cls_best): [0.31352887, tensor(0.9302)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_5.m', + 0.9302499890327454)]) +Noise: 10 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_10.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 945 examples, only 0.9000845844787482 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.1... +Data clsnoise0.1, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_10.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.503315 0.294730 0.903000 +2 0.520918 0.404052 0.908000 +3 0.496451 0.623593 0.825000 +4 0.484413 0.212174 0.950000 +5 0.397562 0.271711 0.929000 +6 0.393162 0.455898 0.908000 +7 0.329386 0.306643 0.922000 +8 0.285580 0.283296 0.921000 +Total time: 11:17 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_10.m +Loss and accuracy using (cls_best): [0.2845364, tensor(0.9258)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_10.m', + 0.9257500171661377)]) +Noise: 15 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_15.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 1418 examples, only 0.8500740114189046 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.15... +Data clsnoise0.15, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_15.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.641261 0.416610 0.848000 +2 0.602479 0.356870 0.901000 +3 0.652983 0.582720 0.812000 +4 0.582735 22.428156 0.942000 +5 0.565184 7.965161 0.855000 +6 0.489513 0.751756 0.898000 +7 0.418973 6.047875 0.859000 +8 0.337963 9.058367 0.845000 +Total time: 11:23 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_15.m +Loss and accuracy using (cls_best): [11.41559, tensor(0.8332)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_15.m', + 0.8332499861717224)]) +Noise: 20 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_20.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 1891 examples, only 0.8000634383590611 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.2... +Data clsnoise0.2, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_20.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.735070 0.430566 0.875000 +2 0.777014 1.136241 0.711000 +3 0.762944 10.364647 0.624000 +4 0.698076 1.523277 0.806000 +5 0.646547 1.162918 0.833000 +6 0.581218 1.385464 0.869000 +7 0.489041 1.207447 0.843000 +8 0.398324 0.668090 0.849000 +Total time: 11:16 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_20.m +Loss and accuracy using (cls_best): [0.55489075, tensor(0.8595)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_20.m', + 0.859499990940094)]) +Noise: 25 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_25.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 2364 examples, only 0.7500528652992176 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.25... +Data clsnoise0.25, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_25.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.877402 0.428968 0.871000 +2 0.874461 0.585437 0.772000 +3 0.866958 0.598036 0.807000 +4 0.805568 0.964052 0.893000 +5 0.760956 5.249828 0.800000 +6 0.671527 1.038407 0.864000 +7 0.597481 0.767250 0.855000 +8 0.539228 0.695689 0.853000 +Total time: 11:29 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_25.m +Loss and accuracy using (cls_best): [0.61637276, tensor(0.8630)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_25.m', + 0.8629999756813049)]) +Noise: 30 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_30.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 2837 examples, only 0.7000422922393741 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.3... +Data clsnoise0.3, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_30.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.908243 0.480479 0.818000 +2 0.906832 0.562728 0.773000 +3 0.896891 3.017841 0.750000 +4 0.900984 2.464900 0.892000 +5 0.874264 2.142747 0.833000 +6 0.775662 0.491934 0.876000 +7 0.614158 0.568567 0.840000 +8 0.553789 3.695555 0.797000 +Total time: 11:32 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_30.m +Loss and accuracy using (cls_best): [2.999626, tensor(0.7897)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_30.m', + 0.7897499799728394)]) +Noise: 35 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_35.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 3310 examples, only 0.6500317191795305 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.35... +Data clsnoise0.35, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_35.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.983643 0.519147 0.891000 +2 1.010607 0.945973 0.580000 +3 0.994139 3.134296 0.461000 +4 0.984234 1.258144 0.864000 +5 0.923507 9.997900 0.759000 +6 0.886210 9.016973 0.667000 +7 0.791056 7.744195 0.798000 +8 0.683184 4.557703 0.787000 +Total time: 11:28 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_35.m +Loss and accuracy using (cls_best): [4.7776446, tensor(0.7642)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_35.m', + 0.7642499804496765)]) +Noise: 40 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_40.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 3783 examples, only 0.600021146119687 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.4... +Data clsnoise0.4, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_40.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.029997 0.882142 0.651000 +2 1.044314 1.591835 0.618000 +3 1.054599 0.522415 0.862000 +4 1.033360 0.998230 0.747000 +5 1.002742 3.083807 0.803000 +6 0.951022 11.693688 0.741000 +7 0.859333 4.469426 0.674000 +8 0.770237 1.737012 0.712000 +Total time: 11:26 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_40.m +Loss and accuracy using (cls_best): [1.721135, tensor(0.7135)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_40.m', + 0.7135000228881836)]) +Noise: 45 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_45.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 4256 examples, only 0.5500105730598435 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.45... +Data clsnoise0.45, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_45.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.118304 0.964211 0.720000 +2 1.105983 1.031464 0.459000 +3 1.090640 1.993658 0.545000 +4 1.094765 2.314941 0.550000 +5 1.075982 8.962537 0.682000 +6 1.044638 4.634834 0.743000 +7 0.997964 1.342650 0.711000 +8 0.953207 1.860142 0.728000 +Total time: 11:14 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_45.m +Loss and accuracy using (cls_best): [2.065772, tensor(0.7682)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_45.m', + 0.7682499885559082)]) +Noise: 50 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_50.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 4729 examples, only 0.5 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.5... +Data clsnoise0.5, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_50.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.131220 0.884647 0.677000 +2 1.118103 0.786608 0.760000 +3 1.135599 1.448700 0.654000 +4 1.131183 1.091670 0.655000 +5 1.087481 1.086206 0.755000 +6 1.071505 47.305462 0.752000 +7 1.008586 22.109758 0.724000 +8 0.978073 8.738451 0.704000 +Total time: 11:20 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_50.m +Loss and accuracy using (cls_best): [7.9533467, tensor(0.6925)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_50.m', + 0.6924999952316284)]) +Noise: 55 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_55.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 5201 examples, only 0.4500951575385917 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.55... +Data clsnoise0.55, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_55.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.161361 0.962813 0.596000 +2 1.150018 1.615359 0.542000 +3 1.154646 5.588564 0.572000 +4 1.145129 1.064145 0.541000 +5 1.138727 0.868817 0.716000 +6 1.092911 3.203760 0.673000 +7 1.027020 2.577705 0.615000 +8 0.962490 2.068990 0.607000 +Total time: 11:23 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_55.m +Loss and accuracy using (cls_best): [1.8640332, tensor(0.6400)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_55.m', + 0.6399999856948853)]) +Noise: 60 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_60.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 5674 examples, only 0.4000845844787482 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.6... +Data clsnoise0.6, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_60.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.184716 1.159816 0.489000 +2 1.162386 1.785291 0.585000 +3 1.189221 6.945411 0.523000 +4 1.169163 30.551344 0.553000 +5 1.143203 96.431221 0.434000 +6 1.113257 18.963854 0.475000 +7 1.083783 8.783161 0.521000 +8 1.064840 7.930975 0.462000 +Total time: 11:40 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_60.m +Loss and accuracy using (cls_best): [8.483881, tensor(0.4535)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_60.m', + 0.45350000262260437)]) +Noise: 65 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_65.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 6147 examples, only 0.35007401141890465 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.65... +Data clsnoise0.65, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_65.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.163443 1.302967 0.355000 +2 1.181459 1.265248 0.188000 +3 1.189486 6.507479 0.466000 +4 1.192368 23.974792 0.525000 +5 1.171117 2.516582 0.473000 +6 1.139664 1.357649 0.398000 +7 1.091899 1.510042 0.437000 +8 1.070406 1.511683 0.392000 +Total time: 11:29 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_65.m +Loss and accuracy using (cls_best): [1.5956299, tensor(0.3887)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_65.m', + 0.38874998688697815)]) +Noise: 70 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_70.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 6620 examples, only 0.30006343835906113 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.7... +Data clsnoise0.7, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_70.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.186880 1.400069 0.164000 +2 1.193253 3.277115 0.258000 +3 1.173004 2.955142 0.184000 +4 1.162564 3.158204 0.105000 +5 1.158516 8.930537 0.178000 +6 1.117573 25.560179 0.178000 +7 1.070933 3.045306 0.183000 +8 1.025959 10.724563 0.192000 +Total time: 11:31 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_70.m +Loss and accuracy using (cls_best): [9.275186, tensor(0.2153)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_70.m', + 0.21525000035762787)]) +Noise: 75 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_75.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 7093 examples, only 0.2500528652992176 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.75... +Data clsnoise0.75, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_75.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.173947 1.503179 0.149000 +2 1.155268 3.008709 0.140000 +3 1.166699 4.902722 0.181000 +4 1.160365 4.787449 0.190000 +5 1.138678 3.777317 0.084000 +6 1.071183 3.077753 0.123000 +7 1.000427 2.818838 0.166000 +8 0.941079 3.680126 0.187000 +Total time: 11:42 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_75.m +Loss and accuracy using (cls_best): [3.2255151, tensor(0.1877)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_75.m', + 0.18774999678134918)]) +{'data/mldoc/es-10/models/sp15k/qrnn_nl4_0.m': 0.9465000033378601, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_5.m': 0.9302499890327454, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_10.m': 0.9257500171661377, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_15.m': 0.8332499861717224, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_20.m': 0.859499990940094, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_25.m': 0.8629999756813049, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_30.m': 0.7897499799728394, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_35.m': 0.7642499804496765, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_40.m': 0.7135000228881836, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_45.m': 0.7682499885559082, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_50.m': 0.6924999952316284, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_55.m': 0.6399999856948853, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_60.m': 0.45350000262260437, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_65.m': 0.38874998688697815, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_70.m': 0.21525000035762787, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_75.m': 0.18774999678134918} +```` \ No newline at end of file From 015f04ec087f85dbb89667ce3eb63be9d0281edc Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Fri, 22 Feb 2019 12:02:18 +0100 Subject: [PATCH 2/3] Training with noise & label smoothing --- tests/test_end_to_end.py | 20 ++++++++++++++++++++ ulmfit/__main__.py | 37 ++++++++++++++++++++++++++++--------- ulmfit/pretrain_lm.py | 23 +++++++++++++++++++---- ulmfit/train_clas.py | 35 +++++++++++++++++++++++------------ 4 files changed, 90 insertions(+), 25 deletions(-) diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index 266bc3f..73ded8a 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -115,6 +115,26 @@ def test_ulmfit_fastai_end_to_end(): exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, ) +def test_ulmfit_fastai_end_to_end_label_smoothing(): + """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset. + """ + test_data, wt2 = get_test_data() + lm_name = 'end-to-end-test-fastai' + + exp = ulmfit.pretrain_lm.LMHyperParams( + dataset_path=wt2, + lang='en', + cuda_id=cuda_id, + qrnn=False, + tokenizer='f', + max_vocab=100, + name=lm_name, + ) + exp.train_lm(num_epochs=1, bs=2, label_smoothing_eps=0.1) + exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) + exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, label_smoothing_eps=0.1 ) + + def test_ulmfit_fastai_bidir_end_to_end(): """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset. """ diff --git a/ulmfit/__main__.py b/ulmfit/__main__.py index 9af2ee4..13a3922 100644 --- a/ulmfit/__main__.py +++ b/ulmfit/__main__.py @@ -1,12 +1,13 @@ import gc import os import pprint +import tarfile import shutil from collections import OrderedDict from functools import wraps - +import pandas as pd import fire -from .pretrain_lm import LMHyperParams, np +from .pretrain_lm import LMHyperParams from .train_clas import CLSHyperParams from pathlib import Path from string import Template @@ -49,19 +50,37 @@ class ULMFiT: return FireView(train=params.train_cls, validate_cls=params.validate_cls) - def eval_noise_resistance(self, lang="de"): - results = {} + def eval_noise_resistance(self, lang="de", size=1, prefix_name="", model="sp15k/qrnn_nl4.m"): + def first_or_default(l, default=None): + l = list(l) + if l: + return l[0] + return default + results= [] for noise in range(0, 80, 5): print("Noise: ", noise) - d = self.eval(glob=f"mldoc/{lang}-1/models/sp15k/qrnn_nl4.m", - name=f"nl4_{noise}", + d = self.eval(glob=f"mldoc/{lang}-1/models/{model}", + name=f"nl4_{prefix_name}{noise}", noise=noise/100, + dataset_template='${lang}-'+str(size), num_cls_epochs=8, bs=18, lr_sched="1cycle") - results.update(d) - np.save('results.npy', results) - print(results) + val = first_or_default(d.values(), default=-1) + results.append((noise/100, val)) + df = pd.DataFrame(results, columns=["noise", "accuracy"]) + df.to_csv(f"noise_{lang}-{size}{prefix_name}.csv") + print(df) + + def tar(self, model_path): + params = CLSHyperParams.from_json(model_path) + tar_name = f"models/{params.lang}-{params.tokenizer_prefix}-{params.model_name}.tar" + print("Storing model in", tar_name) + with tarfile.open(tar_name, mode="w") as tar: + for g in map(params.model_dir.glob, ['*_last.*', 'info.json', 'info.json', '../spm.*', '../itos.*',]): + for f in g: + print("Adding", f, f.relative_to("data")) + tar.add(f, f.relative_to("data")) def eval(self, glob="mldoc/*-1/models/sp30k/lstm_nl4.m", dataset_template='${lang}-1', name="tmp-100", num_lm_epochs=0, cuda_id=0, **trn_params): results = OrderedDict() diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 655116c..5b70ef6 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -171,10 +171,10 @@ class LMHyperParams: with (self.model_dir / 'info.json').open("w") as fp: json.dump(vals, fp) print("Saving info", self.model_dir / 'info.json') - def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, lr=5e-3): + def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, lr=5e-3, label_smoothing_eps=0.0): self.model_dir.mkdir(exist_ok=True, parents=True) data_lm = self.load_wiki_data(bs=bs) if data_lm is None else data_lm - learn = self.create_lm_learner(data_lm, drop_mult=drop_mult) + learn = self.create_lm_learner(data_lm, drop_mult=drop_mult, label_smoothing_eps=label_smoothing_eps) learn.true_wd = true_wd if num_epochs > 0: @@ -204,7 +204,7 @@ class LMHyperParams: # do we need to return `learn'? it adds noise to Fire output #return learn - def create_lm_learner(self, data_lm, dps=None, **kwargs): + def create_lm_learner(self, data_lm, dps=None, label_smoothing_eps=0.0, **kwargs): assert self.bidir == False, "bidirectional model is not yet supported" config = dict(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, pad_token=PAD_TOKEN_ID, qrnn=self.qrnn, tie_weights=True, out_bias=True) @@ -230,6 +230,8 @@ class LMHyperParams: learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/lm-history"), # partial(SaveModelCallback, every='improvement', name='lm') disabled due to Memory issues ] + if label_smoothing_eps > 0.0: + learn.loss_func = LabelSmoothingCrossEntropy(eps=label_smoothing_eps) return learn def load_train_text(self): @@ -316,13 +318,26 @@ class LMHyperParams: d.update(kwargs) return cls(**d) @classmethod - def from_json(cls, model_path, **kwargs): + def from_json(cls, model_path:Path, **kwargs): model_path = Path(model_path).resolve() + name = re.search(r"[a-z]+_(.+).m", model_path.name).group(1) with open(model_path / 'info.json', 'r') as f: d = json.load(f) d.update(kwargs) + d['name'] = name + dataset_path = path_strip(model_path, "data", "models").parent + d['dataset_path'] = str(dataset_path) + d['lang'] = infer_lang_from_dataset(dataset_path.name) return cls(**d) +def infer_lang_from_dataset(name:str): + return name.split("-")[0] + +def path_strip(path, from_folder, to_folder): + to_p = [p for p in path.parents if p.name == to_folder][0] + from_p = [p for p in path.parents if p.name == from_folder][0] + return to_p.relative_to(from_p.parent) + def validate_lm(self): if not self.exp.subword and self.exp.max_vocab is None: raise NotImplementedError("figure out how to validate and save results") diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index e23688e..278a629 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -64,7 +64,8 @@ class CLSHyperParams(LMHyperParams): learn.fit_one_cycle(num_cls_epochs-4, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7), wd=1e-7) def train_cls(self, num_lm_epochs, unfreeze=True, num_cls_frozen_epochs=1, bs=40, drop_mul_lm=0.3, drop_mul_cls=0.5, - use_test_for_validation=False, num_cls_epochs=2, limit=None, noise=0.0, cls_max_len=20*70, lr_sched='layered'): + use_test_for_validation=False, num_cls_epochs=2, limit=None, noise=0.0, cls_max_len=20*70, lr_sched='layered', + label_smoothing_eps=0.0): assert use_test_for_validation == False, "use_test_for_validation=True is not supported" self.model_dir.mkdir(exist_ok=True, parents=True) @@ -73,8 +74,8 @@ class CLSHyperParams(LMHyperParams): data_clas, data_lm, data_tst = self.load_cls_data(bs, limit=limit, noise=noise) - if self.need_fine_tune_lm: self.train_lm(num_lm_epochs, data_lm=data_lm, drop_mult=drop_mul_lm) - learn = self.create_cls_learner(data_clas, drop_mult=drop_mul_cls, max_len=cls_max_len) + if self.need_fine_tune_lm: self.train_lm(num_lm_epochs, data_lm=data_lm, drop_mult=drop_mul_lm, label_smoothing_eps=label_smoothing_eps) + learn = self.create_cls_learner(data_clas, drop_mult=drop_mul_cls, max_len=cls_max_len, label_smoothing_eps=label_smoothing_eps) try: learn.load('cls_last') print("Loading last classifier") @@ -84,6 +85,8 @@ class CLSHyperParams(LMHyperParams): if hasattr(self, 'lr_schedule_'+lr_sched): learn.true_wd = True getattr(self, 'lr_schedule_'+lr_sched)(learn, num_cls_epochs) + else: + raise ValueError(f"Wrong lr_sched: {lr_sched}") print(f"Saving models at {learn.path / learn.model_dir}") learn.save('cls_last', with_opt=False) @@ -102,7 +105,7 @@ class CLSHyperParams(LMHyperParams): print(f"Loss and accuracy using ({save_name}):", results) return list(map(float, results)) - def create_cls_learner(self, data_clas, dps=None, **kwargs): + def create_cls_learner(self, data_clas, dps=None, label_smoothing_eps=0.0, **kwargs): assert self.bidir == False, "bidirectional model is not yet supported" config = dict(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, pad_token=PAD_TOKEN_ID, qrnn=self.qrnn) config.update(dps or self.dps) @@ -121,6 +124,8 @@ class CLSHyperParams(LMHyperParams): learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/cls-history"), #partial(SaveModelCallback, every='improvement', name='cls_best') disabled due to memory issues ] + if label_smoothing_eps > 0.0: + learn.loss_func = LabelSmoothingCrossEntropy(eps=label_smoothing_eps) return learn def load_cls_data(self, bs, **kwargs): @@ -178,6 +183,17 @@ class CLSHyperParams(LMHyperParams): kwargs.update(dict(trn_df=trn_df, val_df=val_df, tst_df=tst_df, unsup_df=unsup_df)) return kwargs + def add_noise(self, trn_df, noise): + count = len(trn_df) + labels = trn_df[0].unique() + assert np.issubdtype(labels.dtype, np.integer), "noise only works on numerical numbers" + modulo = labels.max() + 1 + idx_to_distrub = np.random.permutation(count)[:int(count * noise)] + trn_df.loc[idx_to_distrub, [0]] = (np.random.randint(1, modulo - 1, size=len(idx_to_distrub)) + + trn_df.loc[idx_to_distrub][0]) % modulo + print(f"Added noise to {len(idx_to_distrub)} examples, only {(count - len(idx_to_distrub)) / count} have correct labels") + return trn_df + def databunches(self, bs, trn_df, val_df, tst_df, unsup_df, add_trn_to_lm=True, use_moses=False, force=False, limit=None, noise=0.0): lm_trn_df = pd.concat([unsup_df, val_df, tst_df] + ([trn_df] if add_trn_to_lm else [])) val_len = max(int(len(lm_trn_df) * 0.1), 2) @@ -192,14 +208,9 @@ class CLSHyperParams(LMHyperParams): cls_name=f'{cls_name}limit{limit}' if noise > 0.0: - count = len(trn_df) - labels = trn_df[0].unique() - assert np.issubdtype(labels.dtype, np.integer), "noise only works on numerical numbers" - modulo = labels.max()+1 - idx_to_distrub = np.random.permutation(count)[:int(count * noise)] - trn_df.loc[idx_to_distrub, [0]] = (np.random.randint(1, modulo-1, size=len(idx_to_distrub)) + trn_df.loc[idx_to_distrub][0]) % modulo - print(f"Added noise to {len(idx_to_distrub)} examples, only {(count-len(idx_to_distrub))/count} have correct labels") - cls_name = f'{cls_name}noise{noise}' + trn_df = self.add_noise(trn_df, noise) + val_df = self.add_noise(val_df, noise) + cls_name = f'{cls_name}noise{noise}tv' args = self.tokenizer_to_fastai_args(sp_data_func=lambda: trn_df[1], use_moses=use_moses) data_lm = self.lm_databunch('lm', train_df=lm_trn_df, valid_df=lm_val_df, bs=bs, force=force, **args) From 260faa703cf4f5d1eb8584bc5e3a5b1b10ae40fc Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Fri, 22 Feb 2019 16:55:35 +0100 Subject: [PATCH 3/3] Correct the label smoothing implementation --- results/MLDoc.md | 4 +- results/logs/label_smoothing.md | 223 ++++++++++++++++++++++++++++++++ results/logs/qrnn-ru.md | 12 ++ tests/test_end_to_end.py | 4 +- ulmfit/__main__.py | 2 +- ulmfit/pretrain_lm.py | 2 +- ulmfit/train_clas.py | 6 +- 7 files changed, 246 insertions(+), 7 deletions(-) create mode 100644 results/logs/label_smoothing.md diff --git a/results/MLDoc.md b/results/MLDoc.md index 67cdbf0..5448079 100644 --- a/results/MLDoc.md +++ b/results/MLDoc.md @@ -10,7 +10,9 @@ |ULMFiT L30k | | 95.4 | 95.15 | 93.67 | 88.42 | 89.20 | **87.27** | 90.20 | |ULMFiT sp-fixed | | **95.6** | 94.80 | 94.20 | 88.52 | 88.72 | 86.85 | 90.47 | |ULMFIT Q15k 1cyc| **94.62** | **95.65** | 95.15 | **94.42** | **89.92** | 89.60 | | 90.78/89.82 | -|ULMFIT L30k 1cyc| | **95.85** | **96.32** | **94.82** | 89.87 | **90.45** | **87.94** | **92.02/91.64** | +|ULMFIT Q15k 1c l| | | | | | | | **92.22** | +|ULMFIT L30k 1cyc| | **95.85** | **96.32** | **94.82** | 89.87 | **90.45** | **87.94** | 92.02/91.64 | + - L30k - LSTM sp30k trained using gradual unfreezing - L30k-100 - --||-- **on 100 samples** - ULMFiT sp-fixed - --||-- with fixed tokenization diff --git a/results/logs/label_smoothing.md b/results/logs/label_smoothing.md new file mode 100644 index 0000000..c52df1e --- /dev/null +++ b/results/logs/label_smoothing.md @@ -0,0 +1,223 @@ +# MLDoc + +## QRNN 15k + +Exec 1 +``` +python -m ulmfit eval --glob="mldoc/zh-1/models/sp15k/qrnn_nl4.m" --name nl4-1cyc-sl --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.1 +Processing data/mldoc/zh-1/models/sp15k/qrnn_nl4.m +zh-1 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.859153 0.767961 0.864000 +2 0.768888 0.775161 0.904000 +3 0.658956 0.685653 0.902000 +4 0.589073 0.618438 0.923000 +5 0.540008 0.622157 0.915000 +6 0.508080 0.606979 0.914000 +7 0.487228 0.599491 0.918000 +8 0.477516 0.602196 0.923000 +Total time: 02:18 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl.m +Loss and accuracy using (cls_best): [0.2829206, tensor(0.9205)] +OrderedDict([('data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl.m', + 0.9204999804496765)]) +data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl.m: 0.9204999804496765 +``` +Exec 2 +````python -m ulmfit eval --glob="mldoc/zh-1/models/sp15k/qrnn_nl4.m" --name nl4-1cyc-sl1 --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.1 +Processing data/mldoc/zh-1/models/sp15k/qrnn_nl4.m +zh-1 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl1.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl1.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.881513 0.712176 0.865000 +2 0.743687 0.665091 0.906000 +3 0.677436 0.687689 0.873000 +4 0.595139 0.626483 0.920000 +5 0.542732 0.600652 0.914000 +6 0.512080 0.597546 0.916000 +7 0.487021 0.597065 0.912000 +8 0.476598 0.596792 0.914000 +Total time: 02:20 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl1.m +Loss and accuracy using (cls_best): [0.29172945, tensor(0.9178)] +OrderedDict([('data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl1.m', + 0.9177500009536743)]) +data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl1.m: 0.9177500009536743 +```` +Exec 4 +```bash +python -m ulmfit eval --glob="mldoc/zh-1/models/sp15k/qrnn_nl4.m" --name nl4-1cyc-sl-e4 --num-cls-epochs=4 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.1 ✘ 130 +Processing data/mldoc/zh-1/models/sp15k/qrnn_nl4.m +zh-1 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.880415 0.677291 0.901000 +2 0.729670 0.659975 0.911000 +3 0.624817 0.603056 0.921000 +4 0.542027 0.601961 0.921000 +Total time: 01:08 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m +Loss and accuracy using (cls_best): [0.28558904, tensor(0.9222)] +OrderedDict([('data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m', + 0.922249972820282)]) +data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m: 0.922249972820282 +``` +## LSTM sp30k +### 0.1 +```bash + python -m ulmfit eval --glob="mldoc/zh-1/models/sp30k/lstm_nl4.m" --name nl4-1cyc-sl --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.1 +Processing data/mldoc/zh-1/models/sp30k/lstm_nl4.m +zh-1 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.870432 0.670671 0.882000 +2 0.754248 0.824157 0.895000 +3 0.654601 0.727428 0.885000 +4 0.602772 0.668668 0.901000 +5 0.542110 0.625137 0.903000 +6 0.506150 0.617842 0.913000 +7 0.480944 0.616885 0.912000 +8 0.472876 0.614381 0.911000 +Total time: 06:38 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl.m +Loss and accuracy using (cls_best): [0.2977172, tensor(0.9233)] +OrderedDict([('data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl.m', + 0.9232500195503235)]) +data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl.m: 0.9232500195503235 +``` +### 0.2 +```bash +python -m ulmfit eval --glob="mldoc/zh-1/models/sp30k/lstm_nl4.m" --name nl4-1cyc-sl2 --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.2 +Processing data/mldoc/zh-1/models/sp30k/lstm_nl4.m +zh-1 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl2.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl2.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.045619 0.908213 0.874000 +2 0.957379 0.857977 0.921000 +3 0.891791 0.852157 0.905000 +4 0.845289 0.849923 0.914000 +5 0.818228 0.848613 0.921000 +6 0.787021 0.840483 0.920000 +7 0.776123 0.844006 0.919000 +8 0.762384 0.857240 0.916000 +Total time: 06:33 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl2.m +Loss and accuracy using (cls_best): [0.40299156, tensor(0.9170)] +OrderedDict([('data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl2.m', + 0.9169999957084656)]) +data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl2.m: 0.9169999957084656 +``` +### 0.4 +```bash + python -m ulmfit eval --glob="mldoc/zh-1/models/sp30k/lstm_nl4.m" --name nl4-1cyc-sl4 --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.4 +Processing data/mldoc/zh-1/models/sp30k/lstm_nl4.m +zh-1 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl4.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.251581 1.183341 0.898000 +2 1.214358 1.201266 0.834000 +3 1.190343 1.165525 0.919000 +4 1.168018 1.172510 0.903000 +5 1.149965 1.161660 0.914000 +6 1.140140 1.161689 0.915000 +7 1.135877 1.159853 0.912000 +8 1.134425 1.160039 0.911000 +Total time: 06:34 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl4.m +Loss and accuracy using (cls_best): [0.64041936, tensor(0.9195)] +OrderedDict([('data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl4.m', + 0.9194999933242798)]) +data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl4.m: 0.9194999933242798 +``` \ No newline at end of file diff --git a/results/logs/qrnn-ru.md b/results/logs/qrnn-ru.md index 36ae8d4..a213dd6 100644 --- a/results/logs/qrnn-ru.md +++ b/results/logs/qrnn-ru.md @@ -1,4 +1,16 @@ # QRNN RU +## SP15k nl8 +data/wiki/ru-100/models/sp15k/qrnn_nl8.m + +export CUDA_VISIBLE_DEVICES=0 +LANG=ru +python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/sp15k/qrnn_nl8.m --lang=${LANG} --name 'nl8' - train 20 --bs 20 --num-cls-epochs=8 --lr_sched=1cycle + +export CUDA_VISIBLE_DEVICES=0 +LANG=de +python -m ulmfit cls --dataset-path data/mldoc-m/${LANG}-1 --base-lm-path data/wiki-m/${LANG}-100/models/sp15k/lstm_nl4.m --lang=${LANG} --name 'nl4' - train 20 --bs 20 --num-cls-epochs=8 --lr_sched=1cycle + + ## SP30k nl4 ### LM ``` diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index 73ded8a..127c8fa 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -39,7 +39,8 @@ def get_test_data(): copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.valid.tokens', n=600*sz) copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.test.tokens', n=600*sz) copy_head(imdb / 'train.csv', test_imdb / 'train.csv', n=10*sz) - copy_head(imdb / 'train.csv', test_imdb / 'test.csv', n=6*sz) + copy_head(imdb / 'train.csv', test_imdb / 'test.csv', n=6 * sz) + copy_head(imdb / 'train.csv', test_imdb / 'dev.csv', n=6 * sz) copy_head(imdb / 'train.csv', test_imdb / 'unsup.csv', n=1*sz) return test_data, test_wt @@ -109,6 +110,7 @@ def test_ulmfit_fastai_end_to_end(): qrnn=False, tokenizer='f', max_vocab=100, + nl=1, name=lm_name, ) exp.train_lm(num_epochs=1, bs=2) diff --git a/ulmfit/__main__.py b/ulmfit/__main__.py index 13a3922..ffeabc1 100644 --- a/ulmfit/__main__.py +++ b/ulmfit/__main__.py @@ -90,7 +90,7 @@ class ULMFiT: try: params = CLSHyperParams.from_lm(dataset_path, base_model, lang=lang, name=name, cuda_id=cuda_id) key = str(params.model_dir.relative_to(Path.cwd())) - if (params.model_dir/"cls_last.pth").exists(): + if (params.model_dir/"cls_best.pth").exists(): print("Evaluating previously trained model") results[key] = params.validate_cls()[1] else: diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 5b70ef6..0dfecaa 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -231,7 +231,7 @@ class LMHyperParams: # partial(SaveModelCallback, every='improvement', name='lm') disabled due to Memory issues ] if label_smoothing_eps > 0.0: - learn.loss_func = LabelSmoothingCrossEntropy(eps=label_smoothing_eps) + learn.loss_func = FlattenedLoss(LabelSmoothingCrossEntropy, eps=label_smoothing_eps) return learn def load_train_text(self): diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 278a629..cda378d 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -77,7 +77,7 @@ class CLSHyperParams(LMHyperParams): if self.need_fine_tune_lm: self.train_lm(num_lm_epochs, data_lm=data_lm, drop_mult=drop_mul_lm, label_smoothing_eps=label_smoothing_eps) learn = self.create_cls_learner(data_clas, drop_mult=drop_mul_cls, max_len=cls_max_len, label_smoothing_eps=label_smoothing_eps) try: - learn.load('cls_last') + learn.load('cls_best') print("Loading last classifier") except FileNotFoundError: learn.load_encoder(ENC_BEST) @@ -94,7 +94,7 @@ class CLSHyperParams(LMHyperParams): del learn return self.validate_cls('cls_best', bs=bs, data_tst=data_tst, learn=None) - def validate_cls(self, save_name='cls_last', bs=40, data_tst=None, learn=None): + def validate_cls(self, save_name='cls_best', bs=40, data_tst=None, learn=None): if data_tst is None: _, _, data_tst = self.load_cls_data(bs) if learn is None: @@ -125,7 +125,7 @@ class CLSHyperParams(LMHyperParams): #partial(SaveModelCallback, every='improvement', name='cls_best') disabled due to memory issues ] if label_smoothing_eps > 0.0: - learn.loss_func = LabelSmoothingCrossEntropy(eps=label_smoothing_eps) + learn.loss_func = FlattenedLoss(LabelSmoothingCrossEntropy, eps=label_smoothing_eps) return learn def load_cls_data(self, bs, **kwargs):