diff --git a/results/logs/Timing.md b/results/logs/Timing.md new file mode 100644 index 0000000..3b13fd2 --- /dev/null +++ b/results/logs/Timing.md @@ -0,0 +1,24 @@ + +## QRNN sp15k timing +``` +time python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4-2' --max-vocab 15000 --lang ${LANG} --qrnn=True - train 1 --bs=50 --drop_mult=0 --label-smoothing-eps=0.1 +Max vocab: 15000 +Cache dir: data/wiki/ru-100/models/sp15k +Model dir: data/wiki/ru-100/models/sp15k/qrnn_nl4-2.m +Wiki text was split to 193047 articles +Wiki text was split to 460 articles +Data lm, trn: 193047, val: 460 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', '▁на', 'х'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Training lm from random weights +epoch train_loss valid_loss accuracy +1 3.943105 3.860063 0.477620 +Total time: 1:05:03 +data/wiki/ru-100/models/sp15k +Saving info data/wiki/ru-100/models/sp15k/qrnn_nl4-2.m/info.json + +real 65m30,341s +user 48m49,047s +sys 16m40,688s +``` \ No newline at end of file diff --git a/results/logs/de.md b/results/logs/de.md index 8590f5c..5142dbb 100644 --- a/results/logs/de.md +++ b/results/logs/de.md @@ -169,10 +169,10 @@ epoch train_loss valid_loss accuracy 1 0.175013 0.280545 0.921000 2 0.178333 0.286059 0.923000 Saving models at data/mldoc/de-1/models/vf60k/lstm_nl3.m -Loss and accuracy using (cls_last): [0.28054512, tensor(0.9210)] -Loss and accuracy using (cls_best): [0.28054512, tensor(0.9210)] +Loss and accuracy using (cls_best): [0.16954255, tensor(0.9475)] +OrderedDict([('data/mldoc/de-1/models/vf60k/lstm_nl3.m', 0.9474999904632568)]) ``` -MultiCCA: 93.7% , ulmfit: 92.1% +MultiCCA: 93.7% , ulmfit: 94.74% ## SP30k LSTM nl 4 ### LM ``` diff --git a/results/logs/mldoc/noise/es-random.md b/results/logs/mldoc/noise/es-random.md new file mode 100644 index 0000000..8d79492 --- /dev/null +++ b/results/logs/mldoc/noise/es-random.md @@ -0,0 +1,1085 @@ +# Noise resistance on ES Random +## 1k + + + +## 10k +``` + python -m ulmfit eval_noise_resistance --lang=es --size=10 --prefix-name="rnd2_" --model="sp15k/qrnn_rnd-nl4.m" --label-smoothing-eps=0.1 --random-init=True +Noise: 0 +Processing data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_0.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Data lm, trn: 13013, val: 1445 +Data cls, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: + + !! WARNING !! + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +Your compiler (c++) may be ABI-incompatible with PyTorch! +Please use a compiler that is ABI-compatible with GCC 4.9 and above. +See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. + +See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 +for instructions on how to install GCC 4.9 or higher. +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + !! WARNING !! + + warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 0.674091 1.283792 0.574000 +2 0.624245 11.202841 0.519000 +3 0.634349 3.687749 0.516000 +4 0.596913 4.074763 0.608000 +5 0.594516 1.093833 0.715000 +6 0.557703 1.010482 0.694000 +7 0.543755 0.995253 0.702000 +8 0.537600 0.952028 0.754000 +Total time: 11:03 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_0.m +Loss and accuracy using (cls_best): [0.9988524, tensor(0.7690)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_0.m', + 0.7689999938011169)]) + python -m ulmfit eval_noise_resistance --lang=es --size=10 --prefix-name="rnd2_" --model="sp15k/qrnn_rnd-nl4.m" --label-smoothing-eps=0.1 --random-init=True +Noise: 0 +Processing data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_0.m +Evaluating previously trained model +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Data lm, trn: 13013, val: 1445 +Data cls, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: + + !! WARNING !! + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +Your compiler (c++) may be ABI-incompatible with PyTorch! +Please use a compiler that is ABI-compatible with GCC 4.9 and above. +See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. + +See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 +for instructions on how to install GCC 4.9 or higher. +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + !! WARNING !! + + warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) +Loss and accuracy using (cls_best): [0.99926454, tensor(0.7695)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_0.m', + 0.7695000171661377)]) +Noise: 5 +Processing data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_5.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 472 examples, only 0.9500951575385916 have correct labels +Added noise to 50 examples, only 0.95 have correct labels +Data lm, trn: 13013, val: 1445 +Data clsnoise0.05tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 0.743423 1.354918 0.428000 +2 0.729408 1.153078 0.543000 +3 0.736531 1.465756 0.450000 +4 0.749902 4.479099 0.541000 +5 0.691128 1.322383 0.466000 +6 0.694896 1.274096 0.597000 +7 0.664847 1.020145 0.673000 +8 0.645594 1.027490 0.668000 +Total time: 10:56 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_5.m +Loss and accuracy using (cls_best): [0.68077123, tensor(0.7272)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_5.m', + 0.7272499799728394)]) +Noise: 10 +Processing data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_10.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 945 examples, only 0.9000845844787482 have correct labels +Added noise to 100 examples, only 0.9 have correct labels +Data lm, trn: 13013, val: 1445 +Data clsnoise0.1tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 0.834923 1.409279 0.485000 +2 0.854848 1.614973 0.320000 +3 0.828790 1.701499 0.396000 +4 0.803852 2.424411 0.547000 +5 0.801694 2.024830 0.562000 +6 0.786992 2.021667 0.609000 +7 0.765137 2.020704 0.652000 +8 0.746047 5.385053 0.652000 +Total time: 10:56 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_10.m +Loss and accuracy using (cls_best): [0.80370593, tensor(0.7172)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_10.m', + 0.7172499895095825)]) +Noise: 15 +Processing data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_15.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 1418 examples, only 0.8500740114189046 have correct labels +Added noise to 150 examples, only 0.85 have correct labels +Data lm, trn: 13013, val: 1445 +Data clsnoise0.15tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 0.885186 1.486315 0.448000 +2 0.922848 1.944151 0.485000 +3 0.919082 1.599238 0.400000 +4 0.895376 1.527287 0.451000 +5 0.881112 3.831651 0.478000 +6 0.919497 1.247746 0.508000 +7 0.829800 1.175510 0.508000 +8 0.861669 1.174659 0.504000 +Total time: 10:35 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_15.m +Loss and accuracy using (cls_best): [0.9063169, tensor(0.5955)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_15.m', + 0.5954999923706055)]) +Noise: 20 +Processing data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_20.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 1891 examples, only 0.8000634383590611 have correct labels +Added noise to 200 examples, only 0.8 have correct labels +Data lm, trn: 13013, val: 1445 +Data clsnoise0.2tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 1.000344 1.388730 0.335000 +2 1.007181 12.045410 0.472000 +3 0.983002 1.479323 0.371000 +4 0.958265 1.298010 0.400000 +5 0.939612 4.368020 0.414000 +6 0.928165 3.608808 0.492000 +7 0.926556 2.253188 0.471000 +8 0.905125 2.181204 0.499000 +Total time: 11:06 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_20.m +Loss and accuracy using (cls_best): [0.9292287, tensor(0.5985)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_20.m', + 0.5985000133514404)]) +Noise: 25 +Processing data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_25.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 2364 examples, only 0.7500528652992176 have correct labels +Added noise to 250 examples, only 0.75 have correct labels +Data lm, trn: 13013, val: 1445 +Data clsnoise0.25tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 1.057209 1.246537 0.466000 +2 1.038750 34.810867 0.114000 +3 1.030894 87.796410 0.376000 +4 1.041276 1.264513 0.462000 +5 1.029240 2.345500 0.422000 +6 1.000014 1.293007 0.464000 +7 0.960223 1.266377 0.466000 +8 0.942258 1.348627 0.447000 +Total time: 10:44 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_25.m +Loss and accuracy using (cls_best): [1.0587388, tensor(0.5707)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_25.m', + 0.5707499980926514)]) +Noise: 30 +Processing data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_30.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 2837 examples, only 0.7000422922393741 have correct labels +Added noise to 300 examples, only 0.7 have correct labels +Data lm, trn: 13013, val: 1445 +Data clsnoise0.3tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 1.087622 1.363222 0.435000 +2 1.095437 4.608276 0.221000 +3 1.094145 1.402207 0.371000 +4 1.078363 2.236299 0.388000 +5 1.056162 1.310936 0.431000 +python -m ulmfit eval_noise_resistance --lang=es --size=10 --prefix-name="rnd2_" --model="sp15k/qrnn_rnd-nl4.m" --label-smoo6 1.050225 1.267474 0.425000 +7 1.038286 1.299926 0.421000 +8 1.044985 1.300490 0.422000 +Total time: 10:41 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_30.m +Loss and accuracy using (cls_best): [0.9783086, tensor(0.5817)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_30.m', + 0.5817499756813049)]) +Noise: 35 +Processing data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_35.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 3310 examples, only 0.6500317191795305 have correct labels +Added noise to 350 examples, only 0.65 have correct labels +Data lm, trn: 13013, val: 1445 +Data clsnoise0.35tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 1.153989 1.421947 0.240000 +2 1.138209 1.703624 0.246000 +3 1.124318 112.371529 0.280000 +4 1.133281 1.417321 0.317000 +5 1.138515 1.317285 0.387000 +6 1.101543 1.366418 0.390000 +7 1.085555 1.617780 0.399000 +8 1.092133 1.394255 0.394000 +Total time: 11:06 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_35.m +Loss and accuracy using (cls_best): [1.1545019, tensor(0.5490)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_35.m', + 0.5490000247955322)]) +Noise: 40 +Processing data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_40.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 3783 examples, only 0.600021146119687 have correct labels +Added noise to 400 examples, only 0.6 have correct labels +Data lm, trn: 13013, val: 1445 +Data clsnoise0.4tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 1.199116 1.436976 0.385000 +2 1.182144 3.020936 0.350000 +3 1.169826 1.429357 0.359000 +4 1.184806 2.984490 0.362000 +5 1.142364 2.399835 0.322000 +6 1.136863 25.139421 0.359000 +7 1.139416 12.117358 0.365000 +8 1.132085 2.642181 0.368000 +Total time: 10:59 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_40.m +Loss and accuracy using (cls_best): [2.2691653, tensor(0.5375)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_40.m', + 0.5375000238418579)]) +Noise: 45 +Processing data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_45.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 4256 examples, only 0.5500105730598435 have correct labels +Added noise to 450 examples, only 0.55 have correct labels +Data lm, trn: 13013, val: 1445 +Data clsnoise0.45tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 1.211562 1.438882 0.337000 +2 1.218273 222.648880 0.291000 +3 1.203696 1.414955 0.343000 +4 1.204386 1.364311 0.369000 +5 1.203539 3.488942 0.356000 +6 1.175719 6.136192 0.349000 +7 1.145031 1.604449 0.365000 +8 1.141291 2.528898 0.361000 +Total time: 10:55 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_45.m +Loss and accuracy using (cls_best): [1.6499722, tensor(0.5272)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_45.m', + 0.5272499918937683)]) +Noise: 50 +Processing data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_50.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 4729 examples, only 0.5 have correct labels +Added noise to 500 examples, only 0.5 have correct labels +Data lm, trn: 13013, val: 1445 +Data clsnoise0.5tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 1.232532 1.421933 0.272000 +2 1.237531 1.822951 0.252000 +3 1.254736 1.534031 0.266000 +4 1.252628 1.536729 0.266000 +5 1.234980 1.544544 0.266000 +6 1.234228 1.558047 0.266000 +7 1.240368 1.538028 0.266000 +8 1.253012 1.540725 0.266000 +Total time: 11:01 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_50.m +Loss and accuracy using (cls_best): [1.4770876, tensor(0.3072)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_50.m', + 0.3072499930858612)]) +Noise: 55 +Processing data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_55.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 5201 examples, only 0.4500951575385917 have correct labels +Added noise to 550 examples, only 0.45 have correct labels +Data lm, trn: 13013, val: 1445 +Data clsnoise0.55tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 1.267338 1.647622 0.253000 +2 1.267763 1.562070 0.288000 +3 1.249864 2.255809 0.258000 +4 1.246395 1.470975 0.307000 +5 1.229589 1.466512 0.305000 +6 1.221072 1.472953 0.307000 +7 1.202461 1.430212 0.323000 +8 1.202863 1.416951 0.320000 +Total time: 10:59 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_55.m +Loss and accuracy using (cls_best): [1.1913521, tensor(0.5217)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_55.m', + 0.5217499732971191)]) +Noise: 60 +Processing data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_60.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 5674 examples, only 0.4000845844787482 have correct labels +Added noise to 600 examples, only 0.4 have correct labels +Data lm, trn: 13013, val: 1445 +Data clsnoise0.6tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 1.277718 1.426537 0.253000 +2 1.261508 1.677472 0.229000 +3 1.263114 1.585918 0.276000 +4 1.255874 2.783197 0.293000 +5 1.250460 3.303977 0.353000 +6 1.239010 10.652842 0.326000 +7 1.220644 4.388613 0.317000 +8 1.226328 6.756069 0.327000 +Total time: 11:07 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_60.m +Loss and accuracy using (cls_best): [2.6658566, tensor(0.5073)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_60.m', + 0.5072500109672546)]) +Noise: 65 +Processing data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_65.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 6147 examples, only 0.35007401141890465 have correct labels +Added noise to 650 examples, only 0.35 have correct labels +Data lm, trn: 13013, val: 1445 +Data clsnoise0.65tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 1.263687 1.667461 0.279000 +2 1.287779 8.032739 0.227000 +3 1.289650 1.522185 0.229000 +4 1.287830 1.587737 0.272000 +5 1.267796 1.598588 0.272000 +6 1.276992 1.563360 0.272000 +7 1.276122 1.564773 0.272000 +8 1.268917 1.559838 0.229000 +Total time: 11:10 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_65.m +Loss and accuracy using (cls_best): [1.4682757, tensor(0.3072)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_65.m', + 0.3072499930858612)]) +Noise: 70 +Processing data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_70.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 6620 examples, only 0.30006343835906113 have correct labels +Added noise to 700 examples, only 0.3 have correct labels +Data lm, trn: 13013, val: 1445 +Data clsnoise0.7tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 1.278968 1.395868 0.308000 +2 1.287285 2.381748 0.271000 +3 1.281547 1.510829 0.273000 +4 1.285452 1.517207 0.273000 +5 1.280697 1.535823 0.273000 +6 1.289347 1.505417 0.273000 +7 1.278146 1.527413 0.273000 +8 1.271342 1.522274 0.273000 +Total time: 10:48 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_70.m +Loss and accuracy using (cls_best): [1.4725544, tensor(0.3115)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_70.m', + 0.31150001287460327)]) +Noise: 75 +Processing data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_75.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 7093 examples, only 0.2500528652992176 have correct labels +Added noise to 750 examples, only 0.25 have correct labels +Data lm, trn: 13013, val: 1445 +Data clsnoise0.75tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 1.278254 1.406109 0.294000 +2 1.284943 1.538882 0.268000 +3 1.277497 3.605533 0.256000 +4 1.260796 1.501276 0.259000 +5 1.254600 2.722533 0.273000 +6 1.250827 4.186543 0.269000 +7 1.236754 3.883370 0.273000 +8 1.239405 1.523115 0.291000 +Total time: 11:17 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_75.m +Loss and accuracy using (cls_best): [1.7807395, tensor(0.1478)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_75.m', + 0.14775000512599945)]) + noise accuracy +0 0.00 0.76950 +1 0.05 0.72725 +2 0.10 0.71725 +3 0.15 0.59550 +4 0.20 0.59850 +5 0.25 0.57075 +6 0.30 0.58175 +7 0.35 0.54900 +8 0.40 0.53750 +9 0.45 0.52725 +10 0.50 0.30725 +11 0.55 0.52175 +12 0.60 0.50725 +13 0.65 0.30725 +14 0.70 0.31150 +15 0.75 0.14775 +``` + +# LSTM +``` + python -m ulmfit eval_noise_resistance --lang=es --size=10 --prefix-name="rnd2_" --model="sp30k/lstm_nl4.m" --label-smoothing-eps=0.1 --random-init=True +Noise: 0 +Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m +es-10 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_0.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Data lm, trn: 13013, val: 1445 +Data cls, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 0.659161 4.290442 0.318000 +2 0.750901 2.120531 0.277000 +3 0.780925 2.740564 0.277000 +4 0.761562 4.889140 0.293000 +5 0.743035 14.893404 0.279000 +6 0.726612 3.620207 0.277000 +7 0.760487 2.801041 0.293000 +8 0.751857 2.819516 0.293000 +Total time: 30:06 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_0.m +Loss and accuracy using (cls_best): [2.932457, tensor(0.3072)] +OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_0.m', + 0.3072499930858612)]) +Noise: 5 +Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m +es-10 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_5.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 472 examples, only 0.9500951575385916 have correct labels +Added noise to 50 examples, only 0.95 have correct labels +Data lm, trn: 13013, val: 1445 +Data clsnoise0.05tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 0.744206 2.318675 0.274000 +2 0.845912 1.891602 0.276000 +3 0.831846 3.916036 0.274000 +4 0.831695 1.779413 0.274000 +5 0.830590 1.956843 0.274000 +6 0.831666 1.893295 0.274000 +7 0.850189 1.924907 0.274000 +8 0.829423 1.925661 0.274000 +Total time: 30:02 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_5.m +Loss and accuracy using (cls_best): [1.8306484, tensor(0.3072)] +OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_5.m', + 0.3072499930858612)]) +Noise: 10 +Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m +es-10 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_10.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 945 examples, only 0.9000845844787482 have correct labels +Added noise to 100 examples, only 0.9 have correct labels +Data lm, trn: 13013, val: 1445 +Data clsnoise0.1tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 0.898602 1.545597 0.293000 +2 0.903453 176.651031 0.209000 +3 0.916878 13.823226 0.276000 +4 0.898918 24.522129 0.273000 +5 0.912165 1.825259 0.274000 +6 0.931385 1.808111 0.274000 +7 0.889932 2.430218 0.273000 +8 0.927120 2.782772 0.276000 +Total time: 31:02 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_10.m +Loss and accuracy using (cls_best): [1.7723552, tensor(0.3105)] +OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_10.m', + 0.31049999594688416)]) +Noise: 15 +Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m +es-10 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_15.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 1418 examples, only 0.8500740114189046 have correct labels +Added noise to 150 examples, only 0.85 have correct labels +Data lm, trn: 13013, val: 1445 +Data clsnoise0.15tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 0.973683 6.861582 0.322000 +2 0.966286 2.543203 0.273000 +3 1.016659 3.716355 0.205000 +4 0.955702 1.692103 0.273000 +5 0.967680 3.590133 0.280000 +6 0.982580 8.608456 0.269000 +7 0.940222 2.429212 0.273000 +8 0.996999 2.896548 0.269000 +Total time: 30:04 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_15.m +Loss and accuracy using (cls_best): [1.9653525, tensor(0.3075)] +OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_15.m', + 0.3075000047683716)]) +Noise: 20 +Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m +es-10 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_20.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 1891 examples, only 0.8000634383590611 have correct labels +Added noise to 200 examples, only 0.8 have correct labels +Data lm, trn: 13013, val: 1445 +Data clsnoise0.2tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 1.031621 2.023383 0.255000 +2 1.055999 1.585043 0.255000 +3 1.004598 1.747062 0.255000 +4 1.036506 12.106596 0.257000 +5 1.037610 7.782064 0.257000 +6 1.010497 4.877003 0.255000 +7 1.035525 9.258689 0.257000 +8 1.032628 9.379140 0.257000 +Total time: 30:00 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_20.m +Loss and accuracy using (cls_best): [2.198144, tensor(0.3105)] +OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_20.m', + 0.31049999594688416)]) +Noise: 25 +Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m +es-10 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_25.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 2364 examples, only 0.7500528652992176 have correct labels +Added noise to 250 examples, only 0.75 have correct labels +Data lm, trn: 13013, val: 1445 +Data clsnoise0.25tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 1.062350 1.959029 0.214000 +2 1.061749 1.823441 0.262000 +3 1.083258 12.511975 0.275000 +4 1.089900 2.291190 0.275000 +5 1.086922 8.510952 0.262000 +6 1.090885 2.293679 0.261000 +7 1.075075 2.249190 0.275000 +8 1.068257 2.230014 0.275000 +Total time: 30:26 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_25.m +Loss and accuracy using (cls_best): [2.080095, tensor(0.3072)] +OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_25.m', + 0.3072499930858612)]) +Noise: 30 +Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m +es-10 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_30.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 2837 examples, only 0.7000422922393741 have correct labels +Added noise to 300 examples, only 0.7 have correct labels +Data lm, trn: 13013, val: 1445 +Data clsnoise0.3tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 1.133636 1.806463 0.274000 +2 1.110103 10.866484 0.265000 +3 1.122795 6.634718 0.274000 +4 1.112287 5.359313 0.262000 +5 1.121410 174.423630 0.261000 +6 1.110173 18.667475 0.262000 +7 1.102498 12.188397 0.262000 +8 1.117406 15.158224 0.249000 +Total time: 29:59 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_30.m +Loss and accuracy using (cls_best): [8.974576, tensor(0.2555)] +OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_30.m', + 0.2554999887943268)]) +Noise: 35 +Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m +es-10 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_35.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 3310 examples, only 0.6500317191795305 have correct labels +Added noise to 350 examples, only 0.65 have correct labels +Data lm, trn: 13013, val: 1445 +Data clsnoise0.35tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 1.177323 2.044810 0.260000 +2 1.145307 1.615969 0.254000 +3 1.178776 2.619027 0.260000 +4 1.170236 7.795065 0.253000 +5 1.160120 1.604842 0.254000 +6 1.160545 1.587269 0.254000 +7 1.137365 1.629242 0.254000 +8 1.150483 1.610021 0.254000 +Total time: 29:59 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_35.m +Loss and accuracy using (cls_best): [1.5346162, tensor(0.3072)] +OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_35.m', + 0.3072499930858612)]) +Noise: 40 +Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m +es-10 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_40.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 3783 examples, only 0.600021146119687 have correct labels +Added noise to 400 examples, only 0.6 have correct labels +Data lm, trn: 13013, val: 1445 +Data clsnoise0.4tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 1.200657 1.747926 0.242000 +2 1.213164 33.535637 0.250000 +3 1.188038 1.609861 0.246000 +4 1.185921 2.510113 0.246000 +5 1.218471 4.206014 0.242000 +6 1.187989 1.855121 0.251000 +7 1.210737 4.329615 0.242000 +8 1.193102 5.145269 0.242000 +Total time: 30:09 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_40.m +Loss and accuracy using (cls_best): [2.3451035, tensor(0.3075)] +OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_40.m', + 0.3075000047683716)]) +Noise: 45 +Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m +es-10 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_45.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 4256 examples, only 0.5500105730598435 have correct labels +Added noise to 450 examples, only 0.55 have correct labels +Data lm, trn: 13013, val: 1445 +Data clsnoise0.45tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 1.241620 1.534527 0.278000 +2 1.234608 4.180245 0.251000 +3 1.225962 18.634438 0.251000 +4 1.231608 1.596539 0.245000 +5 1.242316 1.529011 0.245000 +6 1.224234 1.559436 0.245000 +7 1.219737 1.570357 0.245000 +8 1.217654 1.558945 0.245000 +Total time: 29:55 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_45.m +Loss and accuracy using (cls_best): [1.4820943, tensor(0.3072)] +OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_45.m', + 0.3072499930858612)]) +Noise: 50 +Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m +es-10 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_50.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 4729 examples, only 0.5 have correct labels +Added noise to 500 examples, only 0.5 have correct labels +Data lm, trn: 13013, val: 1445 +Data clsnoise0.5tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 1.248102 1.559488 0.241000 +2 1.239413 6.912020 0.242000 +3 1.258250 19.964916 0.240000 +4 1.244972 14.503420 0.240000 +5 1.231926 7.417413 0.237000 +6 1.240820 3.726920 0.240000 +7 1.250362 3.813594 0.240000 +8 1.246718 4.295069 0.240000 +Total time: 29:56 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_50.m +Loss and accuracy using (cls_best): [1.6464796, tensor(0.3105)] +OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_50.m', + 0.31049999594688416)]) +Noise: 55 +Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m +es-10 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_55.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 5201 examples, only 0.4500951575385917 have correct labels +Added noise to 550 examples, only 0.45 have correct labels +Data lm, trn: 13013, val: 1445 +Data clsnoise0.55tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 1.262681 1.390237 0.243000 +2 1.261966 1.613178 0.221000 +3 1.281932 80.386887 0.237000 +4 1.266172 20.980446 0.220000 +5 1.267419 1.564067 0.221000 +6 1.256289 2.409412 0.215000 +7 1.259451 2.007908 0.221000 +8 1.248848 1.682695 0.221000 +Total time: 30:44 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_55.m +Loss and accuracy using (cls_best): [1.5071006, tensor(0.3072)] +OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_55.m', + 0.3072499930858612)]) +Noise: 60 +Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m +es-10 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_60.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 5674 examples, only 0.4000845844787482 have correct labels +Added noise to 600 examples, only 0.4 have correct labels +Data lm, trn: 13013, val: 1445 +Data clsnoise0.6tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 1.299198 1.514330 0.257000 +2 1.276767 1.648853 0.246000 +3 1.273444 1.702971 0.249000 +4 1.275886 5.679698 0.254000 +5 1.282117 15.944865 0.249000 +6 1.273512 4.761624 0.254000 +7 1.283532 5.982591 0.254000 +8 1.266130 4.487469 0.254000 +Total time: 29:48 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_60.m +Loss and accuracy using (cls_best): [2.5968323, tensor(0.3072)] +OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_60.m', + 0.3072499930858612)]) +Noise: 65 +Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m +es-10 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_65.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 6147 examples, only 0.35007401141890465 have correct labels +Added noise to 650 examples, only 0.35 have correct labels +Data lm, trn: 13013, val: 1445 +Data clsnoise0.65tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 1.288758 1.800278 0.266000 +2 1.286780 1.503401 0.272000 +3 1.280267 1.921974 0.278000 +4 1.285067 1.523227 0.259000 +5 1.285825 1.526260 0.272000 +6 1.276409 1.531018 0.267000 +7 1.270072 1.525349 0.259000 +8 1.269815 1.514847 0.242000 +Total time: 29:50 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_65.m +Loss and accuracy using (cls_best): [1.4596524, tensor(0.3072)] +OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_65.m', + 0.3072499930858612)]) +Noise: 70 +Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m +es-10 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_70.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 6620 examples, only 0.30006343835906113 have correct labels +Added noise to 700 examples, only 0.3 have correct labels +Data lm, trn: 13013, val: 1445 +Data clsnoise0.7tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 1.271206 1.656565 0.258000 +2 1.285489 2.193548 0.271000 +3 1.283617 1.947972 0.264000 +4 1.282205 8.228713 0.270000 +5 1.275360 21.401678 0.270000 +6 1.266841 32.251156 0.270000 +7 1.263846 20.252125 0.258000 +8 1.270404 21.024738 0.258000 +Total time: 29:24 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_70.m +Loss and accuracy using (cls_best): [7.6955824, tensor(0.1828)] +OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_70.m', + 0.18275000154972076)]) +Noise: 75 +Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m +es-10 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_75.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 7093 examples, only 0.2500528652992176 have correct labels +Added noise to 750 examples, only 0.25 have correct labels +Data lm, trn: 13013, val: 1445 +Data clsnoise0.75tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 1.282777 1.870099 0.271000 +2 1.271396 3.192471 0.264000 +3 1.282228 11.233303 0.260000 +4 1.276331 35.493816 0.256000 +5 1.269741 3.304629 0.264000 +6 1.266907 6.163653 0.261000 +7 1.279470 3.861671 0.271000 +8 1.269125 2.668693 0.260000 +Total time: 30:42 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_75.m +Loss and accuracy using (cls_best): [1.6692431, tensor(0.1828)] +OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_75.m', + 0.18275000154972076)]) + noise accuracy +0 0.00 0.30725 +1 0.05 0.30725 +2 0.10 0.31050 +3 0.15 0.30750 +4 0.20 0.31050 +5 0.25 0.30725 +6 0.30 0.25550 +7 0.35 0.30725 +8 0.40 0.30750 +9 0.45 0.30725 +10 0.50 0.31050 +11 0.55 0.30725 +12 0.60 0.30725 +13 0.65 0.30725 +14 0.70 0.18275 +15 0.75 0.18275 +``` \ No newline at end of file diff --git a/results/logs/mldoc/noise/random.md b/results/logs/mldoc/noise/random.md new file mode 100644 index 0000000..6b63d78 --- /dev/null +++ b/results/logs/mldoc/noise/random.md @@ -0,0 +1,627 @@ +## Debugging random init +```` + warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) +Loading pretrained model +Unknown tokens 0, first 100: [] +Bptt 70 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_rnd2_0.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +```` + + + + + +## first attempt at random init +``` + python -m ulmfit eval_noise_resistance --lang=de --size=10 --prefix-name="_rnd_" --model="sp15k/qrnn_rnd-nl4.m" --label-smoothing-eps=0.1 +Noise: 0 +Processing data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_0.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: + + !! WARNING !! + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +Your compiler (c++) may be ABI-incompatible with PyTorch! +Please use a compiler that is ABI-compatible with GCC 4.9 and above. +See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. + +See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 +for instructions on how to install GCC 4.9 or higher. +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + !! WARNING !! + + warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) +Loading pretrained model +Unknown tokens 0, first 100: [] +Bptt 70 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_0.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.787174 0.985413 0.701000 +2 0.679534 0.697764 0.875000 +3 0.630987 7.125103 0.873000 +4 0.588259 0.653497 0.915000 +5 0.568135 0.641379 0.942000 +6 0.529713 0.557198 0.948000 +7 0.500168 0.538946 0.958000 +8 0.505462 0.550917 0.954000 +Total time: 19:37 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_0.m +Loss and accuracy using (cls_best): [0.21539633, tensor(0.9613)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_0.m', + 0.9612500071525574)]) +Noise: 5 +Processing data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_5.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 500 examples, only 0.95 have correct labels +Added noise to 50 examples, only 0.95 have correct labels +Data lm, trn: 13500, val: 1500 +Data clsnoise0.05tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +Bptt 70 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_5.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.917813 0.874308 0.790000 +2 0.821460 1.239760 0.671000 +3 0.747909 2.624352 0.667000 +4 0.713649 0.735327 0.893000 +5 0.689947 1.175884 0.844000 +6 0.639073 1.066042 0.862000 +7 0.617660 0.845634 0.875000 +8 0.620402 0.670972 0.901000 +Total time: 19:28 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_5.m +Loss and accuracy using (cls_best): [0.25263783, tensor(0.9560)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_5.m', + 0.9559999704360962)]) +Noise: 10 +Processing data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_10.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 1000 examples, only 0.9 have correct labels +Added noise to 100 examples, only 0.9 have correct labels +Data lm, trn: 13500, val: 1500 +Data clsnoise0.1tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +Bptt 70 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_10.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.976570 1.054016 0.705000 +2 0.885775 0.813666 0.835000 +3 0.861244 0.968860 0.762000 +4 0.790501 0.815453 0.839000 +5 0.754292 0.805088 0.849000 +6 0.742595 0.770547 0.864000 +7 0.712961 0.771171 0.863000 +8 0.695449 0.787870 0.858000 +Total time: 19:48 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_10.m +Loss and accuracy using (cls_best): [0.2744636, tensor(0.9510)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_10.m', + 0.9509999752044678)]) +Noise: 15 +Processing data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_15.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 1500 examples, only 0.85 have correct labels +Added noise to 150 examples, only 0.85 have correct labels +Data lm, trn: 13500, val: 1500 +Data clsnoise0.15tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +Bptt 70 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_15.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.081478 1.075213 0.674000 +2 0.989475 0.939438 0.779000 +3 0.963180 0.984908 0.723000 +4 0.915556 1.209332 0.662000 +5 0.884642 1.000015 0.786000 +6 0.844702 0.884871 0.794000 +7 0.793699 0.882503 0.802000 +8 0.797470 0.871922 0.802000 +Total time: 19:50 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_15.m +Loss and accuracy using (cls_best): [0.32492134, tensor(0.9445)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_15.m', + 0.9445000290870667)]) +Noise: 20 +Processing data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_20.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 2000 examples, only 0.8 have correct labels +Added noise to 200 examples, only 0.8 have correct labels +Data lm, trn: 13500, val: 1500 +Data clsnoise0.2tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +Bptt 70 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_20.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.130177 1.651658 0.424000 +2 1.049187 1.508039 0.286000 +3 1.045260 1.976680 0.578000 +4 0.971859 1.121615 0.735000 +5 0.965327 2.376971 0.684000 +6 0.901961 1.089674 0.744000 +7 0.868971 1.082978 0.750000 +8 0.845376 1.019824 0.740000 +Total time: 19:51 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_20.m +Loss and accuracy using (cls_best): [0.46514454, tensor(0.9438)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_20.m', + 0.9437500238418579)]) +Noise: 25 +Processing data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_25.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 2500 examples, only 0.75 have correct labels +Added noise to 250 examples, only 0.75 have correct labels +Data lm, trn: 13500, val: 1500 +Data clsnoise0.25tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +Bptt 70 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_25.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.204033 1.368233 0.500000 +2 1.121006 1.219437 0.586000 +3 1.057657 1.139297 0.659000 +4 1.054685 1.043641 0.700000 +5 1.023957 1.069890 0.706000 +6 0.992645 1.073037 0.708000 +7 0.948602 1.054931 0.699000 +8 0.945395 1.078187 0.703000 +Total time: 20:09 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_25.m +Loss and accuracy using (cls_best): [0.4676742, tensor(0.9137)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_25.m', + 0.9137499928474426)]) +Noise: 30 +Processing data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_30.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 3000 examples, only 0.7 have correct labels +Added noise to 300 examples, only 0.7 have correct labels +Data lm, trn: 13500, val: 1500 +Data clsnoise0.3tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +Bptt 70 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_30.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.245468 1.243019 0.492000 +2 1.192702 1.644169 0.435000 +3 1.187665 4.143492 0.490000 +4 1.113116 20.139246 0.540000 +5 1.092624 1.189916 0.609000 +6 1.052626 1.264737 0.617000 +7 1.032403 1.317357 0.649000 +8 1.003000 1.187038 0.653000 +Total time: 20:02 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_30.m +Loss and accuracy using (cls_best): [0.5831716, tensor(0.9215)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_30.m', + 0.921500027179718)]) +Noise: 35 +Processing data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_35.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 3500 examples, only 0.65 have correct labels +Added noise to 350 examples, only 0.65 have correct labels +Data lm, trn: 13500, val: 1500 +Data clsnoise0.35tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +Bptt 70 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_35.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.285933 1.719733 0.309000 +2 1.258459 1.465174 0.422000 +3 1.240111 1.205106 0.512000 +4 1.195793 2.153573 0.571000 +5 1.150691 3.427428 0.588000 +6 1.115649 1.933489 0.601000 +7 1.078265 1.214095 0.599000 +8 1.045297 1.140148 0.604000 +Total time: 19:55 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_35.m +Loss and accuracy using (cls_best): [0.5903087, tensor(0.9105)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_35.m', + 0.9104999899864197)]) +Noise: 40 +Processing data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_40.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 4000 examples, only 0.6 have correct labels +Added noise to 400 examples, only 0.6 have correct labels +Data lm, trn: 13500, val: 1500 +Data clsnoise0.4tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +Bptt 70 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_40.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.313393 1.523955 0.316000 +2 1.303059 1.964390 0.418000 +3 1.291624 1.550615 0.458000 +4 1.263588 2.995128 0.390000 +5 1.206715 1.265662 0.524000 +6 1.191890 1.221754 0.536000 +7 1.162122 1.223106 0.527000 +8 1.150922 1.240103 0.531000 +Total time: 19:53 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_40.m +Loss and accuracy using (cls_best): [0.7210464, tensor(0.8583)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_40.m', + 0.8582500219345093)]) +Noise: 45 +Processing data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_45.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 4500 examples, only 0.55 have correct labels +Added noise to 450 examples, only 0.55 have correct labels +Data lm, trn: 13500, val: 1500 +Data clsnoise0.45tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +Bptt 70 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_45.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.345056 1.343081 0.378000 +2 1.281120 11.283777 0.232000 +3 1.284114 14.679921 0.390000 +4 1.267963 2.869378 0.485000 +5 1.227434 1.466781 0.490000 +6 1.209261 1.634938 0.495000 +7 1.170042 1.372811 0.494000 +8 1.162168 2.157310 0.492000 +Total time: 20:05 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_45.m +Loss and accuracy using (cls_best): [1.0457553, tensor(0.8635)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_45.m', + 0.8634999990463257)]) +Noise: 50 +Processing data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_50.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 5000 examples, only 0.5 have correct labels +Added noise to 500 examples, only 0.5 have correct labels +Data lm, trn: 13500, val: 1500 +Data clsnoise0.5tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +Bptt 70 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_50.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.362338 1.361343 0.339000 +2 1.343794 1.358407 0.328000 +3 1.326264 3.336083 0.325000 +4 1.321352 4.200035 0.254000 +5 1.289333 1.363007 0.408000 +6 1.275341 1.449265 0.405000 +7 1.245595 1.358157 0.423000 +8 1.234815 1.346797 0.411000 +Total time: 19:35 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_50.m +Loss and accuracy using (cls_best): [1.3260584, tensor(0.7103)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_50.m', + 0.7102500200271606)]) +Noise: 55 +Processing data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_55.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 5500 examples, only 0.45 have correct labels +Added noise to 550 examples, only 0.45 have correct labels +Data lm, trn: 13500, val: 1500 +Data clsnoise0.55tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +Bptt 70 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_55.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.373838 1.385533 0.265000 +2 1.355375 2.033619 0.316000 +3 1.358652 2.010394 0.260000 +4 1.337999 7.118755 0.351000 +5 1.309082 3.053319 0.361000 +6 1.286589 19.251106 0.359000 +7 1.276432 1.328096 0.379000 +8 1.266364 1.324883 0.378000 +Total time: 19:34 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_55.m +Loss and accuracy using (cls_best): [1.3107486, tensor(0.6503)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_55.m', + 0.6502500176429749)]) +Noise: 60 +Processing data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_60.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 6000 examples, only 0.4 have correct labels +Added noise to 600 examples, only 0.4 have correct labels +Data lm, trn: 13500, val: 1500 +Data clsnoise0.6tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +Bptt 70 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_60.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.378700 1.377784 0.309000 +2 1.359247 9.472390 0.250000 +3 1.343403 1.714557 0.321000 +4 1.336044 1.331355 0.357000 +5 1.322668 1.450317 0.332000 +6 1.283835 2.692688 0.349000 +7 1.261502 1.541230 0.335000 +8 1.230086 1.839382 0.340000 +Total time: 19:58 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_60.m +Loss and accuracy using (cls_best): [1.1523782, tensor(0.5580)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_60.m', + 0.5580000281333923)]) +Noise: 65 +Processing data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_65.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 6500 examples, only 0.35 have correct labels +Added noise to 650 examples, only 0.35 have correct labels +Data lm, trn: 13500, val: 1500 +Data clsnoise0.65tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +Bptt 70 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_65.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.374414 1.361766 0.327000 +2 1.367130 1.353700 0.341000 +3 1.365781 1.421649 0.269000 +4 1.358339 1.385666 0.280000 +5 1.357855 3.068685 0.334000 +6 1.343958 1.586822 0.316000 +7 1.330202 2.436025 0.324000 +8 1.322320 1.743209 0.330000 +Total time: 19:52 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_65.m +Loss and accuracy using (cls_best): [1.7415464, tensor(0.4467)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_65.m', + 0.4467499852180481)]) +Noise: 70 +Processing data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_70.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 7000 examples, only 0.3 have correct labels +Added noise to 700 examples, only 0.3 have correct labels +Data lm, trn: 13500, val: 1500 +Data clsnoise0.7tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +Bptt 70 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_70.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.386991 1.377855 0.302000 +2 1.370086 1.741303 0.298000 +3 1.371910 1.402328 0.316000 +4 1.349717 1.378567 0.277000 +5 1.360438 1.471136 0.298000 +6 1.345680 1.395034 0.312000 +7 1.327264 1.611867 0.312000 +8 1.327243 1.657344 0.312000 +Total time: 20:09 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_70.m +Loss and accuracy using (cls_best): [2.7352421, tensor(0.2693)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_70.m', + 0.2692500054836273)]) +Noise: 75 +Processing data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_75.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 7500 examples, only 0.25 have correct labels +Added noise to 750 examples, only 0.25 have correct labels +Data lm, trn: 13500, val: 1500 +Data clsnoise0.75tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +Bptt 70 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_75.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.376097 1.392457 0.263000 +2 1.372446 1.367712 0.320000 +3 1.356304 1.354679 0.297000 +4 1.342981 1.350475 0.335000 +5 1.340915 1.337473 0.343000 +6 1.320882 1.904698 0.357000 +7 1.291946 1.368179 0.339000 +8 1.283206 1.466608 0.353000 +Total time: 19:50 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_75.m +Loss and accuracy using (cls_best): [1.4704828, tensor(0.1248)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_75.m', + 0.12475000321865082)]) + noise accuracy +0 0.00 0.96125 +1 0.05 0.95600 +2 0.10 0.95100 +3 0.15 0.94450 +4 0.20 0.94375 +5 0.25 0.91375 +6 0.30 0.92150 +7 0.35 0.91050 +8 0.40 0.85825 +9 0.45 0.86350 +10 0.50 0.71025 +11 0.55 0.65025 +12 0.60 0.55800 +13 0.65 0.44675 +14 0.70 0.26925 +15 0.75 0.12475 +``` \ No newline at end of file diff --git a/results/logs/mldoc/random-init.md b/results/logs/mldoc/random-init.md new file mode 100644 index 0000000..f5805fd --- /dev/null +++ b/results/logs/mldoc/random-init.md @@ -0,0 +1,485 @@ + + +### MLDoc laser zero shoot 10k +data/mldoc/de-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m: 0.9052500128746033 +data/mldoc/es-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m: 0.6974999904632568 +data/mldoc/fr-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m: 0.8740000128746033 +data/mldoc/it-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m: 0.7272499799728394 +data/mldoc/zh-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m: 0.8144999742507935 + +``` +python -m ulmfit eval --glob="mldoc/*-1/models/sp15k/qrnn_nl4.m" --dataset_template='${lang}-10-laser-en1' --name rnd_nl4 --num-cls-epochs=8 --label-smoothing-eps=0.1 --lr_sched=1cycle --bs=18 --random-init=True +python -m ulmfit eval --glob="mldoc/*-1/models/sp15k/qrnn_nl4.m" --dataset_template='${lang}-10-laser-en1' --name rnd_nl4 --num-cls-epochs=8 --label-smoothing-eps=0.1 --lr_sched=1cycle --bs=18 --random-init=True +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10-laser-en1 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10-laser-en1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10-laser-en1/de.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: + + !! WARNING !! + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +Your compiler (c++) may be ABI-incompatible with PyTorch! +Please use a compiler that is ABI-compatible with GCC 4.9 and above. +See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. + +See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 +for instructions on how to install GCC 4.9 or higher. +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + !! WARNING !! + + warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 0.864752 0.760891 0.844000 +2 0.734504 1.077554 0.676000 +3 0.681645 0.703327 0.885000 +4 0.670696 0.779010 0.898000 +5 0.620256 0.664871 0.910000 +6 0.591837 1.077103 0.915000 +7 0.550238 0.607863 0.913000 +8 0.543874 0.607274 0.918000 +Total time: 19:55 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m +Loss and accuracy using (cls_best): [0.35624045, tensor(0.9053)] +Processing data/mldoc/en-1/models/sp15k/qrnn_nl4.m +en-10-laser-en1 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10-laser-en1 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10-laser-en1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10-laser-en1/es.dev.csv +Data lm, trn: 13013, val: 1445 +Data cls, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 0.755512 1.360725 0.500000 +2 0.760655 1.362604 0.399000 +3 0.760876 20.748863 0.607000 +4 0.730208 8.120344 0.369000 +5 0.707735 1.149775 0.700000 +6 0.679102 1.010318 0.746000 +7 0.639611 3.087066 0.713000 +8 0.608591 1.327793 0.750000 +Total time: 11:38 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m +Loss and accuracy using (cls_best): [1.3680531, tensor(0.6975)] +Processing data/mldoc/fr-1/models/sp15k/qrnn_nl4.m +fr-10-laser-en1 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-10-laser-en1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-10-laser-en1/fr.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', 's', '▁.', "'", '▁la', '▁le', '▁et', '▁l', '▁à'] +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 0.922996 1.406875 0.519000 +2 0.833284 1.172545 0.640000 +3 0.749922 0.697733 0.863000 +4 0.724680 0.735842 0.837000 +5 0.652541 0.679455 0.876000 +6 0.641541 0.671731 0.868000 +7 0.577571 0.734958 0.868000 +8 0.579186 0.703696 0.883000 +Total time: 19:15 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m +Loss and accuracy using (cls_best): [0.47713563, tensor(0.8740)] +Processing data/mldoc/it-1/models/sp15k/qrnn_nl4.m +it-10-laser-en1 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-10-laser-en1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-10-laser-en1/it.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', '▁e', "▁&'", "'", '▁il', '▁la', '▁in', 'e'] +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 0.991065 1.602189 0.381000 +2 0.935880 0.888808 0.734000 +3 0.860670 0.868564 0.781000 +4 0.818734 0.945302 0.791000 +5 0.751467 3.113552 0.808000 +6 0.687606 0.921033 0.795000 +7 0.677044 1.222023 0.807000 +8 0.645511 1.418593 0.805000 +Total time: 11:44 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m +Loss and accuracy using (cls_best): [1.1276722, tensor(0.7272)] +Processing data/mldoc/ja-1/models/sp15k/qrnn_nl4.m +ja-10-laser-en1 +Processing data/mldoc/zh-1/models/sp15k/qrnn_nl4.m +zh-10-laser-en1 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-10-laser-en1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-10-laser-en1/zh.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] +Starting classifier from random weights +Single training schedule +epoch train_loss valid_loss accuracy +1 0.920411 1.159853 0.629000 +2 0.937020 1.371089 0.527000 +3 0.892036 3.091183 0.615000 +4 0.839919 0.939323 0.724000 +5 0.797184 1.174206 0.735000 +6 0.774195 0.914951 0.733000 +7 0.744524 0.875888 0.762000 +8 0.721782 0.825969 0.788000 +Total time: 19:53 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m +Loss and accuracy using (cls_best): [0.54084456, tensor(0.8145)] +OrderedDict([('data/mldoc/de-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m', + 0.9052500128746033), + ('data/mldoc/es-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m', + 0.6974999904632568), + ('data/mldoc/fr-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m', + 0.8740000128746033), + ('data/mldoc/it-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m', + 0.7272499799728394), + ('data/mldoc/zh-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m', + 0.8144999742507935)]) +data/mldoc/de-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m: 0.9052500128746033 +data/mldoc/es-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m: 0.6974999904632568 +data/mldoc/fr-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m: 0.8740000128746033 +data/mldoc/it-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m: 0.7272499799728394 +data/mldoc/zh-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m: 0.8144999742507935 +``` + +### MLDoc Classification on 1k + +data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m: 0.9024999737739563 +data/mldoc/en-1/models/sp15k/qrnn_rnd-nl4.m: 0.8149999976158142 +data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m: 0.8964999914169312 +data/mldoc/fr-1/models/sp15k/qrnn_rnd-nl4.m: 0.8220000267028809 +data/mldoc/it-1/models/sp15k/qrnn_rnd-nl4.m: 0.7889999747276306 +data/mldoc/ja-1/models/sp15k/qrnn_rnd-nl4.m: 0.8302500247955322 +data/mldoc/ru-1/models/sp15k/qrnn_rnd-nl4.m: 0.7319999933242798 +data/mldoc/zh-1/models/sp15k/qrnn_rnd-nl4.m: 0.8452500104904175 + + +``` +python -m ulmfit eval --glob="wiki/*-100/models/sp15k/qrnn_rnd-nl4.m" --name rnd-nl4 --dataset-template='../mldoc/${lang}-1' --num-lm-epochs=0 --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.1 +Processing data/wiki/de-100/models/sp15k/qrnn_rnd-nl4.m +../mldoc/de-1 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/de.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: + + !! WARNING !! + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +Your compiler (c++) may be ABI-incompatible with PyTorch! +Please use a compiler that is ABI-compatible with GCC 4.9 and above. +See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. + +See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 +for instructions on how to install GCC 4.9 or higher. +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + !! WARNING !! + + warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) +Loading pretrained model +Unknown tokens 0, first 100: [] +Bptt 70 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.411651 1.386593 0.265000 +2 1.287022 1.488027 0.361000 +3 1.084153 2.390431 0.370000 +4 0.904227 0.936213 0.769000 +5 0.740495 1.311880 0.538000 +6 0.642756 0.754690 0.833000 +7 0.582816 0.661088 0.892000 +8 0.548893 0.683635 0.875000 +Total time: 02:13 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m +Loss and accuracy using (cls_best): [0.33525154, tensor(0.9025)] +Processing data/wiki/en-100/models/sp15k/qrnn_rnd-nl4.m +../mldoc/en-1 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k/qrnn_rnd-nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/en.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁the', '▁,', 's', '▁.', '▁of', '▁and', '▁in', '▁to', '▁a', 'ed'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +Bptt 70 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k/qrnn_rnd-nl4.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.369466 1.356636 0.374000 +2 1.263357 2.515120 0.320000 +3 1.119744 1.250081 0.569000 +4 0.949653 1.033515 0.666000 +5 0.802069 0.875799 0.779000 +6 0.676997 0.842525 0.807000 +7 0.613777 0.794573 0.826000 +8 0.571342 0.781615 0.837000 +Total time: 02:26 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k/qrnn_rnd-nl4.m +Loss and accuracy using (cls_best): [0.5295334, tensor(0.8150)] +Processing data/wiki/es-100/models/sp15k/qrnn_rnd-nl4.m +../mldoc/es-1 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv +Data lm, trn: 13013, val: 1445 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 2621, first 100: ['▁sa', '▁i', 'ncia', '▁ka', '▁k', '▁tra', '▁fi', '▁volvi', '▁g', '▁man', '▁pasó', '▁tropas', 'pon', 'tuvieron', '▁x', '▁les', '▁empez', 'ieron', '▁bas', 'sco', '▁cam', '▁adapta', 'sion', '▁mol', 'pico', 'siones', '▁obstante', '▁!', '▁w', 'cular', 'puesta', '▁inten', '▁produj', 'clu', 'simismo', '▁pas', 'fla', '▁amerindio', 'aje', '▁deja', '▁fre', '▁jo', '▁2.', 'american', '▁cre', 'bajo', '▁medi', 'gla', '▁dirigi', 'hol', '▁aparición', 'aciones', 'vivi', 'eras', 'spe', '▁continu', '▁permaneci', '▁ber', 'usa', 'bió', '▁permitió', '▁municipios', '▁regres', 'rt', 'mbi', '▁pr', '▁ofreci', 'emi', 'misiones', '▁cap', '▁ram', 'icio', '▁wal', 'fru', '▁gen', '▁originalmente', '▁eva', '▁ferr', '▁descubri', '▁aparecen', '▁fon', 'capi', 'estre', 'pec', '▁vendi', 'iéndose', 'eja', 'liber', 'nsa', 'ológico', 'ío', 'blo', '▁tro', '▁aviones', 'cara', '▁activo', 'mostró', 'disciplina', '▁ara', 'estra'] +Bptt 70 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.378275 1.354129 0.314000 +2 1.209560 1.333649 0.507000 +3 1.022200 0.820093 0.801000 +4 0.854187 1.782254 0.389000 +5 0.722861 1.031932 0.692000 +6 0.640640 0.762994 0.853000 +7 0.583225 0.677089 0.901000 +8 0.556481 0.652575 0.904000 +Total time: 01:59 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m +Loss and accuracy using (cls_best): [0.35487488, tensor(0.8965)] +Processing data/wiki/fr-100/models/sp15k/qrnn_rnd-nl4.m +../mldoc/fr-1 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_rnd-nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/fr.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', 's', '▁.', "'", '▁la', '▁le', '▁et', '▁l', '▁à'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +Bptt 70 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_rnd-nl4.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.360408 1.298057 0.444000 +2 1.251207 2.454086 0.393000 +3 1.098488 1.152682 0.544000 +4 0.926239 1.256870 0.622000 +5 0.806994 0.911339 0.732000 +6 0.717139 0.945148 0.726000 +7 0.635195 0.781772 0.825000 +8 0.602214 0.763521 0.825000 +Total time: 02:17 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_rnd-nl4.m +Loss and accuracy using (cls_best): [0.52210134, tensor(0.8220)] +Processing data/wiki/it-100/models/sp15k/qrnn_rnd-nl4.m +../mldoc/it-1 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k/qrnn_rnd-nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/it.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', '▁e', "▁&'", "'", '▁il', '▁la', '▁in', 'e'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +Bptt 70 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k/qrnn_rnd-nl4.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.357973 1.353393 0.293000 +2 1.282061 1.535401 0.390000 +3 1.144333 1.480346 0.533000 +4 0.985930 1.360542 0.540000 +5 0.862014 1.285450 0.661000 +6 0.720891 1.140574 0.629000 +7 0.625376 0.840085 0.791000 +8 0.572435 0.828283 0.793000 +Total time: 01:21 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k/qrnn_rnd-nl4.m +Loss and accuracy using (cls_best): [0.5931235, tensor(0.7890)] +Processing data/wiki/ja-100/models/sp15k/qrnn_rnd-nl4.m +../mldoc/ja-1 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k/qrnn_rnd-nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +Bptt 70 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k/qrnn_rnd-nl4.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.377756 1.402221 0.254000 +2 1.243837 7.998792 0.254000 +3 1.066383 2.645358 0.354000 +4 0.903686 1.348676 0.541000 +5 0.843216 0.945152 0.743000 +6 0.759674 0.801283 0.810000 +7 0.689767 0.786832 0.820000 +8 0.674777 0.778615 0.818000 +Total time: 02:48 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k/qrnn_rnd-nl4.m +Loss and accuracy using (cls_best): [0.5008588, tensor(0.8303)] +Processing data/wiki/ru-100/models/sp15k/qrnn_rnd-nl4.m +../mldoc/ru-1 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_rnd-nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv +Running tokenization lm... +Data lm, trn: 9195, val: 1021 +Running tokenization cls... +Data cls, trn: 1000, val: 1000 +Running tokenization tst... +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', '▁на', 'х'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +Bptt 70 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_rnd-nl4.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.394592 1.393761 0.265000 +2 1.381886 1.476836 0.293000 +3 1.258016 1.153065 0.555000 +4 1.105392 1.323574 0.556000 +5 0.948704 1.049486 0.703000 +6 0.848964 1.480141 0.605000 +7 0.757975 1.001765 0.723000 +8 0.684587 0.982136 0.741000 +Total time: 03:07 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_rnd-nl4.m +Loss and accuracy using (cls_best): [0.7138667, tensor(0.7320)] +Processing data/wiki/zh-100/models/sp15k/qrnn_rnd-nl4.m +../mldoc/zh-1 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_rnd-nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +Bptt 70 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_rnd-nl4.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.341714 1.208123 0.569000 +2 1.059330 1.169385 0.662000 +3 0.926222 0.771242 0.824000 +4 0.843994 1.997928 0.524000 +5 0.800537 0.874480 0.756000 +6 0.710552 0.909481 0.758000 +7 0.657595 0.719883 0.854000 +8 0.617662 0.727267 0.852000 +Total time: 02:20 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_rnd-nl4.m +Loss and accuracy using (cls_best): [0.48266637, tensor(0.8453)] +OrderedDict([('data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m', + 0.9024999737739563), + ('data/mldoc/en-1/models/sp15k/qrnn_rnd-nl4.m', + 0.8149999976158142), + ('data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m', + 0.8964999914169312), + ('data/mldoc/fr-1/models/sp15k/qrnn_rnd-nl4.m', + 0.8220000267028809), + ('data/mldoc/it-1/models/sp15k/qrnn_rnd-nl4.m', + 0.7889999747276306), + ('data/mldoc/ja-1/models/sp15k/qrnn_rnd-nl4.m', + 0.8302500247955322), + ('data/mldoc/ru-1/models/sp15k/qrnn_rnd-nl4.m', + 0.7319999933242798), + ('data/mldoc/zh-1/models/sp15k/qrnn_rnd-nl4.m', + 0.8452500104904175)]) +data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m: 0.9024999737739563 +data/mldoc/en-1/models/sp15k/qrnn_rnd-nl4.m: 0.8149999976158142 +data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m: 0.8964999914169312 +data/mldoc/fr-1/models/sp15k/qrnn_rnd-nl4.m: 0.8220000267028809 +data/mldoc/it-1/models/sp15k/qrnn_rnd-nl4.m: 0.7889999747276306 +data/mldoc/ja-1/models/sp15k/qrnn_rnd-nl4.m: 0.8302500247955322 +data/mldoc/ru-1/models/sp15k/qrnn_rnd-nl4.m: 0.7319999933242798 +data/mldoc/zh-1/models/sp15k/qrnn_rnd-nl4.m: 0.8452500104904175 +``` \ No newline at end of file diff --git a/results/logs/ru/5epochs.md b/results/logs/ru/5epochs.md new file mode 100644 index 0000000..ca357c3 --- /dev/null +++ b/results/logs/ru/5epochs.md @@ -0,0 +1,91 @@ +``` +export CUDA_VISIBLE_DEVICES=0 +LANG=ru +python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/sp15k/qrnn_nl4.m --lang=${LANG} --name 'nl4-lm5' - train 5 --bs 20 --num-cls-epochs=8 --lr_sched=1cycle --label-smoothing-eps=0.1 + +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4-lm5.m +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv +Data lm, trn: 9195, val: 1021 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', '▁на', 'х'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: + + !! WARNING !! + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +Your compiler (c++) may be ABI-incompatible with PyTorch! +Please use a compiler that is ABI-compatible with GCC 4.9 and above. +See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. + +See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 +for instructions on how to install GCC 4.9 or higher. +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + !! WARNING !! + + warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) +Loading pretrained model +Unknown tokens 0, first 100: [] +Bptt 70 +Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp15k/qrnn_nl4.m/../itos')] +epoch train_loss valid_loss accuracy +1 4.549059 3.763798 0.472608 +Total time: 02:05 +epoch train_loss valid_loss accuracy +1 3.543295 3.263310 0.567992 +2 3.166918 2.968566 0.619391 +3 3.057842 2.812808 0.648944 +4 2.842979 2.726823 0.665521 +5 2.872606 2.703771 0.670281 +Total time: 14:40 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4-lm5.m/info.json +``` +``` +python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/sp15k/qrnn_nl4.m --lang=${LANG} --name 'nl4-lm5' - train 5 --bs 18 --num-cls-epochs=8 --lr_sched=1cycle --label-smoothing-eps=0.1 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4-lm5.m +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv +Data lm, trn: 9195, val: 1021 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', '▁на', 'х'] +/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: + + !! WARNING !! + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +Your compiler (c++) may be ABI-incompatible with PyTorch! +Please use a compiler that is ABI-compatible with GCC 4.9 and above. +See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. + +See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 +for instructions on how to install GCC 4.9 or higher. +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + !! WARNING !! + + warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) +Single training schedule +epoch train_loss valid_loss accuracy +1 1.021985 0.763919 0.822000 +2 0.903123 0.756099 0.849000 +3 0.831409 0.852466 0.832000 +4 0.744423 0.753127 0.858000 +5 0.669933 0.747895 0.862000 +6 0.607411 0.744035 0.869000 +7 0.554080 0.706676 0.872000 +8 0.532403 0.719503 0.870000 +Total time: 03:12 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4-lm5.m +Loss and accuracy using (cls_best): [0.41288647, tensor(0.8615)] +0.41288647055625916 +0.8615000247955322 +``` \ No newline at end of file diff --git a/results/logs/ru/nl8.md b/results/logs/ru/nl8.md new file mode 100644 index 0000000..ba03d02 --- /dev/null +++ b/results/logs/ru/nl8.md @@ -0,0 +1,78 @@ +python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/sp15k/qrnn_${NAME}.m --lang=${LANG} --name ${NAME}-2 - train 20 --bs 18 --num-cls-epochs=4 --lr_sched=1cycle --label-smoothing-eps=0.1 +Max vocab: 15000 +Cache dir: /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k +Model dir: /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl8-2.m +Loading validation /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv +Data lm, trn: 9195, val: 1021 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', '▁на', 'х'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +Training lm from: [PosixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp15k/qrnn_nl8.m/lm_best'), PosixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp15k/qrnn_nl8.m/../itos')] +epoch train_loss valid_loss accuracy +1 3.966292 3.450758 0.527065 +Total time: 02:58 +epoch train_loss valid_loss accuracy +1 3.495761 3.276329 0.560047 +2 3.319947 3.102911 0.593742 +3 3.137904 2.955317 0.620171 +4 3.040286 2.839161 0.642270 +5 2.869962 2.753622 0.658331 +6 2.905739 2.680881 0.672860 +7 2.836454 2.620925 0.685026 +8 2.857271 2.569716 0.695722 +9 2.702872 2.520050 0.705589 +10 2.701559 2.473591 0.715346 +11 2.740815 2.429558 0.725597 +12 2.646513 2.389550 0.735010 +13 2.587685 2.349614 0.744885 +14 2.546527 2.311087 0.754463 +15 2.568136 2.278581 0.762980 +16 2.492115 2.252367 0.769275 +17 2.338561 2.230529 0.775072 +18 2.447506 2.218215 0.778437 +19 2.364424 2.212115 0.780085 +20 2.367132 2.210520 0.780424 +Total time: 1:30:47 +/home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k +Saving info /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl8-2.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.969255 0.769736 0.843000 +2 0.846340 0.813483 0.839000 +3 0.718175 0.705339 0.867000 +4 0.609513 0.726442 0.875000 +Total time: 02:54 +Saving models at /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl8-2.m +Loss and accuracy using (cls_best): [0.4056449, tensor(0.8698)] +0.40564489364624023 + + + +(fastaiv1) n-waves@GV100:~/workspace/ulmfit-multilingual$ CUDA_VISIBLE_DEVICES=0 python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4-merity-wide2' --max-vocab 15000 --lang ${LANG} --qrnn=True --bptt=140 --nh 3100 - train 10 --bs=50 --drop_mult=0 --label-smoothing-eps=0.1 +Max vocab: 15000 +Cache dir: data/wiki/ru-100/models/sp15k +Model dir: data/wiki/ru-100/models/sp15k/qrnn_nl4-merity-wide2.m +Wiki text was split to 193047 articles +Wiki text was split to 460 articles +Data lm, trn: 193047, val: 460 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', '▁на', 'х'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Training lm from random weights +epoch train_loss valid_loss accuracy +1 3.871650 3.908779 0.467000 +2 3.834093 3.884629 0.467916 +3 3.741005 3.870331 0.469612 +4 3.785444 3.818511 0.476906 +5 3.741888 3.752743 0.486148 +6 3.678481 3.672177 0.499054 +7 3.570398 3.581498 0.512801 +8 3.455193 3.482614 0.530569 +9 3.379779 3.409405 0.543477 +10 3.384574 3.387195 0.548881 +Total time: 27:24:33 +data/wiki/ru-100/models/sp15k \ No newline at end of file diff --git a/results/logs/tokenization/ru.md b/results/logs/tokenization/ru.md new file mode 100644 index 0000000..6adaa2e --- /dev/null +++ b/results/logs/tokenization/ru.md @@ -0,0 +1,29 @@ +``` +LANG=ru +python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='vf' --nl 4 --name 'nl4' --max-vocab 60000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0 --label-smoothing-eps=0.1 +Max vocab: 60000 +Cache dir: data/wiki/ru-100/models/vf60k +Model dir: data/wiki/ru-100/models/vf60k/qrnn_nl4.m +Wiki text was split to 193047 articles +Wiki text was split to 460 articles +Running tokenization lm... +Data lm, trn: 193047, val: 460 +Size of vocabulary: 60003 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', ',', '\n', '.', 'в', 'и', ')', '(', 'на', '—', '«', '»', 'с'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Training lm from random weights +epoch train_loss valid_loss accuracy +1 4.586900 4.478803 0.413023 +2 4.483496 4.400461 0.418495 +3 4.484620 4.390928 0.418422 +4 4.373594 4.350045 0.422567 +5 4.350337 4.307665 0.427411 +6 4.314571 4.249700 0.436324 +7 4.232540 4.183857 0.446341 +8 4.252573 4.119820 0.455522 +9 4.136978 4.088805 0.462345 +10 4.116755 4.079840 0.465394 +Total time: 11:24:03 +data/wiki/ru-100/models/vf60k +``` + diff --git a/results/time_benchmark/logs.md b/results/time_benchmark/logs.md new file mode 100644 index 0000000..d5ec78f --- /dev/null +++ b/results/time_benchmark/logs.md @@ -0,0 +1,50 @@ +# Results + +## Set-up. + - Num Tokens 15K + - GPU V100 + - LM BPTT = 70 + - LM BS = 64 + - CLAS BS = 32 + +| Model | LSTM | QRNN | +|----------------|-----------|-----------| +| LM ms/batch | 143ms | 71ms | +| CLAS ms/batch | 467ms | 156ms | + + +``` +> python results/time_benchmark/qrnn_benchmark.py + +Vocab size 14513 +QRNN +LM +epoch train_loss valid_loss accuracy +1 6.326089 +Total time: 00:11 +Batch size torch.Size([64, 70]) +Params = 22 MM +Training time is 71.0 ms per batch +CLAS +epoch train_loss valid_loss accuracy +1 0.712603 +Total time: 00:10 +Batch size torch.Size([32, 1445]) +Params = 22 MM +Training time is 156.0 ms per batch +LSTM +LM +epoch train_loss valid_loss accuracy +1 6.262911 +Total time: 00:21 +Batch size torch.Size([64, 70]) +Params = 37 MM +Training time is 143.0 ms per batch +CLAS +epoch train_loss valid_loss accuracy +1 0.706715 +Total time: 00:32 +Batch size torch.Size([32, 1445]) +Params = 37 MM +Training time is 467.0 ms per batch +``` \ No newline at end of file diff --git a/results/time_benchmark/qrnn_benchmark.py b/results/time_benchmark/qrnn_benchmark.py new file mode 100644 index 0000000..f6dab86 --- /dev/null +++ b/results/time_benchmark/qrnn_benchmark.py @@ -0,0 +1,52 @@ +import glob +import shutil +import time +from fastai.text import * + +orig_path = untar_data(URLs.IMDB) +path = Path('data') / 'imdb_small' +path.mkdir(parents=True, exist_ok=True) + +for mode in ['train', 'test']: + for label in ['pos', 'neg']: + tgt_path = path / mode / label + tgt_path.mkdir(parents=True, exist_ok=True) + # Keep just 10% of the files + pattern = str(orig_path / mode / label / '3*.txt') + for file in glob.glob(pattern): + shutil.copy(file, tgt_path) + +data_lm = TextLMDataBunch.from_folder(path, valid='test') +data_clas = TextClasDataBunch.from_folder(path, bs=32, vocab=data_lm.train_ds.vocab, valid='test') + +print('Vocab size', len(data_lm.train_ds.vocab.itos)) + + +def count_parameters(model, requires_grad): + return sum(p.numel() for p in model.parameters() if p.requires_grad == requires_grad) + + +def test(qrnn, func, config, data, arch=AWD_LSTM): + total = len(list(data.train_dl)) + config = config.copy() + config['qrnn'] = qrnn + + learn = func(data, AWD_LSTM, config=config, pretrained=False) + learn.unfreeze() + params = count_parameters(learn.model, True) + total = len(list(data.train_dl)) + start_time = time.clock() + learn.fit(1) + diff = time.clock() - start_time + + print('Batch size', data.one_batch()[0].shape) + print(f'Params = {params // 1000000} MM') + print(f'Training time is {1000 * diff // total} ms per batch') + + +for qrnn in [True, False]: + print('QRNN' if qrnn else 'LSTM') + print('LM') + test(qrnn, language_model_learner, config=awd_lstm_lm_config, data=data_lm) + print('CLAS') + test(qrnn, text_classifier_learner, config=awd_lstm_clas_config, data=data_clas) \ No newline at end of file diff --git a/ulmfit/__main__.py b/ulmfit/__main__.py index 011e999..38b7f76 100644 --- a/ulmfit/__main__.py +++ b/ulmfit/__main__.py @@ -51,12 +51,7 @@ class ULMFiT: def eval_noise_resistance(self, lang="de", size=1, prefix_name="", model="sp15k/qrnn_nl4.m", - num_cls_epochs=8, bs=18, lr_sched="1cycle", label_smoothing_eps=0.0): - def first_or_default(l, default=None): - l = list(l) - if l: - return l[0] - return default + num_cls_epochs=8, bs=18, lr_sched="1cycle", label_smoothing_eps=0.0, **kwargs): results= [] for noise in range(0, 80, 5): print("Noise: ", noise) @@ -67,22 +62,27 @@ class ULMFiT: num_cls_epochs=num_cls_epochs, bs=bs, lr_sched=lr_sched, - label_smoothing_eps=label_smoothing_eps) - val = first_or_default(d.values(), default=-1) + label_smoothing_eps=label_smoothing_eps, + **kwargs) + val = next(iter(d.values()), -1) results.append((noise/100, val)) df = pd.DataFrame(results, columns=["noise", "accuracy"]) df.to_csv(f"noise_{lang}-{size}{prefix_name}.csv") print(df) def tar(self, model_path): + data_dir = (Path.cwd()/"data").resolve() params = CLSHyperParams.from_json(model_path) - tar_name = f"models/{params.lang}-{params.tokenizer_prefix}-{params.model_name}.tar" + name = str(params.dataset_dir.resolve().relative_to(data_dir)).replace("/", "-") + + tar_name = f"models/{name}-{params.tokenizer_prefix}-{params.model_name}.tar" print("Storing model in", tar_name) with tarfile.open(tar_name, mode="w") as tar: - for g in map(params.model_dir.glob, ['*_last.*', 'info.json', 'info.json', '../spm.*', '../itos.*',]): + for g in map(params.model_dir.glob, ['*_best.pth', 'info.json', '../spm.*', '../itos.*',]): for f in g: - print("Adding", f, f.relative_to("data")) - tar.add(f, f.relative_to("data")) + dest = f.resolve().relative_to(Path.cwd()) + print("Adding", f, dest) + tar.add(f, dest) def eval(self, glob="mldoc/*-1/models/sp30k/lstm_nl4.m", dataset_template='${lang}-1', name="tmp-100", num_lm_epochs=0, cuda_id=0, **trn_params): results = OrderedDict() diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 737af26..da66aac 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -65,7 +65,7 @@ class CLSHyperParams(LMHyperParams): def train_cls(self, num_lm_epochs, unfreeze=True, num_cls_frozen_epochs=1, bs=40, drop_mul_lm=0.3, drop_mul_cls=0.5, use_test_for_validation=False, num_cls_epochs=2, limit=None, noise=0.0, cls_max_len=20*70, lr_sched='layered', - label_smoothing_eps=0.0): + label_smoothing_eps=0.0, random_init=False): assert use_test_for_validation == False, "use_test_for_validation=True is not supported" self.model_dir.mkdir(exist_ok=True, parents=True) @@ -74,13 +74,22 @@ class CLSHyperParams(LMHyperParams): data_clas, data_lm, data_tst = self.load_cls_data(bs, limit=limit, noise=noise) - if self.need_fine_tune_lm: self.train_lm(num_lm_epochs, data_lm=data_lm, drop_mult=drop_mul_lm, label_smoothing_eps=label_smoothing_eps) - learn = self.create_cls_learner(data_clas, drop_mult=drop_mul_cls, max_len=cls_max_len, label_smoothing_eps=label_smoothing_eps) - try: - learn.load('cls_best') - print("Loading last classifier") - except FileNotFoundError: - learn.load_encoder(ENC_BEST) + if self.need_fine_tune_lm and not random_init: + if not (self.model_dir/(ENC_BEST+".pth")).exists(): + self.train_lm(num_lm_epochs, data_lm=data_lm, drop_mult=drop_mul_lm, label_smoothing_eps=label_smoothing_eps) + else: + print("Language model already exist, skipping finetuning") + learn = self.create_cls_learner(data_clas, drop_mult=drop_mul_cls, max_len=cls_max_len, + label_smoothing_eps=label_smoothing_eps, random_init=random_init) + if not random_init: + try: + learn.load('cls_best') + print("Loading last classifier") + except FileNotFoundError: + learn.load_encoder(ENC_BEST) + else: + print("Starting classifier from random weights") + if hasattr(self, 'lr_schedule_'+lr_sched): learn.true_wd = True @@ -105,7 +114,7 @@ class CLSHyperParams(LMHyperParams): print(f"Loss and accuracy using ({save_name}):", results) return list(map(float, results)) - def create_cls_learner(self, data_clas, dps=None, label_smoothing_eps=0.0, **kwargs): + def create_cls_learner(self, data_clas, dps=None, label_smoothing_eps=0.0, random_init=False, **kwargs): assert self.bidir == False, "bidirectional model is not yet supported" config = dict(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, pad_token=PAD_TOKEN_ID, qrnn=self.qrnn) config.update(dps or self.dps) @@ -114,8 +123,8 @@ class CLSHyperParams(LMHyperParams): learn = text_classifier_learner(data_clas, AWD_LSTM, config=config, pretrained=False, path=self.model_dir.parent, model_dir=self.model_dir.name, **trn_args) - if self.pretrained_model is not None: - print("Loading pretrained model") + if self.pretrained_model is not None and not random_init: + print("Loading pretrained model", self.pretrained_model) model_path = untar_data(self.pretrained_model, data=False) fnames = [list(model_path.glob(f'*.{ext}'))[0] for ext in ['pth', 'pkl']] learn.load_pretrained(*fnames, strict=False)