diff --git a/MLDoc.md b/MLDoc.md new file mode 100644 index 0000000..bf319f0 --- /dev/null +++ b/MLDoc.md @@ -0,0 +1 @@ +# MLDoc \ No newline at end of file diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py index dce29d3..ebd9674 100644 --- a/fastai_contrib/utils.py +++ b/fastai_contrib/utils.py @@ -66,6 +66,8 @@ class SentencePieceTokenizer(Tokenizer): toks = super().process_text(t, tok) toks = tok.sp.EncodeAsPieces(" ".join(toks)) return toks +full_char_coverage_langs = ["bg", "cs", "da", "de", "el", "en", "es", "et", "fi", "fr", "ga", "hr", "hu", + "it","lt","lv","mt","nl","pl","pt","ro","sk","sl","sv"] # all European langus def get_sentencepiece(cache_dir:PathOrStr, load_text, pre_rules: ListRules=None, post_rules:ListRules=None, vocab_size:int=30000, model_type:str='unigram', input_sentence_size:int=1E7, lang='en'): @@ -93,9 +95,11 @@ def get_sentencepiece(cache_dir:PathOrStr, load_text, pre_rules: ListRules=None, raw_text_path = cache_dir / 'all_text.txt' with open(raw_text_path, 'w') as f: f.write("\n".join(text)) + char_coverage = 1 if lang in full_char_coverage_langs else 0.99 + sp_params = [ f"--input={raw_text_path}", - f"--character_coverage=1.0", + f"--character_coverage={char_coverage}", f"--unk_id={len(defaults.text_spec_tok)}", f"--pad_id=-1", f"--bos_id=-1", diff --git a/results/MLDoc.md b/results/MLDoc.md index c39f213..5448079 100644 --- a/results/MLDoc.md +++ b/results/MLDoc.md @@ -5,14 +5,21 @@ |LASER 0 shot | 80.75 | 87.03 | 82.60 | 82.83 | 73.25 | 60.95 | 68.83 | 72.90 | |LASER | 90.73 | 92.70 | 88.75 | 90.80 | 85.93 | 85.15 | 84.65 | 88.98 | |MultiCCA | 92.2 | 93.70 | 94.45 | 92.05 | 85.55 | 85.35 | 85.65 | 87.30 | -|ULMFiT 100 | | 91.35 | 83.32 | 88.77 | 77.99 | 71.12 | 72.20 | | -|ULMFiT Zeroshot from Laser | | 94.48 | 86.93 | 88.78 | 79.35 | | 72.88 | 85.55 | -|ULMFiT | | 95.4 | **95.15** | 93.67 | 88.42 | **89.20** | **87.27** | 90.20 | -|ULMFiT sp-fixed | | **95.6** | 94.80 | **94.20** | **88.52** | 88.72 | 86.85 | 90.47 | -|Bert Multi | 93.23% | 94.0% | **95.15** | 93.20 | 85.82 | 87.48 | 86.85 | **90.72** | +|Bert Multi | 93.23 | 94.0 | 95.15 | 93.20 | 85.82 | 87.48 | 86.85 | 90.72 | +|ULMFiT L30k-100 | | 91.35 | 83.32 | 88.77 | 77.99 | 71.12 | 72.20 | | +|ULMFiT L30k | | 95.4 | 95.15 | 93.67 | 88.42 | 89.20 | **87.27** | 90.20 | +|ULMFiT sp-fixed | | **95.6** | 94.80 | 94.20 | 88.52 | 88.72 | 86.85 | 90.47 | +|ULMFIT Q15k 1cyc| **94.62** | **95.65** | 95.15 | **94.42** | **89.92** | 89.60 | | 90.78/89.82 | +|ULMFIT Q15k 1c l| | | | | | | | **92.22** | +|ULMFIT L30k 1cyc| | **95.85** | **96.32** | **94.82** | 89.87 | **90.45** | **87.94** | 92.02/91.64 | +- L30k - LSTM sp30k trained using gradual unfreezing +- L30k-100 - --||-- **on 100 samples** +- ULMFiT sp-fixed - --||-- with fixed tokenization +- Q15k 1cyc - QRNN sp15k trained using 1cycle learning rate schedule +- L30k 1cyc - LSTM sp30k trained using 1cycle learning rate schedule -## Zero shot approaches +## Zero shot approaches - LSTM | Model | de | es | fr | it | ru | zh | |----------------------|------------|------------|-----------|-----------|-----------|-----------| diff --git a/results/logs/Training qrnn on ML.txt b/results/logs/Training qrnn on ML.txt new file mode 100644 index 0000000..214804d --- /dev/null +++ b/results/logs/Training qrnn on ML.txt @@ -0,0 +1,448 @@ +Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/de-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/de-100/models/sp15k/qrnn_nl4.m/../itos')] +epoch train_loss valid_loss accuracy +1 3.467852 2.558666 0.525457 +Total time: 02:25 +epoch train_loss valid_loss accuracy +1 2.722157 2.387366 0.548566 +2 2.477095 2.170018 0.580988 +3 2.182971 1.981363 0.609205 +4 2.078041 1.836848 0.629900 +5 1.975613 1.744062 0.642769 +6 1.866875 1.656678 0.655799 +7 1.831995 1.595655 0.665479 +8 1.768020 1.540487 0.673880 +9 1.751569 1.488140 0.682557 +10 1.647143 1.441723 0.690275 +11 1.712795 1.399652 0.697534 +12 1.529405 1.350384 0.706170 +13 1.549134 1.313349 0.713210 +14 1.585015 1.278395 0.719908 +15 1.475010 1.248854 0.725591 +16 1.532636 1.221373 0.731053 +17 1.445181 1.203350 0.734503 +18 1.396236 1.191440 0.737102 +19 1.316587 1.186497 0.738052 +20 1.374460 1.185027 0.738290 +Total time: 1:11:26 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_nl4.m/info.json +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_nl4.m +Loss and accuracy using (cls_best): [1.3879647, tensor(0.2595)] +Processing data/wiki/en-100/models/sp15k/qrnn_nl4.m + + +------ + + + + +$ python -m ulmfit eval --glob="wiki/*-100/models/sp15k/qrnn_nl4.m" --name nl4 --dataset-template='../mldoc/${lang}-1' --num-lm-epochs=20 --num-cls-epochs=8 --bs=18 --lr_sched=1cycle + +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁the', '▁,', 's', '▁.', '▁of', '▁and', '▁in', '▁to', '▁a', 'ed'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/en-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/en-100/models/sp15k/qrnn_nl4.m/../itos')] +epoch train_loss valid_loss accuracy +1 4.567319 3.820788 0.346054 +Total time: 02:26 +epoch train_loss valid_loss accuracy +1 3.889761 3.601505 0.374049 +2 3.570620 3.357854 0.406134 +3 3.389516 3.153452 0.432199 +4 3.217872 2.985234 0.452187 +5 3.063675 2.851744 0.468071 +6 3.023959 2.754062 0.480278 +7 2.907327 2.647027 0.493494 +8 2.786187 2.562560 0.505051 +9 2.737610 2.500068 0.513554 +10 2.696695 2.430095 0.523029 +11 2.658439 2.380829 0.530339 +12 2.598193 2.318927 0.539454 +13 2.558214 2.275014 0.546136 +14 2.520342 2.230543 0.553176 +15 2.475964 2.190341 0.559245 +16 2.370359 2.161100 0.564223 +17 2.430078 2.136685 0.568197 +18 2.383946 2.125458 0.569950 +19 2.389433 2.117541 0.571265 +20 2.297921 2.116168 0.571367 +Total time: 1:11:10 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k/qrnn_nl4.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.705876 0.241169 0.918000 +2 0.450546 0.239528 0.926000 +3 0.335179 0.221836 0.931000 +4 0.202048 0.208652 0.951000 +5 0.144956 0.223669 0.954000 +6 0.073117 0.277062 0.953000 +7 0.045186 0.258046 0.962000 +8 0.022987 0.265977 0.961000 +Total time: 02:33 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k/qrnn_nl4.m +Loss and accuracy using (cls_best): [0.29402012, tensor(0.9460)] +Processing data/wiki/es-100/models/sp15k/qrnn_nl4.m +../mldoc/es-1 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4.m +Evaluating previously trained model +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv +Data lm, trn: 13013, val: 1445 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Loss and accuracy using (cls_last): [0.19222946, tensor(0.9515)] +Processing data/wiki/fr-100/models/sp15k/qrnn_nl4.m +../mldoc/fr-1 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/fr.dev.csv +Running tokenization lm... +Data lm, trn: 13500, val: 1500 +Running tokenization cls... +Data cls, trn: 1000, val: 1000 +Running tokenization tst... +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', 's', '▁.', "'", '▁la', '▁le', '▁et', '▁l', '▁à'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp15k/qrnn_nl4.m/../itos')] +epoch train_loss valid_loss accuracy +1 3.375679 2.676224 0.454405 +Total time: 02:19 +epoch train_loss valid_loss accuracy +1 2.901917 2.540690 0.475910 +2 2.593614 2.370477 0.504601 +3 2.423170 2.205713 0.530328 +4 2.287688 2.083261 0.549087 +5 2.161118 1.984955 0.564804 +6 2.221017 1.912810 0.575434 +7 2.111272 1.837854 0.588076 +8 2.032289 1.775163 0.598341 +9 1.984161 1.720519 0.607980 +10 1.904775 1.668184 0.617407 +11 1.829098 1.621347 0.626292 +12 1.855409 1.577870 0.634512 +13 1.843696 1.536835 0.642584 +14 1.767968 1.496428 0.650317 +15 1.741591 1.463305 0.656908 +16 1.682118 1.438706 0.662601 +17 1.666425 1.418383 0.666283 +18 1.623713 1.406877 0.668710 +19 1.645482 1.401546 0.669716 +20 1.579352 1.399608 0.670167 +Total time: 1:06:50 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_nl4.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.614734 0.262120 0.906000 +2 0.377443 0.327852 0.917000 +3 0.296728 0.392655 0.903000 +4 0.179866 0.423420 0.928000 +5 0.114529 0.398973 0.935000 +6 0.082004 0.325470 0.944000 +7 0.047604 0.359636 0.945000 +8 0.032579 0.354014 0.944000 +Total time: 02:22 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_nl4.m +Loss and accuracy using (cls_best): [0.33020702, tensor(0.9450)] +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp15k/qrnn_nl4.m/../itos')] +epoch train_loss valid_loss accuracy +1 3.524609 2.582580 0.512131 +Total time: 02:34 +epoch train_loss valid_loss accuracy +1 2.656181 2.315530 0.550262 +2 2.265949 2.019140 0.598469 +3 1.985897 1.769565 0.638234 +4 1.831071 1.617888 0.660760 +5 1.735492 1.509642 0.677379 +6 1.637924 1.427618 0.690664 +7 1.564483 1.363384 0.700825 +8 1.508054 1.318165 0.708210 +9 1.471599 1.267787 0.716080 +10 1.398376 1.232899 0.722340 +11 1.311976 1.199602 0.728811 +12 1.401354 1.162299 0.735328 +13 1.385588 1.132408 0.740850 +14 1.256193 1.106556 0.745935 +15 1.289892 1.083529 0.750840 +16 1.220951 1.063360 0.754845 +17 1.259715 1.050884 0.757371 +18 1.165468 1.042870 0.759241 +19 1.242660 1.038160 0.760036 +20 1.194239 1.037506 0.760167 +Total time: 1:13:55 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k/qrnn_nl4.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.760780 0.391838 0.872000 +2 0.592674 0.427392 0.870000 +3 0.446265 0.593488 0.838000 +4 0.318780 0.605533 0.858000 +5 0.226914 0.665538 0.872000 +6 0.135525 0.742310 0.891000 +7 0.063984 0.778616 0.892000 +8 0.039366 0.827663 0.884000 +Total time: 02:49 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k/qrnn_nl4.m +Loss and accuracy using (cls_best): [0.70555997, tensor(0.8960)] +Processing data/wiki/zh-100/models/sp15k/qrnn_nl4.m +../mldoc/zh-1 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4.m +Evaluating previously trained model +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] +Loss and accuracy using (cls_last): [0.30052844, tensor(0.8982)] +OrderedDict([('data/mldoc/de-1/models/sp15k/qrnn_nl4.m', 0.2592499852180481), + ('data/mldoc/en-1/models/sp15k/qrnn_nl4.m', 0.9462500214576721), + ('data/mldoc/es-1/models/sp15k/qrnn_nl4.m', 0.9514999985694885), + ('data/mldoc/fr-1/models/sp15k/qrnn_nl4.m', 0.9442499876022339), + ('data/mldoc/ja-1/models/sp15k/qrnn_nl4.m', 0.8960000276565552), + ('data/mldoc/zh-1/models/sp15k/qrnn_nl4.m', 0.8982499837875366)]) + + + + + + + + + !! WARNING !! + + warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) +Single training schedule +epoch train_loss valid_loss accuracy +1 0.610785 0.239165 0.923000 +2 0.392899 0.281254 0.937000 +3 0.268695 0.444383 0.909000 +4 0.162150 0.427744 0.931000 +5 0.109248 0.422351 0.948000 +6 0.061984 0.411351 0.947000 +7 0.033645 0.413174 0.951000 +8 0.018704 0.404264 0.947000 +Total time: 02:16 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_nl4.m +Loss and accuracy using (cls_best): [0.29712877, tensor(0.9565)] +Processing data/wiki/en-100/models/sp15k/qrnn_nl4.m +../mldoc/en-1 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k/qrnn_nl4.m +Evaluating previously trained model +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/en.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁the', '▁,', 's', '▁.', '▁of', '▁and', '▁in', '▁to', '▁a', 'ed'] +Loss and accuracy using (cls_last): [0.29526812, tensor(0.9463)] +Processing data/wiki/es-100/models/sp15k/qrnn_nl4.m +../mldoc/es-1 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4.m +Evaluating previously trained model +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv +Data lm, trn: 13013, val: 1445 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Loss and accuracy using (cls_last): [0.19222946, tensor(0.9515)] +Processing data/wiki/fr-100/models/sp15k/qrnn_nl4.m +../mldoc/fr-1 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_nl4.m +Evaluating previously trained model +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/fr.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', 's', '▁.', "'", '▁la', '▁le', '▁et', '▁l', '▁à'] +Loss and accuracy using (cls_last): [0.33129737, tensor(0.9442)] +Processing data/wiki/it-100/models/sp15k/qrnn_nl4.m +../mldoc/it-1 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k/qrnn_nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/it.dev.csv +Running tokenization lm... +Data lm, trn: 13500, val: 1500 +Running tokenization cls... +Data cls, trn: 1000, val: 1000 +Running tokenization tst... +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', '▁e', "▁&'", "'", '▁il', '▁la', '▁in', 'e'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/it-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/it-100/models/sp15k/qrnn_nl4.m/../itos')] +epoch train_loss valid_loss accuracy +1 3.929510 2.916437 0.434550 +Total time: 01:20 +epoch train_loss valid_loss accuracy +1 3.231523 2.723415 0.461407 +2 2.823881 2.498599 0.496812 +3 2.586648 2.283846 0.530524 +4 2.417038 2.125690 0.553137 +5 2.278636 1.995558 0.572757 +6 2.199877 1.887804 0.589102 +7 2.090629 1.799082 0.603201 +8 2.046975 1.725273 0.615247 +9 1.935966 1.654829 0.626968 +10 1.921190 1.590797 0.638228 +11 1.894758 1.528087 0.649369 +12 1.792718 1.477532 0.658754 +13 1.679359 1.428426 0.668648 +14 1.723383 1.377170 0.678987 +15 1.597491 1.339658 0.686348 +16 1.620966 1.307664 0.692993 +17 1.568962 1.284500 0.697923 +18 1.533934 1.271438 0.700628 +19 1.496832 1.264714 0.701968 +20 1.486198 1.262870 0.702333 +Total time: 39:33 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k/qrnn_nl4.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.784124 0.414312 0.847000 +2 0.550873 0.413405 0.861000 +3 0.445371 0.363693 0.877000 +4 0.271702 0.426771 0.899000 +5 0.165902 0.556069 0.881000 +6 0.091403 0.628809 0.897000 +7 0.065516 0.693292 0.893000 +8 0.033616 0.675199 0.897000 +Total time: 01:24 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k/qrnn_nl4.m +Loss and accuracy using (cls_best): [0.7380945, tensor(0.8992)] +Processing data/wiki/ja-100/models/sp15k/qrnn_nl4.m +../mldoc/ja-1 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k/qrnn_nl4.m +Evaluating previously trained model +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)'] +Loss and accuracy using (cls_last): [0.7049702, tensor(0.8953)] +Processing data/wiki/zh-100/models/sp15k/qrnn_nl4.m +../mldoc/zh-1 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4.m +Evaluating previously trained model +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] +Loss and accuracy using (cls_last): [0.30052844, tensor(0.8982)] +OrderedDict([('data/mldoc/de-1/models/sp15k/qrnn_nl4.m', 0.9564999938011169), + ('data/mldoc/en-1/models/sp15k/qrnn_nl4.m', 0.9462500214576721), + ('data/mldoc/es-1/models/sp15k/qrnn_nl4.m', 0.9514999985694885), + ('data/mldoc/fr-1/models/sp15k/qrnn_nl4.m', 0.9442499876022339), + ('data/mldoc/it-1/models/sp15k/qrnn_nl4.m', 0.8992499709129333), + ('data/mldoc/ja-1/models/sp15k/qrnn_nl4.m', 0.8952500224113464), + ('data/mldoc/zh-1/models/sp15k/qrnn_nl4.m', 0.8982499837875366)]) + + + + + + +## DE +---------------------------------------- +Training issues + + +1/2nd -- That was without fine tuning !!! 0 shot:) +``` +python -m ulmfit load_cls data/mldoc/de-1/models/sp15k/qrnn_nl4.m --lang=de - train 0 --num-cls-epochs 8 --bs=18 --lr-sched=1cycle ✘ 1 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_None.m +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/de.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: + + !! WARNING !! + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +Your compiler (c++) may be ABI-incompatible with PyTorch! +Please use a compiler that is ABI-compatible with GCC 4.9 and above. +See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. + +See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 +for instructions on how to install GCC 4.9 or higher. +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + !! WARNING !! + + warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) +Single training schedule +epoch train_loss valid_loss accuracy +1 1.310368 1.177105 0.520000 +2 1.096284 0.899281 0.739000 +3 0.860910 0.668378 0.864000 +4 0.676764 0.733304 0.868000 +5 0.573360 0.590983 0.885000 +6 0.438448 0.446631 0.918000 +7 0.397323 0.531330 0.919000 +8 0.339557 0.437841 0.922000 +Total time: 02:25 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_None.m +Loss and accuracy using (cls_best): [0.3380329, tensor(0.9295)] +0.33803290128707886 +0.9294999837875366 +``` + +3rd aproach +``` +Single training schedule +epoch train_loss valid_loss accuracy +1 0.610785 0.239165 0.923000 +2 0.392899 0.281254 0.937000 +3 0.268695 0.444383 0.909000 +4 0.162150 0.427744 0.931000 +5 0.109248 0.422351 0.948000 +6 0.061984 0.411351 0.947000 +7 0.033645 0.413174 0.951000 +8 0.018704 0.404264 0.947000 +Total time: 02:16 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_nl4.m +Loss and accuracy using (cls_best): [0.29712877, tensor(0.9565)] +``` \ No newline at end of file diff --git a/results/logs/_experiment-log.md b/results/logs/_experiment-log.md new file mode 100644 index 0000000..0d0d650 --- /dev/null +++ b/results/logs/_experiment-log.md @@ -0,0 +1,57 @@ + + + +## QRNN sp15k +``` +cd fastai # go to fast ai +git checkout ulfit_multilingual +git pull + +cd ../ulmfit-multilingual # go to ulmfit +git checkout master +git pull + +export CUDA_VISIBLE_DEVICES=1 +LANG=fr +python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0 + +## Jeremy +export CUDA_VISIBLE_DEVICES=2 +LANG=it +python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0 + +export CUDA_VISIBLE_DEVICES=3 +LANG=ru +python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0 +``` + + + +``` +# +export CUDA_VISIBLE_DEVICES=0 +LANG=de +python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0 +# + +## Piotr +export CUDA_VISIBLE_DEVICES=1 +LANG=es +python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0 + +export CUDA_VISIBLE_DEVICES=0 +LANG=zh +python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0 +``` + +# trained +export CUDA_VISIBLE_DEVICES=0 +LANG=ja +python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0 + +done V100 +``` +export CUDA_VISIBLE_DEVICES=0 +LANG=en +python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0 +``` \ No newline at end of file diff --git a/results/logs/cls.md b/results/logs/cls.md new file mode 100644 index 0000000..bf45ebd --- /dev/null +++ b/results/logs/cls.md @@ -0,0 +1,172 @@ +# without col merge +```` +python -m ulmfit cls --dataset-path data/cls/${LANG}-books --base-lm-path data/wiki-m/${LANG}-100/models/sp30k/lstm_nl4.m --lang=${LANG} --name 'nl4' - train 20 --bs 20 --num-cls-epochs=8 --lr-sched=single +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k/lstm_nl4.m +Validation set not found using 10% of trn +Data lm, trn: 33183, val: 3687 +Data cls, trn: 1800, val: 200 +Running tokenization tst... +Data tst, trn: 200, val: 2000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki-m/fr-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki-m/fr-100/models/sp30k/lstm_nl4.m/../itos')] +epoch train_loss valid_loss accuracy +1 3.356221 2.821012 0.518492 +Total time: 00:21 +epoch train_loss valid_loss accuracy +1 3.041214 2.734799 0.524577 +2 2.919576 2.648412 0.535661 +3 2.822292 2.542236 0.549206 +4 2.721790 2.414110 0.561852 +5 2.596515 2.276732 0.579841 +6 2.453715 2.140479 0.600370 +7 2.333764 2.000186 0.621349 +8 2.231092 1.873927 0.644259 +9 2.101130 1.765473 0.660529 +10 2.006949 1.666797 0.682196 +11 1.905025 1.584023 0.696058 +12 1.820798 1.513958 0.709841 +13 1.751217 1.456632 0.720846 +14 1.689076 1.410359 0.729947 +15 1.646113 1.371438 0.739868 +16 1.594153 1.346142 0.744577 +17 1.564375 1.332298 0.746693 +18 1.536557 1.322925 0.748995 +19 1.532926 1.319159 0.749444 +20 1.525449 1.318028 0.749815 +Total time: 08:51 +/home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k/lstm_nl4.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.588118 0.581087 0.700000 +2 0.504373 0.583527 0.720000 +3 0.412651 0.538866 0.750000 +4 0.295401 0.658459 0.750000 +5 0.212442 1.054068 0.720000 +6 0.126090 1.302099 0.745000 +7 0.078312 1.307932 0.760000 +8 0.050346 1.339740 0.745000 +Total time: 00:35 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k/lstm_nl4.m +Loss and accuracy using (cls_best): [1.3513571, tensor(0.7700)] +1.351357102394104 +0.7699999809265137 +```` + +### FR books + +````bash +python -m ulmfit cls --dataset-path data/cls/${LANG}-books --base-lm-path data/wiki-m/${LANG}-100/models/sp30k/lstm_nl4.m --lang=${LANG} --name 'nl4' - train 20 --bs 20 --num-cls-epochs=8 --lr-sched=single +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k/lstm_nl4.m +Validation set not found using 10% of trn +Running tokenization lm... +Data lm, trn: 33183, val: 3687 +Running tokenization cls... +Data cls, trn: 1800, val: 200 +Running tokenization tst... +Data tst, trn: 200, val: 2000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki-m/fr-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki-m/fr-100/models/sp30k/lstm_nl4.m/../itos')] +epoch train_loss valid_loss accuracy +1 3.790325 3.409294 0.367234 +Total time: 06:02 +epoch train_loss valid_loss accuracy +1 3.526303 3.326439 0.378936 +2 3.466923 3.226977 0.392378 +3 3.342312 3.111874 0.406997 +4 3.244619 2.992510 0.422330 +5 3.156150 2.877498 0.437467 +6 3.070326 2.762509 0.453874 +7 2.956969 2.651613 0.471552 +8 2.878008 2.535935 0.491058 +9 2.790110 2.438724 0.508560 +10 2.684145 2.323467 0.528415 +11 2.633781 2.231418 0.547093 +12 2.535126 2.143523 0.564889 +13 2.464436 2.055402 0.582077 +14 2.330094 1.989257 0.596582 +15 2.372371 1.924338 0.610048 +16 2.190224 1.866912 0.621738 +17 2.176868 1.834098 0.629221 +18 2.168293 1.809196 0.633879 +19 2.151132 1.797144 0.636382 +20 2.130476 1.793351 0.637044 +Total time: 2:30:05 +/home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k/lstm_nl4.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.314315 0.530879 0.865000 +2 0.336746 0.468635 0.865000 +3 0.255810 0.324242 0.870000 +4 0.149121 0.480570 0.885000 +5 0.093909 0.613743 0.890000 +6 0.091678 0.660452 0.885000 +7 0.049993 0.649642 0.910000 +8 0.034218 0.640008 0.910000 +Total time: 04:19 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k/lstm_nl4.m +Loss and accuracy using (cls_best): [0.5418505, tensor(0.9100)] +0.5418505072593689 +0.9100000262260437 +```` + +``` +Loading pretrained model +Unknown tokens 0, first 100: [] +Training lm from: [PosixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp15k/qrnn_nl4.m/../itos')] +epoch train_loss valid_loss accuracy +1 4.037580 3.312217 0.364410 +Total time: 01:44 +epoch train_loss valid_loss accuracy +1 3.709982 3.256320 0.371825 +2 3.459413 3.150574 0.386972 +3 3.296628 3.037327 0.402039 +4 3.186458 2.914899 0.418413 +5 3.092632 2.817097 0.431216 +6 2.966957 2.726081 0.442906 +7 2.924824 2.647339 0.453871 +8 2.818279 2.561596 0.466795 +9 2.773893 2.501994 0.475877 +10 2.736084 2.438490 0.485978 +11 2.688937 2.370927 0.496899 +12 2.615245 2.314875 0.506508 +13 2.583292 2.260717 0.515725 +14 2.535631 2.220295 0.522666 +15 2.466035 2.179093 0.530148 +16 2.461427 2.151952 0.535315 +17 2.390641 2.131065 0.538749 +18 2.376235 2.116927 0.541430 +19 2.407630 2.115370 0.542039 +20 2.391378 2.112687 0.542522 +Total time: 46:33 +/home/n-waves/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp15k +Saving info /home/n-waves/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp15k/qrnn_nl4.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.466671 0.534682 0.745000 +2 0.358965 0.372612 0.875000 +3 0.251557 0.311034 0.900000 +4 0.166484 0.585425 0.865000 +5 0.101803 0.726341 0.900000 +6 0.072025 0.587875 0.885000 +7 0.045328 0.760989 0.890000 +8 0.027765 0.727203 0.890000 +Total time: 01:17 +Saving models at /home/n-waves/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp15k/qrnn_nl4.m +Loss and accuracy using (cls_best): [0.55982095, tensor(0.8970)] +0.5598209500312805 +0.8970000147819519 +``` \ No newline at end of file diff --git a/results/logs/common.md b/results/logs/common.md index 2f84604..1c23a0d 100644 --- a/results/logs/common.md +++ b/results/logs/common.md @@ -1,4 +1,503 @@ # MLDoc +### Different training schedules + +### 1cycle -lstm +``` +(fastaiv1) pczapla@galatea ~/w/ulmfit-multilingual ❯❯❯ python -m ulmfit eval --glob="mldoc/*-1/models/sp30k/lstm_nl4.m" --name nl4-1cyc --num-cls-epochs=8 --bs=18 --lr_sched=1cycle ✘ 1 +Processing data/mldoc/de-1/models/sp30k/lstm_nl4.m +de-1 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-1cyc.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/de.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-1cyc.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.610423 0.287707 0.920000 +2 0.390499 0.266688 0.948000 +3 0.366716 0.302463 0.933000 +4 0.248321 0.305547 0.937000 +5 0.166564 0.411075 0.948000 +6 0.083940 0.406182 0.950000 +7 0.033326 0.388105 0.949000 +8 0.014658 0.397507 0.948000 +Total time: 06:42 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-1cyc.m +Loss and accuracy using (cls_best): [0.3040595, tensor(0.9585)] +Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m +es-1 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-1cyc.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv +Data lm, trn: 13013, val: 1445 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-1cyc.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.610724 0.278892 0.925000 +2 0.372022 0.348428 0.937000 +3 0.310411 0.386958 0.927000 +4 0.215536 0.273834 0.958000 +5 0.163195 0.319600 0.958000 +6 0.085268 0.313287 0.961000 +7 0.037369 0.347500 0.961000 +8 0.016851 0.338436 0.963000 +Total time: 05:36 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-1cyc.m +Loss and accuracy using (cls_best): [0.31034237, tensor(0.9632)] +Processing data/mldoc/fr-1/models/sp30k/lstm_nl4.m +fr-1 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4-1cyc.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/fr.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4-1cyc.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.642809 0.240702 0.928000 +2 0.420564 0.658542 0.852000 +3 0.443345 0.244053 0.927000 +4 0.338779 0.335634 0.914000 +5 0.224778 0.263748 0.928000 +6 0.116705 0.280655 0.944000 +7 0.072063 0.287557 0.945000 +8 0.048084 0.289200 0.946000 +Total time: 06:33 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4-1cyc.m +Loss and accuracy using (cls_best): [0.29398218, tensor(0.9482)] +Processing data/mldoc/it-1/models/sp30k/lstm_nl4.m +it-1 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4-1cyc.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/it.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', "▁&'", "'", '▁e', '▁il', '▁la', 'e', '▁in'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4-1cyc.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.736070 0.391008 0.859000 +2 0.512531 0.614638 0.860000 +3 0.343422 0.594530 0.862000 +4 0.370786 0.540225 0.884000 +5 0.234727 0.591903 0.892000 +6 0.141539 0.589971 0.906000 +7 0.073315 0.544248 0.906000 +8 0.038477 0.580940 0.904000 +Total time: 03:48 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4-1cyc.m +Loss and accuracy using (cls_best): [0.6642357, tensor(0.8988)] +Processing data/mldoc/ja-1/models/sp30k/lstm_nl4.m +ja-1 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-1cyc.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-1cyc.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.804430 0.435780 0.837000 +2 0.576932 0.452903 0.839000 +3 0.499791 0.640789 0.806000 +4 0.418024 0.610898 0.839000 +5 0.259578 0.582953 0.868000 +6 0.188161 0.719131 0.888000 +7 0.101508 0.766175 0.877000 +8 0.072221 0.795415 0.883000 +Total time: 08:19 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-1cyc.m +Loss and accuracy using (cls_best): [0.636261, tensor(0.9045)] +Processing data/mldoc/ru-1/models/sp30k/lstm_nl4.m +ru-1 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4-1cyc.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv +Data lm, trn: 9195, val: 1021 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', '▁и', 'е', 'и', 'й', '▁на', 'х'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4-1cyc.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.821141 0.532432 0.814000 +2 0.611059 0.457023 0.866000 +3 0.464408 0.484035 0.870000 +4 0.446560 0.477454 0.858000 +5 0.299095 0.906959 0.858000 +6 0.176480 0.709579 0.875000 +7 0.089683 0.781081 0.876000 +8 0.047031 0.772935 0.877000 +Total time: 08:58 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4-1cyc.m +Loss and accuracy using (cls_best): [0.8528109, tensor(0.8795)] +Processing data/mldoc/zh-1/models/sp30k/lstm_nl4.m +zh-1 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.696346 0.335083 0.895000 +2 0.505075 0.360600 0.906000 +3 0.427076 0.462661 0.883000 +4 0.351177 0.489026 0.919000 +5 0.244958 0.415151 0.918000 +6 0.156452 0.494367 0.926000 +7 0.085807 0.471611 0.927000 +8 0.046887 0.484232 0.929000 +Total time: 06:41 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc.m +Loss and accuracy using (cls_best): [0.50999963, tensor(0.9165)] +OrderedDict([('data/mldoc/de-1/models/sp30k/lstm_nl4-1cyc.m', + 0.9585000276565552), + ('data/mldoc/es-1/models/sp30k/lstm_nl4-1cyc.m', + 0.9632499814033508), + ('data/mldoc/fr-1/models/sp30k/lstm_nl4-1cyc.m', + 0.9482499957084656), + ('data/mldoc/it-1/models/sp30k/lstm_nl4-1cyc.m', + 0.8987500071525574), + ('data/mldoc/ja-1/models/sp30k/lstm_nl4-1cyc.m', + 0.9045000076293945), + ('data/mldoc/ru-1/models/sp30k/lstm_nl4-1cyc.m', + 0.8794999718666077), + ('data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc.m', + 0.9164999723434448)]) +data/mldoc/de-1/models/sp30k/lstm_nl4-1cyc.m: 0.9585000276565552 +data/mldoc/es-1/models/sp30k/lstm_nl4-1cyc.m: 0.9632499814033508 +data/mldoc/fr-1/models/sp30k/lstm_nl4-1cyc.m: 0.9482499957084656 +data/mldoc/it-1/models/sp30k/lstm_nl4-1cyc.m: 0.8987500071525574 +data/mldoc/ja-1/models/sp30k/lstm_nl4-1cyc.m: 0.9045000076293945 +data/mldoc/ru-1/models/sp30k/lstm_nl4-1cyc.m: 0.8794999718666077 +data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc.m: 0.9164999723434448 +``` + + +### 2cycle +```bash +python -m ulmfit eval --glob="mldoc/*-1/models/sp15k/qrnn_nl4.m" --name nl4-8e-2cycle --num-cls-epochs=8 --bs=18 --lr_sched=2cycle +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-8e-2cycle.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv +Data lm, trn: 13013, val: 1445 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: + + !! WARNING !! + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +Your compiler (c++) may be ABI-incompatible with PyTorch! +Please use a compiler that is ABI-compatible with GCC 4.9 and above. +See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. + +See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 +for instructions on how to install GCC 4.9 or higher. +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + !! WARNING !! + + warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-8e-2cycle.m/info.json +2cycle training schedule +epoch train_loss valid_loss accuracy +1 0.600965 0.232749 0.937000 +Total time: 00:07 +epoch train_loss valid_loss accuracy +1 0.304946 0.202946 0.946000 +2 0.326092 0.207825 0.954000 +3 0.286274 0.290416 0.943000 +4 0.230937 0.263474 0.950000 +5 0.153293 0.293336 0.962000 +6 0.080219 0.328380 0.960000 +7 0.065156 0.343692 0.961000 +8 0.046342 0.367162 0.962000 +9 0.028884 0.396987 0.960000 +10 0.034997 0.366203 0.960000 +Total time: 02:27 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-8e-2cycle.m +Loss and accuracy using (cls_best): [0.35007542, tensor(0.9528)] +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-8e-2cycle.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-8e-2cycle.m/info.json +2cycle training schedule +epoch train_loss valid_loss accuracy +1 0.675992 0.460828 0.836000 +Total time: 00:09 +epoch train_loss valid_loss accuracy +1 0.439060 0.314642 0.888000 +2 0.382209 0.374305 0.893000 +3 0.338183 0.361669 0.911000 +4 0.260323 0.431681 0.901000 +5 0.146894 0.597865 0.899000 +6 0.090651 0.589435 0.910000 +7 0.079902 0.624589 0.918000 +8 0.043067 0.558498 0.918000 +9 0.022371 0.568702 0.921000 +10 0.022498 0.576052 0.922000 +Total time: 02:53 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-8e-2cycle.m +Loss and accuracy using (cls_best): [0.6146808, tensor(0.9150)] +OrderedDict([('data/mldoc/es-1/models/sp15k/qrnn_nl4-8e-2cycle.m', + 0.952750027179718), + ('data/mldoc/zh-1/models/sp15k/qrnn_nl4-8e-2cycle.m', + 0.9150000214576721)]) +``` + +### SIUNGLE 2epochs +``` +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-2e-single.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.626836 0.318460 0.912000 +2 0.386851 0.327937 0.918000 +Total time: 00:34 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-2e-single.m +Loss and accuracy using (cls_best): [0.32642558, tensor(0.9135)] +OrderedDict([('data/mldoc/es-1/models/sp15k/qrnn_nl4-2e-single.m', + 0.9539999961853027), + ('data/mldoc/zh-1/models/sp15k/qrnn_nl4-2e-single.m', + 0.9135000109672546)]) +``` +### SINGLE 4epochs +``` +OrderedDict([('data/mldoc/es-1/models/sp15k/qrnn_nl4-4e-single.m', + 0.9539999961853027), + ('data/mldoc/zh-1/models/sp15k/qrnn_nl4-4e-single.m', + 0.9210000038146973)]) +``` + + +### SINGLE 5 epochs +``` + python -m ulmfit eval --glob="mldoc/*-1/models/sp15k/qrnn_nl4.m" --name nl4-5e-single --num-cls-epochs=5 --bs=18 --lr_sched=single +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-5e-single.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv +Data lm, trn: 13013, val: 1445 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: + + !! WARNING !! + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +Your compiler (c++) may be ABI-incompatible with PyTorch! +Please use a compiler that is ABI-compatible with GCC 4.9 and above. +See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. + +See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 +for instructions on how to install GCC 4.9 or higher. +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + !! WARNING !! + + warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-5e-single.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.559638 0.250022 0.931000 +2 0.366238 0.348553 0.932000 +3 0.246830 0.243392 0.954000 +4 0.136335 0.242888 0.960000 +5 0.106189 0.254603 0.965000 +Total time: 01:14 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-5e-single.m +Loss and accuracy using (cls_best): [0.24189772, tensor(0.9588)] +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-5e-single.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-5e-single.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.641053 0.334561 0.891000 +2 0.486526 0.374095 0.894000 +3 0.309409 0.359222 0.908000 +4 0.179104 0.403468 0.920000 +5 0.083530 0.414904 0.919000 +Total time: 01:24 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-5e-single.m +Loss and accuracy using (cls_best): [0.44048822, tensor(0.9110)] +OrderedDict([('data/mldoc/es-1/models/sp15k/qrnn_nl4-5e-single.m', + 0.9587500095367432), + ('data/mldoc/zh-1/models/sp15k/qrnn_nl4-5e-single.m', + 0.9110000133514404)]) +``` + + +#### SINGLE 11epochs +``` +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-8e-single.m/info.json +Starting classifier training +Single training schedule +epoch train_loss valid_loss accuracy +1 0.676591 0.205210 0.947000 +2 0.402020 0.461279 0.912000 +3 0.287975 0.496294 0.921000 +4 0.258515 0.243489 0.954000 +5 0.219352 0.274136 0.949000 +6 0.149339 0.352294 0.956000 +7 0.092821 0.378696 0.962000 +8 0.055485 0.367379 0.963000 +9 0.042695 0.367151 0.964000 +10 0.034858 0.386749 0.961000 +11 0.021245 0.392899 0.963000 +Total time: 02:38 + +Loss and accuracy using (cls_last): [0.4098273, tensor(0.9595)] +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-8e-single.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-8e-single.m/info.json +Starting classifier training +Single training schedule +epoch train_loss valid_loss accuracy +1 0.710727 0.355552 0.895000 +2 0.523836 0.328691 0.895000 +3 0.408339 0.440722 0.894000 +4 0.326642 0.422076 0.909000 +5 0.217328 0.539624 0.906000 +6 0.156753 0.583433 0.912000 +7 0.102770 0.549827 0.921000 +8 0.053252 0.533528 0.928000 +9 0.032845 0.568053 0.927000 +10 0.022289 0.604236 0.926000 +11 0.015289 0.587667 0.929000 +Total time: 03:12 +OrderedDict([('data/mldoc/es-1/models/sp15k/qrnn_nl4-8e-single.m', + 0.9595000147819519), + ('data/mldoc/zh-1/models/sp15k/qrnn_nl4-8e-single.m', + 0.9202499985694885)]) + +``` + ## Limiit to 100 examples ``` python -m ulmfit eval --glob="mldoc/*-1/models/sp30k/lstm_nl4.m" --name nl4-100e8 --cuda-id=1 --limit=100 --num-cls-epochs=8 diff --git a/results/logs/de.md b/results/logs/de.md index dc44577..39a6e58 100644 --- a/results/logs/de.md +++ b/results/logs/de.md @@ -1,4 +1,28 @@ # DE +## SP15k LSTM nl4 +``` +$ python -m ulmfit lm --dataset-path data/wiki/de-100 --cuda-id=0 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang de --qrnn=False - train 10 --bs=100 --drop_mult=0 + +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', +'▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Training lm from random weights +epoch train_loss valid_loss accuracy +1 2.519809 2.600072 0.529963 +2 2.436580 2.538897 0.534651 +3 2.402220 2.510569 0.537314 +4 2.305741 2.439347 0.546574 +5 2.265683 2.376482 0.553794 +6 2.210663 2.305362 0.562672 +7 2.134196 2.230041 0.572958 +8 2.085375 2.150917 0.584621 +9 2.037781 2.097170 0.593747 +10 1.986773 2.081469 0.595799 +Total time: 19:18:33 +data/wiki/de-100/models/sp15k +Saving info data/wiki/de-100/models/sp15k/lstm_nl4.m/info.jso +``` + ## VF60k LSTM nl 3 ### LM ``` diff --git a/results/logs/es.md b/results/logs/es.md index 833014e..1e686cd 100644 --- a/results/logs/es.md +++ b/results/logs/es.md @@ -80,4 +80,190 @@ epoch train_loss valid_loss accuracy 2 0.262074 0.202975 0.945000 Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m Loss and accuracy using (cls_best): [0.1749019, tensor(0.9515)] -``` \ No newline at end of file +``` + + + +## ES optimization + + + +### Smaler vocab 15k +#### LM +``` +python -m ulmfit lm --dataset-path data/wiki-m/es-100 --cuda-id=0 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 \ ✘ 1 +--lang es --qrnn=False - train 10 --bs=50 --drop_mult=0 +Max vocab: 15000 +Cache dir: data/wiki-m/es-100/models/sp15k +Model dir: data/wiki-m/es-100/models/sp15k/lstm_nl4.m +Tokenized data loaded +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Training lm from random weights +epoch train_loss valid_loss accuracy +1 2.961702 3.187043 0.403149 +Better model found at epoch 1 with val_loss value: 3.1870434284210205. +2 2.928991 3.170802 0.402026 +Better model found at epoch 2 with val_loss value: 3.170802354812622. +3 2.931906 3.128328 0.407816 +Better model found at epoch 3 with val_loss value: 3.128328323364258. +4 2.869332 3.072160 0.414345 +Better model found at epoch 4 with val_loss value: 3.072160243988037. +5 2.803377 2.997071 0.424847 +Better model found at epoch 5 with val_loss value: 2.997070550918579. +6 2.758087 2.927369 0.432256 +Better model found at epoch 6 with val_loss value: 2.927368640899658. +7 2.657733 2.825029 0.446440 +Better model found at epoch 7 with val_loss value: 2.8250293731689453. +8 2.563273 2.728652 0.459271 +Better model found at epoch 8 with val_loss value: 2.7286524772644043. +9 2.475741 2.654844 0.470864 +Better model found at epoch 9 with val_loss value: 2.654844045639038. +10 2.428898 2.634355 0.474821 +Better model found at epoch 10 with val_loss value: 2.634355306625366. +Total time: 17:53:59 +data/wiki-m/es-100/models/sp15k +Saving info data/wiki-m/es-100/models/sp15k/lstm_nl4.m/info.json +``` + +#### MLDoc +``` + python -m ulmfit cls --dataset-path data/mldoc/es-1 --base-lm-path data/wiki-m/es-100/models/sp15k/lstm_nl4.m --lang=es --name 'nl4' --cuda-id=0 - train 20 --bs 20 --num-cls-epochs=8 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/lstm_nl4.m +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv +Running tokenization... +Saving tokenized: cls.trn 13013, cls.val 1445 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki-m/es-100/models/sp15k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki-m/es-100/models/sp15k/lstm_nl4.m/../itos')] +epoch train_loss valid_loss accuracy +1 2.355042 1.850417 0.589128 +Better model found at epoch 1 with val_loss value: 1.850416898727417. +Total time: 03:25 +epoch train_loss valid_loss accuracy +1 2.087364 1.677666 0.619909 +Better model found at epoch 1 with val_loss value: 1.6776657104492188. +2 1.880730 1.522996 0.648134 +Better model found at epoch 2 with val_loss value: 1.5229955911636353. +3 1.767530 1.403644 0.668117 +Better model found at epoch 3 with val_loss value: 1.4036436080932617. +4 1.659950 1.309353 0.684900 +Better model found at epoch 4 with val_loss value: 1.3093526363372803. +5 1.546585 1.232220 0.699358 +Better model found at epoch 5 with val_loss value: 1.2322196960449219. +6 1.592862 1.161846 0.713034 +Better model found at epoch 6 with val_loss value: 1.1618456840515137. +7 1.444965 1.098108 0.726811 +Better model found at epoch 7 with val_loss value: 1.0981075763702393. +8 1.340874 1.029193 0.741337 +Better model found at epoch 8 with val_loss value: 1.0291931629180908. +9 1.351407 0.974317 0.753408 +Better model found at epoch 9 with val_loss value: 0.9743167757987976. +10 1.231713 0.915328 0.767088 +Better model found at epoch 10 with val_loss value: 0.9153280854225159. +11 1.151926 0.852391 0.782414 +Better model found at epoch 11 with val_loss value: 0.852391242980957. +12 1.163565 0.794699 0.797228 +Better model found at epoch 12 with val_loss value: 0.7946987152099609. +13 1.054929 0.743652 0.810518 +Better model found at epoch 13 with val_loss value: 0.74365234375. +14 0.974651 0.695024 0.823344 +Better model found at epoch 14 with val_loss value: 0.6950243711471558. +15 0.869718 0.651691 0.834510 +Better model found at epoch 15 with val_loss value: 0.6516908407211304. +16 0.889763 0.615112 0.844947 +Better model found at epoch 16 with val_loss value: 0.6151121258735657. +17 0.843503 0.590130 0.851694 +Better model found at epoch 17 with val_loss value: 0.5901297926902771. +18 0.752870 0.575217 0.855496 +Better model found at epoch 18 with val_loss value: 0.5752172470092773. +19 0.807087 0.567187 0.857605 +Better model found at epoch 19 with val_loss value: 0.5671872496604919. +20 0.784531 0.566082 0.857827 +Better model found at epoch 20 with val_loss value: 0.5660821199417114. +Total time: 1:26:48 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/lstm_nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.616963 0.275447 0.923000 +Better model found at epoch 1 with val_loss value: 0.27544698119163513. +Total time: 00:24 +epoch train_loss valid_loss accuracy +1 0.425890 0.201535 0.932000 +Better model found at epoch 1 with val_loss value: 0.2015346735715866. +Total time: 00:27 +epoch train_loss valid_loss accuracy +1 0.265563 0.186434 0.951000 +Better model found at epoch 1 with val_loss value: 0.18643426895141602. +Total time: 00:32 +epoch train_loss valid_loss accuracy +1 0.162097 0.180026 0.955000 +Better model found at epoch 1 with val_loss value: 0.1800260841846466. +2 0.161748 0.187014 0.957000 +3 0.142789 0.166486 0.961000 +Better model found at epoch 3 with val_loss value: 0.1664857715368271. +4 0.105920 0.173207 0.963000 +5 0.078164 0.184849 0.962000 +6 0.070540 0.189451 0.962000 +7 0.051490 0.208019 0.959000 +8 0.047614 0.193699 0.962000 +Total time: 05:20 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/lstm_nl4.m +Loss and accuracy using (cls_best): [0.1623594, tensor(0.9538)] +0.16235940158367157 +0.9537500143051147 +``` + +### Larger dropout - no luck +``` +python -m ulmfit cls --dataset-path data/mldoc/es-1 --base-lm-path data/wiki-m/es-100/models/sp30k/lstm_nl4.m --lang=es --name 'nl4-drop' --cuda-id=0 - train 0 --bs 20 --num-cls-epochs=8 --drop-mul-lm=0.5 --drop-mul-cls=0.8 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-drop.m +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv +Running tokenization... +Saving tokenized: cls.trn 13013, cls.val 1445 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] +Starting classifier training +epoch train_loss valid_loss accuracy +1 1.310594 0.903076 0.724000 +Better model found at epoch 1 with val_loss value: 0.9030755758285522. +Total time: 00:22 +epoch train_loss valid_loss accuracy +1 1.209348 0.714140 0.756000 +Better model found at epoch 1 with val_loss value: 0.714139997959137. +Total time: 00:23 +epoch train_loss valid_loss accuracy +1 1.122636 0.627526 0.797000 +Better model found at epoch 1 with val_loss value: 0.6275263428688049. +Total time: 00:29 +epoch train_loss valid_loss accuracy +1 1.102433 0.593338 0.801000 +Better model found at epoch 1 with val_loss value: 0.5933384895324707. +2 1.096480 0.543266 0.818000 +Better model found at epoch 2 with val_loss value: 0.5432664155960083. +3 1.082919 0.501089 0.837000 +Better model found at epoch 3 with val_loss value: 0.5010889172554016. +4 1.069694 0.518807 0.812000 +5 1.040208 0.508399 0.825000 +6 1.032841 0.512187 0.838000 +7 1.031225 0.504557 0.825000 +8 1.016486 0.502335 0.837000 +Total time: 04:56 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-drop.m +Loss and accuracy using (cls_best): [0.52473265, tensor(0.8160)] +0.5247326493263245 +0.8159999847412109 +``` diff --git a/results/logs/fastai.errors.txt b/results/logs/fastai.errors.txt new file mode 100644 index 0000000..7464fb6 --- /dev/null +++ b/results/logs/fastai.errors.txt @@ -0,0 +1,76 @@ +python -m ulmfit eval --glob="mldoc/*-1/models/sp15k/qrnn_nl4.m" --name nl4-8e-single --num-cls-epochs=8 --bs=18 --single=True +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-8e-single.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv +Data lm, trn: 13013, val: 1445 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: + + !! WARNING !! + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +Your compiler (c++) may be ABI-incompatible with PyTorch! +Please use a compiler that is ABI-compatible with GCC 4.9 and above. +See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. + +See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 +for instructions on how to install GCC 4.9 or higher. +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + !! WARNING !! + + warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-8e-single.m/info.json +Starting classifier training +Single training schedule +epoch train_loss valid_loss accuracy +1 0.676591 0.205210 0.947000 +2 0.402020 0.461279 0.912000 +3 0.287975 0.496294 0.921000 +4 0.258515 0.243489 0.954000 +5 0.219352 0.274136 0.949000 +6 0.149339 0.352294 0.956000 +7 0.092821 0.378696 0.962000 +8 0.055485 0.367379 0.963000 +9 0.042695 0.367151 0.964000 +10 0.034858 0.386749 0.961000 +11 0.021245 0.392899 0.963000 +Total time: 02:38 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-8e-single.m +Traceback (most recent call last): + File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/runpy.py", line 193, in _run_module_as_main + "__main__", mod_spec) + File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/runpy.py", line 85, in _run_code + exec(code, run_globals) + File "/home/pczapla/workspace/ulmfit-multilingual/ulmfit/__main__.py", line 73, in + fire.Fire(ULMFiT()) + File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/fire/core.py", line 127, in Fire + component_trace = _Fire(component, args, context, name) + File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/fire/core.py", line 366, in _Fire + component, remaining_args) + File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/fire/core.py", line 542, in _CallCallable + result = fn(*varargs, **kwargs) + File "/home/pczapla/workspace/ulmfit-multilingual/ulmfit/__main__.py", line 55, in eval + results[key] = params.train_cls(num_lm_epochs=num_lm_epochs, **trn_params)[1] + File "/home/pczapla/workspace/ulmfit-multilingual/ulmfit/train_clas.py", line 76, in train_cls + return self.validate_cls('cls_best', bs=bs, data_tst=data_tst, learn=learn) + File "/home/pczapla/workspace/ulmfit-multilingual/ulmfit/train_clas.py", line 84, in validate_cls + learn.load(save_name) + File "/home/pczapla/workspace/_oss/fastai/fastai/fastai/basic_train.py", line 243, in load + if purge: self.purge(clear_opt=ifnone(with_opt, False)) + File "/home/pczapla/workspace/_oss/fastai/fastai/fastai/basic_train.py", line 293, in purge + self.opt = OptimWrapper.load_with_state_and_layer_group(state['opt'], self.layer_groups) + File "/home/pczapla/workspace/_oss/fastai/fastai/fastai/callback.py", line 130, in load_with_state_and_layer_group + res.load_state_dict(state['opt_state']) + File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/optim/optimizer.py", line 108, in load_state_dict + raise ValueError("loaded state dict contains a parameter group " +ValueError: loaded state dict contains a parameter group that doesn't match the size of optimizer's group \ No newline at end of file diff --git a/results/logs/fr.md b/results/logs/fr.md index ff87353..9b6c456 100644 --- a/results/logs/fr.md +++ b/results/logs/fr.md @@ -1,4 +1,37 @@ # FR + +## SP15k QRNN nl 4 +``` +python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0 +Max vocab: 15000 +Cache dir: data/wiki/fr-100/models/sp15k +Model dir: data/wiki/fr-100/models/sp15k/qrnn_nl4.m +Wiki text was split to 174227 articles +Wiki text was split to 491 articles +Running tokenization lm... +Data lm, trn: 174227, val: 491 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', 's', '▁.', "'", '▁la', '▁le', +'▁et', '▁l', '▁à'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Training lm from random weights +epoch train_loss valid_loss accuracy +1 2.881558 2.790402 0.465847 +2 2.824942 2.732005 0.471660 +3 2.758845 2.672040 0.478273 +4 2.715069 2.602380 0.489159 +5 2.677029 2.553575 0.494752 +6 2.602514 2.476142 0.507337 +7 2.564386 2.388670 0.518902 +8 2.470835 2.304033 0.532000 +9 2.366890 2.243269 0.542781 +10 2.390439 2.223538 0.546622 +Total time: 9:09:26 +data/wiki/fr-100/models/sp15k +Saving info data/wiki/fr-100/models/sp15k/qrnn_nl4.m/info.json + +``` + ## SP30k LSTM nl 4 ### LM ``` diff --git a/results/logs/it.md b/results/logs/it.md index 472e123..a0aa981 100644 --- a/results/logs/it.md +++ b/results/logs/it.md @@ -1,4 +1,28 @@ # FR +## SP15k QRNN NL4 +### LM + +``` +export CUDA_VISIBLE_DEVICES=0 +LANG=it +python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0 +epoch train_loss valid_loss accuracy +1 3.171145 3.516659 0.359233 +2 3.045057 3.472802 0.359628 +3 3.023009 3.401181 0.367101 +4 2.985105 3.351916 0.372709 +5 2.858441 3.280903 0.380848 +6 2.862504 3.210976 0.390263 +7 2.758775 3.122354 0.402106 +8 2.683234 3.035321 0.413798 +9 2.593757 2.964551 0.424886 +10 2.535500 2.947672 0.427958 +Total time: 11:30:03 +data/wiki/it-100/models/sp15k +Saving info data/wiki/it-100/models/sp15k/qrnn_nl4.m/info.json +``` +## xx + ## SP30k LSTM nl 4 ### LM ``` @@ -79,4 +103,4 @@ epoch train_loss valid_loss accuracy 2 0.340658 0.315946 0.877000 Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4bs100.m Loss and accuracy using (cls_best): [0.32998973, tensor(0.8842)] -``` \ No newline at end of file +``` diff --git a/results/logs/ja.md b/results/logs/ja.md index 8d8dd4a..3a81ca6 100644 --- a/results/logs/ja.md +++ b/results/logs/ja.md @@ -30,6 +30,8 @@ Saving info data/wiki/ja-100/models/sp30k/lstm_nl4.m/info.json ``` ### MLDoc + +#### CLS 1 MultiCCA 85.35%, ULMFiT 89.20% ``` python -m ulmfit cls --dataset-path data/mldoc/ja-1 --base-lm-path data/wiki/ja-100/models/sp30k/lstm_nl4.m --lang=ja --name 'nl4' --cuda-id=1 - train 20 --bs 40 --num-cls-epochs=8 @@ -89,6 +91,66 @@ epoch train_loss valid_loss accuracy 8 0.278896 0.358145 0.877000 Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m Loss and accuracy using (cls_best): [0.29789856, tensor(0.8920)] + + +$ mv /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4x8.m + +$ python -m ulmfit eval --glob="mldoc/ja-1/models/sp30k/lstm_nl4x8.m" --name nl4 --cuda-id=0 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Tokenized data loaded, cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4x8.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4x8.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.757615 0.562652 0.825000 +epoch train_loss valid_loss accuracy +1 0.609298 0.382412 0.870000 +epoch train_loss valid_loss accuracy +1 0.544682 0.379602 0.871000 +epoch train_loss valid_loss accuracy +1 0.435453 0.360421 0.885000 +2 0.426099 0.350480 0.885000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m +Loss and accuracy using (cls_best): [0.32738593, tensor(0.8905)] +OrderedDict([('data/mldoc/ja-1/models/sp30k/lstm_nl4.m', 0.890500009059906)]) + +python -m ulmfit eval --glob="mldoc/ja-1/models/sp30k/lstm_nl4x8.m" --name nl4x2 --cuda-id=0 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4x2.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Tokenized data loaded, cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4x8.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4x8.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4x2.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.778722 0.690746 0.805000 +epoch train_loss valid_loss accuracy +1 0.574789 0.386483 0.862000 +epoch train_loss valid_loss accuracy +1 0.518843 0.361983 0.869000 +epoch train_loss valid_loss accuracy +1 0.435260 0.350808 0.869000 +2 0.386701 0.352221 0.875000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4x2.m +Loss and accuracy using (cls_best): [0.31783763, tensor(0.8892)] +OrderedDict([('data/mldoc/ja-1/models/sp30k/lstm_nl4x2.m', 0.8892499804496765)]) + ``` ### JA on 100 elements @@ -150,4 +212,245 @@ epoch train_loss valid_loss accuracy 8 0.668421 0.984848 0.590000 Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-100.m Loss and accuracy using (cls_best): [0.81621724, tensor(0.7437)] -``` \ No newline at end of file +``` + + +### Japanese fixed sentence piece +``` +python -m ulmfit cls --dataset-path data/mldoc/ja-1 --base-lm-path data/mldoc/ja-1/models/bsp30k/lstm_nl4.m --lang=ja --name 'nl4' --cuda-id=0 - train 1 --bs 40 --num-cls-epochs=2 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Tokenized data loaded, cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/../itos')] +epoch train_loss valid_loss accuracy +1 1.222162 0.986234 0.765830 +epoch train_loss valid_loss accuracy +1 1.237291 0.983976 0.766659 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.759230 0.619306 0.826000 +epoch train_loss valid_loss accuracy +1 0.599281 0.423162 0.841000 +epoch train_loss valid_loss accuracy +1 0.485808 0.360609 0.869000 +epoch train_loss valid_loss accuracy +1 0.415960 0.390202 0.872000 +2 0.371651 0.365374 0.876000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m +Loss and accuracy using (cls_best): [0.330675, tensor(0.8873)] +0.33067500591278076 +0.8872500061988831 +(fastaiv1) pczapla@galatea ~/w/ulmfit-multilingual ❯❯❯ python -m ulmfit eval --glob="mldoc/ja-1/models/sp30k/lstm_nl4.m" --name nl4-2nd --cuda-id=0 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-2nd.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Tokenized data loaded, cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-2nd.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.844110 0.700549 0.743000 +epoch train_loss valid_loss accuracy +1 0.610796 0.400912 0.853000 +epoch train_loss valid_loss accuracy +1 0.449974 0.358793 0.870000 +epoch train_loss valid_loss accuracy +1 0.407609 0.397678 0.868000 +2 0.367383 0.373168 0.869000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-2nd.m +Loss and accuracy using (cls_best): [0.33044776, tensor(0.8863)] +OrderedDict([('data/mldoc/ja-1/models/sp30k/lstm_nl4-2nd.m', + 0.8862500190734863)]) +``` +``` +python -m ulmfit cls --dataset-path data/mldoc/ja-1 --base-lm-path data/mldoc/ja-1/models/bsp30k/lstm_nl4.m --lang=ja --name '2nd-nl4' --cuda-id=0 - train 1 --bs 40 --num-cls-epochs=2 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_2nd-nl4.m +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Tokenized data loaded, cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/../itos')] +epoch train_loss valid_loss accuracy +1 1.208774 0.984775 0.766183 +epoch train_loss valid_loss accuracy +1 1.198147 0.984786 0.766730 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_2nd-nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.735084 0.613895 0.803000 +epoch train_loss valid_loss accuracy +1 0.550159 0.406097 0.867000 +epoch train_loss valid_loss accuracy +1 0.468788 0.404081 0.862000 +epoch train_loss valid_loss accuracy +1 0.395969 0.380797 0.870000 +2 0.349470 0.386497 0.866000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_2nd-nl4.m +Loss and accuracy using (cls_best): [0.32550755, tensor(0.8857)] +0.3255075514316559 +0.8857499957084656 + +``` +```bash +python -m ulmfit cls --dataset-path data/mldoc/ja-1 --base-lm-path data/mldoc/ja-1/models/bsp30k/lstm_nl4.m --lang=ja --name '3nd-nl4' --cuda-id=0 - train 0 --bs 40 --num-cls-epochs=2 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_3nd-nl4.m +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Tokenized data loaded, cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_3nd-nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.777661 0.617013 0.786000 +epoch train_loss valid_loss accuracy +1 0.603897 0.388985 0.867000 +epoch train_loss valid_loss accuracy +1 0.510845 0.374942 0.874000 +epoch train_loss valid_loss accuracy +1 0.468642 0.379503 0.872000 +2 0.430415 0.365797 0.880000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_3nd-nl4.m +Loss and accuracy using (cls_best): [0.33084384, tensor(0.8882)] +0.33084383606910706 +0.8882499933242798 + +python -m ulmfit cls --dataset-path data/mldoc/ja-1 --base-lm-path data/mldoc/ja-1/models/bsp30k/lstm_nl4.m --lang=ja --tokenizer=bsp --name '3nd-nl4' --cuda-id=0 - train 0 --bs 40 --num-cls-epochs=2 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_3nd-nl4.m +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Tokenized data loaded, cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_3nd-nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.824216 0.604706 0.825000 +epoch train_loss valid_loss accuracy +1 0.606317 0.409647 0.854000 +epoch train_loss valid_loss accuracy +1 0.500782 0.381826 0.862000 +epoch train_loss valid_loss accuracy +1 0.403516 0.366863 0.866000 +2 0.394599 0.357580 0.874000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_3nd-nl4.m +Loss and accuracy using (cls_best): [0.32903105, tensor(0.8848)] +0.3290310502052307 +0.8847500085830688 + +python -m ulmfit cls --dataset-path data/mldoc/ja-1 --base-lm-path data/mldoc/ja-1/models/bsp30k/lstm_nl4.m --lang=ja --tokenizer=bsp --name '3nd-nl4' --cuda-id=0 - train 0 --bs 40 --num-cls-epochs=2 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_3nd-nl4.m +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Tokenized data loaded, cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] +Loading last classifier +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.460803 0.451998 0.855000 +epoch train_loss valid_loss accuracy +1 0.460069 0.421900 0.867000 +epoch train_loss valid_loss accuracy +1 0.361791 0.447982 0.859000 +epoch train_loss valid_loss accuracy +1 0.301233 0.404477 0.868000 +2 0.269350 0.406427 0.870000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_3nd-nl4.m +Loss and accuracy using (cls_best): [0.34159982, tensor(0.8925)] +0.34159982204437256 +0.8924999833106995 + +``` +## SP60k +``` +python -m ulmfit lm --dataset-path data/wiki/ja-100 --cuda-id=1 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 60000 \ +--lang ja --qrnn=False - train 10 --bs=50 --drop_mult=0 + +Running tokenization +Wiki text was split to 98375 articles +Wiki text was split to 138 articles +Size of vocabulary: 60000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': None, 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: [0.25 0.1 0.2 0.02 0.15] +Training lm from random weights +epoch train_loss valid_loss accuracy +1 3.557822 3.682454 0.366108 +2 3.377493 3.614226 0.369889 +3 3.391634 3.562171 0.377114 +4 3.328160 3.497388 0.385236 +5 3.290285 3.424971 0.394655 +6 3.159867 3.337317 0.407095 +7 3.139091 3.250999 0.417750 +8 3.103923 3.153146 0.433443 +9 2.979789 3.092179 0.443405 +10 2.984099 3.077171 0.446887 +data/wiki/ja-100/models/sp60k +Saving info data/wiki/ja-100/models/sp60k/lstm_nl4.m/info.json +``` +## MLDoc +````bash +python -m ulmfit cls --dataset-path data/mldoc/ja-1 --base-lm-path data/wiki/ja-100/models/sp60k/lstm_nl4.m --lang=ja --name 'nl4' --cuda-id=1 - train 20 --bs 40 --num-cls-epochs=2 +Max vocab: 60000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp60k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp60k/lstm_nl4.m +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 60000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp60k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp60k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp60k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp60k/lstm_nl4.m/../itos')] +epoch train_loss valid_loss accuracy +1 3.019972 2.548868 0.503754 +epoch train_loss valid_loss accuracy +1 2.643698 2.363415 0.532341 +2 2.403588 2.149524 0.567359 +3 2.218298 1.969651 0.597484 +4 2.059648 1.829897 0.619758 +5 1.941803 1.722339 0.636215 +6 1.862969 1.630191 0.650293 +7 1.796515 1.551929 0.663782 +8 1.727768 1.481659 0.675489 +9 1.667709 1.417764 0.687287 +10 1.606343 1.357994 0.697264 +11 1.553344 1.303901 0.707811 +12 1.539182 1.251784 0.718038 +Traceback (most recent call last): +```` \ No newline at end of file diff --git a/results/logs/label_smoothing.md b/results/logs/label_smoothing.md new file mode 100644 index 0000000..c52df1e --- /dev/null +++ b/results/logs/label_smoothing.md @@ -0,0 +1,223 @@ +# MLDoc + +## QRNN 15k + +Exec 1 +``` +python -m ulmfit eval --glob="mldoc/zh-1/models/sp15k/qrnn_nl4.m" --name nl4-1cyc-sl --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.1 +Processing data/mldoc/zh-1/models/sp15k/qrnn_nl4.m +zh-1 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.859153 0.767961 0.864000 +2 0.768888 0.775161 0.904000 +3 0.658956 0.685653 0.902000 +4 0.589073 0.618438 0.923000 +5 0.540008 0.622157 0.915000 +6 0.508080 0.606979 0.914000 +7 0.487228 0.599491 0.918000 +8 0.477516 0.602196 0.923000 +Total time: 02:18 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl.m +Loss and accuracy using (cls_best): [0.2829206, tensor(0.9205)] +OrderedDict([('data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl.m', + 0.9204999804496765)]) +data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl.m: 0.9204999804496765 +``` +Exec 2 +````python -m ulmfit eval --glob="mldoc/zh-1/models/sp15k/qrnn_nl4.m" --name nl4-1cyc-sl1 --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.1 +Processing data/mldoc/zh-1/models/sp15k/qrnn_nl4.m +zh-1 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl1.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl1.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.881513 0.712176 0.865000 +2 0.743687 0.665091 0.906000 +3 0.677436 0.687689 0.873000 +4 0.595139 0.626483 0.920000 +5 0.542732 0.600652 0.914000 +6 0.512080 0.597546 0.916000 +7 0.487021 0.597065 0.912000 +8 0.476598 0.596792 0.914000 +Total time: 02:20 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl1.m +Loss and accuracy using (cls_best): [0.29172945, tensor(0.9178)] +OrderedDict([('data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl1.m', + 0.9177500009536743)]) +data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl1.m: 0.9177500009536743 +```` +Exec 4 +```bash +python -m ulmfit eval --glob="mldoc/zh-1/models/sp15k/qrnn_nl4.m" --name nl4-1cyc-sl-e4 --num-cls-epochs=4 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.1 ✘ 130 +Processing data/mldoc/zh-1/models/sp15k/qrnn_nl4.m +zh-1 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.880415 0.677291 0.901000 +2 0.729670 0.659975 0.911000 +3 0.624817 0.603056 0.921000 +4 0.542027 0.601961 0.921000 +Total time: 01:08 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m +Loss and accuracy using (cls_best): [0.28558904, tensor(0.9222)] +OrderedDict([('data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m', + 0.922249972820282)]) +data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m: 0.922249972820282 +``` +## LSTM sp30k +### 0.1 +```bash + python -m ulmfit eval --glob="mldoc/zh-1/models/sp30k/lstm_nl4.m" --name nl4-1cyc-sl --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.1 +Processing data/mldoc/zh-1/models/sp30k/lstm_nl4.m +zh-1 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.870432 0.670671 0.882000 +2 0.754248 0.824157 0.895000 +3 0.654601 0.727428 0.885000 +4 0.602772 0.668668 0.901000 +5 0.542110 0.625137 0.903000 +6 0.506150 0.617842 0.913000 +7 0.480944 0.616885 0.912000 +8 0.472876 0.614381 0.911000 +Total time: 06:38 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl.m +Loss and accuracy using (cls_best): [0.2977172, tensor(0.9233)] +OrderedDict([('data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl.m', + 0.9232500195503235)]) +data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl.m: 0.9232500195503235 +``` +### 0.2 +```bash +python -m ulmfit eval --glob="mldoc/zh-1/models/sp30k/lstm_nl4.m" --name nl4-1cyc-sl2 --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.2 +Processing data/mldoc/zh-1/models/sp30k/lstm_nl4.m +zh-1 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl2.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl2.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.045619 0.908213 0.874000 +2 0.957379 0.857977 0.921000 +3 0.891791 0.852157 0.905000 +4 0.845289 0.849923 0.914000 +5 0.818228 0.848613 0.921000 +6 0.787021 0.840483 0.920000 +7 0.776123 0.844006 0.919000 +8 0.762384 0.857240 0.916000 +Total time: 06:33 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl2.m +Loss and accuracy using (cls_best): [0.40299156, tensor(0.9170)] +OrderedDict([('data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl2.m', + 0.9169999957084656)]) +data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl2.m: 0.9169999957084656 +``` +### 0.4 +```bash + python -m ulmfit eval --glob="mldoc/zh-1/models/sp30k/lstm_nl4.m" --name nl4-1cyc-sl4 --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.4 +Processing data/mldoc/zh-1/models/sp30k/lstm_nl4.m +zh-1 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl4.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 1000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl4.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.251581 1.183341 0.898000 +2 1.214358 1.201266 0.834000 +3 1.190343 1.165525 0.919000 +4 1.168018 1.172510 0.903000 +5 1.149965 1.161660 0.914000 +6 1.140140 1.161689 0.915000 +7 1.135877 1.159853 0.912000 +8 1.134425 1.160039 0.911000 +Total time: 06:34 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl4.m +Loss and accuracy using (cls_best): [0.64041936, tensor(0.9195)] +OrderedDict([('data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl4.m', + 0.9194999933242798)]) +data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl4.m: 0.9194999933242798 +``` \ No newline at end of file diff --git a/results/logs/noise/de10k-noise.md b/results/logs/noise/de10k-noise.md new file mode 100644 index 0000000..4708277 --- /dev/null +++ b/results/logs/noise/de10k-noise.md @@ -0,0 +1,1158 @@ +# Correct Val data +``` +python -m ulmfit eval_noise_resistance --lang=de --size=10 --prefix-name="val_" +Noise: 0 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_0.m +Evaluating previously trained model +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Data lm, trn: 13500, val: 1500 +Data cls, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: + + !! WARNING !! + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +Your compiler (c++) may be ABI-incompatible with PyTorch! +Please use a compiler that is ABI-compatible with GCC 4.9 and above. +See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. + +See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 +for instructions on how to install GCC 4.9 or higher. +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + !! WARNING !! + + warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) +Loss and accuracy using (cls_last): [0.3033199, tensor(0.9712)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_0.m', + 0.9712499976158142)]) +Noise: 5 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_5.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 500 examples, only 0.95 have correct labels +Added noise to 50 examples, only 0.95 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.05tv... +Data clsnoise0.05tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_5.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.436501 0.416959 0.909000 +2 0.460737 0.426361 0.904000 +3 0.360029 0.502090 0.907000 +4 0.362643 0.404165 0.916000 +5 0.283832 0.478411 0.911000 +6 0.224228 0.538951 0.915000 +7 0.159160 0.662829 0.909000 +8 0.096864 0.699768 0.910000 +Total time: 18:20 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_5.m +Loss and accuracy using (cls_best): [0.22873034, tensor(0.9565)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_5.m', + 0.9564999938011169)]) +Noise: 10 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_10.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 1000 examples, only 0.9 have correct labels +Added noise to 100 examples, only 0.9 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.1tv... +Data clsnoise0.1tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_10.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.608604 0.614120 0.851000 +2 0.563473 0.521960 0.858000 +3 0.534108 0.564487 0.867000 +4 0.473966 0.600789 0.870000 +5 0.450963 0.579453 0.869000 +6 0.365191 0.641545 0.864000 +7 0.282259 0.785328 0.850000 +8 0.227324 0.844709 0.848000 +Total time: 18:56 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_10.m +Loss and accuracy using (cls_best): [0.24939896, tensor(0.9435)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_10.m', + 0.9434999823570251)]) +Noise: 15 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_15.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 1500 examples, only 0.85 have correct labels +Added noise to 150 examples, only 0.85 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.15tv... +Data clsnoise0.15tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_15.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.679152 0.757799 0.816000 +2 0.715950 0.683697 0.810000 +3 0.694975 0.646247 0.817000 +4 0.629321 0.658788 0.818000 +5 0.588069 0.728769 0.805000 +6 0.517607 0.815471 0.796000 +7 0.402233 0.961845 0.781000 +8 0.318577 0.961705 0.779000 +Total time: 18:38 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_15.m +Loss and accuracy using (cls_best): [0.6598115, tensor(0.9300)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_15.m', + 0.9300000071525574)]) +Noise: 20 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_20.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 2000 examples, only 0.8 have correct labels +Added noise to 200 examples, only 0.8 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.2tv... +Data clsnoise0.2tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_20.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.814279 0.829818 0.755000 +2 0.824145 0.879503 0.763000 +3 0.778472 0.853143 0.766000 +4 0.782468 0.779830 0.762000 +5 0.700055 0.840979 0.766000 +6 0.600067 0.887506 0.748000 +7 0.477300 1.100056 0.734000 +8 0.372687 1.117019 0.729000 +Total time: 18:50 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_20.m +Loss and accuracy using (cls_best): [0.32372993, tensor(0.9053)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_20.m', + 0.9052500128746033)]) +Noise: 25 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_25.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 2500 examples, only 0.75 have correct labels +Added noise to 250 examples, only 0.75 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.25tv... +Data clsnoise0.25tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_25.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.930055 0.914462 0.709000 +2 0.875372 0.947374 0.723000 +3 0.903772 0.995556 0.716000 +4 0.830826 0.843742 0.727000 +5 0.761732 0.948633 0.719000 +6 0.652835 1.037753 0.698000 +7 0.522116 1.141807 0.687000 +8 0.456265 1.270369 0.675000 +Total time: 18:53 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_25.m +Loss and accuracy using (cls_best): [0.42412135, tensor(0.8655)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_25.m', + 0.8654999732971191)]) +Noise: 30 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_30.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 3000 examples, only 0.7 have correct labels +Added noise to 300 examples, only 0.7 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.3tv... +Data clsnoise0.3tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_30.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.979780 0.992838 0.647000 +2 0.964802 0.975392 0.650000 +3 0.984818 0.938703 0.664000 +4 0.947048 0.971088 0.667000 +5 0.835190 0.978763 0.651000 +6 0.779068 1.057726 0.654000 +7 0.640580 1.204921 0.617000 +8 0.581258 1.248827 0.613000 +Total time: 18:57 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_30.m +Loss and accuracy using (cls_best): [0.48816764, tensor(0.8470)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_30.m', + 0.847000002861023)]) +Noise: 35 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_35.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 3500 examples, only 0.65 have correct labels +Added noise to 350 examples, only 0.65 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.35tv... +Data clsnoise0.35tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_35.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.055006 0.998555 0.620000 +2 1.044137 0.994613 0.631000 +3 1.002715 1.070116 0.624000 +4 0.986447 1.054147 0.621000 +5 0.967057 1.016487 0.622000 +6 0.833950 1.251236 0.580000 +7 0.717729 1.236029 0.571000 +8 0.628723 1.342408 0.559000 +Total time: 19:03 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_35.m +Loss and accuracy using (cls_best): [0.6226422, tensor(0.7990)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_35.m', + 0.7990000247955322)]) +Noise: 40 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_40.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 4000 examples, only 0.6 have correct labels +Added noise to 400 examples, only 0.6 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.4tv... +Data clsnoise0.4tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_40.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.103033 1.086538 0.570000 +2 1.107361 1.056345 0.569000 +3 1.081129 1.063769 0.548000 +4 1.095896 1.061330 0.556000 +5 0.991429 1.102777 0.569000 +6 0.915104 1.178768 0.525000 +7 0.779928 1.327309 0.513000 +8 0.686467 1.426985 0.502000 +Total time: 18:50 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_40.m +Loss and accuracy using (cls_best): [0.8506142, tensor(0.7530)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_40.m', + 0.753000020980835)]) +Noise: 45 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_45.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 4500 examples, only 0.55 have correct labels +Added noise to 450 examples, only 0.55 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.45tv... +Data clsnoise0.45tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_45.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.140645 1.110783 0.512000 +2 1.136563 1.121161 0.535000 +3 1.129539 1.099606 0.509000 +4 1.108058 1.073747 0.524000 +5 1.048675 1.126527 0.504000 +6 0.956287 1.194883 0.479000 +7 0.818828 1.354711 0.461000 +8 0.758420 1.486676 0.448000 +Total time: 19:12 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_45.m +Loss and accuracy using (cls_best): [0.82442254, tensor(0.7000)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_45.m', + 0.699999988079071)]) +Noise: 50 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_50.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 5000 examples, only 0.5 have correct labels +Added noise to 500 examples, only 0.5 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.5tv... +Data clsnoise0.5tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_50.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.179953 1.145395 0.438000 +2 1.165212 1.175898 0.483000 +3 1.181145 1.127925 0.477000 +4 1.139963 1.151791 0.471000 +5 1.124215 1.130031 0.466000 +6 1.029675 1.185899 0.450000 +7 0.911210 1.320619 0.427000 +8 0.838111 1.383757 0.419000 +Total time: 18:22 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_50.m +Loss and accuracy using (cls_best): [0.9008743, tensor(0.6545)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_50.m', + 0.6545000076293945)]) +Noise: 55 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_55.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 5500 examples, only 0.45 have correct labels +Added noise to 550 examples, only 0.45 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.55tv... +Data clsnoise0.55tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_55.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.183246 1.166291 0.448000 +2 1.196840 1.163279 0.404000 +3 1.193226 1.180232 0.407000 +4 1.183507 1.153738 0.425000 +5 1.164359 1.166123 0.396000 +6 1.095628 1.207745 0.387000 +7 0.982455 1.293190 0.383000 +8 0.911601 1.383209 0.369000 +Total time: 19:04 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_55.m +Loss and accuracy using (cls_best): [1.0093645, tensor(0.6365)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_55.m', + 0.6365000009536743)]) +Noise: 60 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_60.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 6000 examples, only 0.4 have correct labels +Added noise to 600 examples, only 0.4 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.6tv... +Data clsnoise0.6tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_60.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.234308 1.216432 0.370000 +2 1.218607 1.166292 0.360000 +3 1.206626 1.179762 0.335000 +4 1.192636 1.164426 0.332000 +5 1.167782 1.226321 0.362000 +6 1.118646 1.225461 0.360000 +7 1.057706 1.246721 0.358000 +8 1.011610 1.281406 0.346000 +Total time: 18:43 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_60.m +Loss and accuracy using (cls_best): [1.0831424, tensor(0.4775)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_60.m', + 0.47749999165534973)]) +Noise: 65 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_65.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 6500 examples, only 0.35 have correct labels +Added noise to 650 examples, only 0.35 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.65tv... +Data clsnoise0.65tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_65.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.215925 1.200209 0.326000 +2 1.235970 1.173920 0.313000 +3 1.218506 1.251267 0.310000 +4 1.204741 1.182633 0.347000 +5 1.173637 1.171351 0.333000 +6 1.109750 1.226862 0.329000 +7 1.054864 1.330735 0.309000 +8 1.002966 1.325511 0.323000 +Total time: 18:20 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_65.m +Loss and accuracy using (cls_best): [1.2715616, tensor(0.3695)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_65.m', + 0.3695000112056732)]) +Noise: 70 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_70.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 7000 examples, only 0.3 have correct labels +Added noise to 700 examples, only 0.3 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.7tv... +Data clsnoise0.7tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_70.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.217860 1.219061 0.351000 +2 1.222316 1.191306 0.332000 +3 1.222368 1.254068 0.343000 +4 1.208153 1.216313 0.339000 +5 1.180702 1.166293 0.336000 +6 1.128917 1.211504 0.333000 +7 1.080312 1.271096 0.337000 +8 1.017895 1.335092 0.339000 +Total time: 18:47 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_70.m +Loss and accuracy using (cls_best): [1.4655445, tensor(0.2688)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_70.m', + 0.26875001192092896)]) +Noise: 75 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_75.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 7500 examples, only 0.25 have correct labels +Added noise to 750 examples, only 0.25 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.75tv... +Data clsnoise0.75tv, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_75.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.203315 1.147710 0.380000 +2 1.212273 1.190271 0.333000 +3 1.200970 1.208472 0.367000 +4 1.169665 1.165628 0.358000 +5 1.166296 1.213566 0.374000 +6 1.112285 1.462085 0.380000 +7 1.054748 1.266731 0.389000 +8 0.981918 1.692906 0.364000 +Total time: 18:34 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_75.m +Loss and accuracy using (cls_best): [1.957079, tensor(0.1252)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_75.m', + 0.12524999678134918)]) +{'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_0.m': 0.9712499976158142, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_5.m': 0.9564999938011169, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_10.m': 0.9434999823570251, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_15.m': 0.9300000071525574, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_20.m': 0.9052500128746033, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_25.m': 0.8654999732971191, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_30.m': 0.847000002861023, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_35.m': 0.7990000247955322, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_40.m': 0.753000020980835, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_45.m': 0.699999988079071, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_50.m': 0.6545000076293945, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_55.m': 0.6365000009536743, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_60.m': 0.47749999165534973, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_65.m': 0.3695000112056732, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_70.m': 0.26875001192092896, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_75.m': 0.12524999678134918} +``` +# Incorrect Val data + +``` python -m ulmfit eval_noise_resistance --size=10 +Noise: 0 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_0.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Running tokenization lm... +Data lm, trn: 13500, val: 1500 +Running tokenization cls... +Data cls, trn: 10000, val: 1000 +Running tokenization tst... +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: + + !! WARNING !! + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +Your compiler (c++) may be ABI-incompatible with PyTorch! +Please use a compiler that is ABI-compatible with GCC 4.9 and above. +See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. + +See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 +for instructions on how to install GCC 4.9 or higher. +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + !! WARNING !! + + warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_0.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.248545 0.179782 0.959000 +2 0.189914 0.262213 0.960000 +3 0.166565 0.489769 0.942000 +4 0.126547 0.252591 0.956000 +5 0.125812 0.265295 0.959000 +6 0.052031 0.339353 0.966000 +7 0.055669 0.452688 0.965000 +8 0.019922 0.439593 0.963000 +Total time: 18:52 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_0.m +Loss and accuracy using (cls_best): [0.3236656, tensor(0.9720)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_0.m', 0.972000002861023)]) +Noise: 5 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_5.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 500 examples, only 0.95 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.05... +Data clsnoise0.05, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_5.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.467763 0.204974 0.949000 +2 0.403194 0.166174 0.957000 +3 0.412454 0.176702 0.958000 +4 0.337252 0.957854 0.964000 +5 0.314917 0.179809 0.959000 +6 0.244516 0.202331 0.959000 +7 0.145377 0.225054 0.960000 +8 0.129216 0.239015 0.960000 +Total time: 19:15 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_5.m +Loss and accuracy using (cls_best): [0.18567306, tensor(0.9620)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_5.m', + 0.9620000123977661)]) +Noise: 10 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_10.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 1000 examples, only 0.9 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.1... +Data clsnoise0.1, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_10.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.611526 0.220041 0.950000 +2 0.523582 0.209251 0.959000 +3 0.566672 0.190530 0.960000 +4 0.530805 0.214880 0.949000 +5 0.470179 0.238882 0.954000 +6 0.368767 0.224209 0.950000 +7 0.276495 0.254708 0.942000 +8 0.242059 0.260761 0.940000 +Total time: 19:23 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_10.m +Loss and accuracy using (cls_best): [0.19178627, tensor(0.9557)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_10.m', + 0.9557499885559082)]) +Noise: 15 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_15.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 1500 examples, only 0.85 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.15... +Data clsnoise0.15, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_15.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.712348 0.224500 0.948000 +2 0.702935 0.188924 0.956000 +3 0.645956 0.224620 0.957000 +4 0.644018 0.248982 0.952000 +5 0.580399 0.255238 0.958000 +6 0.497618 0.297617 0.930000 +7 0.339819 0.272245 0.936000 +8 0.323421 0.297615 0.926000 +Total time: 18:50 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_15.m +Loss and accuracy using (cls_best): [0.25138617, tensor(0.9350)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_15.m', + 0.9350000023841858)]) +Noise: 20 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_20.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 2000 examples, only 0.8 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.2... +Data clsnoise0.2, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_20.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.821758 0.270898 0.954000 +2 0.831244 0.440837 0.923000 +3 0.787815 0.313425 0.962000 +4 0.730238 0.255505 0.958000 +5 0.748861 0.302285 0.959000 +6 0.603880 0.324315 0.933000 +7 0.539476 0.306479 0.931000 +8 0.431666 0.315723 0.916000 +Total time: 19:28 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_20.m +Loss and accuracy using (cls_best): [0.28679955, tensor(0.9202)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_20.m', + 0.9202499985694885)]) +Noise: 25 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_25.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 2500 examples, only 0.75 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.25... +Data clsnoise0.25, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_25.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.913877 0.325624 0.956000 +2 0.914296 0.380646 0.956000 +3 0.862887 0.400977 0.909000 +4 0.859236 0.291990 0.957000 +5 0.832637 0.447769 0.944000 +6 0.643751 0.319626 0.935000 +7 0.562953 0.444973 0.875000 +8 0.471118 0.437798 0.876000 +Total time: 19:29 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_25.m +Loss and accuracy using (cls_best): [0.53222567, tensor(0.8808)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_25.m', + 0.8807500004768372)]) +Noise: 30 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_30.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 3000 examples, only 0.7 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.3... +Data clsnoise0.3, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_30.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.967625 0.434347 0.921000 +2 1.015967 0.491977 0.927000 +3 0.943155 0.499572 0.935000 +4 0.953488 0.362500 0.951000 +5 0.859923 0.440134 0.944000 +6 0.761165 0.879942 0.894000 +7 0.662025 0.566052 0.857000 +8 0.572539 0.546405 0.826000 +Total time: 19:17 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_30.m +Loss and accuracy using (cls_best): [0.47183233, tensor(0.8540)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_30.m', + 0.8539999723434448)]) +Noise: 35 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_35.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 3500 examples, only 0.65 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.35... +Data clsnoise0.35, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_35.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.079700 0.486082 0.944000 +2 1.032614 0.470132 0.943000 +3 1.007776 0.460361 0.948000 +4 1.018879 0.476889 0.909000 +5 0.943363 1.618245 0.871000 +6 0.897375 0.604067 0.844000 +7 0.731791 10.002838 0.829000 +8 0.631117 6.527773 0.809000 +Total time: 19:26 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_35.m +Loss and accuracy using (cls_best): [0.79261243, tensor(0.8185)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_35.m', + 0.8184999823570251)]) +Noise: 40 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_40.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 4000 examples, only 0.6 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.4... +Data clsnoise0.4, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_40.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.105347 0.646775 0.915000 +2 1.121720 0.547566 0.943000 +3 1.084002 0.581386 0.949000 +4 1.058243 0.486947 0.953000 +5 1.020843 1.061957 0.945000 +6 0.944650 0.561371 0.872000 +7 0.852374 2.072736 0.815000 +8 0.745296 0.637481 0.807000 +Total time: 18:42 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_40.m +Loss and accuracy using (cls_best): [0.6664878, tensor(0.7915)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_40.m', + 0.7914999723434448)]) +Noise: 45 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_45.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 4500 examples, only 0.55 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.45... +Data clsnoise0.45, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_45.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.149571 0.715101 0.835000 +2 1.130510 0.812303 0.671000 +3 1.137401 0.566132 0.938000 +4 1.124969 0.654854 0.944000 +5 1.066340 0.624603 0.928000 +6 1.000705 0.700255 0.836000 +7 0.895356 0.726855 0.784000 +8 0.813315 0.740214 0.747000 +Total time: 19:12 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_45.m +Loss and accuracy using (cls_best): [0.74985105, tensor(0.7402)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_45.m', + 0.7402499914169312)]) +Noise: 50 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_50.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 5000 examples, only 0.5 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.5... +Data clsnoise0.5, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_50.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.176207 0.770225 0.894000 +2 1.146716 0.698875 0.916000 +3 1.159391 0.808971 0.908000 +4 1.148178 0.817309 0.715000 +5 1.142510 0.755769 0.872000 +6 1.068410 0.837932 0.839000 +7 0.985223 1.619010 0.714000 +8 0.900792 0.850288 0.672000 +Total time: 18:43 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_50.m +Loss and accuracy using (cls_best): [0.87151223, tensor(0.6888)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_50.m', + 0.6887500286102295)]) +Noise: 55 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_55.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 5500 examples, only 0.45 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.55... +Data clsnoise0.55, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_55.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.194526 0.829596 0.896000 +2 1.206947 0.800475 0.856000 +3 1.205839 0.832071 0.895000 +4 1.193005 8.321078 0.884000 +5 1.135684 3.162405 0.759000 +6 1.087508 0.866630 0.751000 +7 0.969481 0.909597 0.654000 +8 0.903410 1.040992 0.612000 +Total time: 18:41 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_55.m +Loss and accuracy using (cls_best): [0.944759, tensor(0.6118)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_55.m', + 0.6117500066757202)]) +Noise: 60 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_60.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 6000 examples, only 0.4 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.6... +Data clsnoise0.6, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_60.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.228728 1.235617 0.451000 +2 1.231521 1.103163 0.465000 +3 1.205319 1.067342 0.535000 +4 1.208439 1.171245 0.738000 +5 1.166922 1.071724 0.481000 +6 1.105134 0.913977 0.680000 +7 0.996566 1.203313 0.546000 +8 0.933625 1.092093 0.506000 +Total time: 18:30 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_60.m +Loss and accuracy using (cls_best): [1.0928471, tensor(0.5045)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_60.m', + 0.5044999718666077)]) +Noise: 65 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_65.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 6500 examples, only 0.35 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.65... +Data clsnoise0.65, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_65.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.233954 1.126451 0.312000 +2 1.228535 1.022103 0.514000 +3 1.223979 1.275958 0.040000 +4 1.201343 1.281353 0.233000 +5 1.174795 1.088655 0.714000 +6 1.137927 1.304352 0.430000 +7 1.061154 2.778949 0.450000 +8 0.992379 1.507618 0.409000 +Total time: 18:44 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_65.m +Loss and accuracy using (cls_best): [1.7004116, tensor(0.4027)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_65.m', + 0.4027499854564667)]) +Noise: 70 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_70.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 7000 examples, only 0.3 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.7... +Data clsnoise0.7, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_70.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.212556 1.284562 0.230000 +2 1.221429 1.494182 0.025000 +3 1.208761 1.601475 0.044000 +4 1.196318 1.222757 0.493000 +5 1.170390 1.235789 0.240000 +6 1.114086 1.487270 0.244000 +7 1.054812 1.470409 0.244000 +8 0.998996 1.476172 0.270000 +Total time: 19:12 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_70.m +Loss and accuracy using (cls_best): [1.4951355, tensor(0.2860)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_70.m', + 0.28600001335144043)]) +Noise: 75 +Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m +de-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_75.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv +Added noise to 7500 examples, only 0.25 have correct labels +Data lm, trn: 13500, val: 1500 +Running tokenization clsnoise0.75... +Data clsnoise0.75, trn: 10000, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_75.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.206901 1.635707 0.035000 +2 1.226015 1.508531 0.222000 +3 1.203663 1.570317 0.020000 +4 1.201733 1.404657 0.054000 +5 1.161620 1.456142 0.040000 +6 1.142974 1.518606 0.037000 +7 1.067800 1.744556 0.098000 +8 0.983552 1.720158 0.117000 +Total time: 18:41 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_75.m +Loss and accuracy using (cls_best): [1.7742158, tensor(0.1138)] +OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_75.m', + 0.11375000327825546)]) +{'data/mldoc/de-10/models/sp15k/qrnn_nl4_0.m': 0.972000002861023, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_5.m': 0.9620000123977661, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_10.m': 0.9557499885559082, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_15.m': 0.9350000023841858, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_20.m': 0.9202499985694885, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_25.m': 0.8807500004768372, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_30.m': 0.8539999723434448, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_35.m': 0.8184999823570251, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_40.m': 0.7914999723434448, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_45.m': 0.7402499914169312, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_50.m': 0.6887500286102295, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_55.m': 0.6117500066757202, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_60.m': 0.5044999718666077, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_65.m': 0.4027499854564667, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_70.m': 0.28600001335144043, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_75.m': 0.11375000327825546} +``` \ No newline at end of file diff --git a/results/logs/noise/es10k-noise.md b/results/logs/noise/es10k-noise.md new file mode 100644 index 0000000..e39e197 --- /dev/null +++ b/results/logs/noise/es10k-noise.md @@ -0,0 +1,1159 @@ +# Correct VAL +``` +python -m ulmfit eval_noise_resistance --lang=es --size=10 --prefix-name="val_" +Noise: 0 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_0.m +Evaluating previously trained model +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Data lm, trn: 13013, val: 1445 +Data cls, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: + + !! WARNING !! + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +Your compiler (c++) may be ABI-incompatible with PyTorch! +Please use a compiler that is ABI-compatible with GCC 4.9 and above. +See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. + +See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 +for instructions on how to install GCC 4.9 or higher. +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + !! WARNING !! + + warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) +Loss and accuracy using (cls_last): [0.38580656, tensor(0.9402)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_0.m', + 0.9402499794960022)]) +Noise: 5 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_5.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 472 examples, only 0.9500951575385916 have correct labels +Added noise to 50 examples, only 0.95 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.05tv... +Data clsnoise0.05tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_5.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.360209 0.664265 0.831000 +2 0.367246 0.505887 0.884000 +3 0.339675 0.646535 0.862000 +4 0.324250 0.914779 0.849000 +5 0.303626 0.592339 0.894000 +6 0.225317 0.699065 0.878000 +7 0.220272 0.726604 0.868000 +8 0.160895 0.767783 0.863000 +Total time: 11:24 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_5.m +Loss and accuracy using (cls_best): [0.36014587, tensor(0.9170)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_5.m', + 0.9169999957084656)]) +Noise: 10 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_10.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 945 examples, only 0.9000845844787482 have correct labels +Added noise to 100 examples, only 0.9 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.1tv... +Data clsnoise0.1tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_10.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.535271 0.740620 0.824000 +2 0.496433 1.023953 0.669000 +3 0.474564 0.993422 0.832000 +4 0.468398 6.249076 0.785000 +5 0.480843 0.850946 0.830000 +6 0.395878 0.789419 0.851000 +7 0.298389 2.299487 0.823000 +8 0.253688 2.279287 0.824000 +Total time: 11:21 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_10.m +Loss and accuracy using (cls_best): [0.871454, tensor(0.9075)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_10.m', + 0.9075000286102295)]) +Noise: 15 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_15.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 1418 examples, only 0.8500740114189046 have correct labels +Added noise to 150 examples, only 0.85 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.15tv... +Data clsnoise0.15tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_15.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.652501 1.015270 0.741000 +2 0.654432 1.025377 0.748000 +3 0.617786 1.639322 0.792000 +4 0.639355 1.662912 0.747000 +5 0.577110 5.244443 0.743000 +6 0.527992 3.701031 0.766000 +7 0.393420 3.620962 0.759000 +8 0.361106 3.019649 0.780000 +Total time: 11:31 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_15.m +Loss and accuracy using (cls_best): [0.35248652, tensor(0.9043)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_15.m', + 0.9042500257492065)]) +Noise: 20 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_20.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 1891 examples, only 0.8000634383590611 have correct labels +Added noise to 200 examples, only 0.8 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.2tv... +Data clsnoise0.2tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_20.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.749732 1.118057 0.710000 +2 0.785584 6.410913 0.607000 +3 0.713528 2.003799 0.754000 +4 0.752761 2.294566 0.724000 +5 0.652672 19.180836 0.708000 +6 0.596508 2.582183 0.711000 +7 0.489806 2.648639 0.712000 +8 0.467246 3.001610 0.698000 +Total time: 11:36 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_20.m +Loss and accuracy using (cls_best): [1.6113901, tensor(0.8475)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_20.m', + 0.8475000262260437)]) +Noise: 25 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_25.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 2364 examples, only 0.7500528652992176 have correct labels +Added noise to 250 examples, only 0.75 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.25tv... +Data clsnoise0.25tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_25.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.844129 1.047693 0.663000 +2 0.810037 1.353432 0.655000 +3 0.820446 3.722627 0.632000 +4 0.779205 1.445673 0.630000 +5 0.770405 1.024933 0.653000 +6 0.695584 1.400032 0.676000 +7 0.582116 1.612832 0.649000 +8 0.542117 1.422031 0.648000 +Total time: 11:48 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_25.m +Loss and accuracy using (cls_best): [0.62123287, tensor(0.8300)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_25.m', + 0.8299999833106995)]) +Noise: 30 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_30.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 2837 examples, only 0.7000422922393741 have correct labels +Added noise to 300 examples, only 0.7 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.3tv... +Data clsnoise0.3tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_30.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.946058 1.269022 0.625000 +2 0.942436 2.030650 0.636000 +3 0.920219 5.243942 0.463000 +4 0.890941 14.097425 0.605000 +5 0.835836 5.551986 0.617000 +6 0.708443 2.086009 0.624000 +7 0.639320 2.845402 0.582000 +8 0.537422 1.683678 0.597000 +Total time: 11:21 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_30.m +Loss and accuracy using (cls_best): [0.6909822, tensor(0.7915)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_30.m', + 0.7914999723434448)]) +Noise: 35 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_35.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 3310 examples, only 0.6500317191795305 have correct labels +Added noise to 350 examples, only 0.65 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.35tv... +Data clsnoise0.35tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_35.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.980721 1.266943 0.471000 +2 0.980537 1.438193 0.492000 +3 1.000082 5.081447 0.539000 +4 0.964535 1.640518 0.547000 +5 0.924568 2.078268 0.588000 +6 0.915998 1.612291 0.562000 +7 0.790721 1.804647 0.522000 +8 0.716465 2.185110 0.520000 +Total time: 11:48 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_35.m +Loss and accuracy using (cls_best): [1.8067851, tensor(0.7657)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_35.m', + 0.765749990940094)]) +Noise: 40 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_40.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 3783 examples, only 0.600021146119687 have correct labels +Added noise to 400 examples, only 0.6 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.4tv... +Data clsnoise0.4tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_40.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.031590 1.277689 0.458000 +2 1.056317 2.782609 0.349000 +3 1.044410 1.631716 0.449000 +4 1.034872 2.103323 0.478000 +5 0.966710 1.642946 0.472000 +6 0.943393 1.632070 0.490000 +7 0.856185 1.868247 0.479000 +8 0.780535 1.958063 0.483000 +Total time: 11:36 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_40.m +Loss and accuracy using (cls_best): [0.8928685, tensor(0.7508)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_40.m', + 0.7507500052452087)]) +Noise: 45 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_45.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 4256 examples, only 0.5500105730598435 have correct labels +Added noise to 450 examples, only 0.55 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.45tv... +Data clsnoise0.45tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_45.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.064314 1.302667 0.444000 +2 1.094542 2.054388 0.439000 +3 1.071187 2.209423 0.450000 +4 1.077246 6.365521 0.471000 +5 1.051128 3.242248 0.406000 +6 0.998142 2.802290 0.464000 +7 0.918943 2.492715 0.468000 +8 0.864835 8.349169 0.457000 +Total time: 11:39 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_45.m +Loss and accuracy using (cls_best): [6.738212, tensor(0.6977)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_45.m', + 0.6977499723434448)]) +Noise: 50 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_50.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 4729 examples, only 0.5 have correct labels +Added noise to 500 examples, only 0.5 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.5tv... +Data clsnoise0.5tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_50.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.101441 1.358480 0.450000 +2 1.150545 1.360492 0.392000 +3 1.120479 1.384937 0.437000 +4 1.110555 1.345856 0.401000 +5 1.089176 1.815834 0.420000 +6 1.051251 2.371094 0.422000 +7 1.004491 1.379894 0.410000 +8 0.936776 1.549806 0.424000 +Total time: 11:37 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_50.m +Loss and accuracy using (cls_best): [1.1773515, tensor(0.6180)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_50.m', + 0.6179999709129333)]) +Noise: 55 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_55.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 5201 examples, only 0.4500951575385917 have correct labels +Added noise to 550 examples, only 0.45 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.55tv... +Data clsnoise0.55tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_55.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.149922 1.401648 0.383000 +2 1.144570 9.186795 0.301000 +3 1.162765 1.583447 0.330000 +4 1.154295 1.568725 0.349000 +5 1.117640 1.624917 0.352000 +6 1.097049 2.510476 0.362000 +7 1.048231 3.798041 0.362000 +8 1.020844 3.568361 0.366000 +Total time: 11:25 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_55.m +Loss and accuracy using (cls_best): [1.6726145, tensor(0.6102)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_55.m', + 0.6102499961853027)]) +Noise: 60 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_60.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 5674 examples, only 0.4000845844787482 have correct labels +Added noise to 600 examples, only 0.4 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.6tv... +Data clsnoise0.6tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_60.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.166367 1.629363 0.358000 +2 1.175406 1.307703 0.341000 +3 1.172992 1.437346 0.342000 +4 1.163454 10.670442 0.339000 +5 1.158758 2.170272 0.325000 +6 1.122279 11.275146 0.351000 +7 1.080830 24.562853 0.359000 +8 1.055806 19.967308 0.354000 +Total time: 11:25 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_60.m +Loss and accuracy using (cls_best): [9.046943, tensor(0.4873)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_60.m', + 0.4872500002384186)]) +Noise: 65 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_65.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 6147 examples, only 0.35007401141890465 have correct labels +Added noise to 650 examples, only 0.35 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.65tv... +Data clsnoise0.65tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_65.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.188989 1.408018 0.326000 +2 1.182579 1.576379 0.279000 +3 1.180070 1.482971 0.257000 +4 1.164961 1.580750 0.283000 +5 1.166036 1.530756 0.294000 +6 1.136221 1.670132 0.297000 +7 1.102457 2.092621 0.291000 +8 1.058724 2.033105 0.287000 +Total time: 11:22 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_65.m +Loss and accuracy using (cls_best): [5.145201, tensor(0.4150)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_65.m', + 0.41499999165534973)]) +Noise: 70 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_70.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 6620 examples, only 0.30006343835906113 have correct labels +Added noise to 700 examples, only 0.3 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.7tv... +Data clsnoise0.7tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_70.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.181723 1.387360 0.342000 +2 1.179477 11.148307 0.343000 +3 1.178954 2.155812 0.293000 +4 1.177750 2.038787 0.308000 +5 1.156294 8.581575 0.319000 +6 1.122391 1.584792 0.346000 +7 1.095312 2.261911 0.346000 +8 1.048026 1.873787 0.337000 +Total time: 11:40 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_70.m +Loss and accuracy using (cls_best): [2.1144376, tensor(0.2125)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_70.m', + 0.21250000596046448)]) +Noise: 75 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_75.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 7093 examples, only 0.2500528652992176 have correct labels +Added noise to 750 examples, only 0.25 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.75tv... +Data clsnoise0.75tv, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_75.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.163953 1.370035 0.298000 +2 1.167424 1.883017 0.354000 +3 1.145917 1.854918 0.353000 +4 1.155910 1.328685 0.329000 +5 1.141541 1.895262 0.342000 +6 1.091663 22.743765 0.326000 +7 1.047826 50.595406 0.315000 +8 0.997792 39.199989 0.317000 +Total time: 11:30 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_75.m +Loss and accuracy using (cls_best): [38.87539, tensor(0.1922)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_75.m', + 0.19224999845027924)]) +{'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_0.m': 0.9402499794960022, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_5.m': 0.9169999957084656, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_10.m': 0.9075000286102295, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_15.m': 0.9042500257492065, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_20.m': 0.8475000262260437, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_25.m': 0.8299999833106995, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_30.m': 0.7914999723434448, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_35.m': 0.765749990940094, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_40.m': 0.7507500052452087, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_45.m': 0.6977499723434448, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_50.m': 0.6179999709129333, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_55.m': 0.6102499961853027, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_60.m': 0.4872500002384186, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_65.m': 0.41499999165534973, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_70.m': 0.21250000596046448, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_75.m': 0.19224999845027924} +``` + +# Incorrect Val data +``` +Noise: 0 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_0.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Running tokenization lm... +Data lm, trn: 13013, val: 1445 +Running tokenization cls... +Data cls, trn: 9458, val: 1000 +Running tokenization tst... +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: + + !! WARNING !! + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +Your compiler (c++) may be ABI-incompatible with PyTorch! +Please use a compiler that is ABI-compatible with GCC 4.9 and above. +See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. + +See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 +for instructions on how to install GCC 4.9 or higher. +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + !! WARNING !! + + warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_0.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.118179 0.265471 0.939000 +2 0.131938 0.823511 0.839000 +3 0.139424 0.496273 0.906000 +4 0.065336 0.315253 0.951000 +5 0.050494 0.366971 0.938000 +6 0.041825 0.296858 0.965000 +7 0.023676 0.330993 0.957000 +8 0.014899 0.337952 0.952000 +Total time: 11:14 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_0.m +Loss and accuracy using (cls_best): [0.37119418, tensor(0.9465)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_0.m', + 0.9465000033378601)]) +Noise: 5 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_5.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 472 examples, only 0.9500951575385916 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.05... +Data clsnoise0.05, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_5.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.381904 0.318090 0.893000 +2 0.366108 0.304496 0.907000 +3 0.350814 0.222635 0.954000 +4 0.305924 0.316391 0.937000 +5 0.294849 0.365258 0.938000 +6 0.226888 0.265155 0.953000 +7 0.203635 0.277920 0.944000 +8 0.180686 0.326215 0.935000 +Total time: 11:05 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_5.m +Loss and accuracy using (cls_best): [0.31352887, tensor(0.9302)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_5.m', + 0.9302499890327454)]) +Noise: 10 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_10.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 945 examples, only 0.9000845844787482 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.1... +Data clsnoise0.1, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_10.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.503315 0.294730 0.903000 +2 0.520918 0.404052 0.908000 +3 0.496451 0.623593 0.825000 +4 0.484413 0.212174 0.950000 +5 0.397562 0.271711 0.929000 +6 0.393162 0.455898 0.908000 +7 0.329386 0.306643 0.922000 +8 0.285580 0.283296 0.921000 +Total time: 11:17 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_10.m +Loss and accuracy using (cls_best): [0.2845364, tensor(0.9258)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_10.m', + 0.9257500171661377)]) +Noise: 15 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_15.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 1418 examples, only 0.8500740114189046 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.15... +Data clsnoise0.15, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_15.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.641261 0.416610 0.848000 +2 0.602479 0.356870 0.901000 +3 0.652983 0.582720 0.812000 +4 0.582735 22.428156 0.942000 +5 0.565184 7.965161 0.855000 +6 0.489513 0.751756 0.898000 +7 0.418973 6.047875 0.859000 +8 0.337963 9.058367 0.845000 +Total time: 11:23 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_15.m +Loss and accuracy using (cls_best): [11.41559, tensor(0.8332)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_15.m', + 0.8332499861717224)]) +Noise: 20 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_20.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 1891 examples, only 0.8000634383590611 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.2... +Data clsnoise0.2, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_20.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.735070 0.430566 0.875000 +2 0.777014 1.136241 0.711000 +3 0.762944 10.364647 0.624000 +4 0.698076 1.523277 0.806000 +5 0.646547 1.162918 0.833000 +6 0.581218 1.385464 0.869000 +7 0.489041 1.207447 0.843000 +8 0.398324 0.668090 0.849000 +Total time: 11:16 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_20.m +Loss and accuracy using (cls_best): [0.55489075, tensor(0.8595)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_20.m', + 0.859499990940094)]) +Noise: 25 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_25.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 2364 examples, only 0.7500528652992176 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.25... +Data clsnoise0.25, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_25.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.877402 0.428968 0.871000 +2 0.874461 0.585437 0.772000 +3 0.866958 0.598036 0.807000 +4 0.805568 0.964052 0.893000 +5 0.760956 5.249828 0.800000 +6 0.671527 1.038407 0.864000 +7 0.597481 0.767250 0.855000 +8 0.539228 0.695689 0.853000 +Total time: 11:29 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_25.m +Loss and accuracy using (cls_best): [0.61637276, tensor(0.8630)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_25.m', + 0.8629999756813049)]) +Noise: 30 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_30.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 2837 examples, only 0.7000422922393741 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.3... +Data clsnoise0.3, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_30.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.908243 0.480479 0.818000 +2 0.906832 0.562728 0.773000 +3 0.896891 3.017841 0.750000 +4 0.900984 2.464900 0.892000 +5 0.874264 2.142747 0.833000 +6 0.775662 0.491934 0.876000 +7 0.614158 0.568567 0.840000 +8 0.553789 3.695555 0.797000 +Total time: 11:32 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_30.m +Loss and accuracy using (cls_best): [2.999626, tensor(0.7897)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_30.m', + 0.7897499799728394)]) +Noise: 35 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_35.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 3310 examples, only 0.6500317191795305 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.35... +Data clsnoise0.35, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_35.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 0.983643 0.519147 0.891000 +2 1.010607 0.945973 0.580000 +3 0.994139 3.134296 0.461000 +4 0.984234 1.258144 0.864000 +5 0.923507 9.997900 0.759000 +6 0.886210 9.016973 0.667000 +7 0.791056 7.744195 0.798000 +8 0.683184 4.557703 0.787000 +Total time: 11:28 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_35.m +Loss and accuracy using (cls_best): [4.7776446, tensor(0.7642)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_35.m', + 0.7642499804496765)]) +Noise: 40 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_40.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 3783 examples, only 0.600021146119687 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.4... +Data clsnoise0.4, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_40.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.029997 0.882142 0.651000 +2 1.044314 1.591835 0.618000 +3 1.054599 0.522415 0.862000 +4 1.033360 0.998230 0.747000 +5 1.002742 3.083807 0.803000 +6 0.951022 11.693688 0.741000 +7 0.859333 4.469426 0.674000 +8 0.770237 1.737012 0.712000 +Total time: 11:26 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_40.m +Loss and accuracy using (cls_best): [1.721135, tensor(0.7135)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_40.m', + 0.7135000228881836)]) +Noise: 45 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_45.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 4256 examples, only 0.5500105730598435 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.45... +Data clsnoise0.45, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_45.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.118304 0.964211 0.720000 +2 1.105983 1.031464 0.459000 +3 1.090640 1.993658 0.545000 +4 1.094765 2.314941 0.550000 +5 1.075982 8.962537 0.682000 +6 1.044638 4.634834 0.743000 +7 0.997964 1.342650 0.711000 +8 0.953207 1.860142 0.728000 +Total time: 11:14 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_45.m +Loss and accuracy using (cls_best): [2.065772, tensor(0.7682)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_45.m', + 0.7682499885559082)]) +Noise: 50 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_50.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 4729 examples, only 0.5 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.5... +Data clsnoise0.5, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_50.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.131220 0.884647 0.677000 +2 1.118103 0.786608 0.760000 +3 1.135599 1.448700 0.654000 +4 1.131183 1.091670 0.655000 +5 1.087481 1.086206 0.755000 +6 1.071505 47.305462 0.752000 +7 1.008586 22.109758 0.724000 +8 0.978073 8.738451 0.704000 +Total time: 11:20 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_50.m +Loss and accuracy using (cls_best): [7.9533467, tensor(0.6925)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_50.m', + 0.6924999952316284)]) +Noise: 55 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_55.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 5201 examples, only 0.4500951575385917 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.55... +Data clsnoise0.55, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_55.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.161361 0.962813 0.596000 +2 1.150018 1.615359 0.542000 +3 1.154646 5.588564 0.572000 +4 1.145129 1.064145 0.541000 +5 1.138727 0.868817 0.716000 +6 1.092911 3.203760 0.673000 +7 1.027020 2.577705 0.615000 +8 0.962490 2.068990 0.607000 +Total time: 11:23 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_55.m +Loss and accuracy using (cls_best): [1.8640332, tensor(0.6400)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_55.m', + 0.6399999856948853)]) +Noise: 60 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_60.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 5674 examples, only 0.4000845844787482 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.6... +Data clsnoise0.6, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_60.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.184716 1.159816 0.489000 +2 1.162386 1.785291 0.585000 +3 1.189221 6.945411 0.523000 +4 1.169163 30.551344 0.553000 +5 1.143203 96.431221 0.434000 +6 1.113257 18.963854 0.475000 +7 1.083783 8.783161 0.521000 +8 1.064840 7.930975 0.462000 +Total time: 11:40 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_60.m +Loss and accuracy using (cls_best): [8.483881, tensor(0.4535)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_60.m', + 0.45350000262260437)]) +Noise: 65 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_65.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 6147 examples, only 0.35007401141890465 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.65... +Data clsnoise0.65, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_65.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.163443 1.302967 0.355000 +2 1.181459 1.265248 0.188000 +3 1.189486 6.507479 0.466000 +4 1.192368 23.974792 0.525000 +5 1.171117 2.516582 0.473000 +6 1.139664 1.357649 0.398000 +7 1.091899 1.510042 0.437000 +8 1.070406 1.511683 0.392000 +Total time: 11:29 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_65.m +Loss and accuracy using (cls_best): [1.5956299, tensor(0.3887)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_65.m', + 0.38874998688697815)]) +Noise: 70 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_70.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 6620 examples, only 0.30006343835906113 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.7... +Data clsnoise0.7, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_70.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.186880 1.400069 0.164000 +2 1.193253 3.277115 0.258000 +3 1.173004 2.955142 0.184000 +4 1.162564 3.158204 0.105000 +5 1.158516 8.930537 0.178000 +6 1.117573 25.560179 0.178000 +7 1.070933 3.045306 0.183000 +8 1.025959 10.724563 0.192000 +Total time: 11:31 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_70.m +Loss and accuracy using (cls_best): [9.275186, tensor(0.2153)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_70.m', + 0.21525000035762787)]) +Noise: 75 +Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m +es-10 +Max vocab: 15000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_75.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv +Added noise to 7093 examples, only 0.2500528652992176 have correct labels +Data lm, trn: 13013, val: 1445 +Running tokenization clsnoise0.75... +Data clsnoise0.75, trn: 9458, val: 1000 +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Loading pretrained model +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_75.m/info.json +Single training schedule +epoch train_loss valid_loss accuracy +1 1.173947 1.503179 0.149000 +2 1.155268 3.008709 0.140000 +3 1.166699 4.902722 0.181000 +4 1.160365 4.787449 0.190000 +5 1.138678 3.777317 0.084000 +6 1.071183 3.077753 0.123000 +7 1.000427 2.818838 0.166000 +8 0.941079 3.680126 0.187000 +Total time: 11:42 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_75.m +Loss and accuracy using (cls_best): [3.2255151, tensor(0.1877)] +OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_75.m', + 0.18774999678134918)]) +{'data/mldoc/es-10/models/sp15k/qrnn_nl4_0.m': 0.9465000033378601, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_5.m': 0.9302499890327454, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_10.m': 0.9257500171661377, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_15.m': 0.8332499861717224, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_20.m': 0.859499990940094, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_25.m': 0.8629999756813049, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_30.m': 0.7897499799728394, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_35.m': 0.7642499804496765, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_40.m': 0.7135000228881836, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_45.m': 0.7682499885559082, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_50.m': 0.6924999952316284, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_55.m': 0.6399999856948853, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_60.m': 0.45350000262260437, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_65.m': 0.38874998688697815, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_70.m': 0.21525000035762787, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_75.m': 0.18774999678134918} +```` \ No newline at end of file diff --git a/results/logs/qrnn-en.md b/results/logs/qrnn-en.md index 9c76223..2b0943b 100644 --- a/results/logs/qrnn-en.md +++ b/results/logs/qrnn-en.md @@ -1,4 +1,37 @@ # QRNN EN + +## SP15k nl 4 +``` +python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --m +ax-vocab 15000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0 +Max vocab: 15000 +Cache dir: data/wiki/en-100/models/sp15k +Model dir: data/wiki/en-100/models/sp15k/qrnn_nl4.m +Wiki text was split to 28476 articles +Wiki text was split to 60 articles +Data lm, trn: 28476, val: 60 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁the', '▁,', 's', '▁.', '▁of', '▁and', '▁in', '▁to', '▁a', 'ed'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +Training lm from random weights +epoch train_loss valid_loss accuracy +1 3.080874 3.197244 0.431796 +2 3.021043 3.147150 0.433593 +3 2.933366 3.125982 0.435766 +4 2.905764 3.103272 0.437356 +5 2.867981 3.032923 0.445030 +6 2.815294 2.958662 0.453979 +7 2.733671 2.869483 0.466015 +8 2.744779 2.785220 0.475833 +9 2.717722 2.704370 0.487687 +10 2.666089 2.675301 0.493602 +Total time: 9:07:27 +data/wiki/en-100/models/sp15k +Saving info data/wiki/en-100/models/sp15k/qrnn_nl4.m/info.json +``` + + + ## SP30k nl 4 ### LM diff --git a/results/logs/qrnn-es.md b/results/logs/qrnn-es.md index 835446c..5c3da86 100644 --- a/results/logs/qrnn-es.md +++ b/results/logs/qrnn-es.md @@ -1,5 +1,44 @@ # QRNN ES +## SP15k nl 4 +`` +export CUDA_VISIBLE_DEVICES=1 +LANG=es +python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0 + +Wiki text was split to 161509 articles +Wiki text was split to 78 articles +Running tokenization lm... +Data lm, trn: 161509, val: 78 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', '▁la', 's', '▁el', '▁en', '▁y', '▁a', "▁&'"] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: + +Training lm from random weights +epoch train_loss valid_loss accuracy +1 2.851575 3.398695 0.372940 +2 2.801543 3.353015 0.372648 +3 2.807216 3.290132 0.380787 +4 2.696361 3.220115 0.388937 +5 2.668488 3.132770 0.399528 +6 2.565685 3.062742 0.408880 +7 2.503054 2.985069 0.419262 +8 2.448338 2.895266 0.431797 +9 2.411213 2.829787 0.441973 +10 2.403536 2.811063 0.445468 +Total time: 11:52:32 +data/wiki/es-100/models/sp15k +Saving info data/wiki/es-100/models/sp15k/qrnn_nl4.m/info.json +`` + +```bash +export CUDA_VISIBLE_DEVICES=1 +LANG=es +python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/sp15k/qrnn_nl4.m --lang=${LANG} --name 'nl4' - train 20 --bs 20 --num-cls-epochs=8 +``` + + ## SP30k nl 4 ### LM ``` diff --git a/results/logs/qrnn-ru.md b/results/logs/qrnn-ru.md index 36ae8d4..a213dd6 100644 --- a/results/logs/qrnn-ru.md +++ b/results/logs/qrnn-ru.md @@ -1,4 +1,16 @@ # QRNN RU +## SP15k nl8 +data/wiki/ru-100/models/sp15k/qrnn_nl8.m + +export CUDA_VISIBLE_DEVICES=0 +LANG=ru +python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/sp15k/qrnn_nl8.m --lang=${LANG} --name 'nl8' - train 20 --bs 20 --num-cls-epochs=8 --lr_sched=1cycle + +export CUDA_VISIBLE_DEVICES=0 +LANG=de +python -m ulmfit cls --dataset-path data/mldoc-m/${LANG}-1 --base-lm-path data/wiki-m/${LANG}-100/models/sp15k/lstm_nl4.m --lang=${LANG} --name 'nl4' - train 20 --bs 20 --num-cls-epochs=8 --lr_sched=1cycle + + ## SP30k nl4 ### LM ``` diff --git a/results/logs/qrnn-zh.md b/results/logs/qrnn-zh.md new file mode 100644 index 0000000..ce7b125 --- /dev/null +++ b/results/logs/qrnn-zh.md @@ -0,0 +1,136 @@ + +# +``` + export CUDA_VISIBLE_DEVICES=0 +LANG=zh +python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0 + +Wiki text was split to 103929 articles +Wiki text was split to 113 articles +Running tokenization lm... +Data lm, trn: 103929, val: 113 +Size of vocabulary: 15000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} +/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: + + !! WARNING !! + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +Your compiler (c++) may be ABI-incompatible with PyTorch! +Please use a compiler that is ABI-compatible with GCC 4.9 and above. +See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. + +See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 +for instructions on how to install GCC 4.9 or higher. +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + !! WARNING !! + + warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) +Training lm from random weights +epoch train_loss valid_loss accuracy +1 2.489521 2.734049 0.482433 +2 2.427567 2.662464 0.488089 +3 2.415744 2.613971 0.494118 +4 2.334062 2.560180 0.501209 +5 2.343723 2.503271 0.507307 +6 2.260171 2.444533 0.516768 +7 2.198721 2.367407 0.526631 +8 2.161857 2.308182 0.535856 +9 2.142125 2.252678 0.544535 +10 2.087831 2.234440 0.548529 +Total time: 11:01:47 +data/wiki/zh-100/models/sp15k +Saving info data/wiki/zh-100/models/sp15k/qrnn_nl4.m/info.json +``` + +## MLDoc +```bash +export CUDA_VISIBLE_DEVICES=0 +LANG=zh +python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/sp15k/qrnn_nl4.m --lang=${LANG} --name 'nl4' - train 20 --bs 20 --num-cls-epochs=8 + +Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/zh-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/zh-100/models/sp15k/qrnn_nl4.m/../itos')] +epoch train_loss valid_loss accuracy +1 2.723684 2.148748 0.571206 +Total time: 02:13 +epoch train_loss valid_loss accuracy +1 2.157829 1.937637 0.601026 +2 1.898958 1.712967 0.637379 +3 1.722818 1.547745 0.664276 +4 1.570266 1.427551 0.682546 +5 1.503477 1.344690 0.696379 +6 1.434701 1.289549 0.704813 +7 1.425267 1.217570 0.717714 +8 1.373606 1.174655 0.725217 +9 1.297397 1.116406 0.735997 +10 1.211259 1.062999 0.745848 +11 1.248108 1.024482 0.754134 +12 1.198918 0.980273 0.762664 +13 1.121848 0.937985 0.771961 +14 1.111386 0.898821 0.780796 +15 1.120596 0.866009 0.787908 +16 1.056925 0.836998 0.794833 +17 1.020636 0.816387 0.799694 +18 1.002068 0.802623 0.802859 +19 0.998480 0.796877 0.804212 +20 0.959919 0.794685 0.804594 +Total time: 1:02:57 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.666322 0.433893 0.855000 +Total time: 00:08 +epoch train_loss valid_loss accuracy +1 0.448371 0.317440 0.889000 +Total time: 00:09 +epoch train_loss valid_loss accuracy +1 0.336693 0.309876 0.900000 +Total time: 00:10 +epoch train_loss valid_loss accuracy +1 0.266735 0.302003 0.903000 +2 0.222821 0.294501 0.905000 +3 0.207295 0.293751 0.908000 +4 0.179668 0.296945 0.911000 +5 0.153803 0.293158 0.911000 +Traceback (most recent call last): + File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/runpy.py", line 193, in _run_module_as_main + "__main__", mod_spec) + File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/runpy.py", line 85, in _run_code + exec(code, run_globals) + File "/home/pczapla/workspace/ulmfit-multilingual/ulmfit/__main__.py", line 73, in + fire.Fire(ULMFiT()) + File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/fire/core.py", line 127, in Fire + component_trace = _Fire(component, args, context, name) + File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/fire/core.py", line 366, in _Fire + component, remaining_args) + File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/fire/core.py", line 542, in _CallCallable + result = fn(*varargs, **kwargs) + File "/home/pczapla/workspace/ulmfit-multilingual/ulmfit/train_clas.py", line 54, in train_cls + learn.fit_one_cycle(num_cls_epochs, slice(1e-3 / (2.6 ** 4), 1e-3), moms=(0.8, 0.7)) + File "/home/pczapla/workspace/_oss/fastai/fastai/fastai/train.py", line 22, in fit_one_cycle + learn.fit(cyc_len, max_lr, wd=wd, callbacks=callbacks) + File "/home/pczapla/workspace/_oss/fastai/fastai/fastai/basic_train.py", line 178, in fit + callbacks=self.callbacks+callbacks) + File "/home/pczapla/workspace/_oss/fastai/fastai/fastai/utils/mem.py", line 77, in wrapper + return func(*args, **kwargs) + File "/home/pczapla/workspace/_oss/fastai/fastai/fastai/basic_train.py", line 90, in fit + loss = loss_batch(model, xb, yb, loss_func, opt, cb_handler) + File "/home/pczapla/workspace/_oss/fastai/fastai/fastai/basic_train.py", line 20, in loss_batch + out = model(*xb) + File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/nn/modules/module.py", line 477, in __call__ + result = self.forward(*input, **kwargs) + File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/nn/modules/container.py", line 92, in forward + input = module(input) + File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/nn/modules/module.py", line 477, in __call__ + result = self.forward(*input, **kwargs) + File "/home/pczapla/workspace/_oss/fastai/fastai/fastai/text/learner.py", line 235, in forward + return self.concat(raw_outputs), self.concat(outputs) + File "/home/pczapla/workspace/_oss/fastai/fastai/fastai/text/learner.py", line 221, in concat + return [torch.cat([l[si] for l in arrs], dim=1) for si in range_of(arrs[0])] + File "/home/pczapla/workspace/_oss/fastai/fastai/fastai/text/learner.py", line 221, in + return [torch.cat([l[si] for l in arrs], dim=1) for si in range_of(arrs[0])] +RuntimeError: CUDA error: out of memory +``` \ No newline at end of file diff --git a/results/logs/ru.md b/results/logs/ru.md index 04e5bdd..f309aa0 100644 --- a/results/logs/ru.md +++ b/results/logs/ru.md @@ -1,4 +1,24 @@ # RU +## SP15k nl4 +``` +Training lm from random weights +epoch train_loss valid_loss accuracy +1 3.053061 3.070487 0.450466 +2 2.874137 2.999093 0.455027 +3 2.864496 2.969308 0.458116 +4 2.890568 2.903564 0.466970 +5 2.746530 2.839789 0.474205 +6 2.683900 2.750476 0.486806 +7 2.674458 2.658535 0.499701 +8 2.595780 2.573735 0.512515 +9 2.530827 2.512999 0.522372 +10 2.505664 2.491850 0.526431 +Total time: 10:43:03 +data/wiki/ru-100/models/sp15k +Saving info data/wiki/ru-100/models/sp15k/qrnn_nl4.m/info.json +``` + + ## SP30k nl4 ### LM ``` @@ -21,7 +41,7 @@ epoch train_loss valid_loss accuracy data/wiki/ru-100/models/sp30k Saving info data/wiki/ru-100/models/sp30k/lstm_nl4.m/info.json ``` -### MLDoc +### MLDoc - bsp MultiCCA: 85.65% ulmfit: 87.27% ``` python -m ulmfit cls --dataset-path data/mldoc/ru-1 --base-lm-path data/wiki/ru-100/models/sp30k/lstm_nl4.m --lang=ru --name 'nl4-100' --cuda-id=1 - train 20 --bs 40 --num-cls-epochs=2 @@ -75,4 +95,69 @@ epoch train_loss valid_loss accuracy 2 0.417901 0.369961 0.882000 Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m Loss and accuracy using (cls_best): [0.38499942, tensor(0.8727)] +``` + +### MLDoc run 2x sp +``` +python -m ulmfit cls --dataset-path data/mldoc/ru-1 --base-lm-path data/wiki/ru-100/models/sp30k/lstm_nl4.m --lang=ru --name 'nl4' --cuda-id=1 - train 20 --bs 40 --num-cls-epochs=2 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv +Running tokenization... +Saving tokenized: cls.trn 9195, cls.val 1021 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', '▁и', 'е', 'и', 'й', '▁на', 'х'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp30k/lstm_nl4.m/../itos')] +epoch train_loss valid_loss accuracy +1 2.662225 2.284158 0.552927 +epoch train_loss valid_loss accuracy +1 2.436114 2.151187 0.574219 +2 2.260576 2.012279 0.595820 +3 2.067110 1.862512 0.620246 +4 2.000703 1.729883 0.641713 +5 1.860899 1.609955 0.661346 +6 1.751010 1.522195 0.676297 +7 1.705993 1.420628 0.694044 +8 1.592143 1.338552 0.708978 +9 1.524927 1.270614 0.722596 +10 1.475408 1.198585 0.736638 +11 1.438226 1.134858 0.749314 +12 1.408821 1.076875 0.761448 +13 1.345137 1.020660 0.773432 +14 1.321399 0.978076 0.783070 +15 1.235357 0.936674 0.791642 +16 1.204204 0.906822 0.798548 +17 1.198709 0.884949 0.803528 +18 1.176732 0.874523 0.805585 +19 1.111195 0.871806 0.806239 +20 1.031497 0.869280 0.806826 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.834704 0.615589 0.786000 +epoch train_loss valid_loss accuracy +1 0.679823 0.418461 0.851000 +epoch train_loss valid_loss accuracy +1 0.555612 0.426877 0.861000 +epoch train_loss valid_loss accuracy +1 0.468084 0.391777 0.873000 +2 0.434714 0.388670 0.882000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m +Loss and accuracy using (cls_best): [0.3987146, tensor(0.8680)] +0.3987146019935608 +0.8679999709129333 +``` + +``` +Second execution +epoch train_loss valid_loss accuracy +1 2.749340 2.284773 0.552775 +epoch train_loss valid_loss accuracy +1 2.418463 2.157943 0.572302 ``` \ No newline at end of file diff --git a/results/logs/zh.md b/results/logs/zh.md index c3760e2..c308f9d 100644 --- a/results/logs/zh.md +++ b/results/logs/zh.md @@ -1,5 +1,7 @@ # ZH +## SP15k QRNN + ## SP30k LSTM nl 4 ### LM ``` @@ -79,6 +81,8 @@ Loss and accuracy using (cls_best): [0.28411642, tensor(0.9020)] 0.9020000100135803 ``` + + ## SP60k LSTM nl 4 ### LM ``` diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index 266bc3f..127c8fa 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -39,7 +39,8 @@ def get_test_data(): copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.valid.tokens', n=600*sz) copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.test.tokens', n=600*sz) copy_head(imdb / 'train.csv', test_imdb / 'train.csv', n=10*sz) - copy_head(imdb / 'train.csv', test_imdb / 'test.csv', n=6*sz) + copy_head(imdb / 'train.csv', test_imdb / 'test.csv', n=6 * sz) + copy_head(imdb / 'train.csv', test_imdb / 'dev.csv', n=6 * sz) copy_head(imdb / 'train.csv', test_imdb / 'unsup.csv', n=1*sz) return test_data, test_wt @@ -109,12 +110,33 @@ def test_ulmfit_fastai_end_to_end(): qrnn=False, tokenizer='f', max_vocab=100, + nl=1, name=lm_name, ) exp.train_lm(num_epochs=1, bs=2) exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, ) +def test_ulmfit_fastai_end_to_end_label_smoothing(): + """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset. + """ + test_data, wt2 = get_test_data() + lm_name = 'end-to-end-test-fastai' + + exp = ulmfit.pretrain_lm.LMHyperParams( + dataset_path=wt2, + lang='en', + cuda_id=cuda_id, + qrnn=False, + tokenizer='f', + max_vocab=100, + name=lm_name, + ) + exp.train_lm(num_epochs=1, bs=2, label_smoothing_eps=0.1) + exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) + exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, label_smoothing_eps=0.1 ) + + def test_ulmfit_fastai_bidir_end_to_end(): """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset. """ diff --git a/ulmfit/__main__.py b/ulmfit/__main__.py index 72eb527..ffeabc1 100644 --- a/ulmfit/__main__.py +++ b/ulmfit/__main__.py @@ -1,10 +1,11 @@ import gc import os import pprint +import tarfile import shutil from collections import OrderedDict from functools import wraps - +import pandas as pd import fire from .pretrain_lm import LMHyperParams from .train_clas import CLSHyperParams @@ -25,7 +26,9 @@ def get_lang_from_dataset_path(ds): def get_dataset_path(p, dataset_template): ds = [x for x in p.parents if x.name == "models"][0].parent lang = get_lang_from_dataset_path(ds) - for ds_path in ds.parent.glob(Template(dataset_template).substitute(lang=lang, ds_name=ds.name)): + pattern = Template(dataset_template).substitute(lang=lang, ds_name=ds.name) + print(pattern) + for ds_path in ds.parent.glob(pattern): yield lang, ds_path class ULMFiT: @@ -41,22 +44,65 @@ class ULMFiT: params = CLSHyperParams.from_lm(dataset_path, base_lm_path, **changes) return FireView(train=params.train_cls, validate_cls=params.validate_cls) + @wraps(CLSHyperParams) + def load_cls(self, model_path, **changes): + params = CLSHyperParams.from_json(model_path, **changes) + return FireView(train=params.train_cls, validate_cls=params.validate_cls) + + + def eval_noise_resistance(self, lang="de", size=1, prefix_name="", model="sp15k/qrnn_nl4.m"): + def first_or_default(l, default=None): + l = list(l) + if l: + return l[0] + return default + results= [] + for noise in range(0, 80, 5): + print("Noise: ", noise) + d = self.eval(glob=f"mldoc/{lang}-1/models/{model}", + name=f"nl4_{prefix_name}{noise}", + noise=noise/100, + dataset_template='${lang}-'+str(size), + num_cls_epochs=8, + bs=18, + lr_sched="1cycle") + val = first_or_default(d.values(), default=-1) + results.append((noise/100, val)) + df = pd.DataFrame(results, columns=["noise", "accuracy"]) + df.to_csv(f"noise_{lang}-{size}{prefix_name}.csv") + print(df) + + def tar(self, model_path): + params = CLSHyperParams.from_json(model_path) + tar_name = f"models/{params.lang}-{params.tokenizer_prefix}-{params.model_name}.tar" + print("Storing model in", tar_name) + with tarfile.open(tar_name, mode="w") as tar: + for g in map(params.model_dir.glob, ['*_last.*', 'info.json', 'info.json', '../spm.*', '../itos.*',]): + for f in g: + print("Adding", f, f.relative_to("data")) + tar.add(f, f.relative_to("data")) + def eval(self, glob="mldoc/*-1/models/sp30k/lstm_nl4.m", dataset_template='${lang}-1', name="tmp-100", num_lm_epochs=0, cuda_id=0, **trn_params): results = OrderedDict() for base_model in sorted(Path("data").glob(glob)): + print("Processing", base_model) for lang, dataset_path in sorted(get_dataset_path(base_model, dataset_template)): - params = CLSHyperParams.from_lm(dataset_path, base_model, lang=lang, name=name, cuda_id=cuda_id) - key = str(params.model_dir.relative_to(Path.cwd())) - if (params.model_dir/"cls_last.pth").exists(): - print("Evaluating previously trained model") - results[key] = params.validate_cls()[1] - else: - print("Training") - results[key] = params.train_cls(num_lm_epochs=num_lm_epochs, **trn_params)[1] - del params + try: + params = CLSHyperParams.from_lm(dataset_path, base_model, lang=lang, name=name, cuda_id=cuda_id) + key = str(params.model_dir.relative_to(Path.cwd())) + if (params.model_dir/"cls_best.pth").exists(): + print("Evaluating previously trained model") + results[key] = params.validate_cls()[1] + else: + print("Training") + results[key] = params.train_cls(num_lm_epochs=num_lm_epochs, **trn_params)[1] + del params + except Exception as e: + print("Error", e) gc.collect() pprint.pprint(results) + return results def remove_lm_saves(self): for lm_save in Path("data").glob("**/lm_*.pth"): diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 812f707..0dfecaa 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -171,10 +171,10 @@ class LMHyperParams: with (self.model_dir / 'info.json').open("w") as fp: json.dump(vals, fp) print("Saving info", self.model_dir / 'info.json') - def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, lr=5e-3): + def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, lr=5e-3, label_smoothing_eps=0.0): self.model_dir.mkdir(exist_ok=True, parents=True) data_lm = self.load_wiki_data(bs=bs) if data_lm is None else data_lm - learn = self.create_lm_learner(data_lm, drop_mult=drop_mult) + learn = self.create_lm_learner(data_lm, drop_mult=drop_mult, label_smoothing_eps=label_smoothing_eps) learn.true_wd = true_wd if num_epochs > 0: @@ -204,7 +204,7 @@ class LMHyperParams: # do we need to return `learn'? it adds noise to Fire output #return learn - def create_lm_learner(self, data_lm, dps=None, **kwargs): + def create_lm_learner(self, data_lm, dps=None, label_smoothing_eps=0.0, **kwargs): assert self.bidir == False, "bidirectional model is not yet supported" config = dict(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, pad_token=PAD_TOKEN_ID, qrnn=self.qrnn, tie_weights=True, out_bias=True) @@ -230,6 +230,8 @@ class LMHyperParams: learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/lm-history"), # partial(SaveModelCallback, every='improvement', name='lm') disabled due to Memory issues ] + if label_smoothing_eps > 0.0: + learn.loss_func = FlattenedLoss(LabelSmoothingCrossEntropy, eps=label_smoothing_eps) return learn def load_train_text(self): @@ -297,8 +299,8 @@ class LMHyperParams: @classmethod def from_lm(cls, dataset_path, base_lm_path, **kwargs) -> 'LMHyperParams': - base_lm_path = Path(base_lm_path).resolve() dataset_path = Path(dataset_path).resolve() + base_lm_path = Path(base_lm_path).resolve() with open(base_lm_path/'info.json', 'r') as f: d = json.load(f) d['dataset_path'] = dataset_path d['base_lm_path'] = base_lm_path @@ -315,6 +317,26 @@ class LMHyperParams: d.update(kwargs) return cls(**d) + @classmethod + def from_json(cls, model_path:Path, **kwargs): + model_path = Path(model_path).resolve() + name = re.search(r"[a-z]+_(.+).m", model_path.name).group(1) + with open(model_path / 'info.json', 'r') as f: + d = json.load(f) + d.update(kwargs) + d['name'] = name + dataset_path = path_strip(model_path, "data", "models").parent + d['dataset_path'] = str(dataset_path) + d['lang'] = infer_lang_from_dataset(dataset_path.name) + return cls(**d) + +def infer_lang_from_dataset(name:str): + return name.split("-")[0] + +def path_strip(path, from_folder, to_folder): + to_p = [p for p in path.parents if p.name == to_folder][0] + from_p = [p for p in path.parents if p.name == from_folder][0] + return to_p.relative_to(from_p.parent) def validate_lm(self): if not self.exp.subword and self.exp.max_vocab is None: diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 7c3f980..bc04885 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -26,50 +26,75 @@ class CLSHyperParams(LMHyperParams): @property def need_fine_tune_lm(self): return not (self.model_dir/f"enc_best.pth").exists() - def train_cls(self, num_lm_epochs, unfreeze=True, num_cls_frozen_epochs=1, bs=40, true_wd=True, drop_mul_lm=0.3, drop_mul_cls=0.5, - use_test_for_validation=False, num_cls_epochs=2, limit=None, noise=0.0, cls_max_len=20*70): + def lr_schedule_layered(self, learn, num_cls_epochs): + learn.freeze_to(-1) + learn.fit_one_cycle(1, 2e-2, moms=(0.8, 0.7)) + if num_cls_epochs > 1: + learn.freeze_to(-2) + learn.fit_one_cycle(1, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7)) + learn.freeze_to(-3) + learn.fit_one_cycle(1, slice(5e-3 / (2.6 ** 4), 5e-3), moms=(0.8, 0.7)) + learn.unfreeze() + learn.fit_one_cycle(num_cls_epochs, slice(1e-3 / (2.6 ** 4), 1e-3), moms=(0.8, 0.7)) + + def lr_schedule_2cycle(self, learn, num_cls_epochs): + print("2cycle training schedule") + learn.freeze_to(-1) + learn.fit_one_cycle(1, 2e-2, moms=(0.8, 0.7)) + learn.unfreeze() + if num_cls_epochs > 1: + learn.fit_one_cycle(num_cls_epochs -1, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7)) + + def lr_schedule_1cycle(self, learn, num_cls_epochs): + print("Single training schedule") + learn.unfreeze() + learn.fit_one_cycle(num_cls_epochs, slice(1e-2 / (2.6 ** 4), 2e-2), moms=(0.8, 0.7)) + + def lr_schedule_false_wd(self, learn, num_cls_epochs): + learn.true_wd = False + print("Starting classifier training") + learn.fit_one_cycle(1, 5e-2, moms=(0.8, 0.7), wd=1e-7) + if num_cls_epochs > 1: + learn.freeze_to(-2) + learn.fit_one_cycle(1, slice(5e-2 / (2.6 ** 4), 5e-2), moms=(0.8, 0.7), wd=1e-7) + learn.freeze_to(-3) + learn.fit_one_cycle(1, slice(5e-4 / (2.6 ** 4), 5e-4), moms=(0.8, 0.7), wd=1e-7) + learn.unfreeze() + if num_cls_epochs > 5: + learn.fit_one_cycle(num_cls_epochs-4, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7), wd=1e-7) + + def train_cls(self, num_lm_epochs, unfreeze=True, num_cls_frozen_epochs=1, bs=40, drop_mul_lm=0.3, drop_mul_cls=0.5, + use_test_for_validation=False, num_cls_epochs=2, limit=None, noise=0.0, cls_max_len=20*70, lr_sched='layered', + label_smoothing_eps=0.0): assert use_test_for_validation == False, "use_test_for_validation=True is not supported" self.model_dir.mkdir(exist_ok=True, parents=True) + if not unfreeze: + num_cls_epochs = 1 + data_clas, data_lm, data_tst = self.load_cls_data(bs, limit=limit, noise=noise) - if self.need_fine_tune_lm: self.train_lm(num_lm_epochs, data_lm=data_lm, true_wd=true_wd, drop_mult=drop_mul_lm) - learn = self.create_cls_learner(data_clas, drop_mult=drop_mul_cls, max_len=cls_max_len) + if self.need_fine_tune_lm: self.train_lm(num_lm_epochs, data_lm=data_lm, drop_mult=drop_mul_lm, label_smoothing_eps=label_smoothing_eps) + learn = self.create_cls_learner(data_clas, drop_mult=drop_mul_cls, max_len=cls_max_len, label_smoothing_eps=label_smoothing_eps) try: - learn.load('cls_last') + learn.load('cls_best') print("Loading last classifier") except FileNotFoundError: learn.load_encoder(ENC_BEST) - if true_wd: + + if hasattr(self, 'lr_schedule_'+lr_sched): learn.true_wd = True - print("Starting classifier training") - learn.freeze_to(-1) - learn.fit_one_cycle(num_cls_frozen_epochs, 2e-2, moms=(0.8, 0.7)) - if unfreeze: - learn.freeze_to(-2) - learn.fit_one_cycle(1, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7)) - learn.freeze_to(-3) - learn.fit_one_cycle(1, slice(5e-3 / (2.6 ** 4), 5e-3), moms=(0.8, 0.7)) - learn.unfreeze() - learn.fit_one_cycle(num_cls_epochs, slice(1e-3 / (2.6 ** 4), 1e-3), moms=(0.8, 0.7)) + getattr(self, 'lr_schedule_'+lr_sched)(learn, num_cls_epochs) else: - learn.true_wd = False - print("Starting classifier training") - learn.fit_one_cycle(num_cls_frozen_epochs, 5e-2, moms=(0.8, 0.7), wd=1e-7) - if unfreeze: - learn.freeze_to(-2) - learn.fit_one_cycle(1, slice(5e-2 / (2.6 ** 4), 5e-2), moms=(0.8, 0.7), wd=1e-7) - learn.freeze_to(-3) - learn.fit_one_cycle(1, slice(5e-4 / (2.6 ** 4), 5e-4), moms=(0.8, 0.7), wd=1e-7) - learn.unfreeze() - learn.fit_one_cycle(num_cls_epochs, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7), wd=1e-7) + raise ValueError(f"Wrong lr_sched: {lr_sched}") + print(f"Saving models at {learn.path / learn.model_dir}") learn.save('cls_last', with_opt=False) learn.save('cls_best', with_opt=False) # we don't use early stopping for the time being + del learn + return self.validate_cls('cls_best', bs=bs, data_tst=data_tst, learn=None) - return self.validate_cls('cls_best', bs=bs, data_tst=data_tst, learn=learn) - - def validate_cls(self, save_name='cls_last', bs=40, data_tst=None, learn=None): + def validate_cls(self, save_name='cls_best', bs=40, data_tst=None, learn=None): if data_tst is None: _, _, data_tst = self.load_cls_data(bs) if learn is None: @@ -80,7 +105,7 @@ class CLSHyperParams(LMHyperParams): print(f"Loss and accuracy using ({save_name}):", results) return list(map(float, results)) - def create_cls_learner(self, data_clas, dps=None, **kwargs): + def create_cls_learner(self, data_clas, dps=None, label_smoothing_eps=0.0, **kwargs): assert self.bidir == False, "bidirectional model is not yet supported" config = dict(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, pad_token=PAD_TOKEN_ID, qrnn=self.qrnn) config.update(dps or self.dps) @@ -99,6 +124,8 @@ class CLSHyperParams(LMHyperParams): learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/cls-history"), #partial(SaveModelCallback, every='improvement', name='cls_best') disabled due to memory issues ] + if label_smoothing_eps > 0.0: + learn.loss_func = FlattenedLoss(LabelSmoothingCrossEntropy, eps=label_smoothing_eps) return learn def load_cls_data(self, bs, **kwargs): @@ -121,6 +148,16 @@ class CLSHyperParams(LMHyperParams): **kwargs) return self.databunches(bs, **data) + def merge_cols(self, df): + if len(df.columns) <= 2: + return df + ndf = df[[0,1]].copy() + for i in range(2, len(df.columns)): + ndf[1] += ("\n" + FLD + "\n") + df[i].fillna(" ") + + assert ndf[1].isna().sum().sum() == 0, f"You have NaN values in column(s) of your dataframe, please fix it." + return ndf + def load_data(self, lang='', **kwargs): prefix = '' if lang == '' else lang+'.' trn_df = pd.read_csv(self.dataset_path / f'{prefix}train.csv', header=None) @@ -139,10 +176,24 @@ class CLSHyperParams(LMHyperParams): val_len = max(int(len(trn_df) * 0.1), 2) trn_len = len(trn_df) - val_len trn_df, val_df = trn_df[:trn_len], trn_df[trn_len:] - + trn_df = self.merge_cols(trn_df) + val_df = self.merge_cols(val_df) + tst_df = self.merge_cols(tst_df) + unsup_df = self.merge_cols(unsup_df) kwargs.update(dict(trn_df=trn_df, val_df=val_df, tst_df=tst_df, unsup_df=unsup_df)) return kwargs + def add_noise(self, trn_df, noise): + count = len(trn_df) + labels = trn_df[0].unique() + assert np.issubdtype(labels.dtype, np.integer), "noise only works on numerical numbers" + modulo = labels.max() + 1 + idx_to_distrub = np.random.permutation(count)[:int(count * noise)] + trn_df.loc[idx_to_distrub, [0]] = (np.random.randint(1, modulo - 1, size=len(idx_to_distrub)) + + trn_df.loc[idx_to_distrub][0]) % modulo + print(f"Added noise to {len(idx_to_distrub)} examples, only {(count - len(idx_to_distrub)) / count} have correct labels") + return trn_df + def databunches(self, bs, trn_df, val_df, tst_df, unsup_df, add_trn_to_lm=True, use_moses=False, force=False, limit=None, noise=0.0): lm_trn_df = pd.concat([unsup_df, val_df, tst_df] + ([trn_df] if add_trn_to_lm else [])) val_len = max(int(len(lm_trn_df) * 0.1), 2) @@ -157,14 +208,9 @@ class CLSHyperParams(LMHyperParams): cls_name=f'{cls_name}limit{limit}' if noise > 0.0: - count = len(trn_df) - labels = trn_df[0].unique() - assert np.issubdtype(labels.dtype, np.integer), "noise only works on numerical numbers" - modulo = labels.max()+1 - idx_to_distrub = np.random.permutation(count)[:int(count * noise)] - trn_df.loc[idx_to_distrub, [0]] = (trn_df.loc[idx_to_distrub, [0]] + 1) % modulo - print(f"Added noise to {len(idx_to_distrub)} examples, only {(count-len(idx_to_distrub))/count} have correct labels") - cls_name = f'{cls_name}noise{noise}' + trn_df = self.add_noise(trn_df, noise) + val_df = self.add_noise(val_df, noise) + cls_name = f'{cls_name}noise{noise}tv' args = self.tokenizer_to_fastai_args(sp_data_func=lambda: trn_df[1], use_moses=use_moses) args['text_cols'] = list(trn_df.columns.values)[1:]