diff --git a/results/MLDoc.md b/results/MLDoc.md deleted file mode 100644 index b8a9ff1..0000000 --- a/results/MLDoc.md +++ /dev/null @@ -1,95 +0,0 @@ - -## Supervised classification results on MLDoc -| Model | en | de | es | fr | it | ja | ru | zh | -|----------------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|------------| -|LASER 0 shot | 80.75 | 87.03 | 82.60 | 82.83 | 73.25 | 60.95 | 68.83 | 72.90 | -|LASER | 90.73 | 92.70 | 88.75 | 90.80 | 85.93 | 85.15 | 84.65 | 88.98 | -|MultiCCA | 92.2 | 93.70 | 94.45 | 92.05 | 85.55 | 85.35 | 85.65 | 87.30 | -|Bert Multi | 93.23 | 94.0 | 95.15 | 93.20 | 85.82 | 87.48 | 86.85 | 90.72 | -|ULMFiT L30k-100 | | 91.35 | 83.32 | 88.77 | 77.99 | 71.12 | 72.20 | | -|ULMFiT L30k | | 95.4 | 95.15 | 93.67 | 88.42 | 89.20 | **87.27** | 90.20 | -|ULMFiT sp-fixed | | **95.6** | 94.80 | 94.20 | 88.52 | 88.72 | 86.85 | 90.47 | -|ULMFIT Q15k 1cyc| 94.62 | **95.65** | 95.15 | **94.42** | 89.92 | 89.60 | | 90.78/89.82 | -|ULMFIT Q15k 1c l| **94.99** | | 95.64 | 94.34 | **90.32** | 89.67 | 87.67^ | **92.22** | -|ULMFIT Q15k 1cfl| **95.55** | **96.10** | 95.82 | 94.80 | **90.04** | 89.87 | 87.17 | **91.90** | -|ULMFIT L30k 1cyc| | **95.85** | **96.32** | **94.82** | 89.87 | **90.45** | **87.94** | 92.02/91.64 | - -- L30k - LSTM sp30k trained using gradual unfreezing -- L30k-100 - --||-- **on 100 samples** -- ULMFiT sp-fixed - --||-- with fixed tokenization -- Q15k 1cyc - QRNN sp15k trained using 1cycle learning rate schedule -- L30k 1cyc - LSTM sp30k trained using 1cycle learning rate schedule -- We checked LSTM on sp15k on DE and got 95.53% accuracy which is comparable to QRNN sp15k -- ^ - 16 epochs qrnn_nl4sl-bs500 - -## Zero shot approaches - LSTM - -| Model | de | es | fr | it | ru | zh | -|----------------------|------------|------------|-----------|-----------|-----------|-----------| -| LASER-de | | 81.40 | 81.50 | 74.53 | 64.58 | 73.20 | -| LASER-fr | 88.75 | 80.12 | | 72.58 | 67.35 | 79.40 | -| LASER-en | 87.65 | 75.48 | 84.00 | 71.18 | 66.58 | 76.65 | -| | | | | | | | -| ULMFiT on LASER-de | | **85.50** | 87.37 | **78.75** | 66.95 | 72.32 | -| ULMFiT on LASER-fr | 92.22 | 81.00 | | 76.88 | 68.33 | **84.65** | -| ULMFiT on LASER-en | **92.95** | 80.50 | **88.78** | 76.20 | **70.05** | 80.45 | -| | | | | | | | -| % impr over LASER-de | | 22% | 32% | 17% | 7% | *-3%* | -| % impr over LASER-fr | 31% | 4% | | 16% | 3% | 25% | -| % impr over LASER-en | 43% | 20% | 30% | 17% | 10% | 16% | -| ULMFiT 100 for comp. | 91.35 | 83.32 | 88.77 | 77.99 | 71.12 | | -| | | | | | | | -| Bert Multilingual-EN | 74.50 | 61.85 | 69.77 | 57.73 | 51.10 | 64.08 | - - -### From Laser trained on French data -| Model Name | de | es | fr | it | ru | zh | -|---------------------------|-------|-------|----|-------|-------|-------| -| LASER fr 10k | 91.65 | 81.05 | | 75.08 | 70.73 | 76.33 | -| LASER fr 1k | 88.75 | 80.12 | | 72.58 | 67.35 | 79.4 | -| ULMFiT 10k on LASER-fr10k | 94.48 | 84.10 | | 77.93 | 72.87 | 84.53 | -| ULMFiT 10k on LASER-fr1k | 92.30 | 82.10 | | 75.52 | 69.52 | 85.55 | -| ULMFiT 1k on LASER-fr1k | 92.22 | 81.00 | | 76.88 | 68.33 | 84.65 | -| | | | | | | | -| Impr 10k over 10k | 34% | 16% | | 11% | 7% | 35% | -| Impr 10k over 1k | 32% | 10% | | 11% | 7% | 30% | -| Impr 1k over 1k | 31% | 4% | | 16% | 3% | 25% | - -### From Laser trained on German data -| Model Name | de | es | fr | it | ru | zh | -|---------------------------|----|-------|-------|-------|-------|-------| -| LASER de 10k | | 83.5 | 82.85 | 76.6 | 68.8 | 73.12 | -| LASER de 1k | | 81.4 | 81.5 | 74.53 | 64.58 | 73.2 | -| ULMFiT 10k on LASER-de10k | | 86.92 | 87.17 | 79.35 | 70.15 | 78.15 | -| ULMFiT 10k on LASER-de1k | | 84.65 | 87.48 | 78.70 | 67.65 | 77.50 | -| ULMFiT 1k on LASER-de1k | | 85.5 | 87.37 | 78.75 | 66.95 | 72.32 | -| | | | | | | | -| Impr 10k over 10k | | 21% | 25% | 12% | 4% | 19% | -| Impr 10k over 1k | | 17% | 32% | 16% | 9% | 16% | -| Impr 1k over 1k | | 22% | 32% | 17% | 7% | -3% | - -### From Laser trained on English data -| Model Name | de | es | fr | it | ru | zh | -|---------------------------|-------|-------|-------|-------|-------|-------| -| LASER en 10k | 87.43 | 77.38 | 78.7 | 72.53 | 67.7 | 75.18 | -| LASER en 1k | 87.65 | 75.48 | 84 | 71.18 | 66.58 | 76.65 | -| ULMFiT 10k on LASER-en10k | 92.05 | 80.05 | 86.95 | 76.65 | 70.57 | 80.85 | -| ULMFiT 10k on LASER-en1k | 91.80 | 80.10 | 88.67 | 77.32 | 70.25 | 82.73 | -| ULMFiT 1k on LASER-en1k | 92.95 | 80.50 | 88.78 | 76.20 | 70.05 | 80.45 | -| | | | | | | | -| Impr 10k over 10k | 37% | 12% | 39% | 15% | 9% | 23% | -| Impr 10k over 1k | 34% | 19% | 29% | 21% | 11% | 26% | -| Impr 1k over 1k | 43% | 20% | 30% | 17% | 10% | 16% | -| ULMFiT qrnn on 1k LSRen1k | 91.32 | 78.92 | 89.45 | 75.99 | | 82.45 | -| ULMFiT qrnn on 10k LSRen1k| 91.90 | 78.79 | 88.47 | 76.05 | | | - - -## Noise resistance - -| Model | en | de | es | fr | it | ja | ru | zh | -|---------------------------------|------------|-----------|-----------|-----------|-----------|-----------|-----------|------------| -|LASER 0 shot | 80.75 (en) | 87.03 (fr)| 82.60 (it)| 82.83 (de)| 73.25 (de)| 60.95 (en)| 68.83 (it)| 72.90 (de) | -|ULMFiT | | **95.4** | **95.15** | **93.67** | **88.42** | **89.20** | **87.27** | | -| % of noise | 20% | 13% | 18% | 18% | 27% | 40% | 32% | 28% | -|ULMFiT trained on 1k noisy exmp. | | 94.49 | 93.12 | 90.49 | 83.72 | 74.72 | 75.67 | | - diff --git a/results/ensemble-laser.md b/results/ensemble-laser.md deleted file mode 100644 index 171be0b..0000000 --- a/results/ensemble-laser.md +++ /dev/null @@ -1,49 +0,0 @@ -``` -$ python -m ulmfit ensemble --glob="data/mldoc/*laser*" --file_template='${dataset_path}/${lang}.train.csv' --gold_labels_template='data/mldoc/${lang}-1/${lang}.train.csv' --key_template='${lang}' --out_template='data/mldoc/${key}-1-ensemble/${key}.train.csv' --exclude_re=".*([a-z][a-z])-1-laser-probs-\1.*" -Skipping /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-probs-es1 -Skipping /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1-laser-probs-ja1 -Skipping /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-probs-fr1 -Skipping /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-probs-zh1 -Skipping /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1-laser-probs-en1 -Skipping /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-probs-it1 -Skipping /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-probs-de1 -Skipping /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-probs-ru1 -{'Key': 'ru', 'Test Accuracy': 0.682, 'on': PosixPath('data/mldoc/ru-1/ru.train.csv'), 'files_count': 7} -{'File saved to': PosixPath('data/mldoc/ru-1-ensemble/ru.train.csv')} -{'Key': 'en', 'Test Accuracy': 0.82, 'on': PosixPath('data/mldoc/en-1/en.train.csv'), 'files_count': 7} -{'File saved to': PosixPath('data/mldoc/en-1-ensemble/en.train.csv')} -{'Key': 'es', 'Test Accuracy': 0.821, 'on': PosixPath('data/mldoc/es-1/es.train.csv'), 'files_count': 7} -{'File saved to': PosixPath('data/mldoc/es-1-ensemble/es.train.csv')} -{'Key': 'it', 'Test Accuracy': 0.782, 'on': PosixPath('data/mldoc/it-1/it.train.csv'), 'files_count': 7} -{'File saved to': PosixPath('data/mldoc/it-1-ensemble/it.train.csv')} -{'Key': 'ja', 'Test Accuracy': 0.685, 'on': PosixPath('data/mldoc/ja-1/ja.train.csv'), 'files_count': 7} -{'File saved to': PosixPath('data/mldoc/ja-1-ensemble/ja.train.csv')} -{'Key': 'zh', 'Test Accuracy': 0.789, 'on': PosixPath('data/mldoc/zh-1/zh.train.csv'), 'files_count': 7} -{'File saved to': PosixPath('data/mldoc/zh-1-ensemble/zh.train.csv')} -{'Key': 'de', 'Test Accuracy': 0.905, 'on': PosixPath('data/mldoc/de-1/de.train.csv'), 'files_count': 7} -{'File saved to': PosixPath('data/mldoc/de-1-ensemble/de.train.csv')} -{'Key': 'fr', 'Test Accuracy': 0.86, 'on': PosixPath('data/mldoc/fr-1/fr.train.csv'), 'files_count': 7} -{'File saved to': PosixPath('data/mldoc/fr-1-ensemble/fr.train.csv')} -``` - -``` - - - -ln -s /home/pczapla/workspace/ulmfit-multilingual/data-archive/mldoc/de-1/models/sp15k/qrnn_nl4_0.m data-archive/mldoc/de-1/models/sp15k/qrnn_base.m -ln -s /home/pczapla/workspace/ulmfit-multilingual/data-archive/mldoc/en-1/models/sp15k/qrnn_nl4_tls.m data-archive/mldoc/en-1/models/sp15k/qrnn_base.m -ln -s /home/pczapla/workspace/ulmfit-multilingual/data-archive/mldoc/es-1/models/sp15k/qrnn_nl4_0.m data-archive/mldoc/es-1/models/sp15k/qrnn_base.m -ln -s /home/pczapla/workspace/ulmfit-multilingual/data-archive/mldoc/fr-1/models/sp15k/qrnn_nl4_tls.m data-archive/mldoc/fr-1/models/sp15k/qrnn_base.m -ln -s /home/pczapla/workspace/ulmfit-multilingual/data-archive/mldoc/it-1/models/sp15k/qrnn_nl4_tls.m data-archive/mldoc/it-1/models/sp15k/qrnn_base.m -ln -s /home/pczapla/workspace/ulmfit-multilingual/data-archive/mldoc/ja-1/models/sp15k/qrnn_nl4_tls.m data-archive/mldoc/ja-1/models/sp15k/qrnn_base.m -ln -s /home/pczapla/workspace/ulmfit-multilingual/data-archive/mldoc/ru-1/models/sp15k/qrnn_nl4_tls.m data-archive/mldoc/ru-1/models/sp15k/qrnn_base.m -ln -s /home/pczapla/workspace/ulmfit-multilingual/data-archive/mldoc/zh-1/models/sp15k/qrnn_nl4_tls.m data-archive/mldoc/zh-1/models/sp15k/qrnn_base.m - - -for a in data-archive/mldoc/*-1; do cp $a/*unsup.csv $a/*test.csv $a/*dev.csv ${a/-archive/}-ensemble; done -python -m ulmfit ls --glob 'data-archive/mldoc/*-1/models/sp15k/qrnn_base.m' --dataset_template='data/mldoc/${lang}-ensemble' - - -python -m ulmfit eval --glob 'data-archive/mldoc/*-1/models/sp15k/qrnn_base.m' --dataset_template='../../data/mldoc/${lang}-ensemble' --num_lm_epochs=0 --num_cls_epochs=8 --early_stopping=False --bs=20 --label-smoothing-eps=0.1 --lr_sched=1cycle --skip_on_error=False - -``` \ No newline at end of file diff --git a/results/logs/100examples.md b/results/logs/100examples.md deleted file mode 100644 index 602ec9b..0000000 --- a/results/logs/100examples.md +++ /dev/null @@ -1,1017 +0,0 @@ -# MLDoc classification using 100 examples - -| Language | 8 epochs (1) | 4 epochs (1) | 4 epochs (2) | 8 epochs (2) | -|-------------|--------------------|---------------------|------------------|----------| -| de | 92.37 | 91.79 | 84.60 | 91.27 | -| en | 77.14 | 66.02 | 70.85 | 87.00 | -| es | 89.52 | 87.55 | 80.17 | 89.57 | -| fr | 81.44 | 74.25 | 79.97 | 88.15 | -| it | 81.15 | 69.24 | 74.17 | 77.54 | -| ja | 78.87 | 70.30 | 69.74 | 78.64 | -| zh | 83.57 | 70.47 | 77.39 | 87.17 | - -- (1) - a larger dropout value for output_p=0.7 instead of output_p=0.2, and wd=1e-1 -- (2) - normal dropout but still wd=1e-1 -- (3) - normal dropout but and normal wd=1e-2 - -## QRNN sp15k - normal dropout, normal wd - - - -## QRNN sp15k - normal dropout, wd=1e-1 - -### 8 epochs -```python -m ulmfit eval --glob="mldoc/*-1/models/sp15k/qrnn_nl4.m" --name nl4-100-e8-normal-dp --limit=100 --num-cls-epochs=8 --lr_sched=1cycle --label-smoothing-eps=0.1 --bs=18 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_nl4-100-e8-normal-dp.m -Training -Dropout {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/de.dev.csv -Limiting data set to: 100 -Data lm, trn: 13500, val: 1500 -Data clslimit100, trn: 100, val: 100 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: - - !! WARNING !! - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Your compiler (c++) may be ABI-incompatible with PyTorch! -Please use a compiler that is ABI-compatible with GCC 4.9 and above. -See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. - -See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 -for instructions on how to install GCC 4.9 or higher. -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - !! WARNING !! - - warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_nl4-100-e8-normal-dp.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.346962 1.354738 0.360000 -2 1.083917 1.077360 0.700000 -3 0.919886 0.868146 0.820000 -4 0.820790 0.769731 0.930000 -5 0.743299 0.783199 0.840000 -6 0.705464 0.722734 0.910000 -7 0.673815 0.693874 0.940000 -8 0.640916 0.677240 0.940000 -Total time: 00:22 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_nl4-100-e8-normal-dp.m -Loss and accuracy using (cls_best): [0.49304065, tensor(0.9128)] -Processing data/mldoc/en-1/models/sp15k/qrnn_nl4.m -en-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k/qrnn_nl4-100-e8-normal-dp.m -Training -Dropout {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/en.dev.csv -Limiting data set to: 100 -Data lm, trn: 13500, val: 1500 -Data clslimit100, trn: 100, val: 100 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁the', '▁,', 's', '▁.', '▁of', '▁and', '▁in', '▁to', '▁a', 'ed'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k/qrnn_nl4-100-e8-normal-dp.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.351400 1.369022 0.290000 -2 1.125228 1.163452 0.670000 -3 0.966749 1.370436 0.350000 -4 0.876823 0.914753 0.760000 -5 0.797715 0.889629 0.760000 -6 0.739767 0.833061 0.800000 -7 0.699772 0.787709 0.810000 -8 0.669680 0.758130 0.820000 -Total time: 00:21 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k/qrnn_nl4-100-e8-normal-dp.m -Loss and accuracy using (cls_best): [0.55098367, tensor(0.8700)] -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-100-e8-normal-dp.m -Training -Dropout {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv -Limiting data set to: 100 -Data lm, trn: 13013, val: 1445 -Data clslimit100, trn: 100, val: 100 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-100-e8-normal-dp.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.324274 1.354593 0.450000 -2 1.104991 1.030298 0.810000 -3 0.937743 0.761312 0.890000 -4 0.846890 0.772061 0.860000 -5 0.762197 0.816325 0.850000 -6 0.711926 0.747758 0.890000 -7 0.669704 0.714664 0.910000 -8 0.635587 0.706967 0.910000 -Total time: 00:19 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-100-e8-normal-dp.m -Loss and accuracy using (cls_best): [0.50185955, tensor(0.8957)] -Processing data/mldoc/fr-1/models/sp15k/qrnn_nl4.m -fr-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_nl4-100-e8-normal-dp.m -Training -Dropout {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/fr.dev.csv -Limiting data set to: 100 -Data lm, trn: 13500, val: 1500 -Data clslimit100, trn: 100, val: 100 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', 's', '▁.', "'", '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_nl4-100-e8-normal-dp.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.311578 1.391519 0.200000 -2 1.070487 1.097862 0.730000 -3 0.908473 0.982642 0.700000 -4 0.847152 0.954593 0.670000 -5 0.773764 0.740223 0.890000 -6 0.720973 0.732911 0.860000 -7 0.680840 0.725618 0.880000 -8 0.646478 0.711748 0.880000 -Total time: 00:22 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_nl4-100-e8-normal-dp.m -Loss and accuracy using (cls_best): [0.5118136, tensor(0.8815)] -Processing data/mldoc/it-1/models/sp15k/qrnn_nl4.m -it-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k/qrnn_nl4-100-e8-normal-dp.m -Training -Dropout {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/it.dev.csv -Limiting data set to: 100 -Data lm, trn: 13500, val: 1500 -Data clslimit100, trn: 100, val: 100 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', '▁e', "▁&'", "'", '▁il', '▁la', '▁in', 'e'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k/qrnn_nl4-100-e8-normal-dp.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.342177 1.374349 0.230000 -2 1.180374 1.193104 0.720000 -3 1.057902 0.981448 0.750000 -4 0.964218 1.267683 0.390000 -5 0.865437 0.927968 0.720000 -6 0.801164 0.933495 0.740000 -7 0.753568 0.910974 0.740000 -8 0.710150 0.893166 0.750000 -Total time: 00:15 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k/qrnn_nl4-100-e8-normal-dp.m -Loss and accuracy using (cls_best): [0.6821853, tensor(0.7755)] -Processing data/mldoc/ja-1/models/sp15k/qrnn_nl4.m -ja-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k/qrnn_nl4-100-e8-normal-dp.m -Training -Dropout {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv -Limiting data set to: 100 -Data lm, trn: 13500, val: 1500 -Data clslimit100, trn: 100, val: 100 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k/qrnn_nl4-100-e8-normal-dp.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.390914 1.373147 0.450000 -2 1.179142 1.231387 0.400000 -3 0.984903 1.356539 0.410000 -4 0.934303 1.067314 0.570000 -5 0.862403 0.971283 0.660000 -6 0.789983 0.919483 0.700000 -7 0.738834 0.877783 0.770000 -8 0.698692 0.849805 0.790000 -Total time: 00:24 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k/qrnn_nl4-100-e8-normal-dp.m -Loss and accuracy using (cls_best): [0.7138088, tensor(0.7865)] -Processing data/mldoc/zh-1/models/sp15k/qrnn_nl4.m -zh-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-100-e8-normal-dp.m -Training -Dropout {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv -Limiting data set to: 100 -Data lm, trn: 13500, val: 1500 -Data clslimit100, trn: 100, val: 100 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-100-e8-normal-dp.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.281217 1.359122 0.370000 -2 1.047607 1.106353 0.720000 -3 0.879789 0.842300 0.790000 -4 0.796210 0.830705 0.810000 -5 0.733291 0.782483 0.830000 -6 0.681939 0.757719 0.870000 -7 0.642002 0.751473 0.880000 -8 0.611668 0.735970 0.860000 -Total time: 00:20 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-100-e8-normal-dp.m -Loss and accuracy using (cls_best): [0.56022245, tensor(0.8717)] -OrderedDict([('data/mldoc/de-1/models/sp15k/qrnn_nl4-100-e8-normal-dp.m', - 0.9127500057220459), - ('data/mldoc/en-1/models/sp15k/qrnn_nl4-100-e8-normal-dp.m', - 0.8700000047683716), - ('data/mldoc/es-1/models/sp15k/qrnn_nl4-100-e8-normal-dp.m', - 0.8957499861717224), - ('data/mldoc/fr-1/models/sp15k/qrnn_nl4-100-e8-normal-dp.m', - 0.8815000057220459), - ('data/mldoc/it-1/models/sp15k/qrnn_nl4-100-e8-normal-dp.m', - 0.7754999995231628), - ('data/mldoc/ja-1/models/sp15k/qrnn_nl4-100-e8-normal-dp.m', - 0.7864999771118164), - ('data/mldoc/zh-1/models/sp15k/qrnn_nl4-100-e8-normal-dp.m', - 0.871749997138977)]) -data/mldoc/de-1/models/sp15k/qrnn_nl4-100-e8-normal-dp.m: 0.9127500057220459 -data/mldoc/en-1/models/sp15k/qrnn_nl4-100-e8-normal-dp.m: 0.8700000047683716 -data/mldoc/es-1/models/sp15k/qrnn_nl4-100-e8-normal-dp.m: 0.8957499861717224 -data/mldoc/fr-1/models/sp15k/qrnn_nl4-100-e8-normal-dp.m: 0.8815000057220459 -data/mldoc/it-1/models/sp15k/qrnn_nl4-100-e8-normal-dp.m: 0.7754999995231628 -data/mldoc/ja-1/models/sp15k/qrnn_nl4-100-e8-normal-dp.m: 0.7864999771118164 -data/mldoc/zh-1/models/sp15k/qrnn_nl4-100-e8-normal-dp.m: 0.871749997138977 -``` - - -### 4 epochs -``` -python -m ulmfit eval --glob="mldoc/*-1/models/sp15k/qrnn_nl4.m" --name nl4-100-e4-normal-dp --limit=100 --num-cls-epochs=4 --lr_sched=1cycle --label-smoothing-eps=0.1 --bs=18 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_nl4-100-e4-normal-dp.m -Training -Dropout {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/de.dev.csv -Limiting data set to: 100 -Data lm, trn: 13500, val: 1500 -Data clslimit100, trn: 100, val: 100 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: - - !! WARNING !! - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Your compiler (c++) may be ABI-incompatible with PyTorch! -Please use a compiler that is ABI-compatible with GCC 4.9 and above. -See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. - -See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 -for instructions on how to install GCC 4.9 or higher. -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - !! WARNING !! - - warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_nl4-100-e4-normal-dp.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.284350 1.332569 0.280000 -2 1.016471 0.983958 0.840000 -3 0.867660 0.927123 0.910000 -4 0.761539 0.911453 0.840000 -Total time: 00:11 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_nl4-100-e4-normal-dp.m -Loss and accuracy using (cls_best): [0.819043, tensor(0.8460)] -Processing data/mldoc/en-1/models/sp15k/qrnn_nl4.m -en-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k/qrnn_nl4-100-e4-normal-dp.m -Training -Dropout {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/en.dev.csv -Limiting data set to: 100 -Data lm, trn: 13500, val: 1500 -Data clslimit100, trn: 100, val: 100 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁the', '▁,', 's', '▁.', '▁of', '▁and', '▁in', '▁to', '▁a', 'ed'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k/qrnn_nl4-100-e4-normal-dp.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.290013 1.338139 0.370000 -2 1.052497 1.222131 0.380000 -3 0.904921 1.098593 0.440000 -4 0.809407 0.984065 0.680000 -Total time: 00:10 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k/qrnn_nl4-100-e4-normal-dp.m -Loss and accuracy using (cls_best): [0.89315945, tensor(0.7085)] -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-100-e4-normal-dp.m -Training -Dropout {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv -Limiting data set to: 100 -Data lm, trn: 13013, val: 1445 -Data clslimit100, trn: 100, val: 100 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-100-e4-normal-dp.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.286507 1.319209 0.390000 -2 1.063101 1.230659 0.450000 -3 0.925757 1.026052 0.600000 -4 0.812203 0.907415 0.800000 -Total time: 00:09 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-100-e4-normal-dp.m -Loss and accuracy using (cls_best): [0.7879555, tensor(0.8018)] -Processing data/mldoc/fr-1/models/sp15k/qrnn_nl4.m -fr-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_nl4-100-e4-normal-dp.m -Training -Dropout {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/fr.dev.csv -Limiting data set to: 100 -Data lm, trn: 13500, val: 1500 -Data clslimit100, trn: 100, val: 100 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', 's', '▁.', "'", '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_nl4-100-e4-normal-dp.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.267655 1.295306 0.460000 -2 1.034598 1.207809 0.360000 -3 0.869741 0.965482 0.690000 -4 0.772019 0.871552 0.810000 -Total time: 00:10 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_nl4-100-e4-normal-dp.m -Loss and accuracy using (cls_best): [0.7594081, tensor(0.7997)] -Processing data/mldoc/it-1/models/sp15k/qrnn_nl4.m -it-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k/qrnn_nl4-100-e4-normal-dp.m -Training -Dropout {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/it.dev.csv -Limiting data set to: 100 -Data lm, trn: 13500, val: 1500 -Data clslimit100, trn: 100, val: 100 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', '▁e', "▁&'", "'", '▁il', '▁la', '▁in', 'e'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k/qrnn_nl4-100-e4-normal-dp.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.338249 1.345434 0.320000 -2 1.179869 1.195013 0.480000 -3 1.027482 1.109100 0.560000 -4 0.900706 1.021052 0.730000 -Total time: 00:07 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k/qrnn_nl4-100-e4-normal-dp.m -Loss and accuracy using (cls_best): [0.8819375, tensor(0.7418)] -Processing data/mldoc/ja-1/models/sp15k/qrnn_nl4.m -ja-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k/qrnn_nl4-100-e4-normal-dp.m -Training -Dropout {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv -Limiting data set to: 100 -Data lm, trn: 13500, val: 1500 -Data clslimit100, trn: 100, val: 100 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k/qrnn_nl4-100-e4-normal-dp.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.329939 1.365095 0.200000 -2 1.109706 1.175099 0.420000 -3 0.978721 1.141211 0.440000 -4 0.870699 1.028071 0.700000 -Total time: 00:12 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k/qrnn_nl4-100-e4-normal-dp.m -Loss and accuracy using (cls_best): [0.9101604, tensor(0.6975)] -Processing data/mldoc/zh-1/models/sp15k/qrnn_nl4.m -zh-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-100-e4-normal-dp.m -Training -Dropout {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv -Limiting data set to: 100 -Data lm, trn: 13500, val: 1500 -Data clslimit100, trn: 100, val: 100 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-100-e4-normal-dp.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.256988 1.290418 0.370000 -2 1.000337 0.967188 0.780000 -3 0.837002 0.955509 0.760000 -4 0.740278 0.937393 0.780000 -Total time: 00:10 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-100-e4-normal-dp.m -Loss and accuracy using (cls_best): [0.8290863, tensor(0.7740)] -OrderedDict([('data/mldoc/de-1/models/sp15k/qrnn_nl4-100-e4-normal-dp.m', - 0.8460000157356262), - ('data/mldoc/en-1/models/sp15k/qrnn_nl4-100-e4-normal-dp.m', - 0.7085000276565552), - ('data/mldoc/es-1/models/sp15k/qrnn_nl4-100-e4-normal-dp.m', - 0.8017500042915344), - ('data/mldoc/fr-1/models/sp15k/qrnn_nl4-100-e4-normal-dp.m', - 0.7997499704360962), - ('data/mldoc/it-1/models/sp15k/qrnn_nl4-100-e4-normal-dp.m', - 0.7417500019073486), - ('data/mldoc/ja-1/models/sp15k/qrnn_nl4-100-e4-normal-dp.m', - 0.6974999904632568), - ('data/mldoc/zh-1/models/sp15k/qrnn_nl4-100-e4-normal-dp.m', - 0.7739999890327454)]) -data/mldoc/de-1/models/sp15k/qrnn_nl4-100-e4-normal-dp.m: 0.8460000157356262 -data/mldoc/en-1/models/sp15k/qrnn_nl4-100-e4-normal-dp.m: 0.7085000276565552 -data/mldoc/es-1/models/sp15k/qrnn_nl4-100-e4-normal-dp.m: 0.8017500042915344 -data/mldoc/fr-1/models/sp15k/qrnn_nl4-100-e4-normal-dp.m: 0.7997499704360962 -data/mldoc/it-1/models/sp15k/qrnn_nl4-100-e4-normal-dp.m: 0.7417500019073486 -data/mldoc/ja-1/models/sp15k/qrnn_nl4-100-e4-normal-dp.m: 0.6974999904632568 -data/mldoc/zh-1/models/sp15k/qrnn_nl4-100-e4-normal-dp.m: 0.7739999890327454 -``` - - -## QRNN sp15k - larger dropout -The following dropouts were used: `output_p=0.7` compared to other experimejnts where `output_p=0.2` - -| Model | Accuracy 8 epochs | Accuracy 4 epochs -|-----------------------------|--------------------|---------------------| -| de-1 sp15k/qrnn_nl4-100e8.m | 92.37 | 91.79 | -| en-1 sp15k/qrnn_nl4-100e8.m | 77.14 | 66.02 | -| es-1 sp15k/qrnn_nl4-100e8.m | 89.52 | 87.55 | -| fr-1 sp15k/qrnn_nl4-100e8.m | 81.44 | 74.25 | -| it-1 sp15k/qrnn_nl4-100e8.m | 81.15 | 69.24 | -| ja-1 sp15k/qrnn_nl4-100e8.m | 78.87 | 70.30 | -| zh-1 sp15k/qrnn_nl4-100e8.m | 83.57 | 70.47 | - -### 8 epochs of training -`self.dps = dict(output_p=0.7, hidden_p=0.1, input_p=0.2, embed_p=0.02, weight_p=0.15)` -```` -python -m ulmfit eval --glob="mldoc/*-1/models/sp15k/qrnn_nl4.m" --name nl4-100e8 --limit=100 --num-cls-epochs=8 --lr_sched=1cycle --label-smoothing-eps=0.1 --bs=18 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_nl4-100e8.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/de.dev.csv -Limiting data set to: 100 -Data lm, trn: 13500, val: 1500 -Data clslimit100, trn: 100, val: 100 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: - - !! WARNING !! - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Your compiler (c++) may be ABI-incompatible with PyTorch! -Please use a compiler that is ABI-compatible with GCC 4.9 and above. -See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. - -See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 -for instructions on how to install GCC 4.9 or higher. -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - !! WARNING !! - - warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) -Single training schedule -epoch train_loss valid_loss accuracy -1 1.354484 1.379077 0.280000 -2 1.152421 1.152570 0.620000 -3 0.980928 1.018830 0.660000 -4 0.871912 0.872564 0.820000 -5 0.803019 0.789034 0.900000 -6 0.748509 0.711389 0.940000 -7 0.701289 0.690140 0.950000 -8 0.664756 0.669448 0.950000 -Total time: 00:23 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_nl4-100e8.m -Loss and accuracy using (cls_best): [0.48774713, tensor(0.9237)] -Processing data/mldoc/en-1/models/sp15k/qrnn_nl4.m -en-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k/qrnn_nl4-100e8.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/en.dev.csv -Limiting data set to: 100 -Data lm, trn: 13500, val: 1500 -Data clslimit100, trn: 100, val: 100 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁the', '▁,', 's', '▁.', '▁of', '▁and', '▁in', '▁to', '▁a', 'ed'] -Single training schedule -epoch train_loss valid_loss accuracy -1 1.366923 1.380464 0.310000 -2 1.177802 1.236425 0.500000 -3 0.999587 0.912387 0.740000 -4 0.898221 0.990741 0.660000 -5 0.818071 1.066497 0.580000 -6 0.757039 0.921970 0.680000 -7 0.712545 0.956183 0.660000 -8 0.681197 0.911881 0.710000 -Total time: 00:22 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k/qrnn_nl4-100e8.m -Loss and accuracy using (cls_best): [0.66596776, tensor(0.7715)] -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-100e8.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv -Limiting data set to: 100 -Data lm, trn: 13013, val: 1445 -Data clslimit100, trn: 100, val: 100 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Single training schedule -epoch train_loss valid_loss accuracy -1 1.253322 1.366251 0.220000 -2 1.051253 1.117081 0.650000 -3 0.905148 0.928994 0.770000 -4 0.825990 0.855558 0.810000 -5 0.754039 0.828022 0.810000 -6 0.704161 0.752015 0.880000 -7 0.661291 0.739049 0.880000 -8 0.631924 0.731397 0.880000 -Total time: 00:19 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-100e8.m -Loss and accuracy using (cls_best): [0.5359205, tensor(0.8953)] -Processing data/mldoc/fr-1/models/sp15k/qrnn_nl4.m -fr-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_nl4-100e8.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/fr.dev.csv -Limiting data set to: 100 -Data lm, trn: 13500, val: 1500 -Data clslimit100, trn: 100, val: 100 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', 's', '▁.', "'", '▁la', '▁le', '▁et', '▁l', '▁à'] -Single training schedule -epoch train_loss valid_loss accuracy -1 1.315270 1.368941 0.220000 -2 1.117436 1.137058 0.630000 -3 0.959049 0.955980 0.750000 -4 0.856808 0.749577 0.900000 -5 0.800600 0.859013 0.740000 -6 0.739985 0.873451 0.720000 -7 0.696956 0.820335 0.780000 -8 0.660949 0.784974 0.820000 -Total time: 00:22 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_nl4-100e8.m -Loss and accuracy using (cls_best): [0.6222675, tensor(0.8145)] -Processing data/mldoc/it-1/models/sp15k/qrnn_nl4.m -it-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k/qrnn_nl4-100e8.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/it.dev.csv -Limiting data set to: 100 -Data lm, trn: 13500, val: 1500 -Data clslimit100, trn: 100, val: 100 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', '▁e', "▁&'", "'", '▁il', '▁la', '▁in', 'e'] -Single training schedule -epoch train_loss valid_loss accuracy -1 1.406870 1.370643 0.250000 -2 1.248155 1.226949 0.620000 -3 1.098265 1.045475 0.650000 -4 0.989543 0.999063 0.660000 -5 0.904690 0.908897 0.750000 -6 0.838460 0.906191 0.770000 -7 0.790075 0.867051 0.810000 -8 0.748298 0.844697 0.820000 -Total time: 00:16 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k/qrnn_nl4-100e8.m -Loss and accuracy using (cls_best): [0.6499791, tensor(0.8115)] -Processing data/mldoc/ja-1/models/sp15k/qrnn_nl4.m -ja-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k/qrnn_nl4-100e8.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv -Limiting data set to: 100 -Data lm, trn: 13500, val: 1500 -Data clslimit100, trn: 100, val: 100 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)'] -Single training schedule -epoch train_loss valid_loss accuracy -1 1.284972 1.374997 0.390000 -2 1.112592 1.235691 0.490000 -3 0.990320 1.235275 0.470000 -4 0.884597 0.985890 0.680000 -5 0.814330 1.017439 0.660000 -6 0.756775 0.912423 0.780000 -7 0.757921 0.860843 0.840000 -8 0.731727 0.835498 0.810000 -Total time: 00:24 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k/qrnn_nl4-100e8.m -Loss and accuracy using (cls_best): [0.68836695, tensor(0.7887)] -Processing data/mldoc/zh-1/models/sp15k/qrnn_nl4.m -zh-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-100e8.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv -Limiting data set to: 100 -Data lm, trn: 13500, val: 1500 -Data clslimit100, trn: 100, val: 100 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] -Single training schedule -epoch train_loss valid_loss accuracy -1 1.282265 1.340392 0.670000 -2 1.029523 1.092677 0.600000 -3 0.890638 1.027698 0.650000 -4 0.799896 0.919632 0.730000 -5 0.735252 0.840827 0.800000 -6 0.684259 0.791598 0.860000 -7 0.645315 0.794246 0.840000 -8 0.615542 0.782311 0.840000 -Total time: 00:20 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-100e8.m -Loss and accuracy using (cls_best): [0.63012207, tensor(0.8357)] -OrderedDict([('data/mldoc/de-1/models/sp15k/qrnn_nl4-100e8.m', - 0.9237499833106995), - ('data/mldoc/en-1/models/sp15k/qrnn_nl4-100e8.m', - 0.7714999914169312), - ('data/mldoc/es-1/models/sp15k/qrnn_nl4-100e8.m', - 0.8952500224113464), - ('data/mldoc/fr-1/models/sp15k/qrnn_nl4-100e8.m', - 0.8144999742507935), - ('data/mldoc/it-1/models/sp15k/qrnn_nl4-100e8.m', - 0.8115000128746033), - ('data/mldoc/ja-1/models/sp15k/qrnn_nl4-100e8.m', - 0.7887499928474426), - ('data/mldoc/zh-1/models/sp15k/qrnn_nl4-100e8.m', - 0.8357499837875366)]) -data/mldoc/de-1/models/sp15k/qrnn_nl4-100e8.m: 0.9237499833106995 -data/mldoc/en-1/models/sp15k/qrnn_nl4-100e8.m: 0.7714999914169312 -data/mldoc/es-1/models/sp15k/qrnn_nl4-100e8.m: 0.8952500224113464 -data/mldoc/fr-1/models/sp15k/qrnn_nl4-100e8.m: 0.8144999742507935 -data/mldoc/it-1/models/sp15k/qrnn_nl4-100e8.m: 0.8115000128746033 -data/mldoc/ja-1/models/sp15k/qrnn_nl4-100e8.m: 0.7887499928474426 -data/mldoc/zh-1/models/sp15k/qrnn_nl4-100e8.m: 0.8357499837875366 -```` - -### 4 epochs of training - -```` -python -m ulmfit eval --glob="mldoc/*-1/models/sp15k/qrnn_nl4.m" --name nl4-100e4 --limit=100 --num-cls-epochs=4 --lr_sched=1cycle --label-smoothing-eps=0.1 --bs=18 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_nl4-100e4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/de.dev.csv -Limiting data set to: 100 -Data lm, trn: 13500, val: 1500 -Data clslimit100, trn: 100, val: 100 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.7, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: - - !! WARNING !! - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Your compiler (c++) may be ABI-incompatible with PyTorch! -Please use a compiler that is ABI-compatible with GCC 4.9 and above. -See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. - -See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 -for instructions on how to install GCC 4.9 or higher. -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - !! WARNING !! - - warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_nl4-100e4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.268574 1.327856 0.330000 -2 1.006592 1.120209 0.470000 -3 0.865163 0.915048 0.860000 -4 0.775797 0.878599 0.910000 -Total time: 00:11 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_nl4-100e4.m -Loss and accuracy using (cls_best): [0.7670934, tensor(0.9180)] -Processing data/mldoc/en-1/models/sp15k/qrnn_nl4.m -en-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k/qrnn_nl4-100e4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/en.dev.csv -Limiting data set to: 100 -Data lm, trn: 13500, val: 1500 -Data clslimit100, trn: 100, val: 100 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁the', '▁,', 's', '▁.', '▁of', '▁and', '▁in', '▁to', '▁a', 'ed'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.7, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k/qrnn_nl4-100e4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.346742 1.344590 0.400000 -2 1.097295 1.131111 0.570000 -3 0.927231 0.992626 0.650000 -4 0.823589 1.027419 0.600000 -Total time: 00:10 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k/qrnn_nl4-100e4.m -Loss and accuracy using (cls_best): [0.8792457, tensor(0.6603)] -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-100e4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv -Limiting data set to: 100 -Data lm, trn: 13013, val: 1445 -Data clslimit100, trn: 100, val: 100 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.7, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-100e4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.249590 1.328701 0.370000 -2 0.977393 1.031057 0.540000 -3 0.858498 0.886364 0.850000 -4 0.777858 0.868647 0.890000 -Total time: 00:10 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-100e4.m -Loss and accuracy using (cls_best): [0.72082895, tensor(0.8755)] -Processing data/mldoc/fr-1/models/sp15k/qrnn_nl4.m -fr-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_nl4-100e4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/fr.dev.csv -Limiting data set to: 100 -Data lm, trn: 13500, val: 1500 -Data clslimit100, trn: 100, val: 100 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', 's', '▁.', "'", '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.7, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_nl4-100e4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.305505 1.362959 0.200000 -2 1.073055 1.068938 0.700000 -3 0.892900 0.981664 0.620000 -4 0.796118 0.950983 0.700000 -Total time: 00:11 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_nl4-100e4.m -Loss and accuracy using (cls_best): [0.8226887, tensor(0.7425)] -Processing data/mldoc/it-1/models/sp15k/qrnn_nl4.m -it-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k/qrnn_nl4-100e4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/it.dev.csv -Limiting data set to: 100 -Data lm, trn: 13500, val: 1500 -Data clslimit100, trn: 100, val: 100 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', '▁e', "▁&'", "'", '▁il', '▁la', '▁in', 'e'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.7, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k/qrnn_nl4-100e4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.372190 1.350273 0.500000 -2 1.207576 1.173455 0.750000 -3 1.040837 1.107950 0.590000 -4 0.931903 1.119772 0.540000 -Total time: 00:07 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k/qrnn_nl4-100e4.m -Loss and accuracy using (cls_best): [0.95755553, tensor(0.6925)] -Processing data/mldoc/ja-1/models/sp15k/qrnn_nl4.m -ja-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k/qrnn_nl4-100e4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv -Limiting data set to: 100 -Data lm, trn: 13500, val: 1500 -Data clslimit100, trn: 100, val: 100 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.7, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k/qrnn_nl4-100e4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.300252 1.326448 0.330000 -2 1.175279 1.211864 0.500000 -3 1.019274 1.129269 0.460000 -4 0.895053 1.059557 0.640000 -Total time: 00:11 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k/qrnn_nl4-100e4.m -Loss and accuracy using (cls_best): [0.96151954, tensor(0.7030)] -Processing data/mldoc/zh-1/models/sp15k/qrnn_nl4.m -zh-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-100e4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv -Limiting data set to: 100 -Data lm, trn: 13500, val: 1500 -Data clslimit100, trn: 100, val: 100 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.7, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-100e4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.230801 1.302317 0.660000 -2 0.944098 0.992352 0.710000 -3 0.810783 1.026886 0.630000 -4 0.736866 0.985942 0.680000 -Total time: 00:09 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-100e4.m -Loss and accuracy using (cls_best): [0.8833954, tensor(0.7048)] -OrderedDict([('data/mldoc/de-1/models/sp15k/qrnn_nl4-100e4.m', - 0.9179999828338623), - ('data/mldoc/en-1/models/sp15k/qrnn_nl4-100e4.m', - 0.6602500081062317), - ('data/mldoc/es-1/models/sp15k/qrnn_nl4-100e4.m', - 0.8755000233650208), - ('data/mldoc/fr-1/models/sp15k/qrnn_nl4-100e4.m', - 0.7425000071525574), - ('data/mldoc/it-1/models/sp15k/qrnn_nl4-100e4.m', - 0.6924999952316284), - ('data/mldoc/ja-1/models/sp15k/qrnn_nl4-100e4.m', - 0.703000009059906), - ('data/mldoc/zh-1/models/sp15k/qrnn_nl4-100e4.m', - 0.7047500014305115)]) -data/mldoc/de-1/models/sp15k/qrnn_nl4-100e4.m: 0.9179999828338623 -data/mldoc/en-1/models/sp15k/qrnn_nl4-100e4.m: 0.6602500081062317 -data/mldoc/es-1/models/sp15k/qrnn_nl4-100e4.m: 0.8755000233650208 -data/mldoc/fr-1/models/sp15k/qrnn_nl4-100e4.m: 0.7425000071525574 -data/mldoc/it-1/models/sp15k/qrnn_nl4-100e4.m: 0.6924999952316284 -data/mldoc/ja-1/models/sp15k/qrnn_nl4-100e4.m: 0.703000009059906 -data/mldoc/zh-1/models/sp15k/qrnn_nl4-100e4.m: 0.7047500014305115 -```` \ No newline at end of file diff --git a/results/logs/Timing.md b/results/logs/Timing.md deleted file mode 100644 index 3b13fd2..0000000 --- a/results/logs/Timing.md +++ /dev/null @@ -1,24 +0,0 @@ - -## QRNN sp15k timing -``` -time python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4-2' --max-vocab 15000 --lang ${LANG} --qrnn=True - train 1 --bs=50 --drop_mult=0 --label-smoothing-eps=0.1 -Max vocab: 15000 -Cache dir: data/wiki/ru-100/models/sp15k -Model dir: data/wiki/ru-100/models/sp15k/qrnn_nl4-2.m -Wiki text was split to 193047 articles -Wiki text was split to 460 articles -Data lm, trn: 193047, val: 460 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', '▁на', 'х'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Training lm from random weights -epoch train_loss valid_loss accuracy -1 3.943105 3.860063 0.477620 -Total time: 1:05:03 -data/wiki/ru-100/models/sp15k -Saving info data/wiki/ru-100/models/sp15k/qrnn_nl4-2.m/info.json - -real 65m30,341s -user 48m49,047s -sys 16m40,688s -``` \ No newline at end of file diff --git a/results/logs/Training qrnn on ML.txt b/results/logs/Training qrnn on ML.txt deleted file mode 100644 index 214804d..0000000 --- a/results/logs/Training qrnn on ML.txt +++ /dev/null @@ -1,448 +0,0 @@ -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/de-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/de-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.467852 2.558666 0.525457 -Total time: 02:25 -epoch train_loss valid_loss accuracy -1 2.722157 2.387366 0.548566 -2 2.477095 2.170018 0.580988 -3 2.182971 1.981363 0.609205 -4 2.078041 1.836848 0.629900 -5 1.975613 1.744062 0.642769 -6 1.866875 1.656678 0.655799 -7 1.831995 1.595655 0.665479 -8 1.768020 1.540487 0.673880 -9 1.751569 1.488140 0.682557 -10 1.647143 1.441723 0.690275 -11 1.712795 1.399652 0.697534 -12 1.529405 1.350384 0.706170 -13 1.549134 1.313349 0.713210 -14 1.585015 1.278395 0.719908 -15 1.475010 1.248854 0.725591 -16 1.532636 1.221373 0.731053 -17 1.445181 1.203350 0.734503 -18 1.396236 1.191440 0.737102 -19 1.316587 1.186497 0.738052 -20 1.374460 1.185027 0.738290 -Total time: 1:11:26 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_nl4.m/info.json -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [1.3879647, tensor(0.2595)] -Processing data/wiki/en-100/models/sp15k/qrnn_nl4.m - - ------- - - - - -$ python -m ulmfit eval --glob="wiki/*-100/models/sp15k/qrnn_nl4.m" --name nl4 --dataset-template='../mldoc/${lang}-1' --num-lm-epochs=20 --num-cls-epochs=8 --bs=18 --lr_sched=1cycle - -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁the', '▁,', 's', '▁.', '▁of', '▁and', '▁in', '▁to', '▁a', 'ed'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/en-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/en-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 4.567319 3.820788 0.346054 -Total time: 02:26 -epoch train_loss valid_loss accuracy -1 3.889761 3.601505 0.374049 -2 3.570620 3.357854 0.406134 -3 3.389516 3.153452 0.432199 -4 3.217872 2.985234 0.452187 -5 3.063675 2.851744 0.468071 -6 3.023959 2.754062 0.480278 -7 2.907327 2.647027 0.493494 -8 2.786187 2.562560 0.505051 -9 2.737610 2.500068 0.513554 -10 2.696695 2.430095 0.523029 -11 2.658439 2.380829 0.530339 -12 2.598193 2.318927 0.539454 -13 2.558214 2.275014 0.546136 -14 2.520342 2.230543 0.553176 -15 2.475964 2.190341 0.559245 -16 2.370359 2.161100 0.564223 -17 2.430078 2.136685 0.568197 -18 2.383946 2.125458 0.569950 -19 2.389433 2.117541 0.571265 -20 2.297921 2.116168 0.571367 -Total time: 1:11:10 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.705876 0.241169 0.918000 -2 0.450546 0.239528 0.926000 -3 0.335179 0.221836 0.931000 -4 0.202048 0.208652 0.951000 -5 0.144956 0.223669 0.954000 -6 0.073117 0.277062 0.953000 -7 0.045186 0.258046 0.962000 -8 0.022987 0.265977 0.961000 -Total time: 02:33 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.29402012, tensor(0.9460)] -Processing data/wiki/es-100/models/sp15k/qrnn_nl4.m -../mldoc/es-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Loss and accuracy using (cls_last): [0.19222946, tensor(0.9515)] -Processing data/wiki/fr-100/models/sp15k/qrnn_nl4.m -../mldoc/fr-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/fr.dev.csv -Running tokenization lm... -Data lm, trn: 13500, val: 1500 -Running tokenization cls... -Data cls, trn: 1000, val: 1000 -Running tokenization tst... -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', 's', '▁.', "'", '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.375679 2.676224 0.454405 -Total time: 02:19 -epoch train_loss valid_loss accuracy -1 2.901917 2.540690 0.475910 -2 2.593614 2.370477 0.504601 -3 2.423170 2.205713 0.530328 -4 2.287688 2.083261 0.549087 -5 2.161118 1.984955 0.564804 -6 2.221017 1.912810 0.575434 -7 2.111272 1.837854 0.588076 -8 2.032289 1.775163 0.598341 -9 1.984161 1.720519 0.607980 -10 1.904775 1.668184 0.617407 -11 1.829098 1.621347 0.626292 -12 1.855409 1.577870 0.634512 -13 1.843696 1.536835 0.642584 -14 1.767968 1.496428 0.650317 -15 1.741591 1.463305 0.656908 -16 1.682118 1.438706 0.662601 -17 1.666425 1.418383 0.666283 -18 1.623713 1.406877 0.668710 -19 1.645482 1.401546 0.669716 -20 1.579352 1.399608 0.670167 -Total time: 1:06:50 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.614734 0.262120 0.906000 -2 0.377443 0.327852 0.917000 -3 0.296728 0.392655 0.903000 -4 0.179866 0.423420 0.928000 -5 0.114529 0.398973 0.935000 -6 0.082004 0.325470 0.944000 -7 0.047604 0.359636 0.945000 -8 0.032579 0.354014 0.944000 -Total time: 02:22 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.33020702, tensor(0.9450)] -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.524609 2.582580 0.512131 -Total time: 02:34 -epoch train_loss valid_loss accuracy -1 2.656181 2.315530 0.550262 -2 2.265949 2.019140 0.598469 -3 1.985897 1.769565 0.638234 -4 1.831071 1.617888 0.660760 -5 1.735492 1.509642 0.677379 -6 1.637924 1.427618 0.690664 -7 1.564483 1.363384 0.700825 -8 1.508054 1.318165 0.708210 -9 1.471599 1.267787 0.716080 -10 1.398376 1.232899 0.722340 -11 1.311976 1.199602 0.728811 -12 1.401354 1.162299 0.735328 -13 1.385588 1.132408 0.740850 -14 1.256193 1.106556 0.745935 -15 1.289892 1.083529 0.750840 -16 1.220951 1.063360 0.754845 -17 1.259715 1.050884 0.757371 -18 1.165468 1.042870 0.759241 -19 1.242660 1.038160 0.760036 -20 1.194239 1.037506 0.760167 -Total time: 1:13:55 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.760780 0.391838 0.872000 -2 0.592674 0.427392 0.870000 -3 0.446265 0.593488 0.838000 -4 0.318780 0.605533 0.858000 -5 0.226914 0.665538 0.872000 -6 0.135525 0.742310 0.891000 -7 0.063984 0.778616 0.892000 -8 0.039366 0.827663 0.884000 -Total time: 02:49 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.70555997, tensor(0.8960)] -Processing data/wiki/zh-100/models/sp15k/qrnn_nl4.m -../mldoc/zh-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] -Loss and accuracy using (cls_last): [0.30052844, tensor(0.8982)] -OrderedDict([('data/mldoc/de-1/models/sp15k/qrnn_nl4.m', 0.2592499852180481), - ('data/mldoc/en-1/models/sp15k/qrnn_nl4.m', 0.9462500214576721), - ('data/mldoc/es-1/models/sp15k/qrnn_nl4.m', 0.9514999985694885), - ('data/mldoc/fr-1/models/sp15k/qrnn_nl4.m', 0.9442499876022339), - ('data/mldoc/ja-1/models/sp15k/qrnn_nl4.m', 0.8960000276565552), - ('data/mldoc/zh-1/models/sp15k/qrnn_nl4.m', 0.8982499837875366)]) - - - - - - - - - !! WARNING !! - - warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) -Single training schedule -epoch train_loss valid_loss accuracy -1 0.610785 0.239165 0.923000 -2 0.392899 0.281254 0.937000 -3 0.268695 0.444383 0.909000 -4 0.162150 0.427744 0.931000 -5 0.109248 0.422351 0.948000 -6 0.061984 0.411351 0.947000 -7 0.033645 0.413174 0.951000 -8 0.018704 0.404264 0.947000 -Total time: 02:16 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.29712877, tensor(0.9565)] -Processing data/wiki/en-100/models/sp15k/qrnn_nl4.m -../mldoc/en-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k/qrnn_nl4.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/en.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁the', '▁,', 's', '▁.', '▁of', '▁and', '▁in', '▁to', '▁a', 'ed'] -Loss and accuracy using (cls_last): [0.29526812, tensor(0.9463)] -Processing data/wiki/es-100/models/sp15k/qrnn_nl4.m -../mldoc/es-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Loss and accuracy using (cls_last): [0.19222946, tensor(0.9515)] -Processing data/wiki/fr-100/models/sp15k/qrnn_nl4.m -../mldoc/fr-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_nl4.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/fr.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', 's', '▁.', "'", '▁la', '▁le', '▁et', '▁l', '▁à'] -Loss and accuracy using (cls_last): [0.33129737, tensor(0.9442)] -Processing data/wiki/it-100/models/sp15k/qrnn_nl4.m -../mldoc/it-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k/qrnn_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/it.dev.csv -Running tokenization lm... -Data lm, trn: 13500, val: 1500 -Running tokenization cls... -Data cls, trn: 1000, val: 1000 -Running tokenization tst... -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', '▁e', "▁&'", "'", '▁il', '▁la', '▁in', 'e'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/it-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/it-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.929510 2.916437 0.434550 -Total time: 01:20 -epoch train_loss valid_loss accuracy -1 3.231523 2.723415 0.461407 -2 2.823881 2.498599 0.496812 -3 2.586648 2.283846 0.530524 -4 2.417038 2.125690 0.553137 -5 2.278636 1.995558 0.572757 -6 2.199877 1.887804 0.589102 -7 2.090629 1.799082 0.603201 -8 2.046975 1.725273 0.615247 -9 1.935966 1.654829 0.626968 -10 1.921190 1.590797 0.638228 -11 1.894758 1.528087 0.649369 -12 1.792718 1.477532 0.658754 -13 1.679359 1.428426 0.668648 -14 1.723383 1.377170 0.678987 -15 1.597491 1.339658 0.686348 -16 1.620966 1.307664 0.692993 -17 1.568962 1.284500 0.697923 -18 1.533934 1.271438 0.700628 -19 1.496832 1.264714 0.701968 -20 1.486198 1.262870 0.702333 -Total time: 39:33 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.784124 0.414312 0.847000 -2 0.550873 0.413405 0.861000 -3 0.445371 0.363693 0.877000 -4 0.271702 0.426771 0.899000 -5 0.165902 0.556069 0.881000 -6 0.091403 0.628809 0.897000 -7 0.065516 0.693292 0.893000 -8 0.033616 0.675199 0.897000 -Total time: 01:24 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.7380945, tensor(0.8992)] -Processing data/wiki/ja-100/models/sp15k/qrnn_nl4.m -../mldoc/ja-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k/qrnn_nl4.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)'] -Loss and accuracy using (cls_last): [0.7049702, tensor(0.8953)] -Processing data/wiki/zh-100/models/sp15k/qrnn_nl4.m -../mldoc/zh-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] -Loss and accuracy using (cls_last): [0.30052844, tensor(0.8982)] -OrderedDict([('data/mldoc/de-1/models/sp15k/qrnn_nl4.m', 0.9564999938011169), - ('data/mldoc/en-1/models/sp15k/qrnn_nl4.m', 0.9462500214576721), - ('data/mldoc/es-1/models/sp15k/qrnn_nl4.m', 0.9514999985694885), - ('data/mldoc/fr-1/models/sp15k/qrnn_nl4.m', 0.9442499876022339), - ('data/mldoc/it-1/models/sp15k/qrnn_nl4.m', 0.8992499709129333), - ('data/mldoc/ja-1/models/sp15k/qrnn_nl4.m', 0.8952500224113464), - ('data/mldoc/zh-1/models/sp15k/qrnn_nl4.m', 0.8982499837875366)]) - - - - - - -## DE ----------------------------------------- -Training issues - - -1/2nd -- That was without fine tuning !!! 0 shot:) -``` -python -m ulmfit load_cls data/mldoc/de-1/models/sp15k/qrnn_nl4.m --lang=de - train 0 --num-cls-epochs 8 --bs=18 --lr-sched=1cycle ✘ 1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_None.m -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/de.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: - - !! WARNING !! - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Your compiler (c++) may be ABI-incompatible with PyTorch! -Please use a compiler that is ABI-compatible with GCC 4.9 and above. -See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. - -See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 -for instructions on how to install GCC 4.9 or higher. -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - !! WARNING !! - - warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) -Single training schedule -epoch train_loss valid_loss accuracy -1 1.310368 1.177105 0.520000 -2 1.096284 0.899281 0.739000 -3 0.860910 0.668378 0.864000 -4 0.676764 0.733304 0.868000 -5 0.573360 0.590983 0.885000 -6 0.438448 0.446631 0.918000 -7 0.397323 0.531330 0.919000 -8 0.339557 0.437841 0.922000 -Total time: 02:25 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_None.m -Loss and accuracy using (cls_best): [0.3380329, tensor(0.9295)] -0.33803290128707886 -0.9294999837875366 -``` - -3rd aproach -``` -Single training schedule -epoch train_loss valid_loss accuracy -1 0.610785 0.239165 0.923000 -2 0.392899 0.281254 0.937000 -3 0.268695 0.444383 0.909000 -4 0.162150 0.427744 0.931000 -5 0.109248 0.422351 0.948000 -6 0.061984 0.411351 0.947000 -7 0.033645 0.413174 0.951000 -8 0.018704 0.404264 0.947000 -Total time: 02:16 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.29712877, tensor(0.9565)] -``` \ No newline at end of file diff --git a/results/logs/cls/cls_all.md b/results/logs/cls/cls_all.md deleted file mode 100644 index b0eaa88..0000000 --- a/results/logs/cls/cls_all.md +++ /dev/null @@ -1,1133 +0,0 @@ -# Overall No Label Smoothing - -## DE BOOKS LSTM - -```bash -(fastai) julian@dl-box-spot:~/ulmfit-multilingual$ python -m ulmfit cls --dataset-path data/cls/de-books --base-lm-path ../data/wiki/de-100/models/sp30k/lstm_nl4.m --tokenizer='sp' --lang=de --name 'nl4' - train 20 --bs 40 --lr-sched=1cycle --num-cls-epochs=8 -Max vocab: 30000 -Cache dir: /home/julian/ulmfit-multilingual/data/cls/de-books/models/sp30k -Model dir: /home/julian/ulmfit-multilingual/data/cls/de-books/models/sp30k/lstm_nl4.m -Validation set not found using 10% of trn -Running tokenization lm... -Data lm, trn: 30600, val: 3400 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 2000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/julian/data/wiki/de-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/julian/data/wiki/de-100/models/sp30k/lstm_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.553707 3.077508 0.458865 -Total time: 09:49 -epoch train_loss valid_loss accuracy -1 3.233023 2.998775 0.469265 -2 3.130155 2.900699 0.481559 -3 3.038409 2.793384 0.494638 -4 2.970666 2.693831 0.506061 -5 2.899437 2.601532 0.515972 -6 2.803581 2.516531 0.526783 -7 2.732246 2.443080 0.536339 -8 2.675900 2.375012 0.544895 -9 2.636490 2.313508 0.553726 -10 2.604711 2.253531 0.562466 -11 2.550045 2.202728 0.570852 -12 2.501192 2.145478 0.579989 -13 2.484679 2.092014 0.588614 -14 2.409206 2.044224 0.596671 -15 2.344645 2.008057 0.603097 -16 2.346225 1.976991 0.608867 -17 2.313172 1.954794 0.612839 -18 2.269678 1.937210 0.615802 -19 2.277551 1.930322 0.617028 -20 2.246812 1.928822 0.617285 -Total time: 3:38:47 -/home/julian/ulmfit-multilingual/data/cls/de-books/models/sp30k -Saving info /home/julian/ulmfit-multilingual/data/cls/de-books/models/sp30k/lstm_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.406082 0.238703 0.905000 -2 0.313938 0.477311 0.865000 3 0.255491 0.228014 0.890000 -4 0.155123 0.384204 0.900000 -5 0.107264 0.374567 0.905000 -6 0.070755 0.468389 0.900000 -7 0.035681 0.243386 0.945000 -8 0.022694 0.242060 0.920000 -Total time: 05:49 -Saving models at /home/julian/ulmfit-multilingual/data/cls/de-books/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.43779454, tensor(0.9170)] -0.4377945363521576 -0.9169999957084656 -``` - -## DE BOOKS QRNN - -```bash -(fastai) julian@dl-box-spot:~/ulmfit-multilingual$ python -m ulmfit cls --dataset-path data/cls/de-books --base-lm-path ../data/wiki/de-100/models/sp15k/qrnn_nl4.m --tokenizer='sp' --lang=de --name 'nl4' - train 20 --bs 20 --lr-sched=1cycle --num-cls-epochs=8 - -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/julian/data/wiki/de-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/julian/data/wiki/de-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.865708 3.093886 0.446700 -Total time: 03:56 -epoch train_loss valid_loss accuracy -1 3.355898 2.985934 0.461723 -2 3.105310 2.841031 0.480453 -3 2.973803 2.708443 0.496344 -4 2.846477 2.598246 0.509789 -5 2.772356 2.528287 0.517500 -6 2.664962 2.439845 0.528694 -7 2.635260 2.375823 0.536753 -8 2.593652 2.320358 0.544354 -9 2.555928 2.274414 0.550421 -10 2.509033 2.230369 0.556924 -11 2.474522 2.189939 0.562351 -12 2.450867 2.152792 0.567782 -13 2.430727 2.118766 0.573289 -14 2.386325 2.086252 0.578170 -15 2.345515 2.061625 0.582117 -16 2.348084 2.041799 0.585584 -17 2.322886 2.025411 0.588263 -18 2.309043 2.016290 0.589943 -19 2.273257 2.011463 0.590589 -20 2.293612 2.010254 0.590786 -Total time: 1:44:20 -/home/julian/ulmfit-multilingual/data/cls/de-books/models/sp15k -Saving info /home/julian/ulmfit-multilingual/data/cls/de-books/models/sp15k/qrnn_nl4.m/info.json - -Single training schedule -epoch train_loss valid_loss accuracy -1 0.432808 0.280206 0.890000 -2 0.321715 0.225741 0.930000 -3 0.226988 0.583458 0.765000 -4 0.155420 0.281753 0.925000 -5 0.095285 0.330694 0.910000 -6 0.066643 0.484801 0.885000 -7 0.035414 0.373012 0.920000 -8 0.018784 0.392293 0.920000 -Total time: 02:43 -Saving models at /home/julian/ulmfit-multilingual/data/cls/de-books/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.6044611, tensor(0.9175)] -0.6044610738754272 -0.9175000190734863 - -``` - -## FR BOOKS LSTM - -```bash -python -m ulmfit cls --dataset-path data/cls/${LANG}-books --base-lm-path data/wiki-m/${LANG}-100/models/sp30k/lstm_nl4.m --lang=${LANG} --name 'nl4' - train 20 --bs 20 --num-cls-epochs=8 --lr-sched=single -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k/lstm_nl4.m -Validation set not found using 10% of trn -Running tokenization lm... -Data lm, trn: 33183, val: 3687 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 2000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki-m/fr-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki-m/fr-100/models/sp30k/lstm_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.790325 3.409294 0.367234 -Total time: 06:02 -epoch train_loss valid_loss accuracy -1 3.526303 3.326439 0.378936 -2 3.466923 3.226977 0.392378 -3 3.342312 3.111874 0.406997 -4 3.244619 2.992510 0.422330 -5 3.156150 2.877498 0.437467 -6 3.070326 2.762509 0.453874 -7 2.956969 2.651613 0.471552 -8 2.878008 2.535935 0.491058 -9 2.790110 2.438724 0.508560 -10 2.684145 2.323467 0.528415 -11 2.633781 2.231418 0.547093 -12 2.535126 2.143523 0.564889 -13 2.464436 2.055402 0.582077 -14 2.330094 1.989257 0.596582 -15 2.372371 1.924338 0.610048 -16 2.190224 1.866912 0.621738 -17 2.176868 1.834098 0.629221 -18 2.168293 1.809196 0.633879 -19 2.151132 1.797144 0.636382 -20 2.130476 1.793351 0.637044 -Total time: 2:30:05 -/home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k/lstm_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.314315 0.530879 0.865000 -2 0.336746 0.468635 0.865000 -3 0.255810 0.324242 0.870000 -4 0.149121 0.480570 0.885000 -5 0.093909 0.613743 0.890000 -6 0.091678 0.660452 0.885000 -7 0.049993 0.649642 0.910000 -8 0.034218 0.640008 0.910000 -Total time: 04:19 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.5418505, tensor(0.9100)] -0.5418505072593689 -0.9100000262260437 -``` - -## FR BOOKS QRNN - -``` -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 4.037580 3.312217 0.364410 -Total time: 01:44 -epoch train_loss valid_loss accuracy -1 3.709982 3.256320 0.371825 -2 3.459413 3.150574 0.386972 -3 3.296628 3.037327 0.402039 -4 3.186458 2.914899 0.418413 -5 3.092632 2.817097 0.431216 -6 2.966957 2.726081 0.442906 -7 2.924824 2.647339 0.453871 -8 2.818279 2.561596 0.466795 -9 2.773893 2.501994 0.475877 -10 2.736084 2.438490 0.485978 -11 2.688937 2.370927 0.496899 -12 2.615245 2.314875 0.506508 -13 2.583292 2.260717 0.515725 -14 2.535631 2.220295 0.522666 -15 2.466035 2.179093 0.530148 -16 2.461427 2.151952 0.535315 -17 2.390641 2.131065 0.538749 -18 2.376235 2.116927 0.541430 -19 2.407630 2.115370 0.542039 -20 2.391378 2.112687 0.542522 -Total time: 46:33 -/home/n-waves/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp15k -Saving info /home/n-waves/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.466671 0.534682 0.745000 -2 0.358965 0.372612 0.875000 -3 0.251557 0.311034 0.900000 -4 0.166484 0.585425 0.865000 -5 0.101803 0.726341 0.900000 -6 0.072025 0.587875 0.885000 -7 0.045328 0.760989 0.890000 -8 0.027765 0.727203 0.890000 -Total time: 01:17 -Saving models at /home/n-waves/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.55982095, tensor(0.8970)] -0.5598209500312805 -0.8970000147819519 -``` - -## JA BOOKS LSTM - -```bash -(fastai) julian@dl-box-spot:~/ulmfit-multilingual$ python -m ulmfit cls --dataset-path data/cls/ja-books --base-lm-path ../data/wiki/ja-100/models/sp30k/lstm_nl4.m --tokenizer='sp' --lang=ja --name 'nl4' - train 20 --bs 40 --lr-sched=1cycle --num-cls-epochs=8 - -Max vocab: 30000 -Cache dir: /home/julian/ulmfit-multilingual/data/cls/ja-books/models/sp30k -Model dir: /home/julian/ulmfit-multilingual/data/cls/ja-books/models/sp30k/lstm_nl4.m -Validation set not found using 10% of trn -Running tokenization lm... -Data lm, trn: 30600, val: 3399 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 1999 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/julian/data/wiki/ja-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/julian/data/wiki/ja-100/models/sp30k/lstm_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.847485 3.422460 0.376126 - -Total time: 06:28 -epoch train_loss valid_loss accuracy -1 3.588070 3.347799 0.386644 -2 3.481361 3.262498 0.398140 -3 3.393415 3.164333 0.410120 -4 3.305471 3.069857 0.421340 -5 3.246775 2.972239 0.433235 -6 3.127283 2.886817 0.443638 -7 3.085512 2.806101 0.454796 -8 3.016604 2.738343 0.463713 -9 2.947214 2.667280 0.473756 -10 2.919253 2.602734 0.483177 -12 2.799891 2.488073 0.501841 -13 2.772961 2.432371 0.511213 -14 2.706188 2.389203 0.518911 -15 2.653137 2.346985 0.526103 -16 2.624165 2.316532 0.531397 -17 2.578764 2.293599 0.535356 -18 2.568077 2.279164 0.537922 -19 2.529823 2.271825 0.539241 -20 2.558044 2.270341 0.539438 - -Total time: 2:35:18 -/home/julian/ulmfit-multilingual/data/cls/ja-books/models/sp30k -Saving info /home/julian/ulmfit-multilingual/data/cls/ja-books/models/sp30k/lstm_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.492803 0.584423 0.770000 -2 0.415298 0.697332 0.675000 -3 0.321692 0.742086 0.705000 -4 0.281904 1.092880 0.730000 -5 0.168274 1.050856 0.820000 -6 0.112483 0.895169 0.795000 -7 0.065752 1.082333 0.795000 -8 0.038848 1.138289 0.805000 -Total time: 05:46 -Saving models at /home/julian/ulmfit-multilingual/data/cls/ja-books/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.7825211, tensor(0.8514)] -0.78252112865448 -0.8514257073402405 -``` - -## JA BOOKS QRNN -``` -(fbashastai) julian@dl-box-spot:~/ulmfit-multilingual$ python -m ulmfit cls --dataset-path data/cls/ja-books --base-lm-path ../data/wiki/ja-100/models/sp15k/qrnn_nl4.m/ --tokenizer='sp' --lang=ja --name 'nl4' - train 20 --bs 20 --lr-sched=1cycle --num-cls-epochs=8 -Max vocab: 15000 -Cache dir: /home/julian/ulmfit-multilingual/data/cls/ja-books/models/sp15k -Model dir: /home/julian/ulmfit-multilingual/data/cls/ja-books/models/sp15k/qrnn_nl4.m -Validation set not found using 10% of trn -Running tokenization lm... -Data lm, trn: 30600, val: 3399 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 1999 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/julian/data/wiki/ja-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/julian/data/wiki/ja-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 4.507299 3.523476 0.375921 -Total time: 02:56 -epoch train_loss valid_loss accuracy -1 3.743898 3.405970 0.388302 -2 3.470031 3.225125 0.410167 -3 3.257088 3.058917 0.429464 -4 3.124567 2.932191 0.443150 -5 3.024665 2.830841 0.454890 -6 2.945257 2.753084 0.463231 -8 2.825861 2.637334 0.477286 -9 2.799877 2.594100 0.482632 -10 2.727558 2.548748 0.488096 -11 2.709687 2.512753 0.492869 -12 2.677212 2.475403 0.498646 -13 2.635228 2.447274 0.502558 -14 2.604527 2.418465 0.506572 -15 2.593657 2.394549 0.510170 -16 2.563881 2.378627 0.512493 -17 2.547412 2.363772 0.514894 -18 2.523795 2.355130 0.516159 -19 2.524199 2.351448 0.516690 -20 2.504680 2.350773 0.516817 -Total time: 1:19:10 -/home/julian/ulmfit-multilingual/data/cls/ja-books/models/sp15k -Saving info /home/julian/ulmfit-multilingual/data/cls/ja-books/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.541095 0.580246 0.715000 -2 0.459145 0.636574 0.720000 -3 0.370347 0.669777 0.710000 -4 0.271052 0.781472 0.740000 -5 0.198006 1.307968 0.750000 -6 0.139997 1.702155 0.715000 -7 0.112580 1.866208 0.735000 -8 0.065695 2.046943 0.740000 -Total time: 02:11 -Saving models at /home/julian/ulmfit-multilingual/data/cls/ja-books/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [1.0670499, tensor(0.8349)] -1.0670498609542847 -0.8349174857139587 -``` - - -## DE DVD LSTM - -```bash -(fastai) julian@dl-box-spot:~/ulmfit-multilingual$ python -m ulmfit cls --dataset-path data/cls/de-dvd --base-lm-path ../data/wiki/de-100/models/sp30k/lstm_nl4.m --tokenizer='sp' --lang=de --name 'nl4' - train 20 --bs 40 --lr-sched=1cycle --num-cls-epochs=8 - -Max vocab: 30000 -Cache dir: /home/julian/ulmfit-multilingual/data/cls/de-dvd/models/sp30k -Model dir: /home/julian/ulmfit-multilingual/data/cls/de-dvd/models/sp30k/lstm_nl4.m -Validation set not found using 10% of trn -Running tokenization lm... -Data lm, trn: 30600, val: 3400 - -Running tokenization cls... -Data cls, trn: 1800, val: 200 - -Running tokenization tst... -Data tst, trn: 200, val: 2000 - -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', -'', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, ' -hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/julian/data/wiki/de-100/models/sp30k/lstm_nl4.m/lm_best'), Po -sixPath('/home/julian/data/wiki/de-100/models/sp30k/lstm_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.564329 3.126153 0.456060 -Total time: 09:33 -epoch train_loss valid_loss accuracy -1 3.270905 3.030011 0.467729 -2 3.127578 2.923014 0.481434 -3 3.027712 2.804503 0.495315 -4 2.922042 2.698406 0.507350 -5 2.833169 2.605587 0.518419 -6 2.765501 2.521508 0.528258 -7 2.684195 2.443519 0.538367 -8 2.644001 2.373817 0.547404 -9 2.586362 2.309439 0.556455 -10 2.554237 2.253083 0.564804 -11 2.500762 2.196377 0.573611 -12 2.469062 2.144791 0.581450 -13 2.423278 2.093090 0.590096 -14 2.343388 2.043406 0.598047 -15 2.321417 2.008692 0.604748 -16 2.265463 1.972947 0.610544 -17 2.248210 1.948689 0.615224 -18 2.222042 1.934402 0.617739 -19 2.184187 1.926367 0.619161 -20 2.225068 1.925283 0.619336 -Total time: 3:47:59 - -Single training schedule -epoch train_loss valid_loss accuracy -1 0.494345 0.347763 0.875000 -2 0.424967 0.466457 0.810000 -3 0.321692 0.440244 0.870000 -4 0.212389 0.323907 0.895000 -5 0.142327 0.532973 0.900000 -6 0.080535 0.452185 0.885000 -7 0.039367 0.456267 0.895000 -8 0.021793 0.470200 0.890000 -Total time: 06:18 -Saving models at /home/julian/ulmfit-multilingual/data/cls/de-dvd/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.56412625, tensor(0.8835)] -0.5641262531280518 -0.8834999799728394 -``` - -## DE DVD QRNN - -```bash -(fastai) julian@dl-box-spot:~/ulmfit-multilingual$ python -m ulmfit cls --dataset-path data/cls/de-dvd --base-lm-path ../data/wiki/de-100/models/sp15k/qrnn_nl4.m --tokenizer='sp' --lang=de --name 'nl4' - train 20 --bs 20 --lr-sched=1cycle --num-cls-epochs=8 - -Max vocab: 15000 -Cache dir: /home/julian/ulmfit-multilingual/data/cls/de-dvd/models/sp15k -Model dir: /home/julian/ulmfit-multilingual/data/cls/de-dvd/models/sp15k/qrnn_nl4.m -Validation set not found using 10% of trn -Running tokenization lm... -Data lm, trn: 30600, val: 3400 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 2000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/julian/data/wiki/de-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/julian/data/wiki/de-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.849395 3.198631 0.440844 -Total time: 04:35 -epoch train_loss valid_loss accuracy -1 3.342962 3.041377 0.460921 -2 3.085583 2.859099 0.483095 -3 2.897264 2.715346 0.500916 -4 2.801560 2.604200 0.513484 -5 2.740901 2.519784 0.523829 -6 2.680627 2.449723 0.532085 -7 2.621618 2.391428 0.539286 -8 2.558427 2.341838 0.546166 -9 2.531594 2.295952 0.552346 -10 2.507932 2.255107 0.557783 -11 2.473801 2.214491 0.563843 -12 2.475930 2.174546 0.569883 -13 2.377409 2.133925 0.576207 -14 2.384114 2.097871 0.582316 -15 2.341124 2.069540 0.586640 -16 2.296975 2.042327 0.591220 -17 2.310770 2.020913 0.594684 -18 2.253782 2.009811 0.596532 -19 2.243294 2.002954 0.597705 -20 2.220592 2.001262 0.597922 -Total time: 2:06:03 -/home/julian/ulmfit-multilingual/data/cls/de-dvd/models/sp15k -Saving info /home/julian/ulmfit-multilingual/data/cls/de-dvd/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.404121 0.396830 0.840000 -2 0.340004 0.301888 0.890000 -3 0.231930 0.635746 0.815000 -4 0.173531 0.369872 0.915000 -5 0.112006 0.450329 0.930000 -6 0.068657 0.580160 0.920000 -7 0.025398 0.703478 0.925000 -8 0.015986 0.684468 0.920000 -Total time: 02:54 -Saving models at /home/julian/ulmfit-multilingual/data/cls/de-dvd/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.6911931, tensor(0.8840)] -0.6911931037902832 -0.8840000033378601 -``` - -## FR DVD LSTM - -```bash -(fastai) julian@dl-box-spot:~/ulmfit-multilingual$ python -m ulmfit cls --dataset-path data/cls/fr-dvd --base-lm-path ../data/wiki/fr-100/models/sp30k/lstm_nl4.m --tokenizer='sp' --lang=fr --name 'nl4' - train 20 --bs 40 --lr-sched=1cycle --num-cls-epochs=8 - -Max vocab: 30000 -Cache dir: /home/julian/ulmfit-multilingual/data/cls/fr-dvd/models/sp30k -Model dir: /home/julian/ulmfit-multilingual/data/cls/fr-dvd/models/sp30k/lstm_nl4.m -Validation set not found using 10% of trn -Running tokenization lm... -Data lm, trn: 12021, val: 1335 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 2000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/julian/data/wiki/fr-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/julian/data/wiki/fr-100/models/sp30k/lstm_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.825279 3.341784 0.382891 -Total time: 01:36 -epoch train_loss valid_loss accuracy -1 3.548075 3.280657 0.390619 -2 3.489078 3.206246 0.400057 -3 3.378426 3.099167 0.414316 -4 3.278241 2.985728 0.428153 -5 3.164112 2.868728 0.442793 -6 3.081279 2.740803 0.459863 -7 2.951635 2.615327 0.477424 -8 2.860259 2.511515 0.493157 -9 2.761055 2.386526 0.512397 -10 2.628587 2.277270 0.531014 -11 2.572315 2.181750 0.548689 -12 2.452535 2.083487 0.566041 -13 2.389231 1.998139 0.581409 -14 2.313358 1.927491 0.594620 -15 2.263673 1.873754 0.605384 -16 2.196958 1.827021 0.614506 -17 2.169217 1.797702 0.619863 -18 2.126882 1.777056 0.623906 -19 2.116131 1.767786 0.625270 -20 2.090418 1.765665 0.625703 -Total time: 37:20 -/home/julian/ulmfit-multilingual/data/cls/fr-dvd/models/sp30k -Saving info /home/julian/ulmfit-multilingual/data/cls/fr-dvd/models/sp30k/lstm_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.586308 0.504827 0.750000 -2 0.478227 0.411526 0.860000 -3 0.417158 0.314054 0.890000 -4 0.286224 0.263725 0.900000 -5 0.163930 0.387664 0.880000 -6 0.095715 0.282535 0.930000 -7 0.051098 0.294014 0.930000 -8 0.028741 0.301007 0.930000 -Total time: 02:57 -Saving models at /home/julian/ulmfit-multilingual/data/cls/fr-dvd/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.5228756, tensor(0.8920)] -0.5228756070137024 -0.8920000195503235 -``` - -## FR DVD QRNN - -```bash -(fastai) julian@dl-box-spot:~/ulmfit-multilingual$ python -m ulmfit cls --dataset-path data/cls/fr-dvd --base-lm-path ../data/wiki/fr-100/models/sp15k/qrnn_nl4.m --tokenizer='sp' --lang=fr --name 'nl4' - train 20 --bs 20 --lr-sched=1cycle --num-cls-epochs=8 - -Max vocab: 15000 -Cache dir: /home/julian/ulmfit-multilingual/data/cls/fr-dvd/models/sp15k -Model dir: /home/julian/ulmfit-multilingual/data/cls/fr-dvd/models/sp15k/qrnn_nl4.m -Validation set not found using 10% of trn -Running tokenization lm... -Data lm, trn: 12021, val: 1335 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 2000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', 's', '▁.', "'", '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/julian/data/wiki/fr-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/julian/data/wiki/fr-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 4.098866 3.325340 0.371045 -Total time: 00:38 -epoch train_loss valid_loss accuracy -1 3.766878 3.274258 0.378042 -2 3.552927 3.168638 0.392944 -3 3.344454 3.048812 0.409167 -4 3.237171 2.934779 0.422993 -5 3.116725 2.819759 0.438676 -6 3.054311 2.695264 0.454934 -7 2.959086 2.596110 0.469993 -8 2.854205 2.502099 0.482847 -9 2.764695 2.409211 0.497850 -10 2.717763 2.330320 0.510213 -11 2.625616 2.237966 0.525547 -12 2.569468 2.172560 0.536735 -13 2.478079 2.097768 0.550401 -14 2.392206 2.028578 0.562645 -15 2.354875 1.981342 0.571711 -16 2.323998 1.942189 0.578875 -17 2.289521 1.915991 0.584164 -18 2.245144 1.899356 0.587052 -19 2.261464 1.892248 0.588192 -20 2.215216 1.889717 0.588575 -Total time: 17:34 -/home/julian/ulmfit-multilingual/data/cls/fr-dvd/models/sp15k -Saving info /home/julian/ulmfit-multilingual/data/cls/fr-dvd/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.483799 0.268736 0.895000 -2 0.401154 0.192629 0.930000 -3 0.269437 0.262472 0.905000 -4 0.200453 0.246643 0.915000 -5 0.119864 0.434285 0.900000 -6 0.073772 0.423398 0.950000 -7 0.039172 0.456106 0.945000 -8 0.026730 0.401908 0.945000 -Total time: 01:13 -Saving models at /home/julian/ulmfit-multilingual/data/cls/fr-dvd/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.75294757, tensor(0.8885)] -0.7529475688934326 -``` - -## JA DVD LSTM - -```bash -(fastai) julian@dl-box-spot:~/ulmfit-multilingual$ python -m ulmfit cls --dataset-path data/cls/ja-dvd --base-lm-path ../data/wiki/ja-100/models/sp30k/lstm_nl4.m --tokenizer='sp' --lang=ja --name 'nl4' - train 20 --bs 40 --lr-sched=1cycle --num-cls-epochs=8 - -Max vocab: 30000 -Cache dir: /home/julian/ulmfit-multilingual/data/cls/ja-dvd/models/sp30k -Model dir: /home/julian/ulmfit-multilingual/data/cls/ja-dvd/models/sp30k/lstm_nl4.m -Validation set not found using 10% of trn -Running tokenization lm... -Data lm, trn: 30600, val: 3400 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 2000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/julian/data/wiki/ja-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/julian/data/wiki/ja-100/models/sp30k/lstm_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.904242 3.480109 0.375665 -Total time: 04:37 -epoch train_loss valid_loss accuracy -1 3.671750 3.403708 0.386209 -2 3.565445 3.310030 0.398628 -3 3.443218 3.204042 0.411568 -4 3.374648 3.098158 0.424107 -5 3.288731 3.005343 0.435621 -6 3.187409 2.913090 0.446806 -7 3.135114 2.829881 0.457120 -8 3.073141 2.753949 0.467695 -9 2.996589 2.682856 0.478041 -10 2.909743 2.613899 0.487629 -11 2.859827 2.550690 0.497565 -12 2.818285 2.492902 0.507112 -13 2.779268 2.435685 0.516448 -14 2.718145 2.387462 0.525241 -15 2.664007 2.346267 0.532140 -16 2.641343 2.312850 0.537994 -17 2.599257 2.288488 0.542193 -18 2.579481 2.274002 0.544809 -19 2.571687 2.267283 0.545827 -20 2.560343 2.265548 0.546052 -Total time: 1:47:55 -/home/julian/ulmfit-multilingual/data/cls/ja-dvd/models/sp30k -Saving info /home/julian/ulmfit-multilingual/data/cls/ja-dvd/models/sp30k/lstm_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.525819 0.402837 0.830000 -2 0.453459 0.425072 0.820000 -3 0.401383 0.482119 0.770000 -4 0.337860 0.502686 0.775000 -5 0.234284 0.805287 0.805000 -6 0.134409 0.729153 0.815000 -7 0.072370 0.895428 0.805000 -8 0.040945 0.832303 0.800000 -Total time: 03:09 -Saving models at /home/julian/ulmfit-multilingual/data/cls/ja-dvd/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.72511286, tensor(0.8395)] -0.7251128554344177 -0.8395000100135803 -``` - -## JA DVD QRNN - -```bash -(fastai) julian@dl-box-spot:~/ulmfit-multilingual$ python -m ulmfit cls --dataset-path data/cls/ja-dvd --base-lm-path ../data/wiki/ja-100/models/sp15k/qrnn_nl4.m --tokenizer='sp' --lang=ja --name 'nl4' - train 20 --bs 20 --lr-sched=1cycle --num-cls-epochs=8 - -Max vocab: 15000 -Cache dir: /home/julian/ulmfit-multilingual/data/cls/ja-dvd/models/sp15k -Model dir: /home/julian/ulmfit-multilingual/data/cls/ja-dvd/models/sp15k/qrnn_nl4.m -Validation set not found using 10% of trn -Running tokenization lm... -Data lm, trn: 30600, val: 3400 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 2000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/julian/data/wiki/ja-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/julian/data/wiki/ja-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 4.426582 3.629627 0.375135 -Total time: 03:07 -epoch train_loss valid_loss accuracy -1 3.805171 3.477644 0.391030 -2 3.488298 3.267601 0.414995 -3 3.285561 3.090409 0.434407 -4 3.132227 2.956780 0.448842 -5 3.038989 2.863759 0.458349 -6 2.945142 2.775436 0.468323 -7 2.899916 2.711322 0.476454 -8 2.868184 2.657508 0.482740 -9 2.826075 2.605312 0.489217 -10 2.785546 2.566676 0.494568 -11 2.702636 2.519315 0.500817 -12 2.713773 2.482188 0.505794 -13 2.672333 2.451759 0.510435 -14 2.644996 2.420648 0.515061 -15 2.613210 2.397427 0.518870 -16 2.569429 2.370109 0.522590 -17 2.570986 2.355417 0.525179 -18 2.537443 2.344698 0.527021 -19 2.520152 2.340001 0.527741 -20 2.538636 2.338896 0.527936 -Total time: 1:25:44 -/home/julian/ulmfit-multilingual/data/cls/ja-dvd/models/sp15k -Saving info /home/julian/ulmfit-multilingual/data/cls/ja-dvd/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.523285 0.524549 0.745000 -2 0.455993 0.435848 0.800000 -3 0.330351 0.636708 0.815000 -4 0.240356 0.732489 0.820000 -5 0.148504 0.834724 0.820000 -6 0.095455 1.066170 0.810000 -7 0.057415 1.286681 0.815000 -8 0.041691 1.246479 0.825000 -Total time: 02:09 -Saving models at /home/julian/ulmfit-multilingual/data/cls/ja-dvd/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [1.0560615, tensor(0.8405)] -1.0560615062713623 -0.840499997138977 -``` - -## DE MUSIC QRNN - -```bash -(fastai) julian@dl-box-spot:~/ulmfit-multilingual$ python -m ulmfit cls --dataset-path data/cls/de-music --base-lm-path ../data/wiki/de-100/models/sp15k/qrnn_nl4.m --tokenizer='sp' --lang=de --name 'nl4' - train 20 --bs 20 --lr-sched=1cycle --num-cls-epochs=8 - -Max vocab: 15000 -Cache dir: /home/julian/ulmfit-multilingual/data/cls/de-music/models/sp15k -Model dir: /home/julian/ulmfit-multilingual/data/cls/de-music/models/sp15k/qrnn_nl4.m -Validation set not found using 10% of trn -Running tokenization lm... -Data lm, trn: 30600, val: 3400 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 2000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/julian/data/wiki/de-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/julian/data/wiki/de-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.911617 3.177610 0.439701 -Total time: 02:34 -epoch train_loss valid_loss accuracy -1 3.346166 3.029566 0.459678 -2 3.049080 2.844766 0.481775 -3 2.894477 2.690567 0.499936 -4 2.759127 2.573302 0.513566 -5 2.697380 2.489542 0.523565 -6 2.656406 2.407121 0.533571 -7 2.627402 2.345922 0.542248 -8 2.536563 2.284745 0.550096 -9 2.526928 2.235896 0.557415 -10 2.438895 2.186658 0.564642 -11 2.407938 2.137030 0.572150 -12 2.372165 2.094725 0.578804 -13 2.396612 2.055748 0.585109 -14 2.288415 2.010951 0.592412 -15 2.261529 1.980380 0.597678 -16 2.253213 1.954077 0.602550 -17 2.205776 1.932198 0.606216 -18 2.215024 1.921017 0.608381 -19 2.202204 1.913123 0.609632 -20 2.191336 1.911685 0.609869 -Total time: 1:13:39 -/home/julian/ulmfit-multilingual/data/cls/de-music/models/sp15k -Saving info /home/julian/ulmfit-multilingual/data/cls/de-music/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.431190 0.313631 0.845000 -2 0.332442 0.370172 0.855000 -3 0.231262 0.320309 0.885000 -4 0.169519 0.521468 0.850000 -5 0.119292 0.451171 0.885000 -6 0.090633 0.377672 0.905000 -7 0.040820 0.412475 0.895000 -8 0.020507 0.423591 0.890000 -Total time: 01:44 -Saving models at /home/julian/ulmfit-multilingual/data/cls/de-music/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.5168868, tensor(0.9145)] -0.5168867707252502 -0.9144999980926514 -``` - -## JA MUSIC QRNN - -```bash -(fastai) julian@dl-box-spot:~/ulmfit-multilingual$ python -m ulmfit cls --dataset-path data/cls/ja-music --base-lm-path ../data/wiki/ja-100/models/sp15k/qrnn_nl4.m --tokenizer='sp' --lang=ja --name 'nl4' - train 20 --bs 20 --lr-sched=1cycle --num-cls-epochs=8 - -Max vocab: 15000 -Cache dir: /home/julian/ulmfit-multilingual/data/cls/ja-music/models/sp15k -Model dir: /home/julian/ulmfit-multilingual/data/cls/ja-music/models/sp15k/qrnn_nl4.m -Validation set not found using 10% of trn -Running tokenization lm... -Data lm, trn: 30600, val: 3399 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 1999 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/julian/data/wiki/ja-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/julian/data/wiki/ja-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 4.556378 3.658723 0.373246 -Total time: 02:37 -epoch train_loss valid_loss accuracy -1 3.817752 3.491593 0.390537 -2 3.540558 3.294715 0.413274 -3 3.334061 3.111620 0.433772 -4 3.125960 2.963380 0.450106 -5 3.041799 2.858605 0.461084 -6 2.971459 2.762077 0.472102 -7 2.871483 2.690570 0.480306 -8 2.835200 2.623168 0.488692 -9 2.770324 2.581520 0.493827 -10 2.742213 2.516730 0.502326 -11 2.710455 2.471337 0.509197 -12 2.644516 2.412517 0.518069 -13 2.621891 2.378112 0.522952 -14 2.571613 2.346067 0.527796 -15 2.571703 2.311945 0.533637 -16 2.518094 2.287681 0.537693 -17 2.500024 2.266639 0.540884 -18 2.506000 2.254209 0.543006 -19 2.479864 2.249063 0.543790 -20 2.442601 2.247427 0.544025 -Total time: 1:11:55 -/home/julian/ulmfit-multilingual/data/cls/ja-music/models/sp15k -Saving info /home/julian/ulmfit-multilingual/data/cls/ja-music/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.506969 0.522131 0.770000 -2 0.441373 0.474673 0.830000 -3 0.353533 0.529668 0.820000 -4 0.254606 1.358136 0.750000 -5 0.234732 1.085676 0.800000 -6 0.150228 1.233079 0.820000 -7 0.075722 1.429222 0.825000 -8 0.076240 1.380577 0.815000 -Total time: 01:48 -Saving models at /home/julian/ulmfit-multilingual/data/cls/ja-music/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.9297137, tensor(0.8619)] -0.9297137260437012 -0.8619309663772583 -``` - -## FR MUSIC QRNN - -```bash -(fastai) julian@dl-box-spot:~/ulmfit-multilingual$ python -m ulmfit cls --dataset-path data/cls/fr-music --base-lm-path ../data/wiki/fr-100/models/sp15k/qrnn_nl4.m --tokenizer='sp' --lang=fr --name 'nl4' - train 20 --bs 20 --lr-sched=1cycle --num-cls-epochs=8 - -Max vocab: 15000 -Cache dir: /home/julian/ulmfit-multilingual/data/cls/fr-music/models/sp15k -Model dir: /home/julian/ulmfit-multilingual/data/cls/fr-music/models/sp15k/qrnn_nl4.m -Validation set not found using 10% of trn -Running tokenization lm... -Data lm, trn: 17946, val: 1993 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 2000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', 's', '▁.', "'", '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/julian/data/wiki/fr-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/julian/data/wiki/fr-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 4.065422 3.303533 0.375714 -Total time: 01:45 -epoch train_loss valid_loss accuracy -1 3.680984 3.222789 0.387669 -2 3.401064 3.078735 0.408392 -3 3.229715 2.933169 0.427501 -4 3.068981 2.768277 0.449853 -5 2.927764 2.633488 0.468013 -6 2.836827 2.490237 0.489544 -7 2.658033 2.333551 0.515255 -8 2.586924 2.215265 0.536613 -9 2.499067 2.099867 0.558580 -10 2.428197 1.985598 0.581145 -11 2.279977 1.884761 0.602102 -12 2.205373 1.801624 0.618907 -13 2.160284 1.727773 0.634058 -14 2.004652 1.659159 0.647294 -15 1.994962 1.622999 0.655824 -16 1.908620 1.570057 0.666497 -17 1.871175 1.541314 0.672019 -18 1.873397 1.524069 0.675352 -19 1.843536 1.519989 0.676418 -20 1.922381 1.517763 0.676880 -Total time: 48:16 -/home/julian/ulmfit-multilingual/data/cls/fr-music/models/sp15k -Saving info /home/julian/ulmfit-multilingual/data/cls/fr-music/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.391277 0.377619 0.870000 -2 0.293999 0.289887 0.900000 -3 0.199309 0.408675 0.915000 -4 0.134351 0.413012 0.915000 -5 0.100232 0.504149 0.925000 -6 0.060509 0.593078 0.925000 -7 0.044179 0.499701 0.930000 -8 0.050736 0.508113 0.920000 -Total time: 02:07 -Saving models at /home/julian/ulmfit-multilingual/data/cls/fr-music/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.41011345, tensor(0.9285)] -0.41011345386505127 -0.9284999966621399 -``` - -## DE MUSIC LSTM - -```bash -(fastai) julian@dl-box-spot:~/ulmfit-multilingual$ python -m ulmfit cls --dataset-path data/cls/de-music --base-lm-path ../data/wiki/de-100/models/sp30k/lstm_nl4.m --tokenizer='sp' --lang=de --name 'nl4' - train 20 --bs 20 --lr-sched=1cycle --num-cls-epochs=8 - -Max vocab: 30000 -Cache dir: /home/julian/ulmfit-multilingual/data/cls/de-music/models/sp30k -Model dir: /home/julian/ulmfit-multilingual/data/cls/de-music/models/sp30k/lstm_nl4.m -Validation set not found using 10% of trn -Running tokenization lm... -Data lm, trn: 30600, val: 3400 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 2000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/julian/data/wiki/de-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/julian/data/wiki/de-100/models/sp30k/lstm_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 4.002521 3.526757 0.376006 -Total time: 06:32 -epoch train_loss valid_loss accuracy -1 3.738713 3.425875 0.389128 -2 3.607711 3.316313 0.403729 -3 3.441798 3.188569 0.418975 -4 3.347294 3.070090 0.432434 -5 3.261581 2.959181 0.447016 -6 3.161360 2.850135 0.460998 -7 3.092968 2.747133 0.475559 -8 2.999860 2.661642 0.488256 -9 2.946614 2.572933 0.502634 -10 2.834917 2.477213 0.517815 -11 2.768194 2.394673 0.532000 -12 2.743433 2.325108 0.545050 -13 2.596613 2.255300 0.557315 -14 2.347057 1.965440 0.616312 -15 2.289672 1.907880 0.626826 -16 2.238047 1.870341 0.634285 -17 2.169917 1.829818 0.641958 -18 2.145997 1.811395 0.645616 -19 2.111888 1.800622 0.647706 -20 2.067247 1.797927 0.648219 -Total time: 3:50:18 -/home/julian/ulmfit-multilingual/data/cls/de-music/models/sp30k -Saving info /home/julian/ulmfit-multilingual/data/cls/de-music/models/sp30k/lstm_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.392259 0.319764 0.875000 -2 0.373723 0.401961 0.850000 -3 0.315902 0.415566 0.850000 -4 0.185113 0.312382 0.890000 -5 0.122869 0.399712 0.865000 -6 0.084130 0.435429 0.910000 -7 0.057294 0.394715 0.890000 -8 0.028046 0.391238 0.900000 -Total time: 06:34 -Saving models at /home/julian/ulmfit-multilingual/data/cls/de-music/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.4154407, tensor(0.9210)] -0.41544070839881897 -0.9210000038146973 -``` - -## JA MUSIC LSTM - -```bash -(fastai) julian@dl-box-spot:~/ulmfit-multilingual$ python -m ulmfit cls --dataset-path data/cls/ja-music --base-lm-path ../data/wiki/ja-100/models/sp30k/lstm_nl4.m --tokenizer='sp' --lang=ja --name 'nl4' - train 20 --bs 20 --lr-sched=1cycle --num-cls-epochs=8 - -Data lm, trn: 30600, val: 3399 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 1999 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/julian/data/wiki/ja-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/julian/data/wiki/ja-100/models/sp30k/lstm_nl4.m/../itos')] - -epoch train_loss valid_loss accuracy -1 3.680415 3.162607 0.451358 -Total time: 09:43 -epoch train_loss valid_loss accuracy -1 3.258039 3.043514 0.467028 -2 3.082901 2.909904 0.482319 -3 3.009017 2.784519 0.496660 -4 2.900604 2.671213 0.510406 -5 2.776977 2.570581 0.522192 -6 2.789654 2.488552 0.532739 -7 2.710876 2.407913 0.543447 -8 2.655036 2.342028 0.553344 -9 2.571593 2.281001 0.562552 -10 2.539299 2.207963 0.574177 -11 2.466461 2.139726 0.585225 -12 2.441152 2.081656 0.595266 -13 2.434502 2.018719 0.606514 -14 2.576859 2.190329 0.569373 -15 2.543341 2.137856 0.579508 -16 2.467283 2.092796 0.587677 -17 2.417593 2.061508 0.593782 -18 2.375962 2.038786 0.598027 -19 2.391491 2.029075 0.599871 -20 2.352595 2.026604 0.600235 -Total time: 2:41:18 -/home/julian/ulmfit-multilingual/data/cls/ja-music/models/sp30k -Saving info /home/julian/ulmfit-multilingual/data/cls/ja-music/models/sp30k/lstm_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.458052 0.371875 0.860000 -2 0.473817 0.539201 0.730000 -3 0.423880 0.390433 0.845000 -4 0.310703 0.402607 0.855000 -5 0.211760 0.607136 0.865000 -6 0.108535 0.845904 0.860000 -7 0.053125 0.897018 0.860000 -8 0.024544 0.891663 0.855000 -Total time: 04:29 -Saving models at /home/julian/ulmfit-multilingual/data/cls/ja-music/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.777393, tensor(0.8644)] -0.7773929834365845 -0.8644322156906128 -``` - -## FR MUSIC LSTM - -```bash -(fastai) julian@dl-box-spot:~/ulmfit-multilingual$ python -m ulmfit cls --dataset-path data/cls/fr-music --base-lm-path ../data/wiki/fr-100/models/sp30k/lstm_nl4.m --tokenizer='sp' --lang=fr --name 'nl4' - train 20 --bs 40 --lr-sched=1cycle --num-cls-epochs=8 - -Running tokenization tst... -Data tst, trn: 200, val: 2000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/julian/data/wiki/fr-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/julian/data/wiki/fr-100/models/sp30k/lstm_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.686174 3.271906 0.394277 -Total time: 02:47 -epoch train_loss valid_loss accuracy -1 3.490278 3.197725 0.403639 -2 3.356067 3.100815 0.415938 -3 3.269037 2.970971 0.433533 -4 3.102959 2.819620 0.453009 -5 2.957009 2.647919 0.478246 -6 2.793691 2.481614 0.504928 -7 2.646251 2.316360 0.534853 -8 2.532166 2.140370 0.566817 -9 2.361445 1.982554 0.596333 -10 2.258446 1.855159 0.621765 -11 2.155252 1.772740 0.640348 -12 2.071291 1.668775 0.660405 -13 1.887608 1.579711 0.677771 -14 1.873631 1.493046 0.694815 -15 1.824689 1.438728 0.705296 -16 1.766544 1.398732 0.714093 -17 1.646138 1.372478 0.719408 -18 1.684073 1.350950 0.723633 -19 1.650602 1.344994 0.724889 -20 1.602114 1.341957 0.725314 -Total time: 1:04:53 -/home/julian/ulmfit-multilingual/data/cls/fr-music/models/sp30k -Saving info /home/julian/ulmfit-multilingual/data/cls/fr-music/models/sp30k/lstm_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.415938 0.214479 0.930000 -2 0.373839 0.334263 0.880000 -3 0.317772 0.660272 0.795000 -4 0.207807 0.440546 0.880000 -5 0.146999 0.377026 0.890000 -6 0.095834 0.288273 0.925000 -7 0.048218 0.350355 0.895000 -8 0.023682 0.325598 0.915000 -Total time: 03:18 -Saving models at /home/julian/ulmfit-multilingual/data/cls/fr-music/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.32345334, tensor(0.9295)] -0.3234533369541168 -0.9294999837875366 -``` \ No newline at end of file diff --git a/results/logs/cls/ulmfit on cls lstm.md b/results/logs/cls/ulmfit on cls lstm.md deleted file mode 100644 index 4d99729..0000000 --- a/results/logs/cls/ulmfit on cls lstm.md +++ /dev/null @@ -1,577 +0,0 @@ -# Overall - -## DE BOOKS LSTM - -```bash -(fastai) julian@dl-box-spot:~/ulmfit-multilingual$ python -m ulmfit cls --dataset-path data/cls/de-books --base-lm-path ../data/wiki/de-100/models/sp30k/lstm_nl4.m --tokenizer='sp' --lang=de --name 'nl4' - train 20 --bs 40 --lr-sched=1cycle --num-cls-epochs=8 -Max vocab: 30000 -Cache dir: /home/julian/ulmfit-multilingual/data/cls/de-books/models/sp30k -Model dir: /home/julian/ulmfit-multilingual/data/cls/de-books/models/sp30k/lstm_nl4.m -Validation set not found using 10% of trn -Running tokenization lm... -Data lm, trn: 30600, val: 3400 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 2000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/julian/data/wiki/de-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/julian/data/wiki/de-100/models/sp30k/lstm_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.553707 3.077508 0.458865 -Total time: 09:49 -epoch train_loss valid_loss accuracy -1 3.233023 2.998775 0.469265 -2 3.130155 2.900699 0.481559 -3 3.038409 2.793384 0.494638 -4 2.970666 2.693831 0.506061 -5 2.899437 2.601532 0.515972 -6 2.803581 2.516531 0.526783 -7 2.732246 2.443080 0.536339 -8 2.675900 2.375012 0.544895 -9 2.636490 2.313508 0.553726 -10 2.604711 2.253531 0.562466 -11 2.550045 2.202728 0.570852 -12 2.501192 2.145478 0.579989 -13 2.484679 2.092014 0.588614 -14 2.409206 2.044224 0.596671 -15 2.344645 2.008057 0.603097 -16 2.346225 1.976991 0.608867 -17 2.313172 1.954794 0.612839 -18 2.269678 1.937210 0.615802 -19 2.277551 1.930322 0.617028 -20 2.246812 1.928822 0.617285 -Total time: 3:38:47 -/home/julian/ulmfit-multilingual/data/cls/de-books/models/sp30k -Saving info /home/julian/ulmfit-multilingual/data/cls/de-books/models/sp30k/lstm_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.406082 0.238703 0.905000 -2 0.313938 0.477311 0.865000 3 0.255491 0.228014 0.890000 -4 0.155123 0.384204 0.900000 -5 0.107264 0.374567 0.905000 -6 0.070755 0.468389 0.900000 -7 0.035681 0.243386 0.945000 -8 0.022694 0.242060 0.920000 -Total time: 05:49 -Saving models at /home/julian/ulmfit-multilingual/data/cls/de-books/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.43779454, tensor(0.9170)] -0.4377945363521576 -0.9169999957084656 -``` - -## FR BOOKS LSTM - -```bash -python -m ulmfit cls --dataset-path data/cls/${LANG}-books --base-lm-path data/wiki-m/${LANG}-100/models/sp30k/lstm_nl4.m --lang=${LANG} --name 'nl4' - train 20 --bs 20 --num-cls-epochs=8 --lr-sched=single -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k/lstm_nl4.m -Validation set not found using 10% of trn -Running tokenization lm... -Data lm, trn: 33183, val: 3687 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 2000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki-m/fr-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki-m/fr-100/models/sp30k/lstm_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.790325 3.409294 0.367234 -Total time: 06:02 -epoch train_loss valid_loss accuracy -1 3.526303 3.326439 0.378936 -2 3.466923 3.226977 0.392378 -3 3.342312 3.111874 0.406997 -4 3.244619 2.992510 0.422330 -5 3.156150 2.877498 0.437467 -6 3.070326 2.762509 0.453874 -7 2.956969 2.651613 0.471552 -8 2.878008 2.535935 0.491058 -9 2.790110 2.438724 0.508560 -10 2.684145 2.323467 0.528415 -11 2.633781 2.231418 0.547093 -12 2.535126 2.143523 0.564889 -13 2.464436 2.055402 0.582077 -14 2.330094 1.989257 0.596582 -15 2.372371 1.924338 0.610048 -16 2.190224 1.866912 0.621738 -17 2.176868 1.834098 0.629221 -18 2.168293 1.809196 0.633879 -19 2.151132 1.797144 0.636382 -20 2.130476 1.793351 0.637044 -Total time: 2:30:05 -/home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k/lstm_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.314315 0.530879 0.865000 -2 0.336746 0.468635 0.865000 -3 0.255810 0.324242 0.870000 -4 0.149121 0.480570 0.885000 -5 0.093909 0.613743 0.890000 -6 0.091678 0.660452 0.885000 -7 0.049993 0.649642 0.910000 -8 0.034218 0.640008 0.910000 -Total time: 04:19 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.5418505, tensor(0.9100)] -0.5418505072593689 -0.9100000262260437 -``` - -## JA BOOKS LSTM - -```bash -(fastai) julian@dl-box-spot:~/ulmfit-multilingual$ python -m ulmfit cls --dataset-path data/cls/ja-books --base-lm-path ../data/wiki/ja-100/models/sp30k/lstm_nl4.m --tokenizer='sp' --lang=ja --name 'nl4' - train 20 --bs 40 --lr-sched=1cycle --num-cls-epochs=8 - -Max vocab: 30000 -Cache dir: /home/julian/ulmfit-multilingual/data/cls/ja-books/models/sp30k -Model dir: /home/julian/ulmfit-multilingual/data/cls/ja-books/models/sp30k/lstm_nl4.m -Validation set not found using 10% of trn -Running tokenization lm... -Data lm, trn: 30600, val: 3399 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 1999 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/julian/data/wiki/ja-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/julian/data/wiki/ja-100/models/sp30k/lstm_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.847485 3.422460 0.376126 - -Total time: 06:28 -epoch train_loss valid_loss accuracy -1 3.588070 3.347799 0.386644 -2 3.481361 3.262498 0.398140 -3 3.393415 3.164333 0.410120 -4 3.305471 3.069857 0.421340 -5 3.246775 2.972239 0.433235 -6 3.127283 2.886817 0.443638 -7 3.085512 2.806101 0.454796 -8 3.016604 2.738343 0.463713 -9 2.947214 2.667280 0.473756 -10 2.919253 2.602734 0.483177 -12 2.799891 2.488073 0.501841 -13 2.772961 2.432371 0.511213 -14 2.706188 2.389203 0.518911 -15 2.653137 2.346985 0.526103 -16 2.624165 2.316532 0.531397 -17 2.578764 2.293599 0.535356 -18 2.568077 2.279164 0.537922 -19 2.529823 2.271825 0.539241 -20 2.558044 2.270341 0.539438 - -Total time: 2:35:18 -/home/julian/ulmfit-multilingual/data/cls/ja-books/models/sp30k -Saving info /home/julian/ulmfit-multilingual/data/cls/ja-books/models/sp30k/lstm_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.492803 0.584423 0.770000 -2 0.415298 0.697332 0.675000 -3 0.321692 0.742086 0.705000 -4 0.281904 1.092880 0.730000 -5 0.168274 1.050856 0.820000 -6 0.112483 0.895169 0.795000 -7 0.065752 1.082333 0.795000 -8 0.038848 1.138289 0.805000 -Total time: 05:46 -Saving models at /home/julian/ulmfit-multilingual/data/cls/ja-books/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.7825211, tensor(0.8514)] -0.78252112865448 -0.8514257073402405 -``` - -## DE DVD LSTM - -```bash -(fastai) julian@dl-box-spot:~/ulmfit-multilingual$ python -m ulmfit cls --dataset-path data/cls/de-dvd --base-lm-path ../data/wiki/de-100/models/sp30k/lstm_nl4.m --tokenizer='sp' --lang=de --name 'nl4' - train 20 --bs 40 --lr-sched=1cycle --num-cls-epochs=8 - -Max vocab: 30000 -Cache dir: /home/julian/ulmfit-multilingual/data/cls/de-dvd/models/sp30k -Model dir: /home/julian/ulmfit-multilingual/data/cls/de-dvd/models/sp30k/lstm_nl4.m -Validation set not found using 10% of trn -Running tokenization lm... -Data lm, trn: 30600, val: 3400 - -Running tokenization cls... -Data cls, trn: 1800, val: 200 - -Running tokenization tst... -Data tst, trn: 200, val: 2000 - -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', -'', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, ' -hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/julian/data/wiki/de-100/models/sp30k/lstm_nl4.m/lm_best'), Po -sixPath('/home/julian/data/wiki/de-100/models/sp30k/lstm_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.564329 3.126153 0.456060 -Total time: 09:33 -epoch train_loss valid_loss accuracy -1 3.270905 3.030011 0.467729 -2 3.127578 2.923014 0.481434 -3 3.027712 2.804503 0.495315 -4 2.922042 2.698406 0.507350 -5 2.833169 2.605587 0.518419 -6 2.765501 2.521508 0.528258 -7 2.684195 2.443519 0.538367 -8 2.644001 2.373817 0.547404 -9 2.586362 2.309439 0.556455 -10 2.554237 2.253083 0.564804 -11 2.500762 2.196377 0.573611 -12 2.469062 2.144791 0.581450 -13 2.423278 2.093090 0.590096 -14 2.343388 2.043406 0.598047 -15 2.321417 2.008692 0.604748 -16 2.265463 1.972947 0.610544 -17 2.248210 1.948689 0.615224 -18 2.222042 1.934402 0.617739 -19 2.184187 1.926367 0.619161 -20 2.225068 1.925283 0.619336 -Total time: 3:47:59 - -Single training schedule -epoch train_loss valid_loss accuracy -1 0.494345 0.347763 0.875000 -2 0.424967 0.466457 0.810000 -3 0.321692 0.440244 0.870000 -4 0.212389 0.323907 0.895000 -5 0.142327 0.532973 0.900000 -6 0.080535 0.452185 0.885000 -7 0.039367 0.456267 0.895000 -8 0.021793 0.470200 0.890000 -Total time: 06:18 -Saving models at /home/julian/ulmfit-multilingual/data/cls/de-dvd/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.56412625, tensor(0.8835)] -0.5641262531280518 -0.8834999799728394 -``` - -## FR DVD LSTM - -```bash -(fastai) julian@dl-box-spot:~/ulmfit-multilingual$ python -m ulmfit cls --dataset-path data/cls/fr-dvd --base-lm-path ../data/wiki/fr-100/models/sp30k/lstm_nl4.m --tokenizer='sp' --lang=fr --name 'nl4' - train 20 --bs 40 --lr-sched=1cycle --num-cls-epochs=8 - -Max vocab: 30000 -Cache dir: /home/julian/ulmfit-multilingual/data/cls/fr-dvd/models/sp30k -Model dir: /home/julian/ulmfit-multilingual/data/cls/fr-dvd/models/sp30k/lstm_nl4.m -Validation set not found using 10% of trn -Running tokenization lm... -Data lm, trn: 12021, val: 1335 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 2000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/julian/data/wiki/fr-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/julian/data/wiki/fr-100/models/sp30k/lstm_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.825279 3.341784 0.382891 -Total time: 01:36 -epoch train_loss valid_loss accuracy -1 3.548075 3.280657 0.390619 -2 3.489078 3.206246 0.400057 -3 3.378426 3.099167 0.414316 -4 3.278241 2.985728 0.428153 -5 3.164112 2.868728 0.442793 -6 3.081279 2.740803 0.459863 -7 2.951635 2.615327 0.477424 -8 2.860259 2.511515 0.493157 -9 2.761055 2.386526 0.512397 -10 2.628587 2.277270 0.531014 -11 2.572315 2.181750 0.548689 -12 2.452535 2.083487 0.566041 -13 2.389231 1.998139 0.581409 -14 2.313358 1.927491 0.594620 -15 2.263673 1.873754 0.605384 -16 2.196958 1.827021 0.614506 -17 2.169217 1.797702 0.619863 -18 2.126882 1.777056 0.623906 -19 2.116131 1.767786 0.625270 -20 2.090418 1.765665 0.625703 -Total time: 37:20 -/home/julian/ulmfit-multilingual/data/cls/fr-dvd/models/sp30k -Saving info /home/julian/ulmfit-multilingual/data/cls/fr-dvd/models/sp30k/lstm_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.586308 0.504827 0.750000 -2 0.478227 0.411526 0.860000 -3 0.417158 0.314054 0.890000 -4 0.286224 0.263725 0.900000 -5 0.163930 0.387664 0.880000 -6 0.095715 0.282535 0.930000 -7 0.051098 0.294014 0.930000 -8 0.028741 0.301007 0.930000 -Total time: 02:57 -Saving models at /home/julian/ulmfit-multilingual/data/cls/fr-dvd/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.5228756, tensor(0.8920)] -0.5228756070137024 -0.8920000195503235 -``` - -## JA DVD LSTM - -```bash -(fastai) julian@dl-box-spot:~/ulmfit-multilingual$ python -m ulmfit cls --dataset-path data/cls/ja-dvd --base-lm-path ../data/wiki/ja-100/models/sp30k/lstm_nl4.m --tokenizer='sp' --lang=ja --name 'nl4' - train 20 --bs 40 --lr-sched=1cycle --num-cls-epochs=8 - -Max vocab: 30000 -Cache dir: /home/julian/ulmfit-multilingual/data/cls/ja-dvd/models/sp30k -Model dir: /home/julian/ulmfit-multilingual/data/cls/ja-dvd/models/sp30k/lstm_nl4.m -Validation set not found using 10% of trn -Running tokenization lm... -Data lm, trn: 30600, val: 3400 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 2000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/julian/data/wiki/ja-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/julian/data/wiki/ja-100/models/sp30k/lstm_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.904242 3.480109 0.375665 -Total time: 04:37 -epoch train_loss valid_loss accuracy -1 3.671750 3.403708 0.386209 -2 3.565445 3.310030 0.398628 -3 3.443218 3.204042 0.411568 -4 3.374648 3.098158 0.424107 -5 3.288731 3.005343 0.435621 -6 3.187409 2.913090 0.446806 -7 3.135114 2.829881 0.457120 -8 3.073141 2.753949 0.467695 -9 2.996589 2.682856 0.478041 -10 2.909743 2.613899 0.487629 -11 2.859827 2.550690 0.497565 -12 2.818285 2.492902 0.507112 -13 2.779268 2.435685 0.516448 -14 2.718145 2.387462 0.525241 -15 2.664007 2.346267 0.532140 -16 2.641343 2.312850 0.537994 -17 2.599257 2.288488 0.542193 -18 2.579481 2.274002 0.544809 -19 2.571687 2.267283 0.545827 -20 2.560343 2.265548 0.546052 -Total time: 1:47:55 -/home/julian/ulmfit-multilingual/data/cls/ja-dvd/models/sp30k -Saving info /home/julian/ulmfit-multilingual/data/cls/ja-dvd/models/sp30k/lstm_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.525819 0.402837 0.830000 -2 0.453459 0.425072 0.820000 -3 0.401383 0.482119 0.770000 -4 0.337860 0.502686 0.775000 -5 0.234284 0.805287 0.805000 -6 0.134409 0.729153 0.815000 -7 0.072370 0.895428 0.805000 -8 0.040945 0.832303 0.800000 -Total time: 03:09 -Saving models at /home/julian/ulmfit-multilingual/data/cls/ja-dvd/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.72511286, tensor(0.8395)] -0.7251128554344177 -0.8395000100135803 -``` - -## DE MUSIC LSTM - -```bash -(fastai) julian@dl-box-spot:~/ulmfit-multilingual$ python -m ulmfit cls --dataset-path data/cls/de-music --base-lm-path ../data/wiki/de-100/models/sp30k/lstm_nl4.m --tokenizer='sp' --lang=de --name 'nl4' - train 20 --bs 20 --lr-sched=1cycle --num-cls-epochs=8 - -Max vocab: 30000 -Cache dir: /home/julian/ulmfit-multilingual/data/cls/de-music/models/sp30k -Model dir: /home/julian/ulmfit-multilingual/data/cls/de-music/models/sp30k/lstm_nl4.m -Validation set not found using 10% of trn -Running tokenization lm... -Data lm, trn: 30600, val: 3400 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 2000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/julian/data/wiki/de-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/julian/data/wiki/de-100/models/sp30k/lstm_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 4.002521 3.526757 0.376006 -Total time: 06:32 -epoch train_loss valid_loss accuracy -1 3.738713 3.425875 0.389128 -2 3.607711 3.316313 0.403729 -3 3.441798 3.188569 0.418975 -4 3.347294 3.070090 0.432434 -5 3.261581 2.959181 0.447016 -6 3.161360 2.850135 0.460998 -7 3.092968 2.747133 0.475559 -8 2.999860 2.661642 0.488256 -9 2.946614 2.572933 0.502634 -10 2.834917 2.477213 0.517815 -11 2.768194 2.394673 0.532000 -12 2.743433 2.325108 0.545050 -13 2.596613 2.255300 0.557315 -14 2.347057 1.965440 0.616312 -15 2.289672 1.907880 0.626826 -16 2.238047 1.870341 0.634285 -17 2.169917 1.829818 0.641958 -18 2.145997 1.811395 0.645616 -19 2.111888 1.800622 0.647706 -20 2.067247 1.797927 0.648219 -Total time: 3:50:18 -/home/julian/ulmfit-multilingual/data/cls/de-music/models/sp30k -Saving info /home/julian/ulmfit-multilingual/data/cls/de-music/models/sp30k/lstm_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.392259 0.319764 0.875000 -2 0.373723 0.401961 0.850000 -3 0.315902 0.415566 0.850000 -4 0.185113 0.312382 0.890000 -5 0.122869 0.399712 0.865000 -6 0.084130 0.435429 0.910000 -7 0.057294 0.394715 0.890000 -8 0.028046 0.391238 0.900000 -Total time: 06:34 -Saving models at /home/julian/ulmfit-multilingual/data/cls/de-music/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.4154407, tensor(0.9210)] -0.41544070839881897 -0.9210000038146973 -``` - -## JA MUSIC LSTM - -```bash -(fastai) julian@dl-box-spot:~/ulmfit-multilingual$ python -m ulmfit cls --dataset-path data/cls/ja-music --base-lm-path ../data/wiki/ja-100/models/sp30k/lstm_nl4.m --tokenizer='sp' --lang=ja --name 'nl4' - train 20 --bs 20 --lr-sched=1cycle --num-cls-epochs=8 - -Data lm, trn: 30600, val: 3399 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 1999 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/julian/data/wiki/ja-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/julian/data/wiki/ja-100/models/sp30k/lstm_nl4.m/../itos')] - -epoch train_loss valid_loss accuracy -1 3.680415 3.162607 0.451358 -Total time: 09:43 -epoch train_loss valid_loss accuracy -1 3.258039 3.043514 0.467028 -2 3.082901 2.909904 0.482319 -3 3.009017 2.784519 0.496660 -4 2.900604 2.671213 0.510406 -5 2.776977 2.570581 0.522192 -6 2.789654 2.488552 0.532739 -7 2.710876 2.407913 0.543447 -8 2.655036 2.342028 0.553344 -9 2.571593 2.281001 0.562552 -10 2.539299 2.207963 0.574177 -11 2.466461 2.139726 0.585225 -12 2.441152 2.081656 0.595266 -13 2.434502 2.018719 0.606514 -14 2.576859 2.190329 0.569373 -15 2.543341 2.137856 0.579508 -16 2.467283 2.092796 0.587677 -17 2.417593 2.061508 0.593782 -18 2.375962 2.038786 0.598027 -19 2.391491 2.029075 0.599871 -20 2.352595 2.026604 0.600235 -Total time: 2:41:18 -/home/julian/ulmfit-multilingual/data/cls/ja-music/models/sp30k -Saving info /home/julian/ulmfit-multilingual/data/cls/ja-music/models/sp30k/lstm_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.458052 0.371875 0.860000 -2 0.473817 0.539201 0.730000 -3 0.423880 0.390433 0.845000 -4 0.310703 0.402607 0.855000 -5 0.211760 0.607136 0.865000 -6 0.108535 0.845904 0.860000 -7 0.053125 0.897018 0.860000 -8 0.024544 0.891663 0.855000 -Total time: 04:29 -Saving models at /home/julian/ulmfit-multilingual/data/cls/ja-music/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.777393, tensor(0.8644)] -0.7773929834365845 -0.8644322156906128 -``` - -## FR MUSIC LSTM - -```bash -(fastai) julian@dl-box-spot:~/ulmfit-multilingual$ python -m ulmfit cls --dataset-path data/cls/fr-music --base-lm-path ../data/wiki/fr-100/models/sp30k/lstm_nl4.m --tokenizer='sp' --lang=fr --name 'nl4' - train 20 --bs 40 --lr-sched=1cycle --num-cls-epochs=8 - -Running tokenization tst... -Data tst, trn: 200, val: 2000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/julian/data/wiki/fr-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/julian/data/wiki/fr-100/models/sp30k/lstm_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.686174 3.271906 0.394277 -Total time: 02:47 -epoch train_loss valid_loss accuracy -1 3.490278 3.197725 0.403639 -2 3.356067 3.100815 0.415938 -3 3.269037 2.970971 0.433533 -4 3.102959 2.819620 0.453009 -5 2.957009 2.647919 0.478246 -6 2.793691 2.481614 0.504928 -7 2.646251 2.316360 0.534853 -8 2.532166 2.140370 0.566817 -9 2.361445 1.982554 0.596333 -10 2.258446 1.855159 0.621765 -11 2.155252 1.772740 0.640348 -12 2.071291 1.668775 0.660405 -13 1.887608 1.579711 0.677771 -14 1.873631 1.493046 0.694815 -15 1.824689 1.438728 0.705296 -16 1.766544 1.398732 0.714093 -17 1.646138 1.372478 0.719408 -18 1.684073 1.350950 0.723633 -19 1.650602 1.344994 0.724889 -20 1.602114 1.341957 0.725314 -Total time: 1:04:53 -/home/julian/ulmfit-multilingual/data/cls/fr-music/models/sp30k -Saving info /home/julian/ulmfit-multilingual/data/cls/fr-music/models/sp30k/lstm_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.415938 0.214479 0.930000 -2 0.373839 0.334263 0.880000 -3 0.317772 0.660272 0.795000 -4 0.207807 0.440546 0.880000 -5 0.146999 0.377026 0.890000 -6 0.095834 0.288273 0.925000 -7 0.048218 0.350355 0.895000 -8 0.023682 0.325598 0.915000 -Total time: 03:18 -Saving models at /home/julian/ulmfit-multilingual/data/cls/fr-music/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.32345334, tensor(0.9295)] -0.3234533369541168 -0.9294999837875366 -``` \ No newline at end of file diff --git a/results/logs/cls/ulmfit on cls.md b/results/logs/cls/ulmfit on cls.md deleted file mode 100644 index 2195f13..0000000 --- a/results/logs/cls/ulmfit on cls.md +++ /dev/null @@ -1,1092 +0,0 @@ -# CLS execution logs - -| model name | Accuracy | -|--------------------------------------------|----------| -| data/cls/de-books/models/sp15k/qrnn_nl4.m: | 93.19 | -| data/cls/de-dvd/models/sp15k/qrnn_nl4.m: | 90.54 | -| data/cls/de-music/models/sp15k/qrnn_nl4.m: | 93.00 | -| data/cls/en-books/models/sp15k/qrnn_nl4.m: | 90.75 | -| data/cls/en-dvd/models/sp15k/qrnn_nl4.m: | 89.30 | -| data/cls/en-music/models/sp15k/qrnn_nl4.m: | 89.45 | -| data/cls/fr-books/models/sp15k/qrnn_nl4.m: | 91.25 | -| data/cls/fr-dvd/models/sp15k/qrnn_nl4.m: | 89.55 | -| data/cls/fr-music/models/sp15k/qrnn_nl4.m: | 93.40 | -| data/cls/ja-books/models/sp15k/qrnn_nl4.m: | 86.29 | -| data/cls/ja-dvd/models/sp15k/qrnn_nl4.m: | 85.75 | -| data/cls/ja-music/models/sp15k/qrnn_nl4.m: | 86.59 | - -### All langs books - -``` -python -m ulmfit eval --glob="wiki/*-100/models/sp15k/qrnn_nl4.m" --name nl4 --dataset-template='../cls/${lang}-books' --num-lm-epochs=20 --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.1 -Processing data/wiki/de-100/models/sp15k/qrnn_nl4.m -../cls/de-books -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/de-books/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/de-books/models/sp15k/qrnn_nl4.m -Training -Validation set not found using 10% of trn -Error You have NaN values in column(s) of your dataframe, please fix it. -Processing data/wiki/en-100/models/sp15k/qrnn_nl4.m -../cls/en-books -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/en-books/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/en-books/models/sp15k/qrnn_nl4.m -Training -Validation set not found using 10% of trn -Running tokenization lm... -Data lm, trn: 48600, val: 5400 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 2000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁the', '▁,', 's', '▁.', '▁of', '▁and', '▁in', '▁to', '▁a', 'ed'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: - - !! WARNING !! - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Your compiler (c++) may be ABI-incompatible with PyTorch! -Please use a compiler that is ABI-compatible with GCC 4.9 and above. -See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. - -See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 -for instructions on how to install GCC 4.9 or higher. -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - !! WARNING !! - - warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/en-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/en-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 5.411075 4.929663 0.299280 -Total time: 05:18 -epoch train_loss valid_loss accuracy -1 4.911766 4.729969 0.328842 -2 4.749516 4.571162 0.352025 -3 4.574124 4.432505 0.369642 -4 4.496035 4.315759 0.384135 -5 4.402706 4.238598 0.393190 -6 4.336373 4.176495 0.401667 -7 4.305870 4.111050 0.410795 -8 4.251368 4.055121 0.419378 -9 4.215136 4.005383 0.426659 -10 4.242251 3.959000 0.434112 -11 4.128317 3.919106 0.440762 -12 4.102277 3.881066 0.447852 -13 4.109466 3.840265 0.455235 -14 4.074686 3.798793 0.463265 -15 4.034000 3.765705 0.469861 -16 3.984559 3.736501 0.475469 -17 3.961936 3.718403 0.479141 -18 3.986044 3.708190 0.481330 -19 3.934352 3.700921 0.482685 -20 3.977421 3.698027 0.483279 -Total time: 2:33:09 -/home/pczapla/workspace/ulmfit-multilingual/data/cls/en-books/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/cls/en-books/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.511342 0.485253 0.890000 -2 0.476206 0.497634 0.875000 -3 0.423625 0.495749 0.880000 -4 0.383525 0.472684 0.890000 -5 0.371009 0.455094 0.895000 -6 0.359045 0.453902 0.905000 -7 0.354646 0.453879 0.910000 -8 0.348541 0.462702 0.885000 -Total time: 02:30 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/cls/en-books/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.27204505, tensor(0.9060)] -Processing data/wiki/es-100/models/sp15k/qrnn_nl4.m -../cls/es-books -Processing data/wiki/fr-100/models/sp15k/qrnn_nl4.m -../cls/fr-books -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp15k/qrnn_nl4.m -Training -Validation set not found using 10% of trn -Running tokenization lm... -Data lm, trn: 33183, val: 3687 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 2000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', 's', '▁.', "'", '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 4.844616 4.303658 0.358888 -Total time: 02:32 -epoch train_loss valid_loss accuracy -1 4.558980 4.236440 0.368490 -2 4.336353 4.111036 0.389355 -3 4.215691 3.997827 0.406318 -4 4.124804 3.892600 0.422174 -5 4.052222 3.802997 0.435620 -6 3.958529 3.729386 0.446996 -7 3.889615 3.646023 0.460472 -8 3.860830 3.590203 0.470407 -9 3.763688 3.521166 0.482480 -10 3.735285 3.465606 0.493439 -11 3.699683 3.411256 0.504397 -12 3.666552 3.347546 0.517700 -13 3.580596 3.308754 0.526673 -14 3.551433 3.255782 0.537871 -15 3.517213 3.215991 0.546388 -16 3.480192 3.188064 0.553690 -17 3.416178 3.162537 0.559454 -18 3.391725 3.151500 0.561885 -19 3.400107 3.144848 0.563330 -20 3.400756 3.143354 0.563702 -Total time: 1:12:34 -/home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.532118 0.548179 0.820000 -2 0.495962 0.509888 0.860000 -3 0.427645 0.489982 0.900000 -4 0.385989 0.513428 0.850000 -5 0.372433 0.446763 0.905000 -6 0.364960 0.467484 0.885000 -7 0.350225 0.455279 0.905000 -8 0.347019 0.466804 0.885000 -Total time: 01:48 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.27588147, tensor(0.9130)] -Processing data/wiki/it-100/models/sp15k/qrnn_nl4.m -../cls/it-books -Processing data/wiki/ja-100/models/sp15k/qrnn_nl4.m -../cls/ja-books -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/ja-books/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/ja-books/models/sp15k/qrnn_nl4.m -Training -Validation set not found using 10% of trn -Error You have NaN values in column(s) of your dataframe, please fix it. -Processing data/wiki/ru-100/models/sp15k/qrnn_nl4.m -../cls/ru-books -Processing data/wiki/zh-100/models/sp15k/qrnn_nl4.m -../cls/zh-books -OrderedDict([('data/cls/en-books/models/sp15k/qrnn_nl4.m', 0.906000018119812), - ('data/cls/fr-books/models/sp15k/qrnn_nl4.m', 0.9129999876022339)]) -data/cls/en-books/models/sp15k/qrnn_nl4.m: 0.906000018119812 -``` -### All datasets -```bash -python -m ulmfit eval --glob="wiki/*-100/models/sp15k/qrnn_nl4.m" --name nl4 --dataset-template='../cls/${lang}-*' --num-lm-epochs=20 --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.1 -Processing data/wiki/de-100/models/sp15k/qrnn_nl4.m -../cls/de-* -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/de-books/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/de-books/models/sp15k/qrnn_nl4.m -Evaluating previously trained model -Validation set not found using 10% of trn -Data lm, trn: 152523, val: 16947 -Data cls, trn: 1800, val: 200 -Data tst, trn: 200, val: 2000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: - - !! WARNING !! - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Your compiler (c++) may be ABI-incompatible with PyTorch! -Please use a compiler that is ABI-compatible with GCC 4.9 and above. -See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. - -See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 -for instructions on how to install GCC 4.9 or higher. -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - !! WARNING !! - - warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) -Loss and accuracy using (cls_best): [0.23488419, tensor(0.9320)] -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/de-dvd/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/de-dvd/models/sp15k/qrnn_nl4.m -Training -Validation set not found using 10% of trn -Running tokenization lm... -Data lm, trn: 85965, val: 9551 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 2000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/de-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/de-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 4.596065 4.161942 0.436606 -Total time: 11:33 -epoch train_loss valid_loss accuracy -1 4.127364 3.947751 0.466350 -2 3.918618 3.776825 0.490808 -3 3.800280 3.655171 0.506950 -4 3.690864 3.582598 0.516176 -5 3.698156 3.535314 0.522662 -6 3.610991 3.498398 0.527869 -7 3.592985 3.466045 0.532311 -8 3.566916 3.444989 0.535283 -9 3.563903 3.422111 0.539163 -10 3.557671 3.400061 0.542508 -11 3.486254 3.377414 0.546246 -12 3.489782 3.356982 0.549719 -13 3.451218 3.333650 0.553665 -14 3.464072 3.312912 0.557414 -15 3.431759 3.292514 0.560929 -16 3.387984 3.274624 0.564236 -17 3.405241 3.262639 0.566367 -18 3.374643 3.253385 0.568177 -19 3.374621 3.248559 0.569108 -20 3.377307 3.247425 0.569270 -Total time: 5:31:07 -/home/pczapla/workspace/ulmfit-multilingual/data/cls/de-dvd/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/cls/de-dvd/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.519669 0.485357 0.865000 -2 0.479460 0.426823 0.920000 -3 0.431123 0.432494 0.885000 -4 0.386461 0.431496 0.915000 -5 0.365019 0.427543 0.920000 -6 0.355551 0.415073 0.930000 -7 0.346821 0.412974 0.930000 -8 0.347380 0.413793 0.935000 -Total time: 02:57 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/cls/de-dvd/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.2746336, tensor(0.9055)] -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/de-music/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/de-music/models/sp15k/qrnn_nl4.m -Training -Validation set not found using 10% of trn -Running tokenization lm... -Data lm, trn: 57953, val: 6439 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 2000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/de-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/de-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 4.650980 4.152651 0.433679 -Total time: 07:28 -epoch train_loss valid_loss accuracy -1 4.209763 3.965040 0.459827 -2 3.935442 3.798236 0.482987 -3 3.841647 3.667273 0.500565 -4 3.699245 3.578814 0.512209 -5 3.652425 3.512940 0.521151 -6 3.658080 3.467384 0.527574 -7 3.583572 3.427991 0.533427 -8 3.558736 3.393701 0.538799 -9 3.497921 3.360512 0.544095 -10 3.513804 3.333250 0.548685 -11 3.495725 3.303407 0.553937 -12 3.442093 3.278184 0.558270 -13 3.431648 3.249587 0.563309 -14 3.394415 3.224657 0.567926 -15 3.418715 3.201570 0.572469 -16 3.334841 3.180893 0.576305 -17 3.356635 3.166274 0.579287 -18 3.339209 3.156838 0.581144 -19 3.307631 3.151842 0.582141 -20 3.313863 3.150544 0.582361 -Total time: 3:28:27 -/home/pczapla/workspace/ulmfit-multilingual/data/cls/de-music/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/cls/de-music/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.518283 0.456440 0.880000 -2 0.474427 0.466113 0.865000 -3 0.420782 0.572000 0.850000 -4 0.377621 0.467714 0.865000 -5 0.361156 0.430405 0.915000 -6 0.345733 0.429622 0.890000 -7 0.347064 0.419322 0.910000 -8 0.342849 0.419672 0.900000 -Total time: 02:41 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/cls/de-music/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.24861218, tensor(0.9300)] -Processing data/wiki/en-100/models/sp15k/qrnn_nl4.m -../cls/en-* -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/en-books/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/en-books/models/sp15k/qrnn_nl4.m -Evaluating previously trained model -Validation set not found using 10% of trn -Data lm, trn: 48600, val: 5400 -Data cls, trn: 1800, val: 200 -Data tst, trn: 200, val: 2000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁the', '▁,', 's', '▁.', '▁of', '▁and', '▁in', '▁to', '▁a', 'ed'] -Loss and accuracy using (cls_best): [0.2731999, tensor(0.9075)] -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/en-dvd/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/en-dvd/models/sp15k/qrnn_nl4.m -Training -Validation set not found using 10% of trn -Running tokenization lm... -Data lm, trn: 30600, val: 3400 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 2000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁the', '▁,', 's', '▁.', '▁of', '▁and', '▁in', '▁to', '▁a', 'ed'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/en-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/en-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 5.459846 4.880299 0.304455 -Total time: 03:40 -epoch train_loss valid_loss accuracy -1 4.914763 4.696301 0.329923 -2 4.679338 4.542424 0.351552 -3 4.545784 4.381050 0.373713 -4 4.435306 4.261026 0.388227 -5 4.381686 4.146891 0.402335 -6 4.339701 4.057630 0.414286 -7 4.193101 3.985903 0.425350 -8 4.188499 3.904590 0.437700 -9 4.141142 3.831017 0.449915 -10 4.073668 3.769752 0.460387 -11 4.034776 3.704058 0.472169 -12 4.017927 3.645482 0.483226 -13 3.939799 3.571460 0.498373 -14 3.954270 3.544665 0.504316 -15 3.891879 3.494558 0.514514 -16 3.816200 3.460811 0.522036 -17 3.817204 3.436978 0.527317 -18 3.837135 3.423382 0.530152 -19 3.786701 3.413906 0.532207 -20 3.794206 3.412058 0.532526 -Total time: 1:43:34 -/home/pczapla/workspace/ulmfit-multilingual/data/cls/en-dvd/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/cls/en-dvd/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.546413 0.469370 0.895000 -2 0.487449 0.455633 0.880000 -3 0.434380 0.471036 0.890000 -4 0.402179 0.457251 0.885000 -5 0.375344 0.444693 0.910000 -6 0.356960 0.440818 0.900000 -7 0.352019 0.427976 0.900000 -8 0.347301 0.427571 0.905000 -Total time: 02:39 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/cls/en-dvd/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.28846368, tensor(0.8930)] -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/en-music/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/en-music/models/sp15k/qrnn_nl4.m -Training -Validation set not found using 10% of trn -Running tokenization lm... -Data lm, trn: 26298, val: 2922 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 2000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁the', '▁,', 's', '▁.', '▁of', '▁and', '▁in', '▁to', '▁a', 'ed'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/en-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/en-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 5.360476 4.782286 0.319278 -Total time: 02:41 -epoch train_loss valid_loss accuracy -1 4.751396 4.553875 0.351461 -2 4.583414 4.389054 0.374384 -3 4.383287 4.250410 0.394109 -4 4.299458 4.122505 0.409391 -5 4.248813 4.014976 0.423715 -6 4.149672 3.927886 0.434895 -7 4.099454 3.838499 0.448373 -8 4.001449 3.762594 0.459935 -9 3.931528 3.687276 0.472542 -10 3.904028 3.625594 0.483216 -11 3.892965 3.560731 0.495610 -12 3.851050 3.500596 0.506427 -13 3.748619 3.447429 0.517146 -14 3.783010 3.407018 0.526054 -15 3.737605 3.356801 0.536732 -16 3.642893 3.315823 0.544464 -17 3.664337 3.291245 0.550013 -18 3.636087 3.277205 0.553098 -19 3.614743 3.269276 0.554796 -20 3.615276 3.267146 0.555189 -Total time: 1:16:19 -/home/pczapla/workspace/ulmfit-multilingual/data/cls/en-music/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/cls/en-music/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.558971 0.567553 0.840000 -2 0.512537 0.503813 0.855000 -3 0.456603 0.525335 0.830000 -4 0.404400 0.528679 0.870000 -5 0.380277 0.508553 0.850000 -6 0.370005 0.473182 0.875000 -7 0.353120 0.475434 0.875000 -8 0.349033 0.476645 0.875000 -Total time: 02:17 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/cls/en-music/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.30121252, tensor(0.8945)] -Processing data/wiki/es-100/models/sp15k/qrnn_nl4.m -../cls/es-* -Processing data/wiki/fr-100/models/sp15k/qrnn_nl4.m -../cls/fr-* -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp15k/qrnn_nl4.m -Evaluating previously trained model -Validation set not found using 10% of trn -Data lm, trn: 33183, val: 3687 -Data cls, trn: 1800, val: 200 -Data tst, trn: 200, val: 2000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', 's', '▁.', "'", '▁la', '▁le', '▁et', '▁l', '▁à'] -Loss and accuracy using (cls_best): [0.27542278, tensor(0.9125)] -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-dvd/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-dvd/models/sp15k/qrnn_nl4.m -Training -Validation set not found using 10% of trn -Running tokenization lm... -Data lm, trn: 12023, val: 1335 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 2000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', 's', '▁.', "'", '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 5.066386 4.331831 0.359093 -Total time: 00:59 -epoch train_loss valid_loss accuracy -1 4.724524 4.284094 0.365222 -2 4.508605 4.191759 0.380180 -3 4.362468 4.066796 0.399366 -4 4.244251 3.960160 0.414862 -5 4.175961 3.862056 0.428568 -6 4.089641 3.769989 0.443156 -7 3.996232 3.692802 0.455285 -8 3.911177 3.609659 0.469151 -9 3.834589 3.517031 0.485899 -10 3.753677 3.444421 0.499121 -11 3.693849 3.378458 0.512520 -12 3.656308 3.304272 0.527849 -13 3.619973 3.243898 0.539743 -14 3.538301 3.187478 0.551900 -15 3.542358 3.146652 0.561410 -16 3.433141 3.108872 0.569471 -17 3.397583 3.086333 0.574742 -18 3.420117 3.073466 0.577351 -19 3.364507 3.063923 0.579112 -20 3.371653 3.063201 0.579386 -Total time: 28:17 -/home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-dvd/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-dvd/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.566780 0.456278 0.895000 -2 0.502839 0.438660 0.910000 -3 0.451509 0.439451 0.915000 -4 0.409567 0.458298 0.905000 -5 0.375980 0.451819 0.875000 -6 0.365615 0.429869 0.895000 -7 0.354849 0.430782 0.900000 -8 0.351484 0.429099 0.900000 -Total time: 01:58 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-dvd/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.29015526, tensor(0.8955)] -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-music/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-music/models/sp15k/qrnn_nl4.m -Training -Validation set not found using 10% of trn -Running tokenization lm... -Data lm, trn: 17946, val: 1994 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 2000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', 's', '▁.', "'", '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 4.902670 4.297194 0.366058 -Total time: 01:43 -epoch train_loss valid_loss accuracy -1 4.611322 4.230604 0.375839 -2 4.375579 4.099567 0.396295 -3 4.221557 3.955517 0.419109 -4 4.053433 3.821369 0.439689 -5 3.936422 3.676852 0.463543 -6 3.822320 3.571660 0.481323 -7 3.738963 3.448354 0.505107 -8 3.660149 3.338475 0.528736 -9 3.517678 3.231433 0.552705 -10 3.506994 3.129620 0.575138 -11 3.405409 3.064906 0.591049 -12 3.381207 2.970587 0.613629 -13 3.252796 2.915546 0.627047 -14 3.214425 2.860644 0.640598 -15 3.154248 2.808179 0.653110 -16 3.101139 2.774935 0.661620 -17 3.052113 2.748979 0.667787 -18 2.992890 2.738183 0.670336 -19 3.059078 2.730828 0.671936 -20 2.990193 2.729673 0.672222 -Total time: 48:59 -/home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-music/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-music/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.506996 0.478671 0.880000 -2 0.468533 0.474114 0.900000 -3 0.426250 0.440435 0.910000 -4 0.382522 0.441184 0.920000 -5 0.368826 0.428872 0.925000 -6 0.359141 0.421278 0.925000 -7 0.345824 0.418836 0.925000 -8 0.348028 0.422893 0.920000 -Total time: 02:09 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-music/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.23497759, tensor(0.9340)] -Processing data/wiki/it-100/models/sp15k/qrnn_nl4.m -../cls/it-* -Processing data/wiki/ja-100/models/sp15k/qrnn_nl4.m -../cls/ja-* -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/ja-books/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/ja-books/models/sp15k/qrnn_nl4.m -Evaluating previously trained model -Validation set not found using 10% of trn -Data lm, trn: 156402, val: 17378 -Data cls, trn: 1800, val: 200 -Data tst, trn: 200, val: 2000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)'] -Loss and accuracy using (cls_best): [0.34613457, tensor(0.8630)] -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/ja-dvd/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/ja-dvd/models/sp15k/qrnn_nl4.m -Training -Validation set not found using 10% of trn -Running tokenization lm... -Data lm, trn: 65094, val: 7232 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 2000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 5.160996 4.517923 0.367362 -Total time: 06:17 -epoch train_loss valid_loss accuracy -1 4.543629 4.324594 0.393028 -2 4.285625 4.133171 0.418336 -3 4.129646 3.980379 0.436407 -4 3.999502 3.868094 0.450510 -5 3.955512 3.801580 0.458919 -6 3.918293 3.744513 0.466177 -7 3.851580 3.696822 0.472531 -8 3.797014 3.660165 0.478426 -9 3.784533 3.629932 0.482919 -10 3.750223 3.600744 0.487628 -11 3.746155 3.572618 0.492204 -12 3.731790 3.549776 0.495799 -13 3.721592 3.523684 0.500092 -14 3.664623 3.501803 0.504120 -15 3.653851 3.482831 0.507431 -16 3.655076 3.466049 0.510687 -17 3.621635 3.454597 0.512781 -18 3.619130 3.447068 0.514104 -19 3.598743 3.443191 0.514797 -20 3.585334 3.441960 0.515080 -Total time: 2:55:48 -/home/pczapla/workspace/ulmfit-multilingual/data/cls/ja-dvd/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/cls/ja-dvd/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.572977 0.500271 0.845000 -2 0.530233 0.530061 0.815000 -3 0.478677 0.534202 0.845000 -4 0.422542 0.536495 0.855000 -5 0.382862 0.508948 0.820000 -6 0.369442 0.500411 0.860000 -7 0.364325 0.500543 0.840000 -8 0.353970 0.498926 0.840000 -Total time: 02:12 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/cls/ja-dvd/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.351684, tensor(0.8575)] -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/ja-music/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/ja-music/models/sp15k/qrnn_nl4.m -Training -Validation set not found using 10% of trn -Running tokenization lm... -Data lm, trn: 53903, val: 5989 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 2000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 5.209339 4.578931 0.361889 -Total time: 04:32 -epoch train_loss valid_loss accuracy -1 4.667555 4.387390 0.386283 -2 4.378979 4.193202 0.412063 -3 4.182499 4.027168 0.434250 -4 4.075960 3.904261 0.449052 -5 3.978663 3.817029 0.459996 -6 3.933847 3.751421 0.468568 -7 3.878065 3.694970 0.476144 -8 3.796052 3.649740 0.483267 -9 3.771566 3.604177 0.490818 -10 3.737682 3.573293 0.495396 -11 3.690553 3.530821 0.503269 -12 3.654486 3.498471 0.508781 -13 3.628883 3.467230 0.514346 -14 3.606242 3.443541 0.519053 -15 3.586881 3.420868 0.523608 -16 3.581954 3.402043 0.527174 -17 3.492296 3.383712 0.530720 -18 3.522726 3.376451 0.532236 -19 3.512796 3.371758 0.533116 -20 3.461179 3.370992 0.533302 -Total time: 2:06:47 -/home/pczapla/workspace/ulmfit-multilingual/data/cls/ja-music/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/cls/ja-music/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.572634 0.558102 0.795000 -2 0.538641 0.521115 0.830000 -3 0.480845 0.544550 0.830000 -4 0.420194 0.540962 0.830000 -5 0.394283 0.502248 0.860000 -6 0.371833 0.520424 0.840000 -7 0.360100 0.511556 0.855000 -8 0.353953 0.511548 0.850000 -Total time: 01:51 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/cls/ja-music/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.34172097, tensor(0.8660)] -Processing data/wiki/ru-100/models/sp15k/qrnn_nl4.m -../cls/ru-* -Processing data/wiki/zh-100/models/sp15k/qrnn_nl4.m -../cls/zh-* -OrderedDict([('data/cls/de-books/models/sp15k/qrnn_nl4.m', 0.9319999814033508), - ('data/cls/de-dvd/models/sp15k/qrnn_nl4.m', 0.9054999947547913), - ('data/cls/de-music/models/sp15k/qrnn_nl4.m', 0.9300000071525574), - ('data/cls/en-books/models/sp15k/qrnn_nl4.m', 0.9075000286102295), - ('data/cls/en-dvd/models/sp15k/qrnn_nl4.m', 0.8930000066757202), - ('data/cls/en-music/models/sp15k/qrnn_nl4.m', 0.8945000171661377), - ('data/cls/fr-books/models/sp15k/qrnn_nl4.m', 0.9125000238418579), - ('data/cls/fr-dvd/models/sp15k/qrnn_nl4.m', 0.8955000042915344), - ('data/cls/fr-music/models/sp15k/qrnn_nl4.m', 0.9340000152587891), - ('data/cls/ja-books/models/sp15k/qrnn_nl4.m', 0.8629999756813049), - ('data/cls/ja-dvd/models/sp15k/qrnn_nl4.m', 0.8575000166893005), - ('data/cls/ja-music/models/sp15k/qrnn_nl4.m', 0.8659999966621399)]) -data/cls/de-books/models/sp15k/qrnn_nl4.m: 0.9319999814033508 -data/cls/de-dvd/models/sp15k/qrnn_nl4.m: 0.9054999947547913 -data/cls/de-music/models/sp15k/qrnn_nl4.m: 0.9300000071525574 -data/cls/en-books/models/sp15k/qrnn_nl4.m: 0.9075000286102295 -data/cls/en-dvd/models/sp15k/qrnn_nl4.m: 0.8930000066757202 -data/cls/en-music/models/sp15k/qrnn_nl4.m: 0.8945000171661377 -data/cls/fr-books/models/sp15k/qrnn_nl4.m: 0.9125000238418579 -data/cls/fr-dvd/models/sp15k/qrnn_nl4.m: 0.8955000042915344 -data/cls/fr-music/models/sp15k/qrnn_nl4.m: 0.9340000152587891 -data/cls/ja-books/models/sp15k/qrnn_nl4.m: 0.8629999756813049 -data/cls/ja-dvd/models/sp15k/qrnn_nl4.m: 0.8575000166893005 -data/cls/ja-music/models/sp15k/qrnn_nl4.m: 0.8659999966621399 -``` -# Debugging issues -## Without col merge -```` -python -m ulmfit cls --dataset-path data/cls/${LANG}-books --base-lm-path data/wiki-m/${LANG}-100/models/sp30k/lstm_nl4.m --lang=${LANG} --name 'nl4' - train 20 --bs 20 --num-cls-epochs=8 --lr-sched=single -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k/lstm_nl4.m -Validation set not found using 10% of trn -Data lm, trn: 33183, val: 3687 -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 2000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki-m/fr-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki-m/fr-100/models/sp30k/lstm_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.356221 2.821012 0.518492 -Total time: 00:21 -epoch train_loss valid_loss accuracy -1 3.041214 2.734799 0.524577 -2 2.919576 2.648412 0.535661 -3 2.822292 2.542236 0.549206 -4 2.721790 2.414110 0.561852 -5 2.596515 2.276732 0.579841 -6 2.453715 2.140479 0.600370 -7 2.333764 2.000186 0.621349 -8 2.231092 1.873927 0.644259 -9 2.101130 1.765473 0.660529 -10 2.006949 1.666797 0.682196 -11 1.905025 1.584023 0.696058 -12 1.820798 1.513958 0.709841 -13 1.751217 1.456632 0.720846 -14 1.689076 1.410359 0.729947 -15 1.646113 1.371438 0.739868 -16 1.594153 1.346142 0.744577 -17 1.564375 1.332298 0.746693 -18 1.536557 1.322925 0.748995 -19 1.532926 1.319159 0.749444 -20 1.525449 1.318028 0.749815 -Total time: 08:51 -/home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k/lstm_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.588118 0.581087 0.700000 -2 0.504373 0.583527 0.720000 -3 0.412651 0.538866 0.750000 -4 0.295401 0.658459 0.750000 -5 0.212442 1.054068 0.720000 -6 0.126090 1.302099 0.745000 -7 0.078312 1.307932 0.760000 -8 0.050346 1.339740 0.745000 -Total time: 00:35 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [1.3513571, tensor(0.7700)] -1.351357102394104 -0.7699999809265137 -```` - -### FR books -#### CLS second run 91.00% -````bash -python -m ulmfit cls --dataset-path data/cls/${LANG}-books --base-lm-path data/wiki-m/${LANG}-100/models/sp30k/lstm_nl4.m --lang=${LANG} --name 'nl4' - train 20 --bs 20 --num-cls-epochs=8 --lr-sched=single -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k/lstm_nl4.m -Validation set not found using 10% of trn -Running tokenization lm... -Data lm, trn: 33183, val: 3687 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 2000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki-m/fr-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki-m/fr-100/models/sp30k/lstm_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.790325 3.409294 0.367234 -Total time: 06:02 -epoch train_loss valid_loss accuracy -1 3.526303 3.326439 0.378936 -2 3.466923 3.226977 0.392378 -3 3.342312 3.111874 0.406997 -4 3.244619 2.992510 0.422330 -5 3.156150 2.877498 0.437467 -6 3.070326 2.762509 0.453874 -7 2.956969 2.651613 0.471552 -8 2.878008 2.535935 0.491058 -9 2.790110 2.438724 0.508560 -10 2.684145 2.323467 0.528415 -11 2.633781 2.231418 0.547093 -12 2.535126 2.143523 0.564889 -13 2.464436 2.055402 0.582077 -14 2.330094 1.989257 0.596582 -15 2.372371 1.924338 0.610048 -16 2.190224 1.866912 0.621738 -17 2.176868 1.834098 0.629221 -18 2.168293 1.809196 0.633879 -19 2.151132 1.797144 0.636382 -20 2.130476 1.793351 0.637044 -Total time: 2:30:05 -/home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k/lstm_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.314315 0.530879 0.865000 -2 0.336746 0.468635 0.865000 -3 0.255810 0.324242 0.870000 -4 0.149121 0.480570 0.885000 -5 0.093909 0.613743 0.890000 -6 0.091678 0.660452 0.885000 -7 0.049993 0.649642 0.910000 -8 0.034218 0.640008 0.910000 -Total time: 04:19 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.5418505, tensor(0.9100)] -0.5418505072593689 -0.9100000262260437 -```` -#### CLS second run 89.70% -``` -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 4.037580 3.312217 0.364410 -Total time: 01:44 -epoch train_loss valid_loss accuracy -1 3.709982 3.256320 0.371825 -2 3.459413 3.150574 0.386972 -3 3.296628 3.037327 0.402039 -4 3.186458 2.914899 0.418413 -5 3.092632 2.817097 0.431216 -6 2.966957 2.726081 0.442906 -7 2.924824 2.647339 0.453871 -8 2.818279 2.561596 0.466795 -9 2.773893 2.501994 0.475877 -10 2.736084 2.438490 0.485978 -11 2.688937 2.370927 0.496899 -12 2.615245 2.314875 0.506508 -13 2.583292 2.260717 0.515725 -14 2.535631 2.220295 0.522666 -15 2.466035 2.179093 0.530148 -16 2.461427 2.151952 0.535315 -17 2.390641 2.131065 0.538749 -18 2.376235 2.116927 0.541430 -19 2.407630 2.115370 0.542039 -20 2.391378 2.112687 0.542522 -Total time: 46:33 -/home/n-waves/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp15k -Saving info /home/n-waves/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.466671 0.534682 0.745000 -2 0.358965 0.372612 0.875000 -3 0.251557 0.311034 0.900000 -4 0.166484 0.585425 0.865000 -5 0.101803 0.726341 0.900000 -6 0.072025 0.587875 0.885000 -7 0.045328 0.760989 0.890000 -8 0.027765 0.727203 0.890000 -Total time: 01:17 -Saving models at /home/n-waves/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.55982095, tensor(0.8970)] -0.5598209500312805 -0.8970000147819519 -``` - -````bash - python -m ulmfit eval --glob="wiki/*-100/models/sp15k/qrnn_nl4.m" --name nl4 --dataset-template='../cls/${lang}-books' --num-lm-epochs=20 --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.1 -Processing data/wiki/de-100/models/sp15k/qrnn_nl4.m -../cls/de-books -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/de-books/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/de-books/models/sp15k/qrnn_nl4.m -Training -Validation set not found using 10% of trn -Running tokenization lm... -Data lm, trn: 152523, val: 16947 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 2000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: - - !! WARNING !! - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Your compiler (c++) may be ABI-incompatible with PyTorch! -Please use a compiler that is ABI-compatible with GCC 4.9 and above. -See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. - -See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 -for instructions on how to install GCC 4.9 or higher. -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - !! WARNING !! - - warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/de-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/de-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 4.659840 4.155068 0.438915 -Total time: 19:49 -epoch train_loss valid_loss accuracy -1 4.047657 3.906174 0.473263 -2 3.838604 3.750033 0.495044 -3 3.759745 3.645392 0.508066 -4 3.687292 3.590345 0.515256 -5 3.627978 3.558625 0.519270 -6 3.645294 3.535127 0.522458 -7 3.598592 3.514614 0.525545 -8 3.624991 3.498831 0.527508 -9 3.579310 3.484266 0.529842 -10 3.583765 3.466230 0.532676 -11 3.582043 3.449772 0.535180 -12 3.555684 3.431577 0.537932 -13 3.547026 3.412131 0.541149 -14 3.494887 3.394598 0.543857 -15 3.508622 3.377836 0.546771 -16 3.509469 3.362931 0.549247 -17 3.491808 3.351186 0.551273 -18 3.493448 3.342843 0.552928 -19 3.430579 3.338350 0.553565 -20 3.464007 3.337320 0.553756 -Total time: 9:30:17 -/home/pczapla/workspace/ulmfit-multilingual/data/cls/de-books/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/cls/de-books/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.483316 0.432666 0.930000 -2 0.439075 0.419435 0.900000 -3 0.413648 0.415576 0.930000 -4 0.378170 0.421523 0.925000 -5 0.362544 0.410525 0.935000 -6 0.348756 0.407976 0.935000 -7 0.345272 0.403396 0.935000 -8 0.347165 0.408822 0.935000 -Total time: 02:45 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/cls/de-books/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.23490253, tensor(0.9315)] -Processing data/wiki/en-100/models/sp15k/qrnn_nl4.m -../cls/en-books -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/en-books/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/en-books/models/sp15k/qrnn_nl4.m -Evaluating previously trained model -Validation set not found using 10% of trn -Data lm, trn: 48600, val: 5400 -Data cls, trn: 1800, val: 200 -Data tst, trn: 200, val: 2000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁the', '▁,', 's', '▁.', '▁of', '▁and', '▁in', '▁to', '▁a', 'ed'] -Loss and accuracy using (cls_best): [0.2731999, tensor(0.9075)] -Processing data/wiki/es-100/models/sp15k/qrnn_nl4.m -../cls/es-books -Processing data/wiki/fr-100/models/sp15k/qrnn_nl4.m -../cls/fr-books -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp15k/qrnn_nl4.m -Evaluating previously trained model -Validation set not found using 10% of trn -Data lm, trn: 33183, val: 3687 -Data cls, trn: 1800, val: 200 -Data tst, trn: 200, val: 2000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', 's', '▁.', "'", '▁la', '▁le', '▁et', '▁l', '▁à'] -Loss and accuracy using (cls_best): [0.27542278, tensor(0.9125)] -Processing data/wiki/it-100/models/sp15k/qrnn_nl4.m -../cls/it-books -Processing data/wiki/ja-100/models/sp15k/qrnn_nl4.m -../cls/ja-books -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/ja-books/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/ja-books/models/sp15k/qrnn_nl4.m -Training -Validation set not found using 10% of trn -Running tokenization lm... -Data lm, trn: 156402, val: 17378 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 2000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 4.851995 4.372256 0.377945 -Total time: 15:17 -epoch train_loss valid_loss accuracy -1 4.281115 4.121970 0.414018 -2 4.072245 3.940537 0.438534 -3 3.903796 3.822031 0.453624 -4 3.863336 3.748684 0.462770 -5 3.818518 3.706158 0.468450 -6 3.755432 3.675938 0.472679 -7 3.738127 3.648453 0.476536 -8 3.717736 3.625716 0.479841 -9 3.720993 3.605317 0.483234 -10 3.695235 3.585903 0.486179 -11 3.696312 3.567543 0.489165 -12 3.688561 3.550402 0.492248 -13 3.654837 3.533271 0.495245 -14 3.667123 3.515244 0.498298 -15 3.610541 3.499110 0.501204 -16 3.589734 3.484431 0.503880 -17 3.586404 3.474879 0.505702 -18 3.590800 3.466892 0.507203 -19 3.570113 3.462836 0.508037 -20 3.544942 3.461981 0.508177 -Total time: 7:11:56 -/home/pczapla/workspace/ulmfit-multilingual/data/cls/ja-books/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/cls/ja-books/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.536773 0.625124 0.775000 -2 0.509836 0.827109 0.735000 -3 0.455667 0.636377 0.805000 -4 0.411549 0.650928 0.725000 -5 0.371732 0.594132 0.795000 -6 0.363084 0.580896 0.810000 -7 0.351636 0.569889 0.790000 -8 0.343057 0.575629 0.790000 -Total time: 02:15 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/cls/ja-books/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.3464668, tensor(0.8630)] -Processing data/wiki/ru-100/models/sp15k/qrnn_nl4.m -../cls/ru-books -Processing data/wiki/zh-100/models/sp15k/qrnn_nl4.m -../cls/zh-books -OrderedDict([('data/cls/de-books/models/sp15k/qrnn_nl4.m', 0.9315000176429749), - ('data/cls/en-books/models/sp15k/qrnn_nl4.m', 0.9075000286102295), - ('data/cls/fr-books/models/sp15k/qrnn_nl4.m', 0.9125000238418579), - ('data/cls/ja-books/models/sp15k/qrnn_nl4.m', 0.8629999756813049)]) -```` diff --git a/results/logs/cls/zeroshoot.md b/results/logs/cls/zeroshoot.md deleted file mode 100644 index d6b7c71..0000000 --- a/results/logs/cls/zeroshoot.md +++ /dev/null @@ -1,118 +0,0 @@ -# Results on books dataset - -| | de | fr | -|-------------------------|-------|-------| -| laser zero shot from en | 84.15 | 83.90 | -| with ULMFIT QRNN sp15k | 89.60 | 87.84 | - -## Laser results - -| | en | de | fr | -|------|--------|-------|------| -| en: | 84.55 | 84.15 | 83.90| -| de: | 82.60 | 85.20 | 83.05| -| fr: | 77.20 | 82.95 | 84.85| - -## ULMFiT improvment -``` -data/cls/de-books-laser-en1/models/sp15k/qrnn_nl4.m: 0.8960000276565552 -data/cls/fr-books-laser-en1/models/sp15k/qrnn_nl4.m: 0.8784999847412109 -``` - -### Execution log -``` -python -m ulmfit eval --glob="cls/*-books/models/sp15k/qrnn_nl4.m" --name nl4 --dataset-template='${lang}-books-laser-en1' --num-lm-epochs=0 --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.1 -Processing data/cls/de-books/models/sp15k/qrnn_nl4.m -de-books-laser-en1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/de-books-laser-en1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/de-books-laser-en1/models/sp15k/qrnn_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/cls/de-books-laser-en1/de.dev.csv -Running tokenization lm... -Data lm, trn: 152523, val: 16947 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 2000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: - - !! WARNING !! - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Your compiler (c++) may be ABI-incompatible with PyTorch! -Please use a compiler that is ABI-compatible with GCC 4.9 and above. -See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. - -See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 -for instructions on how to install GCC 4.9 or higher. -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - !! WARNING !! - - warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/cls/de-books-laser-en1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/cls/de-books-laser-en1/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.541837 0.487596 0.870000 -2 0.498016 0.490300 0.885000 -3 0.442417 0.479205 0.875000 -4 0.395640 0.528897 0.855000 -5 0.369408 0.521830 0.855000 -6 0.361129 0.481892 0.880000 -7 0.351095 0.481634 0.885000 -8 0.343147 0.481654 0.880000 -Total time: 02:35 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/cls/de-books-laser-en1/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.29498395, tensor(0.8960)] -Processing data/cls/en-books/models/sp15k/qrnn_nl4.m -en-books-laser-en1 -Processing data/cls/fr-books/models/sp15k/qrnn_nl4.m -fr-books-laser-en1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books-laser-en1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books-laser-en1/models/sp15k/qrnn_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books-laser-en1/fr.dev.csv -Running tokenization lm... -Data lm, trn: 33183, val: 3687 -Running tokenization cls... -Data cls, trn: 1800, val: 200 -Running tokenization tst... -Data tst, trn: 200, val: 2000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', 's', '▁.', "'", '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books-laser-en1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books-laser-en1/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.551931 0.532421 0.835000 -2 0.509913 0.524893 0.880000 -3 0.433385 0.502657 0.860000 -4 0.397314 0.487201 0.880000 -5 0.365447 0.467523 0.885000 -6 0.356587 0.520736 0.855000 -7 0.353801 0.487093 0.875000 -8 0.343812 0.484453 0.880000 -Total time: 01:45 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books-laser-en1/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.32666296, tensor(0.8785)] -Processing data/cls/ja-books/models/sp15k/qrnn_nl4.m -ja-books-laser-en1 -OrderedDict([('data/cls/de-books-laser-en1/models/sp15k/qrnn_nl4.m', - 0.8960000276565552), - ('data/cls/fr-books-laser-en1/models/sp15k/qrnn_nl4.m', - 0.8784999847412109)]) -data/cls/de-books-laser-en1/models/sp15k/qrnn_nl4.m: 0.8960000276565552 -data/cls/fr-books-laser-en1/models/sp15k/qrnn_nl4.m: 0.8784999847412109 -``` - diff --git a/results/logs/common.md b/results/logs/common.md deleted file mode 100644 index 1c23a0d..0000000 --- a/results/logs/common.md +++ /dev/null @@ -1,1021 +0,0 @@ -# MLDoc -### Different training schedules - -### 1cycle -lstm -``` -(fastaiv1) pczapla@galatea ~/w/ulmfit-multilingual ❯❯❯ python -m ulmfit eval --glob="mldoc/*-1/models/sp30k/lstm_nl4.m" --name nl4-1cyc --num-cls-epochs=8 --bs=18 --lr_sched=1cycle ✘ 1 -Processing data/mldoc/de-1/models/sp30k/lstm_nl4.m -de-1 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-1cyc.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/de.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-1cyc.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.610423 0.287707 0.920000 -2 0.390499 0.266688 0.948000 -3 0.366716 0.302463 0.933000 -4 0.248321 0.305547 0.937000 -5 0.166564 0.411075 0.948000 -6 0.083940 0.406182 0.950000 -7 0.033326 0.388105 0.949000 -8 0.014658 0.397507 0.948000 -Total time: 06:42 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-1cyc.m -Loss and accuracy using (cls_best): [0.3040595, tensor(0.9585)] -Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m -es-1 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-1cyc.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-1cyc.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.610724 0.278892 0.925000 -2 0.372022 0.348428 0.937000 -3 0.310411 0.386958 0.927000 -4 0.215536 0.273834 0.958000 -5 0.163195 0.319600 0.958000 -6 0.085268 0.313287 0.961000 -7 0.037369 0.347500 0.961000 -8 0.016851 0.338436 0.963000 -Total time: 05:36 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-1cyc.m -Loss and accuracy using (cls_best): [0.31034237, tensor(0.9632)] -Processing data/mldoc/fr-1/models/sp30k/lstm_nl4.m -fr-1 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4-1cyc.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/fr.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4-1cyc.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.642809 0.240702 0.928000 -2 0.420564 0.658542 0.852000 -3 0.443345 0.244053 0.927000 -4 0.338779 0.335634 0.914000 -5 0.224778 0.263748 0.928000 -6 0.116705 0.280655 0.944000 -7 0.072063 0.287557 0.945000 -8 0.048084 0.289200 0.946000 -Total time: 06:33 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4-1cyc.m -Loss and accuracy using (cls_best): [0.29398218, tensor(0.9482)] -Processing data/mldoc/it-1/models/sp30k/lstm_nl4.m -it-1 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4-1cyc.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/it.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', "▁&'", "'", '▁e', '▁il', '▁la', 'e', '▁in'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4-1cyc.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.736070 0.391008 0.859000 -2 0.512531 0.614638 0.860000 -3 0.343422 0.594530 0.862000 -4 0.370786 0.540225 0.884000 -5 0.234727 0.591903 0.892000 -6 0.141539 0.589971 0.906000 -7 0.073315 0.544248 0.906000 -8 0.038477 0.580940 0.904000 -Total time: 03:48 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4-1cyc.m -Loss and accuracy using (cls_best): [0.6642357, tensor(0.8988)] -Processing data/mldoc/ja-1/models/sp30k/lstm_nl4.m -ja-1 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-1cyc.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-1cyc.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.804430 0.435780 0.837000 -2 0.576932 0.452903 0.839000 -3 0.499791 0.640789 0.806000 -4 0.418024 0.610898 0.839000 -5 0.259578 0.582953 0.868000 -6 0.188161 0.719131 0.888000 -7 0.101508 0.766175 0.877000 -8 0.072221 0.795415 0.883000 -Total time: 08:19 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-1cyc.m -Loss and accuracy using (cls_best): [0.636261, tensor(0.9045)] -Processing data/mldoc/ru-1/models/sp30k/lstm_nl4.m -ru-1 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4-1cyc.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv -Data lm, trn: 9195, val: 1021 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', '▁и', 'е', 'и', 'й', '▁на', 'х'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4-1cyc.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.821141 0.532432 0.814000 -2 0.611059 0.457023 0.866000 -3 0.464408 0.484035 0.870000 -4 0.446560 0.477454 0.858000 -5 0.299095 0.906959 0.858000 -6 0.176480 0.709579 0.875000 -7 0.089683 0.781081 0.876000 -8 0.047031 0.772935 0.877000 -Total time: 08:58 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4-1cyc.m -Loss and accuracy using (cls_best): [0.8528109, tensor(0.8795)] -Processing data/mldoc/zh-1/models/sp30k/lstm_nl4.m -zh-1 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.696346 0.335083 0.895000 -2 0.505075 0.360600 0.906000 -3 0.427076 0.462661 0.883000 -4 0.351177 0.489026 0.919000 -5 0.244958 0.415151 0.918000 -6 0.156452 0.494367 0.926000 -7 0.085807 0.471611 0.927000 -8 0.046887 0.484232 0.929000 -Total time: 06:41 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc.m -Loss and accuracy using (cls_best): [0.50999963, tensor(0.9165)] -OrderedDict([('data/mldoc/de-1/models/sp30k/lstm_nl4-1cyc.m', - 0.9585000276565552), - ('data/mldoc/es-1/models/sp30k/lstm_nl4-1cyc.m', - 0.9632499814033508), - ('data/mldoc/fr-1/models/sp30k/lstm_nl4-1cyc.m', - 0.9482499957084656), - ('data/mldoc/it-1/models/sp30k/lstm_nl4-1cyc.m', - 0.8987500071525574), - ('data/mldoc/ja-1/models/sp30k/lstm_nl4-1cyc.m', - 0.9045000076293945), - ('data/mldoc/ru-1/models/sp30k/lstm_nl4-1cyc.m', - 0.8794999718666077), - ('data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc.m', - 0.9164999723434448)]) -data/mldoc/de-1/models/sp30k/lstm_nl4-1cyc.m: 0.9585000276565552 -data/mldoc/es-1/models/sp30k/lstm_nl4-1cyc.m: 0.9632499814033508 -data/mldoc/fr-1/models/sp30k/lstm_nl4-1cyc.m: 0.9482499957084656 -data/mldoc/it-1/models/sp30k/lstm_nl4-1cyc.m: 0.8987500071525574 -data/mldoc/ja-1/models/sp30k/lstm_nl4-1cyc.m: 0.9045000076293945 -data/mldoc/ru-1/models/sp30k/lstm_nl4-1cyc.m: 0.8794999718666077 -data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc.m: 0.9164999723434448 -``` - - -### 2cycle -```bash -python -m ulmfit eval --glob="mldoc/*-1/models/sp15k/qrnn_nl4.m" --name nl4-8e-2cycle --num-cls-epochs=8 --bs=18 --lr_sched=2cycle -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-8e-2cycle.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: - - !! WARNING !! - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Your compiler (c++) may be ABI-incompatible with PyTorch! -Please use a compiler that is ABI-compatible with GCC 4.9 and above. -See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. - -See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 -for instructions on how to install GCC 4.9 or higher. -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - !! WARNING !! - - warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-8e-2cycle.m/info.json -2cycle training schedule -epoch train_loss valid_loss accuracy -1 0.600965 0.232749 0.937000 -Total time: 00:07 -epoch train_loss valid_loss accuracy -1 0.304946 0.202946 0.946000 -2 0.326092 0.207825 0.954000 -3 0.286274 0.290416 0.943000 -4 0.230937 0.263474 0.950000 -5 0.153293 0.293336 0.962000 -6 0.080219 0.328380 0.960000 -7 0.065156 0.343692 0.961000 -8 0.046342 0.367162 0.962000 -9 0.028884 0.396987 0.960000 -10 0.034997 0.366203 0.960000 -Total time: 02:27 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-8e-2cycle.m -Loss and accuracy using (cls_best): [0.35007542, tensor(0.9528)] -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-8e-2cycle.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-8e-2cycle.m/info.json -2cycle training schedule -epoch train_loss valid_loss accuracy -1 0.675992 0.460828 0.836000 -Total time: 00:09 -epoch train_loss valid_loss accuracy -1 0.439060 0.314642 0.888000 -2 0.382209 0.374305 0.893000 -3 0.338183 0.361669 0.911000 -4 0.260323 0.431681 0.901000 -5 0.146894 0.597865 0.899000 -6 0.090651 0.589435 0.910000 -7 0.079902 0.624589 0.918000 -8 0.043067 0.558498 0.918000 -9 0.022371 0.568702 0.921000 -10 0.022498 0.576052 0.922000 -Total time: 02:53 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-8e-2cycle.m -Loss and accuracy using (cls_best): [0.6146808, tensor(0.9150)] -OrderedDict([('data/mldoc/es-1/models/sp15k/qrnn_nl4-8e-2cycle.m', - 0.952750027179718), - ('data/mldoc/zh-1/models/sp15k/qrnn_nl4-8e-2cycle.m', - 0.9150000214576721)]) -``` - -### SIUNGLE 2epochs -``` -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-2e-single.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.626836 0.318460 0.912000 -2 0.386851 0.327937 0.918000 -Total time: 00:34 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-2e-single.m -Loss and accuracy using (cls_best): [0.32642558, tensor(0.9135)] -OrderedDict([('data/mldoc/es-1/models/sp15k/qrnn_nl4-2e-single.m', - 0.9539999961853027), - ('data/mldoc/zh-1/models/sp15k/qrnn_nl4-2e-single.m', - 0.9135000109672546)]) -``` -### SINGLE 4epochs -``` -OrderedDict([('data/mldoc/es-1/models/sp15k/qrnn_nl4-4e-single.m', - 0.9539999961853027), - ('data/mldoc/zh-1/models/sp15k/qrnn_nl4-4e-single.m', - 0.9210000038146973)]) -``` - - -### SINGLE 5 epochs -``` - python -m ulmfit eval --glob="mldoc/*-1/models/sp15k/qrnn_nl4.m" --name nl4-5e-single --num-cls-epochs=5 --bs=18 --lr_sched=single -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-5e-single.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: - - !! WARNING !! - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Your compiler (c++) may be ABI-incompatible with PyTorch! -Please use a compiler that is ABI-compatible with GCC 4.9 and above. -See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. - -See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 -for instructions on how to install GCC 4.9 or higher. -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - !! WARNING !! - - warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-5e-single.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.559638 0.250022 0.931000 -2 0.366238 0.348553 0.932000 -3 0.246830 0.243392 0.954000 -4 0.136335 0.242888 0.960000 -5 0.106189 0.254603 0.965000 -Total time: 01:14 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-5e-single.m -Loss and accuracy using (cls_best): [0.24189772, tensor(0.9588)] -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-5e-single.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-5e-single.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.641053 0.334561 0.891000 -2 0.486526 0.374095 0.894000 -3 0.309409 0.359222 0.908000 -4 0.179104 0.403468 0.920000 -5 0.083530 0.414904 0.919000 -Total time: 01:24 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-5e-single.m -Loss and accuracy using (cls_best): [0.44048822, tensor(0.9110)] -OrderedDict([('data/mldoc/es-1/models/sp15k/qrnn_nl4-5e-single.m', - 0.9587500095367432), - ('data/mldoc/zh-1/models/sp15k/qrnn_nl4-5e-single.m', - 0.9110000133514404)]) -``` - - -#### SINGLE 11epochs -``` -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-8e-single.m/info.json -Starting classifier training -Single training schedule -epoch train_loss valid_loss accuracy -1 0.676591 0.205210 0.947000 -2 0.402020 0.461279 0.912000 -3 0.287975 0.496294 0.921000 -4 0.258515 0.243489 0.954000 -5 0.219352 0.274136 0.949000 -6 0.149339 0.352294 0.956000 -7 0.092821 0.378696 0.962000 -8 0.055485 0.367379 0.963000 -9 0.042695 0.367151 0.964000 -10 0.034858 0.386749 0.961000 -11 0.021245 0.392899 0.963000 -Total time: 02:38 - -Loss and accuracy using (cls_last): [0.4098273, tensor(0.9595)] -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-8e-single.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-8e-single.m/info.json -Starting classifier training -Single training schedule -epoch train_loss valid_loss accuracy -1 0.710727 0.355552 0.895000 -2 0.523836 0.328691 0.895000 -3 0.408339 0.440722 0.894000 -4 0.326642 0.422076 0.909000 -5 0.217328 0.539624 0.906000 -6 0.156753 0.583433 0.912000 -7 0.102770 0.549827 0.921000 -8 0.053252 0.533528 0.928000 -9 0.032845 0.568053 0.927000 -10 0.022289 0.604236 0.926000 -11 0.015289 0.587667 0.929000 -Total time: 03:12 -OrderedDict([('data/mldoc/es-1/models/sp15k/qrnn_nl4-8e-single.m', - 0.9595000147819519), - ('data/mldoc/zh-1/models/sp15k/qrnn_nl4-8e-single.m', - 0.9202499985694885)]) - -``` - -## Limiit to 100 examples -``` -python -m ulmfit eval --glob="mldoc/*-1/models/sp30k/lstm_nl4.m" --name nl4-100e8 --cuda-id=1 --limit=100 --num-cls-epochs=8 -{ - 'data/mldoc/it-1/models/sp30k/lstm_nl4-100e8.m': 0.7799999713897705, - 'data/mldoc/de-1/models/sp30k/lstm_nl4-100e8.m': 0.9135000109672546, - 'data/mldoc/ja-1/models/sp30k/lstm_nl4-100e8.m': 0.7112500071525574, - 'data/mldoc/fr-1/models/sp30k/lstm_nl4-100e8.m': 0.8877500295639038, - 'data/mldoc/ru-1/models/sp30k/lstm_nl4-100e8.m': 0.722000002861023, - 'data/mldoc/es-1/models/sp30k/lstm_nl4-100e8.m': 0.8169999718666077 -} -``` - - -## Noise - -``` -noise=0.13 -lang=de -python -m ulmfit eval --glob="mldoc/${lang}-1/models/sp30k/lstm_nl4.m" --name nl4-noise --cuda-id=1 --num-cls-epochs=2 --noise=${noise} -{'data/mldoc/de-1/models/sp30k/lstm_nl4-noise.m': 0.9449999928474426} - -noise=0.18 -lang=es -python -m ulmfit eval --glob="mldoc/${lang}-1/models/sp30k/lstm_nl4.m" --name nl4-noise --cuda-id=1 --num-cls-epochs=2 --noise=${noise} -{'data/mldoc/es-1/models/sp30k/lstm_nl4-noise.m': 0.9312499761581421} - -noise=0.18 -lang=fr -python -m ulmfit eval --glob="mldoc/${lang}-1/models/sp30k/lstm_nl4.m" --name nl4-noise --cuda-id=1 --num-cls-epochs=2 --noise=${noise} -{'data/mldoc/fr-1/models/sp30k/lstm_nl4-noise.m': 0.9049999713897705} - -noise=0.27 -lang=it -python -m ulmfit eval --glob="mldoc/${lang}-1/models/sp30k/lstm_nl4.m" --name nl4-noise --cuda-id=1 --num-cls-epochs=2 --noise=${noise} -{'data/mldoc/it-1/models/sp30k/lstm_nl4-noise.m': 0.8372499942779541} - -noise=0.4 -lang=ja -python -m ulmfit eval --glob="mldoc/${lang}-1/models/sp30k/lstm_nl4.m" --name nl4-noise --cuda-id=1 --num-cls-epochs=2 --noise=${noise} -{'data/mldoc/ja-1/models/sp30k/lstm_nl4-noise.m': 0.7472500205039978 - -noise=0.32 -lang=ru -python -m ulmfit eval --glob="mldoc/${lang}-1/models/sp30k/lstm_nl4.m" --name nl4-noise --cuda-id=1 --num-cls-epochs=2 --noise=${noise} -{'data/mldoc/ru-1/models/sp30k/lstm_nl4-noise.m': 0.7567499876022339} - -noise=0.28 -lang=zh -python -m ulmfit eval --glob="mldoc/${lang}-1/models/sp30k/lstm_nl4.m" --name nl4-noise --cuda-id=1 --num-cls-epochs=2 --noise=${noise} -``` - - - -## Fix the sentence piece tokenizer -``` -python -m ulmfit eval --glob="mldoc/*-1/models/bsp30k/lstm_nl4.m" --name nl4 --cuda-id=0 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/de.dev.csv -Running tokenization... -Saving tokenized: cls.trn 13500, cls.val 1500 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/bsp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/bsp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.480367 0.263987 0.930000 -epoch train_loss valid_loss accuracy -1 0.340463 0.209449 0.940000 -epoch train_loss valid_loss accuracy -1 0.235290 0.214566 0.952000 -epoch train_loss valid_loss accuracy -1 0.169588 0.217762 0.952000 -2 0.175439 0.215570 0.946000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.15467079, tensor(0.9563)] -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv -Running tokenization... -Saving tokenized: cls.trn 13013, cls.val 1445 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/bsp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/bsp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.609305 0.343920 0.914000 -epoch train_loss valid_loss accuracy -1 0.380833 0.204708 0.947000 -epoch train_loss valid_loss accuracy -1 0.311563 0.210382 0.943000 -epoch train_loss valid_loss accuracy -1 0.266655 0.192309 0.955000 -2 0.235553 0.183249 0.953000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.19250762, tensor(0.9427)] -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/fr.dev.csv -Running tokenization... -Saving tokenized: cls.trn 13500, cls.val 1500 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/bsp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/bsp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.527443 0.342664 0.901000 -epoch train_loss valid_loss accuracy -1 0.359752 0.203693 0.936000 -epoch train_loss valid_loss accuracy -1 0.272514 0.188933 0.938000 -epoch train_loss valid_loss accuracy -1 0.191520 0.183208 0.938000 -2 0.201412 0.178796 0.942000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.18170285, tensor(0.9420)] -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/it.dev.csv -Running tokenization... -Saving tokenized: cls.trn 13500, cls.val 1500 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', "▁&'", "'", '▁e', '▁il', '▁la', 'e', '▁in'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/bsp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/bsp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.754286 0.599928 0.783000 -epoch train_loss valid_loss accuracy -1 0.501399 0.379078 0.851000 -epoch train_loss valid_loss accuracy -1 0.408768 0.345188 0.867000 -epoch train_loss valid_loss accuracy -1 0.324877 0.335110 0.872000 -2 0.291118 0.336596 0.879000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.33244577, tensor(0.8852)] -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Tokenized data loaded, cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] -Loss and accuracy using (cls_last): [0.330675, tensor(0.8873)] -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv -Tokenized data loaded, lm.trn 9195, lm.val 1021 -Tokenized data loaded, cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', '▁и', 'е', 'и', 'й', '▁на', 'х'] -Loss and accuracy using (cls_last): [0.3987146, tensor(0.8680)] -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv -Running tokenization... -Saving tokenized: cls.trn 13500, cls.val 1500 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/bsp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/bsp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.612067 0.487164 0.860000 -epoch train_loss valid_loss accuracy -1 0.441519 0.343330 0.886000 -epoch train_loss valid_loss accuracy -1 0.384575 0.318655 0.897000 -epoch train_loss valid_loss accuracy -1 0.298987 0.305157 0.899000 -2 0.293190 0.312572 0.901000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.28432375, tensor(0.9047)] -OrderedDict([('data/mldoc/de-1/models/sp30k/lstm_nl4.m', 0.956250011920929), - ('data/mldoc/es-1/models/sp30k/lstm_nl4.m', 0.9427499771118164), - ('data/mldoc/fr-1/models/sp30k/lstm_nl4.m', 0.9419999718666077), - ('data/mldoc/it-1/models/sp30k/lstm_nl4.m', 0.8852499723434448), - ('data/mldoc/ja-1/models/sp30k/lstm_nl4.m', 0.8872500061988831), - ('data/mldoc/ru-1/models/sp30k/lstm_nl4.m', 0.8679999709129333), - ('data/mldoc/zh-1/models/sp30k/lstm_nl4.m', 0.9047499895095825)]) - ---- Additonal run on ru -python -m ulmfit eval --glob="mldoc/ru-1/models/bsp30k/lstm_nl4.m" --name nl4 --cuda-id=0 ✘ 1 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv -Running tokenization... -Saving tokenized: cls.trn 9195, cls.val 1021 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', '▁и', 'е', 'и', 'й', '▁на', 'х'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/bsp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/bsp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.814752 0.555627 0.808000 -epoch train_loss valid_loss accuracy -1 0.659532 0.427135 0.855000 -epoch train_loss valid_loss accuracy -1 0.508609 0.427321 0.851000 -epoch train_loss valid_loss accuracy -1 0.440108 0.396991 0.872000 -2 0.440024 0.388976 0.866000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.3959306, tensor(0.8685)] ----- - ----- - python -m ulmfit eval --glob="mldoc/es-1/models/bsp30k/lstm_nl4.m" --name nl4-2nd --cuda-id=0 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-2nd.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv -Tokenized data loaded, lm.trn 13013, lm.val 1445 -Tokenized data loaded, cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/bsp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/bsp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-2nd.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.654984 0.453887 0.818000 -epoch train_loss valid_loss accuracy -1 0.451552 0.220058 0.934000 -epoch train_loss valid_loss accuracy -1 0.323974 0.193342 0.949000 -epoch train_loss valid_loss accuracy -1 0.244755 0.201804 0.945000 -2 0.237175 0.183736 0.953000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-2nd.m -Loss and accuracy using (cls_best): [0.18296617, tensor(0.9480)] -OrderedDict([('data/mldoc/es-1/models/sp30k/lstm_nl4-2nd.m', - 0.9480000138282776)]) ----- - -``` - -### LIMIT LOgs -``` -python -m ulmfit eval --name nl4-100e8 --cuda-id=1 --limit=100 --num-cls-epochs=8 ✘ 130 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4-100e8.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/it.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Limiting data set to: 100 -Tokenized data loaded, cls.trn 100, cls.val 100 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', "▁&'", "'", '▁e', '▁il', '▁la', 'e', '▁in'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4-100e8.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 1.214805 1.382632 0.280000 -epoch train_loss valid_loss accuracy -1 0.977314 1.269534 0.450000 -epoch train_loss valid_loss accuracy -1 0.856274 1.223441 0.530000 -epoch train_loss valid_loss accuracy -1 0.718223 1.188048 0.620000 -2 0.735718 1.130525 0.730000 -3 0.730894 1.069027 0.710000 -4 0.715334 1.015253 0.710000 -5 0.716080 0.965223 0.720000 -6 0.695554 0.918456 0.730000 -7 0.689949 0.892840 0.730000 -8 0.675208 0.876222 0.720000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4-100e8.m -Loss and accuracy using (cls_best): [0.7090041, tensor(0.7800)] -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-100e8.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/de.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Limiting data set to: 100 -Running tokenization... -Saving tokenized: cls.trn 100, cls.val 100 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-100e8.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 1.141527 1.328262 0.280000 -epoch train_loss valid_loss accuracy -1 0.703434 1.170250 0.510000 -epoch train_loss valid_loss accuracy -1 0.568693 1.051980 0.780000 -epoch train_loss valid_loss accuracy -1 0.455238 0.990438 0.800000 -2 0.475659 0.928943 0.850000 -3 0.477652 0.848537 0.920000 -4 0.455583 0.769415 0.930000 -5 0.450824 0.690618 0.930000 -6 0.443699 0.633900 0.940000 -7 0.430881 0.563667 0.950000 -8 0.419999 0.524655 0.950000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-100e8.m -Loss and accuracy using (cls_best): [0.45835665, tensor(0.9135)] -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-100e8.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Limiting data set to: 100 -Tokenized data loaded, cls.trn 100, cls.val 100 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-100e8.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 1.342269 1.399389 0.230000 -epoch train_loss valid_loss accuracy -1 0.957341 1.344665 0.280000 -epoch train_loss valid_loss accuracy -1 0.881869 1.301798 0.450000 -epoch train_loss valid_loss accuracy -1 0.887575 1.280226 0.440000 -2 0.835731 1.257639 0.450000 -3 0.813987 1.219512 0.510000 -4 0.792665 1.181309 0.520000 -5 0.785690 1.151372 0.510000 -6 0.784095 1.152232 0.500000 -7 0.768115 1.133895 0.520000 -8 0.769684 1.124231 0.530000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-100e8.m -Loss and accuracy using (cls_best): [0.8863698, tensor(0.7113)] -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4-100e8.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/fr.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Limiting data set to: 100 -Running tokenization... -Saving tokenized: cls.trn 100, cls.val 100 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4-100e8.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 1.220506 1.413276 0.200000 -epoch train_loss valid_loss accuracy -1 0.791777 1.306999 0.290000 -epoch train_loss valid_loss accuracy -1 0.572241 1.190053 0.580000 -epoch train_loss valid_loss accuracy -1 0.502800 1.130456 0.710000 -2 0.515115 1.056434 0.770000 -3 0.522720 0.974482 0.780000 -4 0.518296 0.881002 0.840000 -5 0.496588 0.825646 0.880000 -6 0.490416 0.771587 0.860000 -7 0.497172 0.722874 0.850000 -8 0.491894 0.682278 0.850000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4-100e8.m -Loss and accuracy using (cls_best): [0.5428351, tensor(0.8878)] -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4-100e8.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv -Tokenized data loaded, lm.trn 9195, lm.val 1021 -Limiting data set to: 100 -Running tokenization... -Saving tokenized: cls.trn 100, cls.val 100 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', '▁и', 'е', 'и', 'й', '▁на', 'х'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4-100e8.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 1.367201 1.409767 0.240000 -epoch train_loss valid_loss accuracy -1 1.099071 1.320811 0.330000 -epoch train_loss valid_loss accuracy -1 0.875845 1.253172 0.410000 -epoch train_loss valid_loss accuracy -1 0.775657 1.215067 0.580000 -2 0.774420 1.171324 0.660000 -3 0.766028 1.118901 0.680000 -4 0.744478 1.074021 0.680000 -5 0.738797 1.033736 0.660000 -6 0.733380 0.997304 0.660000 -7 0.723470 0.977280 0.670000 -8 0.710699 0.953586 0.640000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4-100e8.m -Loss and accuracy using (cls_best): [0.8535175, tensor(0.7220)] -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-100e8.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv -Tokenized data loaded, lm.trn 13013, lm.val 1445 -Limiting data set to: 100 -Running tokenization... -Saving tokenized: cls.trn 100, cls.val 100 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-100e8.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 1.142170 1.330161 0.300000 -epoch train_loss valid_loss accuracy -1 0.767807 1.212253 0.420000 -epoch train_loss valid_loss accuracy -1 0.636803 1.099303 0.540000 -epoch train_loss valid_loss accuracy -1 0.584241 0.997207 0.610000 -2 0.578480 0.907674 0.710000 -3 0.548451 0.830268 0.730000 -4 0.535560 0.762040 0.750000 -5 0.522172 0.746566 0.740000 -6 0.506584 0.676038 0.770000 -7 0.493665 0.651112 0.770000 -8 0.493031 0.621689 0.770000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-100e8.m -Loss and accuracy using (cls_best): [0.54911107, tensor(0.8170)] -{'data/mldoc/it-1/models/sp30k/lstm_nl4-100e8.m': 0.7799999713897705, 'data/mldoc/de-1/models/sp30k/lstm_nl4-100e8.m': 0.9135000109672546, 'data/mldoc/ja-1/models/sp30k/lstm_nl4-100e8.m': 0.7112500071525574, 'data/mldoc/fr-1/models/sp30k/lstm_nl4-100e8.m': 0.8877500295639038, 'data/mldoc/ru-1/models/sp30k/lstm_nl4-100e8.m': 0.722000002861023, 'data/mldoc/es-1/models/sp30k/lstm_nl4-100e8.m': 0.8169999718666077} - -python -m ulmfit eval --glob="mldoc/es-1/models/sp30k/lstm_nl4.m" --name nl4-100-2nd --cuda-id=1 --num-cls-epochs=8 --limit=100 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-100-2nd.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv -Tokenized data loaded, lm.trn 13013, lm.val 1445 -Limiting data set to: 100 -Tokenized data loaded, cls.trn 100, cls.val 100 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-100-2nd.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 1.243127 1.354496 0.290000 -epoch train_loss valid_loss accuracy -1 0.840900 1.213333 0.460000 -epoch train_loss valid_loss accuracy -1 0.656407 1.055138 0.750000 -epoch train_loss valid_loss accuracy -1 0.558013 0.983957 0.780000 -2 0.554590 0.915244 0.750000 -3 0.536740 0.840074 0.770000 -4 0.521179 0.759908 0.790000 -5 0.515218 0.692961 0.810000 -6 0.500587 0.639504 0.810000 -7 0.486596 0.593410 0.840000 -8 0.472318 0.550126 0.830000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-100-2nd.m -Loss and accuracy using (cls_best): [0.5382241, tensor(0.8332)] -{'data/mldoc/es-1/models/sp30k/lstm_nl4-100-2nd.m': 0.8332499861717224} - -``` \ No newline at end of file diff --git a/results/logs/de.md b/results/logs/de.md deleted file mode 100644 index 5142dbb..0000000 --- a/results/logs/de.md +++ /dev/null @@ -1,605 +0,0 @@ -# DE -## SP15k LSTM nl4 -``` -$ python -m ulmfit lm --dataset-path data/wiki/de-100 --cuda-id=0 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang de --qrnn=False - train 10 --bs=100 --drop_mult=0 - -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', -'▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Training lm from random weights -epoch train_loss valid_loss accuracy -1 2.519809 2.600072 0.529963 -2 2.436580 2.538897 0.534651 -3 2.402220 2.510569 0.537314 -4 2.305741 2.439347 0.546574 -5 2.265683 2.376482 0.553794 -6 2.210663 2.305362 0.562672 -7 2.134196 2.230041 0.572958 -8 2.085375 2.150917 0.584621 -9 2.037781 2.097170 0.593747 -10 1.986773 2.081469 0.595799 -Total time: 19:18:33 -data/wiki/de-100/models/sp15k -Saving info data/wiki/de-100/models/sp15k/lstm_nl4.m/info.jso -``` -### MLDoc -```bash -LANG=de -python -m ulmfit cls --dataset-path data/mldoc-m/${LANG}-1 --base-lm-path data/wiki-m/${LANG}-100/models/sp15k/lstm_nl4.m --lang=${LANG} --name 'nl4' - train 20 --bs 20 --num-cls- -epochs=8 --lr_sched=1cycle -Max vocab: 15000 -Cache dir: /home/test/workspace/ulmfit-multilingual/data/mldoc-m/de-1/models/sp15k -Model dir: /home/test/workspace/ulmfit-multilingual/data/mldoc-m/de-1/models/sp15k/lstm_nl4.m -Loading validation /home/test/workspace/ulmfit-multilingual/data/mldoc-m/de-1/de.dev.csv -Running tokenization lm... -Data lm, trn: 13500, val: 1500 -Running tokenization cls... -Data cls, trn: 1000, val: 1000 -Running tokenization tst... -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/test/workspace/ulmfit-multilingual/data/wiki-m/de-100/models/sp15k/lstm_nl4.m/lm_best'), PosixPath('/home/test/workspace/ulmfit-multilingual/data/wiki-m/de-100/models/sp15k/lstm_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 2.333600 2.005051 0.596875 -Total time: 07:40 -epoch train_loss valid_loss accuracy -1 2.120653 1.886799 0.615784 -2 1.980713 1.763139 0.636041 -3 1.805195 1.655620 0.654068 -4 1.729641 1.564017 0.668772 -5 1.681813 1.491185 0.680613 -6 1.682965 1.422562 0.692458 -7 1.580731 1.357177 0.703143 -8 1.506753 1.297219 0.714487 -9 1.515824 1.235473 0.725413 -10 1.427750 1.178680 0.737216 -11 1.371839 1.118909 0.749590 -12 1.342978 1.068754 0.760473 -13 1.286842 1.011940 0.772384 -14 1.254822 0.960727 0.784244 -15 1.195136 0.919377 0.793910 -16 1.118260 0.881799 0.802814 -17 1.071546 0.855769 0.809040 -18 1.079081 0.839280 0.812895 -19 1.052724 0.831323 0.814723 -20 1.024207 0.829737 0.815070 -Total time: 3:08:58 -/home/test/workspace/ulmfit-multilingual/data/mldoc-m/de-1/models/sp15k -Saving info /home/test/workspace/ulmfit-multilingual/data/mldoc-m/de-1/models/sp15k/lstm_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.539181 0.239851 0.938000 -2 0.326801 0.374512 0.917000 -3 0.225103 0.330872 0.945000 -4 0.121660 0.444890 0.938000 -5 0.078411 0.422513 0.948000 -6 0.061354 0.509489 0.949000 -7 0.029890 0.438118 0.949000 -8 0.014213 0.441808 0.949000 -Total time: 09:00 -Saving models at /home/test/workspace/ulmfit-multilingual/data/mldoc-m/de-1/models/sp15k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.3710725, tensor(0.9553)] - -0.3710725009441376 -0.9552500247955322 -``` - - -## VF60k LSTM nl 3 -### LM -``` -python -m ulmfit lm --dataset-path data/wiki/de-100 --cuda-id=1 --tokenizer='vf' --nl 3 --name 'nl3' --max-vocab 60000 --lang de --qrnn=False - train 10 --bs=50 --drop_mult=0 -Max vocab: 60000 -Cache dir: data/wiki/de-100/models/vf60k -Model dir: data/wiki/de-100/models/vf60k/lstm_nl3.m -Running tokenization -Wiki text was split to 175965 articles -Wiki text was split to 110 articles -Size of vocabulary: 60003 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', ',', 'der', '.', 'und', 'die', 'in', "&'", 'von', 'den', '(', 'im', ')'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': None, 'pretrained_model': None, 'drop_mult': 0} dps: [0.25 0.1 0.2 0.02 0.15] -Training lm from random weights -epoch train_loss valid_loss accuracy -1 3.214624 3.573368 0.397312 -2 3.194401 3.549021 0.396143 -3 3.116934 3.535322 0.398108 -4 3.159205 3.498862 0.400490 -5 3.104538 3.454015 0.405504 -6 2.996653 3.410940 0.409791 -7 2.987909 3.359425 0.413711 -8 2.941863 3.311215 0.419416 -9 2.914403 3.285807 0.423674 -10 2.857530 3.278313 0.425131 -data/wiki/de-100/models/vf60k -Saving info data/wiki/de-100/models/vf60k/lstm_nl3.m/info.json -``` -### MLDocs -``` -python -m ulmfit cls --dataset-path data/mldoc/de-1 --base-lm-path data/wiki/de-100/models/vf60k/lstm_nl3.m --lang=de --name 'nl3' - train 20 --bs 40 -Max vocab: 60000 -Cache dir: data/mldoc/de-1/models/vf60k -Model dir: data/mldoc/de-1/models/vf60k/lstm_nl3.m -Loading validation data/mldoc/de-1/de.dev.csv -Running tokenization... -Saving tokenized: cls.trn 13500, cls.val 1500 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 39171 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '.', 'der', ',', 'die', ')', '(', 'in', 'und', 'auf', 'von', 'den', 'im'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('data/wiki/de-100/models/vf60k/lstm_nl3.m/lm_best'), PosixPath('data/wiki/de-100/models/vf60k/lstm_nl3.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 20582, first 100: ['"', 'vh', 'ös', 'brs', '&', 'geg', 'lpo', 'vormonat', 'fgc', 'waigel', 'tcs', 'mic', 'bund-future', 'ajs', 'brn', 'dih', 'analysten', 'mrd', 'rpk', 'notierten', 'dividende', 'feb', 'aktienmarkt', 'rev', 'rentenmarkt', 'basispunkte', 'müßten', 'gewinnmitnahmen', 'aktienbörse', 'rußland', 'volkswirte', 'fls', 'steuerreform', 'kontrakte', 'kps', 'mge', 'zählern', 'vortagesschluß', 'umsätzen', 'prozent.', 'snb', 'dow-jones-index', 'reingewinn', 'notierungen', "\\'", 'gesamtmarkt', 'industrieproduktion', 'akr', 'kjf', '49-69-7565', 'abl', 'hoh', 'finanzdienst', 'atx', 'feinunze', 'zinserhöhung', 'zugelegt', 'netanjahu', 'verbraucherpreise', 'pence', 'ticks', 'arafat', 'kursgewinne', 'ker', 'aktienindex', 'rlb', 'smi', 'vorbörslich', 'dst', 'mkl', 'kontrakten', 'calls', 'veraenderung', 'gwa', 'gesamtjahr', 'auftragseingang', 'überschuß', 'erwarte', 'verlautete', 'eju', 'tms', 'jahresvergleich', 'vorjahreszeitraum', 'werden.', 'betriebsergebnis', 'rin', 'bobl-future', 'puts', 'fri', '4.50', 'schluß', 'ewu', 'standardwerte', 'jahresüberschuß', 'rechne', '49-69-756525', '16.00', 'peh', 'hmh', 'dtb'] -Training lm from: [PosixPath('data/wiki/de-100/models/vf60k/lstm_nl3.m/lm_best'), PosixPath('data/wiki/de-100/models/vf60k/lstm_nl3.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.532079 3.059487 0.465283 -epoch train_loss valid_loss accuracy -1 3.219148 2.945357 0.475736 -2 3.014822 2.804256 0.494567 -3 2.896143 2.652700 0.513166 -4 2.756027 2.516747 0.528836 -5 2.629735 2.383480 0.543956 -6 2.515785 2.281831 0.556083 -7 2.422463 2.178855 0.567950 -8 2.351060 2.091266 0.579531 -9 2.297676 2.017783 0.590206 -10 2.205688 1.937085 0.601936 -11 2.155664 1.871271 0.612579 -12 2.065812 1.806647 0.623888 -13 2.038635 1.748420 0.634389 -14 1.957434 1.696571 0.643807 -15 1.895242 1.653865 0.651743 -16 1.910458 1.618776 0.658140 -17 1.843909 1.598143 0.662129 -18 1.837299 1.583182 0.664999 -19 1.788718 1.573136 0.666785 -20 1.780236 1.574308 0.666625 -data/mldoc/de-1/models/vf60k -Saving info data/mldoc/de-1/models/vf60k/lstm_nl3.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.526303 0.328480 0.892000 -epoch train_loss valid_loss accuracy -1 0.346665 0.238605 0.920000 -epoch train_loss valid_loss accuracy -1 0.266841 0.285444 0.921000 -epoch train_loss valid_loss accuracy -1 0.175013 0.280545 0.921000 -2 0.178333 0.286059 0.923000 -Saving models at data/mldoc/de-1/models/vf60k/lstm_nl3.m -Loss and accuracy using (cls_best): [0.16954255, tensor(0.9475)] -OrderedDict([('data/mldoc/de-1/models/vf60k/lstm_nl3.m', 0.9474999904632568)]) -``` -MultiCCA: 93.7% , ulmfit: 94.74% -## SP30k LSTM nl 4 -### LM -``` -python -m ulmfit lm --dataset-path data/wiki/de-100 --cuda-id=0 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 30000 --lang sp --qrnn=False - train 10 --bs=50 --drop_mult=0 -1,2.833101,3.174348,0.472863 -2,2.788717,3.171983,0.471377 -3,2.831292,3.187135,0.471068 -4,2.723390,3.133801,0.475572 -5,2.681617,3.064743,0.481984 -6,2.662792,2.984701,0.489080 -7,2.542035,2.892254,0.499275 -8,2.422225,2.806846,0.508663 -9,2.462655,2.736171,0.517994 -10,2.396778,2.714520,0.521145 -data/wiki/de-100/models/sp30k/lstm_nl4.m/lm-history.csv -``` - -### MLDocs -``` -python -m ulmfit cls --dataset-path data/mldoc/de-1 --base-lm-path data/wiki/de-100/models/sp30k/lstm_nl4.m --lang=de --name 'nl4' - train 20 --bs 40 ✘ 1 -Max vocab: 30000 -Cache dir: data/mldoc/de-1/models/sp30k -Model dir: data/mldoc/de-1/models/sp30k/lstm_nl4.m -Loading validation data/mldoc/de-1/de.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Tokenized data loaded, cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('data/wiki/de-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('data/wiki/de-100/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('data/wiki/de-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('data/wiki/de-100/models/sp30k/lstm_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.042075 2.457199 0.547201 -epoch train_loss valid_loss accuracy -1 2.581403 2.305440 0.565500 -2 2.366814 2.139165 0.589417 -3 2.187646 1.986698 0.612081 -4 2.054434 1.857322 0.630642 -5 1.948663 1.758499 0.644389 -6 1.850596 1.673632 0.655852 -7 1.813331 1.593225 0.668256 -8 1.738136 1.523946 0.678633 -9 1.683469 1.463405 0.688561 -10 1.609236 1.410462 0.697171 -11 1.599416 1.356008 0.706997 -12 1.526982 1.308399 0.715433 -13 1.487115 1.263120 0.723749 -14 1.430917 1.224060 0.731837 -15 1.410333 1.191501 0.738267 -16 1.385961 1.166404 0.743477 -17 1.349813 1.144801 0.747553 -18 1.345938 1.132679 0.750188 -19 1.311102 1.127321 0.751208 -20 1.355743 1.126064 0.751384 -Saving info data/mldoc/de-1/models/sp30k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.490199 0.246640 0.940000 -epoch train_loss valid_loss accuracy -1 0.302251 0.243051 0.932000 -epoch train_loss valid_loss accuracy -1 0.211028 0.249550 0.932000 -epoch train_loss valid_loss accuracy -1 0.159555 0.230822 0.947000 -2 0.144418 0.226450 0.943000 -Saving models at data/mldoc/de-1/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_last): [0.16306259, tensor(0.9540)] -``` -MultiCCA: 93.7% , ulmfit: 95.4% - ``` - Max vocab: 30000 -Cache dir: data/mldoc/de-1/models/sp30k -Model dir: data/mldoc/de-1/models/sp30k/lstm_nl4-2nd.m -Loading validation data/mldoc/de-1/de.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Tokenized data loaded, cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -data/mldoc/de-1/models/sp30k -Saving info data/mldoc/de-1/models/sp30k/lstm_nl4-2nd.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.464957 0.258905 0.928000 -epoch train_loss valid_loss accuracy -1 0.284900 0.243053 0.937000 -epoch train_loss valid_loss accuracy -1 0.298546 0.204188 0.948000 -epoch train_loss valid_loss accuracy -1 0.159097 0.199651 0.952000 -2 0.112476 0.203827 0.953000 -Saving models at data/mldoc/de-1/models/sp30k/lstm_nl4-2nd.m -Loss and accuracy using (cls_last): [0.1689675, tensor(0.9550)] - ``` -### examples limited to 100 - -#### 2x run -first run -``` -python -m ulmfit cls --dataset-path data/mldoc/de-1 --base-lm-path data/mldoc/de-1/models/sp30k/lstm_nl4.m --lang=de --name 'nl4-100-2x' --cuda-id=1 - train 0 --bs 40 --limit=100 --drop-mult-cls=0.3 -Max vocab: 30000 -Cache dir: data/mldoc/de-1/models/sp30k -Model dir: data/mldoc/de-1/models/sp30k/lstm_nl4-100-2x.m -Loading validation data/mldoc/de-1/de.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Limiting data set to: 100 -Running tokenization... -Saving tokenized: cls.trn 100, cls.val 100 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -data/mldoc/de-1/models/sp30k -Saving info data/mldoc/de-1/models/sp30k/lstm_nl4-100-2x.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 1.181207 1.315258 0.300000 -epoch train_loss valid_loss accuracy -1 0.749909 1.204297 0.660000 -epoch train_loss valid_loss accuracy -1 0.558658 1.083666 0.830000 -epoch train_loss valid_loss accuracy -1 0.486175 1.020435 0.850000 -2 0.485117 0.958238 0.880000 -Saving models at data/mldoc/de-1/models/sp30k/lstm_nl4-100-2x.m - ..? .. -``` -2nd run -``` -python -m ulmfit cls --dataset-path data/mldoc/de-1 --base-lm-path data/mldoc/de-1/models/sp30k/lstm_nl4.m --lang=de --name 'nl4-100-2x' --cuda-id=1 - train 0 --bs 40 --limit=100 -Max vocab: 30000 -Cache dir: data/mldoc/de-1/models/sp30k -Model dir: data/mldoc/de-1/models/sp30k/lstm_nl4-100-2x.m -Loading validation data/mldoc/de-1/de.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Limiting data set to: 100 -Tokenized data loaded, cls.trn 100, cls.val 100 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] -Loading last classifier -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.441340 0.716396 0.840000 -epoch train_loss valid_loss accuracy -1 0.312035 0.532610 0.910000 -epoch train_loss valid_loss accuracy -1 0.267714 0.462694 0.920000 -epoch train_loss valid_loss accuracy -1 0.242031 0.430018 0.930000 -2 0.231161 0.398335 0.930000 -Saving models at data/mldoc/de-1/models/sp30k/lstm_nl4-100-2x.m -Loss and accuracy using (cls_last): [0.33284584, tensor(0.9252)] -``` -#### 8 epoches at the end -``` -python -m ulmfit cls --dataset-path data/mldoc/de-1 --base-lm-path data/mldoc/de-1/models/sp30k/lstm_nl4.m --lang=de --name 'nl4-100-e8' --cuda-id=1 - train 0 --bs 40 --limit=100 --num-cls-epochs=8 --drop-mult-cls=0.3 -Max vocab: 30000 -Cache dir: data/mldoc/de-1/models/sp30k -Model dir: data/mldoc/de-1/models/sp30k/lstm_nl4-100-e8.m -Loading validation data/mldoc/de-1/de.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Limiting data set to: 100 -Tokenized data loaded, cls.trn 100, cls.val 100 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -data/mldoc/de-1/models/sp30k -Saving info data/mldoc/de-1/models/sp30k/lstm_nl4-100-e8.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 1.208816 1.324359 0.280000 -epoch train_loss valid_loss accuracy -1 0.716811 1.195012 0.440000 -epoch train_loss valid_loss accuracy -1 0.535809 1.075753 0.590000 -epoch train_loss valid_loss accuracy -1 0.499198 1.018431 0.760000 -2 0.480971 0.948658 0.880000 -3 0.468659 0.866477 0.860000 -4 0.460322 0.770794 0.880000 -5 0.461138 0.704613 0.900000 -6 0.442423 0.623944 0.900000 -7 0.422423 0.568031 0.920000 -8 0.417041 0.527571 0.930000 -Saving models at data/mldoc/de-1/models/sp30k/lstm_nl4-100-e8.m -Loss and accuracy using (cls_last): [0.47343642, tensor(0.9070)] -``` -Dropout 0.6 -``` -python -m ulmfit cls --dataset-path data/mldoc/de-1 --base-lm-path data/mldoc/de-1/models/sp30k/lstm_nl4.m --lang=de --name 'nl4-100-e8dp6' --cuda-id=1 - train 0 --bs 40 --limit=100 --num-cls-epochs=8 --drop-mult-cls=0.6 -Max vocab: 30000 -Cache dir: data/mldoc/de-1/models/sp30k -Model dir: data/mldoc/de-1/models/sp30k/lstm_nl4-100-e8dp6.m -Loading validation data/mldoc/de-1/de.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Limiting data set to: 100 -Tokenized data loaded, cls.trn 100, cls.val 100 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -data/mldoc/de-1/models/sp30k -Saving info data/mldoc/de-1/models/sp30k/lstm_nl4-100-e8dp6.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 1.151162 1.323036 0.280000 -epoch train_loss valid_loss accuracy -1 0.745338 1.160084 0.610000 -epoch train_loss valid_loss accuracy -1 0.535118 1.041519 0.770000 -epoch train_loss valid_loss accuracy -1 0.459913 0.995187 0.860000 -2 0.451289 0.949036 0.830000 -3 0.460395 0.885940 0.800000 -4 0.454847 0.848194 0.770000 -5 0.447404 0.788741 0.810000 -6 0.428524 0.748181 0.760000 -7 0.419571 0.696069 0.760000 -8 0.408938 0.661937 0.770000 -Saving models at data/mldoc/de-1/models/sp30k/lstm_nl4-100-e8dp6.m -Loss and accuracy using (cls_last): [0.53202456, tensor(0.8830)] -``` -``` -python -m ulmfit cls --dataset-path data/mldoc/de-1 --base-lm-path data/mldoc/de-1/models/sp30k/lstm_nl4.m --lang=de --name 'nl4-100-e8dp6x2' --cuda-id=1 - train 0 --bs 40 --limit=100 --num-cls-epochs=8 --drop-mult-cls=0.6 -Max vocab: 30000 -Cache dir: data/mldoc/de-1/models/sp30k -Model dir: data/mldoc/de-1/models/sp30k/lstm_nl4-100-e8dp6x2.m -Loading validation data/mldoc/de-1/de.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Limiting data set to: 100 -Tokenized data loaded, cls.trn 100, cls.val 100 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -data/mldoc/de-1/models/sp30k -Saving info data/mldoc/de-1/models/sp30k/lstm_nl4-100-e8dp6x2.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 1.126586 1.338514 0.470000 -epoch train_loss valid_loss accuracy -1 0.751379 1.181071 0.550000 -epoch train_loss valid_loss accuracy -1 0.559534 1.083532 0.810000 -epoch train_loss valid_loss accuracy -1 0.444137 1.034607 0.870000 -2 0.438850 0.983929 0.830000 -3 0.436560 0.906958 0.840000 -4 0.447400 0.847952 0.840000 -5 0.431961 0.783818 0.850000 -6 0.422364 0.713126 0.850000 -7 0.414145 0.662799 0.840000 -8 0.407066 0.630168 0.840000 -Saving models at data/mldoc/de-1/models/sp30k/lstm_nl4-100-e8dp6x2.m -Loss and accuracy using (cls_last): [0.46259913, tensor(0.9147)] -``` - -``` -python -m ulmfit cls --dataset-path data/mldoc/de-1 --base-lm-path data/mldoc/de-1/models/sp30k/lstm_nl4.m --lang=de --name 'nl4-100-e8dp2' --cuda-id=1 - train 0 --bs 40 --limit=100 --num-cls-epochs=8 --drop-mult-cls=0.2 -Max vocab: 30000 -Cache dir: data/mldoc/de-1/models/sp30k -Model dir: data/mldoc/de-1/models/sp30k/lstm_nl4-100-e8dp2.m -Loading validation data/mldoc/de-1/de.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Limiting data set to: 100 -Tokenized data loaded, cls.trn 100, cls.val 100 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -data/mldoc/de-1/models/sp30k -Saving info data/mldoc/de-1/models/sp30k/lstm_nl4-100-e8dp2.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 1.172059 1.311985 0.280000 -epoch train_loss valid_loss accuracy -1 0.721259 1.180611 0.720000 -epoch train_loss valid_loss accuracy -1 0.495393 1.051538 0.770000 -epoch train_loss valid_loss accuracy -1 0.445929 0.984670 0.830000 -2 0.430556 0.897431 0.870000 -3 0.442683 0.800808 0.900000 -4 0.427033 0.711604 0.880000 -5 0.411931 0.624835 0.890000 -6 0.397705 0.560819 0.900000 -7 0.387848 0.506201 0.900000 -8 0.380063 0.459507 0.900000 -Saving models at data/mldoc/de-1/models/sp30k/lstm_nl4-100-e8dp2.m -Loss and accuracy using (cls_last): [0.41103342, tensor(0.9105)] -``` - -#### 2x e8 -``` -python -m ulmfit cls --dataset-path data/mldoc/de-1 --base-lm-path data/mldoc/de-1/models/sp30k/lstm_nl4.m --lang=de --name 'nl4-100-2nd-2x' --cuda-id=1 - train 0 --bs 40 --limit=100 --num-cls-epochs=8 --drop-mult-cls=0.2 -Max vocab: 30000 -Cache dir: data/mldoc/de-1/models/sp30k -Model dir: data/mldoc/de-1/models/sp30k/lstm_nl4-100-2nd-2x.m -Loading validation data/mldoc/de-1/de.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Limiting data set to: 100 -Tokenized data loaded, cls.trn 100, cls.val 100 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -data/mldoc/de-1/models/sp30k -Saving info data/mldoc/de-1/models/sp30k/lstm_nl4-100-2nd-2x.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 1.122616 1.290291 0.300000 -epoch train_loss valid_loss accuracy -1 0.746805 1.166377 0.730000 -epoch train_loss valid_loss accuracy -1 0.535163 1.058924 0.900000 -epoch train_loss valid_loss accuracy -1 0.437773 1.022764 0.850000 -2 0.449220 0.949963 0.820000 -3 0.443732 0.854293 0.860000 -4 0.432721 0.746918 0.900000 -5 0.423113 0.718745 0.840000 -6 0.403492 0.671295 0.820000 -7 0.399091 0.539798 0.900000 -8 0.394950 0.508265 0.900000 -Saving models at data/mldoc/de-1/models/sp30k/lstm_nl4-100-2nd-2x.m -Loading validation data/mldoc/de-1/de.dev.csv -Loss and accuracy using (cls_last): [0.44688165, tensor(0.9062)] -Loss and accuracy using (cls_best): [0.44688165, tensor(0.9062)] -``` -``` -python -m ulmfit cls --dataset-path data/mldoc/de-1 --base-lm-path data/mldoc/de-1/models/sp30k/lstm_nl4.m --lang=de --name 'nl4-100-2nd-2x' --cuda-id=1 - train 0 --bs 40 --limit=100 --num-cls-epochs=8 --drop-mult-cls=0.2 -Max vocab: 30000 -Cache dir: data/mldoc/de-1/models/sp30k -Model dir: data/mldoc/de-1/models/sp30k/lstm_nl4-100-2nd-2x.m -Loading validation data/mldoc/de-1/de.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Limiting data set to: 100 -Tokenized data loaded, cls.trn 100, cls.val 100 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] -Loading last classifier -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.422151 0.797943 0.720000 -epoch train_loss valid_loss accuracy -1 0.357166 0.736997 0.780000 -epoch train_loss valid_loss accuracy -1 0.238496 1.497305 0.660000 -epoch train_loss valid_loss accuracy -1 0.312356 1.522862 0.660000 -2 0.267801 1.518249 0.660000 -3 0.244077 1.110030 0.680000 -4 0.264972 0.798898 0.770000 -5 0.236816 0.398245 0.860000 -6 0.251284 0.415783 0.860000 -7 0.244988 0.417737 0.860000 -8 0.240362 0.415114 0.860000 -Saving models at data/mldoc/de-1/models/sp30k/lstm_nl4-100-2nd-2x.m -Loss and accuracy using (cls_last): [0.27954015, tensor(0.9125)] -Loss and accuracy using (cls_best): [0.27954015, tensor(0.9125)] -``` -### Adding noise -#### 40% -``` -python -m ulmfit cls --dataset-path data/mldoc/de-1 --base-lm-path data/mldoc/de-1/models/sp30k/lstm_nl4.m --lang=de --name 'nl4-noise0.4' --cuda-id=1 - train 0 --bs 40 --noise=0.4 --num-cls-epochs=8 --drop-mult-cls=0.2 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-noise0.4.m -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/de.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Added noise to 400 examples, only 0.6 have correct labels -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-noise0.4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 1.053928 0.938391 0.535000 -epoch train_loss valid_loss accuracy -1 0.941778 0.599400 0.836000 -epoch train_loss valid_loss accuracy -1 0.858363 0.675211 0.760000 -epoch train_loss valid_loss accuracy -1 0.768678 0.645293 0.788000 -2 0.758538 0.636551 0.780000 -3 0.753799 0.673323 0.708000 -4 0.731245 0.638630 0.736000 -5 0.691206 0.659491 0.717000 -6 0.691426 0.682510 0.696000 -7 0.672320 0.668610 0.702000 -8 0.653569 0.669633 0.694000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-noise0.4.m -Loss and accuracy using (cls_last): [0.62477165, tensor(0.7717)] -``` -#### 15% -``` -python -m ulmfit cls --dataset-path data/mldoc/de-1 --base-lm-path data/mldoc/de-1/models/sp30k/lstm_nl4.m --lang=de --name 'nl4-noise0.15' --cuda-id=1 - train 0 --bs 40 --noise=0.15 --num-cls-epochs=2 --drop-mult-cls=0.2 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-noise0.15.m -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/de.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Added noise to 150 examples, only 0.85 have correct labels -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-noise0.15.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.836104 0.584330 0.897000 -epoch train_loss valid_loss accuracy -1 0.692108 0.303470 0.930000 -epoch train_loss valid_loss accuracy -1 0.653277 0.330520 0.924000 -epoch train_loss valid_loss accuracy -1 0.541086 0.331944 0.922000 -2 0.523274 0.335986 0.922000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-noise0.15.m -Loss and accuracy using (cls_last): [0.28749043, tensor(0.9355)] -``` \ No newline at end of file diff --git a/results/logs/en.md b/results/logs/en.md deleted file mode 100644 index 8f68020..0000000 --- a/results/logs/en.md +++ /dev/null @@ -1,6 +0,0 @@ -# EN -## SP30k LSTM nl 4 -### LM - -### MLDoc - diff --git a/results/logs/es.md b/results/logs/es.md deleted file mode 100644 index 1e686cd..0000000 --- a/results/logs/es.md +++ /dev/null @@ -1,269 +0,0 @@ -# ES - -## SP30k LSTM nl 4 -### LM -```` -python -m ulmfit lm --dataset-path data/wiki/es-100 --cuda-id=0 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 30000 --lang es --qrnn=False - train 10 --bs=50 --drop_mult=0 -Running tokenization -Wiki text was split to 96224 articles -Wiki text was split to 105 articles -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': None, 'pretrained_model': None, 'drop_mult': 0} dps: [0.25 0.1 0.2 0.02 0.15] -Training lm from random weights -epoch train_loss valid_loss accuracy -1 3.269541 3.451855 0.387471 -2 3.161740 3.423016 0.386158 -3 3.187431 3.419638 0.388626 -4 3.115763 3.357066 0.393877 -5 2.996527 3.291787 0.402488 -6 3.021759 3.202183 0.410873 -7 2.998267 3.104373 0.422624 -8 2.827225 3.006537 0.436010 -9 2.784576 2.937735 0.446654 -10 2.789913 2.918509 0.450055 -data/wiki/es-100/models/sp30k -Saving info data/wiki/es-100/models/sp30k/lstm_nl4.m/info.json -```` - -### MLDoc - -``` -python -m ulmfit cls --dataset-path data/mldoc/es-1 --base-lm-path data/wiki/es-100/models/sp30k/lstm_nl4.m --lang=es --name 'nl4' --cuda-id=1 - train 20 --bs 40 --num-cls-epochs=2 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv -Running tokenization... -Saving tokenized: cls.trn 13013, cls.val 1445 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/es-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/es-100/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/es-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/es-100/models/sp30k/lstm_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 2.805415 2.188974 0.537779 -epoch train_loss valid_loss accuracy -1 2.429727 1.989691 0.569048 -2 2.218828 1.794969 0.603721 -3 2.015097 1.644815 0.629609 -4 1.877210 1.537773 0.646898 -5 1.775648 1.450283 0.660861 -6 1.749334 1.377085 0.672146 -7 1.601073 1.311101 0.684400 -8 1.564420 1.251074 0.694900 -9 1.532728 1.197607 0.704779 -10 1.391921 1.145408 0.716044 -11 1.379958 1.093550 0.726937 -12 1.324111 1.048308 0.735890 -13 1.344113 1.007926 0.745691 -14 1.243085 0.969521 0.754591 -15 1.230809 0.937330 0.762675 -16 1.162501 0.913408 0.768044 -17 1.170092 0.894892 0.773239 -18 1.110860 0.884449 0.775603 -19 1.115907 0.880448 0.776671 -20 1.083033 0.878421 0.776931 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.621574 0.391042 0.856000 -epoch train_loss valid_loss accuracy -1 0.411668 0.215625 0.935000 -epoch train_loss valid_loss accuracy -1 0.340519 0.222422 0.935000 -epoch train_loss valid_loss accuracy -1 0.281729 0.192193 0.949000 -2 0.262074 0.202975 0.945000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.1749019, tensor(0.9515)] -``` - - - -## ES optimization - - - -### Smaler vocab 15k -#### LM -``` -python -m ulmfit lm --dataset-path data/wiki-m/es-100 --cuda-id=0 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 \ ✘ 1 ---lang es --qrnn=False - train 10 --bs=50 --drop_mult=0 -Max vocab: 15000 -Cache dir: data/wiki-m/es-100/models/sp15k -Model dir: data/wiki-m/es-100/models/sp15k/lstm_nl4.m -Tokenized data loaded -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Training lm from random weights -epoch train_loss valid_loss accuracy -1 2.961702 3.187043 0.403149 -Better model found at epoch 1 with val_loss value: 3.1870434284210205. -2 2.928991 3.170802 0.402026 -Better model found at epoch 2 with val_loss value: 3.170802354812622. -3 2.931906 3.128328 0.407816 -Better model found at epoch 3 with val_loss value: 3.128328323364258. -4 2.869332 3.072160 0.414345 -Better model found at epoch 4 with val_loss value: 3.072160243988037. -5 2.803377 2.997071 0.424847 -Better model found at epoch 5 with val_loss value: 2.997070550918579. -6 2.758087 2.927369 0.432256 -Better model found at epoch 6 with val_loss value: 2.927368640899658. -7 2.657733 2.825029 0.446440 -Better model found at epoch 7 with val_loss value: 2.8250293731689453. -8 2.563273 2.728652 0.459271 -Better model found at epoch 8 with val_loss value: 2.7286524772644043. -9 2.475741 2.654844 0.470864 -Better model found at epoch 9 with val_loss value: 2.654844045639038. -10 2.428898 2.634355 0.474821 -Better model found at epoch 10 with val_loss value: 2.634355306625366. -Total time: 17:53:59 -data/wiki-m/es-100/models/sp15k -Saving info data/wiki-m/es-100/models/sp15k/lstm_nl4.m/info.json -``` - -#### MLDoc -``` - python -m ulmfit cls --dataset-path data/mldoc/es-1 --base-lm-path data/wiki-m/es-100/models/sp15k/lstm_nl4.m --lang=es --name 'nl4' --cuda-id=0 - train 20 --bs 20 --num-cls-epochs=8 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/lstm_nl4.m -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv -Running tokenization... -Saving tokenized: cls.trn 13013, cls.val 1445 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki-m/es-100/models/sp15k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki-m/es-100/models/sp15k/lstm_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 2.355042 1.850417 0.589128 -Better model found at epoch 1 with val_loss value: 1.850416898727417. -Total time: 03:25 -epoch train_loss valid_loss accuracy -1 2.087364 1.677666 0.619909 -Better model found at epoch 1 with val_loss value: 1.6776657104492188. -2 1.880730 1.522996 0.648134 -Better model found at epoch 2 with val_loss value: 1.5229955911636353. -3 1.767530 1.403644 0.668117 -Better model found at epoch 3 with val_loss value: 1.4036436080932617. -4 1.659950 1.309353 0.684900 -Better model found at epoch 4 with val_loss value: 1.3093526363372803. -5 1.546585 1.232220 0.699358 -Better model found at epoch 5 with val_loss value: 1.2322196960449219. -6 1.592862 1.161846 0.713034 -Better model found at epoch 6 with val_loss value: 1.1618456840515137. -7 1.444965 1.098108 0.726811 -Better model found at epoch 7 with val_loss value: 1.0981075763702393. -8 1.340874 1.029193 0.741337 -Better model found at epoch 8 with val_loss value: 1.0291931629180908. -9 1.351407 0.974317 0.753408 -Better model found at epoch 9 with val_loss value: 0.9743167757987976. -10 1.231713 0.915328 0.767088 -Better model found at epoch 10 with val_loss value: 0.9153280854225159. -11 1.151926 0.852391 0.782414 -Better model found at epoch 11 with val_loss value: 0.852391242980957. -12 1.163565 0.794699 0.797228 -Better model found at epoch 12 with val_loss value: 0.7946987152099609. -13 1.054929 0.743652 0.810518 -Better model found at epoch 13 with val_loss value: 0.74365234375. -14 0.974651 0.695024 0.823344 -Better model found at epoch 14 with val_loss value: 0.6950243711471558. -15 0.869718 0.651691 0.834510 -Better model found at epoch 15 with val_loss value: 0.6516908407211304. -16 0.889763 0.615112 0.844947 -Better model found at epoch 16 with val_loss value: 0.6151121258735657. -17 0.843503 0.590130 0.851694 -Better model found at epoch 17 with val_loss value: 0.5901297926902771. -18 0.752870 0.575217 0.855496 -Better model found at epoch 18 with val_loss value: 0.5752172470092773. -19 0.807087 0.567187 0.857605 -Better model found at epoch 19 with val_loss value: 0.5671872496604919. -20 0.784531 0.566082 0.857827 -Better model found at epoch 20 with val_loss value: 0.5660821199417114. -Total time: 1:26:48 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.616963 0.275447 0.923000 -Better model found at epoch 1 with val_loss value: 0.27544698119163513. -Total time: 00:24 -epoch train_loss valid_loss accuracy -1 0.425890 0.201535 0.932000 -Better model found at epoch 1 with val_loss value: 0.2015346735715866. -Total time: 00:27 -epoch train_loss valid_loss accuracy -1 0.265563 0.186434 0.951000 -Better model found at epoch 1 with val_loss value: 0.18643426895141602. -Total time: 00:32 -epoch train_loss valid_loss accuracy -1 0.162097 0.180026 0.955000 -Better model found at epoch 1 with val_loss value: 0.1800260841846466. -2 0.161748 0.187014 0.957000 -3 0.142789 0.166486 0.961000 -Better model found at epoch 3 with val_loss value: 0.1664857715368271. -4 0.105920 0.173207 0.963000 -5 0.078164 0.184849 0.962000 -6 0.070540 0.189451 0.962000 -7 0.051490 0.208019 0.959000 -8 0.047614 0.193699 0.962000 -Total time: 05:20 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.1623594, tensor(0.9538)] -0.16235940158367157 -0.9537500143051147 -``` - -### Larger dropout - no luck -``` -python -m ulmfit cls --dataset-path data/mldoc/es-1 --base-lm-path data/wiki-m/es-100/models/sp30k/lstm_nl4.m --lang=es --name 'nl4-drop' --cuda-id=0 - train 0 --bs 20 --num-cls-epochs=8 --drop-mul-lm=0.5 --drop-mul-cls=0.8 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-drop.m -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv -Running tokenization... -Saving tokenized: cls.trn 13013, cls.val 1445 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Starting classifier training -epoch train_loss valid_loss accuracy -1 1.310594 0.903076 0.724000 -Better model found at epoch 1 with val_loss value: 0.9030755758285522. -Total time: 00:22 -epoch train_loss valid_loss accuracy -1 1.209348 0.714140 0.756000 -Better model found at epoch 1 with val_loss value: 0.714139997959137. -Total time: 00:23 -epoch train_loss valid_loss accuracy -1 1.122636 0.627526 0.797000 -Better model found at epoch 1 with val_loss value: 0.6275263428688049. -Total time: 00:29 -epoch train_loss valid_loss accuracy -1 1.102433 0.593338 0.801000 -Better model found at epoch 1 with val_loss value: 0.5933384895324707. -2 1.096480 0.543266 0.818000 -Better model found at epoch 2 with val_loss value: 0.5432664155960083. -3 1.082919 0.501089 0.837000 -Better model found at epoch 3 with val_loss value: 0.5010889172554016. -4 1.069694 0.518807 0.812000 -5 1.040208 0.508399 0.825000 -6 1.032841 0.512187 0.838000 -7 1.031225 0.504557 0.825000 -8 1.016486 0.502335 0.837000 -Total time: 04:56 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-drop.m -Loss and accuracy using (cls_best): [0.52473265, tensor(0.8160)] -0.5247326493263245 -0.8159999847412109 -``` diff --git a/results/logs/fastai.errors.txt b/results/logs/fastai.errors.txt deleted file mode 100644 index 7464fb6..0000000 --- a/results/logs/fastai.errors.txt +++ /dev/null @@ -1,76 +0,0 @@ -python -m ulmfit eval --glob="mldoc/*-1/models/sp15k/qrnn_nl4.m" --name nl4-8e-single --num-cls-epochs=8 --bs=18 --single=True -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-8e-single.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: - - !! WARNING !! - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Your compiler (c++) may be ABI-incompatible with PyTorch! -Please use a compiler that is ABI-compatible with GCC 4.9 and above. -See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. - -See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 -for instructions on how to install GCC 4.9 or higher. -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - !! WARNING !! - - warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-8e-single.m/info.json -Starting classifier training -Single training schedule -epoch train_loss valid_loss accuracy -1 0.676591 0.205210 0.947000 -2 0.402020 0.461279 0.912000 -3 0.287975 0.496294 0.921000 -4 0.258515 0.243489 0.954000 -5 0.219352 0.274136 0.949000 -6 0.149339 0.352294 0.956000 -7 0.092821 0.378696 0.962000 -8 0.055485 0.367379 0.963000 -9 0.042695 0.367151 0.964000 -10 0.034858 0.386749 0.961000 -11 0.021245 0.392899 0.963000 -Total time: 02:38 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-8e-single.m -Traceback (most recent call last): - File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/runpy.py", line 193, in _run_module_as_main - "__main__", mod_spec) - File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/runpy.py", line 85, in _run_code - exec(code, run_globals) - File "/home/pczapla/workspace/ulmfit-multilingual/ulmfit/__main__.py", line 73, in - fire.Fire(ULMFiT()) - File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/fire/core.py", line 127, in Fire - component_trace = _Fire(component, args, context, name) - File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/fire/core.py", line 366, in _Fire - component, remaining_args) - File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/fire/core.py", line 542, in _CallCallable - result = fn(*varargs, **kwargs) - File "/home/pczapla/workspace/ulmfit-multilingual/ulmfit/__main__.py", line 55, in eval - results[key] = params.train_cls(num_lm_epochs=num_lm_epochs, **trn_params)[1] - File "/home/pczapla/workspace/ulmfit-multilingual/ulmfit/train_clas.py", line 76, in train_cls - return self.validate_cls('cls_best', bs=bs, data_tst=data_tst, learn=learn) - File "/home/pczapla/workspace/ulmfit-multilingual/ulmfit/train_clas.py", line 84, in validate_cls - learn.load(save_name) - File "/home/pczapla/workspace/_oss/fastai/fastai/fastai/basic_train.py", line 243, in load - if purge: self.purge(clear_opt=ifnone(with_opt, False)) - File "/home/pczapla/workspace/_oss/fastai/fastai/fastai/basic_train.py", line 293, in purge - self.opt = OptimWrapper.load_with_state_and_layer_group(state['opt'], self.layer_groups) - File "/home/pczapla/workspace/_oss/fastai/fastai/fastai/callback.py", line 130, in load_with_state_and_layer_group - res.load_state_dict(state['opt_state']) - File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/optim/optimizer.py", line 108, in load_state_dict - raise ValueError("loaded state dict contains a parameter group " -ValueError: loaded state dict contains a parameter group that doesn't match the size of optimizer's group \ No newline at end of file diff --git a/results/logs/fr.md b/results/logs/fr.md deleted file mode 100644 index 9b6c456..0000000 --- a/results/logs/fr.md +++ /dev/null @@ -1,180 +0,0 @@ -# FR - -## SP15k QRNN nl 4 -``` -python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0 -Max vocab: 15000 -Cache dir: data/wiki/fr-100/models/sp15k -Model dir: data/wiki/fr-100/models/sp15k/qrnn_nl4.m -Wiki text was split to 174227 articles -Wiki text was split to 491 articles -Running tokenization lm... -Data lm, trn: 174227, val: 491 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', 's', '▁.', "'", '▁la', '▁le', -'▁et', '▁l', '▁à'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Training lm from random weights -epoch train_loss valid_loss accuracy -1 2.881558 2.790402 0.465847 -2 2.824942 2.732005 0.471660 -3 2.758845 2.672040 0.478273 -4 2.715069 2.602380 0.489159 -5 2.677029 2.553575 0.494752 -6 2.602514 2.476142 0.507337 -7 2.564386 2.388670 0.518902 -8 2.470835 2.304033 0.532000 -9 2.366890 2.243269 0.542781 -10 2.390439 2.223538 0.546622 -Total time: 9:09:26 -data/wiki/fr-100/models/sp15k -Saving info data/wiki/fr-100/models/sp15k/qrnn_nl4.m/info.json - -``` - -## SP30k LSTM nl 4 -### LM -``` -python -m ulmfit lm --dataset-path data/wiki/fr-100 --cuda-id=1 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 30000 \ ✘ 130 ---lang fr --qrnn=False - train 10 --bs=50 --drop_mult=0 -Max vocab: 30000 -Cache dir: data/wiki/fr-100/models/sp30k -Model dir: data/wiki/fr-100/models/sp30k/lstm_nl4.m -Running tokenization -Wiki text was split to 113288 articles -Wiki text was split to 88 articles -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': None, 'pretrained_model': None, 'drop_mult': 0} dps: [0.25 0.1 0.2 0.02 0.15] -Training lm from random weights -epoch train_loss valid_loss accuracy -1 3.120035 3.449028 0.383274 -2 3.070353 3.431372 0.382520 -3 3.092097 3.406521 0.384604 -4 3.035016 3.356502 0.391155 -5 2.936505 3.297572 0.396365 -6 2.926953 3.192980 0.407546 -7 2.841542 3.115280 0.417741 -8 2.805254 3.008793 0.429512 -9 2.681713 2.944207 0.439959 -10 2.644920 2.923765 0.442415 -data/wiki/fr-100/models/sp30k -Saving info data/wiki/fr-100/models/sp30k/lstm_nl4.m/info.json -``` - -### MLDocs -#### First run -MultiCCA 92.05, ulmfit 93.90 -``` -python -m ulmfit cls --dataset-path data/mldoc/fr-1 --base-lm-path data/wiki/fr-100/models/sp30k/lstm_nl4.m --lang=fr --name 'nl4' --cuda-id=1 - train 20 --bs 40 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/fr.dev.csv -Running tokenization... -Saving tokenized: cls.trn 13500, cls.val 1500 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp30k/lstm_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.072937 2.621444 0.468314 -epoch train_loss valid_loss accuracy -1 2.737065 2.485359 0.486546 -2 2.625827 2.353188 0.507669 -3 2.408049 2.224600 0.527609 -4 2.332804 2.113603 0.544255 -5 2.242967 2.016229 0.560002 -6 2.162170 1.925214 0.574050 -7 2.094163 1.843778 0.587944 -8 2.011285 1.773228 0.599802 -9 1.931492 1.708201 0.611245 -10 1.883735 1.643842 0.623145 -11 1.793858 1.583394 0.635366 -12 1.759305 1.526640 0.646132 -13 1.741412 1.474198 0.657485 -14 1.675670 1.430597 0.666407 -15 1.624235 1.390453 0.674829 -16 1.588415 1.359892 0.681364 -17 1.594124 1.336594 0.686985 -18 1.567758 1.322139 0.689745 -19 1.536472 1.315883 0.690974 -20 1.530144 1.314872 0.691084 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.528480 0.428756 0.853000 -epoch train_loss valid_loss accuracy -1 0.365323 0.224117 0.928000 -epoch train_loss valid_loss accuracy -1 0.300881 0.199623 0.936000 -epoch train_loss valid_loss accuracy -1 0.217855 0.198016 0.937000 -2 0.206357 0.212208 0.938000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.18914989, tensor(0.9390)] -``` - -#### Second run -MultiCCA 92.05, ulmfit 93.67 -``` -python -m ulmfit cls --dataset-path data/mldoc/fr-1 --base-lm-path data/wiki/fr-100/models/sp30k/lstm_nl4.m --lang=fr --name 'nl4-2nd' --cuda-id=1 - train 20 --bs 40 --num-cls-epochs=8 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4-2nd.m -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/fr.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Tokenized data loaded, cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp30k/lstm_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.080331 2.625329 0.467306 -epoch train_loss valid_loss accuracy -1 2.750554 2.485271 0.486776 -2 2.578659 2.353940 0.507637 -3 2.422981 2.224983 0.527749 -4 2.342781 2.113364 0.545006 -5 2.254575 2.007775 0.560709 -6 2.124016 1.920536 0.575680 -7 2.068470 1.847463 0.586699 -8 2.013289 1.775580 0.599840 -9 1.929649 1.705369 0.612201 -10 1.916013 1.646228 0.623175 -11 1.825515 1.586714 0.634298 -12 1.795780 1.529771 0.645840 -13 1.725532 1.476651 0.656197 -14 1.673942 1.429790 0.666030 -15 1.639384 1.392116 0.674128 -16 1.605681 1.359316 0.681356 -17 1.560283 1.337794 0.686116 -18 1.543926 1.323153 0.689276 -19 1.531950 1.318164 0.690415 -20 1.494068 1.316459 0.690586 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4-2nd.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.537720 0.395818 0.886000 -epoch train_loss valid_loss accuracy -1 0.332603 0.232112 0.930000 -epoch train_loss valid_loss accuracy -1 0.267323 0.230307 0.927000 -epoch train_loss valid_loss accuracy -1 0.216402 0.226042 0.930000 -2 0.231040 0.232696 0.936000 -3 0.182048 0.217882 0.934000 -4 0.170389 0.212531 0.937000 -5 0.148332 0.214293 0.937000 -6 0.124968 0.210322 0.936000 -7 0.117591 0.234207 0.936000 -8 0.109146 0.218597 0.938000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4-2nd.m -Loss and accuracy using (cls_best): [0.21502711, tensor(0.9367)] -``` \ No newline at end of file diff --git a/results/logs/it.md b/results/logs/it.md deleted file mode 100644 index a0aa981..0000000 --- a/results/logs/it.md +++ /dev/null @@ -1,106 +0,0 @@ -# FR -## SP15k QRNN NL4 -### LM - -``` -export CUDA_VISIBLE_DEVICES=0 -LANG=it -python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0 -epoch train_loss valid_loss accuracy -1 3.171145 3.516659 0.359233 -2 3.045057 3.472802 0.359628 -3 3.023009 3.401181 0.367101 -4 2.985105 3.351916 0.372709 -5 2.858441 3.280903 0.380848 -6 2.862504 3.210976 0.390263 -7 2.758775 3.122354 0.402106 -8 2.683234 3.035321 0.413798 -9 2.593757 2.964551 0.424886 -10 2.535500 2.947672 0.427958 -Total time: 11:30:03 -data/wiki/it-100/models/sp15k -Saving info data/wiki/it-100/models/sp15k/qrnn_nl4.m/info.json -``` -## xx - -## SP30k LSTM nl 4 -### LM -``` -python -m ulmfit lm --dataset-path data/wiki/it-100 --lang=it --bidir=False --qrnn=False --max-vocab 30000 --nl 4 --tokenizer=sp --name 'nl4bs100' - train 10 --bs 100 --dropout-mult=0 -Wiki text was split to 164583 articles -Wiki text was split to 98 articles -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', "▁&'", "'", '▁e', '▁il', '▁la', 'e', '▁in'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': None, 'pretrained_model': None, 'drop_mult': 0.0} dps: [0.25 0.1 0.2 0.02 0.15] -Training lm from random weights -epoch train_loss valid_loss accuracy -1 3.306743 3.717148 0.353641 -2 3.126413 3.606443 0.360839 -3 3.062586 3.545493 0.365721 -4 3.055600 3.474823 0.373451 -5 2.927211 3.406635 0.380311 -6 2.924096 3.321370 0.389487 -7 2.779998 3.233350 0.399968 -8 2.722100 3.147745 0.410365 -9 2.615910 3.087420 0.419097 -10 2.565747 3.075364 0.420906 -data/wiki/it-100/models/sp30k -Saving info data/wiki/it-100/models/sp30k/lstm_nl4bs100.m/info.json -``` - - -### MLDoc -MultiCCA: 85.55%, ULMFiT 88.42% -``` -python -m ulmfit cls --dataset-path data/mldoc/it-1 --base-lm-path data/wiki/it-100/models/sp30k/lstm_nl4bs100.m --lang=it --name 'nl4bs100' --cuda-id=1 - train 20 --bs 40 --num-cls-epochs=2 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4bs100.m -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/it.dev.csv -Running tokenization... -Saving tokenized: cls.trn 13500, cls.val 1500 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', "▁&'", "'", '▁e', '▁il', '▁la', 'e', '▁in'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/it-100/models/sp30k/lstm_nl4bs100.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/it-100/models/sp30k/lstm_nl4bs100.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/it-100/models/sp30k/lstm_nl4bs100.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/it-100/models/sp30k/lstm_nl4bs100.m/../itos')] -epoch train_loss valid_loss accuracy -1 2.826957 2.518636 0.492175 -epoch train_loss valid_loss accuracy -1 2.606302 2.397623 0.509596 -2 2.470586 2.260363 0.531301 -3 2.334087 2.113640 0.554089 -4 2.176830 1.988222 0.572687 -5 2.123101 1.869944 0.591537 -6 2.011187 1.770606 0.606682 -7 1.934953 1.676852 0.622504 -8 1.889363 1.592609 0.637525 -9 1.774590 1.517665 0.652233 -10 1.725905 1.435543 0.666759 -11 1.670903 1.365167 0.681168 -12 1.610080 1.302561 0.694462 -13 1.522876 1.242124 0.708201 -14 1.478528 1.193259 0.718366 -15 1.423993 1.150854 0.728324 -16 1.389901 1.115550 0.735836 -17 1.365959 1.094267 0.740730 -18 1.347579 1.079465 0.744019 -19 1.321906 1.074090 0.745281 -20 1.332676 1.073143 0.745453 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4bs100.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.632703 0.463210 0.831000 -epoch train_loss valid_loss accuracy -1 0.527650 0.390041 0.858000 -epoch train_loss valid_loss accuracy -1 0.436223 0.326409 0.871000 -epoch train_loss valid_loss accuracy -1 0.361738 0.321380 0.875000 -2 0.340658 0.315946 0.877000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4bs100.m -Loss and accuracy using (cls_best): [0.32998973, tensor(0.8842)] -``` diff --git a/results/logs/ja.md b/results/logs/ja.md deleted file mode 100644 index 3a81ca6..0000000 --- a/results/logs/ja.md +++ /dev/null @@ -1,456 +0,0 @@ -# JA -## SP30k LSTM nl 4 -### LM -``` -python -m ulmfit lm --dataset-path data/wiki/ja-100 --cuda-id=0 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 30000 \ ---lang ja --qrnn=False - train 10 --bs=50 --drop_mult=0 -Max vocab: 30000 -Cache dir: data/wiki/ja-100/models/sp30k -Model dir: data/wiki/ja-100/models/sp30k/lstm_nl4.m -Running tokenization -Wiki text was split to 98375 articles -Wiki text was split to 138 articles -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': None, 'pretrained_model': None, 'drop_mult': 0} dps: [0.25 0.1 0.2 0.02 0.15] -Training lm from random weights -epoch train_loss valid_loss accuracy -1 3.211025 3.328014 0.396197 -2 3.119410 3.286294 0.395946 -3 3.042064 3.247161 0.403915 -4 3.023840 3.161323 0.413816 -5 2.944752 3.102044 0.423163 -6 2.907167 3.015610 0.434095 -7 2.796073 2.927566 0.447088 -8 2.715568 2.828766 0.461556 -9 2.717255 2.747889 0.473289 -10 2.619846 2.731164 0.477403 -data/wiki/ja-100/models/sp30k -Saving info data/wiki/ja-100/models/sp30k/lstm_nl4.m/info.json -``` - -### MLDoc - -#### CLS 1 -MultiCCA 85.35%, ULMFiT 89.20% -``` -python -m ulmfit cls --dataset-path data/mldoc/ja-1 --base-lm-path data/wiki/ja-100/models/sp30k/lstm_nl4.m --lang=ja --name 'nl4' --cuda-id=1 - train 20 --bs 40 --num-cls-epochs=8 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv -Running tokenization... -Saving tokenized: cls.trn 13500, cls.val 1500 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp30k/lstm_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 2.828645 2.386208 0.518716 -epoch train_loss valid_loss accuracy -1 2.462274 2.191761 0.549783 -2 2.229925 1.982564 0.586238 -3 2.043816 1.805435 0.616238 -4 1.885779 1.674736 0.637964 -5 1.773445 1.575366 0.653925 -6 1.713029 1.490263 0.667570 -7 1.660558 1.419641 0.680072 -8 1.579792 1.357093 0.690826 -9 1.459628 1.298609 0.701452 -10 1.433604 1.251296 0.710232 -11 1.439143 1.202794 0.719104 -12 1.399083 1.158469 0.728430 -13 1.310390 1.120877 0.736382 -14 1.322389 1.085479 0.744013 -15 1.272924 1.056051 0.750401 -16 1.235312 1.034233 0.755225 -17 1.227864 1.016682 0.759288 -18 1.209589 1.007038 0.761234 -19 1.173158 1.001694 0.762281 -20 1.189994 1.000854 0.762526 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.745803 0.554439 0.819000 -epoch train_loss valid_loss accuracy -1 0.620647 0.392026 0.856000 -epoch train_loss valid_loss accuracy -1 0.489173 0.369560 0.869000 -epoch train_loss valid_loss accuracy -1 0.406491 0.365988 0.872000 -2 0.392645 0.351823 0.876000 -3 0.386403 0.331737 0.880000 -4 0.361338 0.333245 0.882000 -5 0.319456 0.347253 0.879000 -6 0.295419 0.350348 0.885000 -7 0.286144 0.348592 0.879000 -8 0.278896 0.358145 0.877000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.29789856, tensor(0.8920)] - - -$ mv /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4x8.m - -$ python -m ulmfit eval --glob="mldoc/ja-1/models/sp30k/lstm_nl4x8.m" --name nl4 --cuda-id=0 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Tokenized data loaded, cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4x8.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4x8.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.757615 0.562652 0.825000 -epoch train_loss valid_loss accuracy -1 0.609298 0.382412 0.870000 -epoch train_loss valid_loss accuracy -1 0.544682 0.379602 0.871000 -epoch train_loss valid_loss accuracy -1 0.435453 0.360421 0.885000 -2 0.426099 0.350480 0.885000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.32738593, tensor(0.8905)] -OrderedDict([('data/mldoc/ja-1/models/sp30k/lstm_nl4.m', 0.890500009059906)]) - -python -m ulmfit eval --glob="mldoc/ja-1/models/sp30k/lstm_nl4x8.m" --name nl4x2 --cuda-id=0 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4x2.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Tokenized data loaded, cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4x8.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4x8.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4x2.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.778722 0.690746 0.805000 -epoch train_loss valid_loss accuracy -1 0.574789 0.386483 0.862000 -epoch train_loss valid_loss accuracy -1 0.518843 0.361983 0.869000 -epoch train_loss valid_loss accuracy -1 0.435260 0.350808 0.869000 -2 0.386701 0.352221 0.875000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4x2.m -Loss and accuracy using (cls_best): [0.31783763, tensor(0.8892)] -OrderedDict([('data/mldoc/ja-1/models/sp30k/lstm_nl4x2.m', 0.8892499804496765)]) - -``` - -### JA on 100 elements -``` -python -m ulmfit cls --dataset-path data/mldoc/ja-1 --base-lm-path data/wiki/ja-100/models/sp30k/lstm_nl4.m --lang=ja --name 'nl4-100' --cuda-id=1 - train 20 --bs 40 --num-cls-epochs=8 --limit=100 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-100.m -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Limiting data set to: 100 -Running tokenization... -Saving tokenized: cls.trn 100, cls.val 100 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp30k/lstm_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 2.837937 2.387255 0.518590 -epoch train_loss valid_loss accuracy -1 2.466900 2.193583 0.549492 -2 2.232762 1.983981 0.586658 -3 2.026505 1.810167 0.615649 -4 1.918111 1.679784 0.636613 -5 1.748909 1.577095 0.653108 -6 1.708709 1.491436 0.667657 -7 1.640415 1.420449 0.679619 -8 1.577434 1.359511 0.690194 -9 1.551961 1.302819 0.700306 -10 1.475623 1.252393 0.710039 -11 1.435565 1.208159 0.718740 -12 1.354910 1.161781 0.727927 -13 1.351157 1.123244 0.736009 -14 1.299070 1.086383 0.743896 -15 1.258739 1.055745 0.750383 -16 1.210775 1.035209 0.754965 -17 1.228421 1.018373 0.758963 -18 1.179444 1.007714 0.761158 -19 1.197443 1.003041 0.762068 -20 1.163223 1.001939 0.762211 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-100.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 1.269222 1.360420 0.340000 -epoch train_loss valid_loss accuracy -1 0.969350 1.314497 0.400000 -epoch train_loss valid_loss accuracy -1 0.832396 1.263416 0.550000 -epoch train_loss valid_loss accuracy -1 0.780991 1.225439 0.600000 -2 0.765755 1.183010 0.600000 -3 0.749420 1.139053 0.600000 -4 0.731800 1.093319 0.610000 -5 0.711152 1.054695 0.610000 -6 0.694611 1.029465 0.580000 -7 0.680276 1.004366 0.580000 -8 0.668421 0.984848 0.590000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-100.m -Loss and accuracy using (cls_best): [0.81621724, tensor(0.7437)] -``` - - -### Japanese fixed sentence piece -``` -python -m ulmfit cls --dataset-path data/mldoc/ja-1 --base-lm-path data/mldoc/ja-1/models/bsp30k/lstm_nl4.m --lang=ja --name 'nl4' --cuda-id=0 - train 1 --bs 40 --num-cls-epochs=2 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Tokenized data loaded, cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 1.222162 0.986234 0.765830 -epoch train_loss valid_loss accuracy -1 1.237291 0.983976 0.766659 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.759230 0.619306 0.826000 -epoch train_loss valid_loss accuracy -1 0.599281 0.423162 0.841000 -epoch train_loss valid_loss accuracy -1 0.485808 0.360609 0.869000 -epoch train_loss valid_loss accuracy -1 0.415960 0.390202 0.872000 -2 0.371651 0.365374 0.876000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.330675, tensor(0.8873)] -0.33067500591278076 -0.8872500061988831 -(fastaiv1) pczapla@galatea ~/w/ulmfit-multilingual ❯❯❯ python -m ulmfit eval --glob="mldoc/ja-1/models/sp30k/lstm_nl4.m" --name nl4-2nd --cuda-id=0 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-2nd.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Tokenized data loaded, cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-2nd.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.844110 0.700549 0.743000 -epoch train_loss valid_loss accuracy -1 0.610796 0.400912 0.853000 -epoch train_loss valid_loss accuracy -1 0.449974 0.358793 0.870000 -epoch train_loss valid_loss accuracy -1 0.407609 0.397678 0.868000 -2 0.367383 0.373168 0.869000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-2nd.m -Loss and accuracy using (cls_best): [0.33044776, tensor(0.8863)] -OrderedDict([('data/mldoc/ja-1/models/sp30k/lstm_nl4-2nd.m', - 0.8862500190734863)]) -``` -``` -python -m ulmfit cls --dataset-path data/mldoc/ja-1 --base-lm-path data/mldoc/ja-1/models/bsp30k/lstm_nl4.m --lang=ja --name '2nd-nl4' --cuda-id=0 - train 1 --bs 40 --num-cls-epochs=2 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_2nd-nl4.m -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Tokenized data loaded, cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 1.208774 0.984775 0.766183 -epoch train_loss valid_loss accuracy -1 1.198147 0.984786 0.766730 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_2nd-nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.735084 0.613895 0.803000 -epoch train_loss valid_loss accuracy -1 0.550159 0.406097 0.867000 -epoch train_loss valid_loss accuracy -1 0.468788 0.404081 0.862000 -epoch train_loss valid_loss accuracy -1 0.395969 0.380797 0.870000 -2 0.349470 0.386497 0.866000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_2nd-nl4.m -Loss and accuracy using (cls_best): [0.32550755, tensor(0.8857)] -0.3255075514316559 -0.8857499957084656 - -``` -```bash -python -m ulmfit cls --dataset-path data/mldoc/ja-1 --base-lm-path data/mldoc/ja-1/models/bsp30k/lstm_nl4.m --lang=ja --name '3nd-nl4' --cuda-id=0 - train 0 --bs 40 --num-cls-epochs=2 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_3nd-nl4.m -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Tokenized data loaded, cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_3nd-nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.777661 0.617013 0.786000 -epoch train_loss valid_loss accuracy -1 0.603897 0.388985 0.867000 -epoch train_loss valid_loss accuracy -1 0.510845 0.374942 0.874000 -epoch train_loss valid_loss accuracy -1 0.468642 0.379503 0.872000 -2 0.430415 0.365797 0.880000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_3nd-nl4.m -Loss and accuracy using (cls_best): [0.33084384, tensor(0.8882)] -0.33084383606910706 -0.8882499933242798 - -python -m ulmfit cls --dataset-path data/mldoc/ja-1 --base-lm-path data/mldoc/ja-1/models/bsp30k/lstm_nl4.m --lang=ja --tokenizer=bsp --name '3nd-nl4' --cuda-id=0 - train 0 --bs 40 --num-cls-epochs=2 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_3nd-nl4.m -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Tokenized data loaded, cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_3nd-nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.824216 0.604706 0.825000 -epoch train_loss valid_loss accuracy -1 0.606317 0.409647 0.854000 -epoch train_loss valid_loss accuracy -1 0.500782 0.381826 0.862000 -epoch train_loss valid_loss accuracy -1 0.403516 0.366863 0.866000 -2 0.394599 0.357580 0.874000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_3nd-nl4.m -Loss and accuracy using (cls_best): [0.32903105, tensor(0.8848)] -0.3290310502052307 -0.8847500085830688 - -python -m ulmfit cls --dataset-path data/mldoc/ja-1 --base-lm-path data/mldoc/ja-1/models/bsp30k/lstm_nl4.m --lang=ja --tokenizer=bsp --name '3nd-nl4' --cuda-id=0 - train 0 --bs 40 --num-cls-epochs=2 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_3nd-nl4.m -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Tokenized data loaded, cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] -Loading last classifier -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.460803 0.451998 0.855000 -epoch train_loss valid_loss accuracy -1 0.460069 0.421900 0.867000 -epoch train_loss valid_loss accuracy -1 0.361791 0.447982 0.859000 -epoch train_loss valid_loss accuracy -1 0.301233 0.404477 0.868000 -2 0.269350 0.406427 0.870000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_3nd-nl4.m -Loss and accuracy using (cls_best): [0.34159982, tensor(0.8925)] -0.34159982204437256 -0.8924999833106995 - -``` -## SP60k -``` -python -m ulmfit lm --dataset-path data/wiki/ja-100 --cuda-id=1 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 60000 \ ---lang ja --qrnn=False - train 10 --bs=50 --drop_mult=0 - -Running tokenization -Wiki text was split to 98375 articles -Wiki text was split to 138 articles -Size of vocabulary: 60000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': None, 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: [0.25 0.1 0.2 0.02 0.15] -Training lm from random weights -epoch train_loss valid_loss accuracy -1 3.557822 3.682454 0.366108 -2 3.377493 3.614226 0.369889 -3 3.391634 3.562171 0.377114 -4 3.328160 3.497388 0.385236 -5 3.290285 3.424971 0.394655 -6 3.159867 3.337317 0.407095 -7 3.139091 3.250999 0.417750 -8 3.103923 3.153146 0.433443 -9 2.979789 3.092179 0.443405 -10 2.984099 3.077171 0.446887 -data/wiki/ja-100/models/sp60k -Saving info data/wiki/ja-100/models/sp60k/lstm_nl4.m/info.json -``` -## MLDoc -````bash -python -m ulmfit cls --dataset-path data/mldoc/ja-1 --base-lm-path data/wiki/ja-100/models/sp60k/lstm_nl4.m --lang=ja --name 'nl4' --cuda-id=1 - train 20 --bs 40 --num-cls-epochs=2 -Max vocab: 60000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp60k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp60k/lstm_nl4.m -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 60000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp60k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp60k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp60k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp60k/lstm_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.019972 2.548868 0.503754 -epoch train_loss valid_loss accuracy -1 2.643698 2.363415 0.532341 -2 2.403588 2.149524 0.567359 -3 2.218298 1.969651 0.597484 -4 2.059648 1.829897 0.619758 -5 1.941803 1.722339 0.636215 -6 1.862969 1.630191 0.650293 -7 1.796515 1.551929 0.663782 -8 1.727768 1.481659 0.675489 -9 1.667709 1.417764 0.687287 -10 1.606343 1.357994 0.697264 -11 1.553344 1.303901 0.707811 -12 1.539182 1.251784 0.718038 -Traceback (most recent call last): -```` \ No newline at end of file diff --git a/results/logs/label_smoothing.md b/results/logs/label_smoothing.md deleted file mode 100644 index 4d17cee..0000000 --- a/results/logs/label_smoothing.md +++ /dev/null @@ -1,1174 +0,0 @@ -# MLDoc - -## QRNN 15k - -## LM+CLS training -``` -python -m ulmfit eval --glob="wiki/*-100/models/sp15k/qrnn_nl4.m" --name nl4-sl --dataset-template='../mldoc/${lang}-1' --num-lm-epochs=20 --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.1 -Processing data/wiki/de-100/models/sp15k/qrnn_nl4.m -../mldoc/de-1 -Max vocab: 15000 -Cache dir: /home/test/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k -Model dir: /home/test/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_nl4-sl.m -Training -Loading validation /home/test/workspace/ulmfit-multilingual/data/mldoc/de-1/de.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/test/workspace/ulmfit-multilingual/data/wiki/de-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/test/workspace/ulmfit-multilingual/data/wiki/de-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 4.336932 3.600409 0.524499 - -Total time: 01:55 -epoch train_loss valid_loss accuracy -1 3.867303 3.446689 0.547847 - -2 3.475966 3.257031 0.578702 - -3 3.233705 3.085521 0.607024 - -4 3.170250 2.964533 0.627028 - -5 3.059212 2.876878 0.641094 - -6 2.965926 2.797152 0.654905 - -7 2.974514 2.743403 0.663470 - -8 2.858759 2.690824 0.672891 - -9 2.866673 2.646101 0.680956 - -10 2.814579 2.610239 0.687777 - -11 2.806775 2.577145 0.694683 - -12 2.741160 2.540292 0.702336 - -13 2.753407 2.506782 0.709361 - -14 2.720171 2.480167 0.715709 - -15 2.631490 2.452236 0.721706 - -16 2.587928 2.431256 0.726922 - -17 2.608380 2.417473 0.729975 - -18 2.564811 2.407112 0.732487 - -19 2.599782 2.403481 0.733463 - -20 2.603479 2.402438 0.733552 - -Total time: 1:18:42 -/home/test/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k -Saving info /home/test/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_nl4-sl.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.790572 0.610132 0.925000 - -2 0.655777 0.619519 0.945000 - -3 0.591051 0.654355 0.930000 - -4 0.541992 0.582572 0.944000 - -5 0.514052 0.574799 0.939000 - -6 0.490411 0.550166 0.949000 - -7 0.476344 0.553866 0.947000 - -8 0.469109 0.548871 0.947000 - -Total time: 02:43 -Saving models at /home/test/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_nl4-sl.m -Loss and accuracy using (cls_best): [0.2010318, tensor(0.9610)] - -Processing data/wiki/en-100/models/sp15k/qrnn_nl4.m -../mldoc/en-1 -Max vocab: 15000 -Cache dir: /home/test/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k -Model dir: /home/test/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k/qrnn_nl4-sl.m -Training -Loading validation /home/test/workspace/ulmfit-multilingual/data/mldoc/en-1/en.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁the', '▁,', 's', '▁.', '▁of', '▁and', '▁in', '▁to', '▁a', 'ed'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/test/workspace/ulmfit-multilingual/data/wiki/en-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/test/workspace/ulmfit-multilingual/data/wiki/en-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 5.361436 4.703674 0.343924 - -Total time: 02:55 -epoch train_loss valid_loss accuracy -1 4.757253 4.493189 0.372715 - -2 4.515738 4.291261 0.404511 - -3 4.301139 4.110973 0.430612 - -4 4.162306 3.964101 0.450053 - -5 4.062487 3.841125 0.466488 - -6 3.898028 3.740108 0.480628 - -7 3.876914 3.660982 0.493164 - -8 3.793781 3.593925 0.502977 - -9 3.736873 3.528259 0.513241 - -10 3.695738 3.477659 0.521709 - -11 3.668749 3.431972 0.529821 - -12 3.642119 3.385145 0.537860 - -13 3.556678 3.343567 0.545521 - -14 3.548823 3.305735 0.552502 - -15 3.520068 3.272878 0.558736 - -16 3.439619 3.247021 0.563504 - -17 3.391731 3.228240 0.567151 - -18 3.398134 3.217466 0.569319 - -19 3.402400 3.212110 0.570352 - -20 3.375334 3.210727 0.570589 - -Total time: 1:19:32 -/home/test/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k -Saving info /home/test/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k/qrnn_nl4-sl.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.855421 0.644234 0.903000 - -2 0.722607 0.665451 0.954000 - -3 0.629362 0.590857 0.945000 - -4 0.555874 0.562738 0.950000 - -5 0.522701 0.549048 0.953000 - -6 0.506758 0.536445 0.961000 - -7 0.489250 0.527361 0.963000 - -8 0.482953 0.528452 0.961000 - -Total time: 02:54 -Saving models at /home/test/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k/qrnn_nl4-sl.m -Loss and accuracy using (cls_best): [0.20434816, tensor(0.9555)] - -Processing data/wiki/es-100/models/sp15k/qrnn_nl4.m -../mldoc/es-1 -Max vocab: 15000 -Cache dir: /home/test/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k -Model dir: /home/test/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-sl.m -Training -Loading validation /home/test/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', '▁la', 's', '▁el', '▁en', '▁y', '▁a', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/test/workspace/ulmfit-multilingual/data/wiki/es-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/test/workspace/ulmfit-multilingual/data/wiki/es-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.925853 3.293504 0.535753 - -Total time: 01:42 -epoch train_loss valid_loss accuracy -1 3.584036 3.131868 0.563887 - -2 3.341769 2.924815 0.605171 - -3 3.064540 2.760458 0.637288 - -4 2.979203 2.651128 0.657270 - -5 2.913760 2.569732 0.670629 - -6 2.901400 2.507033 0.682515 - -7 2.884516 2.454021 0.692617 - -8 2.759039 2.404587 0.703479 - -9 2.730353 2.367218 0.711349 - -10 2.657660 2.325339 0.720632 - -11 2.638513 2.292851 0.728599 - -12 2.629284 2.258947 0.737086 - -13 2.542013 2.226581 0.744815 - -14 2.464086 2.202000 0.750827 - -15 2.489060 2.177043 0.757989 - -16 2.446775 2.158471 0.762447 - -17 2.388175 2.144888 0.766083 - -18 2.415777 2.136921 0.768290 - -19 2.454445 2.132960 0.769424 - -20 2.346935 2.132173 0.769667 - -Total time: 47:14 -/home/test/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k -Saving info /home/test/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-sl.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.842615 0.671844 0.919000 - -2 0.705828 0.669006 0.948000 - -3 0.627629 0.572335 0.944000 - -4 0.576609 0.600053 0.953000 - -5 0.532899 0.542761 0.962000 - -6 0.503425 0.548742 0.961000 - -7 0.495654 0.535397 0.959000 - -8 0.487190 0.545798 0.958000 - -Total time: 02:16 -Saving models at /home/test/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-sl.m -Loss and accuracy using (cls_best): [0.18526463, tensor(0.9582)] - -Processing data/wiki/fr-100/models/sp15k/qrnn_nl4.m -../mldoc/fr-1 -Max vocab: 15000 -Cache dir: /home/test/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k -Model dir: /home/test/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_nl4-sl.m -Training -Loading validation /home/test/workspace/ulmfit-multilingual/data/mldoc/fr-1/fr.dev.csv -Data lm, trn: 13500, val: 1500 -Running tokenization cls... -Data cls, trn: 1000, val: 1000 - -Running tokenization tst... -Data tst, trn: 1000, val: 4000 - -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', 's', '▁.', "'", '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/test/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/test/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 4.408209 3.734384 0.451908 - -Total time: 02:09 -epoch train_loss valid_loss accuracy -1 3.911323 3.613580 0.472285 - -2 3.678445 3.445187 0.503284 - -3 3.489969 3.296292 0.528945 - -4 3.381153 3.180339 0.549296 - -5 3.291956 3.100773 0.562257 - -6 3.184217 3.027092 0.575632 - -7 3.215341 2.965142 0.586544 - -8 3.119935 2.915341 0.596039 - -9 3.081539 2.870155 0.605426 - -10 3.096917 2.826453 0.614306 - -11 3.024909 2.786344 0.622573 - -12 2.940282 2.743246 0.632480 - -13 2.939400 2.713417 0.639012 - -14 2.920471 2.682074 0.646323 - -15 2.836955 2.652518 0.653338 - -16 2.873827 2.631899 0.658241 - -17 2.847641 2.615588 0.662295 - -18 2.856253 2.605571 0.664535 - -19 2.817845 2.601670 0.665505 - -20 2.827199 2.600590 0.665696 - -Total time: 1:13:04 -/home/test/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k -Saving info /home/test/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_nl4-sl.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.830361 0.645785 0.913000 - -2 0.718334 0.713889 0.903000 - -3 0.625460 0.608466 0.936000 - -4 0.549874 0.573567 0.938000 - -5 0.513573 0.563112 0.938000 - -6 0.497791 0.559489 0.948000 - -7 0.482823 0.547815 0.944000 - -8 0.473484 0.543763 0.946000 - -Total time: 02:36 -Saving models at /home/test/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_nl4-sl.m -Loss and accuracy using (cls_best): [0.21287616, tensor(0.9480)] - -Processing data/wiki/it-100/models/sp15k/qrnn_nl4.m -../mldoc/it-1 -Max vocab: 15000 -Cache dir: /home/test/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k -Model dir: /home/test/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k/qrnn_nl4-sl.m -Training -Loading validation /home/test/workspace/ulmfit-multilingual/data/mldoc/it-1/it.dev.csv -Running tokenization lm... -Data lm, trn: 13500, val: 1500 - -Running tokenization cls... -Data cls, trn: 1000, val: 1000 - -Running tokenization tst... -Data tst, trn: 1000, val: 4000 - -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', '▁e', "▁&'", "'", '▁il', '▁la', '▁in', 'e'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/test/workspace/ulmfit-multilingual/data/wiki/it-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/test/workspace/ulmfit-multilingual/data/wiki/it-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 4.802729 3.930606 0.435333 - -Total time: 01:03 -epoch train_loss valid_loss accuracy -1 4.208684 3.763728 0.460475 - -2 3.899941 3.558932 0.495172 - -3 3.627373 3.365080 0.528145 - -4 3.479348 3.217953 0.552994 - -5 3.345057 3.106383 0.571176 - -6 3.203966 3.013706 0.587515 - -7 3.159807 2.928730 0.601662 - -8 3.123686 2.863181 0.613407 - -9 3.091257 2.802755 0.625887 - -10 2.993412 2.747775 0.636441 - -11 2.923112 2.694130 0.647843 - -12 2.910893 2.644428 0.658610 - -13 2.912500 2.606808 0.667447 - -14 2.809009 2.564682 0.676342 - -15 2.813561 2.530405 0.684753 - -16 2.768533 2.505601 0.691209 - -17 2.714267 2.487358 0.695374 - -18 2.704343 2.474279 0.698375 - -19 2.721650 2.469191 0.699875 - -20 2.692483 2.468237 0.700245 - -Total time: 43:31 -/home/test/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k -Saving info /home/test/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k/qrnn_nl4-sl.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.926403 0.718438 0.853000 - -2 0.791397 0.850076 0.824000 - -3 0.710437 0.707848 0.902000 - -4 0.626952 0.700860 0.882000 - -5 0.551851 0.648725 0.900000 - -6 0.527778 0.632797 0.906000 - -7 0.502474 0.621409 0.911000 - -8 0.489953 0.621797 0.910000 - -Total time: 01:32 -Saving models at /home/test/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k/qrnn_nl4-sl.m -Loss and accuracy using (cls_best): [0.3240368, tensor(0.9005)] - -Processing data/wiki/ja-100/models/sp15k/qrnn_nl4.m -../mldoc/ja-1 -Max vocab: 15000 -Cache dir: /home/test/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k -Model dir: /home/test/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k/qrnn_nl4-sl.m -Training -Loading validation /home/test/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv -Running tokenization lm... -Data lm, trn: 13500, val: 1500 - -Running tokenization cls... -Data cls, trn: 1000, val: 1000 - -Running tokenization tst... -Data tst, trn: 1000, val: 4000 - -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/test/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/test/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 4.467716 3.639215 0.510628 - -Total time: 02:43 -epoch train_loss valid_loss accuracy -1 3.640961 3.393765 0.547888 - -2 3.245662 3.111493 0.597234 - -3 3.091152 2.893684 0.636629 - -4 2.874470 2.753393 0.660614 - -5 2.774781 2.660047 0.677299 - -6 2.818495 2.584401 0.690161 - -7 2.763403 2.525782 0.699487 - -8 2.689764 2.481472 0.708918 - -9 2.471829 2.443742 0.715523 - -10 2.558768 2.411052 0.722205 - -11 2.583986 2.380159 0.728743 - -12 2.416061 2.352447 0.734377 - -13 2.422695 2.327425 0.739690 - -14 2.447176 2.302086 0.745426 - -15 2.409782 2.280813 0.749981 - -16 2.431124 2.265426 0.753981 - -17 2.409947 2.255584 0.756481 - -18 2.426040 2.246758 0.758470 - -19 2.363041 2.244397 0.759102 - -20 2.397845 2.243302 0.759366 - -Total time: 1:20:37 -/home/test/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k -Saving info /home/test/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k/qrnn_nl4-sl.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.962460 0.721570 0.861000 - -2 0.850875 0.732539 0.873000 - -3 0.733097 0.733598 0.880000 - -4 0.639531 0.743423 0.882000 - -5 0.570058 0.702896 0.870000 - -6 0.525673 0.663320 0.892000 - -7 0.514369 0.668241 0.887000 - -8 0.500725 0.663006 0.886000 - -Total time: 03:16 -Saving models at /home/test/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k/qrnn_nl4-sl.m -Loss and accuracy using (cls_best): [0.32758784, tensor(0.8988)] - -Processing data/wiki/ru-100/models/sp15k/qrnn_nl4.m -../mldoc/ru-1 -Max vocab: 15000 -Cache dir: /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k -Model dir: /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4-sl.m -Training -Loading validation /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv -Running tokenization lm... -Data lm, trn: 9195, val: 1021 - -Running tokenization cls... -Data cls, trn: 1000, val: 1000 - -Running tokenization tst... -Data tst, trn: 1000, val: 4000 - -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', '▁на', 'х'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/test/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/test/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 4.599347 3.756284 0.476954 - -Total time: 02:08 -epoch train_loss valid_loss accuracy -1 3.814071 3.501726 0.524080 - -2 3.543633 3.248534 0.571844 - -3 3.245142 3.051544 0.607123 - -4 3.074194 2.917155 0.630383 - -5 3.015727 2.814585 0.648273 - -6 2.936803 2.731585 0.663555 - -7 2.801445 2.659986 0.676864 - -8 2.829947 2.602137 0.688181 - -9 2.784838 2.547982 0.699379 - -10 2.705119 2.501527 0.709368 - -11 2.763923 2.456791 0.719173 - -12 2.597343 2.411362 0.730358 - -13 2.647648 2.374054 0.738579 - -14 2.527706 2.336248 0.747234 - -15 2.543835 2.306177 0.755421 - -16 2.478718 2.282218 0.761426 - -17 2.563173 2.261219 0.766674 - -18 2.480387 2.251179 0.769268 - -19 2.415822 2.244734 0.770911 - -20 2.459630 2.243680 0.771312 - -Total time: 1:04:34 -/home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k -Saving info /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4-sl.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.010382 0.760864 0.848000 - -2 0.885819 0.800701 0.835000 - -3 0.768587 0.828160 0.844000 - -4 0.698592 0.751787 0.857000 - -5 0.620826 0.767858 0.856000 - -6 0.563309 0.727524 0.859000 - -7 0.524067 0.700363 0.878000 - -8 0.499860 0.707688 0.871000 - -Total time: 03:40 -Saving models at /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4-sl.m -Loss and accuracy using (cls_best): [0.40361115, tensor(0.8717)] - -Processing data/wiki/zh-100/models/sp15k/qrnn_nl4.m -../mldoc/zh-1 -Max vocab: 15000 -Cache dir: /home/test/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k -Model dir: /home/test/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-sl.m -Training -Loading validation /home/test/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv -Running tokenization lm... -Data lm, trn: 13500, val: 1500 - -Running tokenization cls... -Data cls, trn: 1000, val: 1000 - -Running tokenization tst... -Data tst, trn: 1000, val: 4000 - -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/test/workspace/ulmfit-multilingual/data/wiki/zh-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/test/workspace/ulmfit-multilingual/data/wiki/zh-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.804344 3.253225 0.571249 - -Total time: 02:19 -epoch train_loss valid_loss accuracy -1 3.309380 3.061855 0.600547 - -2 3.055568 2.865146 0.636541 - -3 2.877304 2.710331 0.663194 - -4 2.749444 2.604171 0.681028 - -5 2.715787 2.528694 0.693581 - -6 2.664271 2.477576 0.702474 - -7 2.598713 2.412279 0.715434 - -8 2.540510 2.367390 0.724008 - -9 2.499321 2.330683 0.731755 - -10 2.513472 2.290408 0.740227 - -11 2.397077 2.248312 0.749950 - -12 2.425433 2.212132 0.757908 - -13 2.364556 2.176752 0.767242 - -14 2.349984 2.142507 0.775855 - -15 2.321824 2.119729 0.781726 - -16 2.313458 2.095738 0.788297 - -17 2.239505 2.078650 0.792735 - -18 2.240292 2.069656 0.795083 - -19 2.250233 2.064083 0.796754 - -20 2.251804 2.063307 0.797006 - -Total time: 1:10:47 -/home/test/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k -Saving info /home/test/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-sl.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.920672 0.685620 0.870000 - -2 0.773749 0.700495 0.907000 - -3 0.663505 0.669162 0.908000 - -4 0.591225 0.621341 0.915000 - -5 0.542783 0.622516 0.919000 - -6 0.517919 0.608709 0.911000 - -7 0.489793 0.605009 0.918000 - -8 0.476783 0.597071 0.916000 - -Total time: 02:37 -Saving models at /home/test/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-sl.m -Loss and accuracy using (cls_best): [0.29685277, tensor(0.9190)] - -OrderedDict([('data/mldoc/de-1/models/sp15k/qrnn_nl4-sl.m', 0.9610000252723694), - ('data/mldoc/en-1/models/sp15k/qrnn_nl4-sl.m', 0.9555000066757202), - ('data/mldoc/es-1/models/sp15k/qrnn_nl4-sl.m', 0.9582499861717224), - ('data/mldoc/fr-1/models/sp15k/qrnn_nl4-sl.m', 0.9480000138282776), - ('data/mldoc/it-1/models/sp15k/qrnn_nl4-sl.m', 0.9004999995231628), - ('data/mldoc/ja-1/models/sp15k/qrnn_nl4-sl.m', 0.8987500071525574), - ('data/mldoc/ru-1/models/sp15k/qrnn_nl4-sl.m', 0.871749997138977), - ('data/mldoc/zh-1/models/sp15k/qrnn_nl4-sl.m', - 0.9190000295639038)]) -data/mldoc/de-1/models/sp15k/qrnn_nl4-sl.m: 0.9610000252723694 -data/mldoc/en-1/models/sp15k/qrnn_nl4-sl.m: 0.9555000066757202 -data/mldoc/es-1/models/sp15k/qrnn_nl4-sl.m: 0.9582499861717224 -data/mldoc/fr-1/models/sp15k/qrnn_nl4-sl.m: 0.9480000138282776 -data/mldoc/it-1/models/sp15k/qrnn_nl4-sl.m: 0.9004999995231628 -data/mldoc/ja-1/models/sp15k/qrnn_nl4-sl.m: 0.8987500071525574 -data/mldoc/ru-1/models/sp15k/qrnn_nl4-sl.m: 0.871749997138977 -data/mldoc/zh-1/models/sp15k/qrnn_nl4-sl.m: 0.9190000295639038 -``` - - -## CLS training -### all -``` -python -m ulmfit eval --glob="mldoc/*-1/models/sp15k/qrnn_nl4.m" --name nl4-1cyc-sl-e4 --num-cls-epochs=4 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.1 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/de.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: - - !! WARNING !! - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Your compiler (c++) may be ABI-incompatible with PyTorch! -Please use a compiler that is ABI-compatible with GCC 4.9 and above. -See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. - -See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 -for instructions on how to install GCC 4.9 or higher. -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - !! WARNING !! - - warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.806209 0.620243 0.938000 -2 0.643088 0.608909 0.944000 -3 0.552762 0.577317 0.943000 -4 0.506282 0.566124 0.944000 -Total time: 01:09 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m -Loss and accuracy using (cls_best): [0.18408646, tensor(0.9597)] -Processing data/mldoc/en-1/models/sp15k/qrnn_nl4.m -en-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/en.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁the', '▁,', 's', '▁.', '▁of', '▁and', '▁in', '▁to', '▁a', 'ed'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.856259 0.637753 0.935000 -2 0.713970 0.627611 0.918000 -3 0.594603 0.550310 0.947000 -4 0.526848 0.549133 0.954000 -Total time: 01:15 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m -Loss and accuracy using (cls_best): [0.20320596, tensor(0.9500)] -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.804829 0.636135 0.906000 -2 0.712160 0.579012 0.952000 -3 0.605291 0.543731 0.965000 -4 0.531676 0.546145 0.966000 -Total time: 01:01 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m -Loss and accuracy using (cls_best): [0.18462537, tensor(0.9565)] -Processing data/mldoc/fr-1/models/sp15k/qrnn_nl4.m -fr-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/fr.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', 's', '▁.', "'", '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.835704 0.657239 0.902000 -2 0.688311 0.679450 0.924000 -3 0.575970 0.579612 0.938000 -4 0.515154 0.565664 0.939000 -Total time: 01:11 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m -Loss and accuracy using (cls_best): [0.20844615, tensor(0.9435)] -Processing data/mldoc/it-1/models/sp15k/qrnn_nl4.m -it-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/it.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', '▁e', "▁&'", "'", '▁il', '▁la', '▁in', 'e'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.914871 0.758207 0.848000 -2 0.799002 0.692626 0.877000 -3 0.658110 0.646415 0.888000 -4 0.567358 0.629301 0.912000 -Total time: 00:42 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m -Loss and accuracy using (cls_best): [0.30882642, tensor(0.9032)] -Processing data/mldoc/ja-1/models/sp15k/qrnn_nl4.m -ja-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.959891 0.771793 0.834000 -2 0.813159 0.684481 0.889000 -3 0.675869 0.698423 0.878000 -4 0.580597 0.689197 0.881000 -Total time: 01:24 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m -Loss and accuracy using (cls_best): [0.3306819, tensor(0.8967)] -Processing data/mldoc/zh-1/models/sp15k/qrnn_nl4.m -zh-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] -Loss and accuracy using (cls_best): [0.28541276, tensor(0.9237)] -OrderedDict([('data/mldoc/de-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m', - 0.9597499966621399), - ('data/mldoc/en-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m', - 0.949999988079071), - ('data/mldoc/es-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m', - 0.9564999938011169), - ('data/mldoc/fr-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m', - 0.9434999823570251), - ('data/mldoc/it-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m', - 0.903249979019165), - ('data/mldoc/ja-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m', - 0.8967499732971191), - ('data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m', - 0.9237499833106995)]) -data/mldoc/de-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m: 0.9597499966621399 -data/mldoc/en-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m: 0.949999988079071 -data/mldoc/es-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m: 0.9564999938011169 -data/mldoc/fr-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m: 0.9434999823570251 -data/mldoc/it-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m: 0.903249979019165 -data/mldoc/ja-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m: 0.8967499732971191 -data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m: 0.9237499833106995 -``` -### ZH -Exec 1 -``` -python -m ulmfit eval --glob="mldoc/zh-1/models/sp15k/qrnn_nl4.m" --name nl4-1cyc-sl --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.1 -Processing data/mldoc/zh-1/models/sp15k/qrnn_nl4.m -zh-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.859153 0.767961 0.864000 -2 0.768888 0.775161 0.904000 -3 0.658956 0.685653 0.902000 -4 0.589073 0.618438 0.923000 -5 0.540008 0.622157 0.915000 -6 0.508080 0.606979 0.914000 -7 0.487228 0.599491 0.918000 -8 0.477516 0.602196 0.923000 -Total time: 02:18 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl.m -Loss and accuracy using (cls_best): [0.2829206, tensor(0.9205)] -OrderedDict([('data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl.m', - 0.9204999804496765)]) -data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl.m: 0.9204999804496765 -``` -Exec 2 -````python -m ulmfit eval --glob="mldoc/zh-1/models/sp15k/qrnn_nl4.m" --name nl4-1cyc-sl1 --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.1 -Processing data/mldoc/zh-1/models/sp15k/qrnn_nl4.m -zh-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl1.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl1.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.881513 0.712176 0.865000 -2 0.743687 0.665091 0.906000 -3 0.677436 0.687689 0.873000 -4 0.595139 0.626483 0.920000 -5 0.542732 0.600652 0.914000 -6 0.512080 0.597546 0.916000 -7 0.487021 0.597065 0.912000 -8 0.476598 0.596792 0.914000 -Total time: 02:20 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl1.m -Loss and accuracy using (cls_best): [0.29172945, tensor(0.9178)] -OrderedDict([('data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl1.m', - 0.9177500009536743)]) -data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl1.m: 0.9177500009536743 -```` -Exec 4 -```bash -python -m ulmfit eval --glob="mldoc/zh-1/models/sp15k/qrnn_nl4.m" --name nl4-1cyc-sl-e4 --num-cls-epochs=4 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.1 ✘ 130 -Processing data/mldoc/zh-1/models/sp15k/qrnn_nl4.m -zh-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.880415 0.677291 0.901000 -2 0.729670 0.659975 0.911000 -3 0.624817 0.603056 0.921000 -4 0.542027 0.601961 0.921000 -Total time: 01:08 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m -Loss and accuracy using (cls_best): [0.28558904, tensor(0.9222)] -OrderedDict([('data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m', - 0.922249972820282)]) -data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m: 0.922249972820282 -``` -## LSTM sp30k -### 0.1 -```bash - python -m ulmfit eval --glob="mldoc/zh-1/models/sp30k/lstm_nl4.m" --name nl4-1cyc-sl --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.1 -Processing data/mldoc/zh-1/models/sp30k/lstm_nl4.m -zh-1 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.870432 0.670671 0.882000 -2 0.754248 0.824157 0.895000 -3 0.654601 0.727428 0.885000 -4 0.602772 0.668668 0.901000 -5 0.542110 0.625137 0.903000 -6 0.506150 0.617842 0.913000 -7 0.480944 0.616885 0.912000 -8 0.472876 0.614381 0.911000 -Total time: 06:38 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl.m -Loss and accuracy using (cls_best): [0.2977172, tensor(0.9233)] -OrderedDict([('data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl.m', - 0.9232500195503235)]) -data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl.m: 0.9232500195503235 -``` -### 0.2 -```bash -python -m ulmfit eval --glob="mldoc/zh-1/models/sp30k/lstm_nl4.m" --name nl4-1cyc-sl2 --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.2 -Processing data/mldoc/zh-1/models/sp30k/lstm_nl4.m -zh-1 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl2.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl2.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.045619 0.908213 0.874000 -2 0.957379 0.857977 0.921000 -3 0.891791 0.852157 0.905000 -4 0.845289 0.849923 0.914000 -5 0.818228 0.848613 0.921000 -6 0.787021 0.840483 0.920000 -7 0.776123 0.844006 0.919000 -8 0.762384 0.857240 0.916000 -Total time: 06:33 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl2.m -Loss and accuracy using (cls_best): [0.40299156, tensor(0.9170)] -OrderedDict([('data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl2.m', - 0.9169999957084656)]) -data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl2.m: 0.9169999957084656 -``` -### 0.4 -```bash - python -m ulmfit eval --glob="mldoc/zh-1/models/sp30k/lstm_nl4.m" --name nl4-1cyc-sl4 --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.4 -Processing data/mldoc/zh-1/models/sp30k/lstm_nl4.m -zh-1 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.251581 1.183341 0.898000 -2 1.214358 1.201266 0.834000 -3 1.190343 1.165525 0.919000 -4 1.168018 1.172510 0.903000 -5 1.149965 1.161660 0.914000 -6 1.140140 1.161689 0.915000 -7 1.135877 1.159853 0.912000 -8 1.134425 1.160039 0.911000 -Total time: 06:34 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl4.m -Loss and accuracy using (cls_best): [0.64041936, tensor(0.9195)] -OrderedDict([('data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl4.m', - 0.9194999933242798)]) -data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl4.m: 0.9194999933242798 -``` \ No newline at end of file diff --git a/results/logs/mldoc/noise/de10k-noise.md b/results/logs/mldoc/noise/de10k-noise.md deleted file mode 100644 index 4708277..0000000 --- a/results/logs/mldoc/noise/de10k-noise.md +++ /dev/null @@ -1,1158 +0,0 @@ -# Correct Val data -``` -python -m ulmfit eval_noise_resistance --lang=de --size=10 --prefix-name="val_" -Noise: 0 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_0.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: - - !! WARNING !! - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Your compiler (c++) may be ABI-incompatible with PyTorch! -Please use a compiler that is ABI-compatible with GCC 4.9 and above. -See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. - -See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 -for instructions on how to install GCC 4.9 or higher. -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - !! WARNING !! - - warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) -Loss and accuracy using (cls_last): [0.3033199, tensor(0.9712)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_0.m', - 0.9712499976158142)]) -Noise: 5 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_5.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 500 examples, only 0.95 have correct labels -Added noise to 50 examples, only 0.95 have correct labels -Data lm, trn: 13500, val: 1500 -Running tokenization clsnoise0.05tv... -Data clsnoise0.05tv, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_5.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.436501 0.416959 0.909000 -2 0.460737 0.426361 0.904000 -3 0.360029 0.502090 0.907000 -4 0.362643 0.404165 0.916000 -5 0.283832 0.478411 0.911000 -6 0.224228 0.538951 0.915000 -7 0.159160 0.662829 0.909000 -8 0.096864 0.699768 0.910000 -Total time: 18:20 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_5.m -Loss and accuracy using (cls_best): [0.22873034, tensor(0.9565)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_5.m', - 0.9564999938011169)]) -Noise: 10 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_10.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 1000 examples, only 0.9 have correct labels -Added noise to 100 examples, only 0.9 have correct labels -Data lm, trn: 13500, val: 1500 -Running tokenization clsnoise0.1tv... -Data clsnoise0.1tv, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_10.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.608604 0.614120 0.851000 -2 0.563473 0.521960 0.858000 -3 0.534108 0.564487 0.867000 -4 0.473966 0.600789 0.870000 -5 0.450963 0.579453 0.869000 -6 0.365191 0.641545 0.864000 -7 0.282259 0.785328 0.850000 -8 0.227324 0.844709 0.848000 -Total time: 18:56 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_10.m -Loss and accuracy using (cls_best): [0.24939896, tensor(0.9435)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_10.m', - 0.9434999823570251)]) -Noise: 15 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_15.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 1500 examples, only 0.85 have correct labels -Added noise to 150 examples, only 0.85 have correct labels -Data lm, trn: 13500, val: 1500 -Running tokenization clsnoise0.15tv... -Data clsnoise0.15tv, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_15.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.679152 0.757799 0.816000 -2 0.715950 0.683697 0.810000 -3 0.694975 0.646247 0.817000 -4 0.629321 0.658788 0.818000 -5 0.588069 0.728769 0.805000 -6 0.517607 0.815471 0.796000 -7 0.402233 0.961845 0.781000 -8 0.318577 0.961705 0.779000 -Total time: 18:38 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_15.m -Loss and accuracy using (cls_best): [0.6598115, tensor(0.9300)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_15.m', - 0.9300000071525574)]) -Noise: 20 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_20.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 2000 examples, only 0.8 have correct labels -Added noise to 200 examples, only 0.8 have correct labels -Data lm, trn: 13500, val: 1500 -Running tokenization clsnoise0.2tv... -Data clsnoise0.2tv, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_20.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.814279 0.829818 0.755000 -2 0.824145 0.879503 0.763000 -3 0.778472 0.853143 0.766000 -4 0.782468 0.779830 0.762000 -5 0.700055 0.840979 0.766000 -6 0.600067 0.887506 0.748000 -7 0.477300 1.100056 0.734000 -8 0.372687 1.117019 0.729000 -Total time: 18:50 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_20.m -Loss and accuracy using (cls_best): [0.32372993, tensor(0.9053)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_20.m', - 0.9052500128746033)]) -Noise: 25 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_25.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 2500 examples, only 0.75 have correct labels -Added noise to 250 examples, only 0.75 have correct labels -Data lm, trn: 13500, val: 1500 -Running tokenization clsnoise0.25tv... -Data clsnoise0.25tv, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_25.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.930055 0.914462 0.709000 -2 0.875372 0.947374 0.723000 -3 0.903772 0.995556 0.716000 -4 0.830826 0.843742 0.727000 -5 0.761732 0.948633 0.719000 -6 0.652835 1.037753 0.698000 -7 0.522116 1.141807 0.687000 -8 0.456265 1.270369 0.675000 -Total time: 18:53 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_25.m -Loss and accuracy using (cls_best): [0.42412135, tensor(0.8655)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_25.m', - 0.8654999732971191)]) -Noise: 30 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_30.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 3000 examples, only 0.7 have correct labels -Added noise to 300 examples, only 0.7 have correct labels -Data lm, trn: 13500, val: 1500 -Running tokenization clsnoise0.3tv... -Data clsnoise0.3tv, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_30.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.979780 0.992838 0.647000 -2 0.964802 0.975392 0.650000 -3 0.984818 0.938703 0.664000 -4 0.947048 0.971088 0.667000 -5 0.835190 0.978763 0.651000 -6 0.779068 1.057726 0.654000 -7 0.640580 1.204921 0.617000 -8 0.581258 1.248827 0.613000 -Total time: 18:57 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_30.m -Loss and accuracy using (cls_best): [0.48816764, tensor(0.8470)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_30.m', - 0.847000002861023)]) -Noise: 35 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_35.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 3500 examples, only 0.65 have correct labels -Added noise to 350 examples, only 0.65 have correct labels -Data lm, trn: 13500, val: 1500 -Running tokenization clsnoise0.35tv... -Data clsnoise0.35tv, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_35.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.055006 0.998555 0.620000 -2 1.044137 0.994613 0.631000 -3 1.002715 1.070116 0.624000 -4 0.986447 1.054147 0.621000 -5 0.967057 1.016487 0.622000 -6 0.833950 1.251236 0.580000 -7 0.717729 1.236029 0.571000 -8 0.628723 1.342408 0.559000 -Total time: 19:03 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_35.m -Loss and accuracy using (cls_best): [0.6226422, tensor(0.7990)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_35.m', - 0.7990000247955322)]) -Noise: 40 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_40.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 4000 examples, only 0.6 have correct labels -Added noise to 400 examples, only 0.6 have correct labels -Data lm, trn: 13500, val: 1500 -Running tokenization clsnoise0.4tv... -Data clsnoise0.4tv, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_40.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.103033 1.086538 0.570000 -2 1.107361 1.056345 0.569000 -3 1.081129 1.063769 0.548000 -4 1.095896 1.061330 0.556000 -5 0.991429 1.102777 0.569000 -6 0.915104 1.178768 0.525000 -7 0.779928 1.327309 0.513000 -8 0.686467 1.426985 0.502000 -Total time: 18:50 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_40.m -Loss and accuracy using (cls_best): [0.8506142, tensor(0.7530)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_40.m', - 0.753000020980835)]) -Noise: 45 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_45.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 4500 examples, only 0.55 have correct labels -Added noise to 450 examples, only 0.55 have correct labels -Data lm, trn: 13500, val: 1500 -Running tokenization clsnoise0.45tv... -Data clsnoise0.45tv, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_45.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.140645 1.110783 0.512000 -2 1.136563 1.121161 0.535000 -3 1.129539 1.099606 0.509000 -4 1.108058 1.073747 0.524000 -5 1.048675 1.126527 0.504000 -6 0.956287 1.194883 0.479000 -7 0.818828 1.354711 0.461000 -8 0.758420 1.486676 0.448000 -Total time: 19:12 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_45.m -Loss and accuracy using (cls_best): [0.82442254, tensor(0.7000)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_45.m', - 0.699999988079071)]) -Noise: 50 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_50.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 5000 examples, only 0.5 have correct labels -Added noise to 500 examples, only 0.5 have correct labels -Data lm, trn: 13500, val: 1500 -Running tokenization clsnoise0.5tv... -Data clsnoise0.5tv, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_50.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.179953 1.145395 0.438000 -2 1.165212 1.175898 0.483000 -3 1.181145 1.127925 0.477000 -4 1.139963 1.151791 0.471000 -5 1.124215 1.130031 0.466000 -6 1.029675 1.185899 0.450000 -7 0.911210 1.320619 0.427000 -8 0.838111 1.383757 0.419000 -Total time: 18:22 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_50.m -Loss and accuracy using (cls_best): [0.9008743, tensor(0.6545)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_50.m', - 0.6545000076293945)]) -Noise: 55 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_55.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 5500 examples, only 0.45 have correct labels -Added noise to 550 examples, only 0.45 have correct labels -Data lm, trn: 13500, val: 1500 -Running tokenization clsnoise0.55tv... -Data clsnoise0.55tv, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_55.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.183246 1.166291 0.448000 -2 1.196840 1.163279 0.404000 -3 1.193226 1.180232 0.407000 -4 1.183507 1.153738 0.425000 -5 1.164359 1.166123 0.396000 -6 1.095628 1.207745 0.387000 -7 0.982455 1.293190 0.383000 -8 0.911601 1.383209 0.369000 -Total time: 19:04 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_55.m -Loss and accuracy using (cls_best): [1.0093645, tensor(0.6365)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_55.m', - 0.6365000009536743)]) -Noise: 60 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_60.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 6000 examples, only 0.4 have correct labels -Added noise to 600 examples, only 0.4 have correct labels -Data lm, trn: 13500, val: 1500 -Running tokenization clsnoise0.6tv... -Data clsnoise0.6tv, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_60.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.234308 1.216432 0.370000 -2 1.218607 1.166292 0.360000 -3 1.206626 1.179762 0.335000 -4 1.192636 1.164426 0.332000 -5 1.167782 1.226321 0.362000 -6 1.118646 1.225461 0.360000 -7 1.057706 1.246721 0.358000 -8 1.011610 1.281406 0.346000 -Total time: 18:43 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_60.m -Loss and accuracy using (cls_best): [1.0831424, tensor(0.4775)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_60.m', - 0.47749999165534973)]) -Noise: 65 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_65.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 6500 examples, only 0.35 have correct labels -Added noise to 650 examples, only 0.35 have correct labels -Data lm, trn: 13500, val: 1500 -Running tokenization clsnoise0.65tv... -Data clsnoise0.65tv, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_65.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.215925 1.200209 0.326000 -2 1.235970 1.173920 0.313000 -3 1.218506 1.251267 0.310000 -4 1.204741 1.182633 0.347000 -5 1.173637 1.171351 0.333000 -6 1.109750 1.226862 0.329000 -7 1.054864 1.330735 0.309000 -8 1.002966 1.325511 0.323000 -Total time: 18:20 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_65.m -Loss and accuracy using (cls_best): [1.2715616, tensor(0.3695)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_65.m', - 0.3695000112056732)]) -Noise: 70 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_70.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 7000 examples, only 0.3 have correct labels -Added noise to 700 examples, only 0.3 have correct labels -Data lm, trn: 13500, val: 1500 -Running tokenization clsnoise0.7tv... -Data clsnoise0.7tv, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_70.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.217860 1.219061 0.351000 -2 1.222316 1.191306 0.332000 -3 1.222368 1.254068 0.343000 -4 1.208153 1.216313 0.339000 -5 1.180702 1.166293 0.336000 -6 1.128917 1.211504 0.333000 -7 1.080312 1.271096 0.337000 -8 1.017895 1.335092 0.339000 -Total time: 18:47 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_70.m -Loss and accuracy using (cls_best): [1.4655445, tensor(0.2688)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_70.m', - 0.26875001192092896)]) -Noise: 75 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_75.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 7500 examples, only 0.25 have correct labels -Added noise to 750 examples, only 0.25 have correct labels -Data lm, trn: 13500, val: 1500 -Running tokenization clsnoise0.75tv... -Data clsnoise0.75tv, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_75.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.203315 1.147710 0.380000 -2 1.212273 1.190271 0.333000 -3 1.200970 1.208472 0.367000 -4 1.169665 1.165628 0.358000 -5 1.166296 1.213566 0.374000 -6 1.112285 1.462085 0.380000 -7 1.054748 1.266731 0.389000 -8 0.981918 1.692906 0.364000 -Total time: 18:34 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_val_75.m -Loss and accuracy using (cls_best): [1.957079, tensor(0.1252)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_val_75.m', - 0.12524999678134918)]) -{'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_0.m': 0.9712499976158142, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_5.m': 0.9564999938011169, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_10.m': 0.9434999823570251, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_15.m': 0.9300000071525574, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_20.m': 0.9052500128746033, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_25.m': 0.8654999732971191, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_30.m': 0.847000002861023, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_35.m': 0.7990000247955322, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_40.m': 0.753000020980835, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_45.m': 0.699999988079071, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_50.m': 0.6545000076293945, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_55.m': 0.6365000009536743, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_60.m': 0.47749999165534973, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_65.m': 0.3695000112056732, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_70.m': 0.26875001192092896, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_val_75.m': 0.12524999678134918} -``` -# Incorrect Val data - -``` python -m ulmfit eval_noise_resistance --size=10 -Noise: 0 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_0.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Running tokenization lm... -Data lm, trn: 13500, val: 1500 -Running tokenization cls... -Data cls, trn: 10000, val: 1000 -Running tokenization tst... -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: - - !! WARNING !! - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Your compiler (c++) may be ABI-incompatible with PyTorch! -Please use a compiler that is ABI-compatible with GCC 4.9 and above. -See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. - -See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 -for instructions on how to install GCC 4.9 or higher. -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - !! WARNING !! - - warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_0.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.248545 0.179782 0.959000 -2 0.189914 0.262213 0.960000 -3 0.166565 0.489769 0.942000 -4 0.126547 0.252591 0.956000 -5 0.125812 0.265295 0.959000 -6 0.052031 0.339353 0.966000 -7 0.055669 0.452688 0.965000 -8 0.019922 0.439593 0.963000 -Total time: 18:52 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_0.m -Loss and accuracy using (cls_best): [0.3236656, tensor(0.9720)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_0.m', 0.972000002861023)]) -Noise: 5 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_5.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 500 examples, only 0.95 have correct labels -Data lm, trn: 13500, val: 1500 -Running tokenization clsnoise0.05... -Data clsnoise0.05, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_5.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.467763 0.204974 0.949000 -2 0.403194 0.166174 0.957000 -3 0.412454 0.176702 0.958000 -4 0.337252 0.957854 0.964000 -5 0.314917 0.179809 0.959000 -6 0.244516 0.202331 0.959000 -7 0.145377 0.225054 0.960000 -8 0.129216 0.239015 0.960000 -Total time: 19:15 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_5.m -Loss and accuracy using (cls_best): [0.18567306, tensor(0.9620)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_5.m', - 0.9620000123977661)]) -Noise: 10 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_10.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 1000 examples, only 0.9 have correct labels -Data lm, trn: 13500, val: 1500 -Running tokenization clsnoise0.1... -Data clsnoise0.1, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_10.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.611526 0.220041 0.950000 -2 0.523582 0.209251 0.959000 -3 0.566672 0.190530 0.960000 -4 0.530805 0.214880 0.949000 -5 0.470179 0.238882 0.954000 -6 0.368767 0.224209 0.950000 -7 0.276495 0.254708 0.942000 -8 0.242059 0.260761 0.940000 -Total time: 19:23 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_10.m -Loss and accuracy using (cls_best): [0.19178627, tensor(0.9557)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_10.m', - 0.9557499885559082)]) -Noise: 15 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_15.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 1500 examples, only 0.85 have correct labels -Data lm, trn: 13500, val: 1500 -Running tokenization clsnoise0.15... -Data clsnoise0.15, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_15.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.712348 0.224500 0.948000 -2 0.702935 0.188924 0.956000 -3 0.645956 0.224620 0.957000 -4 0.644018 0.248982 0.952000 -5 0.580399 0.255238 0.958000 -6 0.497618 0.297617 0.930000 -7 0.339819 0.272245 0.936000 -8 0.323421 0.297615 0.926000 -Total time: 18:50 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_15.m -Loss and accuracy using (cls_best): [0.25138617, tensor(0.9350)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_15.m', - 0.9350000023841858)]) -Noise: 20 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_20.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 2000 examples, only 0.8 have correct labels -Data lm, trn: 13500, val: 1500 -Running tokenization clsnoise0.2... -Data clsnoise0.2, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_20.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.821758 0.270898 0.954000 -2 0.831244 0.440837 0.923000 -3 0.787815 0.313425 0.962000 -4 0.730238 0.255505 0.958000 -5 0.748861 0.302285 0.959000 -6 0.603880 0.324315 0.933000 -7 0.539476 0.306479 0.931000 -8 0.431666 0.315723 0.916000 -Total time: 19:28 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_20.m -Loss and accuracy using (cls_best): [0.28679955, tensor(0.9202)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_20.m', - 0.9202499985694885)]) -Noise: 25 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_25.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 2500 examples, only 0.75 have correct labels -Data lm, trn: 13500, val: 1500 -Running tokenization clsnoise0.25... -Data clsnoise0.25, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_25.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.913877 0.325624 0.956000 -2 0.914296 0.380646 0.956000 -3 0.862887 0.400977 0.909000 -4 0.859236 0.291990 0.957000 -5 0.832637 0.447769 0.944000 -6 0.643751 0.319626 0.935000 -7 0.562953 0.444973 0.875000 -8 0.471118 0.437798 0.876000 -Total time: 19:29 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_25.m -Loss and accuracy using (cls_best): [0.53222567, tensor(0.8808)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_25.m', - 0.8807500004768372)]) -Noise: 30 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_30.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 3000 examples, only 0.7 have correct labels -Data lm, trn: 13500, val: 1500 -Running tokenization clsnoise0.3... -Data clsnoise0.3, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_30.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.967625 0.434347 0.921000 -2 1.015967 0.491977 0.927000 -3 0.943155 0.499572 0.935000 -4 0.953488 0.362500 0.951000 -5 0.859923 0.440134 0.944000 -6 0.761165 0.879942 0.894000 -7 0.662025 0.566052 0.857000 -8 0.572539 0.546405 0.826000 -Total time: 19:17 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_30.m -Loss and accuracy using (cls_best): [0.47183233, tensor(0.8540)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_30.m', - 0.8539999723434448)]) -Noise: 35 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_35.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 3500 examples, only 0.65 have correct labels -Data lm, trn: 13500, val: 1500 -Running tokenization clsnoise0.35... -Data clsnoise0.35, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_35.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.079700 0.486082 0.944000 -2 1.032614 0.470132 0.943000 -3 1.007776 0.460361 0.948000 -4 1.018879 0.476889 0.909000 -5 0.943363 1.618245 0.871000 -6 0.897375 0.604067 0.844000 -7 0.731791 10.002838 0.829000 -8 0.631117 6.527773 0.809000 -Total time: 19:26 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_35.m -Loss and accuracy using (cls_best): [0.79261243, tensor(0.8185)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_35.m', - 0.8184999823570251)]) -Noise: 40 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_40.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 4000 examples, only 0.6 have correct labels -Data lm, trn: 13500, val: 1500 -Running tokenization clsnoise0.4... -Data clsnoise0.4, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_40.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.105347 0.646775 0.915000 -2 1.121720 0.547566 0.943000 -3 1.084002 0.581386 0.949000 -4 1.058243 0.486947 0.953000 -5 1.020843 1.061957 0.945000 -6 0.944650 0.561371 0.872000 -7 0.852374 2.072736 0.815000 -8 0.745296 0.637481 0.807000 -Total time: 18:42 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_40.m -Loss and accuracy using (cls_best): [0.6664878, tensor(0.7915)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_40.m', - 0.7914999723434448)]) -Noise: 45 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_45.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 4500 examples, only 0.55 have correct labels -Data lm, trn: 13500, val: 1500 -Running tokenization clsnoise0.45... -Data clsnoise0.45, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_45.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.149571 0.715101 0.835000 -2 1.130510 0.812303 0.671000 -3 1.137401 0.566132 0.938000 -4 1.124969 0.654854 0.944000 -5 1.066340 0.624603 0.928000 -6 1.000705 0.700255 0.836000 -7 0.895356 0.726855 0.784000 -8 0.813315 0.740214 0.747000 -Total time: 19:12 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_45.m -Loss and accuracy using (cls_best): [0.74985105, tensor(0.7402)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_45.m', - 0.7402499914169312)]) -Noise: 50 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_50.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 5000 examples, only 0.5 have correct labels -Data lm, trn: 13500, val: 1500 -Running tokenization clsnoise0.5... -Data clsnoise0.5, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_50.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.176207 0.770225 0.894000 -2 1.146716 0.698875 0.916000 -3 1.159391 0.808971 0.908000 -4 1.148178 0.817309 0.715000 -5 1.142510 0.755769 0.872000 -6 1.068410 0.837932 0.839000 -7 0.985223 1.619010 0.714000 -8 0.900792 0.850288 0.672000 -Total time: 18:43 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_50.m -Loss and accuracy using (cls_best): [0.87151223, tensor(0.6888)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_50.m', - 0.6887500286102295)]) -Noise: 55 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_55.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 5500 examples, only 0.45 have correct labels -Data lm, trn: 13500, val: 1500 -Running tokenization clsnoise0.55... -Data clsnoise0.55, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_55.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.194526 0.829596 0.896000 -2 1.206947 0.800475 0.856000 -3 1.205839 0.832071 0.895000 -4 1.193005 8.321078 0.884000 -5 1.135684 3.162405 0.759000 -6 1.087508 0.866630 0.751000 -7 0.969481 0.909597 0.654000 -8 0.903410 1.040992 0.612000 -Total time: 18:41 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_55.m -Loss and accuracy using (cls_best): [0.944759, tensor(0.6118)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_55.m', - 0.6117500066757202)]) -Noise: 60 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_60.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 6000 examples, only 0.4 have correct labels -Data lm, trn: 13500, val: 1500 -Running tokenization clsnoise0.6... -Data clsnoise0.6, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_60.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.228728 1.235617 0.451000 -2 1.231521 1.103163 0.465000 -3 1.205319 1.067342 0.535000 -4 1.208439 1.171245 0.738000 -5 1.166922 1.071724 0.481000 -6 1.105134 0.913977 0.680000 -7 0.996566 1.203313 0.546000 -8 0.933625 1.092093 0.506000 -Total time: 18:30 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_60.m -Loss and accuracy using (cls_best): [1.0928471, tensor(0.5045)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_60.m', - 0.5044999718666077)]) -Noise: 65 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_65.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 6500 examples, only 0.35 have correct labels -Data lm, trn: 13500, val: 1500 -Running tokenization clsnoise0.65... -Data clsnoise0.65, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_65.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.233954 1.126451 0.312000 -2 1.228535 1.022103 0.514000 -3 1.223979 1.275958 0.040000 -4 1.201343 1.281353 0.233000 -5 1.174795 1.088655 0.714000 -6 1.137927 1.304352 0.430000 -7 1.061154 2.778949 0.450000 -8 0.992379 1.507618 0.409000 -Total time: 18:44 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_65.m -Loss and accuracy using (cls_best): [1.7004116, tensor(0.4027)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_65.m', - 0.4027499854564667)]) -Noise: 70 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_70.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 7000 examples, only 0.3 have correct labels -Data lm, trn: 13500, val: 1500 -Running tokenization clsnoise0.7... -Data clsnoise0.7, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_70.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.212556 1.284562 0.230000 -2 1.221429 1.494182 0.025000 -3 1.208761 1.601475 0.044000 -4 1.196318 1.222757 0.493000 -5 1.170390 1.235789 0.240000 -6 1.114086 1.487270 0.244000 -7 1.054812 1.470409 0.244000 -8 0.998996 1.476172 0.270000 -Total time: 19:12 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_70.m -Loss and accuracy using (cls_best): [1.4951355, tensor(0.2860)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_70.m', - 0.28600001335144043)]) -Noise: 75 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_75.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 7500 examples, only 0.25 have correct labels -Data lm, trn: 13500, val: 1500 -Running tokenization clsnoise0.75... -Data clsnoise0.75, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_75.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.206901 1.635707 0.035000 -2 1.226015 1.508531 0.222000 -3 1.203663 1.570317 0.020000 -4 1.201733 1.404657 0.054000 -5 1.161620 1.456142 0.040000 -6 1.142974 1.518606 0.037000 -7 1.067800 1.744556 0.098000 -8 0.983552 1.720158 0.117000 -Total time: 18:41 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_75.m -Loss and accuracy using (cls_best): [1.7742158, tensor(0.1138)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4_75.m', - 0.11375000327825546)]) -{'data/mldoc/de-10/models/sp15k/qrnn_nl4_0.m': 0.972000002861023, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_5.m': 0.9620000123977661, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_10.m': 0.9557499885559082, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_15.m': 0.9350000023841858, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_20.m': 0.9202499985694885, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_25.m': 0.8807500004768372, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_30.m': 0.8539999723434448, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_35.m': 0.8184999823570251, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_40.m': 0.7914999723434448, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_45.m': 0.7402499914169312, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_50.m': 0.6887500286102295, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_55.m': 0.6117500066757202, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_60.m': 0.5044999718666077, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_65.m': 0.4027499854564667, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_70.m': 0.28600001335144043, 'data/mldoc/de-10/models/sp15k/qrnn_nl4_75.m': 0.11375000327825546} -``` \ No newline at end of file diff --git a/results/logs/mldoc/noise/es-random.md b/results/logs/mldoc/noise/es-random.md deleted file mode 100644 index 8d79492..0000000 --- a/results/logs/mldoc/noise/es-random.md +++ /dev/null @@ -1,1085 +0,0 @@ -# Noise resistance on ES Random -## 1k - - - -## 10k -``` - python -m ulmfit eval_noise_resistance --lang=es --size=10 --prefix-name="rnd2_" --model="sp15k/qrnn_rnd-nl4.m" --label-smoothing-eps=0.1 --random-init=True -Noise: 0 -Processing data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_0.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: - - !! WARNING !! - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Your compiler (c++) may be ABI-incompatible with PyTorch! -Please use a compiler that is ABI-compatible with GCC 4.9 and above. -See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. - -See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 -for instructions on how to install GCC 4.9 or higher. -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - !! WARNING !! - - warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 0.674091 1.283792 0.574000 -2 0.624245 11.202841 0.519000 -3 0.634349 3.687749 0.516000 -4 0.596913 4.074763 0.608000 -5 0.594516 1.093833 0.715000 -6 0.557703 1.010482 0.694000 -7 0.543755 0.995253 0.702000 -8 0.537600 0.952028 0.754000 -Total time: 11:03 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_0.m -Loss and accuracy using (cls_best): [0.9988524, tensor(0.7690)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_0.m', - 0.7689999938011169)]) - python -m ulmfit eval_noise_resistance --lang=es --size=10 --prefix-name="rnd2_" --model="sp15k/qrnn_rnd-nl4.m" --label-smoothing-eps=0.1 --random-init=True -Noise: 0 -Processing data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_0.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: - - !! WARNING !! - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Your compiler (c++) may be ABI-incompatible with PyTorch! -Please use a compiler that is ABI-compatible with GCC 4.9 and above. -See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. - -See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 -for instructions on how to install GCC 4.9 or higher. -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - !! WARNING !! - - warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) -Loss and accuracy using (cls_best): [0.99926454, tensor(0.7695)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_0.m', - 0.7695000171661377)]) -Noise: 5 -Processing data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_5.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 472 examples, only 0.9500951575385916 have correct labels -Added noise to 50 examples, only 0.95 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.05tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 0.743423 1.354918 0.428000 -2 0.729408 1.153078 0.543000 -3 0.736531 1.465756 0.450000 -4 0.749902 4.479099 0.541000 -5 0.691128 1.322383 0.466000 -6 0.694896 1.274096 0.597000 -7 0.664847 1.020145 0.673000 -8 0.645594 1.027490 0.668000 -Total time: 10:56 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_5.m -Loss and accuracy using (cls_best): [0.68077123, tensor(0.7272)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_5.m', - 0.7272499799728394)]) -Noise: 10 -Processing data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_10.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 945 examples, only 0.9000845844787482 have correct labels -Added noise to 100 examples, only 0.9 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.1tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 0.834923 1.409279 0.485000 -2 0.854848 1.614973 0.320000 -3 0.828790 1.701499 0.396000 -4 0.803852 2.424411 0.547000 -5 0.801694 2.024830 0.562000 -6 0.786992 2.021667 0.609000 -7 0.765137 2.020704 0.652000 -8 0.746047 5.385053 0.652000 -Total time: 10:56 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_10.m -Loss and accuracy using (cls_best): [0.80370593, tensor(0.7172)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_10.m', - 0.7172499895095825)]) -Noise: 15 -Processing data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_15.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 1418 examples, only 0.8500740114189046 have correct labels -Added noise to 150 examples, only 0.85 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.15tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 0.885186 1.486315 0.448000 -2 0.922848 1.944151 0.485000 -3 0.919082 1.599238 0.400000 -4 0.895376 1.527287 0.451000 -5 0.881112 3.831651 0.478000 -6 0.919497 1.247746 0.508000 -7 0.829800 1.175510 0.508000 -8 0.861669 1.174659 0.504000 -Total time: 10:35 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_15.m -Loss and accuracy using (cls_best): [0.9063169, tensor(0.5955)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_15.m', - 0.5954999923706055)]) -Noise: 20 -Processing data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_20.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 1891 examples, only 0.8000634383590611 have correct labels -Added noise to 200 examples, only 0.8 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.2tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 1.000344 1.388730 0.335000 -2 1.007181 12.045410 0.472000 -3 0.983002 1.479323 0.371000 -4 0.958265 1.298010 0.400000 -5 0.939612 4.368020 0.414000 -6 0.928165 3.608808 0.492000 -7 0.926556 2.253188 0.471000 -8 0.905125 2.181204 0.499000 -Total time: 11:06 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_20.m -Loss and accuracy using (cls_best): [0.9292287, tensor(0.5985)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_20.m', - 0.5985000133514404)]) -Noise: 25 -Processing data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_25.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 2364 examples, only 0.7500528652992176 have correct labels -Added noise to 250 examples, only 0.75 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.25tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 1.057209 1.246537 0.466000 -2 1.038750 34.810867 0.114000 -3 1.030894 87.796410 0.376000 -4 1.041276 1.264513 0.462000 -5 1.029240 2.345500 0.422000 -6 1.000014 1.293007 0.464000 -7 0.960223 1.266377 0.466000 -8 0.942258 1.348627 0.447000 -Total time: 10:44 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_25.m -Loss and accuracy using (cls_best): [1.0587388, tensor(0.5707)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_25.m', - 0.5707499980926514)]) -Noise: 30 -Processing data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_30.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 2837 examples, only 0.7000422922393741 have correct labels -Added noise to 300 examples, only 0.7 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.3tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 1.087622 1.363222 0.435000 -2 1.095437 4.608276 0.221000 -3 1.094145 1.402207 0.371000 -4 1.078363 2.236299 0.388000 -5 1.056162 1.310936 0.431000 -python -m ulmfit eval_noise_resistance --lang=es --size=10 --prefix-name="rnd2_" --model="sp15k/qrnn_rnd-nl4.m" --label-smoo6 1.050225 1.267474 0.425000 -7 1.038286 1.299926 0.421000 -8 1.044985 1.300490 0.422000 -Total time: 10:41 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_30.m -Loss and accuracy using (cls_best): [0.9783086, tensor(0.5817)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_30.m', - 0.5817499756813049)]) -Noise: 35 -Processing data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_35.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 3310 examples, only 0.6500317191795305 have correct labels -Added noise to 350 examples, only 0.65 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.35tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 1.153989 1.421947 0.240000 -2 1.138209 1.703624 0.246000 -3 1.124318 112.371529 0.280000 -4 1.133281 1.417321 0.317000 -5 1.138515 1.317285 0.387000 -6 1.101543 1.366418 0.390000 -7 1.085555 1.617780 0.399000 -8 1.092133 1.394255 0.394000 -Total time: 11:06 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_35.m -Loss and accuracy using (cls_best): [1.1545019, tensor(0.5490)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_35.m', - 0.5490000247955322)]) -Noise: 40 -Processing data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_40.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 3783 examples, only 0.600021146119687 have correct labels -Added noise to 400 examples, only 0.6 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.4tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 1.199116 1.436976 0.385000 -2 1.182144 3.020936 0.350000 -3 1.169826 1.429357 0.359000 -4 1.184806 2.984490 0.362000 -5 1.142364 2.399835 0.322000 -6 1.136863 25.139421 0.359000 -7 1.139416 12.117358 0.365000 -8 1.132085 2.642181 0.368000 -Total time: 10:59 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_40.m -Loss and accuracy using (cls_best): [2.2691653, tensor(0.5375)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_40.m', - 0.5375000238418579)]) -Noise: 45 -Processing data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_45.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 4256 examples, only 0.5500105730598435 have correct labels -Added noise to 450 examples, only 0.55 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.45tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 1.211562 1.438882 0.337000 -2 1.218273 222.648880 0.291000 -3 1.203696 1.414955 0.343000 -4 1.204386 1.364311 0.369000 -5 1.203539 3.488942 0.356000 -6 1.175719 6.136192 0.349000 -7 1.145031 1.604449 0.365000 -8 1.141291 2.528898 0.361000 -Total time: 10:55 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_45.m -Loss and accuracy using (cls_best): [1.6499722, tensor(0.5272)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_45.m', - 0.5272499918937683)]) -Noise: 50 -Processing data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_50.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 4729 examples, only 0.5 have correct labels -Added noise to 500 examples, only 0.5 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.5tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 1.232532 1.421933 0.272000 -2 1.237531 1.822951 0.252000 -3 1.254736 1.534031 0.266000 -4 1.252628 1.536729 0.266000 -5 1.234980 1.544544 0.266000 -6 1.234228 1.558047 0.266000 -7 1.240368 1.538028 0.266000 -8 1.253012 1.540725 0.266000 -Total time: 11:01 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_50.m -Loss and accuracy using (cls_best): [1.4770876, tensor(0.3072)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_50.m', - 0.3072499930858612)]) -Noise: 55 -Processing data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_55.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 5201 examples, only 0.4500951575385917 have correct labels -Added noise to 550 examples, only 0.45 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.55tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 1.267338 1.647622 0.253000 -2 1.267763 1.562070 0.288000 -3 1.249864 2.255809 0.258000 -4 1.246395 1.470975 0.307000 -5 1.229589 1.466512 0.305000 -6 1.221072 1.472953 0.307000 -7 1.202461 1.430212 0.323000 -8 1.202863 1.416951 0.320000 -Total time: 10:59 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_55.m -Loss and accuracy using (cls_best): [1.1913521, tensor(0.5217)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_55.m', - 0.5217499732971191)]) -Noise: 60 -Processing data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_60.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 5674 examples, only 0.4000845844787482 have correct labels -Added noise to 600 examples, only 0.4 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.6tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 1.277718 1.426537 0.253000 -2 1.261508 1.677472 0.229000 -3 1.263114 1.585918 0.276000 -4 1.255874 2.783197 0.293000 -5 1.250460 3.303977 0.353000 -6 1.239010 10.652842 0.326000 -7 1.220644 4.388613 0.317000 -8 1.226328 6.756069 0.327000 -Total time: 11:07 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_60.m -Loss and accuracy using (cls_best): [2.6658566, tensor(0.5073)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_60.m', - 0.5072500109672546)]) -Noise: 65 -Processing data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_65.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 6147 examples, only 0.35007401141890465 have correct labels -Added noise to 650 examples, only 0.35 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.65tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 1.263687 1.667461 0.279000 -2 1.287779 8.032739 0.227000 -3 1.289650 1.522185 0.229000 -4 1.287830 1.587737 0.272000 -5 1.267796 1.598588 0.272000 -6 1.276992 1.563360 0.272000 -7 1.276122 1.564773 0.272000 -8 1.268917 1.559838 0.229000 -Total time: 11:10 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_65.m -Loss and accuracy using (cls_best): [1.4682757, tensor(0.3072)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_65.m', - 0.3072499930858612)]) -Noise: 70 -Processing data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_70.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 6620 examples, only 0.30006343835906113 have correct labels -Added noise to 700 examples, only 0.3 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.7tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 1.278968 1.395868 0.308000 -2 1.287285 2.381748 0.271000 -3 1.281547 1.510829 0.273000 -4 1.285452 1.517207 0.273000 -5 1.280697 1.535823 0.273000 -6 1.289347 1.505417 0.273000 -7 1.278146 1.527413 0.273000 -8 1.271342 1.522274 0.273000 -Total time: 10:48 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_70.m -Loss and accuracy using (cls_best): [1.4725544, tensor(0.3115)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_70.m', - 0.31150001287460327)]) -Noise: 75 -Processing data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_75.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 7093 examples, only 0.2500528652992176 have correct labels -Added noise to 750 examples, only 0.25 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.75tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 1.278254 1.406109 0.294000 -2 1.284943 1.538882 0.268000 -3 1.277497 3.605533 0.256000 -4 1.260796 1.501276 0.259000 -5 1.254600 2.722533 0.273000 -6 1.250827 4.186543 0.269000 -7 1.236754 3.883370 0.273000 -8 1.239405 1.523115 0.291000 -Total time: 11:17 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_75.m -Loss and accuracy using (cls_best): [1.7807395, tensor(0.1478)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_rnd2_75.m', - 0.14775000512599945)]) - noise accuracy -0 0.00 0.76950 -1 0.05 0.72725 -2 0.10 0.71725 -3 0.15 0.59550 -4 0.20 0.59850 -5 0.25 0.57075 -6 0.30 0.58175 -7 0.35 0.54900 -8 0.40 0.53750 -9 0.45 0.52725 -10 0.50 0.30725 -11 0.55 0.52175 -12 0.60 0.50725 -13 0.65 0.30725 -14 0.70 0.31150 -15 0.75 0.14775 -``` - -# LSTM -``` - python -m ulmfit eval_noise_resistance --lang=es --size=10 --prefix-name="rnd2_" --model="sp30k/lstm_nl4.m" --label-smoothing-eps=0.1 --random-init=True -Noise: 0 -Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m -es-10 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_0.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 0.659161 4.290442 0.318000 -2 0.750901 2.120531 0.277000 -3 0.780925 2.740564 0.277000 -4 0.761562 4.889140 0.293000 -5 0.743035 14.893404 0.279000 -6 0.726612 3.620207 0.277000 -7 0.760487 2.801041 0.293000 -8 0.751857 2.819516 0.293000 -Total time: 30:06 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_0.m -Loss and accuracy using (cls_best): [2.932457, tensor(0.3072)] -OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_0.m', - 0.3072499930858612)]) -Noise: 5 -Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m -es-10 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_5.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 472 examples, only 0.9500951575385916 have correct labels -Added noise to 50 examples, only 0.95 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.05tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 0.744206 2.318675 0.274000 -2 0.845912 1.891602 0.276000 -3 0.831846 3.916036 0.274000 -4 0.831695 1.779413 0.274000 -5 0.830590 1.956843 0.274000 -6 0.831666 1.893295 0.274000 -7 0.850189 1.924907 0.274000 -8 0.829423 1.925661 0.274000 -Total time: 30:02 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_5.m -Loss and accuracy using (cls_best): [1.8306484, tensor(0.3072)] -OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_5.m', - 0.3072499930858612)]) -Noise: 10 -Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m -es-10 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_10.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 945 examples, only 0.9000845844787482 have correct labels -Added noise to 100 examples, only 0.9 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.1tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 0.898602 1.545597 0.293000 -2 0.903453 176.651031 0.209000 -3 0.916878 13.823226 0.276000 -4 0.898918 24.522129 0.273000 -5 0.912165 1.825259 0.274000 -6 0.931385 1.808111 0.274000 -7 0.889932 2.430218 0.273000 -8 0.927120 2.782772 0.276000 -Total time: 31:02 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_10.m -Loss and accuracy using (cls_best): [1.7723552, tensor(0.3105)] -OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_10.m', - 0.31049999594688416)]) -Noise: 15 -Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m -es-10 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_15.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 1418 examples, only 0.8500740114189046 have correct labels -Added noise to 150 examples, only 0.85 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.15tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 0.973683 6.861582 0.322000 -2 0.966286 2.543203 0.273000 -3 1.016659 3.716355 0.205000 -4 0.955702 1.692103 0.273000 -5 0.967680 3.590133 0.280000 -6 0.982580 8.608456 0.269000 -7 0.940222 2.429212 0.273000 -8 0.996999 2.896548 0.269000 -Total time: 30:04 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_15.m -Loss and accuracy using (cls_best): [1.9653525, tensor(0.3075)] -OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_15.m', - 0.3075000047683716)]) -Noise: 20 -Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m -es-10 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_20.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 1891 examples, only 0.8000634383590611 have correct labels -Added noise to 200 examples, only 0.8 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.2tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 1.031621 2.023383 0.255000 -2 1.055999 1.585043 0.255000 -3 1.004598 1.747062 0.255000 -4 1.036506 12.106596 0.257000 -5 1.037610 7.782064 0.257000 -6 1.010497 4.877003 0.255000 -7 1.035525 9.258689 0.257000 -8 1.032628 9.379140 0.257000 -Total time: 30:00 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_20.m -Loss and accuracy using (cls_best): [2.198144, tensor(0.3105)] -OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_20.m', - 0.31049999594688416)]) -Noise: 25 -Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m -es-10 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_25.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 2364 examples, only 0.7500528652992176 have correct labels -Added noise to 250 examples, only 0.75 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.25tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 1.062350 1.959029 0.214000 -2 1.061749 1.823441 0.262000 -3 1.083258 12.511975 0.275000 -4 1.089900 2.291190 0.275000 -5 1.086922 8.510952 0.262000 -6 1.090885 2.293679 0.261000 -7 1.075075 2.249190 0.275000 -8 1.068257 2.230014 0.275000 -Total time: 30:26 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_25.m -Loss and accuracy using (cls_best): [2.080095, tensor(0.3072)] -OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_25.m', - 0.3072499930858612)]) -Noise: 30 -Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m -es-10 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_30.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 2837 examples, only 0.7000422922393741 have correct labels -Added noise to 300 examples, only 0.7 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.3tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 1.133636 1.806463 0.274000 -2 1.110103 10.866484 0.265000 -3 1.122795 6.634718 0.274000 -4 1.112287 5.359313 0.262000 -5 1.121410 174.423630 0.261000 -6 1.110173 18.667475 0.262000 -7 1.102498 12.188397 0.262000 -8 1.117406 15.158224 0.249000 -Total time: 29:59 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_30.m -Loss and accuracy using (cls_best): [8.974576, tensor(0.2555)] -OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_30.m', - 0.2554999887943268)]) -Noise: 35 -Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m -es-10 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_35.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 3310 examples, only 0.6500317191795305 have correct labels -Added noise to 350 examples, only 0.65 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.35tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 1.177323 2.044810 0.260000 -2 1.145307 1.615969 0.254000 -3 1.178776 2.619027 0.260000 -4 1.170236 7.795065 0.253000 -5 1.160120 1.604842 0.254000 -6 1.160545 1.587269 0.254000 -7 1.137365 1.629242 0.254000 -8 1.150483 1.610021 0.254000 -Total time: 29:59 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_35.m -Loss and accuracy using (cls_best): [1.5346162, tensor(0.3072)] -OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_35.m', - 0.3072499930858612)]) -Noise: 40 -Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m -es-10 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_40.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 3783 examples, only 0.600021146119687 have correct labels -Added noise to 400 examples, only 0.6 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.4tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 1.200657 1.747926 0.242000 -2 1.213164 33.535637 0.250000 -3 1.188038 1.609861 0.246000 -4 1.185921 2.510113 0.246000 -5 1.218471 4.206014 0.242000 -6 1.187989 1.855121 0.251000 -7 1.210737 4.329615 0.242000 -8 1.193102 5.145269 0.242000 -Total time: 30:09 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_40.m -Loss and accuracy using (cls_best): [2.3451035, tensor(0.3075)] -OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_40.m', - 0.3075000047683716)]) -Noise: 45 -Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m -es-10 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_45.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 4256 examples, only 0.5500105730598435 have correct labels -Added noise to 450 examples, only 0.55 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.45tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 1.241620 1.534527 0.278000 -2 1.234608 4.180245 0.251000 -3 1.225962 18.634438 0.251000 -4 1.231608 1.596539 0.245000 -5 1.242316 1.529011 0.245000 -6 1.224234 1.559436 0.245000 -7 1.219737 1.570357 0.245000 -8 1.217654 1.558945 0.245000 -Total time: 29:55 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_45.m -Loss and accuracy using (cls_best): [1.4820943, tensor(0.3072)] -OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_45.m', - 0.3072499930858612)]) -Noise: 50 -Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m -es-10 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_50.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 4729 examples, only 0.5 have correct labels -Added noise to 500 examples, only 0.5 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.5tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 1.248102 1.559488 0.241000 -2 1.239413 6.912020 0.242000 -3 1.258250 19.964916 0.240000 -4 1.244972 14.503420 0.240000 -5 1.231926 7.417413 0.237000 -6 1.240820 3.726920 0.240000 -7 1.250362 3.813594 0.240000 -8 1.246718 4.295069 0.240000 -Total time: 29:56 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_50.m -Loss and accuracy using (cls_best): [1.6464796, tensor(0.3105)] -OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_50.m', - 0.31049999594688416)]) -Noise: 55 -Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m -es-10 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_55.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 5201 examples, only 0.4500951575385917 have correct labels -Added noise to 550 examples, only 0.45 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.55tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 1.262681 1.390237 0.243000 -2 1.261966 1.613178 0.221000 -3 1.281932 80.386887 0.237000 -4 1.266172 20.980446 0.220000 -5 1.267419 1.564067 0.221000 -6 1.256289 2.409412 0.215000 -7 1.259451 2.007908 0.221000 -8 1.248848 1.682695 0.221000 -Total time: 30:44 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_55.m -Loss and accuracy using (cls_best): [1.5071006, tensor(0.3072)] -OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_55.m', - 0.3072499930858612)]) -Noise: 60 -Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m -es-10 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_60.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 5674 examples, only 0.4000845844787482 have correct labels -Added noise to 600 examples, only 0.4 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.6tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 1.299198 1.514330 0.257000 -2 1.276767 1.648853 0.246000 -3 1.273444 1.702971 0.249000 -4 1.275886 5.679698 0.254000 -5 1.282117 15.944865 0.249000 -6 1.273512 4.761624 0.254000 -7 1.283532 5.982591 0.254000 -8 1.266130 4.487469 0.254000 -Total time: 29:48 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_60.m -Loss and accuracy using (cls_best): [2.5968323, tensor(0.3072)] -OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_60.m', - 0.3072499930858612)]) -Noise: 65 -Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m -es-10 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_65.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 6147 examples, only 0.35007401141890465 have correct labels -Added noise to 650 examples, only 0.35 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.65tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 1.288758 1.800278 0.266000 -2 1.286780 1.503401 0.272000 -3 1.280267 1.921974 0.278000 -4 1.285067 1.523227 0.259000 -5 1.285825 1.526260 0.272000 -6 1.276409 1.531018 0.267000 -7 1.270072 1.525349 0.259000 -8 1.269815 1.514847 0.242000 -Total time: 29:50 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_65.m -Loss and accuracy using (cls_best): [1.4596524, tensor(0.3072)] -OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_65.m', - 0.3072499930858612)]) -Noise: 70 -Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m -es-10 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_70.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 6620 examples, only 0.30006343835906113 have correct labels -Added noise to 700 examples, only 0.3 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.7tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 1.271206 1.656565 0.258000 -2 1.285489 2.193548 0.271000 -3 1.283617 1.947972 0.264000 -4 1.282205 8.228713 0.270000 -5 1.275360 21.401678 0.270000 -6 1.266841 32.251156 0.270000 -7 1.263846 20.252125 0.258000 -8 1.270404 21.024738 0.258000 -Total time: 29:24 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_70.m -Loss and accuracy using (cls_best): [7.6955824, tensor(0.1828)] -OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_70.m', - 0.18275000154972076)]) -Noise: 75 -Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m -es-10 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_75.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 7093 examples, only 0.2500528652992176 have correct labels -Added noise to 750 examples, only 0.25 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.75tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 1.282777 1.870099 0.271000 -2 1.271396 3.192471 0.264000 -3 1.282228 11.233303 0.260000 -4 1.276331 35.493816 0.256000 -5 1.269741 3.304629 0.264000 -6 1.266907 6.163653 0.261000 -7 1.279470 3.861671 0.271000 -8 1.269125 2.668693 0.260000 -Total time: 30:42 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_75.m -Loss and accuracy using (cls_best): [1.6692431, tensor(0.1828)] -OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_rnd2_75.m', - 0.18275000154972076)]) - noise accuracy -0 0.00 0.30725 -1 0.05 0.30725 -2 0.10 0.31050 -3 0.15 0.30750 -4 0.20 0.31050 -5 0.25 0.30725 -6 0.30 0.25550 -7 0.35 0.30725 -8 0.40 0.30750 -9 0.45 0.30725 -10 0.50 0.31050 -11 0.55 0.30725 -12 0.60 0.30725 -13 0.65 0.30725 -14 0.70 0.18275 -15 0.75 0.18275 -``` \ No newline at end of file diff --git a/results/logs/mldoc/noise/es10k-noise-lstm.md b/results/logs/mldoc/noise/es10k-noise-lstm.md deleted file mode 100644 index b40dbe1..0000000 --- a/results/logs/mldoc/noise/es10k-noise-lstm.md +++ /dev/null @@ -1,370 +0,0 @@ -``` - python -m ulmfit eval_noise_resistance --lang=es --size=10 --prefix-name="val_" --model="sp30k/lstm_nl4.m" -Noise: 0 -Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m -es-10 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_val_0.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Loss and accuracy using (cls_best): [0.32779965, tensor(0.9515)] -OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_val_0.m', - 0.9514999985694885)]) -Noise: 5 -Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m -es-10 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_val_5.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Loss and accuracy using (cls_best): [0.33051395, tensor(0.9488)] -OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_val_5.m', - 0.9487500190734863)]) -Noise: 10 -Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m -es-10 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_val_10.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Loss and accuracy using (cls_best): [0.22158922, tensor(0.9433)] -OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_val_10.m', - 0.9432500004768372)]) -Noise: 15 -Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m -es-10 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_val_15.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Loss and accuracy using (cls_best): [0.25426567, tensor(0.9358)] -OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_val_15.m', - 0.9357500076293945)]) -Noise: 20 -Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m -es-10 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_val_20.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Loss and accuracy using (cls_best): [0.32246214, tensor(0.9210)] -OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_val_20.m', - 0.9210000038146973)]) -Noise: 25 -Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m -es-10 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_val_25.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Loss and accuracy using (cls_best): [0.823559, tensor(0.9095)] -OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_val_25.m', - 0.909500002861023)]) -Noise: 30 -Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m -es-10 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_val_30.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Loss and accuracy using (cls_best): [0.5010365, tensor(0.8942)] -OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_val_30.m', - 0.8942499756813049)]) -Noise: 35 -Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m -es-10 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_val_35.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Loss and accuracy using (cls_best): [0.95638776, tensor(0.5853)] -OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_val_35.m', - 0.5852500200271606)]) -Noise: 40 -Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m -es-10 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_val_40.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Loss and accuracy using (cls_best): [1.1012905, tensor(0.5642)] -OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_val_40.m', - 0.5642499923706055)]) -Noise: 45 -Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m -es-10 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_val_45.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Loss and accuracy using (cls_best): [1.6009017, tensor(0.3072)] -OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_val_45.m', - 0.3072499930858612)]) -Noise: 50 -Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m -es-10 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_val_50.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Loss and accuracy using (cls_best): [1.5735056, tensor(0.3072)] -OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_val_50.m', - 0.3072499930858612)]) -Noise: 55 -Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m -es-10 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_val_55.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 5201 examples, only 0.4500951575385917 have correct labels -Added noise to 550 examples, only 0.45 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.55tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Single training schedule -epoch train_loss valid_loss accuracy -1 1.150436 1.391014 0.321000 -2 1.190502 5.388964 0.313000 -3 1.216090 1.697217 0.221000 -4 1.221863 1.676644 0.221000 -5 1.213776 1.734900 0.221000 -6 1.195663 1.713853 0.221000 -7 1.211159 1.710040 0.221000 -8 1.197578 1.674693 0.221000 -Total time: 29:10 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_val_55.m -Loss and accuracy using (cls_best): [1.5282942, tensor(0.3072)] -OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_val_55.m', - 0.3072499930858612)]) -Noise: 60 -Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m -es-10 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_val_60.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 5674 examples, only 0.4000845844787482 have correct labels -Added noise to 600 examples, only 0.4 have correct labels -Data lm, trn: 13013, val: 1445 -Running tokenization clsnoise0.6tv... -Data clsnoise0.6tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_val_60.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.176071 1.396755 0.321000 -2 1.211001 1.619019 0.289000 -3 1.229442 1.733743 0.261000 -4 1.190156 1.545205 0.312000 -5 1.182274 1.369377 0.308000 -6 1.169403 1.352204 0.304000 -7 1.166997 1.332295 0.316000 -8 1.165893 1.370641 0.313000 -Total time: 30:23 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_val_60.m -Loss and accuracy using (cls_best): [1.2368572, tensor(0.6102)] -OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_val_60.m', - 0.6102499961853027)]) -Noise: 65 -Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m -es-10 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_val_65.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 6147 examples, only 0.35007401141890465 have correct labels -Added noise to 650 examples, only 0.35 have correct labels -Data lm, trn: 13013, val: 1445 -Running tokenization clsnoise0.65tv... -Data clsnoise0.65tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_val_65.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.179257 1.460323 0.295000 -2 1.220349 1.516707 0.222000 -3 1.211396 1.870125 0.242000 -4 1.187261 1.922184 0.308000 -5 1.201833 1.429372 0.300000 -6 1.187137 1.580070 0.264000 -7 1.162549 1.845004 0.294000 -8 1.162919 1.514930 0.313000 -Total time: 29:23 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_val_65.m -Loss and accuracy using (cls_best): [1.1729655, tensor(0.6385)] -OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_val_65.m', - 0.6384999752044678)]) -Noise: 70 -Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m -es-10 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_val_70.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 6620 examples, only 0.30006343835906113 have correct labels -Added noise to 700 examples, only 0.3 have correct labels -Data lm, trn: 13013, val: 1445 -Running tokenization clsnoise0.7tv... -Data clsnoise0.7tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_val_70.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.155135 1.479137 0.312000 -2 1.190364 1.649113 0.288000 -3 1.220965 3.919039 0.280000 -4 1.222588 1.696949 0.258000 -5 1.220919 1.669896 0.264000 -6 1.217906 2.003806 0.257000 -7 1.216235 1.654473 0.258000 -8 1.217084 1.675933 0.258000 -Total time: 29:04 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_val_70.m -Loss and accuracy using (cls_best): [1.5526773, tensor(0.1828)] -OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_val_70.m', - 0.18275000154972076)]) -Noise: 75 -Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m -es-10 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_val_75.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 7093 examples, only 0.2500528652992176 have correct labels -Added noise to 750 examples, only 0.25 have correct labels -Data lm, trn: 13013, val: 1445 -Running tokenization clsnoise0.75tv... -Data clsnoise0.75tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_val_75.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.170507 1.421675 0.344000 -2 1.221764 1.700004 0.246000 -3 1.215101 2.358311 0.263000 -4 1.243265 1.551931 0.257000 -5 1.222902 1.756996 0.271000 -6 1.215993 1.677014 0.266000 -7 1.225945 4.560951 0.263000 -8 1.219151 2.939914 0.245000 -Total time: 29:52 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp30k/lstm_nl4_val_75.m -Loss and accuracy using (cls_best): [1.7072973, tensor(0.2465)] -OrderedDict([('data/mldoc/es-10/models/sp30k/lstm_nl4_val_75.m', - 0.24650000035762787)]) - noise accuracy -0 0.00 0.95150 -1 0.05 0.94875 -2 0.10 0.94325 -3 0.15 0.93575 -4 0.20 0.92100 -5 0.25 0.90950 -6 0.30 0.89425 -7 0.35 0.58525 -8 0.40 0.56425 -9 0.45 0.30725 -10 0.50 0.30725 -11 0.55 0.30725 -12 0.60 0.61025 -13 0.65 0.63850 -14 0.70 0.18275 -15 0.75 0.24650 -``` \ No newline at end of file diff --git a/results/logs/mldoc/noise/es10k-noise.md b/results/logs/mldoc/noise/es10k-noise.md deleted file mode 100644 index c33efb1..0000000 --- a/results/logs/mldoc/noise/es10k-noise.md +++ /dev/null @@ -1,2137 +0,0 @@ - -# Label Smoothing -## Epochs 4 -``` -python -m ulmfit eval_noise_resistance --lang=es --size=10 --prefix-name="sl-e4" --model="sp15k/qrnn_nl4.m" --num-cls-epochs=4 --label-smoothing-eps=0.1 -python -m ulmfit eval_noise_resistance --lang=es --size=10 --prefix-name="sl-e4" --model="sp15k/qrnn_nl4.m" --num-cls-epochs=4 --label-smoothing-eps=0.1 -Noise: 0 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e40.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: - - !! WARNING !! - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Your compiler (c++) may be ABI-incompatible with PyTorch! -Please use a compiler that is ABI-compatible with GCC 4.9 and above. -See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. - -See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 -for instructions on how to install GCC 4.9 or higher. -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - !! WARNING !! - - warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) -Loss and accuracy using (cls_best): [0.2204297, tensor(0.9553)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e40.m', - 0.9552500247955322)]) -Noise: 5 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e45.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Loss and accuracy using (cls_best): [4.971248, tensor(0.8798)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e45.m', - 0.8797500133514404)]) -Noise: 10 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e410.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Loss and accuracy using (cls_best): [0.5768092, tensor(0.9420)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e410.m', - 0.9419999718666077)]) -Noise: 15 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e415.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Loss and accuracy using (cls_best): [1.2631954, tensor(0.9197)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e415.m', - 0.9197499752044678)]) -Noise: 20 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e420.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Loss and accuracy using (cls_best): [0.42572692, tensor(0.9237)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e420.m', - 0.9237499833106995)]) -Noise: 25 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e425.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Loss and accuracy using (cls_best): [29.74483, tensor(0.8648)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e425.m', - 0.8647500276565552)]) -Noise: 30 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e430.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Loss and accuracy using (cls_best): [0.51494944, tensor(0.9185)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e430.m', - 0.9185000061988831)]) -Noise: 35 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e435.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Loss and accuracy using (cls_best): [0.66567194, tensor(0.8848)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e435.m', - 0.8847500085830688)]) -Noise: 40 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e440.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Loss and accuracy using (cls_best): [2.3167746, tensor(0.8217)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e440.m', - 0.8217499852180481)]) -Noise: 45 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e445.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Loss and accuracy using (cls_best): [1.6398115, tensor(0.8192)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e445.m', - 0.8192499876022339)]) -Noise: 50 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e450.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 4729 examples, only 0.5 have correct labels -Added noise to 500 examples, only 0.5 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.5tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Single training schedule -epoch train_loss valid_loss accuracy -1 1.222034 1.372962 0.382000 -2 1.222928 9.350571 0.374000 -3 1.200542 1.636571 0.413000 -4 1.175799 7.368647 0.417000 -Total time: 05:31 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e450.m -Loss and accuracy using (cls_best): [5.1662917, tensor(0.7197)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e450.m', - 0.7197499871253967)]) -Noise: 55 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e455.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 5201 examples, only 0.4500951575385917 have correct labels -Added noise to 550 examples, only 0.45 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.55tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e455.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.241146 1.574603 0.338000 -2 1.220505 3.343982 0.314000 -3 1.215406 4.805650 0.381000 -4 1.192340 3.459251 0.368000 -Total time: 05:16 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e455.m -Loss and accuracy using (cls_best): [4.879584, tensor(0.6900)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e455.m', - 0.6899999976158142)]) -Noise: 60 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e460.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 5674 examples, only 0.4000845844787482 have correct labels -Added noise to 600 examples, only 0.4 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.6tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e460.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.258633 1.374315 0.319000 -2 1.249057 1.935162 0.279000 -3 1.235318 8.574917 0.325000 -4 1.224898 31.664907 0.356000 -Total time: 05:25 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e460.m -Loss and accuracy using (cls_best): [33.578053, tensor(0.5670)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e460.m', - 0.5669999718666077)]) -Noise: 65 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e465.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 6147 examples, only 0.35007401141890465 have correct labels -Added noise to 650 examples, only 0.35 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.65tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e465.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.259673 1.630593 0.307000 -2 1.256428 1.900148 0.237000 -3 1.241903 3.709883 0.307000 -4 1.218096 1.747542 0.292000 -Total time: 05:31 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e465.m -Loss and accuracy using (cls_best): [1.3016428, tensor(0.5238)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e465.m', - 0.5237500071525574)]) -Noise: 70 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e470.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 6620 examples, only 0.30006343835906113 have correct labels -Added noise to 700 examples, only 0.3 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.7tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e470.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.261316 1.444706 0.331000 -2 1.251065 6.526892 0.285000 -3 1.232021 19.925491 0.335000 -4 1.207373 1.669503 0.329000 -Total time: 05:17 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e470.m -Loss and accuracy using (cls_best): [1.6000191, tensor(0.1885)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e470.m', - 0.18850000202655792)]) -Noise: 75 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e475.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 7093 examples, only 0.2500528652992176 have correct labels -Added noise to 750 examples, only 0.25 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.75tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e475.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.256266 1.862535 0.346000 -2 1.245836 3.535186 0.307000 -3 1.222711 5.987287 0.335000 -4 1.206070 4.709743 0.322000 -Total time: 06:12 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e475.m -Loss and accuracy using (cls_best): [1.847491, tensor(0.1700)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e475.m', - 0.17000000178813934)]) - noise accuracy -0 0.00 0.95525 -1 0.05 0.87975 -2 0.10 0.94200 -3 0.15 0.91975 -4 0.20 0.92375 -5 0.25 0.86475 -6 0.30 0.91850 -7 0.35 0.88475 -8 0.40 0.82175 -9 0.45 0.81925 -10 0.50 0.71975 -11 0.55 0.69000 -12 0.60 0.56700 -13 0.65 0.52375 -14 0.70 0.18850 - -``` - -## Epochs 8 -``` -python -m ulmfit eval_noise_resistance --lang=es --size=10 --prefix-name="sl-e8" --model="sp15k/qrnn_nl4.m" --num-cls-epochs=8 --label-smoothing-eps=0.1 -Noise: 0 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e80.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: - - !! WARNING !! - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Your compiler (c++) may be ABI-incompatible with PyTorch! -Please use a compiler that is ABI-compatible with GCC 4.9 and above. -See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. - -See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 -for instructions on how to install GCC 4.9 or higher. -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - !! WARNING !! - - warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e80.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.520412 0.696753 0.934000 -2 0.509825 0.735629 0.934000 -3 0.486594 0.684108 0.931000 -4 0.488381 0.579037 0.953000 -5 0.471751 0.588278 0.945000 -6 0.459802 0.567890 0.943000 -7 0.455027 0.545390 0.954000 -8 0.462613 0.575517 0.943000 -Total time: 11:12 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e80.m -Loss and accuracy using (cls_best): [0.27576917, tensor(0.9417)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e80.m', - 0.9417499899864197)]) -Noise: 5 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e85.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 472 examples, only 0.9500951575385916 have correct labels -Added noise to 50 examples, only 0.95 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.05tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e85.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.666560 0.852668 0.874000 -2 0.657355 5.546601 0.824000 -3 0.656841 7.308374 0.853000 -4 0.612917 24.591734 0.790000 -5 0.596588 12.358717 0.834000 -6 0.568417 4.996921 0.913000 -7 0.557468 1.554828 0.831000 -8 0.536724 1.115134 0.858000 -Total time: 10:51 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e85.m -Loss and accuracy using (cls_best): [0.6382615, tensor(0.9078)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e85.m', - 0.9077500104904175)]) -Noise: 10 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e810.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 945 examples, only 0.9000845844787482 have correct labels -Added noise to 100 examples, only 0.9 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.1tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e810.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.771364 0.982115 0.801000 -2 0.744242 0.947076 0.840000 -3 0.736343 1.086157 0.842000 -4 0.730165 0.987014 0.827000 -5 0.708371 5.287072 0.763000 -6 0.697188 0.855899 0.833000 -7 0.640460 0.931902 0.835000 -8 0.593337 0.915233 0.837000 -Total time: 10:48 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e810.m -Loss and accuracy using (cls_best): [0.45135674, tensor(0.9170)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e810.m', - 0.9169999957084656)]) -Noise: 15 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e815.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 1418 examples, only 0.8500740114189046 have correct labels -Added noise to 150 examples, only 0.85 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.15tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e815.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.848943 1.361678 0.751000 -2 0.857650 1.120813 0.662000 -3 0.861447 1.036685 0.809000 -4 0.824053 1.000713 0.786000 -5 0.821309 1.111803 0.799000 -6 0.754514 1.129427 0.782000 -7 0.695773 1.394005 0.765000 -8 0.638984 1.081989 0.780000 -Total time: 11:00 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e815.m -Loss and accuracy using (cls_best): [0.36475056, tensor(0.9005)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e815.m', - 0.9004999995231628)]) -Noise: 20 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e820.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 1891 examples, only 0.8000634383590611 have correct labels -Added noise to 200 examples, only 0.8 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.2tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e820.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.931807 1.271276 0.709000 -2 0.923600 1.298985 0.741000 -3 0.925085 5.663558 0.658000 -4 0.929284 1.188406 0.739000 -5 0.888162 3.878520 0.658000 -6 0.819503 1.259068 0.731000 -7 0.752936 1.150221 0.710000 -8 0.735175 1.149028 0.722000 -Total time: 11:07 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e820.m -Loss and accuracy using (cls_best): [0.4438091, tensor(0.8813)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e820.m', - 0.8812500238418579)]) -Noise: 25 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e825.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 2364 examples, only 0.7500528652992176 have correct labels -Added noise to 250 examples, only 0.75 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.25tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e825.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.999227 1.194783 0.646000 -2 0.990589 2.633189 0.626000 -3 1.018390 1.542320 0.659000 -4 0.987891 7.653442 0.605000 -5 0.972299 2.651095 0.646000 -6 0.922192 9.360953 0.637000 -7 0.878368 2.497859 0.640000 -8 0.847166 1.993811 0.639000 -Total time: 10:48 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e825.m -Loss and accuracy using (cls_best): [0.88353807, tensor(0.8367)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e825.m', - 0.8367499709129333)]) -Noise: 30 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e830.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 2837 examples, only 0.7000422922393741 have correct labels -Added noise to 300 examples, only 0.7 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.3tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e830.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.052917 1.325655 0.581000 -2 1.071630 1.998020 0.526000 -3 1.063711 31.339291 0.510000 -4 1.029802 2.206242 0.607000 -5 1.027847 1.763125 0.607000 -6 0.961119 1.473439 0.657000 -7 0.908768 1.629924 0.617000 -8 0.873362 1.462983 0.626000 -Total time: 10:44 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e830.m -Loss and accuracy using (cls_best): [0.61362606, tensor(0.8410)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e830.m', - 0.8410000205039978)]) -Noise: 35 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e835.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 3310 examples, only 0.6500317191795305 have correct labels -Added noise to 350 examples, only 0.65 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.35tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e835.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.124931 1.301690 0.549000 -2 1.098526 1.527998 0.599000 -3 1.109146 3.372168 0.557000 -4 1.085633 9.049232 0.536000 -5 1.040149 2.878901 0.552000 -6 0.999970 1.699484 0.548000 -7 0.924742 2.458920 0.519000 -8 0.916262 5.709455 0.517000 -Total time: 10:39 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e835.m -Loss and accuracy using (cls_best): [7.591171, tensor(0.6967)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e835.m', - 0.6967499852180481)]) -Noise: 40 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e840.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 3783 examples, only 0.600021146119687 have correct labels -Added noise to 400 examples, only 0.6 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.4tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e840.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.165653 1.350154 0.508000 -2 1.172212 1.732053 0.421000 -3 1.159003 9.825891 0.476000 -4 1.147642 2.909990 0.485000 -5 1.085988 4.217392 0.523000 -6 1.073084 3.288731 0.488000 -7 0.998839 1.664031 0.482000 -8 0.932477 1.921165 0.471000 -Total time: 11:06 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e840.m -Loss and accuracy using (cls_best): [0.8643208, tensor(0.7275)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e840.m', - 0.7275000214576721)]) -Noise: 45 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e845.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 4256 examples, only 0.5500105730598435 have correct labels -Added noise to 450 examples, only 0.55 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.45tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e845.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.195873 1.296093 0.433000 -2 1.181843 13.203069 0.375000 -3 1.202427 3.895014 0.350000 -4 1.180791 2.918823 0.442000 -5 1.168890 4.746016 0.477000 -6 1.138908 5.763408 0.452000 -7 1.085301 1.734959 0.487000 -8 1.019007 1.669769 0.467000 -Total time: 10:40 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e845.m -Loss and accuracy using (cls_best): [0.99686193, tensor(0.7107)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e845.m', - 0.7107499837875366)]) -Noise: 50 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e850.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 4729 examples, only 0.5 have correct labels -Added noise to 500 examples, only 0.5 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.5tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e850.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.206193 1.506727 0.400000 -2 1.219487 1.916529 0.383000 -3 1.201234 12.136375 0.383000 -4 1.193026 2.111413 0.403000 -5 1.184301 2.633834 0.438000 -6 1.145973 2.558009 0.437000 -7 1.099467 2.177719 0.410000 -8 1.061634 2.724488 0.401000 -Total time: 11:06 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e850.m -Loss and accuracy using (cls_best): [1.8331982, tensor(0.5920)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e850.m', - 0.5920000076293945)]) -Noise: 55 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e855.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 5201 examples, only 0.4500951575385917 have correct labels -Added noise to 550 examples, only 0.45 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.55tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e855.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.231457 1.337840 0.356000 -2 1.245993 12.007490 0.306000 -3 1.229736 1.784564 0.331000 -4 1.227334 4.005848 0.339000 -5 1.207836 12.369584 0.363000 -6 1.186945 15.869857 0.365000 -7 1.143967 22.024086 0.386000 -8 1.097784 4.130466 0.361000 -Total time: 10:52 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e855.m -Loss and accuracy using (cls_best): [1.1213393, tensor(0.6237)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e855.m', - 0.6237499713897705)]) -Noise: 60 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e860.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 5674 examples, only 0.4000845844787482 have correct labels -Added noise to 600 examples, only 0.4 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.6tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e860.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.253129 1.360811 0.344000 -2 1.259076 1.422704 0.292000 -3 1.249924 2.302250 0.258000 -4 1.238938 7.596085 0.318000 -5 1.226801 19.490450 0.361000 -6 1.220376 45.920719 0.358000 -7 1.186902 92.042252 0.361000 -8 1.169160 47.323799 0.352000 -Total time: 10:51 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e860.m -Loss and accuracy using (cls_best): [54.127697, tensor(0.5253)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e860.m', - 0.5252500176429749)]) -Noise: 65 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e865.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 6147 examples, only 0.35007401141890465 have correct labels -Added noise to 650 examples, only 0.35 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.65tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e865.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.258576 1.400468 0.285000 -2 1.252457 1.485225 0.303000 -3 1.264578 15.317787 0.257000 -4 1.245918 8.976856 0.300000 -5 1.239147 5.338088 0.296000 -6 1.226425 8.540084 0.314000 -7 1.206139 8.959650 0.300000 -8 1.185957 7.868751 0.322000 -Total time: 10:47 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e865.m -Loss and accuracy using (cls_best): [3.7213144, tensor(0.5070)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e865.m', - 0.5070000290870667)]) -Noise: 70 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e870.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 6620 examples, only 0.30006343835906113 have correct labels -Added noise to 700 examples, only 0.3 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.7tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e870.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.257724 1.391102 0.311000 -2 1.259833 1.376604 0.346000 -3 1.256289 4.146193 0.272000 -4 1.248032 21.711473 0.308000 -5 1.237731 104.512573 0.287000 -6 1.226114 14.212803 0.318000 -7 1.190652 3.960902 0.333000 -8 1.186640 2.436945 0.339000 -Total time: 10:38 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e870.m -Loss and accuracy using (cls_best): [2.232332, tensor(0.2713)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e870.m', - 0.27125000953674316)]) -Noise: 75 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e875.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 7093 examples, only 0.2500528652992176 have correct labels -Added noise to 750 examples, only 0.25 have correct labels -Data lm, trn: 13013, val: 1445 -Data clsnoise0.75tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e875.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.246185 1.331556 0.343000 -2 1.247202 1.593753 0.334000 -3 1.248334 4.676108 0.320000 -4 1.235685 27.420618 0.289000 -5 1.221268 10.798445 0.330000 -6 1.201879 4.895516 0.335000 -7 1.164162 3.015471 0.353000 -8 1.146750 2.903884 0.353000 -Total time: 11:04 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e875.m -Loss and accuracy using (cls_best): [3.2696714, tensor(0.1305)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_sl-e875.m', - 0.13050000369548798)]) - noise accuracy -0 0.00 0.94175 -1 0.05 0.90775 -2 0.10 0.91700 -3 0.15 0.90050 -4 0.20 0.88125 -5 0.25 0.83675 -6 0.30 0.84100 -7 0.35 0.69675 -8 0.40 0.72750 -9 0.45 0.71075 -10 0.50 0.59200 -11 0.55 0.62375 -12 0.60 0.52525 -13 0.65 0.50700 -14 0.70 0.27125 -``` - - -# Correct VAL -``` -python -m ulmfit eval_noise_resistance --lang=es --size=10 --prefix-name="val_" -Noise: 0 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_0.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: - - !! WARNING !! - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Your compiler (c++) may be ABI-incompatible with PyTorch! -Please use a compiler that is ABI-compatible with GCC 4.9 and above. -See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. - -See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 -for instructions on how to install GCC 4.9 or higher. -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - !! WARNING !! - - warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) -Loss and accuracy using (cls_last): [0.38580656, tensor(0.9402)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_0.m', - 0.9402499794960022)]) -Noise: 5 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_5.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 472 examples, only 0.9500951575385916 have correct labels -Added noise to 50 examples, only 0.95 have correct labels -Data lm, trn: 13013, val: 1445 -Running tokenization clsnoise0.05tv... -Data clsnoise0.05tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_5.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.360209 0.664265 0.831000 -2 0.367246 0.505887 0.884000 -3 0.339675 0.646535 0.862000 -4 0.324250 0.914779 0.849000 -5 0.303626 0.592339 0.894000 -6 0.225317 0.699065 0.878000 -7 0.220272 0.726604 0.868000 -8 0.160895 0.767783 0.863000 -Total time: 11:24 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_5.m -Loss and accuracy using (cls_best): [0.36014587, tensor(0.9170)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_5.m', - 0.9169999957084656)]) -Noise: 10 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_10.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 945 examples, only 0.9000845844787482 have correct labels -Added noise to 100 examples, only 0.9 have correct labels -Data lm, trn: 13013, val: 1445 -Running tokenization clsnoise0.1tv... -Data clsnoise0.1tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_10.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.535271 0.740620 0.824000 -2 0.496433 1.023953 0.669000 -3 0.474564 0.993422 0.832000 -4 0.468398 6.249076 0.785000 -5 0.480843 0.850946 0.830000 -6 0.395878 0.789419 0.851000 -7 0.298389 2.299487 0.823000 -8 0.253688 2.279287 0.824000 -Total time: 11:21 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_10.m -Loss and accuracy using (cls_best): [0.871454, tensor(0.9075)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_10.m', - 0.9075000286102295)]) -Noise: 15 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_15.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 1418 examples, only 0.8500740114189046 have correct labels -Added noise to 150 examples, only 0.85 have correct labels -Data lm, trn: 13013, val: 1445 -Running tokenization clsnoise0.15tv... -Data clsnoise0.15tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_15.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.652501 1.015270 0.741000 -2 0.654432 1.025377 0.748000 -3 0.617786 1.639322 0.792000 -4 0.639355 1.662912 0.747000 -5 0.577110 5.244443 0.743000 -6 0.527992 3.701031 0.766000 -7 0.393420 3.620962 0.759000 -8 0.361106 3.019649 0.780000 -Total time: 11:31 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_15.m -Loss and accuracy using (cls_best): [0.35248652, tensor(0.9043)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_15.m', - 0.9042500257492065)]) -Noise: 20 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_20.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 1891 examples, only 0.8000634383590611 have correct labels -Added noise to 200 examples, only 0.8 have correct labels -Data lm, trn: 13013, val: 1445 -Running tokenization clsnoise0.2tv... -Data clsnoise0.2tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_20.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.749732 1.118057 0.710000 -2 0.785584 6.410913 0.607000 -3 0.713528 2.003799 0.754000 -4 0.752761 2.294566 0.724000 -5 0.652672 19.180836 0.708000 -6 0.596508 2.582183 0.711000 -7 0.489806 2.648639 0.712000 -8 0.467246 3.001610 0.698000 -Total time: 11:36 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_20.m -Loss and accuracy using (cls_best): [1.6113901, tensor(0.8475)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_20.m', - 0.8475000262260437)]) -Noise: 25 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_25.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 2364 examples, only 0.7500528652992176 have correct labels -Added noise to 250 examples, only 0.75 have correct labels -Data lm, trn: 13013, val: 1445 -Running tokenization clsnoise0.25tv... -Data clsnoise0.25tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_25.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.844129 1.047693 0.663000 -2 0.810037 1.353432 0.655000 -3 0.820446 3.722627 0.632000 -4 0.779205 1.445673 0.630000 -5 0.770405 1.024933 0.653000 -6 0.695584 1.400032 0.676000 -7 0.582116 1.612832 0.649000 -8 0.542117 1.422031 0.648000 -Total time: 11:48 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_25.m -Loss and accuracy using (cls_best): [0.62123287, tensor(0.8300)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_25.m', - 0.8299999833106995)]) -Noise: 30 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_30.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 2837 examples, only 0.7000422922393741 have correct labels -Added noise to 300 examples, only 0.7 have correct labels -Data lm, trn: 13013, val: 1445 -Running tokenization clsnoise0.3tv... -Data clsnoise0.3tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_30.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.946058 1.269022 0.625000 -2 0.942436 2.030650 0.636000 -3 0.920219 5.243942 0.463000 -4 0.890941 14.097425 0.605000 -5 0.835836 5.551986 0.617000 -6 0.708443 2.086009 0.624000 -7 0.639320 2.845402 0.582000 -8 0.537422 1.683678 0.597000 -Total time: 11:21 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_30.m -Loss and accuracy using (cls_best): [0.6909822, tensor(0.7915)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_30.m', - 0.7914999723434448)]) -Noise: 35 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_35.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 3310 examples, only 0.6500317191795305 have correct labels -Added noise to 350 examples, only 0.65 have correct labels -Data lm, trn: 13013, val: 1445 -Running tokenization clsnoise0.35tv... -Data clsnoise0.35tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_35.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.980721 1.266943 0.471000 -2 0.980537 1.438193 0.492000 -3 1.000082 5.081447 0.539000 -4 0.964535 1.640518 0.547000 -5 0.924568 2.078268 0.588000 -6 0.915998 1.612291 0.562000 -7 0.790721 1.804647 0.522000 -8 0.716465 2.185110 0.520000 -Total time: 11:48 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_35.m -Loss and accuracy using (cls_best): [1.8067851, tensor(0.7657)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_35.m', - 0.765749990940094)]) -Noise: 40 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_40.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 3783 examples, only 0.600021146119687 have correct labels -Added noise to 400 examples, only 0.6 have correct labels -Data lm, trn: 13013, val: 1445 -Running tokenization clsnoise0.4tv... -Data clsnoise0.4tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_40.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.031590 1.277689 0.458000 -2 1.056317 2.782609 0.349000 -3 1.044410 1.631716 0.449000 -4 1.034872 2.103323 0.478000 -5 0.966710 1.642946 0.472000 -6 0.943393 1.632070 0.490000 -7 0.856185 1.868247 0.479000 -8 0.780535 1.958063 0.483000 -Total time: 11:36 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_40.m -Loss and accuracy using (cls_best): [0.8928685, tensor(0.7508)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_40.m', - 0.7507500052452087)]) -Noise: 45 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_45.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 4256 examples, only 0.5500105730598435 have correct labels -Added noise to 450 examples, only 0.55 have correct labels -Data lm, trn: 13013, val: 1445 -Running tokenization clsnoise0.45tv... -Data clsnoise0.45tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_45.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.064314 1.302667 0.444000 -2 1.094542 2.054388 0.439000 -3 1.071187 2.209423 0.450000 -4 1.077246 6.365521 0.471000 -5 1.051128 3.242248 0.406000 -6 0.998142 2.802290 0.464000 -7 0.918943 2.492715 0.468000 -8 0.864835 8.349169 0.457000 -Total time: 11:39 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_45.m -Loss and accuracy using (cls_best): [6.738212, tensor(0.6977)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_45.m', - 0.6977499723434448)]) -Noise: 50 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_50.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 4729 examples, only 0.5 have correct labels -Added noise to 500 examples, only 0.5 have correct labels -Data lm, trn: 13013, val: 1445 -Running tokenization clsnoise0.5tv... -Data clsnoise0.5tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_50.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.101441 1.358480 0.450000 -2 1.150545 1.360492 0.392000 -3 1.120479 1.384937 0.437000 -4 1.110555 1.345856 0.401000 -5 1.089176 1.815834 0.420000 -6 1.051251 2.371094 0.422000 -7 1.004491 1.379894 0.410000 -8 0.936776 1.549806 0.424000 -Total time: 11:37 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_50.m -Loss and accuracy using (cls_best): [1.1773515, tensor(0.6180)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_50.m', - 0.6179999709129333)]) -Noise: 55 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_55.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 5201 examples, only 0.4500951575385917 have correct labels -Added noise to 550 examples, only 0.45 have correct labels -Data lm, trn: 13013, val: 1445 -Running tokenization clsnoise0.55tv... -Data clsnoise0.55tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_55.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.149922 1.401648 0.383000 -2 1.144570 9.186795 0.301000 -3 1.162765 1.583447 0.330000 -4 1.154295 1.568725 0.349000 -5 1.117640 1.624917 0.352000 -6 1.097049 2.510476 0.362000 -7 1.048231 3.798041 0.362000 -8 1.020844 3.568361 0.366000 -Total time: 11:25 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_55.m -Loss and accuracy using (cls_best): [1.6726145, tensor(0.6102)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_55.m', - 0.6102499961853027)]) -Noise: 60 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_60.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 5674 examples, only 0.4000845844787482 have correct labels -Added noise to 600 examples, only 0.4 have correct labels -Data lm, trn: 13013, val: 1445 -Running tokenization clsnoise0.6tv... -Data clsnoise0.6tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_60.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.166367 1.629363 0.358000 -2 1.175406 1.307703 0.341000 -3 1.172992 1.437346 0.342000 -4 1.163454 10.670442 0.339000 -5 1.158758 2.170272 0.325000 -6 1.122279 11.275146 0.351000 -7 1.080830 24.562853 0.359000 -8 1.055806 19.967308 0.354000 -Total time: 11:25 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_60.m -Loss and accuracy using (cls_best): [9.046943, tensor(0.4873)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_60.m', - 0.4872500002384186)]) -Noise: 65 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_65.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 6147 examples, only 0.35007401141890465 have correct labels -Added noise to 650 examples, only 0.35 have correct labels -Data lm, trn: 13013, val: 1445 -Running tokenization clsnoise0.65tv... -Data clsnoise0.65tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_65.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.188989 1.408018 0.326000 -2 1.182579 1.576379 0.279000 -3 1.180070 1.482971 0.257000 -4 1.164961 1.580750 0.283000 -5 1.166036 1.530756 0.294000 -6 1.136221 1.670132 0.297000 -7 1.102457 2.092621 0.291000 -8 1.058724 2.033105 0.287000 -Total time: 11:22 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_65.m -Loss and accuracy using (cls_best): [5.145201, tensor(0.4150)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_65.m', - 0.41499999165534973)]) -Noise: 70 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_70.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 6620 examples, only 0.30006343835906113 have correct labels -Added noise to 700 examples, only 0.3 have correct labels -Data lm, trn: 13013, val: 1445 -Running tokenization clsnoise0.7tv... -Data clsnoise0.7tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_70.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.181723 1.387360 0.342000 -2 1.179477 11.148307 0.343000 -3 1.178954 2.155812 0.293000 -4 1.177750 2.038787 0.308000 -5 1.156294 8.581575 0.319000 -6 1.122391 1.584792 0.346000 -7 1.095312 2.261911 0.346000 -8 1.048026 1.873787 0.337000 -Total time: 11:40 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_70.m -Loss and accuracy using (cls_best): [2.1144376, tensor(0.2125)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_70.m', - 0.21250000596046448)]) -Noise: 75 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_75.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 7093 examples, only 0.2500528652992176 have correct labels -Added noise to 750 examples, only 0.25 have correct labels -Data lm, trn: 13013, val: 1445 -Running tokenization clsnoise0.75tv... -Data clsnoise0.75tv, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_75.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.163953 1.370035 0.298000 -2 1.167424 1.883017 0.354000 -3 1.145917 1.854918 0.353000 -4 1.155910 1.328685 0.329000 -5 1.141541 1.895262 0.342000 -6 1.091663 22.743765 0.326000 -7 1.047826 50.595406 0.315000 -8 0.997792 39.199989 0.317000 -Total time: 11:30 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_val_75.m -Loss and accuracy using (cls_best): [38.87539, tensor(0.1922)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_val_75.m', - 0.19224999845027924)]) -{'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_0.m': 0.9402499794960022, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_5.m': 0.9169999957084656, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_10.m': 0.9075000286102295, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_15.m': 0.9042500257492065, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_20.m': 0.8475000262260437, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_25.m': 0.8299999833106995, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_30.m': 0.7914999723434448, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_35.m': 0.765749990940094, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_40.m': 0.7507500052452087, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_45.m': 0.6977499723434448, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_50.m': 0.6179999709129333, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_55.m': 0.6102499961853027, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_60.m': 0.4872500002384186, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_65.m': 0.41499999165534973, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_70.m': 0.21250000596046448, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_val_75.m': 0.19224999845027924} -``` - -# Incorrect Val data -``` -Noise: 0 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_0.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Running tokenization lm... -Data lm, trn: 13013, val: 1445 -Running tokenization cls... -Data cls, trn: 9458, val: 1000 -Running tokenization tst... -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: - - !! WARNING !! - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Your compiler (c++) may be ABI-incompatible with PyTorch! -Please use a compiler that is ABI-compatible with GCC 4.9 and above. -See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. - -See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 -for instructions on how to install GCC 4.9 or higher. -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - !! WARNING !! - - warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_0.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.118179 0.265471 0.939000 -2 0.131938 0.823511 0.839000 -3 0.139424 0.496273 0.906000 -4 0.065336 0.315253 0.951000 -5 0.050494 0.366971 0.938000 -6 0.041825 0.296858 0.965000 -7 0.023676 0.330993 0.957000 -8 0.014899 0.337952 0.952000 -Total time: 11:14 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_0.m -Loss and accuracy using (cls_best): [0.37119418, tensor(0.9465)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_0.m', - 0.9465000033378601)]) -Noise: 5 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_5.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 472 examples, only 0.9500951575385916 have correct labels -Data lm, trn: 13013, val: 1445 -Running tokenization clsnoise0.05... -Data clsnoise0.05, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_5.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.381904 0.318090 0.893000 -2 0.366108 0.304496 0.907000 -3 0.350814 0.222635 0.954000 -4 0.305924 0.316391 0.937000 -5 0.294849 0.365258 0.938000 -6 0.226888 0.265155 0.953000 -7 0.203635 0.277920 0.944000 -8 0.180686 0.326215 0.935000 -Total time: 11:05 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_5.m -Loss and accuracy using (cls_best): [0.31352887, tensor(0.9302)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_5.m', - 0.9302499890327454)]) -Noise: 10 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_10.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 945 examples, only 0.9000845844787482 have correct labels -Data lm, trn: 13013, val: 1445 -Running tokenization clsnoise0.1... -Data clsnoise0.1, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_10.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.503315 0.294730 0.903000 -2 0.520918 0.404052 0.908000 -3 0.496451 0.623593 0.825000 -4 0.484413 0.212174 0.950000 -5 0.397562 0.271711 0.929000 -6 0.393162 0.455898 0.908000 -7 0.329386 0.306643 0.922000 -8 0.285580 0.283296 0.921000 -Total time: 11:17 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_10.m -Loss and accuracy using (cls_best): [0.2845364, tensor(0.9258)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_10.m', - 0.9257500171661377)]) -Noise: 15 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_15.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 1418 examples, only 0.8500740114189046 have correct labels -Data lm, trn: 13013, val: 1445 -Running tokenization clsnoise0.15... -Data clsnoise0.15, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_15.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.641261 0.416610 0.848000 -2 0.602479 0.356870 0.901000 -3 0.652983 0.582720 0.812000 -4 0.582735 22.428156 0.942000 -5 0.565184 7.965161 0.855000 -6 0.489513 0.751756 0.898000 -7 0.418973 6.047875 0.859000 -8 0.337963 9.058367 0.845000 -Total time: 11:23 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_15.m -Loss and accuracy using (cls_best): [11.41559, tensor(0.8332)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_15.m', - 0.8332499861717224)]) -Noise: 20 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_20.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 1891 examples, only 0.8000634383590611 have correct labels -Data lm, trn: 13013, val: 1445 -Running tokenization clsnoise0.2... -Data clsnoise0.2, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_20.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.735070 0.430566 0.875000 -2 0.777014 1.136241 0.711000 -3 0.762944 10.364647 0.624000 -4 0.698076 1.523277 0.806000 -5 0.646547 1.162918 0.833000 -6 0.581218 1.385464 0.869000 -7 0.489041 1.207447 0.843000 -8 0.398324 0.668090 0.849000 -Total time: 11:16 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_20.m -Loss and accuracy using (cls_best): [0.55489075, tensor(0.8595)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_20.m', - 0.859499990940094)]) -Noise: 25 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_25.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 2364 examples, only 0.7500528652992176 have correct labels -Data lm, trn: 13013, val: 1445 -Running tokenization clsnoise0.25... -Data clsnoise0.25, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_25.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.877402 0.428968 0.871000 -2 0.874461 0.585437 0.772000 -3 0.866958 0.598036 0.807000 -4 0.805568 0.964052 0.893000 -5 0.760956 5.249828 0.800000 -6 0.671527 1.038407 0.864000 -7 0.597481 0.767250 0.855000 -8 0.539228 0.695689 0.853000 -Total time: 11:29 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_25.m -Loss and accuracy using (cls_best): [0.61637276, tensor(0.8630)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_25.m', - 0.8629999756813049)]) -Noise: 30 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_30.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 2837 examples, only 0.7000422922393741 have correct labels -Data lm, trn: 13013, val: 1445 -Running tokenization clsnoise0.3... -Data clsnoise0.3, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_30.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.908243 0.480479 0.818000 -2 0.906832 0.562728 0.773000 -3 0.896891 3.017841 0.750000 -4 0.900984 2.464900 0.892000 -5 0.874264 2.142747 0.833000 -6 0.775662 0.491934 0.876000 -7 0.614158 0.568567 0.840000 -8 0.553789 3.695555 0.797000 -Total time: 11:32 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_30.m -Loss and accuracy using (cls_best): [2.999626, tensor(0.7897)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_30.m', - 0.7897499799728394)]) -Noise: 35 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_35.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 3310 examples, only 0.6500317191795305 have correct labels -Data lm, trn: 13013, val: 1445 -Running tokenization clsnoise0.35... -Data clsnoise0.35, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_35.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.983643 0.519147 0.891000 -2 1.010607 0.945973 0.580000 -3 0.994139 3.134296 0.461000 -4 0.984234 1.258144 0.864000 -5 0.923507 9.997900 0.759000 -6 0.886210 9.016973 0.667000 -7 0.791056 7.744195 0.798000 -8 0.683184 4.557703 0.787000 -Total time: 11:28 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_35.m -Loss and accuracy using (cls_best): [4.7776446, tensor(0.7642)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_35.m', - 0.7642499804496765)]) -Noise: 40 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_40.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 3783 examples, only 0.600021146119687 have correct labels -Data lm, trn: 13013, val: 1445 -Running tokenization clsnoise0.4... -Data clsnoise0.4, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_40.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.029997 0.882142 0.651000 -2 1.044314 1.591835 0.618000 -3 1.054599 0.522415 0.862000 -4 1.033360 0.998230 0.747000 -5 1.002742 3.083807 0.803000 -6 0.951022 11.693688 0.741000 -7 0.859333 4.469426 0.674000 -8 0.770237 1.737012 0.712000 -Total time: 11:26 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_40.m -Loss and accuracy using (cls_best): [1.721135, tensor(0.7135)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_40.m', - 0.7135000228881836)]) -Noise: 45 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_45.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 4256 examples, only 0.5500105730598435 have correct labels -Data lm, trn: 13013, val: 1445 -Running tokenization clsnoise0.45... -Data clsnoise0.45, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_45.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.118304 0.964211 0.720000 -2 1.105983 1.031464 0.459000 -3 1.090640 1.993658 0.545000 -4 1.094765 2.314941 0.550000 -5 1.075982 8.962537 0.682000 -6 1.044638 4.634834 0.743000 -7 0.997964 1.342650 0.711000 -8 0.953207 1.860142 0.728000 -Total time: 11:14 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_45.m -Loss and accuracy using (cls_best): [2.065772, tensor(0.7682)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_45.m', - 0.7682499885559082)]) -Noise: 50 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_50.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 4729 examples, only 0.5 have correct labels -Data lm, trn: 13013, val: 1445 -Running tokenization clsnoise0.5... -Data clsnoise0.5, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_50.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.131220 0.884647 0.677000 -2 1.118103 0.786608 0.760000 -3 1.135599 1.448700 0.654000 -4 1.131183 1.091670 0.655000 -5 1.087481 1.086206 0.755000 -6 1.071505 47.305462 0.752000 -7 1.008586 22.109758 0.724000 -8 0.978073 8.738451 0.704000 -Total time: 11:20 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_50.m -Loss and accuracy using (cls_best): [7.9533467, tensor(0.6925)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_50.m', - 0.6924999952316284)]) -Noise: 55 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_55.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 5201 examples, only 0.4500951575385917 have correct labels -Data lm, trn: 13013, val: 1445 -Running tokenization clsnoise0.55... -Data clsnoise0.55, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_55.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.161361 0.962813 0.596000 -2 1.150018 1.615359 0.542000 -3 1.154646 5.588564 0.572000 -4 1.145129 1.064145 0.541000 -5 1.138727 0.868817 0.716000 -6 1.092911 3.203760 0.673000 -7 1.027020 2.577705 0.615000 -8 0.962490 2.068990 0.607000 -Total time: 11:23 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_55.m -Loss and accuracy using (cls_best): [1.8640332, tensor(0.6400)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_55.m', - 0.6399999856948853)]) -Noise: 60 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_60.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 5674 examples, only 0.4000845844787482 have correct labels -Data lm, trn: 13013, val: 1445 -Running tokenization clsnoise0.6... -Data clsnoise0.6, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_60.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.184716 1.159816 0.489000 -2 1.162386 1.785291 0.585000 -3 1.189221 6.945411 0.523000 -4 1.169163 30.551344 0.553000 -5 1.143203 96.431221 0.434000 -6 1.113257 18.963854 0.475000 -7 1.083783 8.783161 0.521000 -8 1.064840 7.930975 0.462000 -Total time: 11:40 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_60.m -Loss and accuracy using (cls_best): [8.483881, tensor(0.4535)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_60.m', - 0.45350000262260437)]) -Noise: 65 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_65.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 6147 examples, only 0.35007401141890465 have correct labels -Data lm, trn: 13013, val: 1445 -Running tokenization clsnoise0.65... -Data clsnoise0.65, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_65.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.163443 1.302967 0.355000 -2 1.181459 1.265248 0.188000 -3 1.189486 6.507479 0.466000 -4 1.192368 23.974792 0.525000 -5 1.171117 2.516582 0.473000 -6 1.139664 1.357649 0.398000 -7 1.091899 1.510042 0.437000 -8 1.070406 1.511683 0.392000 -Total time: 11:29 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_65.m -Loss and accuracy using (cls_best): [1.5956299, tensor(0.3887)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_65.m', - 0.38874998688697815)]) -Noise: 70 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_70.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 6620 examples, only 0.30006343835906113 have correct labels -Data lm, trn: 13013, val: 1445 -Running tokenization clsnoise0.7... -Data clsnoise0.7, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_70.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.186880 1.400069 0.164000 -2 1.193253 3.277115 0.258000 -3 1.173004 2.955142 0.184000 -4 1.162564 3.158204 0.105000 -5 1.158516 8.930537 0.178000 -6 1.117573 25.560179 0.178000 -7 1.070933 3.045306 0.183000 -8 1.025959 10.724563 0.192000 -Total time: 11:31 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_70.m -Loss and accuracy using (cls_best): [9.275186, tensor(0.2153)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_70.m', - 0.21525000035762787)]) -Noise: 75 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_75.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/es.dev.csv -Added noise to 7093 examples, only 0.2500528652992176 have correct labels -Data lm, trn: 13013, val: 1445 -Running tokenization clsnoise0.75... -Data clsnoise0.75, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_75.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.173947 1.503179 0.149000 -2 1.155268 3.008709 0.140000 -3 1.166699 4.902722 0.181000 -4 1.160365 4.787449 0.190000 -5 1.138678 3.777317 0.084000 -6 1.071183 3.077753 0.123000 -7 1.000427 2.818838 0.166000 -8 0.941079 3.680126 0.187000 -Total time: 11:42 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10/models/sp15k/qrnn_nl4_75.m -Loss and accuracy using (cls_best): [3.2255151, tensor(0.1877)] -OrderedDict([('data/mldoc/es-10/models/sp15k/qrnn_nl4_75.m', - 0.18774999678134918)]) -{'data/mldoc/es-10/models/sp15k/qrnn_nl4_0.m': 0.9465000033378601, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_5.m': 0.9302499890327454, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_10.m': 0.9257500171661377, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_15.m': 0.8332499861717224, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_20.m': 0.859499990940094, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_25.m': 0.8629999756813049, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_30.m': 0.7897499799728394, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_35.m': 0.7642499804496765, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_40.m': 0.7135000228881836, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_45.m': 0.7682499885559082, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_50.m': 0.6924999952316284, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_55.m': 0.6399999856948853, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_60.m': 0.45350000262260437, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_65.m': 0.38874998688697815, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_70.m': 0.21525000035762787, 'data/mldoc/es-10/models/sp15k/qrnn_nl4_75.m': 0.18774999678134918} -```` \ No newline at end of file diff --git a/results/logs/mldoc/noise/random.md b/results/logs/mldoc/noise/random.md deleted file mode 100644 index 6b63d78..0000000 --- a/results/logs/mldoc/noise/random.md +++ /dev/null @@ -1,627 +0,0 @@ -## Debugging random init -```` - warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4_rnd2_0.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -```` - - - - - -## first attempt at random init -``` - python -m ulmfit eval_noise_resistance --lang=de --size=10 --prefix-name="_rnd_" --model="sp15k/qrnn_rnd-nl4.m" --label-smoothing-eps=0.1 -Noise: 0 -Processing data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_0.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: - - !! WARNING !! - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Your compiler (c++) may be ABI-incompatible with PyTorch! -Please use a compiler that is ABI-compatible with GCC 4.9 and above. -See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. - -See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 -for instructions on how to install GCC 4.9 or higher. -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - !! WARNING !! - - warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_0.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.787174 0.985413 0.701000 -2 0.679534 0.697764 0.875000 -3 0.630987 7.125103 0.873000 -4 0.588259 0.653497 0.915000 -5 0.568135 0.641379 0.942000 -6 0.529713 0.557198 0.948000 -7 0.500168 0.538946 0.958000 -8 0.505462 0.550917 0.954000 -Total time: 19:37 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_0.m -Loss and accuracy using (cls_best): [0.21539633, tensor(0.9613)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_0.m', - 0.9612500071525574)]) -Noise: 5 -Processing data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_5.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 500 examples, only 0.95 have correct labels -Added noise to 50 examples, only 0.95 have correct labels -Data lm, trn: 13500, val: 1500 -Data clsnoise0.05tv, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_5.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.917813 0.874308 0.790000 -2 0.821460 1.239760 0.671000 -3 0.747909 2.624352 0.667000 -4 0.713649 0.735327 0.893000 -5 0.689947 1.175884 0.844000 -6 0.639073 1.066042 0.862000 -7 0.617660 0.845634 0.875000 -8 0.620402 0.670972 0.901000 -Total time: 19:28 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_5.m -Loss and accuracy using (cls_best): [0.25263783, tensor(0.9560)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_5.m', - 0.9559999704360962)]) -Noise: 10 -Processing data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_10.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 1000 examples, only 0.9 have correct labels -Added noise to 100 examples, only 0.9 have correct labels -Data lm, trn: 13500, val: 1500 -Data clsnoise0.1tv, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_10.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.976570 1.054016 0.705000 -2 0.885775 0.813666 0.835000 -3 0.861244 0.968860 0.762000 -4 0.790501 0.815453 0.839000 -5 0.754292 0.805088 0.849000 -6 0.742595 0.770547 0.864000 -7 0.712961 0.771171 0.863000 -8 0.695449 0.787870 0.858000 -Total time: 19:48 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_10.m -Loss and accuracy using (cls_best): [0.2744636, tensor(0.9510)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_10.m', - 0.9509999752044678)]) -Noise: 15 -Processing data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_15.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 1500 examples, only 0.85 have correct labels -Added noise to 150 examples, only 0.85 have correct labels -Data lm, trn: 13500, val: 1500 -Data clsnoise0.15tv, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_15.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.081478 1.075213 0.674000 -2 0.989475 0.939438 0.779000 -3 0.963180 0.984908 0.723000 -4 0.915556 1.209332 0.662000 -5 0.884642 1.000015 0.786000 -6 0.844702 0.884871 0.794000 -7 0.793699 0.882503 0.802000 -8 0.797470 0.871922 0.802000 -Total time: 19:50 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_15.m -Loss and accuracy using (cls_best): [0.32492134, tensor(0.9445)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_15.m', - 0.9445000290870667)]) -Noise: 20 -Processing data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_20.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 2000 examples, only 0.8 have correct labels -Added noise to 200 examples, only 0.8 have correct labels -Data lm, trn: 13500, val: 1500 -Data clsnoise0.2tv, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_20.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.130177 1.651658 0.424000 -2 1.049187 1.508039 0.286000 -3 1.045260 1.976680 0.578000 -4 0.971859 1.121615 0.735000 -5 0.965327 2.376971 0.684000 -6 0.901961 1.089674 0.744000 -7 0.868971 1.082978 0.750000 -8 0.845376 1.019824 0.740000 -Total time: 19:51 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_20.m -Loss and accuracy using (cls_best): [0.46514454, tensor(0.9438)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_20.m', - 0.9437500238418579)]) -Noise: 25 -Processing data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_25.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 2500 examples, only 0.75 have correct labels -Added noise to 250 examples, only 0.75 have correct labels -Data lm, trn: 13500, val: 1500 -Data clsnoise0.25tv, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_25.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.204033 1.368233 0.500000 -2 1.121006 1.219437 0.586000 -3 1.057657 1.139297 0.659000 -4 1.054685 1.043641 0.700000 -5 1.023957 1.069890 0.706000 -6 0.992645 1.073037 0.708000 -7 0.948602 1.054931 0.699000 -8 0.945395 1.078187 0.703000 -Total time: 20:09 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_25.m -Loss and accuracy using (cls_best): [0.4676742, tensor(0.9137)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_25.m', - 0.9137499928474426)]) -Noise: 30 -Processing data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_30.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 3000 examples, only 0.7 have correct labels -Added noise to 300 examples, only 0.7 have correct labels -Data lm, trn: 13500, val: 1500 -Data clsnoise0.3tv, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_30.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.245468 1.243019 0.492000 -2 1.192702 1.644169 0.435000 -3 1.187665 4.143492 0.490000 -4 1.113116 20.139246 0.540000 -5 1.092624 1.189916 0.609000 -6 1.052626 1.264737 0.617000 -7 1.032403 1.317357 0.649000 -8 1.003000 1.187038 0.653000 -Total time: 20:02 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_30.m -Loss and accuracy using (cls_best): [0.5831716, tensor(0.9215)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_30.m', - 0.921500027179718)]) -Noise: 35 -Processing data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_35.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 3500 examples, only 0.65 have correct labels -Added noise to 350 examples, only 0.65 have correct labels -Data lm, trn: 13500, val: 1500 -Data clsnoise0.35tv, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_35.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.285933 1.719733 0.309000 -2 1.258459 1.465174 0.422000 -3 1.240111 1.205106 0.512000 -4 1.195793 2.153573 0.571000 -5 1.150691 3.427428 0.588000 -6 1.115649 1.933489 0.601000 -7 1.078265 1.214095 0.599000 -8 1.045297 1.140148 0.604000 -Total time: 19:55 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_35.m -Loss and accuracy using (cls_best): [0.5903087, tensor(0.9105)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_35.m', - 0.9104999899864197)]) -Noise: 40 -Processing data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_40.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 4000 examples, only 0.6 have correct labels -Added noise to 400 examples, only 0.6 have correct labels -Data lm, trn: 13500, val: 1500 -Data clsnoise0.4tv, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_40.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.313393 1.523955 0.316000 -2 1.303059 1.964390 0.418000 -3 1.291624 1.550615 0.458000 -4 1.263588 2.995128 0.390000 -5 1.206715 1.265662 0.524000 -6 1.191890 1.221754 0.536000 -7 1.162122 1.223106 0.527000 -8 1.150922 1.240103 0.531000 -Total time: 19:53 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_40.m -Loss and accuracy using (cls_best): [0.7210464, tensor(0.8583)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_40.m', - 0.8582500219345093)]) -Noise: 45 -Processing data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_45.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 4500 examples, only 0.55 have correct labels -Added noise to 450 examples, only 0.55 have correct labels -Data lm, trn: 13500, val: 1500 -Data clsnoise0.45tv, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_45.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.345056 1.343081 0.378000 -2 1.281120 11.283777 0.232000 -3 1.284114 14.679921 0.390000 -4 1.267963 2.869378 0.485000 -5 1.227434 1.466781 0.490000 -6 1.209261 1.634938 0.495000 -7 1.170042 1.372811 0.494000 -8 1.162168 2.157310 0.492000 -Total time: 20:05 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_45.m -Loss and accuracy using (cls_best): [1.0457553, tensor(0.8635)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_45.m', - 0.8634999990463257)]) -Noise: 50 -Processing data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_50.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 5000 examples, only 0.5 have correct labels -Added noise to 500 examples, only 0.5 have correct labels -Data lm, trn: 13500, val: 1500 -Data clsnoise0.5tv, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_50.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.362338 1.361343 0.339000 -2 1.343794 1.358407 0.328000 -3 1.326264 3.336083 0.325000 -4 1.321352 4.200035 0.254000 -5 1.289333 1.363007 0.408000 -6 1.275341 1.449265 0.405000 -7 1.245595 1.358157 0.423000 -8 1.234815 1.346797 0.411000 -Total time: 19:35 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_50.m -Loss and accuracy using (cls_best): [1.3260584, tensor(0.7103)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_50.m', - 0.7102500200271606)]) -Noise: 55 -Processing data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_55.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 5500 examples, only 0.45 have correct labels -Added noise to 550 examples, only 0.45 have correct labels -Data lm, trn: 13500, val: 1500 -Data clsnoise0.55tv, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_55.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.373838 1.385533 0.265000 -2 1.355375 2.033619 0.316000 -3 1.358652 2.010394 0.260000 -4 1.337999 7.118755 0.351000 -5 1.309082 3.053319 0.361000 -6 1.286589 19.251106 0.359000 -7 1.276432 1.328096 0.379000 -8 1.266364 1.324883 0.378000 -Total time: 19:34 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_55.m -Loss and accuracy using (cls_best): [1.3107486, tensor(0.6503)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_55.m', - 0.6502500176429749)]) -Noise: 60 -Processing data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_60.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 6000 examples, only 0.4 have correct labels -Added noise to 600 examples, only 0.4 have correct labels -Data lm, trn: 13500, val: 1500 -Data clsnoise0.6tv, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_60.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.378700 1.377784 0.309000 -2 1.359247 9.472390 0.250000 -3 1.343403 1.714557 0.321000 -4 1.336044 1.331355 0.357000 -5 1.322668 1.450317 0.332000 -6 1.283835 2.692688 0.349000 -7 1.261502 1.541230 0.335000 -8 1.230086 1.839382 0.340000 -Total time: 19:58 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_60.m -Loss and accuracy using (cls_best): [1.1523782, tensor(0.5580)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_60.m', - 0.5580000281333923)]) -Noise: 65 -Processing data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_65.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 6500 examples, only 0.35 have correct labels -Added noise to 650 examples, only 0.35 have correct labels -Data lm, trn: 13500, val: 1500 -Data clsnoise0.65tv, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_65.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.374414 1.361766 0.327000 -2 1.367130 1.353700 0.341000 -3 1.365781 1.421649 0.269000 -4 1.358339 1.385666 0.280000 -5 1.357855 3.068685 0.334000 -6 1.343958 1.586822 0.316000 -7 1.330202 2.436025 0.324000 -8 1.322320 1.743209 0.330000 -Total time: 19:52 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_65.m -Loss and accuracy using (cls_best): [1.7415464, tensor(0.4467)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_65.m', - 0.4467499852180481)]) -Noise: 70 -Processing data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_70.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 7000 examples, only 0.3 have correct labels -Added noise to 700 examples, only 0.3 have correct labels -Data lm, trn: 13500, val: 1500 -Data clsnoise0.7tv, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_70.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.386991 1.377855 0.302000 -2 1.370086 1.741303 0.298000 -3 1.371910 1.402328 0.316000 -4 1.349717 1.378567 0.277000 -5 1.360438 1.471136 0.298000 -6 1.345680 1.395034 0.312000 -7 1.327264 1.611867 0.312000 -8 1.327243 1.657344 0.312000 -Total time: 20:09 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_70.m -Loss and accuracy using (cls_best): [2.7352421, tensor(0.2693)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_70.m', - 0.2692500054836273)]) -Noise: 75 -Processing data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m -de-10 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_75.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/de.dev.csv -Added noise to 7500 examples, only 0.25 have correct labels -Added noise to 750 examples, only 0.25 have correct labels -Data lm, trn: 13500, val: 1500 -Data clsnoise0.75tv, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_75.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.376097 1.392457 0.263000 -2 1.372446 1.367712 0.320000 -3 1.356304 1.354679 0.297000 -4 1.342981 1.350475 0.335000 -5 1.340915 1.337473 0.343000 -6 1.320882 1.904698 0.357000 -7 1.291946 1.368179 0.339000 -8 1.283206 1.466608 0.353000 -Total time: 19:50 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_75.m -Loss and accuracy using (cls_best): [1.4704828, tensor(0.1248)] -OrderedDict([('data/mldoc/de-10/models/sp15k/qrnn_nl4__rnd_75.m', - 0.12475000321865082)]) - noise accuracy -0 0.00 0.96125 -1 0.05 0.95600 -2 0.10 0.95100 -3 0.15 0.94450 -4 0.20 0.94375 -5 0.25 0.91375 -6 0.30 0.92150 -7 0.35 0.91050 -8 0.40 0.85825 -9 0.45 0.86350 -10 0.50 0.71025 -11 0.55 0.65025 -12 0.60 0.55800 -13 0.65 0.44675 -14 0.70 0.26925 -15 0.75 0.12475 -``` \ No newline at end of file diff --git a/results/logs/mldoc/random-init.md b/results/logs/mldoc/random-init.md deleted file mode 100644 index f5805fd..0000000 --- a/results/logs/mldoc/random-init.md +++ /dev/null @@ -1,485 +0,0 @@ - - -### MLDoc laser zero shoot 10k -data/mldoc/de-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m: 0.9052500128746033 -data/mldoc/es-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m: 0.6974999904632568 -data/mldoc/fr-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m: 0.8740000128746033 -data/mldoc/it-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m: 0.7272499799728394 -data/mldoc/zh-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m: 0.8144999742507935 - -``` -python -m ulmfit eval --glob="mldoc/*-1/models/sp15k/qrnn_nl4.m" --dataset_template='${lang}-10-laser-en1' --name rnd_nl4 --num-cls-epochs=8 --label-smoothing-eps=0.1 --lr_sched=1cycle --bs=18 --random-init=True -python -m ulmfit eval --glob="mldoc/*-1/models/sp15k/qrnn_nl4.m" --dataset_template='${lang}-10-laser-en1' --name rnd_nl4 --num-cls-epochs=8 --label-smoothing-eps=0.1 --lr_sched=1cycle --bs=18 --random-init=True -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-10-laser-en1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10-laser-en1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10-laser-en1/de.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: - - !! WARNING !! - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Your compiler (c++) may be ABI-incompatible with PyTorch! -Please use a compiler that is ABI-compatible with GCC 4.9 and above. -See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. - -See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 -for instructions on how to install GCC 4.9 or higher. -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - !! WARNING !! - - warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 0.864752 0.760891 0.844000 -2 0.734504 1.077554 0.676000 -3 0.681645 0.703327 0.885000 -4 0.670696 0.779010 0.898000 -5 0.620256 0.664871 0.910000 -6 0.591837 1.077103 0.915000 -7 0.550238 0.607863 0.913000 -8 0.543874 0.607274 0.918000 -Total time: 19:55 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m -Loss and accuracy using (cls_best): [0.35624045, tensor(0.9053)] -Processing data/mldoc/en-1/models/sp15k/qrnn_nl4.m -en-10-laser-en1 -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-10-laser-en1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10-laser-en1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10-laser-en1/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 9458, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 0.755512 1.360725 0.500000 -2 0.760655 1.362604 0.399000 -3 0.760876 20.748863 0.607000 -4 0.730208 8.120344 0.369000 -5 0.707735 1.149775 0.700000 -6 0.679102 1.010318 0.746000 -7 0.639611 3.087066 0.713000 -8 0.608591 1.327793 0.750000 -Total time: 11:38 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m -Loss and accuracy using (cls_best): [1.3680531, tensor(0.6975)] -Processing data/mldoc/fr-1/models/sp15k/qrnn_nl4.m -fr-10-laser-en1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-10-laser-en1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-10-laser-en1/fr.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', 's', '▁.', "'", '▁la', '▁le', '▁et', '▁l', '▁à'] -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 0.922996 1.406875 0.519000 -2 0.833284 1.172545 0.640000 -3 0.749922 0.697733 0.863000 -4 0.724680 0.735842 0.837000 -5 0.652541 0.679455 0.876000 -6 0.641541 0.671731 0.868000 -7 0.577571 0.734958 0.868000 -8 0.579186 0.703696 0.883000 -Total time: 19:15 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m -Loss and accuracy using (cls_best): [0.47713563, tensor(0.8740)] -Processing data/mldoc/it-1/models/sp15k/qrnn_nl4.m -it-10-laser-en1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-10-laser-en1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-10-laser-en1/it.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', '▁e', "▁&'", "'", '▁il', '▁la', '▁in', 'e'] -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 0.991065 1.602189 0.381000 -2 0.935880 0.888808 0.734000 -3 0.860670 0.868564 0.781000 -4 0.818734 0.945302 0.791000 -5 0.751467 3.113552 0.808000 -6 0.687606 0.921033 0.795000 -7 0.677044 1.222023 0.807000 -8 0.645511 1.418593 0.805000 -Total time: 11:44 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m -Loss and accuracy using (cls_best): [1.1276722, tensor(0.7272)] -Processing data/mldoc/ja-1/models/sp15k/qrnn_nl4.m -ja-10-laser-en1 -Processing data/mldoc/zh-1/models/sp15k/qrnn_nl4.m -zh-10-laser-en1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-10-laser-en1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-10-laser-en1/zh.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 10000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] -Starting classifier from random weights -Single training schedule -epoch train_loss valid_loss accuracy -1 0.920411 1.159853 0.629000 -2 0.937020 1.371089 0.527000 -3 0.892036 3.091183 0.615000 -4 0.839919 0.939323 0.724000 -5 0.797184 1.174206 0.735000 -6 0.774195 0.914951 0.733000 -7 0.744524 0.875888 0.762000 -8 0.721782 0.825969 0.788000 -Total time: 19:53 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m -Loss and accuracy using (cls_best): [0.54084456, tensor(0.8145)] -OrderedDict([('data/mldoc/de-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m', - 0.9052500128746033), - ('data/mldoc/es-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m', - 0.6974999904632568), - ('data/mldoc/fr-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m', - 0.8740000128746033), - ('data/mldoc/it-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m', - 0.7272499799728394), - ('data/mldoc/zh-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m', - 0.8144999742507935)]) -data/mldoc/de-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m: 0.9052500128746033 -data/mldoc/es-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m: 0.6974999904632568 -data/mldoc/fr-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m: 0.8740000128746033 -data/mldoc/it-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m: 0.7272499799728394 -data/mldoc/zh-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m: 0.8144999742507935 -``` - -### MLDoc Classification on 1k - -data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m: 0.9024999737739563 -data/mldoc/en-1/models/sp15k/qrnn_rnd-nl4.m: 0.8149999976158142 -data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m: 0.8964999914169312 -data/mldoc/fr-1/models/sp15k/qrnn_rnd-nl4.m: 0.8220000267028809 -data/mldoc/it-1/models/sp15k/qrnn_rnd-nl4.m: 0.7889999747276306 -data/mldoc/ja-1/models/sp15k/qrnn_rnd-nl4.m: 0.8302500247955322 -data/mldoc/ru-1/models/sp15k/qrnn_rnd-nl4.m: 0.7319999933242798 -data/mldoc/zh-1/models/sp15k/qrnn_rnd-nl4.m: 0.8452500104904175 - - -``` -python -m ulmfit eval --glob="wiki/*-100/models/sp15k/qrnn_rnd-nl4.m" --name rnd-nl4 --dataset-template='../mldoc/${lang}-1' --num-lm-epochs=0 --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.1 -Processing data/wiki/de-100/models/sp15k/qrnn_rnd-nl4.m -../mldoc/de-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/de.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: - - !! WARNING !! - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Your compiler (c++) may be ABI-incompatible with PyTorch! -Please use a compiler that is ABI-compatible with GCC 4.9 and above. -See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. - -See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 -for instructions on how to install GCC 4.9 or higher. -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - !! WARNING !! - - warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.411651 1.386593 0.265000 -2 1.287022 1.488027 0.361000 -3 1.084153 2.390431 0.370000 -4 0.904227 0.936213 0.769000 -5 0.740495 1.311880 0.538000 -6 0.642756 0.754690 0.833000 -7 0.582816 0.661088 0.892000 -8 0.548893 0.683635 0.875000 -Total time: 02:13 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m -Loss and accuracy using (cls_best): [0.33525154, tensor(0.9025)] -Processing data/wiki/en-100/models/sp15k/qrnn_rnd-nl4.m -../mldoc/en-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k/qrnn_rnd-nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/en.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁the', '▁,', 's', '▁.', '▁of', '▁and', '▁in', '▁to', '▁a', 'ed'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k/qrnn_rnd-nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.369466 1.356636 0.374000 -2 1.263357 2.515120 0.320000 -3 1.119744 1.250081 0.569000 -4 0.949653 1.033515 0.666000 -5 0.802069 0.875799 0.779000 -6 0.676997 0.842525 0.807000 -7 0.613777 0.794573 0.826000 -8 0.571342 0.781615 0.837000 -Total time: 02:26 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k/qrnn_rnd-nl4.m -Loss and accuracy using (cls_best): [0.5295334, tensor(0.8150)] -Processing data/wiki/es-100/models/sp15k/qrnn_rnd-nl4.m -../mldoc/es-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv -Data lm, trn: 13013, val: 1445 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 2621, first 100: ['▁sa', '▁i', 'ncia', '▁ka', '▁k', '▁tra', '▁fi', '▁volvi', '▁g', '▁man', '▁pasó', '▁tropas', 'pon', 'tuvieron', '▁x', '▁les', '▁empez', 'ieron', '▁bas', 'sco', '▁cam', '▁adapta', 'sion', '▁mol', 'pico', 'siones', '▁obstante', '▁!', '▁w', 'cular', 'puesta', '▁inten', '▁produj', 'clu', 'simismo', '▁pas', 'fla', '▁amerindio', 'aje', '▁deja', '▁fre', '▁jo', '▁2.', 'american', '▁cre', 'bajo', '▁medi', 'gla', '▁dirigi', 'hol', '▁aparición', 'aciones', 'vivi', 'eras', 'spe', '▁continu', '▁permaneci', '▁ber', 'usa', 'bió', '▁permitió', '▁municipios', '▁regres', 'rt', 'mbi', '▁pr', '▁ofreci', 'emi', 'misiones', '▁cap', '▁ram', 'icio', '▁wal', 'fru', '▁gen', '▁originalmente', '▁eva', '▁ferr', '▁descubri', '▁aparecen', '▁fon', 'capi', 'estre', 'pec', '▁vendi', 'iéndose', 'eja', 'liber', 'nsa', 'ológico', 'ío', 'blo', '▁tro', '▁aviones', 'cara', '▁activo', 'mostró', 'disciplina', '▁ara', 'estra'] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.378275 1.354129 0.314000 -2 1.209560 1.333649 0.507000 -3 1.022200 0.820093 0.801000 -4 0.854187 1.782254 0.389000 -5 0.722861 1.031932 0.692000 -6 0.640640 0.762994 0.853000 -7 0.583225 0.677089 0.901000 -8 0.556481 0.652575 0.904000 -Total time: 01:59 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m -Loss and accuracy using (cls_best): [0.35487488, tensor(0.8965)] -Processing data/wiki/fr-100/models/sp15k/qrnn_rnd-nl4.m -../mldoc/fr-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_rnd-nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/fr.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', 's', '▁.', "'", '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_rnd-nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.360408 1.298057 0.444000 -2 1.251207 2.454086 0.393000 -3 1.098488 1.152682 0.544000 -4 0.926239 1.256870 0.622000 -5 0.806994 0.911339 0.732000 -6 0.717139 0.945148 0.726000 -7 0.635195 0.781772 0.825000 -8 0.602214 0.763521 0.825000 -Total time: 02:17 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_rnd-nl4.m -Loss and accuracy using (cls_best): [0.52210134, tensor(0.8220)] -Processing data/wiki/it-100/models/sp15k/qrnn_rnd-nl4.m -../mldoc/it-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k/qrnn_rnd-nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/it.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', '▁e', "▁&'", "'", '▁il', '▁la', '▁in', 'e'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k/qrnn_rnd-nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.357973 1.353393 0.293000 -2 1.282061 1.535401 0.390000 -3 1.144333 1.480346 0.533000 -4 0.985930 1.360542 0.540000 -5 0.862014 1.285450 0.661000 -6 0.720891 1.140574 0.629000 -7 0.625376 0.840085 0.791000 -8 0.572435 0.828283 0.793000 -Total time: 01:21 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k/qrnn_rnd-nl4.m -Loss and accuracy using (cls_best): [0.5931235, tensor(0.7890)] -Processing data/wiki/ja-100/models/sp15k/qrnn_rnd-nl4.m -../mldoc/ja-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k/qrnn_rnd-nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k/qrnn_rnd-nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.377756 1.402221 0.254000 -2 1.243837 7.998792 0.254000 -3 1.066383 2.645358 0.354000 -4 0.903686 1.348676 0.541000 -5 0.843216 0.945152 0.743000 -6 0.759674 0.801283 0.810000 -7 0.689767 0.786832 0.820000 -8 0.674777 0.778615 0.818000 -Total time: 02:48 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k/qrnn_rnd-nl4.m -Loss and accuracy using (cls_best): [0.5008588, tensor(0.8303)] -Processing data/wiki/ru-100/models/sp15k/qrnn_rnd-nl4.m -../mldoc/ru-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_rnd-nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv -Running tokenization lm... -Data lm, trn: 9195, val: 1021 -Running tokenization cls... -Data cls, trn: 1000, val: 1000 -Running tokenization tst... -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', '▁на', 'х'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_rnd-nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.394592 1.393761 0.265000 -2 1.381886 1.476836 0.293000 -3 1.258016 1.153065 0.555000 -4 1.105392 1.323574 0.556000 -5 0.948704 1.049486 0.703000 -6 0.848964 1.480141 0.605000 -7 0.757975 1.001765 0.723000 -8 0.684587 0.982136 0.741000 -Total time: 03:07 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_rnd-nl4.m -Loss and accuracy using (cls_best): [0.7138667, tensor(0.7320)] -Processing data/wiki/zh-100/models/sp15k/qrnn_rnd-nl4.m -../mldoc/zh-1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_rnd-nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv -Data lm, trn: 13500, val: 1500 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_rnd-nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.341714 1.208123 0.569000 -2 1.059330 1.169385 0.662000 -3 0.926222 0.771242 0.824000 -4 0.843994 1.997928 0.524000 -5 0.800537 0.874480 0.756000 -6 0.710552 0.909481 0.758000 -7 0.657595 0.719883 0.854000 -8 0.617662 0.727267 0.852000 -Total time: 02:20 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_rnd-nl4.m -Loss and accuracy using (cls_best): [0.48266637, tensor(0.8453)] -OrderedDict([('data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m', - 0.9024999737739563), - ('data/mldoc/en-1/models/sp15k/qrnn_rnd-nl4.m', - 0.8149999976158142), - ('data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m', - 0.8964999914169312), - ('data/mldoc/fr-1/models/sp15k/qrnn_rnd-nl4.m', - 0.8220000267028809), - ('data/mldoc/it-1/models/sp15k/qrnn_rnd-nl4.m', - 0.7889999747276306), - ('data/mldoc/ja-1/models/sp15k/qrnn_rnd-nl4.m', - 0.8302500247955322), - ('data/mldoc/ru-1/models/sp15k/qrnn_rnd-nl4.m', - 0.7319999933242798), - ('data/mldoc/zh-1/models/sp15k/qrnn_rnd-nl4.m', - 0.8452500104904175)]) -data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m: 0.9024999737739563 -data/mldoc/en-1/models/sp15k/qrnn_rnd-nl4.m: 0.8149999976158142 -data/mldoc/es-1/models/sp15k/qrnn_rnd-nl4.m: 0.8964999914169312 -data/mldoc/fr-1/models/sp15k/qrnn_rnd-nl4.m: 0.8220000267028809 -data/mldoc/it-1/models/sp15k/qrnn_rnd-nl4.m: 0.7889999747276306 -data/mldoc/ja-1/models/sp15k/qrnn_rnd-nl4.m: 0.8302500247955322 -data/mldoc/ru-1/models/sp15k/qrnn_rnd-nl4.m: 0.7319999933242798 -data/mldoc/zh-1/models/sp15k/qrnn_rnd-nl4.m: 0.8452500104904175 -``` \ No newline at end of file diff --git a/results/logs/new-runs/qrnn-de-1152.md b/results/logs/new-runs/qrnn-de-1152.md deleted file mode 100644 index 639775d..0000000 --- a/results/logs/new-runs/qrnn-de-1152.md +++ /dev/null @@ -1,31 +0,0 @@ -``` -LANG=de -python -m multifit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='fsp' --nl 4 --name '1152' --max-vocab 30000 --lang ${LANG} --qrnn=True --lmseed=1 --nh=1152 - train 10 --bs=50 --drop_mult=0 --label-smoothing-eps=0.1 -Training lm -Max vocab: 30000 -Cache dir: data/wiki/de-100/models/fsp30k -Model dir: data/wiki/de-100/models/fsp30k/qrnn_1152_lmseed-1.m -Setting LM seed to 1 -Wiki text was split to 191112 articles -Wiki text was split to 431 articles -Data lm, trn: 191112, val: 431 -Size of vocabulary: 30000 -First 20 words in vocab: ['▁xxunk', '▁xxpad', '▁xxbos', '▁xxeos', '▁xxfld', '▁xxmaj', '▁xxup', '▁xxrep', '▁xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', "▁&'", 's', '-'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} config: {'emb_sz': 400, 'n_hid': 1152, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.1, 'hidden_p': 0.15, 'input_p': 0.25, 'embed_p': 0.02, 'weight_p': 0.2, 'tie_weights': True, 'out_bias': True} -Bptt 70 -Training lm from random weights -epoch train_loss valid_loss accuracy time -0 4.452324 4.517938 0.397063 1:12:10 -1 4.354969 4.473063 0.399108 1:12:08 -2 4.375927 4.460905 0.400854 1:11:47 -3 4.279962 4.406254 0.406279 1:11:47 -4 4.229251 4.358620 0.413019 1:11:51 -5 4.194514 4.380852 0.409330 1:11:54 -6 4.151179 4.204870 0.431857 1:11:57 -7 4.085239 4.131142 0.442781 1:12:28 -8 4.026665 4.080124 0.451819 1:13:08 -9 4.015738 4.063066 0.454933 1:13:28 -Total time: 12:02:42 -data/wiki/de-100/models/fsp30k -Saving info data/wiki/de-100/models/fsp30k/qrnn_1152_lmseed-1.m/info.json -``` \ No newline at end of file diff --git a/results/logs/new-runs/qrnn-de.md b/results/logs/new-runs/qrnn-de.md deleted file mode 100644 index 0080691..0000000 --- a/results/logs/new-runs/qrnn-de.md +++ /dev/null @@ -1,32 +0,0 @@ -``` -LANG=de -python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='fsp' --nl 4 --name 'nl4' --max-vocab 30000 --lang ${LANG} --qrnn=True --lmseed=1 --nh=1552 - train 10 --bs=50 --drop_mult=0 --label-smoothing-eps=0.1 -Training lm -Max vocab: 30000 -Cache dir: data/wiki/de-100/models/fsp30k -Model dir: data/wiki/de-100/models/fsp30k/qrnn_nl4_lmseed-1.m -Setting LM seed to 1 -Wiki text was split to 191112 articles -Wiki text was split to 431 articles -Data lm, trn: 191112, val: 431 -Size of vocabulary: 30000 -First 20 words in vocab: ['▁xxunk', '▁xxpad', '▁xxbos', '▁xxeos', '▁xxfld', '▁xxmaj', '▁xxup', '▁xxrep', '▁xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', "▁&'", 's', '-'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'input_p': 0.25, 'output_p': 0.1, 'weight_p': 0.2, 'embed_p': 0.02, 'hidden_p': 0.15} -Bptt 70 -Training lm from random weights -epoch train_loss valid_loss accuracy time -0 4.403168 4.469939 0.403841 1:30:29 -1 4.312332 4.432437 0.404200 1:30:30 -2 4.334051 4.423142 0.405613 1:30:03 -3 4.239668 4.376138 0.411162 1:30:05 -4 4.193250 4.324453 0.416959 1:30:04 -5 4.151386 4.248287 0.427130 1:30:16 -6 4.103295 4.160756 0.438965 1:30:32 -7 4.033971 4.086159 0.450292 1:30:16 -8 3.967596 4.029943 0.459819 1:31:20 -9 3.954203 4.013039 0.463228 1:31:14 -Total time: 15:04:52 -data/wiki/de-100/models/fsp30k -Saving info data/wiki/de-100/models/fsp30k/qrnn_nl4_lmseed-1.m/info.json -``` -------- diff --git a/results/logs/new-runs/qrnn-ja-1152.md b/results/logs/new-runs/qrnn-ja-1152.md deleted file mode 100644 index faf9ea8..0000000 --- a/results/logs/new-runs/qrnn-ja-1152.md +++ /dev/null @@ -1,139 +0,0 @@ -```` -LANG=ja ✘ 130 -python -m multifit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='fsp' --nl 4 --name 'nl4-1152' --max-vocab 15000 --lang ${LANG} --qrnn=True --lmseed=1 --nh=1152 - train 10 --bs=50 --drop_mult=0 --label-smoothing-eps=0.1 -Training lm -Max vocab: 15000 -Cache dir: data/wiki/ja-100/models/fsp15k -Model dir: data/wiki/ja-100/models/fsp15k/qrnn_nl4-1152_lmseed-1.m -Setting LM seed to 1 -Wiki text was split to 120037 articles -Wiki text was split to 63 articles -Running tokenization lm... -sentencepiece_trainer.cc(116) LOG(INFO) Running command: --input=/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/fsp15k/all_text.out --max_sentence_length=20480 --character_coverage=0.9998 --unk_id=9 --pad_id=-1 --bos_id=-1 --eos_id=-1 --user_defined_symbols=▁xxunk,▁xxpad,▁xxbos,▁xxeos,▁xxfld,▁xxmaj,▁xxup,▁xxrep,▁xxwrep --model_prefix=/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/fsp15k/spm --vocab_size=15000 --model_type=unigram -sentencepiece_trainer.cc(49) LOG(INFO) Starts training with : -TrainerSpec { - input: /home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/fsp15k/all_text.out - input_format: - model_prefix: /home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/fsp15k/spm - model_type: UNIGRAM - vocab_size: 15000 - self_test_sample_size: 0 - character_coverage: 0.9998 - input_sentence_size: 0 - shuffle_input_sentence: 1 - seed_sentencepiece_size: 1000000 - shrinking_factor: 0.75 - max_sentence_length: 20480 - num_threads: 16 - num_sub_iterations: 2 - max_sentencepiece_length: 16 - split_by_unicode_script: 1 - split_by_number: 1 - split_by_whitespace: 1 - treat_whitespace_as_suffix: 0 - user_defined_symbols: ▁xxunk - user_defined_symbols: ▁xxpad - user_defined_symbols: ▁xxbos - user_defined_symbols: ▁xxeos - user_defined_symbols: ▁xxfld - user_defined_symbols: ▁xxmaj - user_defined_symbols: ▁xxup - user_defined_symbols: ▁xxrep - user_defined_symbols: ▁xxwrep - hard_vocab_limit: 1 - use_all_vocab: 0 - unk_id: 9 - bos_id: -1 - eos_id: -1 - pad_id: -1 - unk_piece: - bos_piece: - eos_piece: - pad_piece: - unk_surface: ⁇ -} -NormalizerSpec { - name: nmt_nfkc - add_dummy_prefix: 1 - remove_extra_whitespaces: 1 - escape_whitespaces: 1 - normalization_rule_tsv: -} - -trainer_interface.cc(267) LOG(INFO) Loading corpus: /home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/fsp15k/all_text.out -trainer_interface.cc(287) LOG(WARNING) Found too long line (74468 > 20480). -trainer_interface.cc(289) LOG(WARNING) Too long lines are skipped in the training. -trainer_interface.cc(290) LOG(WARNING) The maximum length can be changed with --max_sentence_length= flag. -trainer_interface.cc(315) LOG(INFO) Loaded all 115812 sentences -trainer_interface.cc(321) LOG(INFO) Skipped 4225 too long sentences. -trainer_interface.cc(330) LOG(INFO) Adding meta_piece: ▁xxunk -trainer_interface.cc(330) LOG(INFO) Adding meta_piece: ▁xxpad -trainer_interface.cc(330) LOG(INFO) Adding meta_piece: ▁xxbos -trainer_interface.cc(330) LOG(INFO) Adding meta_piece: ▁xxeos -trainer_interface.cc(330) LOG(INFO) Adding meta_piece: ▁xxfld -trainer_interface.cc(330) LOG(INFO) Adding meta_piece: ▁xxmaj -trainer_interface.cc(330) LOG(INFO) Adding meta_piece: ▁xxup -trainer_interface.cc(330) LOG(INFO) Adding meta_piece: ▁xxrep -trainer_interface.cc(330) LOG(INFO) Adding meta_piece: ▁xxwrep -trainer_interface.cc(330) LOG(INFO) Adding meta_piece: -trainer_interface.cc(335) LOG(INFO) Normalizing sentences... -trainer_interface.cc(385) LOG(INFO) all chars count=179924674 -trainer_interface.cc(393) LOG(INFO) Done: 99.98% characters are covered. -trainer_interface.cc(403) LOG(INFO) Alphabet size=4440 -trainer_interface.cc(404) LOG(INFO) Final character coverage=0.9998 -trainer_interface.cc(436) LOG(INFO) Done! preprocessed 115812 sentences. -unigram_model_trainer.cc(129) LOG(INFO) Making suffix array... -unigram_model_trainer.cc(133) LOG(INFO) Extracting frequent sub strings... -unigram_model_trainer.cc(184) LOG(INFO) Initialized 1000000 seed sentencepieces -trainer_interface.cc(442) LOG(INFO) Tokenizing input sentences with whitespace: 115812 -trainer_interface.cc(452) LOG(INFO) Done! 2256013 -unigram_model_trainer.cc(470) LOG(INFO) Using 2256013 sentences for EM training -unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=0 size=589366 obj=9.67447 num_tokens=5565112 num_tokens/piece=9.44254 -unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=1 size=499262 obj=8.54312 num_tokens=5565195 num_tokens/piece=11.1468 -unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=0 size=374209 obj=8.48394 num_tokens=5655360 num_tokens/piece=15.1128 -unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=1 size=373594 obj=8.47578 num_tokens=5655268 num_tokens/piece=15.1375 -unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=0 size=280179 obj=8.57339 num_tokens=5837695 num_tokens/piece=20.8356 -unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=1 size=280142 obj=8.57282 num_tokens=5839303 num_tokens/piece=20.8441 -unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=0 size=210103 obj=8.62784 num_tokens=6061221 num_tokens/piece=28.8488 -unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=1 size=210098 obj=8.62292 num_tokens=6062624 num_tokens/piece=28.8562 -unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=0 size=157573 obj=8.72461 num_tokens=6293760 num_tokens/piece=39.9419 -unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=1 size=157573 obj=8.71413 num_tokens=6295112 num_tokens/piece=39.9504 -unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=0 size=118179 obj=8.87183 num_tokens=6531351 num_tokens/piece=55.2666 -unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=1 size=118178 obj=8.86018 num_tokens=6532542 num_tokens/piece=55.2771 -unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=0 size=88632 obj=9.09891 num_tokens=6780015 num_tokens/piece=76.4962 -unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=1 size=88632 obj=9.08523 num_tokens=6781876 num_tokens/piece=76.5172 -unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=0 size=66474 obj=9.39972 num_tokens=7048669 num_tokens/piece=106.036 -unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=1 size=66474 obj=9.38439 num_tokens=7050941 num_tokens/piece=106.071 -unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=0 size=49855 obj=9.80014 num_tokens=7334836 num_tokens/piece=147.123 -unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=1 size=49855 obj=9.78241 num_tokens=7339902 num_tokens/piece=147.225 -unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=0 size=37391 obj=10.5579 num_tokens=7654233 num_tokens/piece=204.708 -unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=1 size=37391 obj=10.5324 num_tokens=7672187 num_tokens/piece=205.188 -unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=0 size=28043 obj=11.6071 num_tokens=8021953 num_tokens/piece=286.059 -unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=1 size=28043 obj=11.5755 num_tokens=8063782 num_tokens/piece=287.551 -unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=0 size=21032 obj=13.1808 num_tokens=8464260 num_tokens/piece=402.447 -unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=1 size=21032 obj=13.1392 num_tokens=8549719 num_tokens/piece=406.51 -unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=0 size=16500 obj=14.8618 num_tokens=8932725 num_tokens/piece=541.377 -unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=1 size=16500 obj=14.8185 num_tokens=8996465 num_tokens/piece=545.24 -trainer_interface.cc(508) LOG(INFO) Saving model: /home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/fsp15k/spm.model -trainer_interface.cc(532) LOG(INFO) Saving vocabs: /home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/fsp15k/spm.vocab -Data lm, trn: 120037, val: 63 -Size of vocabulary: 15000 -First 20 words in vocab: ['▁xxunk', '▁xxpad', '▁xxbos', '▁xxeos', '▁xxfld', '▁xxmaj', '▁xxup', '▁xxrep', '▁xxwrep', '', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', '▁は', '▁・', '▁('] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} config: {'emb_sz': 400, 'n_hid': 1152, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.1, 'hidden_p': 0.15, 'input_p': 0.25, 'embed_p': 0.02, 'weight_p': 0.2, 'tie_weights': True, 'out_bias': True} -Bptt 70 -Training lm from random weights -epoch train_loss valid_loss accuracy time -0 3.886147 3.948560 0.430394 47:41 -1 3.835572 3.931224 0.429429 47:37 -2 3.830497 3.871733 0.439411 47:25 -3 3.740645 3.824856 0.446847 47:25 -4 3.715881 3.770223 0.455526 47:25 -5 3.698531 3.708276 0.463582 47:25 -6 3.600051 3.639733 0.476275 47:25 -7 3.540591 3.576993 0.487240 47:25 -8 3.487257 3.533188 0.496038 47:25 -9 3.503373 3.515034 0.499575 47:31 -Total time: 7:54:45 -data/wiki/ja-100/models/fsp15k -Saving info data/wiki/ja-100/models/fsp15k/qrnn_nl4-1152_lmseed-1.m/info.json -```` \ No newline at end of file diff --git a/results/logs/new-runs/qrrn-ja.md b/results/logs/new-runs/qrrn-ja.md deleted file mode 100644 index 3cb33cd..0000000 --- a/results/logs/new-runs/qrrn-ja.md +++ /dev/null @@ -1,121 +0,0 @@ -``` -LANG=ja ✘ 130 -python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='fsp' --nl 4 --name 'nl4' --max-vocab 30000 --lang ${LANG} --qrnn=True --lmseed=1 --nh=1552 - train 10 --bs=50 --drop_mult=0 --label-smoothing-eps=0.1 -Training lm -Max vocab: 30000 -Cache dir: data/wiki/ja-100/models/fsp30k -Model dir: data/wiki/ja-100/models/fsp30k/qrnn_nl4_lmseed-1.m -Setting LM seed to 1 -Wiki text was split to 120037 articles -Wiki text was split to 63 articles -Data lm, trn: 120037, val: 63 -Size of vocabulary: 30000 -First 20 words in vocab: ['▁xxunk', '▁xxpad', '▁xxbos', '▁xxeos', '▁xxfld', '▁xxmaj', '▁xxup', '▁xxrep', '▁xxwrep', '', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', '▁は', '▁・', '▁('] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'input_p': 0.25, 'output_p': 0.1, 'weight_p': 0.2, 'embed_p': 0.02, 'hidden_p': 0.15} -Bptt 70 -Training lm from random weights -epoch train_loss valid_loss accuracy time -0 4.346260 4.408598 0.370535 1:12:49 -1 4.253693 4.355113 0.372354 1:12:40 -2 4.190918 4.288729 0.383211 1:12:15 -3 4.148739 4.242265 0.389964 1:12:12 -4 4.136361 4.190885 0.398423 1:12:23 -5 4.051008 4.119476 0.409002 1:12:15 -6 3.966213 4.052222 0.419292 1:12:20 -7 3.928247 3.979634 0.431336 1:12:18 -8 3.840935 3.929402 0.442688 1:12:39 -9 3.909105 3.911067 0.446342 1:12:51 -Total time: 12:04:47 -data/wiki/ja-100/models/fsp30k -Saving info data/wiki/ja-100/models/fsp30k/qrnn_nl4_lmseed-1.m/info.json -``` -------- - -``` -python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/fsp30k/qrnn_nl4_lmseed-1.m --lang=${LANG} --name 'nl4' --clsweightseed=0 - train 20 --bs 20 --lr_sched=1cycle --label-smoothing-eps=0.1 -data/mldoc/ja-1 'data/wiki/ja-100/models/fsp30k/qrnn_nl4_lmseed-1.m' -Training CLS -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/fsp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/fsp30k/qrnn_nl4_lmseed-1-clsweightseed-0.m -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv -Running tokenization lm-notst... -Data lm-notst, trn: 13500, val: 1500 -Running tokenization cls... -Data cls, trn: 1000, val: 1000 -Running tokenization tst... -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 30000 -First 20 words in vocab: ['▁xxunk', '▁xxpad', '▁xxbos', '▁xxeos', '▁xxfld', '▁xxmaj', '▁xxup', '▁xxrep', '▁xxwrep', '', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', '▁は', '▁・', '▁('] -Training lm -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/fsp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/fsp30k/qrnn_nl4_lmseed-1-clsweightseed-0.m -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'input_p': 0.25, 'output_p': 0.1, 'weight_p': 0.2, 'embed_p': 0.02, 'hidden_p': 0.15} -Loading pretrained model -Bptt 70 -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/fsp30k/qrnn_nl4_lmseed-1.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/fsp30k/qrnn_nl4_lmseed-1.m/../itos')] -epoch train_loss valid_loss accuracy time -0 4.921511 4.071093 0.449957 02:49 -Total time: 02:49 -epoch train_loss valid_loss accuracy time -0 4.078950 3.772346 0.490980 03:45 -1 3.727190 3.439734 0.550465 03:47 -2 3.359830 3.201281 0.590776 03:47 -3 3.319000 3.039330 0.618913 03:48 -4 3.127082 2.923124 0.637883 03:49 -5 3.004416 2.842367 0.652876 03:49 -6 2.940764 2.775950 0.664768 03:49 -7 2.961673 2.723808 0.672876 03:48 -8 2.895331 2.680753 0.681351 03:49 -9 2.804141 2.642330 0.688681 03:48 -10 2.867590 2.607323 0.695910 03:49 -11 2.755013 2.570714 0.703812 03:48 -12 2.747998 2.539443 0.710536 03:49 -13 2.710947 2.512321 0.716777 03:49 -14 2.675630 2.486792 0.721944 03:49 -15 2.654088 2.466209 0.726944 03:50 -16 2.668594 2.452018 0.730059 03:49 -17 2.640188 2.444180 0.731973 03:47 -18 2.562034 2.439116 0.732988 03:43 -19 2.641966 2.438401 0.733259 03:42 -Total time: 1:16:04 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/fsp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/fsp30k/qrnn_nl4_lmseed-1-clsweightseed-0.m/info.json -Setting classifier weights seed to 0 -Single training schedule -epoch train_loss valid_loss f_beta precision recall kappa_score matthews_correff accuracy time -/home/pczapla/workspace/_oss/fastai/fastai/fastai/metrics.py:189: UserWarning: average=`binary` was selected for a non binary case. Value for average has now been set to `macro` instead. - warn("average=`binary` was selected for a non binary case. Value for average has now been set to `macro` instead.") -0 0.866883 0.690708 0.826320 0.859738 0.829345 0.770440 0.781947 0.828000 00:13 -Better model found at epoch 0 with f_beta value: 0.826319694519043. -/home/pczapla/workspace/_oss/fastai/fastai/fastai/metrics.py:189: UserWarning: average=`binary` was selected for a non binary case. Value for average has now been set to `macro` instead. - warn("average=`binary` was selected for a non binary case. Value for average has now been set to `macro` instead.") -1 0.676495 0.609408 0.881395 0.888526 0.874378 0.831782 0.835147 0.874000 00:13 -Better model found at epoch 1 with f_beta value: 0.8813954591751099. -Total time: 00:27 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/fsp30k/qrnn_nl4_lmseed-1-clsweightseed-0.m -/home/pczapla/workspace/_oss/fastai/fastai/fastai/metrics.py:189: UserWarning: average=`binary` was selected for a non binary case. Value for average has now been set to `macro` instead. - warn("average=`binary` was selected for a non binary case. Value for average has now been set to `macro` instead.") -Model: nl4 -Evaluation on: test -F1 score bin: 0.8911986351013184 -Loss: 0.32485464215278625 -Precision: 0.8969309329986572 -Recall: 0.8911643028259277 -Accuracy: 0.8914999961853027 -test F1 score bin: 0.8911986351013184 -test Loss: 0.32485464 -test Precision: 0.8969309329986572 -test Recall: 0.8911643028259277 -test Kappa Linear: 0.8553637266159058 -test Matthews Correff: 0.8571781516075134 -test Accuracy: 0.8914999961853027 -``` - ----- -``` -for seed in 1 2 3 4 5 ; do - python -m multifit eval --glob="mldoc/${LANG}-1/models/fsp30k/qrnn_nl4_lmseed-1-clsweightseed-0.m" --name nl4 --clsweightseed=$seed --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.1 -done -``` \ No newline at end of file diff --git a/results/logs/qrnn-de.md b/results/logs/qrnn-de.md deleted file mode 100644 index f033ce1..0000000 --- a/results/logs/qrnn-de.md +++ /dev/null @@ -1,86 +0,0 @@ -# QRNN DE -## SP30k nl -### LM -``` -python -m ulmfit lm --dataset-path data/wiki/de-100 --bidir=False --qrnn=True --nl 4 --tokenizer='sp' --max-vocab 30000 --lang de --name 'nl4' --cuda-id=0 - train 10 --drop-mult=0 --bs=50 - -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', "▁&'", 'en', 's', '-'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Training lm from random weights -epoch train_loss valid_loss accuracy -1 2.790653 2.867094 0.511392 -2 2.742032 2.843288 0.510885 -3 2.696114 2.833874 0.512062 -4 2.671780 2.786312 0.516448 -5 2.611292 2.725993 0.522723 -6 2.542737 2.655713 0.530968 -7 2.572076 2.582141 0.539928 -8 2.465960 2.509654 0.549987 -9 2.405682 2.448580 0.558674 -10 2.339395 2.428111 0.562502 -``` - -### MLDocs -``` -python -m ulmfit cls --dataset-path data/mldoc/de-1 --cuda-id=0 --base-lm-path data-filtered/data/wiki/de-100/models/sp30k/qrnn_nl4.m --lang=de --name 'nl4' - train 20 --bs 40 --cls-max-len 700 - -Max vocab: 30000 -Cache dir: /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/de-1/models/sp30k -Model dir: /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/qrnn_nl4.m -Loading validation /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/de-1/de.dev.csv -Running tokenization... -Saving tokenized: cls.trn 13500, cls.val 1500 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', "▁&'", 'en', 's', '-'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/marcin/github/n-waves/ulmfit-multilingual/data-filtered/data/wiki/de-100/models/sp30k/qrnn_nl4.m/lm_best'), PosixPath('/home/marcin/github/n-waves/ulmfit-multilingual/data-filtered/data/wiki/de-100/models/sp30k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.450698 2.601732 0.527671 -epoch train_loss valid_loss accuracy -1 2.888087 2.477170 0.542949 -2 2.621279 2.300024 0.568743 -3 2.313220 2.120824 0.592728 -4 2.176746 1.973596 0.613343 -5 2.114441 1.857317 0.628628 -6 2.022593 1.765069 0.642017 -7 1.936942 1.696150 0.651549 -8 1.860200 1.622848 0.661923 -9 1.795039 1.549579 0.673416 -10 1.740739 1.500053 0.681305 -11 1.695835 1.448141 0.689201 -12 1.605702 1.402924 0.697096 -13 1.582328 1.354327 0.706123 -14 1.548034 1.316290 0.712870 -15 1.496170 1.282155 0.719413 -16 1.514243 1.255556 0.724801 -17 1.482411 1.236461 0.728380 -18 1.458308 1.223498 0.730708 -19 1.422691 1.218288 0.731713 -20 1.380592 1.217068 0.731893 -/home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/de-1/models/sp30k -Saving info /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/qrnn_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.529402 0.376163 0.900000 -Better model found at epoch 1 with val_loss value: 0.3761630356311798. -epoch train_loss valid_loss accuracy -1 0.290838 0.252989 0.916000 -Better model found at epoch 1 with val_loss value: 0.25298893451690674. -epoch train_loss valid_loss accuracy -1 0.184352 0.204892 0.941000 -Better model found at epoch 1 with val_loss value: 0.20489171147346497. -epoch train_loss valid_loss accuracy -1 0.113328 0.204136 0.947000 -Better model found at epoch 1 with val_loss value: 0.20413607358932495. -2 0.106220 0.200674 0.949000 -Better model found at epoch 2 with val_loss value: 0.20067360997200012. -Saving models at /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.15208693, tensor(0.9532)] -0.15208692848682404 -0.953249990940094 -``` diff --git a/results/logs/qrnn-en.md b/results/logs/qrnn-en.md deleted file mode 100644 index 2b0943b..0000000 --- a/results/logs/qrnn-en.md +++ /dev/null @@ -1,141 +0,0 @@ -# QRNN EN - -## SP15k nl 4 -``` -python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --m -ax-vocab 15000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0 -Max vocab: 15000 -Cache dir: data/wiki/en-100/models/sp15k -Model dir: data/wiki/en-100/models/sp15k/qrnn_nl4.m -Wiki text was split to 28476 articles -Wiki text was split to 60 articles -Data lm, trn: 28476, val: 60 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁the', '▁,', 's', '▁.', '▁of', '▁and', '▁in', '▁to', '▁a', 'ed'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Training lm from random weights -epoch train_loss valid_loss accuracy -1 3.080874 3.197244 0.431796 -2 3.021043 3.147150 0.433593 -3 2.933366 3.125982 0.435766 -4 2.905764 3.103272 0.437356 -5 2.867981 3.032923 0.445030 -6 2.815294 2.958662 0.453979 -7 2.733671 2.869483 0.466015 -8 2.744779 2.785220 0.475833 -9 2.717722 2.704370 0.487687 -10 2.666089 2.675301 0.493602 -Total time: 9:07:27 -data/wiki/en-100/models/sp15k -Saving info data/wiki/en-100/models/sp15k/qrnn_nl4.m/info.json -``` - - - -## SP30k nl 4 -### LM - -``` -python -m ulmfit lm --dataset-path data/wiki/wikitext-103 --bidir=False --qrnn=True --nl 4 --tokenizer='sp' --max-vocab 30000 --lang en --name 'nl4' --cuda-id=1 - train 10 --drop-mult=0 --bs=50 - -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁the', '▁,', '▁.', 's', '▁of', '▁and', '▁in', '▁to', '▁a', 'ed'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.5} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Training lm from random weights -epoch train_loss valid_loss accuracy -1 3.184221 3.256314 0.438527 -2 3.084555 3.241498 0.435628 -3 3.099060 3.258447 0.435060 -4 3.119621 3.220939 0.437597 -5 3.073662 3.165012 0.445108 -6 2.938047 3.086962 0.452921 -7 2.920506 2.998151 0.462940 -8 2.920506 2.899240 0.474378 -9 2.862836 2.835098 0.485305 -10 2.891070 2.810929 0.489867 -``` - -### LM, BS=128, drop-mult=0.5 -``` -python -m ulmfit lm --dataset-path data/wiki/wikitext-103 --bidir=False --qrnn=True --nl 4 --tokenizer='sp' --max-vocab 30000 --lang en --name 'nl4-bs128' --cuda-id=1 - train 10 --drop-mult=0.5 --bs=128 - -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁the', '▁,', '▁.', 's', '▁of', '▁and', '▁in', '▁to', '▁a', 'ed'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.5} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Training lm from random weights -epoch train_loss valid_loss accuracy -1 3.413345 3.280860 0.433011 -2 3.219606 3.129479 0.444172 -3 3.136091 3.094905 0.448493 -4 3.145281 3.033001 0.452830 -5 3.100366 2.980189 0.458984 -6 3.062894 2.923044 0.464841 -7 3.001627 2.834753 0.475316 -8 2.979051 2.792044 0.480915 -9 2.933140 2.733279 0.488346 -10 2.964397 2.720861 0.490423 -``` - -### MLDocs -``` -python -m ulmfit cls --dataset-path data/mldoc/en-1 --cuda-id=0 --base-lm-path data-filtered/data/wiki/wikitext-103/models/sp30k/qrnn_nl4.m --lang=en --name 'nl4' - train 20 --bs 40 --cls-max-len 700 - -Max vocab: 30000 -Cache dir: /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/en-1/models/sp30k -Model dir: /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/en-1/models/sp30k/qrnn_nl4.m -Loading validation /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/en-1/en.dev.csv -Running tokenization... -Saving tokenized: cls.trn 13500, cls.val 1500 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁the', '▁,', '▁.', 's', '▁of', '▁and', '▁in', '▁to', '▁a', 'ed'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/marcin/github/n-waves/ulmfit-multilingual/data-filtered/data/wiki/wikitext-103/models/sp30k/qrnn_nl4.m/lm_best'), PosixPath('/home/marcin/github/n-waves/ulmfit-multilingual/data-filtered/data/wiki/wikitext-103/models/sp30k/qrnn_nl4.m/. -./itos')] -epoch train_loss valid_loss accuracy -1 4.459886 3.692770 0.364677 -epoch train_loss valid_loss accuracy -1 3.962907 3.560222 0.379027 -2 3.673292 3.378484 0.402066 -3 3.460093 3.191662 0.424295 -4 3.296515 3.030681 0.442995 -5 3.161650 2.891829 0.459052 -6 3.022674 2.776469 0.473280 -7 2.974365 2.686321 0.484403 -8 2.869587 2.593854 0.496297 -9 2.785321 2.509093 0.506853 -10 2.677728 2.440328 0.516178 -11 2.641243 2.371950 0.525810 -12 2.652385 2.320008 0.533105 -13 2.547195 2.261057 0.542046 -14 2.491570 2.216933 0.548810 -15 2.454437 2.179364 0.555077 -16 2.414449 2.147612 0.559972 -17 2.358593 2.125351 0.563405 -18 2.362696 2.111580 0.565614 -19 2.341626 2.104268 0.566749 -20 2.342680 2.102918 0.566966 -/home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/en-1/models/sp30k -Saving info /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/en-1/models/sp30k/qrnn_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.622379 0.422002 0.882000 -Better model found at epoch 1 with val_loss value: 0.42200201749801636. -epoch train_loss valid_loss accuracy -1 0.313018 0.275563 0.908000 -Better model found at epoch 1 with val_loss value: 0.27556276321411133. -epoch train_loss valid_loss accuracy -1 0.241521 0.174606 0.933000 -Better model found at epoch 1 with val_loss value: 0.1746061146259308. -epoch train_loss valid_loss accuracy -1 0.125556 0.170286 0.940000 -Better model found at epoch 1 with val_loss value: 0.17028628289699554. -2 0.107322 0.181366 0.939000 -Saving models at /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/en-1/models/sp30k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.18917121, tensor(0.9388)] -0.1891712099313736 -0.9387500286102295 -``` diff --git a/results/logs/qrnn-es.md b/results/logs/qrnn-es.md deleted file mode 100644 index 5c3da86..0000000 --- a/results/logs/qrnn-es.md +++ /dev/null @@ -1,127 +0,0 @@ -# QRNN ES - -## SP15k nl 4 -`` -export CUDA_VISIBLE_DEVICES=1 -LANG=es -python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0 - -Wiki text was split to 161509 articles -Wiki text was split to 78 articles -Running tokenization lm... -Data lm, trn: 161509, val: 78 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', '▁la', 's', '▁el', '▁en', '▁y', '▁a', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: - -Training lm from random weights -epoch train_loss valid_loss accuracy -1 2.851575 3.398695 0.372940 -2 2.801543 3.353015 0.372648 -3 2.807216 3.290132 0.380787 -4 2.696361 3.220115 0.388937 -5 2.668488 3.132770 0.399528 -6 2.565685 3.062742 0.408880 -7 2.503054 2.985069 0.419262 -8 2.448338 2.895266 0.431797 -9 2.411213 2.829787 0.441973 -10 2.403536 2.811063 0.445468 -Total time: 11:52:32 -data/wiki/es-100/models/sp15k -Saving info data/wiki/es-100/models/sp15k/qrnn_nl4.m/info.json -`` - -```bash -export CUDA_VISIBLE_DEVICES=1 -LANG=es -python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/sp15k/qrnn_nl4.m --lang=${LANG} --name 'nl4' - train 20 --bs 20 --num-cls-epochs=8 -``` - - -## SP30k nl 4 -### LM -``` -python -m ulmfit lm --dataset-path data/wiki/es-100 --bidir=False --qrnn=True --nl 4 --tokenizer='sp' --max-vocab 30000 --lang es --name 'nl4' --cuda-id=0 - train 10 --drop-mult=0 --bs=50 - -Wiki text was split to 161509 articles -Wiki text was split to 78 articles -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', '▁la', '▁el', '▁en', '▁y', 's', '▁a', "▁&'"] -Training args: {'clip': 0.12, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Training lm from random weights -epoch train_loss valid_loss accuracy -1 3.067289 3.640350 0.357276 -2 2.958243 3.619773 0.358111 -3 3.033412 3.587700 0.359495 -4 2.933573 3.525202 0.367685 -5 2.904549 3.467990 0.372583 -6 2.798806 3.409506 0.380045 -7 2.733132 3.303108 0.391922 -8 2.675272 3.224150 0.401143 -9 2.635299 3.166430 0.410160 -10 2.656724 3.145599 0.413176 -``` - -### MLDocs -``` -python -m ulmfit cls --dataset-path data/mldoc/es-1 --cuda-id=0 --base-lm-path data-filtered/data/wiki/es-100/models/sp30k/qrnn_nl4.m --lang=es --name 'nl4' - train 20 --bs 40 --cls-max-len 700 - -Max vocab: 30000 -Cache dir: /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/es-1/models/sp30k -Model dir: /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/qrnn_nl4.m -Loading validation /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv -Running tokenization... -Saving tokenized: cls.trn 13013, cls.val 1445 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', '▁la', '▁el', '▁en', '▁y', 's', '▁a', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/marcin/github/n-waves/ulmfit-multilingual/data-filtered/data/wiki/es-100/models/sp30k/qrnn_nl4.m/lm_best'), PosixPath('/home/marcin/github/n-waves/ulmfit-multilingual/data-filtered/data/wiki/es-100/models/sp30k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.352874 2.367255 0.514858 -epoch train_loss valid_loss accuracy -1 2.796090 2.203233 0.536513 -2 2.515840 1.970145 0.576640 -3 2.198857 1.774013 0.610990 -4 2.035614 1.633484 0.633450 -5 1.944539 1.535505 0.649110 -6 1.848854 1.451618 0.661764 -7 1.788579 1.382675 0.673166 -8 1.675414 1.320675 0.683617 -9 1.614536 1.264944 0.694086 -10 1.618723 1.215493 0.702936 -11 1.504875 1.164356 0.712921 -12 1.411316 1.126858 0.721374 -13 1.421174 1.079897 0.731196 -14 1.352116 1.044965 0.738148 -15 1.318876 1.013755 0.745312 -16 1.268569 0.986391 0.751383 -17 1.273424 0.971129 0.754643 -18 1.256196 0.960661 0.757439 -19 1.233202 0.955790 0.758405 -20 1.230536 0.955070 0.758496 -/home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/es-1/models/sp30k -Saving info /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/qrnn_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.739119 0.438338 0.867000 -Better model found at epoch 1 with val_loss value: 0.438338041305542. -epoch train_loss valid_loss accuracy -1 0.425376 0.207067 0.950000 -Better model found at epoch 1 with val_loss value: 0.20706671476364136. -epoch train_loss valid_loss accuracy -1 0.311269 0.172416 0.956000 -Better model found at epoch 1 with val_loss value: 0.17241604626178741. -epoch train_loss valid_loss accuracy -1 0.226164 0.166543 0.958000 -Better model found at epoch 1 with val_loss value: 0.1665433794260025. -2 0.199775 0.167683 0.956000 -Saving models at /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.18184493, tensor(0.9448)] -0.18184493482112885 -0.9447500109672546 -``` diff --git a/results/logs/qrnn-it.md b/results/logs/qrnn-it.md deleted file mode 100644 index 43e0a2f..0000000 --- a/results/logs/qrnn-it.md +++ /dev/null @@ -1,30 +0,0 @@ -# IT -## SP30k QRNN nl 4 -### LM -``` -python -m ulmfit lm --dataset-path data/wiki/it-100/ --cuda-id=0 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 30000 --lang it --qrnn=True - train 10 --bs=50 --drop_mult=0 -Max vocab: 30000 -Cache dir: data/wiki/it-100/models/sp30k -Model dir: data/wiki/it-100/models/sp30k/qrnn_nl4.m -Tokenized data loaded -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', "▁&'", "'", '▁e', '▁il', '▁la', 'e', '▁in'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Training lm from random weights -epoch train_loss valid_loss accuracy -1 3.354224 3.749085 0.350236 -2 3.274838 3.697026 0.351104 -3 3.222462 3.680071 0.352152 -4 3.217652 3.628976 0.357922 -5 3.117965 3.563592 0.364370 -6 3.075397 3.483997 0.372794 -7 3.002098 3.394749 0.383217 -8 2.936974 3.316284 0.393616 -9 2.843549 3.258448 0.401605 -10 2.818070 3.240303 0.404684 -Total time: 10:49:44 -data/wiki/it-100/models/sp30k -Saving info data/wiki/it-100/models/sp30k/qrnn_nl4.m/info.json -``` - -### MLDoc diff --git a/results/logs/qrnn-ja.md b/results/logs/qrnn-ja.md deleted file mode 100644 index ffc1046..0000000 --- a/results/logs/qrnn-ja.md +++ /dev/null @@ -1,69 +0,0 @@ -LANG=ja -python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 30000 --lang ${LANG} --qrnn=True --lmseed=1 - train 10 --bs=50 --drop_mult=0 - -LANG=ja -python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang ${LANG} --qrnn=True "--tokenizer-mod=-fix" --lmseed=1 - train 10 --bs=50 --drop_mult=0 - -python -m ulmfit cls --dataset-path data/mldoc/ja-1 --base-lm-path data/wiki/ja-100/models/sp15k-fix/qrnn_nl4_lmseed-1.m --lang=ja --name 'nl4' - train 20 --bs 20 --lr_sched=1cycle --label-smoothing-eps=0.1set-path data/mldoc/es-1 --base-lm-path data/wiki/ja-100/models/sp30k-fix/qrnn_nl4.m --lang=ja --name 'nl4' - train 20 --bs 40 - -0 4.875985 3.940408 0.479504 02:40 -Total time: 02:40 -epoch train_loss valid_loss accuracy time -0 3.814340 3.574515 0.524143 03:43 -1 3.344660 3.174727 0.591801 03:42 -2 3.062797 2.909801 0.638397 03:42 -3 2.924287 2.753593 0.662699 03:42 -4 2.832728 2.648505 0.679922 03:41 -5 2.673981 2.575237 0.692270 03:41 -6 2.727100 2.521090 0.702180 03:42 -7 2.647422 2.474463 0.710956 03:42 -8 2.557694 2.437784 0.717445 03:41 -9 2.619366 2.398306 0.725727 03:42 -10 2.501441 2.368656 0.731127 03:41 -11 2.501446 2.340000 0.737203 03:41 -12 2.539080 2.316010 0.742567 03:42 -13 2.441686 2.290955 0.748064 03:41 -14 2.406307 2.273114 0.752770 03:41 -15 2.421776 2.256135 0.756699 03:42 -16 2.399470 2.245539 0.758821 03:42 -17 2.336457 2.237878 0.760780 03:42 -18 2.383474 2.234359 0.761501 03:41 -19 2.407631 2.233689 0.761739 03:42 -Total time: 1:14:01 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k-fix -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k-fix/qrnn_nl4_lmseed-1.m/info.json -Single training schedule -epoch train_loss valid_loss f_beta precision recall kappa_score matthews_correff accuracy time -/home/pczapla/workspace/_oss/fastai/fastai/fastai/metrics.py:179: UserWarning: average=`binary` was selected for a non binary case. Value for average has now been set to `macro` instead. - def _precision(self): -0 0.872342 0.839480 0.739856 0.836258 0.741572 0.649270 0.682293 0.737000 00:19 -Better model found at epoch 0 with f_beta value: 0.739856481552124. -/home/pczapla/workspace/_oss/fastai/fastai/fastai/metrics.py:179: UserWarning: average=`binary` was selected for a non binary case. Value for average has now been set to `macro` instead. - def _precision(self): -1 0.678405 0.587759 0.886923 0.888504 0.885348 0.845242 0.846027 0.884000 00:19 -Better model found at epoch 1 with f_beta value: 0.8869231939315796. -Total time: 00:39 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k-fix/qrnn_nl4_lmseed-1.m -/home/pczapla/workspace/_oss/fastai/fastai/fastai/metrics.py:179: UserWarning: average=`binary` was selected for a non binary case. Value for average has now been set to `macro` instead. - def _precision(self): -Model: nl4 -Validation on: test -F1 score bin: 0.9002974033355713 -Loss: 0.3307778239250183 -Precision: 0.9020541906356812 -Recall: 0.9002037048339844 -Accuracy: 0.9007499814033508 -test F1 score bin: 0.9002974033355713 -test Loss: 0.33077782 -test Precision: 0.9020541906356812 -test Recall: 0.9002037048339844 -test Kappa Linear: 0.8676621913909912 -test Matthews Correff: 0.868194043636322 -test Accuracy: 0.9007499814033508 - - - -eval --glob="mldoc/ja-1/models/sp15k/qrnn_nl4.m" --name nl4-1cyc-sl --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.1 - - - diff --git a/results/logs/qrnn-ru.md b/results/logs/qrnn-ru.md deleted file mode 100644 index 6b6b121..0000000 --- a/results/logs/qrnn-ru.md +++ /dev/null @@ -1,204 +0,0 @@ -# QRNN RU -## SP15k nl4 -## LM -export CUDA_VISIBLE_DEVICES=3 -LANG=ru -python -m ulmfit lm --dataset-path data/wiki/ru-100 --bidir=False --qrnn=True --nl 4 --tokenizer='sp' --max-vocab 15000 --lang ru --name 'nl4' - train 10 --drop-mult=0 --bs=50 --label-smoothing-eps=0.1 - -## SP15k nl8 -### LM -``` -5 2.869308 2.905951 0.466976 -6 2.768955 2.782804 0.481852 -7 2.654484 2.676304 0.495593 -8 2.585963 2.591748 0.508447 -9 2.512042 2.526819 0.518860 -10 2.520543 2.509287 0.521890 -Total time: 18:46:01 -data/wiki/ru-100/models/sp15k -Saving info data/wiki/ru-100/models/sp15k/qrnn_nl8.m/info.json -``` -### MLDoc -``` -python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/sp15k/qrnn_nl8.m --lang=${LANG} --name 'nl8' - train 20 --bs 20 --num-cls-epochs=8 --lr_sched=1cycle --label-smoothing-eps=0.1 -Max vocab: 15000 -Cache dir: /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k -Model dir: /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl8.m -Loading validation /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv -Running tokenization lm... -Data lm, trn: 9195, val: 1021 - -Running tokenization cls... -Data cls, trn: 1000, val: 1000 - -Running tokenization tst... -Data tst, trn: 1000, val: 4000 - -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', '▁на', 'х'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp15k/qrnn_nl8.m/lm_best'), PosixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp15k/qrnn_nl8.m/../itos')] -epoch train_loss valid_loss accuracy -1 2.923423 2.334532 0.529978 - -Total time: 02:46 -epoch train_loss valid_loss accuracy -1 2.462077 2.150593 0.563281 - -2 2.230013 1.972095 0.596198 - -3 2.118523 1.812012 0.623204 - -4 1.916368 1.690016 0.644060 - -5 1.842718 1.585770 0.661704 - -6 1.748630 1.513972 0.674130 - -7 1.675032 1.447667 0.686207 - -8 1.628485 1.393949 0.695972 - -9 1.564814 1.330838 0.707272 - -10 1.553933 1.283114 0.715716 - -11 1.441891 1.234810 0.726201 - -12 1.496388 1.185676 0.735977 - -13 1.383019 1.141014 0.745528 - -14 1.256620 1.094201 0.755120 - -15 1.306187 1.052457 0.764280 - -16 1.297933 1.028387 0.769747 - -17 1.319773 1.004256 0.775285 - -18 1.178073 0.989788 0.778480 - -19 1.252248 0.982740 0.780057 - -20 1.177640 0.981201 0.780267 - -Total time: 1:24:58 -/home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k -Saving info /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl8.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.882775 0.510930 0.826000 - -2 0.683476 0.513669 0.847000 - -3 0.556661 0.590375 0.839000 - -4 0.454019 0.757216 0.828000 - -5 0.344460 0.549675 0.870000 - -6 0.246039 0.630242 0.861000 - -7 0.173423 0.649066 0.858000 - -8 0.098640 0.638015 0.867000 - -Total time: 05:11 -Saving models at /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl8.m -Loss and accuracy using (cls_best): [0.64393336, tensor(0.8683)] -``` - -### MLDoc nl8 -2nd -``` - python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/sp15k/qrnn_${NAME}.m --lang=${LANG} --name ${NAME}-2 - train 20 --bs 18 --num-cls-epochs=4 --lr_sched=1cycle --label-smoothing-eps=0 -.1 -Max vocab: 15000 -Cache dir: /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k -Model dir: /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl8-2.m -Loading validation /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv -Data lm, trn: 9195, val: 1021 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', '▁на', 'х'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp15k/qrnn_nl8.m/lm_best'), PosixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp15k/qrnn_nl8.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.966292 3.450758 0.527065 -Total time: 02:58 -epoch train_loss valid_loss accuracy -1 3.495761 3.276329 0.560047 -2 3.319947 3.102911 0.593742 -3 3.137904 2.955317 0.620171 -4 3.040286 2.839161 0.642270 -5 2.869962 2.753622 0.658331 -6 2.905739 2.680881 0.672860 -7 2.836454 2.620925 0.685026 -8 2.857271 2.569716 0.695722 -9 2.702872 2.520050 0.705589 -10 2.701559 2.473591 0.715346 -11 2.740815 2.429558 0.725597 -12 2.646513 2.389550 0.735010 -13 2.587685 2.349614 0.744885 -14 2.546527 2.311087 0.754463 -15 2.568136 2.278581 0.762980 -16 2.492115 2.252367 0.769275 -17 2.338561 2.230529 0.775072 -18 2.447506 2.218215 0.778437 -19 2.364424 2.212115 0.780085 -20 2.367132 2.210520 0.780424 -Total time: 1:30:47 -/home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k -Saving info /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl8-2.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.969255 0.769736 0.843000 -2 0.846340 0.813483 0.839000 -3 0.718175 0.705339 0.867000 -4 0.609513 0.726442 0.875000 -Total time: 02:54 -Saving models at /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl8-2.m -Loss and accuracy using (cls_best): [0.4056449, tensor(0.8698)] -0.40564489364624023 -``` - - - - - - -## cls -``` -export CUDA_VISIBLE_DEVICES=0 -LANG=ru -python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/sp15k/qrnn_nl8.m --lang=${LANG} --name 'nl8' - train 20 --bs 20 --num-cls-epochs=8 --lr_sched=1cycle --label-smoothing-eps=0.1 -``` - -## SP30k nl4 -### LM -``` -python -m ulmfit lm --dataset-path data/wiki/ru-100 --bidir=False --qrnn=True --nl 4 --tokenizer='sp' --max-vocab 30000 --lang ru --name 'nl4' --cuda-id=0 - train 10 --drop-mult=0 --bs=50 - -Wiki text was split to 193047 articles -Wiki text was split to 460 articles -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', '▁и', 'е', 'и', 'й', '▁на', '▁с'] -Training args: {'clip': 0.12, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Training lm from random weights -epoch train_loss valid_loss accuracy -1 3.273207 3.350111 0.429702 -2 3.169897 3.274238 0.433682 -3 3.162197 3.247077 0.435900 -4 3.131630 3.168798 0.445252 -5 3.042942 3.096774 0.453532 -6 2.950550 3.002989 0.465113 -7 2.833593 2.902871 0.478954 -8 2.829737 2.805592 0.492138 -9 2.746991 2.733609 0.503711 -10 2.687201 2.708546 0.508050 -``` diff --git a/results/logs/qrnn-zh.md b/results/logs/qrnn-zh.md deleted file mode 100644 index dcb9e15..0000000 --- a/results/logs/qrnn-zh.md +++ /dev/null @@ -1,140 +0,0 @@ - -# -``` - export CUDA_VISIBLE_DEVICES=0 -LANG=zh -python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0 - -Wiki text was split to 103929 articles -Wiki text was split to 113 articles -Running tokenization lm... -Data lm, trn: 103929, val: 113 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: - - !! WARNING !! - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Your compiler (c++) may be ABI-incompatible with PyTorch! -Please use a compiler that is ABI-compatible with GCC 4.9 and above. -See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. - -See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 -for instructions on how to install GCC 4.9 or higher. -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - !! WARNING !! - - warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) -Training lm from random weights -epoch train_loss valid_loss accuracy -1 2.489521 2.734049 0.482433 -2 2.427567 2.662464 0.488089 -3 2.415744 2.613971 0.494118 -4 2.334062 2.560180 0.501209 -5 2.343723 2.503271 0.507307 -6 2.260171 2.444533 0.516768 -7 2.198721 2.367407 0.526631 -8 2.161857 2.308182 0.535856 -9 2.142125 2.252678 0.544535 -10 2.087831 2.234440 0.548529 -Total time: 11:01:47 -data/wiki/zh-100/models/sp15k -Saving info data/wiki/zh-100/models/sp15k/qrnn_nl4.m/info.json -``` - -## MLDoc -```bash -export CUDA_VISIBLE_DEVICES=0 -LANG=zh -python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/sp15k/qrnn_nl4.m --lang=${LANG} --name 'nl4' - train 20 --bs 20 --num-cls-epochs=8 - -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/zh-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/zh-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 2.723684 2.148748 0.571206 -Total time: 02:13 -epoch train_loss valid_loss accuracy -1 2.157829 1.937637 0.601026 -2 1.898958 1.712967 0.637379 -3 1.722818 1.547745 0.664276 -4 1.570266 1.427551 0.682546 -5 1.503477 1.344690 0.696379 -6 1.434701 1.289549 0.704813 -7 1.425267 1.217570 0.717714 -8 1.373606 1.174655 0.725217 -9 1.297397 1.116406 0.735997 -10 1.211259 1.062999 0.745848 -11 1.248108 1.024482 0.754134 -12 1.198918 0.980273 0.762664 -13 1.121848 0.937985 0.771961 -14 1.111386 0.898821 0.780796 -15 1.120596 0.866009 0.787908 -16 1.056925 0.836998 0.794833 -17 1.020636 0.816387 0.799694 -18 1.002068 0.802623 0.802859 -19 0.998480 0.796877 0.804212 -20 0.959919 0.794685 0.804594 -Total time: 1:02:57 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.666322 0.433893 0.855000 -Total time: 00:08 -epoch train_loss valid_loss accuracy -1 0.448371 0.317440 0.889000 -Total time: 00:09 -epoch train_loss valid_loss accuracy -1 0.336693 0.309876 0.900000 -Total time: 00:10 -epoch train_loss valid_loss accuracy -1 0.266735 0.302003 0.903000 -2 0.222821 0.294501 0.905000 -3 0.207295 0.293751 0.908000 -4 0.179668 0.296945 0.911000 -5 0.153803 0.293158 0.911000 -Traceback (most recent call last): - File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/runpy.py", line 193, in _run_module_as_main - "__main__", mod_spec) - File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/runpy.py", line 85, in _run_code - exec(code, run_globals) - File "/home/pczapla/workspace/ulmfit-multilingual/ulmfit/__main__.py", line 73, in - fire.Fire(ULMFiT()) - File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/fire/core.py", line 127, in Fire - component_trace = _Fire(component, args, context, name) - File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/fire/core.py", line 366, in _Fire - component, remaining_args) - File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/fire/core.py", line 542, in _CallCallable - result = fn(*varargs, **kwargs) - File "/home/pczapla/workspace/ulmfit-multilingual/ulmfit/train_clas.py", line 54, in train_cls - learn.fit_one_cycle(num_cls_epochs, slice(1e-3 / (2.6 ** 4), 1e-3), moms=(0.8, 0.7)) - File "/home/pczapla/workspace/_oss/fastai/fastai/fastai/train.py", line 22, in fit_one_cycle - learn.fit(cyc_len, max_lr, wd=wd, callbacks=callbacks) - File "/home/pczapla/workspace/_oss/fastai/fastai/fastai/basic_train.py", line 178, in fit - callbacks=self.callbacks+callbacks) - File "/home/pczapla/workspace/_oss/fastai/fastai/fastai/utils/mem.py", line 77, in wrapper - return func(*args, **kwargs) - File "/home/pczapla/workspace/_oss/fastai/fastai/fastai/basic_train.py", line 90, in fit - loss = loss_batch(model, xb, yb, loss_func, opt, cb_handler) - File "/home/pczapla/workspace/_oss/fastai/fastai/fastai/basic_train.py", line 20, in loss_batch - out = model(*xb) - File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/nn/modules/module.py", line 477, in __call__ - result = self.forward(*input, **kwargs) - File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/nn/modules/container.py", line 92, in forward - input = module(input) - File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/nn/modules/module.py", line 477, in __call__ - result = self.forward(*input, **kwargs) - File "/home/pczapla/workspace/_oss/fastai/fastai/fastai/text/learner.py", line 235, in forward - return self.concat(raw_outputs), self.concat(outputs) - File "/home/pczapla/workspace/_oss/fastai/fastai/fastai/text/learner.py", line 221, in concat - return [torch.cat([l[si] for l in arrs], dim=1) for si in range_of(arrs[0])] - File "/home/pczapla/workspace/_oss/fastai/fastai/fastai/text/learner.py", line 221, in - return [torch.cat([l[si] for l in arrs], dim=1) for si in range_of(arrs[0])] -RuntimeError: CUDA error: out of memory -``` - -## Fixed sentence piece -LANG=zh -python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang ${LANG} --qrnn=True "--tokenizer-mod=-fix" --lmseed=1 - train 10 --bs=50 --drop_mult=0 diff --git a/results/logs/ru.md b/results/logs/ru.md deleted file mode 100644 index dc469aa..0000000 --- a/results/logs/ru.md +++ /dev/null @@ -1,166 +0,0 @@ -# RU -## SP15k nl4 -``` -Training lm from random weights -epoch train_loss valid_loss accuracy -1 3.053061 3.070487 0.450466 -2 2.874137 2.999093 0.455027 -3 2.864496 2.969308 0.458116 -4 2.890568 2.903564 0.466970 -5 2.746530 2.839789 0.474205 -6 2.683900 2.750476 0.486806 -7 2.674458 2.658535 0.499701 -8 2.595780 2.573735 0.512515 -9 2.530827 2.512999 0.522372 -10 2.505664 2.491850 0.526431 -Total time: 10:43:03 -data/wiki/ru-100/models/sp15k -Saving info data/wiki/ru-100/models/sp15k/qrnn_nl4.m/info.json -``` -```bash -python -m ulmfit cls --dataset-path data/mldoc/ru-1 --base-lm-path data/wiki/ru-100/models/sp30k/lstm_nl4.m --lang=ru --name 'nl4-100' --cuda-id=1 - train 20 --bs 40 --num-cls-epochs=2 - -``` - -## SP30k nl4 -### LM -``` -python -m ulmfit lm --dataset-path data/wiki/ru-100 --cuda-id=0 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 30000 --lang ru --qrnn=False - train 10 --bs=50 --drop_mult=0 -Size of vocabulary: 30000 [39/805] -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', '▁и', 'е', 'и', 'й', '▁на', 'х'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': None, 'pretrained_model': None, 'drop_mult': 0} dps: [0.25 0.1 0.2 0.02 0.15] -Training lm from random weights -epoch train_loss valid_loss accuracy -1 3.200520 3.295865 0.436852 -2 3.027569 3.168700 0.445551 -3 3.007320 3.132495 0.450450 -4 2.940000 3.041745 0.459344 -5 2.876227 2.952338 0.469182 -6 2.742553 2.860888 0.480943 -7 2.684717 2.769994 0.492934 -8 2.569419 2.669971 0.507300 -9 2.525698 2.604086 0.516840 -10 2.495174 2.591011 0.519415 -data/wiki/ru-100/models/sp30k -Saving info data/wiki/ru-100/models/sp30k/lstm_nl4.m/info.json -``` -### MLDoc - bsp -MultiCCA: 85.65% ulmfit: 87.27% -``` -python -m ulmfit cls --dataset-path data/mldoc/ru-1 --base-lm-path data/wiki/ru-100/models/sp30k/lstm_nl4.m --lang=ru --name 'nl4-100' --cuda-id=1 - train 20 --bs 40 --num-cls-epochs=2 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv -Running tokenization... -Saving tokenized: cls.trn 9195, cls.val 1021 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', '▁и', 'е', 'и', 'й', '▁на', 'х'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp30k/lstm_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 2.764138 2.289755 0.552181 -epoch train_loss valid_loss accuracy -1 2.414295 2.161708 0.572407 -2 2.310551 2.013092 0.596075 -3 2.124479 1.864450 0.620103 -4 1.970015 1.723395 0.642392 -5 1.883664 1.623308 0.658949 -6 1.793856 1.513542 0.677954 -7 1.625767 1.424582 0.693092 -8 1.677054 1.335406 0.709802 -9 1.578936 1.264322 0.723626 -10 1.523383 1.194463 0.737942 -11 1.436643 1.129712 0.750586 -12 1.351507 1.072792 0.762524 -13 1.357552 1.020739 0.773266 -14 1.310516 0.975852 0.783653 -15 1.216484 0.940323 0.791262 -16 1.187942 0.909915 0.797675 -17 1.141316 0.885367 0.803305 -18 1.114629 0.871992 0.805929 -19 1.075366 0.867010 0.807009 -20 1.166387 0.865594 0.807241 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.831180 0.610087 0.787000 -epoch train_loss valid_loss accuracy -1 0.678307 0.435860 0.856000 -epoch train_loss valid_loss accuracy -1 0.547668 0.399889 0.870000 -epoch train_loss valid_loss accuracy -1 0.445839 0.396535 0.869000 -2 0.417901 0.369961 0.882000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.38499942, tensor(0.8727)] -``` - -### MLDoc run 2x sp -``` -python -m ulmfit cls --dataset-path data/mldoc/ru-1 --base-lm-path data/wiki/ru-100/models/sp30k/lstm_nl4.m --lang=ru --name 'nl4' --cuda-id=1 - train 20 --bs 40 --num-cls-epochs=2 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv -Running tokenization... -Saving tokenized: cls.trn 9195, cls.val 1021 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', '▁и', 'е', 'и', 'й', '▁на', 'х'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp30k/lstm_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 2.662225 2.284158 0.552927 -epoch train_loss valid_loss accuracy -1 2.436114 2.151187 0.574219 -2 2.260576 2.012279 0.595820 -3 2.067110 1.862512 0.620246 -4 2.000703 1.729883 0.641713 -5 1.860899 1.609955 0.661346 -6 1.751010 1.522195 0.676297 -7 1.705993 1.420628 0.694044 -8 1.592143 1.338552 0.708978 -9 1.524927 1.270614 0.722596 -10 1.475408 1.198585 0.736638 -11 1.438226 1.134858 0.749314 -12 1.408821 1.076875 0.761448 -13 1.345137 1.020660 0.773432 -14 1.321399 0.978076 0.783070 -15 1.235357 0.936674 0.791642 -16 1.204204 0.906822 0.798548 -17 1.198709 0.884949 0.803528 -18 1.176732 0.874523 0.805585 -19 1.111195 0.871806 0.806239 -20 1.031497 0.869280 0.806826 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.834704 0.615589 0.786000 -epoch train_loss valid_loss accuracy -1 0.679823 0.418461 0.851000 -epoch train_loss valid_loss accuracy -1 0.555612 0.426877 0.861000 -epoch train_loss valid_loss accuracy -1 0.468084 0.391777 0.873000 -2 0.434714 0.388670 0.882000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.3987146, tensor(0.8680)] -0.3987146019935608 -0.8679999709129333 -``` - -``` -Second execution -epoch train_loss valid_loss accuracy -1 2.749340 2.284773 0.552775 -epoch train_loss valid_loss accuracy -1 2.418463 2.157943 0.572302 -``` \ No newline at end of file diff --git a/results/logs/ru/5epochs.md b/results/logs/ru/5epochs.md deleted file mode 100644 index ca357c3..0000000 --- a/results/logs/ru/5epochs.md +++ /dev/null @@ -1,91 +0,0 @@ -``` -export CUDA_VISIBLE_DEVICES=0 -LANG=ru -python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/sp15k/qrnn_nl4.m --lang=${LANG} --name 'nl4-lm5' - train 5 --bs 20 --num-cls-epochs=8 --lr_sched=1cycle --label-smoothing-eps=0.1 - -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4-lm5.m -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv -Data lm, trn: 9195, val: 1021 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', '▁на', 'х'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: - - !! WARNING !! - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Your compiler (c++) may be ABI-incompatible with PyTorch! -Please use a compiler that is ABI-compatible with GCC 4.9 and above. -See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. - -See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 -for instructions on how to install GCC 4.9 or higher. -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - !! WARNING !! - - warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) -Loading pretrained model -Unknown tokens 0, first 100: [] -Bptt 70 -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 4.549059 3.763798 0.472608 -Total time: 02:05 -epoch train_loss valid_loss accuracy -1 3.543295 3.263310 0.567992 -2 3.166918 2.968566 0.619391 -3 3.057842 2.812808 0.648944 -4 2.842979 2.726823 0.665521 -5 2.872606 2.703771 0.670281 -Total time: 14:40 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4-lm5.m/info.json -``` -``` -python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/sp15k/qrnn_nl4.m --lang=${LANG} --name 'nl4-lm5' - train 5 --bs 18 --num-cls-epochs=8 --lr_sched=1cycle --label-smoothing-eps=0.1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4-lm5.m -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv -Data lm, trn: 9195, val: 1021 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', '▁на', 'х'] -/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: - - !! WARNING !! - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Your compiler (c++) may be ABI-incompatible with PyTorch! -Please use a compiler that is ABI-compatible with GCC 4.9 and above. -See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. - -See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 -for instructions on how to install GCC 4.9 or higher. -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - !! WARNING !! - - warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) -Single training schedule -epoch train_loss valid_loss accuracy -1 1.021985 0.763919 0.822000 -2 0.903123 0.756099 0.849000 -3 0.831409 0.852466 0.832000 -4 0.744423 0.753127 0.858000 -5 0.669933 0.747895 0.862000 -6 0.607411 0.744035 0.869000 -7 0.554080 0.706676 0.872000 -8 0.532403 0.719503 0.870000 -Total time: 03:12 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4-lm5.m -Loss and accuracy using (cls_best): [0.41288647, tensor(0.8615)] -0.41288647055625916 -0.8615000247955322 -``` \ No newline at end of file diff --git a/results/logs/ru/bs_vs_lr.md b/results/logs/ru/bs_vs_lr.md deleted file mode 100644 index 9d13a63..0000000 --- a/results/logs/ru/bs_vs_lr.md +++ /dev/null @@ -1,123 +0,0 @@ - -## BS=18, lr_mult=1.0 - -epoch train_loss valid_loss accuracy -1 4.427713 3.693394 0.484268 -Total time: 01:32 -epoch train_loss valid_loss accuracy -1 3.758918 3.446661 0.529820 -2 3.394254 3.199054 0.577411 -3 3.235364 3.014517 0.610520 -4 3.125459 2.871101 0.637153 -5 2.994313 2.773862 0.654470 -6 2.915075 2.693080 0.669942 -7 2.855732 2.622858 0.683629 -8 2.755074 2.572147 0.694145 -9 2.697898 2.517524 0.704816 -10 2.689881 2.468190 0.715927 -11 2.579573 2.432807 0.723324 -12 2.659464 2.387878 0.733931 -13 2.520637 2.344804 0.744233 -14 2.482952 2.315014 0.751855 -15 2.564730 2.279045 0.761163 -16 2.552707 2.255916 0.766971 -17 2.511244 2.240169 0.770991 -18 2.461429 2.228213 0.774309 -19 2.426440 2.222140 0.775745 -20 2.425955 2.221128 0.775836 -Total time: 1:14:17 - -## BS=500, lr_mult=1.0 -epoch train_loss valid_loss accuracy -1 5.536769 3.850831 0.444662 -Total time: 01:10 -epoch train_loss valid_loss accuracy -1 4.845898 3.781763 0.461471 -2 4.388605 3.643141 0.491225 -3 4.038255 3.464554 0.526143 - -## BS=500, lr_mult=27 -epoch train_loss valid_loss accuracy -1 7.234749 5.868155 0.312675 -Total time: 01:41 - -## BS=500, lr_mult=10 + BS=50 lr_mult=10 for cls -/data/wiki/ru-100/models/sp15k/qrnn_nl4sl.m/../itos')] -epoch train_loss valid_loss accuracy -1 5.052441 4.082105 0.439539 -Total time: 02:27 -epoch train_loss valid_loss accuracy -1 4.120197 3.712686 0.498216 -2 3.727043 3.373258 0.557896 -3 3.383009 3.109635 0.598970 -4 3.180799 2.938478 0.626816 -5 3.048913 2.812639 0.647257 -6 2.943903 2.727179 0.661784 -7 2.864300 2.650275 0.674248 -8 2.773810 2.583594 0.687063 -9 2.724850 2.529445 0.697573 -10 2.673996 2.473824 0.708698 -11 2.657637 2.431461 0.716904 -12 2.591277 2.372668 0.730318 -13 2.537707 2.323294 0.741157 -14 2.486507 2.280270 0.751768 -15 2.435933 2.238660 0.762545 -16 2.401303 2.208848 0.769561 -17 2.374117 2.184253 0.776400 -18 2.341421 2.169156 0.780388 -19 2.328202 2.163922 0.781700 -20 2.315462 2.161784 0.782105 -Total time: 1:07:09 -------------------- Checking the influence of number of epochs on the accuracy -(multifit) test@test:~/workspace/ulmfit-multilingual$ rm /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4sl-bs500.m/cls* -(multifit) test@test:~/workspace/ulmfit-multilingual$ python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/sp15k/qrnn_${NAME}.m --lang=${LANG} --name ${NAME}-bs500 - train 0 --bs 50 --num-cls-epochs=4 --lr_sched=1cycle --label-smoothing-eps=0.1 --lr_mult=1 -Max vocab: 15000 -Cache dir: /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k -Model dir: /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4sl-bs500.m -Loading validation /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv -Data lm, trn: 9195, val: 1021 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', '▁на', 'х'] -Single training schedule -epoch train_loss valid_loss accuracy -1 1.063845 1.111960 0.601000 -2 0.902245 0.766871 0.817000 -3 0.766261 0.707502 0.861000 -4 0.680053 0.694492 0.866000 -Total time: 01:22 -Saving models at /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4sl-bs500.m -Loss and accuracy using (cls_best): [0.41532615, tensor(0.8630)] -0.41532614827156067 -0.8629999756813049 -(multifit) test@test:~/workspace/ulmfit-multilingual$ python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/sp15k/qrnn_${NAME}.m --lang=${LANG} --name ${NAME}-bs500 - train 0 --bs 50 --num-cls-epochs=16 --lr_sched=1cycle --label-smoothing-eps=0.1 --lr_mult=1 -Max vocab: 15000 -Cache dir: /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k -Model dir: /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4sl-bs500.m -Loading validation /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv -Data lm, trn: 9195, val: 1021 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', '▁на', 'х'] -Loading last classifier -Single training schedule -epoch train_loss valid_loss accuracy -1 0.556688 0.706000 0.873000 -2 0.537578 0.717411 0.865000 -3 0.532326 0.775549 0.854000 -4 0.529178 0.767506 0.861000 -5 0.521306 0.797604 0.860000 -6 0.527344 0.736225 0.868000 -7 0.516393 0.724941 0.878000 -8 0.510422 0.716110 0.873000 -9 0.504320 0.701886 0.869000 -10 0.500323 0.676577 0.878000 -11 0.493490 0.682657 0.873000 -12 0.484450 0.682047 0.878000 -13 0.479248 0.682782 0.880000 -14 0.474778 0.688019 0.873000 -15 0.472664 0.685304 0.874000 -16 0.470747 0.677925 0.878000 -Total time: 07:57 diff --git a/results/logs/ru/lm-opti.md b/results/logs/ru/lm-opti.md deleted file mode 100644 index 122a3b6..0000000 --- a/results/logs/ru/lm-opti.md +++ /dev/null @@ -1,260 +0,0 @@ -CUDA_VISIBLE_DEVICES=0 python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 5 --name 'nl5-merity' --max-vocab 15000 --lang ${LANG} --qrnn=True --bptt=140 --nh 2500 - train 14 --bs=50 --drop_mult=0 --label-smoothing-eps=0.1 -CUDA_VISIBLE_DEVICES=1 python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4-wide2' --max-vocab 15000 --lang ${LANG} --qrnn=True --nh 3100 - train 10 --bs=100 --drop_mult=0 --label-smoothing-eps=0.1 -CUDA_VISIBLE_DEVICES=2 python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4-merity' --max-vocab 15000 --lang ${LANG} --qrnn=True --bptt=140 --nh 2500 - train 14 --bs=50 --drop_mult=0 --label-smoothing-eps=0.1 -CUDA_VISIBLE_DEVICES=3 python -m ulmfit lm --dataset-path data/wiki/ru-100 --bidir=False --qrnn=True --nl 4 --tokenizer=sp --max-vocab 15000 --lang ru --name nl4sl - train 10 --drop-mult=0 --bs=50 --label-smoothing-eps=0.1 - -## 25vocab -CUDA_VISIBLE_DEVICES=3 python -m ulmfit lm --dataset-path data/wiki/ru-100 --bidir=False --qrnn=True --nl 4 --tokenizer=sp --max-vocab 25000 --lang ru --name nl4 - train 10 --drop-mult=0 --bs=50 --label-smoothing-eps=0.1 - -LANG=ru -CUDA_VISIBLE_DEVICES=0 python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4-merity-wide2' --max-vocab 15000 --lang ${LANG} --qrnn=True --bptt=140 --nh 3100 - train 10 --bs=50 --drop_mult=0 --label-smoothing-eps=0.1 - -##### CLS -export CUDA_VISIBLE_DEVICES=0 -LANG=ru -NAME=nl5-merity -python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/sp15k/qrnn_${NAME}.m --lang=${LANG} --name ${NAME} - train 20 --bs 18 --num-cls-epochs=4 --lr_sched=1cycle --label-smoothing-eps=0.1 - -export CUDA_VISIBLE_DEVICES=1 -LANG=ru -NAME=nl4-wide2 -python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/sp15k/qrnn_${NAME}.m --lang=${LANG} --name ${NAME} - train 20 --bs 18 --num-cls-epochs=4 --lr_sched=1cycle --label-smoothing-eps=0.1 - -export CUDA_VISIBLE_DEVICES=2 -LANG=ru -NAME=nl4-merity -python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/sp15k/qrnn_${NAME}.m --lang=${LANG} --name ${NAME} - train 20 --bs 18 --num-cls-epochs=4 --lr_sched=1cycle --label-smoothing-eps=0.1 - -export CUDA_VISIBLE_DEVICES=3 -LANG=ru -NAME=nl4sl -python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/sp15k/qrnn_${NAME}.m --lang=${LANG} --name ${NAME} - train 20 --bs 18 --num-cls-epochs=4 --lr_sched=1cycle --label-smoothing-eps=0.1 - - ------------------------CLS1 -export CUDA_VISIBLE_DEVICES=0 -LANG=ru -NAME=nl4 -python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/sp15k/qrnn_${NAME}.m --lang=${LANG} --name ${NAME} - train 20 --bs 18 --num-cls-epochs=4 --lr_sched=1cycle --label-smoothing-eps=0.1 - - -export CUDA_VISIBLE_DEVICES=0 -LANG=ru -NAME=nl8 -python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/sp15k/qrnn_${NAME}.m --lang=${LANG} --name ${NAME} - train 20 --bs 18 --num-cls-epochs=4 --lr_sched=1cycle --label-smoothing-eps=0.1 - - -python -m ulmfit cls --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 5 --name 'nl5-merity' --max-vocab 15000 --lang ${LANG} --qrnn=True --bptt=140 --nh 2500 - train 14 --bs=50 --drop_mult=0 --label-smoothing-eps=0.1 - -CUDA_VISIBLE_DEVICES=1 python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4-wide2' --max-vocab 15000 --lang ${LANG} --qrnn=True --nh 3100 - train 10 --bs=100 --drop_mult=0 --label-smoothing-eps=0.1 -CUDA_VISIBLE_DEVICES=2 python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4-merity' --max-vocab 15000 --lang ${LANG} --qrnn=True --bptt=140 --nh 2500 - train 14 --bs=50 --drop_mult=0 --label-smoothing-eps=0.1 -CUDA_VISIBLE_DEVICES=3 python -m ulmfit lm --dataset-path data/wiki/ru-100 --bidir=False --qrnn=True --nl 4 --tokenizer=sp --max-vocab 15000 --lang ru --name nl4sl - train 10 --drop-mult=0 --bs=50 --label-smoothing-eps=0.1 -## - ------------------------- - -7 3.680504 3.678406 0.498396 -8 3.556062 3.596037 0.512345 -9 3.553716 3.535783 0.523509 -10 3.523366 3.515352 0.527935 -Total time: 20:03:59 -data/wiki/ru-100/models/sp15k -Saving info data/wiki/ru-100/models/sp15k/qrnn_ nl4sl.m/info.json - -### Ru -``` -export CUDA_VISIBLE_DEVICES=3 -LANG=ru -NAME=nl4sl -python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/sp15k/qrnn_${NAME}.m --lang=${LANG} --name ${NAME} - train 20 --bs 18 --num-cls-epochs=4 --lr_sched=1cycle --label-smoothing-eps=0.1 -Max vocab: 15000 -Cache dir: /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k -Model dir: /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4sl.m -Loading validation /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv -Data lm, trn: 9195, val: 1021 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', '▁на', 'х'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/test/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp15k/qrnn_nl4sl.m/lm_best'), PosixPath('/home/test/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp15k/qrnn_nl4sl.m/../itos')] -epoch train_loss valid_loss accuracy -1 4.427713 3.693394 0.484268 -Total time: 01:32 -epoch train_loss valid_loss accuracy -1 3.758918 3.446661 0.529820 -2 3.394254 3.199054 0.577411 -3 3.235364 3.014517 0.610520 -4 3.125459 2.871101 0.637153 -5 2.994313 2.773862 0.654470 -6 2.915075 2.693080 0.669942 -7 2.855732 2.622858 0.683629 -8 2.755074 2.572147 0.694145 -9 2.697898 2.517524 0.704816 -10 2.689881 2.468190 0.715927 -11 2.579573 2.432807 0.723324 -12 2.659464 2.387878 0.733931 -13 2.520637 2.344804 0.744233 -14 2.482952 2.315014 0.751855 -15 2.564730 2.279045 0.761163 -16 2.552707 2.255916 0.766971 -17 2.511244 2.240169 0.770991 -18 2.461429 2.228213 0.774309 -19 2.426440 2.222140 0.775745 -20 2.425955 2.221128 0.775836 -Total time: 1:14:17 -/home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k -Saving info /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4sl.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.022382 0.779370 0.822000 -2 0.866379 0.792353 0.832000 -3 0.715650 0.698579 0.865000 -4 0.603621 0.693501 0.884000 -Total time: 02:05 -Saving models at /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4sl.m -Loss and accuracy using (cls_best): [0.3978519, tensor(0.8723)] -0.3978519141674042 -0.8722500205039978 -``` - ----- - -```bash -$ export CUDA_VISIBLE_DEVICES=0 -$ LANG=ru -$ NAME=nl4 -$ python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/sp15k/qrnn_${NAME}.m --lang=${LANG} --name ${NAME} - train 20 --bs 18 --num-cls-epochs=4 --lr_sched=1cycle --label-smoothing-eps=0.1 -Max vocab: 15000 -Cache dir: /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k -Model dir: /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4.m -Loading validation /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv -Data lm, trn: 9195, val: 1021 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', '▁на', 'х'] -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', '▁на', 'х'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp15k/qrnn_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 4.531968 3.764185 0.474252 -Total time: 01:44 -epoch train_loss valid_loss accuracy -1 3.770046 3.506013 0.522443 -2 3.546580 3.251341 0.571620 -3 3.320569 3.055680 0.606364 -4 3.130226 2.912925 0.631395 -5 3.072772 2.809725 0.649728 -6 2.765424 2.731825 0.662963 -7 2.959237 2.662104 0.676203 -8 2.807999 2.600417 0.688423 -9 2.771271 2.548279 0.699473 -10 2.809488 2.501688 0.709020 -11 2.707221 2.454946 0.719196 -12 2.597226 2.417315 0.728432 -13 2.609972 2.376176 0.737923 -14 2.590427 2.341666 0.746216 -15 2.572995 2.306599 0.754747 -16 2.496636 2.285632 0.760806 -17 2.508584 2.266456 0.765147 -18 2.441373 2.253839 0.768449 -19 2.430915 2.249204 0.769536 -20 2.426130 2.247966 0.769886 -Total time: 47:33 -/home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k -Saving info /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.060299 0.890710 0.716000 -2 0.884965 0.769866 0.853000 -3 0.722994 0.723213 0.875000 -4 0.609488 0.730594 0.865000 -Total time: 01:14 -Saving models at /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.39589784, tensor(0.8692)] -0.39589783549308777 -0.8692499995231628 -``` - - -## wide 2 -```bash -$ CUDA_VISIBLE_DEVICES=1 python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4-wide2' --max-vocab 15000 --lang ${LANG} --qrnn=True --nh 3100 - train 10 --bs=100 --drop_mult=0 --label-smoothing-eps=0.1 -Max vocab: 15000 -Cache dir: data/wiki/ru-100/models/sp15k -Model dir: data/wiki/ru-100/models/sp15k/qrnn_nl4-wide2.m -Wiki text was split to 193047 articles -Wiki text was split to 460 articles -Data lm, trn: 193047, val: 460 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', '▁на', 'х'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Training lm from random weights -epoch train_loss valid_loss accuracy -1 3.908233 3.950865 0.463469 -2 3.738863 3.815026 0.477703 -3 3.696502 3.779513 0.483625 -4 3.692592 3.720908 0.490143 -5 3.600519 3.652444 0.501671 -6 3.564568 3.582584 0.511550 -7 3.472859 3.493226 0.525943 -8 3.390483 3.407970 0.541749 -9 3.351620 3.344207 0.552758 -10 3.329683 3.330087 0.556380 -Total time: 51:05:43 -data/wiki/ru-100/models/sp15k -Saving info data/wiki/ru-100/models/sp15k/qrnn_nl4-wide2.m/info.json -``` -### MLDoc -export CUDA_VISIBLE_DEVICES=1 -LANG=ru -NAME=nl4-wide2 -python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/sp15k/qrnn_${NAME}.m --lang=${LANG} --name ${NAME} - train 20 --bs 18 --num-cls-epochs=4 --lr_sched=1cycle --label-smoothing-eps=0.1 - - - -## Merity nl4 -```bash - CUDA_VISIBLE_DEVICES=2 python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4-merity' --max-vocab 15000 --lang ${LANG} --qrnn=True --bptt=140 --nh 2500 - train 14 --bs=50 --drop_mult=0 --label-smoothing-eps=0.1 -Max vocab: 15000 -Cache dir: data/wiki/ru-100/models/sp15k -Model dir: data/wiki/ru-100/models/sp15k/qrnn_nl4-merity.m -Wiki text was split to 193047 articles -Wiki text was split to 460 articles -Data lm, trn: 193047, val: 460 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', '▁на', 'х'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Training lm from random weights -epoch train_loss valid_loss accuracy -1 3.965899 3.977734 0.460046 -2 3.806082 3.858176 0.472396 -3 3.839230 3.874757 0.469224 -4 3.762105 3.868653 0.469943 -5 3.800827 3.833991 0.474116 -6 3.755466 3.796329 0.479868 -7 3.691958 3.747888 0.487367 -8 3.660529 3.702986 0.493545 -9 3.593282 3.635035 0.504086 -10 3.585948 3.579200 0.513631 -11 3.473865 3.512114 0.525391 -12 3.451973 3.455807 0.535520 -13 3.418731 3.417129 0.542943 -14 3.385637 3.407541 0.545545 -Total time: 51:32:09 -data/wiki/ru-100/models/sp15k -Saving info data/wiki/ru-100/models/sp15k/qrnn_nl4-merity.m/info.json -``` - -#### MLDoc - -export CUDA_VISIBLE_DEVICES=2 -LANG=ru -NAME=nl4-merity -python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/sp15k/qrnn_${NAME}.m --lang=${LANG} --name ${NAME} - train 20 --bs 18 --num-cls-epochs=4 --lr_sched=1cycle --label-smoothing-eps=0.1 diff --git a/results/logs/ru/merity4.md b/results/logs/ru/merity4.md deleted file mode 100644 index d84b258..0000000 --- a/results/logs/ru/merity4.md +++ /dev/null @@ -1,101 +0,0 @@ -## LM - - -### MLDoc 1 -``` -python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/sp15k/qrnn_${NAME}.m --lang=${LANG} --name ${NAME} - train 20 --bs 18 --num-cls-epochs=4 --lr_sched=1cycle --label-smoothing-eps=0.1 -Max vocab: 15000 -Cache dir: /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k -Model dir: /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4-merity.m -Loading validation /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv -Data lm, trn: 9195, val: 1021 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', '▁на', 'х'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/test/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp15k/qrnn_nl4-merity.m/lm_best'), PosixPath('/home/test/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp15k/qrnn_nl4-merity.m/../itos')] -epoch train_loss valid_loss accuracy -1 4.411345 3.660489 0.486965 -Total time: 02:43 -epoch train_loss valid_loss accuracy -1 3.613334 3.372079 0.544188 -2 3.325245 3.100234 0.596406 -3 3.181919 2.906442 0.631586 -4 3.010830 2.767429 0.656378 -5 2.880418 2.663865 0.676339 -6 2.825526 2.571074 0.694140 -7 2.766901 2.483362 0.711652 -8 2.601965 2.417213 0.726853 -9 2.569160 2.341699 0.744193 -10 2.588142 2.272457 0.760294 -11 2.494011 2.198197 0.779175 -12 2.421921 2.135517 0.795854 -13 2.396429 2.075012 0.812815 -14 2.306572 2.019140 0.828851 -15 2.281730 1.966554 0.843595 -16 2.206670 1.927567 0.854515 -17 2.143836 1.901352 0.862114 -18 2.141715 1.884954 0.867003 -19 2.070353 1.876935 0.869214 -20 2.066195 1.874844 0.869665 -Total time: 2:12:21 -/home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k -Saving info /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4-merity.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.994393 0.755689 0.844000 -2 0.859871 0.822650 0.856000 -3 0.678185 0.721333 0.859000 -4 0.586906 0.693618 0.878000 -Total time: 04:17 -Saving models at /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4-merity.m -Loss and accuracy using (cls_best): [0.3872361, tensor(0.8777)] -0.387236088514328 -0.8777499794960022 -``` - -### MLDoc 2 -``` -(multifit) test@test:~/workspace/ulmfit-multilingual$ python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/mldoc/${LANG}-1/models/sp15k/qrnn_${NAME}. -m --lang=${LANG} --name ${NAME}-16 - train 0 --bs 18 --num-cls-epochs=16 --lr_sched=1cycle --label-smoothing-eps=0.1 -Max vocab: 15000 -Cache dir: /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k -Model dir: /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4-merity-16.m -Loading validation /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv -Data lm, trn: 9195, val: 1021 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', '▁на', 'х'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k -Saving info /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4-merity-16.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.081481 0.837775 0.781000 -2 0.901621 0.798574 0.858000 -3 0.778870 0.826576 0.859000 -4 0.693465 0.787875 0.833000 -5 0.639763 0.841092 0.861000 -6 0.595044 0.731504 0.853000 -7 0.576115 0.796013 0.819000 -8 0.544098 0.744034 0.875000 -9 0.531359 0.699035 0.879000 -10 0.513886 0.698310 0.879000 -11 0.495473 0.686897 0.864000 -12 0.489863 0.688584 0.881000 -13 0.481086 0.675660 0.881000 -14 0.479960 0.684917 0.883000 -15 0.490157 0.687865 0.882000 -16 0.486081 0.679104 0.882000 -Total time: 15:26 -Saving models at /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4-merity-16.m -Loss and accuracy using (cls_best): [0.4047818, tensor(0.8737)] -0.4047817885875702 -0.8737499713897705 -``` \ No newline at end of file diff --git a/results/logs/ru/merity5.md b/results/logs/ru/merity5.md deleted file mode 100644 index fe8bddf..0000000 --- a/results/logs/ru/merity5.md +++ /dev/null @@ -1,38 +0,0 @@ -## LM -```bash -CUDA_VISIBLE_DEVICES=0 python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 5 --name 'nl5-mer -ity' --max-vocab 15000 --lang ${LANG} --qrnn=True --bptt=140 --nh 2500 - train 14 --bs=50 --drop_mult=0 --label-smoothing-eps=0.1 -Max vocab: 15000 -Cache dir: data/wiki/ru-100/models/sp15k -Model dir: data/wiki/ru-100/models/sp15k/qrnn_nl5-merity.m -Wiki text was split to 193047 articles -Wiki text was split to 460 articles -Data lm, trn: 193047, val: 460 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', '▁на', 'х'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Training lm from random weights -epoch train_loss valid_loss accuracy -1 3.969639 4.024787 0.452887 -2 3.814622 3.834142 0.476612 -3 3.798372 3.846118 0.473666 -4 3.742609 3.835311 0.474612 -5 3.715114 3.790690 0.480469 -6 3.652987 3.748408 0.486146 -7 3.573350 3.697325 0.493774 -8 3.589853 3.637134 0.504189 -9 3.558110 3.583030 0.512137 -10 3.501382 3.510491 0.524148 -11 3.408982 3.437177 0.536634 -12 3.402717 3.373548 0.548113 -13 3.293624 3.331311 0.556288 -14 3.309859 3.322777 0.558426 -Total time: 68:05:15 -data/wiki/ru-100/models/sp15k -Saving info data/wiki/ru-100/models/sp15k/qrnn_nl5-merity.m/info.json -``` - -### MLDoc 1 -``` - -``` \ No newline at end of file diff --git a/results/logs/ru/nl8.md b/results/logs/ru/nl8.md deleted file mode 100644 index ba03d02..0000000 --- a/results/logs/ru/nl8.md +++ /dev/null @@ -1,78 +0,0 @@ -python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/sp15k/qrnn_${NAME}.m --lang=${LANG} --name ${NAME}-2 - train 20 --bs 18 --num-cls-epochs=4 --lr_sched=1cycle --label-smoothing-eps=0.1 -Max vocab: 15000 -Cache dir: /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k -Model dir: /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl8-2.m -Loading validation /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv -Data lm, trn: 9195, val: 1021 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', '▁на', 'х'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp15k/qrnn_nl8.m/lm_best'), PosixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp15k/qrnn_nl8.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.966292 3.450758 0.527065 -Total time: 02:58 -epoch train_loss valid_loss accuracy -1 3.495761 3.276329 0.560047 -2 3.319947 3.102911 0.593742 -3 3.137904 2.955317 0.620171 -4 3.040286 2.839161 0.642270 -5 2.869962 2.753622 0.658331 -6 2.905739 2.680881 0.672860 -7 2.836454 2.620925 0.685026 -8 2.857271 2.569716 0.695722 -9 2.702872 2.520050 0.705589 -10 2.701559 2.473591 0.715346 -11 2.740815 2.429558 0.725597 -12 2.646513 2.389550 0.735010 -13 2.587685 2.349614 0.744885 -14 2.546527 2.311087 0.754463 -15 2.568136 2.278581 0.762980 -16 2.492115 2.252367 0.769275 -17 2.338561 2.230529 0.775072 -18 2.447506 2.218215 0.778437 -19 2.364424 2.212115 0.780085 -20 2.367132 2.210520 0.780424 -Total time: 1:30:47 -/home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k -Saving info /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl8-2.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.969255 0.769736 0.843000 -2 0.846340 0.813483 0.839000 -3 0.718175 0.705339 0.867000 -4 0.609513 0.726442 0.875000 -Total time: 02:54 -Saving models at /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl8-2.m -Loss and accuracy using (cls_best): [0.4056449, tensor(0.8698)] -0.40564489364624023 - - - -(fastaiv1) n-waves@GV100:~/workspace/ulmfit-multilingual$ CUDA_VISIBLE_DEVICES=0 python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4-merity-wide2' --max-vocab 15000 --lang ${LANG} --qrnn=True --bptt=140 --nh 3100 - train 10 --bs=50 --drop_mult=0 --label-smoothing-eps=0.1 -Max vocab: 15000 -Cache dir: data/wiki/ru-100/models/sp15k -Model dir: data/wiki/ru-100/models/sp15k/qrnn_nl4-merity-wide2.m -Wiki text was split to 193047 articles -Wiki text was split to 460 articles -Data lm, trn: 193047, val: 460 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', '▁на', 'х'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Training lm from random weights -epoch train_loss valid_loss accuracy -1 3.871650 3.908779 0.467000 -2 3.834093 3.884629 0.467916 -3 3.741005 3.870331 0.469612 -4 3.785444 3.818511 0.476906 -5 3.741888 3.752743 0.486148 -6 3.678481 3.672177 0.499054 -7 3.570398 3.581498 0.512801 -8 3.455193 3.482614 0.530569 -9 3.379779 3.409405 0.543477 -10 3.384574 3.387195 0.548881 -Total time: 27:24:33 -data/wiki/ru-100/models/sp15k \ No newline at end of file diff --git a/results/logs/ru/wide2.md b/results/logs/ru/wide2.md deleted file mode 100644 index 1637791..0000000 --- a/results/logs/ru/wide2.md +++ /dev/null @@ -1,183 +0,0 @@ - 3 2.812496 2.877055 0.468569 - -4 2.705551 2.792535 0.479420 - -5 2.649598 2.726415 0.487439 - -6 2.599835 2.635610 0.499679 - -7 2.574639 2.554657 0.512358 - -8 2.489573 2.475936 0.523280 - -9 2.396540 2.415555 0.534089 - -10 2.374290 2.401968 0.536601 - -Total time: 15:49:20 -data/wiki/ru-100/models/sp15k -Saving info data/wiki/ru-100/models/sp15k/qrnn_nl4-wide2.m/info.json -Fire trace: -1. Initial component -2. Accessed property "lm" (/home/test/workspace/ulmfit-multilingual/ulmfit/__main__.py:32) -3. Called routine "LMHyperParams" (/home/test/workspace/ulmfit-multilingual/ulmfit/__main__.py:32) -4. Accessed property "train" (/home/test/workspace/ulmfit-multilingual/ulmfit/pretrain_lm.py:174) -5. Called routine "train_lm" (/home/test/workspace/ulmfit-multilingual/ulmfit/pretrain_lm.py:174) -6. ('Could not consume arg:', '--nh') - -Type: NoneType -String form: None - -Usage: __main__.py lm --dataset-path data/wiki/ru-100 --tokenizer=sp --nl 4 --name nl4-wide2 --max-vocab 15000 --lang ru --qrnn=True - train 10 --bs=100 --drop_mult=0 - -(multifit) test@test:~/workspace/ulmfit-multilingual$ less data/wiki/ru-100/models/sp15k/qrnn_nl4-wide2.m/info.json -(multifit) test@test:~/workspace/ulmfit-multilingual$ CUDA_VISIBLE_DEVICES=1 python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4-wide2' --max-vocab 15000 --lang ${LANG} --qrnn=True - train 10 --bs=100 --drop_mult=0 ^C100 -- -(multifit) test@test:~/workspace/ulmfit-multilingual$ mv data/wiki/ru-100/models/sp15k/qrnn_nl4-wide2.m/ data/wiki/ru-100/models/sp15k/qrnn_nl4-2.m/ -(multifit) test@test:~/workspace/ulmfit-multilingual$ less data/wiki/ru-100/models/sp15k/qrnn_nl4-wide2.m/info.json^C -(multifit) test@test:~/workspace/ulmfit-multilingual$ CUDA_VISIBLE_DEVICES=1 python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4-wid -e2' --max-vocab 15000 --lang ${LANG} --qrnn=True --nh 3100 - train 10 --bs=100 --drop_mult=0 -Max vocab: 15000 -Cache dir: data/wiki/ru-100/models/sp15k -Model dir: data/wiki/ru-100/models/sp15k/qrnn_nl4-wide2.m -^CTraceback (most recent call last): - File "/home/test/anaconda3/envs/multifit/lib/python3.7/runpy.py", line 193, in _run_module_as_main - "__main__", mod_spec) - File "/home/test/anaconda3/envs/multifit/lib/python3.7/runpy.py", line 85, in _run_code - exec(code, run_globals) - File "/home/test/workspace/ulmfit-multilingual/ulmfit/__main__.py", line 119, in - fire.Fire(ULMFiT()) - File "/home/test/anaconda3/envs/multifit/lib/python3.7/site-packages/fire/core.py", line 127, in Fire - component_trace = _Fire(component, args, context, name) - File "/home/test/anaconda3/envs/multifit/lib/python3.7/site-packages/fire/core.py", line 366, in _Fire - component, remaining_args) - File "/home/test/anaconda3/envs/multifit/lib/python3.7/site-packages/fire/core.py", line 542, in _CallCallable - result = fn(*varargs, **kwargs) - File "/home/test/workspace/ulmfit-multilingual/ulmfit/pretrain_lm.py", line 176, in train_lm - data_lm = self.load_wiki_data(bs=bs) if data_lm is None else data_lm - File "/home/test/workspace/ulmfit-multilingual/ulmfit/pretrain_lm.py", line 253, in load_wiki_data - train_df=read_wiki_articles(trn_path), - File "/home/test/workspace/ulmfit-multilingual/ulmfit/pretrain_lm.py", line 48, in read_wiki_articles - if i < len(lines)-2 and lines[i+1].strip() == "" and istitle(lines[i+2]): - File "/home/test/workspace/ulmfit-multilingual/ulmfit/pretrain_lm.py", line 39, in istitle - return len(re.findall(r'^ ?= [^=]* = ?$', line)) != 0 - File "/home/test/anaconda3/envs/multifit/lib/python3.7/re.py", line 223, in findall - return _compile(pattern, flags).findall(string) -KeyboardInterrupt -^C -(multifit) test@test:~/workspace/ulmfit-multilingual$ CUDA_VISIBLE_DEVICES=1 python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4-wide2' --max-vocab 15000 --lang ${LANG} --qrnn=True --nh 3100 - train 10 --bs=100 --drop_mult=0 --label-smoothing-eps=0.1 -Max vocab: 15000 -Cache dir: data/wiki/ru-100/models/sp15k -Model dir: data/wiki/ru-100/models/sp15k/qrnn_nl4-wide2.m -Wiki text was split to 193047 articles -Wiki text was split to 460 articles -Data lm, trn: 193047, val: 460 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', '▁на', 'х'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Training lm from random weights -epoch train_loss valid_loss accuracy -1 3.908233 3.950865 0.463469 -2 3.738863 3.815026 0.477703 -3 3.696502 3.779513 0.483625 -4 3.692592 3.720908 0.490143 -5 3.600519 3.652444 0.501671 -6 3.564568 3.582584 0.511550 -7 3.472859 3.493226 0.525943 -8 3.390483 3.407970 0.541749 -9 3.351620 3.344207 0.552758 -10 3.329683 3.330087 0.556380 -Total time: 51:05:43 -data/wiki/ru-100/models/sp15k -Saving info data/wiki/ru-100/models/sp15k/qrnn_nl4-wide2.m/info.json -(multifit) test@test:~/workspace/ulmfit-multilingual$ export CUDA_VISIBLE_DEVICES=1 -(multifit) test@test:~/workspace/ulmfit-multilingual$ LANG=ru -(multifit) test@test:~/workspace/ulmfit-multilingual$ NAME=nl4-wide2 -(multifit) test@test:~/workspace/ulmfit-multilingual$ python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/sp15k/qrnn_${NAME}.m --lang=${LANG} --name ${NAME} - train 20 --bs 18 --num-cls-epochs=4 --lr_sched=1cycle --label-smoothing-eps=0.1 -Max vocab: 15000 -Cache dir: /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k -Model dir: /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4-wide2.m -Loading validation /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv -Data lm, trn: 9195, val: 1021 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', '▁на', 'х'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/test/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp15k/qrnn_nl4-wide2.m/lm_best'), PosixPath('/home/test/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp15k/qrnn_nl4-wide2.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.777423 3.222261 0.564503 -Total time: 04:35 -epoch train_loss valid_loss accuracy -1 3.292465 3.029143 0.602257 -2 3.034045 2.858176 0.634576 -3 2.943366 2.710314 0.665116 -4 2.722069 2.596702 0.687515 -5 2.819853 2.508158 0.705020 -6 2.734984 2.417240 0.724748 -7 2.674353 2.332395 0.743694 -8 2.527344 2.251373 0.762892 -9 2.473972 2.168185 0.784043 -10 2.359504 2.093983 0.803255 -11 2.287590 2.019540 0.823566 -12 2.254421 1.943832 0.845138 -13 2.203321 1.884380 0.863381 -14 2.142532 1.824186 0.881509 -15 2.121573 1.777664 0.894901 -16 2.013238 1.740772 0.905824 -17 2.026189 1.715271 0.913569 -18 1.904322 1.700163 0.917917 -19 1.889113 1.692539 0.919811 -20 1.903118 1.691033 0.920319 -Total time: 3:10:09 -/home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k -Saving info /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4-wide2.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.006922 0.880313 0.788000 -2 0.823572 0.782953 0.860000 -3 0.679078 0.749164 0.872000 -4 0.579215 0.707200 0.872000 -Total time: 06:30 -Saving models at /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4-wide2.m -Loss and accuracy using (cls_best): [0.3935929, tensor(0.8708)] -0.393592894077301 -0.8707500100135803 -(multifit) test@test:~/workspace/ulmfit-multilingual$ python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/mldoc/${LANG}-1/models/sp15k/qrnn_${NAME}.m --lang=${LANG} --name ${NAME}-16 - train 0 --bs 18 --num-cls-epochs=16 --lr_sched=1cycle --label-smoothing-eps=0.1 -Max vocab: 15000 -Cache dir: /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k -Model dir: /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4-wide2-16.m -Loading validation /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv -Data lm, trn: 9195, val: 1021 -Data cls, trn: 1000, val: 1000 -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', '▁на', 'х'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k -Saving info /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4-wide2-16.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 1.067446 0.822965 0.824000 -2 0.897088 0.845636 0.826000 -3 0.778055 0.828693 0.847000 -4 0.685080 0.893327 0.823000 -5 0.620457 0.929057 0.800000 -6 0.587644 0.802154 0.859000 -7 0.570255 0.713434 0.872000 -8 0.543071 0.705259 0.871000 -9 0.517465 0.715090 0.867000 -10 0.498291 0.695459 0.876000 -11 0.497857 0.698052 0.862000 -12 0.486924 0.681911 0.878000 -13 0.479041 0.676714 0.874000 -14 0.475131 0.677843 0.878000 -15 0.467238 0.672065 0.876000 -16 0.476889 0.680850 0.875000 -Total time: 23:47 -Saving models at /home/test/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4-wide2-16.m -Loss and accuracy using (cls_best): [0.41155785, tensor(0.8700)] -0.4115578532218933 -0.8700000047683716 \ No newline at end of file diff --git a/results/logs/tokenization/ru.md b/results/logs/tokenization/ru.md deleted file mode 100644 index 6adaa2e..0000000 --- a/results/logs/tokenization/ru.md +++ /dev/null @@ -1,29 +0,0 @@ -``` -LANG=ru -python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='vf' --nl 4 --name 'nl4' --max-vocab 60000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0 --label-smoothing-eps=0.1 -Max vocab: 60000 -Cache dir: data/wiki/ru-100/models/vf60k -Model dir: data/wiki/ru-100/models/vf60k/qrnn_nl4.m -Wiki text was split to 193047 articles -Wiki text was split to 460 articles -Running tokenization lm... -Data lm, trn: 193047, val: 460 -Size of vocabulary: 60003 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', ',', '\n', '.', 'в', 'и', ')', '(', 'на', '—', '«', '»', 'с'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Training lm from random weights -epoch train_loss valid_loss accuracy -1 4.586900 4.478803 0.413023 -2 4.483496 4.400461 0.418495 -3 4.484620 4.390928 0.418422 -4 4.373594 4.350045 0.422567 -5 4.350337 4.307665 0.427411 -6 4.314571 4.249700 0.436324 -7 4.232540 4.183857 0.446341 -8 4.252573 4.119820 0.455522 -9 4.136978 4.088805 0.462345 -10 4.116755 4.079840 0.465394 -Total time: 11:24:03 -data/wiki/ru-100/models/vf60k -``` - diff --git a/results/logs/zeroshot.md b/results/logs/zeroshot.md deleted file mode 100644 index fbb6199..0000000 --- a/results/logs/zeroshot.md +++ /dev/null @@ -1,1340 +0,0 @@ -## Laser Perforamnce - -Accuracy matrix: -### 1k - -| Train | en | de | es | fr | it | ru | zh | -|-------|-------|-------|-------|-------|-------|-------|-------| -| en: | 91.48 | 87.65 | 75.48 | 84.00 | 71.18 | 66.58 | 76.65 | -| de: | 78.23 | 93.50 | 81.40 | 81.50 | 74.53 | 64.58 | 73.20 | -| es: | 71.62 | 84.00 | 93.73 | 78.90 | 73.38 | 53.33 | 55.83 | -| fr: | 81.30 | 88.75 | 80.12 | 90.85 | 72.58 | 67.35 | 79.40 | -| it: | 74.33 | 83.53 | 80.58 | 79.78 | 84.48 | 66.45 | 63.35 | -| ru: | 72.38 | 81.65 | 65.73 | 71.30 | 63.33 | 85.45 | 59.58 | -| zh: | 74.98 | 81.35 | 72.20 | 73.28 | 70.08 | 66.23 | 88.30 | - -### 10k - -| Train | en | de | es | fr | it | ru | zh | -|-------|-------|-------|-------|-------|-------|-------|-------| -| en: | 92.70 | 87.43 | 77.38 | 78.70 | 72.53 | 67.70 | 75.18 | -| de: | 81.60 | 95.40 | 83.50 | 82.85 | 76.60 | 68.80 | 73.12 | -| es: | 73.48 | 87.13 | 94.40 | 81.63 | 76.70 | 58.65 | 72.98 | -| fr: | 85.08 | 91.65 | 81.05 | 93.65 | 75.08 | 70.73 | 76.33 | -| it: | 76.75 | 86.68 | 82.55 | 82.65 | 87.80 | 65.90 | 73.35 | -| ru: | 75.23 | 81.88 | 66.83 | 68.60 | 67.38 | 87.00 | 62.68 | -| zh: | 76.05 | 82.05 | 68.40 | 77.38 | 68.45 | 66.88 | 90.38 | - - - -### laser 1k -#### Building dataset -```bash -for SRC_LANG in en de fr; do ✘ 130 - for LANG in en de es fr it ru zh; do - echo $LANG from $SRC_LANG - python ../../source/classify.py embed10000/mldoc.${SRC_LANG}-${SRC_LANG}.h5 ~/workspace/ulmfit-multilingual/data/mldoc/${LANG}-10 --suffix=10 | grep Test: - done -done -``` - -``` -for SRC_LANG in en de fr; do ✘ 130 - for LANG in en de es fr it ru zh; do - echo $LANG from $SRC_LANG - python ../../source/classify.py embed1000/mldoc.${SRC_LANG}-${SRC_LANG}.h5 ~/workspace/ulmfit-multilingual/data/mldoc/${LANG}-1 | grep Test: - done -done - -en from en - | Test: 91.48% | classes: 23.77 24.90 26.25 25.07 -de from en - | Test: 87.65% | classes: 21.98 24.45 27.65 25.93 -es from en - | Test: 75.48% | classes: 21.60 15.82 22.10 40.48 -fr from en - | Test: 84.00% | classes: 23.18 29.12 27.90 19.80 -it from en - | Test: 71.18% | classes: 23.65 22.88 25.68 27.80 -ru from en - | Test: 66.58% | classes: 29.48 13.78 34.52 22.23 -zh from en - | Test: 76.65% | classes: 30.25 31.30 13.93 24.52 -en from de - | Test: 78.23% | classes: 31.80 17.73 30.15 20.32 -de from de - | Test: 93.50% | classes: 24.45 25.45 26.00 24.10 -es from de - | Test: 81.40% | classes: 24.15 25.77 20.12 29.95 -fr from de - | Test: 81.50% | classes: 25.52 29.45 27.45 17.57 -it from de - | Test: 74.53% | classes: 24.70 27.25 22.43 25.62 -ru from de - | Test: 64.58% | classes: 45.62 9.12 26.73 18.52 -zh from de - | Test: 73.20% | classes: 31.20 43.38 7.60 17.82 -en from fr - | Test: 81.30% | classes: 28.95 18.02 24.98 28.05 -de from fr - | Test: 88.75% | classes: 24.00 23.75 24.85 27.40 -es from fr - | Test: 80.12% | classes: 24.50 14.82 18.40 42.27 -fr from fr - | Test: 90.85% | classes: 24.50 24.75 24.68 26.07 -it from fr - | Test: 72.58% | classes: 25.45 24.10 17.50 32.95 -ru from fr - | Test: 67.35% | classes: 47.15 13.62 16.68 22.55 -zh from fr - | Test: 79.40% | classes: 33.60 31.12 9.07 26.20 -``` - -#### ULMFit zershot on laser-en1k 4 epochs -```bash -python -m ulmfit eval --glob="mldoc/*-1/models/sp15k/qrnn_nl4.m" --dataset_template="{lang}-1*-laser-en1" --name nl4 --num-cls-epochs=4 --label-smoothing-eps=0.1 --lr_sched=1cycle --bs=18 - python -m ulmfit eval --glob="mldoc/*-1/models/sp15k/qrnn_nl4.m" --dataset_template='${lang}-1*-laser-en1' --name nl4 --num-cls-epochs=4 --label-smoothing-eps=0.1 --lr_sched=1cycle --bs=18 -Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m -de-1*-laser-en1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-en1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-en1/models/sp15k/qrnn_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-en1/de.dev.csv -Running tokenization lm... -Data lm, trn: 13500, val: 1500 -Running tokenization cls... -Data cls, trn: 1000, val: 1000 -Running tokenization tst... -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: - - !! WARNING !! - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Your compiler (c++) may be ABI-incompatible with PyTorch! -Please use a compiler that is ABI-compatible with GCC 4.9 and above. -See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html. - -See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6 -for instructions on how to install GCC 4.9 or higher. -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - !! WARNING !! - - warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler)) -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-en1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-en1/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.858069 0.756738 0.853000 -2 0.737577 0.687798 0.909000 -3 0.611922 0.615140 0.919000 -4 0.544274 0.608824 0.909000 -Total time: 01:08 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-en1/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.31768194, tensor(0.9133)] -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10-laser-en1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10-laser-en1/models/sp15k/qrnn_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10-laser-en1/de.dev.csv -Running tokenization lm... -Data lm, trn: 13500, val: 1500 -Running tokenization cls... -Data cls, trn: 10000, val: 1000 -Running tokenization tst... -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10-laser-en1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10-laser-en1/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.665453 0.600660 0.924000 -2 0.624768 0.595788 0.922000 -3 0.576251 0.580166 0.930000 -4 0.520507 0.569867 0.930000 -Total time: 09:38 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-10-laser-en1/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.3278069, tensor(0.9190)] -Processing data/mldoc/en-1/models/sp15k/qrnn_nl4.m -en-1*-laser-en1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1-laser-en1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1-laser-en1/models/sp15k/qrnn_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1-laser-en1/en.dev.csv -Running tokenization lm... -Data lm, trn: 13500, val: 1500 -Running tokenization cls... -Data cls, trn: 1000, val: 1000 -Running tokenization tst... -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁the', '▁,', 's', '▁.', '▁of', '▁and', '▁in', '▁to', '▁a', 'ed'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1-laser-en1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1-laser-en1/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.836131 0.760151 0.887000 -2 0.691355 0.666257 0.905000 -3 0.587016 0.603976 0.932000 -4 0.529849 0.584768 0.943000 -Total time: 01:09 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1-laser-en1/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.20341124, tensor(0.9503)] -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-10-laser-en1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-10-laser-en1/models/sp15k/qrnn_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-10-laser-en1/en.dev.csv -Running tokenization lm... -Data lm, trn: 13500, val: 1500 -Running tokenization cls... -Data cls, trn: 10000, val: 1000 -Running tokenization tst... -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁the', '▁,', 's', '▁.', '▁of', '▁and', '▁in', '▁to', '▁a', 'ed'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-10-laser-en1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-10-laser-en1/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.676697 0.611570 0.931000 -2 0.647868 0.629156 0.916000 -3 0.578947 0.546653 0.943000 -4 0.539242 0.550864 0.949000 -Total time: 10:25 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-10-laser-en1/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.2546631, tensor(0.9490)] -Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m -es-1*-laser-en1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en1/models/sp15k/qrnn_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en1/es.dev.csv -Running tokenization lm... -Data lm, trn: 13013, val: 1445 -Running tokenization cls... -Data cls, trn: 1000, val: 1000 -Running tokenization tst... -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en1/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.947175 0.774158 0.854000 -2 0.827687 0.747528 0.859000 -3 0.698278 0.726713 0.881000 -4 0.603821 0.729449 0.878000 -Total time: 00:58 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en1/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.77420205, tensor(0.7893)] -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10-laser-en1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10-laser-en1/models/sp15k/qrnn_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10-laser-en1/es.dev.csv -Running tokenization lm... -Data lm, trn: 13013, val: 1445 -Running tokenization cls... -Data cls, trn: 9458, val: 1000 -Running tokenization tst... -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10-laser-en1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10-laser-en1/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.654948 0.968341 0.863000 -2 0.634143 0.844979 0.873000 -3 0.559031 0.719894 0.890000 -4 0.524448 0.675569 0.887000 -Total time: 05:36 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-10-laser-en1/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.70765454, tensor(0.7880)] -Processing data/mldoc/fr-1/models/sp15k/qrnn_nl4.m -fr-1*-laser-en1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en1/models/sp15k/qrnn_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en1/fr.dev.csv -Running tokenization lm... -Data lm, trn: 13500, val: 1500 -Running tokenization cls... -Data cls, trn: 1000, val: 1000 -Running tokenization tst... -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', 's', '▁.', "'", '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en1/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.882636 0.720057 0.851000 -2 0.766720 0.790358 0.847000 -3 0.661309 0.669355 0.877000 -4 0.584607 0.676029 0.889000 -Total time: 01:05 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en1/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.32164142, tensor(0.8945)] -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-10-laser-en1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-10-laser-en1/models/sp15k/qrnn_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-10-laser-en1/fr.dev.csv -Running tokenization lm... -Data lm, trn: 13500, val: 1500 -Running tokenization cls... -Data cls, trn: 10000, val: 1000 -Running tokenization tst... -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', 's', '▁.', "'", '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-10-laser-en1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-10-laser-en1/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.721444 0.714190 0.865000 -2 0.684221 0.688866 0.879000 -3 0.616377 0.624161 0.904000 -4 0.576405 0.630186 0.904000 -Total time: 09:37 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-10-laser-en1/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.39518934, tensor(0.8848)] -Processing data/mldoc/it-1/models/sp15k/qrnn_nl4.m -it-1*-laser-en1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-en1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-en1/models/sp15k/qrnn_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-en1/it.dev.csv -Running tokenization lm... -Data lm, trn: 13500, val: 1500 -Running tokenization cls... -Data cls, trn: 1000, val: 1000 -Running tokenization tst... -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', '▁e', "▁&'", "'", '▁il', '▁la', '▁in', 'e'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-en1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-en1/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.963395 0.843734 0.777000 -2 0.892776 0.890245 0.785000 -3 0.755211 0.799284 0.815000 -4 0.629971 0.796769 0.820000 -Total time: 00:40 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-en1/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.76029295, tensor(0.7600)] -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-10-laser-en1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-10-laser-en1/models/sp15k/qrnn_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-10-laser-en1/it.dev.csv -Running tokenization lm... -Data lm, trn: 13500, val: 1500 -Running tokenization cls... -Data cls, trn: 10000, val: 1000 -Running tokenization tst... -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', '▁e', "▁&'", "'", '▁il', '▁la', '▁in', 'e'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-10-laser-en1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-10-laser-en1/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.828429 0.801399 0.820000 -2 0.769261 0.786845 0.816000 -3 0.717665 0.734139 0.845000 -4 0.597664 0.741355 0.843000 -Total time: 05:29 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-10-laser-en1/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.7915614, tensor(0.7605)] -Processing data/mldoc/ja-1/models/sp15k/qrnn_nl4.m -ja-1*-laser-en1 -Processing data/mldoc/zh-1/models/sp15k/qrnn_nl4.m -zh-1*-laser-en1 -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-en1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-en1/models/sp15k/qrnn_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-en1/zh.dev.csv -Running tokenization lm... -Data lm, trn: 13500, val: 1500 -Running tokenization cls... -Data cls, trn: 1000, val: 1000 -Running tokenization tst... -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-en1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-en1/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.964628 1.016147 0.710000 -2 0.836267 0.885741 0.789000 -3 0.694938 0.789230 0.811000 -4 0.594315 0.809480 0.811000 -Total time: 01:08 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-en1/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.5295076, tensor(0.8245)] -Max vocab: 15000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-10-laser-en1/models/sp15k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-10-laser-en1/models/sp15k/qrnn_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-10-laser-en1/zh.dev.csv -Running tokenization lm... -Data lm, trn: 13500, val: 1500 -Running tokenization cls... -Data cls, trn: 10000, val: 1000 -Running tokenization tst... -Data tst, trn: 1000, val: 4000 -Size of vocabulary: 15000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有'] -Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} -Loading pretrained model -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-10-laser-en1/models/sp15k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-10-laser-en1/models/sp15k/qrnn_nl4.m/info.json -Single training schedule -epoch train_loss valid_loss accuracy -1 0.823208 0.774504 0.821000 -2 0.771960 0.769799 0.822000 -3 0.682731 0.724021 0.847000 -4 0.595054 0.744821 0.836000 -Total time: 09:42 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-10-laser-en1/models/sp15k/qrnn_nl4.m -Loss and accuracy using (cls_best): [0.59163076, tensor(0.8127)] -OrderedDict([('data/mldoc/de-1-laser-en1/models/sp15k/qrnn_nl4.m', - 0.9132500290870667), - ('data/mldoc/de-10-laser-en1/models/sp15k/qrnn_nl4.m', - 0.9190000295639038), - ('data/mldoc/en-1-laser-en1/models/sp15k/qrnn_nl4.m', - 0.9502500295639038), - ('data/mldoc/en-10-laser-en1/models/sp15k/qrnn_nl4.m', - 0.9490000009536743), - ('data/mldoc/es-1-laser-en1/models/sp15k/qrnn_nl4.m', - 0.7892500162124634), - ('data/mldoc/es-10-laser-en1/models/sp15k/qrnn_nl4.m', - 0.7879999876022339), - ('data/mldoc/fr-1-laser-en1/models/sp15k/qrnn_nl4.m', - 0.8945000171661377), - ('data/mldoc/fr-10-laser-en1/models/sp15k/qrnn_nl4.m', - 0.8847500085830688), - ('data/mldoc/it-1-laser-en1/models/sp15k/qrnn_nl4.m', - 0.7599999904632568), - ('data/mldoc/it-10-laser-en1/models/sp15k/qrnn_nl4.m', - 0.7605000138282776), - ('data/mldoc/zh-1-laser-en1/models/sp15k/qrnn_nl4.m', - 0.8245000243186951), - ('data/mldoc/zh-10-laser-en1/models/sp15k/qrnn_nl4.m', - 0.812749981880188)]) -data/mldoc/de-1-laser-en1/models/sp15k/qrnn_nl4.m: 0.9132500290870667 -data/mldoc/de-10-laser-en1/models/sp15k/qrnn_nl4.m: 0.9190000295639038 -data/mldoc/en-1-laser-en1/models/sp15k/qrnn_nl4.m: 0.9502500295639038 -data/mldoc/en-10-laser-en1/models/sp15k/qrnn_nl4.m: 0.9490000009536743 -data/mldoc/es-1-laser-en1/models/sp15k/qrnn_nl4.m: 0.7892500162124634 -data/mldoc/es-10-laser-en1/models/sp15k/qrnn_nl4.m: 0.7879999876022339 -data/mldoc/fr-1-laser-en1/models/sp15k/qrnn_nl4.m: 0.8945000171661377 -data/mldoc/fr-10-laser-en1/models/sp15k/qrnn_nl4.m: 0.8847500085830688 -data/mldoc/it-1-laser-en1/models/sp15k/qrnn_nl4.m: 0.7599999904632568 -data/mldoc/it-10-laser-en1/models/sp15k/qrnn_nl4.m: 0.7605000138282776 -data/mldoc/zh-1-laser-en1/models/sp15k/qrnn_nl4.m: 0.8245000243186951 -``` - - -#### Evaluation of Laser 1k Performance -``` -python -m ulmfit eval --glob="mldoc/*-1/models/sp60k/lstm_nl4.m" --dataset_template="{}-laser-*" --name nl4 --cuda-id=0 -Max vocab: 60000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp60k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp60k/lstm_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/zh.dev.csv -Running tokenization... -Saving tokenized: cls.trn 13500, cls.val 1500 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 60000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁是', '▁人'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp60k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp60k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp60k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp60k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.789124 0.620514 0.781000 -epoch train_loss valid_loss accuracy -1 0.621348 0.524669 0.828000 -epoch train_loss valid_loss accuracy -1 0.497774 0.467979 0.842000 -epoch train_loss valid_loss accuracy -1 0.445851 0.479755 0.833000 -2 0.424097 0.468968 0.826000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp60k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.53502685, tensor(0.8235)] -[('data/mldoc/zh-1-laser-fr/models/sp60k/lstm_nl4.m', 0.8234999775886536)] -python -m ulmfit eval --glob="mldoc/*-1/models/sp60k/lstm_nl4.m" --dataset_template="{}-laser-*" --name nl4 --cuda-id=0 ✘ 130 -Max vocab: 60000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp60k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp60k/lstm_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/zh.dev.csv -Running tokenization... -Saving tokenized: cls.trn 13500, cls.val 1500 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 60000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁是', '▁人'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp60k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp60k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp60k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp60k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.789124 0.620514 0.781000 -epoch train_loss valid_loss accuracy -1 0.621348 0.524669 0.828000 -epoch train_loss valid_loss accuracy -1 0.497774 0.467979 0.842000 -epoch train_loss valid_loss accuracy -1 0.445851 0.479755 0.833000 -2 0.424097 0.468968 0.826000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp60k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.53502685, tensor(0.8235)] -[('data/mldoc/zh-1-laser-fr/models/sp60k/lstm_nl4.m', 0.8234999775886536)] -(fastaiv1) pczapla@galatea ~/w/ulmfit-multilingual ❯❯❯ python -m ulmfit eval --glob="mldoc/*-1/models/sp30k/lstm_nl4.m" --dataset_template="{}-laser-*" --name nl4 --cuda-id=0 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/models/sp30k/lstm_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/it.dev.csv -Running tokenization... -Saving tokenized: cls.trn 13500, cls.val 1500 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', "▁&'", "'", '▁e', '▁il', '▁la', 'e', '▁in'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/models/sp30k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.823176 0.588192 0.802000 -epoch train_loss valid_loss accuracy -1 0.654395 0.465622 0.846000 -epoch train_loss valid_loss accuracy -1 0.536948 0.453061 0.847000 -epoch train_loss valid_loss accuracy -1 0.488410 0.454361 0.845000 -2 0.450684 0.448873 0.849000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.6332891, tensor(0.7875)] -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/models/sp30k/lstm_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/de.dev.csv -Running tokenization... -Saving tokenized: cls.trn 13500, cls.val 1500 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/models/sp30k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.566941 0.389549 0.882000 -epoch train_loss valid_loss accuracy -1 0.399470 0.302616 0.898000 -epoch train_loss valid_loss accuracy -1 0.349054 0.336955 0.900000 -epoch train_loss valid_loss accuracy -1 0.278230 0.333488 0.896000 -2 0.275510 0.343370 0.899000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.26227093, tensor(0.9222)] -Traceback (most recent call last): - File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/runpy.py", line 193, in _run_module_as_main - "__main__", mod_spec) - File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/runpy.py", line 85, in _run_code - exec(code, run_globals) - File "/home/pczapla/workspace/ulmfit-multilingual/ulmfit/__main__.py", line 58, in - fire.Fire(ULMFiT()) - File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/fire/core.py", line 127, in Fire - component_trace = _Fire(component, args, context, name) - File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/fire/core.py", line 366, in _Fire - component, remaining_args) - File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/fire/core.py", line 542, in _CallCallable - result = fn(*varargs, **kwargs) - File "/home/pczapla/workspace/ulmfit-multilingual/ulmfit/__main__.py", line 41, in eval - dataset_path = get_dataset_path(base_model, dataset_template) - File "/home/pczapla/workspace/ulmfit-multilingual/ulmfit/__main__.py", line 17, in get_dataset_path - return list(ds.parent.glob(dataset_template.format(ds.name)))[0] -IndexError: list index out of range -(fastaiv1) pczapla@galatea ~/w/ulmfit-multilingual ❯❯❯ python -m ulmfit eval --glob="mldoc/*-1/models/sp30k/lstm_nl4.m" --dataset_template="{}-laser-*" --name nl4 --cuda-id=0 ✘ 1 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/models/sp30k/lstm_nl4.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/it.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Tokenized data loaded, cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', "▁&'", "'", '▁e', '▁il', '▁la', 'e', '▁in'] -Loss and accuracy using (cls_last): [0.6332891, tensor(0.7875)] -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/models/sp30k/lstm_nl4.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/de.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Tokenized data loaded, cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] -Loss and accuracy using (cls_last): [0.26227093, tensor(0.9222)] -Skipping data/mldoc/ja-1/models/sp30k/lstm_nl4.m as template {}-laser-* was not found -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp30k/lstm_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/zh.dev.csv -Running tokenization... -Saving tokenized: cls.trn 13500, cls.val 1500 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp30k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.745636 0.601781 0.812000 -epoch train_loss valid_loss accuracy -1 0.564749 0.435314 0.851000 -epoch train_loss valid_loss accuracy -1 0.485875 0.428803 0.850000 -epoch train_loss valid_loss accuracy -1 0.405431 0.439304 0.847000 -2 0.418333 0.442639 0.845000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.5289812, tensor(0.8465)] -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en/models/sp30k/lstm_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en/fr.dev.csv -Running tokenization... -Saving tokenized: cls.trn 13500, cls.val 1500 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en/models/sp30k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.669493 0.510190 0.852000 -epoch train_loss valid_loss accuracy -1 0.464863 0.349456 0.888000 -epoch train_loss valid_loss accuracy -1 0.396977 0.335358 0.879000 -epoch train_loss valid_loss accuracy -1 0.316100 0.326822 0.882000 -2 0.292052 0.326660 0.874000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.3416499, tensor(0.8878)] -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-fr/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-fr/models/sp30k/lstm_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-fr/ru.dev.csv -Running tokenization... -Saving tokenized: cls.trn 9195, cls.val 1021 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', '▁и', 'е', 'и', 'й', '▁на', 'х'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-fr/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-fr/models/sp30k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.906124 0.592495 0.797000 -epoch train_loss valid_loss accuracy -1 0.751562 0.440800 0.842000 -epoch train_loss valid_loss accuracy -1 0.631221 0.393381 0.860000 -epoch train_loss valid_loss accuracy -1 0.582251 0.376320 0.867000 -2 0.543821 0.374095 0.860000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-fr/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [1.0429544, tensor(0.6833)] -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-de/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-de/models/sp30k/lstm_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-de/es.dev.csv -Running tokenization... -Saving tokenized: cls.trn 13013, cls.val 1445 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-de/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-de/models/sp30k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.667080 0.471187 0.884000 -epoch train_loss valid_loss accuracy -1 0.553853 0.329840 0.904000 -epoch train_loss valid_loss accuracy -1 0.463647 0.309136 0.907000 -epoch train_loss valid_loss accuracy -1 0.396284 0.282263 0.911000 -2 0.368159 0.287222 0.916000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-de/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.5038375, tensor(0.8550)] -[('data/mldoc/de-1-laser-fr/models/sp30k/lstm_nl4.m', 0.922249972820282), ('data/mldoc/es-1-laser-de/models/sp30k/lstm_nl4.m', 0.8550000190734863), ('data/mldoc/fr-1-laser-en/models/sp30k/lstm_nl4.m', 0.8877500295639038), ('data/mldoc/it-1-laser-de/models/sp30k/lstm_nl4.m', 0.7875000238418579), ('data/mldoc/ru-1-laser-fr/models/sp30k/lstm_nl4.m', 0.6832500100135803), ('data/mldoc/zh-1-laser-fr/models/sp30k/lstm_nl4.m', 0.8464999794960022)] -``` -second run -``` -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-de/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-de/models/sp30k/lstm_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-de/de.dev.csv -Running tokenization... -Saving tokenized: cls.trn 13500, cls.val 1500 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-de/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-de/models/sp30k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.467292 0.243158 0.919000 -epoch train_loss valid_loss accuracy -1 0.270090 0.207252 0.941000 -epoch train_loss valid_loss accuracy -1 0.201597 0.219442 0.934000 -epoch train_loss valid_loss accuracy -1 0.193163 0.199092 0.943000 -2 0.169631 0.199501 0.940000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-de/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.16265252, tensor(0.9545)] -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-en/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-en/models/sp30k/lstm_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-en/de.dev.csv -Running tokenization... -Saving tokenized: cls.trn 13500, cls.val 1500 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-en/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-en/models/sp30k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.575276 0.419917 0.879000 -epoch train_loss valid_loss accuracy -1 0.475003 0.263138 0.909000 -epoch train_loss valid_loss accuracy -1 0.345987 0.260215 0.911000 -epoch train_loss valid_loss accuracy -1 0.305776 0.268171 0.906000 -2 0.289134 0.267642 0.911000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-en/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.23464507, tensor(0.9295)] -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/models/sp30k/lstm_nl4.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/de.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Tokenized data loaded, cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] -Loss and accuracy using (cls_last): [0.26227093, tensor(0.9222)] -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-de/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-de/models/sp30k/lstm_nl4.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-de/es.dev.csv -Tokenized data loaded, lm.trn 13013, lm.val 1445 -Tokenized data loaded, cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Loss and accuracy using (cls_last): [0.5038375, tensor(0.8550)] -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en/models/sp30k/lstm_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en/es.dev.csv -Running tokenization... -Saving tokenized: cls.trn 13013, cls.val 1445 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en/models/sp30k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.784271 0.667321 0.741000 -epoch train_loss valid_loss accuracy -1 0.601108 0.471457 0.854000 -epoch train_loss valid_loss accuracy -1 0.489287 0.428631 0.854000 -epoch train_loss valid_loss accuracy -1 0.434144 0.413409 0.864000 -2 0.443724 0.385349 0.869000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.82167965, tensor(0.8050)] -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-fr/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-fr/models/sp30k/lstm_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-fr/es.dev.csv -Running tokenization... -Saving tokenized: cls.trn 13013, cls.val 1445 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-fr/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-fr/models/sp30k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.752788 0.615142 0.786000 -epoch train_loss valid_loss accuracy -1 0.566108 0.403893 0.870000 -epoch train_loss valid_loss accuracy -1 0.503008 0.468810 0.865000 -epoch train_loss valid_loss accuracy -1 0.413641 0.448900 0.873000 -2 0.381155 0.413034 0.879000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-fr/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.7937071, tensor(0.8100)] -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-de/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-de/models/sp30k/lstm_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-de/fr.dev.csv -Running tokenization... -Saving tokenized: cls.trn 13500, cls.val 1500 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-de/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-de/models/sp30k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.674638 0.524605 0.796000 -epoch train_loss valid_loss accuracy -1 0.493693 0.401442 0.851000 -epoch train_loss valid_loss accuracy -1 0.418525 0.394886 0.859000 -epoch train_loss valid_loss accuracy -1 0.343561 0.402565 0.862000 -2 0.335855 0.418237 0.851000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-de/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.44778627, tensor(0.8737)] -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en/models/sp30k/lstm_nl4.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en/fr.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Tokenized data loaded, cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] -Loss and accuracy using (cls_last): [0.3416499, tensor(0.8878)] -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-fr/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-fr/models/sp30k/lstm_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-fr/fr.dev.csv -Running tokenization... -Saving tokenized: cls.trn 13500, cls.val 1500 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-fr/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-fr/models/sp30k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.477812 0.332947 0.894000 -epoch train_loss valid_loss accuracy -1 0.305868 0.201659 0.937000 -epoch train_loss valid_loss accuracy -1 0.208116 0.224481 0.931000 -epoch train_loss valid_loss accuracy -1 0.146847 0.214640 0.941000 -2 0.129603 0.227498 0.929000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-fr/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.19940722, tensor(0.9358)] -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/models/sp30k/lstm_nl4.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/it.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Tokenized data loaded, cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', "▁&'", "'", '▁e', '▁il', '▁la', 'e', '▁in'] -Loss and accuracy using (cls_last): [0.6332891, tensor(0.7875)] -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-en/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-en/models/sp30k/lstm_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-en/it.dev.csv -Running tokenization... -Saving tokenized: cls.trn 13500, cls.val 1500 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', "▁&'", "'", '▁e', '▁il', '▁la', 'e', '▁in'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-en/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-en/models/sp30k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.845312 0.660645 0.769000 -epoch train_loss valid_loss accuracy -1 0.699314 0.584146 0.786000 -epoch train_loss valid_loss accuracy -1 0.556744 0.531658 0.801000 -epoch train_loss valid_loss accuracy -1 0.503091 0.529716 0.805000 -2 0.474142 0.520058 0.806000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-en/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.7639212, tensor(0.7620)] -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-fr/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-fr/models/sp30k/lstm_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-fr/it.dev.csv -Running tokenization... -Saving tokenized: cls.trn 13500, cls.val 1500 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', "▁&'", "'", '▁e', '▁il', '▁la', 'e', '▁in'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-fr/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-fr/models/sp30k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.773207 0.568426 0.803000 -epoch train_loss valid_loss accuracy -1 0.570457 0.516704 0.821000 -epoch train_loss valid_loss accuracy -1 0.527280 0.460192 0.840000 -epoch train_loss valid_loss accuracy -1 0.469201 0.461563 0.841000 -2 0.458892 0.443310 0.836000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-fr/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.80693215, tensor(0.7688)] -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-de/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-de/models/sp30k/lstm_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-de/ru.dev.csv -Running tokenization... -Saving tokenized: cls.trn 9195, cls.val 1021 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', '▁и', 'е', 'и', 'й', '▁на', 'х'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-de/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-de/models/sp30k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.878202 0.549530 0.815000 -epoch train_loss valid_loss accuracy -1 0.747663 0.439798 0.860000 -epoch train_loss valid_loss accuracy -1 0.610381 0.391122 0.878000 -epoch train_loss valid_loss accuracy -1 0.563902 0.393633 0.880000 -2 0.515117 0.403987 0.878000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-de/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [1.3181443, tensor(0.6695)] -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-en/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-en/models/sp30k/lstm_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-en/ru.dev.csv -Running tokenization... -Saving tokenized: cls.trn 9195, cls.val 1021 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', '▁и', 'е', 'и', 'й', '▁на', 'х'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-en/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-en/models/sp30k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.897468 0.570228 0.801000 -epoch train_loss valid_loss accuracy -1 0.704874 0.560132 0.812000 -epoch train_loss valid_loss accuracy -1 0.595008 0.507041 0.816000 -epoch train_loss valid_loss accuracy -1 0.484754 0.479213 0.825000 -2 0.454896 0.501114 0.824000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-en/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [1.1765001, tensor(0.7005)] -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-fr/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-fr/models/sp30k/lstm_nl4.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-fr/ru.dev.csv -Tokenized data loaded, lm.trn 9195, lm.val 1021 -Tokenized data loaded, cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', '▁и', 'е', 'и', 'й', '▁на', 'х'] -Loss and accuracy using (cls_last): [1.0429544, tensor(0.6833)] -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-de/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-de/models/sp30k/lstm_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-de/zh.dev.csv -Running tokenization... -Saving tokenized: cls.trn 13500, cls.val 1500 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-de/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-de/models/sp30k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.759435 0.762386 0.707000 -epoch train_loss valid_loss accuracy -1 0.631534 0.591862 0.786000 -epoch train_loss valid_loss accuracy -1 0.534237 0.589429 0.801000 -epoch train_loss valid_loss accuracy -1 0.454291 0.589220 0.799000 -2 0.446990 0.586956 0.804000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-de/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.8401224, tensor(0.7232)] -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-en/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-en/models/sp30k/lstm_nl4.m -Training -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-en/zh.dev.csv -Running tokenization... -Saving tokenized: cls.trn 13500, cls.val 1500 -Running tokenization... -Saving tokenized: cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-en/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-en/models/sp30k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.827517 0.821250 0.712000 -epoch train_loss valid_loss accuracy -1 0.636761 0.656195 0.772000 -epoch train_loss valid_loss accuracy -1 0.582199 0.675501 0.769000 -epoch train_loss valid_loss accuracy -1 0.511542 0.634232 0.764000 -2 0.508244 0.647197 0.771000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-en/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.5421255, tensor(0.8045)] -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp30k/lstm_nl4.m -Evaluating previously trained model -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/zh.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Tokenized data loaded, cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是'] -Loss and accuracy using (cls_last): [0.5289812, tensor(0.8465)] -OrderedDict([('data/mldoc/de-1-laser-de/models/sp30k/lstm_nl4.m', - 0.9545000195503235), - ('data/mldoc/de-1-laser-en/models/sp30k/lstm_nl4.m', - 0.9294999837875366), - ('data/mldoc/de-1-laser-fr/models/sp30k/lstm_nl4.m', - 0.922249972820282), - ('data/mldoc/es-1-laser-de/models/sp30k/lstm_nl4.m', - 0.8550000190734863), - ('data/mldoc/es-1-laser-en/models/sp30k/lstm_nl4.m', - 0.8050000071525574), - ('data/mldoc/es-1-laser-fr/models/sp30k/lstm_nl4.m', - 0.8100000023841858), - ('data/mldoc/fr-1-laser-de/models/sp30k/lstm_nl4.m', - 0.8737499713897705), - ('data/mldoc/fr-1-laser-en/models/sp30k/lstm_nl4.m', - 0.8877500295639038), - ('data/mldoc/fr-1-laser-fr/models/sp30k/lstm_nl4.m', - 0.9357500076293945), - ('data/mldoc/it-1-laser-de/models/sp30k/lstm_nl4.m', - 0.7875000238418579), - ('data/mldoc/it-1-laser-en/models/sp30k/lstm_nl4.m', - 0.7620000243186951), - ('data/mldoc/it-1-laser-fr/models/sp30k/lstm_nl4.m', - 0.768750011920929), - ('data/mldoc/ru-1-laser-de/models/sp30k/lstm_nl4.m', - 0.6694999933242798), - ('data/mldoc/ru-1-laser-en/models/sp30k/lstm_nl4.m', - 0.7005000114440918), - ('data/mldoc/ru-1-laser-fr/models/sp30k/lstm_nl4.m', - 0.6832500100135803), - ('data/mldoc/zh-1-laser-de/models/sp30k/lstm_nl4.m', - 0.7232499718666077), - ('data/mldoc/zh-1-laser-en/models/sp30k/lstm_nl4.m', - 0.8044999837875366), - ('data/mldoc/zh-1-laser-fr/models/sp30k/lstm_nl4.m', - 0.8464999794960022)]) -``` - - -### Laser 10k -### Building dataset 10k -``` - for SRC_LANG in en de fr; do ✘ 130 - for LANG in en de es fr it ru zh; do - echo $LANG from $SRC_LANG - python ../../source/classify.py embed10000/mldoc.${SRC_LANG}-${SRC_LANG}.h5 ~/workspace/ulmfit-multilingual/data/mldoc/${LANG}-10 --suffix=10 | grep Test: - done -done -zsh: command not found: ✘ -en from en - | Test: 92.70% | classes: 24.65 25.52 26.20 23.62 -de from en - | Test: 87.43% | classes: 20.05 25.55 28.88 25.52 -es from en - | Test: 77.38% | classes: 24.70 15.40 22.27 37.62 -fr from en - | Test: 78.70% | classes: 17.65 29.75 34.60 18.00 -it from en - | Test: 72.53% | classes: 20.75 24.60 28.52 26.12 -ru from en - | Test: 67.70% | classes: 33.40 17.12 29.35 20.12 -zh from en - | Test: 75.18% | classes: 27.70 39.00 12.07 21.23 -zsh: command not found: ✘ -en from de - | Test: 81.60% | classes: 31.62 23.82 24.93 19.62 -de from de - | Test: 95.40% | classes: 24.57 26.00 25.62 23.80 -es from de - | Test: 83.50% | classes: 28.73 22.60 18.65 30.02 -fr from de - | Test: 82.85% | classes: 27.52 29.23 25.23 18.02 -it from de - | Test: 76.60% | classes: 26.52 25.62 21.43 26.43 -ru from de - | Test: 68.80% | classes: 48.48 13.22 19.43 18.88 -zh from de - | Test: 73.12% | classes: 33.12 40.10 10.53 16.25 -zsh: command not found: ✘ -en from fr - | Test: 85.08% | classes: 25.43 23.43 25.43 25.73 -de from fr - | Test: 91.65% | classes: 23.15 27.10 25.00 24.75 -es from fr - | Test: 81.05% | classes: 24.18 17.68 18.93 39.23 -fr from fr - | Test: 93.65% | classes: 24.25 25.10 25.80 24.85 -it from fr - | Test: 75.08% | classes: 24.07 26.85 18.25 30.82 -ru from fr - | Test: 70.73% | classes: 43.17 19.15 16.57 21.10 -zh from fr - | Test: 76.33% | classes: 34.23 34.05 10.95 20.77 - ``` - 1 10 -### Building dataset 1k -``` - for SRC_LANG in en de fr; do - for LANG in en de es fr it ru zh; do - echo $LANG from $SRC_LANG - python ../../source/classify.py embed1000/mldoc.${SRC_LANG}-${SRC_LANG}.h5 ~/workspace/ulmfit-multilingual/data/mldoc/${LANG}-10 --suffix=1 | grep Test: - done - done - for SRC_LANG in en de fr; do - for LANG in en de es fr it ru zh; do - echo $LANG from $SRC_LANG - python ../../source/classify.py embed1000/mldoc.${SRC_LANG}-${SRC_LANG}.h5 ~/workspace/ulmfit-multilingual/data/mldoc/${LANG}-10 --suffix=1 | grep Test: - done - done -en from en - | Test: 91.48% | classes: 23.77 24.90 26.25 25.07 -de from en - | Test: 87.65% | classes: 21.98 24.45 27.65 25.93 -es from en - | Test: 75.48% | classes: 21.60 15.82 22.10 40.48 -fr from en - | Test: 84.00% | classes: 23.18 29.12 27.90 19.80 -it from en - | Test: 71.18% | classes: 23.65 22.88 25.68 27.80 -ru from en - | Test: 66.58% | classes: 29.48 13.78 34.52 22.23 -zh from en - | Test: 76.65% | classes: 30.25 31.30 13.93 24.52 -en from de - | Test: 78.23% | classes: 31.80 17.73 30.15 20.32 -de from de - | Test: 93.50% | classes: 24.45 25.45 26.00 24.10 -es from de - | Test: 81.40% | classes: 24.15 25.77 20.12 29.95 -fr from de - | Test: 81.50% | classes: 25.52 29.45 27.45 17.57 -it from de - | Test: 74.53% | classes: 24.70 27.25 22.43 25.62 -ru from de - | Test: 64.58% | classes: 45.62 9.12 26.73 18.52 -zh from de - | Test: 73.20% | classes: 31.20 43.38 7.60 17.82 -en from fr - | Test: 81.30% | classes: 28.95 18.02 24.98 28.05 -de from fr - | Test: 88.75% | classes: 24.00 23.75 24.85 27.40 -es from fr - | Test: 80.12% | classes: 24.50 14.82 18.40 42.27 -fr from fr - | Test: 90.85% | classes: 24.50 24.75 24.68 26.07 -it from fr - | Test: 72.58% | classes: 25.45 24.10 17.50 32.95 -ru from fr - | Test: 67.35% | classes: 47.15 13.62 16.68 22.55 -zh from fr - | Test: 79.40% | classes: 33.60 31.12 9.07 26.20 -``` -### No Unfreeze -#### one epoch -``` -python -m ulmfit cls --dataset-path data/mldoc/fr-1-laser --base-lm-path data/mldoc/fr-1/models/sp30k/lstm_nl4.m --lang=fr --name 'nl4-no_unfreeze' --cuda-id=1 - train 0 --bs 40 --num-cls-epochs=2 --unfreeze=False -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-no_unfreeze.m -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/fr.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Tokenized data loaded, cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-no_unfreeze.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.800256 0.783174 0.701000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-no_unfreeze.m -Loss and accuracy using (cls_best): [1.1735736, tensor(0.5077)] -1.173573613166809 -0.5077499747276306 -``` -#### 4 epochs -ulmfit: 63.67% -``` -python -m ulmfit cls --dataset-path data/mldoc/fr-1-laser --base-lm-path data/mldoc/fr-1/models/sp30k/lstm_nl4.m --lang=fr --name 'nl4-no_unfreeze2' --cuda-id=1 - train 0 --bs 40 --num-cls-epochs=2 --unfreeze=False --num-cls-frozen-epochs=4 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-no_unfreeze2.m -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/fr.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Tokenized data loaded, cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-no_unfreeze2.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.832118 0.750073 0.717000 -2 0.729266 0.617375 0.749000 -3 0.645946 0.623189 0.751000 -4 0.566385 0.608672 0.760000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-no_unfreeze2.m -Loss and accuracy using (cls_best): [0.97152597, tensor(0.6367)] -``` \ No newline at end of file diff --git a/results/logs/zh.md b/results/logs/zh.md deleted file mode 100644 index c308f9d..0000000 --- a/results/logs/zh.md +++ /dev/null @@ -1,163 +0,0 @@ -# ZH - -## SP15k QRNN - -## SP30k LSTM nl 4 -### LM -``` - python -m ulmfit lm --dataset-path data/wiki/zh-100 --cuda-id=0 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 30000 --lang zh --qrnn=False - train 10 --bs=50 --drop_mult=0 -Max vocab: 30000 -Cache dir: data/wiki/zh-100/models/sp30k -Model dir: data/wiki/zh-100/models/sp30k/lstm_nl4.m -Tokenized data loaded -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': None, 'pretrained_model': None, 'drop_mult': 0} dps: [0.25 0.1 0.2 0.02 0.15] -Training lm from random weights -epoch train_loss valid_loss accuracy -1 2.736679 3.050473 0.428462 -2 2.664505 3.011505 0.432414 -3 2.607435 2.942389 0.439985 -4 2.561503 2.851523 0.451965 -5 2.499060 2.798222 0.459438 -6 2.387191 2.720054 0.471021 -7 2.356725 2.648299 0.479029 -8 2.301895 2.553860 0.493597 -9 2.275601 2.481724 0.505979 -10 2.187606 2.465159 0.509590 -``` -### MLDoc -``` -python -m ulmfit cls --dataset-path data/mldoc/zh-1 --base-lm-path data/wiki/zh-100/models/sp30k/lstm_nl4.m --lang=zh --name 'nl4' --cuda-id=0 - train 20 --bs 40 --num-cls-epochs=2 -Max vocab: 30000 -Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k -Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4.m -Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Tokenized data loaded, cls.trn 1000, cls.val 1000 -Size of vocabulary: 30000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/zh-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/zh-100/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/zh-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/zh-100/models/sp30k/lstm_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 2.604460 2.225315 0.546099 -epoch train_loss valid_loss accuracy -1 2.240892 2.020697 0.578796 -2 2.025043 1.816424 0.613192 -3 1.832658 1.646025 0.640532 -4 1.746628 1.530125 0.659058 -5 1.621672 1.425179 0.675305 -6 1.544814 1.345650 0.689195 -7 1.464704 1.271710 0.702200 -8 1.412583 1.204830 0.714764 -9 1.332440 1.147108 0.725389 -10 1.327941 1.092910 0.736447 -11 1.227284 1.039441 0.747662 -12 1.200814 0.991910 0.758105 -13 1.161579 0.947898 0.768121 -14 1.100010 0.908599 0.776732 -15 1.059006 0.872309 0.785161 -16 1.045412 0.844972 0.791998 -17 1.026688 0.824872 0.796891 -18 1.013831 0.812786 0.799699 -19 0.978586 0.807678 0.800954 -20 0.982473 0.805671 0.801201 -/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k -Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.637427 0.505143 0.836000 -epoch train_loss valid_loss accuracy -1 0.471189 0.317678 0.887000 -epoch train_loss valid_loss accuracy -1 0.384985 0.288901 0.904000 -epoch train_loss valid_loss accuracy -1 0.316358 0.275456 0.906000 -2 0.295534 0.278589 0.907000 -Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_best): [0.28411642, tensor(0.9020)] -0.2841164171695709 -0.9020000100135803 -``` - - - -## SP60k LSTM nl 4 -### LM -``` -Wiki text was split to 153503 articles -Wiki text was split to 145 articles -Size of vocabulary: 60000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁是', '▁人'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': None, 'pretrained_model': None, 'drop_mult': 0} dps: [0.25 0.1 0.2 0.02 0.15] -Training lm from random weights -epoch train_loss valid_loss accuracy -1 3.312704 3.701317 0.334740 -2 3.212988 3.648671 0.336709 -3 3.060103 3.584413 0.344427 -4 3.108131 3.477978 0.356738 -5 2.952951 3.410785 0.365901 -6 2.919397 3.325265 0.376316 -7 2.839392 3.224750 0.391707 -8 2.750095 3.132644 0.404416 -9 2.805704 3.066595 0.415245 -10 2.653435 3.055314 0.417736 -data/wiki/zh-100/models/sp60k -Saving info data/wiki/zh-100/models/sp60k/lstm_nl4.m/info.json -``` -### MLDoc -``` -python -m ulmfit cls --dataset-path data/mldoc/zh-1 --base-lm-path data/wiki/zh-100/models/sp60k/lstm_nl4.m --lang=zh --name 'nl4' --cu -da-id=0 - train 20 --bs 40 --num-cls-epochs=2 -Max vocab: 60000 -Cache dir: /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp60k -Model dir: /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp60k/lstm_nl4.m -Loading validation /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv -Tokenized data loaded, lm.trn 13500, lm.val 1500 -Tokenized data loaded, cls.trn 1000, cls.val 1000 -Size of vocabulary: 60000 -First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁是', '▁人'] -Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/zh-100/models/sp60k/lstm_nl4.m/lm_best'), Po -sixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/zh-100/models/sp60k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] -Unknown tokens 0, first 100: [] -Training lm from: [PosixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/zh-100/models/sp60k/lstm_nl4.m/lm_best'), PosixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/zh- -100/models/sp60k/lstm_nl4.m/../itos')] -epoch train_loss valid_loss accuracy -1 3.055914 2.690310 0.467917 -epoch train_loss valid_loss accuracy -1 2.713421 2.464873 0.503386 -2 2.429520 2.215309 0.543961 -3 2.247576 2.010849 0.578106 -4 2.083628 1.853473 0.602419 -5 1.969939 1.734762 0.621440 -6 1.904438 1.624005 0.640240 -7 1.783416 1.526202 0.656981 -8 1.719215 1.445780 0.671753 -9 1.621891 1.366912 0.687187 -10 1.589463 1.295759 0.701207 -11 1.510032 1.223578 0.716387 -12 1.404720 1.160607 0.729603 -13 1.414636 1.107378 0.741273 -14 1.364716 1.056422 0.753112 -15 1.327804 1.011525 0.763934 -16 1.255990 0.976447 0.771864 -17 1.181438 0.951213 0.778309 -18 1.192709 0.936060 0.781858 -19 1.190164 0.928613 0.783513 -20 1.172130 0.927612 0.783722 -/home/n-waves/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp60k -Saving info /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp60k/lstm_nl4.m/info.json -Starting classifier training -epoch train_loss valid_loss accuracy -1 0.646537 0.516221 0.836000 -epoch train_loss valid_loss accuracy -1 0.441884 0.361802 0.873000 -epoch train_loss valid_loss accuracy -1 0.376583 0.318426 0.893000 -epoch train_loss valid_loss accuracy -1 0.280910 0.314279 0.889000 -2 0.308887 0.309718 0.903000 -Saving models at /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp60k/lstm_nl4.m -Loss and accuracy using (cls_last): [0.30276635, tensor(0.8978)] -``` \ No newline at end of file diff --git a/results/time_benchmark/logs.md b/results/time_benchmark/logs.md deleted file mode 100644 index d5ec78f..0000000 --- a/results/time_benchmark/logs.md +++ /dev/null @@ -1,50 +0,0 @@ -# Results - -## Set-up. - - Num Tokens 15K - - GPU V100 - - LM BPTT = 70 - - LM BS = 64 - - CLAS BS = 32 - -| Model | LSTM | QRNN | -|----------------|-----------|-----------| -| LM ms/batch | 143ms | 71ms | -| CLAS ms/batch | 467ms | 156ms | - - -``` -> python results/time_benchmark/qrnn_benchmark.py - -Vocab size 14513 -QRNN -LM -epoch train_loss valid_loss accuracy -1 6.326089 -Total time: 00:11 -Batch size torch.Size([64, 70]) -Params = 22 MM -Training time is 71.0 ms per batch -CLAS -epoch train_loss valid_loss accuracy -1 0.712603 -Total time: 00:10 -Batch size torch.Size([32, 1445]) -Params = 22 MM -Training time is 156.0 ms per batch -LSTM -LM -epoch train_loss valid_loss accuracy -1 6.262911 -Total time: 00:21 -Batch size torch.Size([64, 70]) -Params = 37 MM -Training time is 143.0 ms per batch -CLAS -epoch train_loss valid_loss accuracy -1 0.706715 -Total time: 00:32 -Batch size torch.Size([32, 1445]) -Params = 37 MM -Training time is 467.0 ms per batch -``` \ No newline at end of file diff --git a/results/time_benchmark/qrnn_benchmark.py b/results/time_benchmark/qrnn_benchmark.py deleted file mode 100644 index f6dab86..0000000 --- a/results/time_benchmark/qrnn_benchmark.py +++ /dev/null @@ -1,52 +0,0 @@ -import glob -import shutil -import time -from fastai.text import * - -orig_path = untar_data(URLs.IMDB) -path = Path('data') / 'imdb_small' -path.mkdir(parents=True, exist_ok=True) - -for mode in ['train', 'test']: - for label in ['pos', 'neg']: - tgt_path = path / mode / label - tgt_path.mkdir(parents=True, exist_ok=True) - # Keep just 10% of the files - pattern = str(orig_path / mode / label / '3*.txt') - for file in glob.glob(pattern): - shutil.copy(file, tgt_path) - -data_lm = TextLMDataBunch.from_folder(path, valid='test') -data_clas = TextClasDataBunch.from_folder(path, bs=32, vocab=data_lm.train_ds.vocab, valid='test') - -print('Vocab size', len(data_lm.train_ds.vocab.itos)) - - -def count_parameters(model, requires_grad): - return sum(p.numel() for p in model.parameters() if p.requires_grad == requires_grad) - - -def test(qrnn, func, config, data, arch=AWD_LSTM): - total = len(list(data.train_dl)) - config = config.copy() - config['qrnn'] = qrnn - - learn = func(data, AWD_LSTM, config=config, pretrained=False) - learn.unfreeze() - params = count_parameters(learn.model, True) - total = len(list(data.train_dl)) - start_time = time.clock() - learn.fit(1) - diff = time.clock() - start_time - - print('Batch size', data.one_batch()[0].shape) - print(f'Params = {params // 1000000} MM') - print(f'Training time is {1000 * diff // total} ms per batch') - - -for qrnn in [True, False]: - print('QRNN' if qrnn else 'LSTM') - print('LM') - test(qrnn, language_model_learner, config=awd_lstm_lm_config, data=data_lm) - print('CLAS') - test(qrnn, text_classifier_learner, config=awd_lstm_clas_config, data=data_clas) \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py deleted file mode 100644 index 6ef9ba5..0000000 --- a/tests/test_end_to_end.py +++ /dev/null @@ -1,183 +0,0 @@ -import multifit.pretrain_lm -import multifit.train_clas -from multifit.datasets.utils import * -from multifit.pretrain_lm import get_data_folder - -import fastai.core -fastai.core.defaults.cpus = 1 -cuda_id=0 - -def copy_head(src_fn, dst_fn, n=1000): - with src_fn.open("r") as s, dst_fn.open("w") as d: - for i in range(n): - d.write(s.readline()) - -def get_test_data(): - data = get_data_folder() - wt = data / "wiki" / "wikitext-2" - imdb = data / "imdb" - - test_data = data / "test" - if test_data.exists(): - shutil.rmtree(test_data) - - test_wt = test_data / 'wikitext-s' - test_imdb = test_data / 'imdb' - test_wt.mkdir(exist_ok=True, parents=True) - test_imdb.mkdir(exist_ok=True, parents=True) - - sz=1 - # we use the same text to see if models overfits - copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.train.tokens', n=100*sz) - copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.valid.tokens', n=60*sz) - copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.test.tokens', n=60*sz) - copy_head(imdb / 'train.csv', test_imdb / 'train.csv', n=10*sz) - copy_head(imdb / 'train.csv', test_imdb / 'test.csv', n=6 * sz) - copy_head(imdb / 'train.csv', test_imdb / 'dev.csv', n=6 * sz) - copy_head(imdb / 'train.csv', test_imdb / 'unsup.csv', n=1*sz) - - return test_data, test_wt - - -def test_evaluate(): - """ Test ulmfit with (default) Moses tokenizer on small wikipedia dataset. - """ - os.chdir(get_data_folder()/"..") - fastai.core.defaults.cpus=0 - test_data, wt2 = get_test_data() - exp = multifit.train_clas.CLSHyperParams(test_data / 'imdb', lang='en', qrnn=False, max_vocab=1000, name="tst") - exp.evaluate_cls(save_name=None, bs=2) - - -def test_ulmfit_works_with_relative_paths(): - """ Test ulmfit with (default) Moses tokenizer on small wikipedia dataset. - """ - os.chdir(get_data_folder()/"..") - - test_data, wt2 = get_test_data() - lm_name = 'end-to-end-test-default' - cuda_id = 0 - exp = multifit.pretrain_lm.LMHyperParams( - dataset_path=wt2.relative_to(Path.cwd()), - lang='en', - qrnn=False, - max_vocab=1000, - name=lm_name, - cuda_id=cuda_id) - - exp.train_lm(num_epochs=1, bs=2) - - #assert exp.results['accuracy'] > 0.02 - - exp2 = multifit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) - exp2.train_cls(num_lm_epochs=1, unfreeze=False, bs=4,) - - # should work for the second time as well - - exp2 = multifit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) - exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, ) - - -def test_ulmfit_default_end_to_end(): - """ Test ulmfit with (default) Moses tokenizer on small wikipedia dataset. - """ - test_data, wt2 = get_test_data() - lm_name = 'end-to-end-test-default' - cuda_id = 0 - exp = multifit.pretrain_lm.LMHyperParams( - dataset_path=wt2, - lang='en', - qrnn=False, - max_vocab=1000, - name=lm_name, - cuda_id=cuda_id) - - exp.train_lm(num_epochs=1, bs=2) - - #assert exp.results['accuracy'] > 0.02 - - exp2 = multifit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) - exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4,) - -def test_ulmfit_fastai_end_to_end(): - """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset. - """ - test_data, wt2 = get_test_data() - lm_name = 'end-to-end-test-fastai' - - exp = multifit.pretrain_lm.LMHyperParams( - dataset_path=wt2, - lang='en', - cuda_id=cuda_id, - qrnn=False, - tokenizer='f', - max_vocab=100, - nl=1, - name=lm_name, - ) - exp.train_lm(num_epochs=1, bs=2) - exp2 = multifit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) - exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, ) - -def test_ulmfit_fastai_end_to_end_label_smoothing(): - """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset. - """ - test_data, wt2 = get_test_data() - lm_name = 'end-to-end-test-fastai-lablel-smoothing' - - exp = multifit.pretrain_lm.LMHyperParams( - dataset_path=wt2, - lang='en', - cuda_id=cuda_id, - qrnn=False, - tokenizer='f', - max_vocab=100, - name=lm_name, - ) - exp.train_lm(num_epochs=1, bs=2, label_smoothing_eps=0.1) - exp2 = multifit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir, name=lm_name) - exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, label_smoothing_eps=0.1 ) - -def test_ulmfit_sentencepiece_end_to_end(): - """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset. - """ - test_data, wt2 = get_test_data() - lm_name = 'end-to-end-test-spm' - - exp = multifit.pretrain_lm.LMHyperParams( - dataset_path=wt2, - lang='en', - cuda_id=cuda_id, - qrnn=False, - tokenizer=multifit.pretrain_lm.Tokenizers.SUBWORD, - max_vocab=200, - name=lm_name, - ) - exp.train_lm(num_epochs=1, bs=2) - # not supported yet - exp2 = multifit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) - exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, ) - -def test_ulmfit_sentencepiece_fastai_impl_end_to_end(): - """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset. - """ - test_data, wt2 = get_test_data() - lm_name = 'end-to-end-test-spm-fa' - - exp = multifit.pretrain_lm.LMHyperParams( - dataset_path=wt2, - lang='en', - cuda_id=cuda_id, - qrnn=False, - tokenizer=multifit.pretrain_lm.Tokenizers.FASTAI_SUBWORD, - max_vocab=200, - name=lm_name, - ) - exp.train_lm(num_epochs=1, bs=2) - # not supported yet - exp2 = multifit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) - exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, ) - -if __name__ == "__main__": - fire.Fire() # allows using all functions via CLI -