From 4e1b76feeec302781508ce9c6bfd2395b57521b0 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Tue, 12 Feb 2019 15:01:42 +0100 Subject: [PATCH] Add MLDoc summary & zeroshot logs --- results/MLDoc.md | 27 ++++ results/logs/common.md | 294 +++++++++++++++++++++++++++++++++++++++ results/logs/zeroshot.md | 265 +++++++++++++++++++++++++++++++++++ 3 files changed, 586 insertions(+) create mode 100644 results/MLDoc.md create mode 100644 results/logs/common.md create mode 100644 results/logs/zeroshot.md diff --git a/results/MLDoc.md b/results/MLDoc.md new file mode 100644 index 0000000..11c5006 --- /dev/null +++ b/results/MLDoc.md @@ -0,0 +1,27 @@ + +# non-zeroshot +| Model | en | de | es | fr | it | ja | ru | zh | +|----------------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|------------| +|LASER | 90.73 | 92.70 | 88.75 | 90.80 | 85.93 | 85.15 | 84.65 | 88.98 | +|MultiCCA | 92.2 | 93.70 | 94.45 | 92.05 | 85.55 | 85.35 | 85.65 | 87.30 | +|ULMFiT | | **95.4** | **95.15** | **93.67** | **88.42** | **89.20** | **87.27** | | +|ULMFiT 100 | | 91.35 | 83.32 | 88.77 | 77.99 | 71.12 | 72.20 | | + +# Zero shot approaches + +| Model | en | de | es | fr | it | ja | ru | zh | +|----------------------|------------|-----------|-----------|-----------|-----------|-----------|-----------|------------| +|LASER 0 shot | 80.75 (en) | 87.03 (fr)| 82.60 (it)| 82.83 (de)| 73.25 (de)| 60.95 (en)| 68.83 (it)| 72.90 (de) | +|LASER base 0 shot | | 86.48 | 79.23 | 76.73 | +|ULMFiT 0 shot | | **91.97**| **85.35** | 85.54 | +|ULMFiT 100 for comp. | | 91.35 | 83.32 | 88.77 | 77.99 | 71.12 | 72.20 | | + + +To simulate ulmfit zero shot we add noise to the training labels to simulate training from Laser labels + +| Model | en | de | es | fr | it | ja | ru | zh | +|----------------------|------------|-----------|-----------|-----------|-----------|-----------|-----------|------------| +|LASER 0 shot | 80.75 (en) | 87.03 (fr)| 82.60 (it)| 82.83 (de)| 73.25 (de)| 60.95 (en)| 68.83 (it)| 72.90 (de) | +|ULMFiT | | **95.4** | **95.15** | **93.67** | **88.42** | **89.20** | **87.27** | | +| Noise | 20% | 13% | 18% | 18% | 27% | 40% | 32% | 28% | +|ULMFiT noise ~ 0 shot | | 94.49 | 93.12 | 90.49 | 83.72 | 74.72 | 75.67 | | diff --git a/results/logs/common.md b/results/logs/common.md new file mode 100644 index 0000000..9142b2b --- /dev/null +++ b/results/logs/common.md @@ -0,0 +1,294 @@ +# MLDoc +## Limiit to 100 examples +``` +python -m ulmfit eval --glob="mldoc/*-1/models/sp30k/lstm_nl4.m" --name nl4-100e8 --cuda-id=1 --limit=100 --num-cls-epochs=8 +{ + 'data/mldoc/it-1/models/sp30k/lstm_nl4-100e8.m': 0.7799999713897705, + 'data/mldoc/de-1/models/sp30k/lstm_nl4-100e8.m': 0.9135000109672546, + 'data/mldoc/ja-1/models/sp30k/lstm_nl4-100e8.m': 0.7112500071525574, + 'data/mldoc/fr-1/models/sp30k/lstm_nl4-100e8.m': 0.8877500295639038, + 'data/mldoc/ru-1/models/sp30k/lstm_nl4-100e8.m': 0.722000002861023, + 'data/mldoc/es-1/models/sp30k/lstm_nl4-100e8.m': 0.8169999718666077 +} +``` + + +## Noise + +``` +noise=0.13 +lang=de +python -m ulmfit eval --glob="mldoc/${lang}-1/models/sp30k/lstm_nl4.m" --name nl4-noise --cuda-id=1 --num-cls-epochs=2 --noise=${noise} +{'data/mldoc/de-1/models/sp30k/lstm_nl4-noise.m': 0.9449999928474426} + +noise=0.18 +lang=es +python -m ulmfit eval --glob="mldoc/${lang}-1/models/sp30k/lstm_nl4.m" --name nl4-noise --cuda-id=1 --num-cls-epochs=2 --noise=${noise} +{'data/mldoc/es-1/models/sp30k/lstm_nl4-noise.m': 0.9312499761581421} + +noise=0.18 +lang=fr +python -m ulmfit eval --glob="mldoc/${lang}-1/models/sp30k/lstm_nl4.m" --name nl4-noise --cuda-id=1 --num-cls-epochs=2 --noise=${noise} +{'data/mldoc/fr-1/models/sp30k/lstm_nl4-noise.m': 0.9049999713897705} + +noise=0.27 +lang=it +python -m ulmfit eval --glob="mldoc/${lang}-1/models/sp30k/lstm_nl4.m" --name nl4-noise --cuda-id=1 --num-cls-epochs=2 --noise=${noise} +{'data/mldoc/it-1/models/sp30k/lstm_nl4-noise.m': 0.8372499942779541} + +noise=0.4 +lang=ja +python -m ulmfit eval --glob="mldoc/${lang}-1/models/sp30k/lstm_nl4.m" --name nl4-noise --cuda-id=1 --num-cls-epochs=2 --noise=${noise} +{'data/mldoc/ja-1/models/sp30k/lstm_nl4-noise.m': 0.7472500205039978 + +noise=0.32 +lang=ru +python -m ulmfit eval --glob="mldoc/${lang}-1/models/sp30k/lstm_nl4.m" --name nl4-noise --cuda-id=1 --num-cls-epochs=2 --noise=${noise} +{'data/mldoc/ru-1/models/sp30k/lstm_nl4-noise.m': 0.7567499876022339} + +noise=0.28 +lang=zh +python -m ulmfit eval --glob="mldoc/${lang}-1/models/sp30k/lstm_nl4.m" --name nl4-noise --cuda-id=1 --num-cls-epochs=2 --noise=${noise} +``` + + + + + +### LIMIT LOgs +``` +python -m ulmfit eval --name nl4-100e8 --cuda-id=1 --limit=100 --num-cls-epochs=8 ✘ 130 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4-100e8.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/it.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Limiting data set to: 100 +Tokenized data loaded, cls.trn 100, cls.val 100 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁di', "▁&'", "'", '▁e', '▁il', '▁la', 'e', '▁in'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4-100e8.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 1.214805 1.382632 0.280000 +epoch train_loss valid_loss accuracy +1 0.977314 1.269534 0.450000 +epoch train_loss valid_loss accuracy +1 0.856274 1.223441 0.530000 +epoch train_loss valid_loss accuracy +1 0.718223 1.188048 0.620000 +2 0.735718 1.130525 0.730000 +3 0.730894 1.069027 0.710000 +4 0.715334 1.015253 0.710000 +5 0.716080 0.965223 0.720000 +6 0.695554 0.918456 0.730000 +7 0.689949 0.892840 0.730000 +8 0.675208 0.876222 0.720000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4-100e8.m +Loss and accuracy using (cls_best): [0.7090041, tensor(0.7800)] +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-100e8.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/de.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Limiting data set to: 100 +Running tokenization... +Saving tokenized: cls.trn 100, cls.val 100 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-100e8.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 1.141527 1.328262 0.280000 +epoch train_loss valid_loss accuracy +1 0.703434 1.170250 0.510000 +epoch train_loss valid_loss accuracy +1 0.568693 1.051980 0.780000 +epoch train_loss valid_loss accuracy +1 0.455238 0.990438 0.800000 +2 0.475659 0.928943 0.850000 +3 0.477652 0.848537 0.920000 +4 0.455583 0.769415 0.930000 +5 0.450824 0.690618 0.930000 +6 0.443699 0.633900 0.940000 +7 0.430881 0.563667 0.950000 +8 0.419999 0.524655 0.950000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-100e8.m +Loss and accuracy using (cls_best): [0.45835665, tensor(0.9135)] +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-100e8.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Limiting data set to: 100 +Tokenized data loaded, cls.trn 100, cls.val 100 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-100e8.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 1.342269 1.399389 0.230000 +epoch train_loss valid_loss accuracy +1 0.957341 1.344665 0.280000 +epoch train_loss valid_loss accuracy +1 0.881869 1.301798 0.450000 +epoch train_loss valid_loss accuracy +1 0.887575 1.280226 0.440000 +2 0.835731 1.257639 0.450000 +3 0.813987 1.219512 0.510000 +4 0.792665 1.181309 0.520000 +5 0.785690 1.151372 0.510000 +6 0.784095 1.152232 0.500000 +7 0.768115 1.133895 0.520000 +8 0.769684 1.124231 0.530000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-100e8.m +Loss and accuracy using (cls_best): [0.8863698, tensor(0.7113)] +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4-100e8.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/fr.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Limiting data set to: 100 +Running tokenization... +Saving tokenized: cls.trn 100, cls.val 100 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4-100e8.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 1.220506 1.413276 0.200000 +epoch train_loss valid_loss accuracy +1 0.791777 1.306999 0.290000 +epoch train_loss valid_loss accuracy +1 0.572241 1.190053 0.580000 +epoch train_loss valid_loss accuracy +1 0.502800 1.130456 0.710000 +2 0.515115 1.056434 0.770000 +3 0.522720 0.974482 0.780000 +4 0.518296 0.881002 0.840000 +5 0.496588 0.825646 0.880000 +6 0.490416 0.771587 0.860000 +7 0.497172 0.722874 0.850000 +8 0.491894 0.682278 0.850000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4-100e8.m +Loss and accuracy using (cls_best): [0.5428351, tensor(0.8878)] +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4-100e8.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv +Tokenized data loaded, lm.trn 9195, lm.val 1021 +Limiting data set to: 100 +Running tokenization... +Saving tokenized: cls.trn 100, cls.val 100 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁,', '▁.', '▁в', 'а', '▁и', 'е', 'и', 'й', '▁на', 'х'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4-100e8.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 1.367201 1.409767 0.240000 +epoch train_loss valid_loss accuracy +1 1.099071 1.320811 0.330000 +epoch train_loss valid_loss accuracy +1 0.875845 1.253172 0.410000 +epoch train_loss valid_loss accuracy +1 0.775657 1.215067 0.580000 +2 0.774420 1.171324 0.660000 +3 0.766028 1.118901 0.680000 +4 0.744478 1.074021 0.680000 +5 0.738797 1.033736 0.660000 +6 0.733380 0.997304 0.660000 +7 0.723470 0.977280 0.670000 +8 0.710699 0.953586 0.640000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4-100e8.m +Loss and accuracy using (cls_best): [0.8535175, tensor(0.7220)] +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-100e8.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv +Tokenized data loaded, lm.trn 13013, lm.val 1445 +Limiting data set to: 100 +Running tokenization... +Saving tokenized: cls.trn 100, cls.val 100 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-100e8.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 1.142170 1.330161 0.300000 +epoch train_loss valid_loss accuracy +1 0.767807 1.212253 0.420000 +epoch train_loss valid_loss accuracy +1 0.636803 1.099303 0.540000 +epoch train_loss valid_loss accuracy +1 0.584241 0.997207 0.610000 +2 0.578480 0.907674 0.710000 +3 0.548451 0.830268 0.730000 +4 0.535560 0.762040 0.750000 +5 0.522172 0.746566 0.740000 +6 0.506584 0.676038 0.770000 +7 0.493665 0.651112 0.770000 +8 0.493031 0.621689 0.770000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-100e8.m +Loss and accuracy using (cls_best): [0.54911107, tensor(0.8170)] +{'data/mldoc/it-1/models/sp30k/lstm_nl4-100e8.m': 0.7799999713897705, 'data/mldoc/de-1/models/sp30k/lstm_nl4-100e8.m': 0.9135000109672546, 'data/mldoc/ja-1/models/sp30k/lstm_nl4-100e8.m': 0.7112500071525574, 'data/mldoc/fr-1/models/sp30k/lstm_nl4-100e8.m': 0.8877500295639038, 'data/mldoc/ru-1/models/sp30k/lstm_nl4-100e8.m': 0.722000002861023, 'data/mldoc/es-1/models/sp30k/lstm_nl4-100e8.m': 0.8169999718666077} + +python -m ulmfit eval --glob="mldoc/es-1/models/sp30k/lstm_nl4.m" --name nl4-100-2nd --cuda-id=1 --num-cls-epochs=8 --limit=100 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-100-2nd.m +Training +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv +Tokenized data loaded, lm.trn 13013, lm.val 1445 +Limiting data set to: 100 +Tokenized data loaded, cls.trn 100, cls.val 100 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-100-2nd.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 1.243127 1.354496 0.290000 +epoch train_loss valid_loss accuracy +1 0.840900 1.213333 0.460000 +epoch train_loss valid_loss accuracy +1 0.656407 1.055138 0.750000 +epoch train_loss valid_loss accuracy +1 0.558013 0.983957 0.780000 +2 0.554590 0.915244 0.750000 +3 0.536740 0.840074 0.770000 +4 0.521179 0.759908 0.790000 +5 0.515218 0.692961 0.810000 +6 0.500587 0.639504 0.810000 +7 0.486596 0.593410 0.840000 +8 0.472318 0.550126 0.830000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-100-2nd.m +Loss and accuracy using (cls_best): [0.5382241, tensor(0.8332)] +{'data/mldoc/es-1/models/sp30k/lstm_nl4-100-2nd.m': 0.8332499861717224} + +``` \ No newline at end of file diff --git a/results/logs/zeroshot.md b/results/logs/zeroshot.md new file mode 100644 index 0000000..e199336 --- /dev/null +++ b/results/logs/zeroshot.md @@ -0,0 +1,265 @@ +# Laser Performance +Accuracy matrix: + +| Train | en | de | es | fr | it | ru | zh | +|-------|-------|-------|-------|-------|-------|-------|-------| +| en: | 90.88 | 86.48 | 67.62 | 61.98 | 69.95 | 22.95 | 11.65 | +| de: | 73.23 | 92.90 | 77.23 | 74.05 | 72.30 | 24.80 | 9.93 | +| es: | 65.62 | 80.58 | 92.03 | 73.28 | 69.03 | 34.10 | 12.58 | +| fr: | 78.35 | 85.45 | 78.20 | 89.68 | 69.85 | 33.88 | 9.68 | +| it: | 73.93 | 84.58 | 79.23 | 76.73 | 84.03 | 34.48 | 11.83 | +| ru: | 57.33 | 63.78 | 45.80 | 52.78 | 51.15 | 66.08 | 36.28 | +| zh: | 26.15 | 28.13 | 21.88 | 29.33 | 30.58 | 34.38 | 75.62 | + +# DE +Laser 0shot: 86.48, ULMFiT 0shot: 91.97 +``` +python ../../source/classify.py embed-2019-02-12/mldoc.en-en.h5 ~/workspace/ulmfit-multilingual/data/mldoc/de-1 + | Test: 86.48% | classes: 24.30 22.77 28.90 24.02 + Making train set + | Train: 85.70% | classes: 27.00 21.40 27.60 24.00 +Accuracy 0.857 + 0 1 +0 3 Tokio (Reuter) - Der Dollar ist am Donnerstag ... +1 3 Kairo (Reuter) - Die ägyptische Zentralbank se... +2 2 Bonn (Reuter) - Wegen einer Bombendrohung ist ... +3 0 Berlin (Reuter) - Die Bahn AG will mit Hilfe p... +4 3 08.15 Uhr MEZ - Deutsche Aktien nach den Rekor... + + Making dev set + | Train: 85.60% | classes: 23.70 22.30 30.60 23.40 +Accuracy 0.856 + 0 1 +0 1 New York (Reuter) - Das Vertrauen der US-Verbr... +1 2 Tokio (Reuter) - Russische Patrouillenboote ha... +2 2 Paris (Reuter) - Bei der Volksabstimmung in Al... +3 2 Belgrad (Reuter) - Die serbische Polizei hat n... +4 0 München (Reuter) - Der Stuttgarter Bosch-Konze... +``` +``` +python -m ulmfit cls --dataset-path data/mldoc/de-1-laser --base-lm-path data/mldoc/de-1/models/sp30k/lstm_nl4.m --lang=de --name 'nl4' --cuda-id=1 - train 0 --bs 40 --num-cls-epochs=2 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser/models/sp30k/lstm_nl4.m +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser/de.dev.csv +Running tokenization... +Saving tokenized: cls.trn 13500, cls.val 1500 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser/models/sp30k/lstm_nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.671869 0.466408 0.863000 +epoch train_loss valid_loss accuracy +1 0.518045 0.388151 0.887000 +epoch train_loss valid_loss accuracy +1 0.375156 0.370652 0.893000 +epoch train_loss valid_loss accuracy +1 0.339284 0.367223 0.891000 +2 0.314325 0.369492 0.891000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser/models/sp30k/lstm_nl4.m +Loss and accuracy using (cls_best): [0.25416428, tensor(0.9197)] +0.25416427850723267 +0.9197499752044678 +``` + + + + + +# ES from IT +``` +python ../../source/classify.py embed-2019-02-12/mldoc.it-it.h5 ~/workspace/ulmfit-multilingual/data/mldoc/es-1 ✘ 130 + | Test: 79.23% | classes: 25.48 16.45 24.18 33.90 + Making train set + | Train: 80.30% | classes: 27.10 19.20 22.60 31.10 +Accuracy 0.803 + 0 1 +0 3 LONDRES, 5 sep (Reuter) - El dólar se mantenía... +1 1 MADRID, 30 dic (Reuter) - La Generalitat de Va... +2 3 PARIS, 30 jun (Reuter) - La Bolsa de París neg... +3 0 MADRID, 23 dic (Reuter) - La agencia de valore... +4 0 MADRID, 4 Feb (Reuter) - El Banco Bilbao Vizca... + + Making dev set + | Train: 79.70% | classes: 25.40 17.50 26.20 30.90 +Accuracy 0.797 + 0 1 +0 0 NUEVA YORK, 11 abr (Reuter) - MCI Communicatio... +1 3 FRANCFORT, 17 jun (Reuter) - La Bolsa de Franc... +2 1 BONN, 3 jun (Reuter) - Un destacado miembro de... +3 2 LONDRES, 3 sep (Reuter) - El secretario de Def... +4 2 MADRID, 3 oct (Reuter) - Las acciones de Pryca... +``` + +``` +python -m ulmfit cls --dataset-path data/mldoc/es-1-laser-it --base-lm-path data/mldoc/es-1/models/sp30k/lstm_nl4.m --lang=es --name 'nl4' --cuda-id=1 - train 0 --bs 40 --num-cls-epochs=2 +``` + +# FR from IT +``` +python ../../source/classify.py embed-2019-02-12/mldoc.it-it.h5 ~/workspace/ulmfit-multilingual/data/mldoc/fr-1 + | Test: 76.73% | classes: 21.65 21.98 31.77 24.60 + Making train set + | Train: 79.20% | classes: 22.20 22.40 31.40 24.00 +Accuracy 0.792 + 0 1 +0 2 WASHINGTON, 13 septembre, Reuter - Les Etats-U... +1 1 PARIS, 10 juillet, Reuter - L'audit des financ... +2 2 MOSCOU, 29 mai, Reuter - Après l'accord interv... +3 2 PARIS, 1er octobre, Reuter - Le groupe communi... +4 0 LONDRES, 3 juin, Reuter - National Grid Group ... + + Making dev set + | Train: 76.60% | classes: 23.30 20.10 33.00 23.60 +Accuracy 0.766 + 0 1 +0 0 PARIS, 30 décembre, Reuter - Zodiac . Chiffre ... +1 0 AJACCIO, 11 décembre, Reuter - Une charge de 7... +2 0 BRUXELLES, 26 décembre, Reuter - 1997 s'annonc... +3 0 PARIS, 26 septembre, Reuter - Alcatel Alsthom ... +4 0 NEW YORK, 25 octobre, Reuter - La hausse plus ... +``` + +``` +python -m ulmfit cls --dataset-path data/mldoc/fr-1-laser-it --base-lm-path data/mldoc/fr-1/models/sp30k/lstm_nl4.m --lang=fr --name 'nl4' --cuda-id=1 - train 0 --bs 40 --num-cls-epochs=2 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-it/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-it/models/sp30k/lstm_nl4.m +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-it/fr.dev.csv +Running tokenization... +Saving tokenized: cls.trn 13500, cls.val 1500 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-it/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-it/models/sp30k/lstm_nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.737947 0.627607 0.793000 +epoch train_loss valid_loss accuracy +1 0.603060 0.513449 0.831000 +epoch train_loss valid_loss accuracy +1 0.481312 0.499689 0.828000 +epoch train_loss valid_loss accuracy +1 0.422958 0.508330 0.825000 +2 0.408061 0.493875 0.839000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-it/models/sp30k/lstm_nl4.m +Loss and accuracy using (cls_best): [0.4295174, tensor(0.8555)] +0.42951738834381104 +0.8554999828338623 +``` +# FR From EN +``` +python ../../source/classify.py embed-2019-02-12/mldoc.en-en.h5 ~/workspace/ulmfit-multilingual/data/mldoc/fr-1 + | Test: 61.98% | classes: 11.85 41.10 40.05 7.00 + Making train set + | Train: 63.70% | classes: 11.70 43.80 38.40 6.10 +Accuracy 0.637 + 0 1 +0 2 WASHINGTON, 13 septembre, Reuter - Les Etats-U... +1 1 PARIS, 10 juillet, Reuter - L'audit des financ... +2 2 MOSCOU, 29 mai, Reuter - Après l'accord interv... +3 2 PARIS, 1er octobre, Reuter - Le groupe communi... +4 0 LONDRES, 3 juin, Reuter - National Grid Group ... + + Making dev set + | Train: 61.60% | classes: 11.90 40.90 39.70 7.50 +Accuracy 0.616 + 0 1 +0 1 PARIS, 30 décembre, Reuter - Zodiac . Chiffre ... +1 0 AJACCIO, 11 décembre, Reuter - Une charge de 7... +2 1 BRUXELLES, 26 décembre, Reuter - 1997 s'annonc... +3 1 PARIS, 26 septembre, Reuter - Alcatel Alsthom ... +4 1 NEW YORK, 25 octobre, Reuter - La hausse plus ... +``` +``` + +python -m ulmfit cls --dataset-path data/mldoc/fr-1-laser --base-lm-path data/mldoc/fr-1/models/sp30k/lstm_nl4.m --lang=fr --name 'nl4-laser' --cuda-id=1 - train 0 --bs 40 --num-cls-epochs=2 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-laser.m +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/fr.dev.csv +Running tokenization... +Saving tokenized: cls.trn 13500, cls.val 1500 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-laser.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.797327 0.697984 0.730000 +epoch train_loss valid_loss accuracy +1 0.639780 0.582377 0.763000 +epoch train_loss valid_loss accuracy +1 0.585295 0.582596 0.762000 +epoch train_loss valid_loss accuracy +1 0.482629 0.582803 0.765000 +2 0.470849 0.582416 0.771000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-laser.m +Loss and accuracy using (cls_best): [0.80327946, tensor(0.6920)] +``` + + + +### No Unfreeze +#### one epoch +``` +python -m ulmfit cls --dataset-path data/mldoc/fr-1-laser --base-lm-path data/mldoc/fr-1/models/sp30k/lstm_nl4.m --lang=fr --name 'nl4-no_unfreeze' --cuda-id=1 - train 0 --bs 40 --num-cls-epochs=2 --unfreeze=False +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-no_unfreeze.m +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/fr.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Tokenized data loaded, cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-no_unfreeze.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.800256 0.783174 0.701000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-no_unfreeze.m +Loss and accuracy using (cls_best): [1.1735736, tensor(0.5077)] +1.173573613166809 +0.5077499747276306 +``` +#### 4 epochs +ulmfit: 63.67% +``` +python -m ulmfit cls --dataset-path data/mldoc/fr-1-laser --base-lm-path data/mldoc/fr-1/models/sp30k/lstm_nl4.m --lang=fr --name 'nl4-no_unfreeze2' --cuda-id=1 - train 0 --bs 40 --num-cls-epochs=2 --unfreeze=False --num-cls-frozen-epochs=4 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-no_unfreeze2.m +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/fr.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Tokenized data loaded, cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-no_unfreeze2.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.832118 0.750073 0.717000 +2 0.729266 0.617375 0.749000 +3 0.645946 0.623189 0.751000 +4 0.566385 0.608672 0.760000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-no_unfreeze2.m +Loss and accuracy using (cls_best): [0.97152597, tensor(0.6367)] +``` \ No newline at end of file