mirror of
https://github.com/wassname/multifit.git
synced 2026-08-22 12:10:15 +08:00
Merge branch 'master' into use-configs
This commit is contained in:
Marcin
@@ -88,3 +88,13 @@ $ git push --set-upstream n-waves ulmfit_multilingual # to automatically push u
|
||||
- `bilm` -- scripts to train biLM ELMo style, Bert style
|
||||
- `class` -- scripts to test classifiers on multiple languages
|
||||
- `xnli` -- scripts to test nli
|
||||
|
||||
|
||||
## Running tests
|
||||
|
||||
To run the tests, the following data is necessary:
|
||||
|
||||
- wikitext-2 (prepared by `./prepare_wiki-en.sh`, along with wikitext-103)
|
||||
- imdb (prepared by `./prepare_imdb.sh`)
|
||||
|
||||
then simply run tests, e.g. `pytest .`
|
||||
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
LANGS
|
||||
for
|
||||
@@ -714,9 +714,9 @@
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python [conda env:fastaiv1]",
|
||||
"display_name": "fastai-dev",
|
||||
"language": "python",
|
||||
"name": "conda-env-fastaiv1-py"
|
||||
"name": "fastai-dev"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
@@ -728,7 +728,7 @@
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.0"
|
||||
"version": "3.6.8"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
|
||||
Regular → Executable
+2
-1
@@ -1,4 +1,5 @@
|
||||
fire>=0.1.3
|
||||
cupy>=5.0.0
|
||||
scikit-learn>=0.20
|
||||
sacremoses>=0.0.5
|
||||
sacremoses>=0.0.5
|
||||
sentencepiece
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
|
||||
# non-zeroshot
|
||||
| Model | en | de | es | fr | it | ja | ru | zh |
|
||||
|----------------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|------------|
|
||||
|LASER | 90.73 | 92.70 | 88.75 | 90.80 | 85.93 | 85.15 | 84.65 | 88.98 |
|
||||
|MultiCCA | 92.2 | 93.70 | 94.45 | 92.05 | 85.55 | 85.35 | 85.65 | 87.30 |
|
||||
|ULMFiT | | **95.4** | **95.15** | **93.67** | **88.42** | **89.20** | **87.27** | |
|
||||
|ULMFiT 100 | | 91.35 | 83.32 | 88.77 | 77.99 | 71.12 | 72.20 | |
|
||||
|
||||
# Zero shot approaches
|
||||
|
||||
| Model | en | de | es | fr | it | ja | ru | zh |
|
||||
|----------------------|------------|-----------|-----------|-----------|-----------|-----------|-----------|------------|
|
||||
|LASER 0 shot | 80.75 (en) | 87.03 (fr)| 82.60 (it)| 82.83 (de)| 73.25 (de)| 60.95 (en)| 68.83 (it)| 72.90 (de) |
|
||||
|LASER base 0 shot | | 86.48 | 79.23 | 76.73 |
|
||||
|ULMFiT 0 shot | | **91.97**| **85.35** | 85.54 |
|
||||
|ULMFiT 100 for comp. | | 91.35 | 83.32 | 88.77 | 77.99 | 71.12 | 72.20 | |
|
||||
|
||||
|
||||
To simulate ulmfit zero shot we add noise to the training labels to simulate training from Laser labels
|
||||
|
||||
| Model | en | de | es | fr | it | ja | ru | zh |
|
||||
|----------------------|------------|-----------|-----------|-----------|-----------|-----------|-----------|------------|
|
||||
|LASER 0 shot | 80.75 (en) | 87.03 (fr)| 82.60 (it)| 82.83 (de)| 73.25 (de)| 60.95 (en)| 68.83 (it)| 72.90 (de) |
|
||||
|ULMFiT | | **95.4** | **95.15** | **93.67** | **88.42** | **89.20** | **87.27** | |
|
||||
| Noise | 20% | 13% | 18% | 18% | 27% | 40% | 32% | 28% |
|
||||
|ULMFiT noise ~ 0 shot | | 94.49 | 93.12 | 90.49 | 83.72 | 74.72 | 75.67 | |
|
||||
@@ -0,0 +1,294 @@
|
||||
# MLDoc
|
||||
## Limiit to 100 examples
|
||||
```
|
||||
python -m ulmfit eval --glob="mldoc/*-1/models/sp30k/lstm_nl4.m" --name nl4-100e8 --cuda-id=1 --limit=100 --num-cls-epochs=8
|
||||
{
|
||||
'data/mldoc/it-1/models/sp30k/lstm_nl4-100e8.m': 0.7799999713897705,
|
||||
'data/mldoc/de-1/models/sp30k/lstm_nl4-100e8.m': 0.9135000109672546,
|
||||
'data/mldoc/ja-1/models/sp30k/lstm_nl4-100e8.m': 0.7112500071525574,
|
||||
'data/mldoc/fr-1/models/sp30k/lstm_nl4-100e8.m': 0.8877500295639038,
|
||||
'data/mldoc/ru-1/models/sp30k/lstm_nl4-100e8.m': 0.722000002861023,
|
||||
'data/mldoc/es-1/models/sp30k/lstm_nl4-100e8.m': 0.8169999718666077
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
## Noise
|
||||
|
||||
```
|
||||
noise=0.13
|
||||
lang=de
|
||||
python -m ulmfit eval --glob="mldoc/${lang}-1/models/sp30k/lstm_nl4.m" --name nl4-noise --cuda-id=1 --num-cls-epochs=2 --noise=${noise}
|
||||
{'data/mldoc/de-1/models/sp30k/lstm_nl4-noise.m': 0.9449999928474426}
|
||||
|
||||
noise=0.18
|
||||
lang=es
|
||||
python -m ulmfit eval --glob="mldoc/${lang}-1/models/sp30k/lstm_nl4.m" --name nl4-noise --cuda-id=1 --num-cls-epochs=2 --noise=${noise}
|
||||
{'data/mldoc/es-1/models/sp30k/lstm_nl4-noise.m': 0.9312499761581421}
|
||||
|
||||
noise=0.18
|
||||
lang=fr
|
||||
python -m ulmfit eval --glob="mldoc/${lang}-1/models/sp30k/lstm_nl4.m" --name nl4-noise --cuda-id=1 --num-cls-epochs=2 --noise=${noise}
|
||||
{'data/mldoc/fr-1/models/sp30k/lstm_nl4-noise.m': 0.9049999713897705}
|
||||
|
||||
noise=0.27
|
||||
lang=it
|
||||
python -m ulmfit eval --glob="mldoc/${lang}-1/models/sp30k/lstm_nl4.m" --name nl4-noise --cuda-id=1 --num-cls-epochs=2 --noise=${noise}
|
||||
{'data/mldoc/it-1/models/sp30k/lstm_nl4-noise.m': 0.8372499942779541}
|
||||
|
||||
noise=0.4
|
||||
lang=ja
|
||||
python -m ulmfit eval --glob="mldoc/${lang}-1/models/sp30k/lstm_nl4.m" --name nl4-noise --cuda-id=1 --num-cls-epochs=2 --noise=${noise}
|
||||
{'data/mldoc/ja-1/models/sp30k/lstm_nl4-noise.m': 0.7472500205039978
|
||||
|
||||
noise=0.32
|
||||
lang=ru
|
||||
python -m ulmfit eval --glob="mldoc/${lang}-1/models/sp30k/lstm_nl4.m" --name nl4-noise --cuda-id=1 --num-cls-epochs=2 --noise=${noise}
|
||||
{'data/mldoc/ru-1/models/sp30k/lstm_nl4-noise.m': 0.7567499876022339}
|
||||
|
||||
noise=0.28
|
||||
lang=zh
|
||||
python -m ulmfit eval --glob="mldoc/${lang}-1/models/sp30k/lstm_nl4.m" --name nl4-noise --cuda-id=1 --num-cls-epochs=2 --noise=${noise}
|
||||
```
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
### LIMIT LOgs
|
||||
```
|
||||
python -m ulmfit eval --name nl4-100e8 --cuda-id=1 --limit=100 --num-cls-epochs=8 ✘ 130
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4-100e8.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/it.dev.csv
|
||||
Tokenized data loaded, lm.trn 13500, lm.val 1500
|
||||
Limiting data set to: 100
|
||||
Tokenized data loaded, cls.trn 100, cls.val 100
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁di', "▁&'", "'", '▁e', '▁il', '▁la', 'e', '▁in']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4-100e8.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 1.214805 1.382632 0.280000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.977314 1.269534 0.450000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.856274 1.223441 0.530000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.718223 1.188048 0.620000
|
||||
2 0.735718 1.130525 0.730000
|
||||
3 0.730894 1.069027 0.710000
|
||||
4 0.715334 1.015253 0.710000
|
||||
5 0.716080 0.965223 0.720000
|
||||
6 0.695554 0.918456 0.730000
|
||||
7 0.689949 0.892840 0.730000
|
||||
8 0.675208 0.876222 0.720000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4-100e8.m
|
||||
Loss and accuracy using (cls_best): [0.7090041, tensor(0.7800)]
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-100e8.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/de.dev.csv
|
||||
Tokenized data loaded, lm.trn 13500, lm.val 1500
|
||||
Limiting data set to: 100
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 100, cls.val 100
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-100e8.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 1.141527 1.328262 0.280000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.703434 1.170250 0.510000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.568693 1.051980 0.780000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.455238 0.990438 0.800000
|
||||
2 0.475659 0.928943 0.850000
|
||||
3 0.477652 0.848537 0.920000
|
||||
4 0.455583 0.769415 0.930000
|
||||
5 0.450824 0.690618 0.930000
|
||||
6 0.443699 0.633900 0.940000
|
||||
7 0.430881 0.563667 0.950000
|
||||
8 0.419999 0.524655 0.950000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-100e8.m
|
||||
Loss and accuracy using (cls_best): [0.45835665, tensor(0.9135)]
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-100e8.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv
|
||||
Tokenized data loaded, lm.trn 13500, lm.val 1500
|
||||
Limiting data set to: 100
|
||||
Tokenized data loaded, cls.trn 100, cls.val 100
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁(']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-100e8.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 1.342269 1.399389 0.230000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.957341 1.344665 0.280000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.881869 1.301798 0.450000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.887575 1.280226 0.440000
|
||||
2 0.835731 1.257639 0.450000
|
||||
3 0.813987 1.219512 0.510000
|
||||
4 0.792665 1.181309 0.520000
|
||||
5 0.785690 1.151372 0.510000
|
||||
6 0.784095 1.152232 0.500000
|
||||
7 0.768115 1.133895 0.520000
|
||||
8 0.769684 1.124231 0.530000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-100e8.m
|
||||
Loss and accuracy using (cls_best): [0.8863698, tensor(0.7113)]
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4-100e8.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/fr.dev.csv
|
||||
Tokenized data loaded, lm.trn 13500, lm.val 1500
|
||||
Limiting data set to: 100
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 100, cls.val 100
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4-100e8.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 1.220506 1.413276 0.200000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.791777 1.306999 0.290000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.572241 1.190053 0.580000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.502800 1.130456 0.710000
|
||||
2 0.515115 1.056434 0.770000
|
||||
3 0.522720 0.974482 0.780000
|
||||
4 0.518296 0.881002 0.840000
|
||||
5 0.496588 0.825646 0.880000
|
||||
6 0.490416 0.771587 0.860000
|
||||
7 0.497172 0.722874 0.850000
|
||||
8 0.491894 0.682278 0.850000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4-100e8.m
|
||||
Loss and accuracy using (cls_best): [0.5428351, tensor(0.8878)]
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4-100e8.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv
|
||||
Tokenized data loaded, lm.trn 9195, lm.val 1021
|
||||
Limiting data set to: 100
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 100, cls.val 100
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁в', 'а', '▁и', 'е', 'и', 'й', '▁на', 'х']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4-100e8.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 1.367201 1.409767 0.240000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 1.099071 1.320811 0.330000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.875845 1.253172 0.410000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.775657 1.215067 0.580000
|
||||
2 0.774420 1.171324 0.660000
|
||||
3 0.766028 1.118901 0.680000
|
||||
4 0.744478 1.074021 0.680000
|
||||
5 0.738797 1.033736 0.660000
|
||||
6 0.733380 0.997304 0.660000
|
||||
7 0.723470 0.977280 0.670000
|
||||
8 0.710699 0.953586 0.640000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4-100e8.m
|
||||
Loss and accuracy using (cls_best): [0.8535175, tensor(0.7220)]
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-100e8.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv
|
||||
Tokenized data loaded, lm.trn 13013, lm.val 1445
|
||||
Limiting data set to: 100
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 100, cls.val 100
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-100e8.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 1.142170 1.330161 0.300000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.767807 1.212253 0.420000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.636803 1.099303 0.540000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.584241 0.997207 0.610000
|
||||
2 0.578480 0.907674 0.710000
|
||||
3 0.548451 0.830268 0.730000
|
||||
4 0.535560 0.762040 0.750000
|
||||
5 0.522172 0.746566 0.740000
|
||||
6 0.506584 0.676038 0.770000
|
||||
7 0.493665 0.651112 0.770000
|
||||
8 0.493031 0.621689 0.770000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-100e8.m
|
||||
Loss and accuracy using (cls_best): [0.54911107, tensor(0.8170)]
|
||||
{'data/mldoc/it-1/models/sp30k/lstm_nl4-100e8.m': 0.7799999713897705, 'data/mldoc/de-1/models/sp30k/lstm_nl4-100e8.m': 0.9135000109672546, 'data/mldoc/ja-1/models/sp30k/lstm_nl4-100e8.m': 0.7112500071525574, 'data/mldoc/fr-1/models/sp30k/lstm_nl4-100e8.m': 0.8877500295639038, 'data/mldoc/ru-1/models/sp30k/lstm_nl4-100e8.m': 0.722000002861023, 'data/mldoc/es-1/models/sp30k/lstm_nl4-100e8.m': 0.8169999718666077}
|
||||
|
||||
python -m ulmfit eval --glob="mldoc/es-1/models/sp30k/lstm_nl4.m" --name nl4-100-2nd --cuda-id=1 --num-cls-epochs=8 --limit=100
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-100-2nd.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv
|
||||
Tokenized data loaded, lm.trn 13013, lm.val 1445
|
||||
Limiting data set to: 100
|
||||
Tokenized data loaded, cls.trn 100, cls.val 100
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-100-2nd.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 1.243127 1.354496 0.290000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.840900 1.213333 0.460000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.656407 1.055138 0.750000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.558013 0.983957 0.780000
|
||||
2 0.554590 0.915244 0.750000
|
||||
3 0.536740 0.840074 0.770000
|
||||
4 0.521179 0.759908 0.790000
|
||||
5 0.515218 0.692961 0.810000
|
||||
6 0.500587 0.639504 0.810000
|
||||
7 0.486596 0.593410 0.840000
|
||||
8 0.472318 0.550126 0.830000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-100-2nd.m
|
||||
Loss and accuracy using (cls_best): [0.5382241, tensor(0.8332)]
|
||||
{'data/mldoc/es-1/models/sp30k/lstm_nl4-100-2nd.m': 0.8332499861717224}
|
||||
|
||||
```
|
||||
@@ -483,3 +483,33 @@ epoch train_loss valid_loss accuracy
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-noise0.4.m
|
||||
Loss and accuracy using (cls_last): [0.62477165, tensor(0.7717)]
|
||||
```
|
||||
#### 15%
|
||||
```
|
||||
python -m ulmfit cls --dataset-path data/mldoc/de-1 --base-lm-path data/mldoc/de-1/models/sp30k/lstm_nl4.m --lang=de --name 'nl4-noise0.15' --cuda-id=1 - train 0 --bs 40 --noise=0.15 --num-cls-epochs=2 --drop-mult-cls=0.2
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-noise0.15.m
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/de.dev.csv
|
||||
Tokenized data loaded, lm.trn 13500, lm.val 1500
|
||||
Added noise to 150 examples, only 0.85 have correct labels
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-noise0.15.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.836104 0.584330 0.897000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.692108 0.303470 0.930000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.653277 0.330520 0.924000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.541086 0.331944 0.922000
|
||||
2 0.523274 0.335986 0.922000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-noise0.15.m
|
||||
Loss and accuracy using (cls_last): [0.28749043, tensor(0.9355)]
|
||||
```
|
||||
@@ -0,0 +1,81 @@
|
||||
````
|
||||
|
||||
|
||||
python -m ulmfit lm --dataset-path data/wiki/es-100 --cuda-id=0 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 30000 --lang es --qrnn=False - train 10 --bs=50 --drop_mult=0
|
||||
Running tokenization
|
||||
Wiki text was split to 96224 articles
|
||||
Wiki text was split to 105 articles
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': None, 'pretrained_model': None, 'drop_mult': 0} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Training lm from random weights
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 3.269541 3.451855 0.387471
|
||||
2 3.161740 3.423016 0.386158
|
||||
3 3.187431 3.419638 0.388626
|
||||
4 3.115763 3.357066 0.393877
|
||||
5 2.996527 3.291787 0.402488
|
||||
6 3.021759 3.202183 0.410873
|
||||
7 2.998267 3.104373 0.422624
|
||||
8 2.827225 3.006537 0.436010
|
||||
9 2.784576 2.937735 0.446654
|
||||
10 2.789913 2.918509 0.450055
|
||||
data/wiki/es-100/models/sp30k
|
||||
Saving info data/wiki/es-100/models/sp30k/lstm_nl4.m/info.json
|
||||
````
|
||||
|
||||
### MLDoc
|
||||
|
||||
```
|
||||
python -m ulmfit cls --dataset-path data/mldoc/es-1 --base-lm-path data/wiki/es-100/models/sp30k/lstm_nl4.m --lang=es --name 'nl4' --cuda-id=1 - train 20 --bs 40 --num-cls-epochs=2
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 13013, cls.val 1445
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/es-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/es-100/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/es-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/es-100/models/sp30k/lstm_nl4.m/../itos')]
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 2.805415 2.188974 0.537779
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 2.429727 1.989691 0.569048
|
||||
2 2.218828 1.794969 0.603721
|
||||
3 2.015097 1.644815 0.629609
|
||||
4 1.877210 1.537773 0.646898
|
||||
5 1.775648 1.450283 0.660861
|
||||
6 1.749334 1.377085 0.672146
|
||||
7 1.601073 1.311101 0.684400
|
||||
8 1.564420 1.251074 0.694900
|
||||
9 1.532728 1.197607 0.704779
|
||||
10 1.391921 1.145408 0.716044
|
||||
11 1.379958 1.093550 0.726937
|
||||
12 1.324111 1.048308 0.735890
|
||||
13 1.344113 1.007926 0.745691
|
||||
14 1.243085 0.969521 0.754591
|
||||
15 1.230809 0.937330 0.762675
|
||||
16 1.162501 0.913408 0.768044
|
||||
17 1.170092 0.894892 0.773239
|
||||
18 1.110860 0.884449 0.775603
|
||||
19 1.115907 0.880448 0.776671
|
||||
20 1.083033 0.878421 0.776931
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.621574 0.391042 0.856000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.411668 0.215625 0.935000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.340519 0.222422 0.935000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.281729 0.192193 0.949000
|
||||
2 0.262074 0.202975 0.945000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.1749019, tensor(0.9515)]
|
||||
```
|
||||
@@ -0,0 +1,82 @@
|
||||
# FR
|
||||
## SP30k LSTM nl 4
|
||||
### LM
|
||||
```
|
||||
python -m ulmfit lm --dataset-path data/wiki/it-100 --lang=it --bidir=False --qrnn=False --max-vocab 30000 --nl 4 --tokenizer=sp --name 'nl4bs100' - train 10 --bs 100 --dropout-mult=0
|
||||
Wiki text was split to 164583 articles
|
||||
Wiki text was split to 98 articles
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁di', "▁&'", "'", '▁e', '▁il', '▁la', 'e', '▁in']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': None, 'pretrained_model': None, 'drop_mult': 0.0} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Training lm from random weights
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 3.306743 3.717148 0.353641
|
||||
2 3.126413 3.606443 0.360839
|
||||
3 3.062586 3.545493 0.365721
|
||||
4 3.055600 3.474823 0.373451
|
||||
5 2.927211 3.406635 0.380311
|
||||
6 2.924096 3.321370 0.389487
|
||||
7 2.779998 3.233350 0.399968
|
||||
8 2.722100 3.147745 0.410365
|
||||
9 2.615910 3.087420 0.419097
|
||||
10 2.565747 3.075364 0.420906
|
||||
data/wiki/it-100/models/sp30k
|
||||
Saving info data/wiki/it-100/models/sp30k/lstm_nl4bs100.m/info.json
|
||||
```
|
||||
|
||||
|
||||
### MLDoc
|
||||
MultiCCA: 85.55%, ULMFiT 88.42%
|
||||
```
|
||||
python -m ulmfit cls --dataset-path data/mldoc/it-1 --base-lm-path data/wiki/it-100/models/sp30k/lstm_nl4bs100.m --lang=it --name 'nl4bs100' --cuda-id=1 - train 20 --bs 40 --num-cls-epochs=2
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4bs100.m
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/it.dev.csv
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 13500, cls.val 1500
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁di', "▁&'", "'", '▁e', '▁il', '▁la', 'e', '▁in']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/it-100/models/sp30k/lstm_nl4bs100.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/it-100/models/sp30k/lstm_nl4bs100.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/it-100/models/sp30k/lstm_nl4bs100.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/it-100/models/sp30k/lstm_nl4bs100.m/../itos')]
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 2.826957 2.518636 0.492175
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 2.606302 2.397623 0.509596
|
||||
2 2.470586 2.260363 0.531301
|
||||
3 2.334087 2.113640 0.554089
|
||||
4 2.176830 1.988222 0.572687
|
||||
5 2.123101 1.869944 0.591537
|
||||
6 2.011187 1.770606 0.606682
|
||||
7 1.934953 1.676852 0.622504
|
||||
8 1.889363 1.592609 0.637525
|
||||
9 1.774590 1.517665 0.652233
|
||||
10 1.725905 1.435543 0.666759
|
||||
11 1.670903 1.365167 0.681168
|
||||
12 1.610080 1.302561 0.694462
|
||||
13 1.522876 1.242124 0.708201
|
||||
14 1.478528 1.193259 0.718366
|
||||
15 1.423993 1.150854 0.728324
|
||||
16 1.389901 1.115550 0.735836
|
||||
17 1.365959 1.094267 0.740730
|
||||
18 1.347579 1.079465 0.744019
|
||||
19 1.321906 1.074090 0.745281
|
||||
20 1.332676 1.073143 0.745453
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4bs100.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.632703 0.463210 0.831000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.527650 0.390041 0.858000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.436223 0.326409 0.871000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.361738 0.321380 0.875000
|
||||
2 0.340658 0.315946 0.877000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4bs100.m
|
||||
Loss and accuracy using (cls_best): [0.32998973, tensor(0.8842)]
|
||||
```
|
||||
@@ -87,4 +87,65 @@ epoch train_loss valid_loss accuracy
|
||||
8 0.278896 0.358145 0.877000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.29789856, tensor(0.8920)]
|
||||
```
|
||||
|
||||
### JA on 100 elements
|
||||
```
|
||||
python -m ulmfit cls --dataset-path data/mldoc/ja-1 --base-lm-path data/wiki/ja-100/models/sp30k/lstm_nl4.m --lang=ja --name 'nl4-100' --cuda-id=1 - train 20 --bs 40 --num-cls-epochs=8 --limit=100
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-100.m
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv
|
||||
Tokenized data loaded, lm.trn 13500, lm.val 1500
|
||||
Limiting data set to: 100
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 100, cls.val 100
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁(']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp30k/lstm_nl4.m/../itos')]
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 2.837937 2.387255 0.518590
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 2.466900 2.193583 0.549492
|
||||
2 2.232762 1.983981 0.586658
|
||||
3 2.026505 1.810167 0.615649
|
||||
4 1.918111 1.679784 0.636613
|
||||
5 1.748909 1.577095 0.653108
|
||||
6 1.708709 1.491436 0.667657
|
||||
7 1.640415 1.420449 0.679619
|
||||
8 1.577434 1.359511 0.690194
|
||||
9 1.551961 1.302819 0.700306
|
||||
10 1.475623 1.252393 0.710039
|
||||
11 1.435565 1.208159 0.718740
|
||||
12 1.354910 1.161781 0.727927
|
||||
13 1.351157 1.123244 0.736009
|
||||
14 1.299070 1.086383 0.743896
|
||||
15 1.258739 1.055745 0.750383
|
||||
16 1.210775 1.035209 0.754965
|
||||
17 1.228421 1.018373 0.758963
|
||||
18 1.179444 1.007714 0.761158
|
||||
19 1.197443 1.003041 0.762068
|
||||
20 1.163223 1.001939 0.762211
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-100.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 1.269222 1.360420 0.340000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.969350 1.314497 0.400000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.832396 1.263416 0.550000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.780991 1.225439 0.600000
|
||||
2 0.765755 1.183010 0.600000
|
||||
3 0.749420 1.139053 0.600000
|
||||
4 0.731800 1.093319 0.610000
|
||||
5 0.711152 1.054695 0.610000
|
||||
6 0.694611 1.029465 0.580000
|
||||
7 0.680276 1.004366 0.580000
|
||||
8 0.668421 0.984848 0.590000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-100.m
|
||||
Loss and accuracy using (cls_best): [0.81621724, tensor(0.7437)]
|
||||
```
|
||||
@@ -0,0 +1,78 @@
|
||||
# RU
|
||||
## SP30k nl4
|
||||
### LM
|
||||
```
|
||||
python -m ulmfit lm --dataset-path data/wiki/ru-100 --cuda-id=0 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 30000 --lang ru --qrnn=False - train 10 --bs=50 --drop_mult=0
|
||||
Size of vocabulary: 30000 [39/805]
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁в', 'а', '▁и', 'е', 'и', 'й', '▁на', 'х']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': None, 'pretrained_model': None, 'drop_mult': 0} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Training lm from random weights
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 3.200520 3.295865 0.436852
|
||||
2 3.027569 3.168700 0.445551
|
||||
3 3.007320 3.132495 0.450450
|
||||
4 2.940000 3.041745 0.459344
|
||||
5 2.876227 2.952338 0.469182
|
||||
6 2.742553 2.860888 0.480943
|
||||
7 2.684717 2.769994 0.492934
|
||||
8 2.569419 2.669971 0.507300
|
||||
9 2.525698 2.604086 0.516840
|
||||
10 2.495174 2.591011 0.519415
|
||||
data/wiki/ru-100/models/sp30k
|
||||
Saving info data/wiki/ru-100/models/sp30k/lstm_nl4.m/info.json
|
||||
```
|
||||
### MLDoc
|
||||
MultiCCA: 85.65% ulmfit: 87.27%
|
||||
```
|
||||
python -m ulmfit cls --dataset-path data/mldoc/ru-1 --base-lm-path data/wiki/ru-100/models/sp30k/lstm_nl4.m --lang=ru --name 'nl4-100' --cuda-id=1 - train 20 --bs 40 --num-cls-epochs=2
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 9195, cls.val 1021
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁в', 'а', '▁и', 'е', 'и', 'й', '▁на', 'х']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp30k/lstm_nl4.m/../itos')]
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 2.764138 2.289755 0.552181
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 2.414295 2.161708 0.572407
|
||||
2 2.310551 2.013092 0.596075
|
||||
3 2.124479 1.864450 0.620103
|
||||
4 1.970015 1.723395 0.642392
|
||||
5 1.883664 1.623308 0.658949
|
||||
6 1.793856 1.513542 0.677954
|
||||
7 1.625767 1.424582 0.693092
|
||||
8 1.677054 1.335406 0.709802
|
||||
9 1.578936 1.264322 0.723626
|
||||
10 1.523383 1.194463 0.737942
|
||||
11 1.436643 1.129712 0.750586
|
||||
12 1.351507 1.072792 0.762524
|
||||
13 1.357552 1.020739 0.773266
|
||||
14 1.310516 0.975852 0.783653
|
||||
15 1.216484 0.940323 0.791262
|
||||
16 1.187942 0.909915 0.797675
|
||||
17 1.141316 0.885367 0.803305
|
||||
18 1.114629 0.871992 0.805929
|
||||
19 1.075366 0.867010 0.807009
|
||||
20 1.166387 0.865594 0.807241
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.831180 0.610087 0.787000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.678307 0.435860 0.856000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.547668 0.399889 0.870000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.445839 0.396535 0.869000
|
||||
2 0.417901 0.369961 0.882000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.38499942, tensor(0.8727)]
|
||||
```
|
||||
@@ -0,0 +1,265 @@
|
||||
# Laser Performance
|
||||
Accuracy matrix:
|
||||
|
||||
| Train | en | de | es | fr | it | ru | zh |
|
||||
|-------|-------|-------|-------|-------|-------|-------|-------|
|
||||
| en: | 90.88 | 86.48 | 67.62 | 61.98 | 69.95 | 22.95 | 11.65 |
|
||||
| de: | 73.23 | 92.90 | 77.23 | 74.05 | 72.30 | 24.80 | 9.93 |
|
||||
| es: | 65.62 | 80.58 | 92.03 | 73.28 | 69.03 | 34.10 | 12.58 |
|
||||
| fr: | 78.35 | 85.45 | 78.20 | 89.68 | 69.85 | 33.88 | 9.68 |
|
||||
| it: | 73.93 | 84.58 | 79.23 | 76.73 | 84.03 | 34.48 | 11.83 |
|
||||
| ru: | 57.33 | 63.78 | 45.80 | 52.78 | 51.15 | 66.08 | 36.28 |
|
||||
| zh: | 26.15 | 28.13 | 21.88 | 29.33 | 30.58 | 34.38 | 75.62 |
|
||||
|
||||
# DE
|
||||
Laser 0shot: 86.48, ULMFiT 0shot: 91.97
|
||||
```
|
||||
python ../../source/classify.py embed-2019-02-12/mldoc.en-en.h5 ~/workspace/ulmfit-multilingual/data/mldoc/de-1
|
||||
| Test: 86.48% | classes: 24.30 22.77 28.90 24.02
|
||||
Making train set
|
||||
| Train: 85.70% | classes: 27.00 21.40 27.60 24.00
|
||||
Accuracy 0.857
|
||||
0 1
|
||||
0 3 Tokio (Reuter) - Der Dollar ist am Donnerstag ...
|
||||
1 3 Kairo (Reuter) - Die ägyptische Zentralbank se...
|
||||
2 2 Bonn (Reuter) - Wegen einer Bombendrohung ist ...
|
||||
3 0 Berlin (Reuter) - Die Bahn AG will mit Hilfe p...
|
||||
4 3 08.15 Uhr MEZ - Deutsche Aktien nach den Rekor...
|
||||
|
||||
Making dev set
|
||||
| Train: 85.60% | classes: 23.70 22.30 30.60 23.40
|
||||
Accuracy 0.856
|
||||
0 1
|
||||
0 1 New York (Reuter) - Das Vertrauen der US-Verbr...
|
||||
1 2 Tokio (Reuter) - Russische Patrouillenboote ha...
|
||||
2 2 Paris (Reuter) - Bei der Volksabstimmung in Al...
|
||||
3 2 Belgrad (Reuter) - Die serbische Polizei hat n...
|
||||
4 0 München (Reuter) - Der Stuttgarter Bosch-Konze...
|
||||
```
|
||||
```
|
||||
python -m ulmfit cls --dataset-path data/mldoc/de-1-laser --base-lm-path data/mldoc/de-1/models/sp30k/lstm_nl4.m --lang=de --name 'nl4' --cuda-id=1 - train 0 --bs 40 --num-cls-epochs=2
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser/models/sp30k/lstm_nl4.m
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser/de.dev.csv
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 13500, cls.val 1500
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser/models/sp30k/lstm_nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.671869 0.466408 0.863000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.518045 0.388151 0.887000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.375156 0.370652 0.893000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.339284 0.367223 0.891000
|
||||
2 0.314325 0.369492 0.891000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser/models/sp30k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.25416428, tensor(0.9197)]
|
||||
0.25416427850723267
|
||||
0.9197499752044678
|
||||
```
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# ES from IT
|
||||
```
|
||||
python ../../source/classify.py embed-2019-02-12/mldoc.it-it.h5 ~/workspace/ulmfit-multilingual/data/mldoc/es-1 ✘ 130
|
||||
| Test: 79.23% | classes: 25.48 16.45 24.18 33.90
|
||||
Making train set
|
||||
| Train: 80.30% | classes: 27.10 19.20 22.60 31.10
|
||||
Accuracy 0.803
|
||||
0 1
|
||||
0 3 LONDRES, 5 sep (Reuter) - El dólar se mantenía...
|
||||
1 1 MADRID, 30 dic (Reuter) - La Generalitat de Va...
|
||||
2 3 PARIS, 30 jun (Reuter) - La Bolsa de París neg...
|
||||
3 0 MADRID, 23 dic (Reuter) - La agencia de valore...
|
||||
4 0 MADRID, 4 Feb (Reuter) - El Banco Bilbao Vizca...
|
||||
|
||||
Making dev set
|
||||
| Train: 79.70% | classes: 25.40 17.50 26.20 30.90
|
||||
Accuracy 0.797
|
||||
0 1
|
||||
0 0 NUEVA YORK, 11 abr (Reuter) - MCI Communicatio...
|
||||
1 3 FRANCFORT, 17 jun (Reuter) - La Bolsa de Franc...
|
||||
2 1 BONN, 3 jun (Reuter) - Un destacado miembro de...
|
||||
3 2 LONDRES, 3 sep (Reuter) - El secretario de Def...
|
||||
4 2 MADRID, 3 oct (Reuter) - Las acciones de Pryca...
|
||||
```
|
||||
|
||||
```
|
||||
python -m ulmfit cls --dataset-path data/mldoc/es-1-laser-it --base-lm-path data/mldoc/es-1/models/sp30k/lstm_nl4.m --lang=es --name 'nl4' --cuda-id=1 - train 0 --bs 40 --num-cls-epochs=2
|
||||
```
|
||||
|
||||
# FR from IT
|
||||
```
|
||||
python ../../source/classify.py embed-2019-02-12/mldoc.it-it.h5 ~/workspace/ulmfit-multilingual/data/mldoc/fr-1
|
||||
| Test: 76.73% | classes: 21.65 21.98 31.77 24.60
|
||||
Making train set
|
||||
| Train: 79.20% | classes: 22.20 22.40 31.40 24.00
|
||||
Accuracy 0.792
|
||||
0 1
|
||||
0 2 WASHINGTON, 13 septembre, Reuter - Les Etats-U...
|
||||
1 1 PARIS, 10 juillet, Reuter - L'audit des financ...
|
||||
2 2 MOSCOU, 29 mai, Reuter - Après l'accord interv...
|
||||
3 2 PARIS, 1er octobre, Reuter - Le groupe communi...
|
||||
4 0 LONDRES, 3 juin, Reuter - National Grid Group ...
|
||||
|
||||
Making dev set
|
||||
| Train: 76.60% | classes: 23.30 20.10 33.00 23.60
|
||||
Accuracy 0.766
|
||||
0 1
|
||||
0 0 PARIS, 30 décembre, Reuter - Zodiac . Chiffre ...
|
||||
1 0 AJACCIO, 11 décembre, Reuter - Une charge de 7...
|
||||
2 0 BRUXELLES, 26 décembre, Reuter - 1997 s'annonc...
|
||||
3 0 PARIS, 26 septembre, Reuter - Alcatel Alsthom ...
|
||||
4 0 NEW YORK, 25 octobre, Reuter - La hausse plus ...
|
||||
```
|
||||
|
||||
```
|
||||
python -m ulmfit cls --dataset-path data/mldoc/fr-1-laser-it --base-lm-path data/mldoc/fr-1/models/sp30k/lstm_nl4.m --lang=fr --name 'nl4' --cuda-id=1 - train 0 --bs 40 --num-cls-epochs=2
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-it/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-it/models/sp30k/lstm_nl4.m
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-it/fr.dev.csv
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 13500, cls.val 1500
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-it/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-it/models/sp30k/lstm_nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.737947 0.627607 0.793000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.603060 0.513449 0.831000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.481312 0.499689 0.828000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.422958 0.508330 0.825000
|
||||
2 0.408061 0.493875 0.839000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-it/models/sp30k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.4295174, tensor(0.8555)]
|
||||
0.42951738834381104
|
||||
0.8554999828338623
|
||||
```
|
||||
# FR From EN
|
||||
```
|
||||
python ../../source/classify.py embed-2019-02-12/mldoc.en-en.h5 ~/workspace/ulmfit-multilingual/data/mldoc/fr-1
|
||||
| Test: 61.98% | classes: 11.85 41.10 40.05 7.00
|
||||
Making train set
|
||||
| Train: 63.70% | classes: 11.70 43.80 38.40 6.10
|
||||
Accuracy 0.637
|
||||
0 1
|
||||
0 2 WASHINGTON, 13 septembre, Reuter - Les Etats-U...
|
||||
1 1 PARIS, 10 juillet, Reuter - L'audit des financ...
|
||||
2 2 MOSCOU, 29 mai, Reuter - Après l'accord interv...
|
||||
3 2 PARIS, 1er octobre, Reuter - Le groupe communi...
|
||||
4 0 LONDRES, 3 juin, Reuter - National Grid Group ...
|
||||
|
||||
Making dev set
|
||||
| Train: 61.60% | classes: 11.90 40.90 39.70 7.50
|
||||
Accuracy 0.616
|
||||
0 1
|
||||
0 1 PARIS, 30 décembre, Reuter - Zodiac . Chiffre ...
|
||||
1 0 AJACCIO, 11 décembre, Reuter - Une charge de 7...
|
||||
2 1 BRUXELLES, 26 décembre, Reuter - 1997 s'annonc...
|
||||
3 1 PARIS, 26 septembre, Reuter - Alcatel Alsthom ...
|
||||
4 1 NEW YORK, 25 octobre, Reuter - La hausse plus ...
|
||||
```
|
||||
```
|
||||
|
||||
python -m ulmfit cls --dataset-path data/mldoc/fr-1-laser --base-lm-path data/mldoc/fr-1/models/sp30k/lstm_nl4.m --lang=fr --name 'nl4-laser' --cuda-id=1 - train 0 --bs 40 --num-cls-epochs=2
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-laser.m
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/fr.dev.csv
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 13500, cls.val 1500
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-laser.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.797327 0.697984 0.730000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.639780 0.582377 0.763000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.585295 0.582596 0.762000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.482629 0.582803 0.765000
|
||||
2 0.470849 0.582416 0.771000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-laser.m
|
||||
Loss and accuracy using (cls_best): [0.80327946, tensor(0.6920)]
|
||||
```
|
||||
|
||||
|
||||
|
||||
### No Unfreeze
|
||||
#### one epoch
|
||||
```
|
||||
python -m ulmfit cls --dataset-path data/mldoc/fr-1-laser --base-lm-path data/mldoc/fr-1/models/sp30k/lstm_nl4.m --lang=fr --name 'nl4-no_unfreeze' --cuda-id=1 - train 0 --bs 40 --num-cls-epochs=2 --unfreeze=False
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-no_unfreeze.m
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/fr.dev.csv
|
||||
Tokenized data loaded, lm.trn 13500, lm.val 1500
|
||||
Tokenized data loaded, cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-no_unfreeze.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.800256 0.783174 0.701000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-no_unfreeze.m
|
||||
Loss and accuracy using (cls_best): [1.1735736, tensor(0.5077)]
|
||||
1.173573613166809
|
||||
0.5077499747276306
|
||||
```
|
||||
#### 4 epochs
|
||||
ulmfit: 63.67%
|
||||
```
|
||||
python -m ulmfit cls --dataset-path data/mldoc/fr-1-laser --base-lm-path data/mldoc/fr-1/models/sp30k/lstm_nl4.m --lang=fr --name 'nl4-no_unfreeze2' --cuda-id=1 - train 0 --bs 40 --num-cls-epochs=2 --unfreeze=False --num-cls-frozen-epochs=4
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-no_unfreeze2.m
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/fr.dev.csv
|
||||
Tokenized data loaded, lm.trn 13500, lm.val 1500
|
||||
Tokenized data loaded, cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-no_unfreeze2.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.832118 0.750073 0.717000
|
||||
2 0.729266 0.617375 0.749000
|
||||
3 0.645946 0.623189 0.751000
|
||||
4 0.566385 0.608672 0.760000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-no_unfreeze2.m
|
||||
Loss and accuracy using (cls_best): [0.97152597, tensor(0.6367)]
|
||||
```
|
||||
@@ -0,0 +1,6 @@
|
||||
|
||||
|
||||
|
||||
```
|
||||
python -m ulmfit lm --dataset-path data/wiki/zh-100 --cuda-id=0 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 60000 --lang zh --qrnn=False - train 10 --bs=50 --drop_mult=0
|
||||
```
|
||||
@@ -25,7 +25,8 @@ def get_test_data():
|
||||
imdb = data / "imdb"
|
||||
|
||||
test_data = data / "test"
|
||||
shutil.rmtree(test_data)
|
||||
if test_data.exists():
|
||||
shutil.rmtree(test_data)
|
||||
|
||||
test_wt = test_data / 'wikitext-s'
|
||||
test_imdb = test_data / 'imdb'
|
||||
|
||||
+10
-13
@@ -22,38 +22,35 @@ def test_should_load_backwards_lm():
|
||||
df = text_df(['neg','pos'])
|
||||
|
||||
data = TextLMDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=0, text_cols=["text"], bs=2,
|
||||
lm_type=contrib_data.LanguageModelType.BwdLM,
|
||||
ld_cls=contrib_data.LanguageModelLoader)
|
||||
lm_type=contrib_data.LanguageModelType.BwdLM)
|
||||
lml = data.train_dl.dl
|
||||
lml.data = lml.batchify(np.concatenate([lml.dataset.x.items[i] for i in range(len(lml.dataset))]))
|
||||
batch = lml.get_batch(lml.data, 0, 70)
|
||||
|
||||
assert batch[0].shape == (70, lml.bs)
|
||||
assert batch[0].shape == (lml.bs, 70)
|
||||
assert batch[1].shape == (70*lml.bs,)
|
||||
|
||||
|
||||
as_text = [lml.dataset.vocab.itos[x] for x in batch[0][:,0]]
|
||||
np.testing.assert_array_equal(as_text[:5], ["world", "hello", '1', 'xxfld', 'project',])
|
||||
as_text = [lml.dataset.vocab.itos[x] for x in batch[0][0]]
|
||||
np.testing.assert_array_equal(as_text[:5], ["world", "hello", 'xxbos', 'project', 'cool'])
|
||||
|
||||
def test_should_load_bi_lm():
|
||||
path = untar_data(URLs.IMDB_SAMPLE)
|
||||
df = text_df(['neg', 'pos'])
|
||||
|
||||
data = TextLMDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=0, text_cols=["text"], bs=2,
|
||||
lm_type=contrib_data.LanguageModelType.BiLM,
|
||||
ld_cls=contrib_data.LanguageModelLoader)
|
||||
lm_type=contrib_data.LanguageModelType.BiLM)
|
||||
lml = data.train_dl.dl
|
||||
lml.data = lml.batchify(np.concatenate([lml.dataset.x.items[i] for i in range(len(lml.dataset))]))
|
||||
batch = lml.get_batch(lml.data, 0, 70)
|
||||
|
||||
assert batch[0].shape == (70, lml.bs, 2)
|
||||
assert batch[0].shape == (lml.bs, 70, 2)
|
||||
assert batch[1].shape == (70*lml.bs, 2)
|
||||
|
||||
as_text = [lml.dataset.vocab.itos[x] for x in batch[0][:, 0, 0]]
|
||||
np.testing.assert_array_equal(as_text[:7], "xxfld 1 fast ai is a cool".split())
|
||||
as_text = [lml.dataset.vocab.itos[x] for x in batch[0][0, :, 0]]
|
||||
np.testing.assert_array_equal(as_text[:7], "xxbos fast ai is a cool project".split())
|
||||
|
||||
as_text = [lml.dataset.vocab.itos[x] for x in batch[0][:,0,1]]
|
||||
np.testing.assert_array_equal(as_text[:5], ["world", "hello", '1', 'xxfld', 'project',])
|
||||
as_text = [lml.dataset.vocab.itos[x] for x in batch[0][0, :, 1]]
|
||||
np.testing.assert_array_equal(as_text[:5], ["world", "hello", 'xxbos', 'project', 'cool'])
|
||||
|
||||
###################### NEW CODE
|
||||
|
||||
|
||||
@@ -42,7 +42,7 @@ def learn():
|
||||
|
||||
def text_df(n_labels):
|
||||
data = []
|
||||
texts = ["fast ai is a cool project", "hello world"]
|
||||
texts = ["fast ai is a cool project", "hello world"] * 20
|
||||
for ind, text in enumerate(texts):
|
||||
sample = {}
|
||||
for label in range(n_labels): sample[label] = ind%2
|
||||
@@ -58,19 +58,21 @@ def test_val_loss(learn):
|
||||
|
||||
|
||||
def test_bilm_classifier_loads_encoder():
|
||||
n_labels=2
|
||||
n_labels=1
|
||||
nl = 1
|
||||
emb_sz = 100
|
||||
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data', 'tmp')
|
||||
os.makedirs(path)
|
||||
try:
|
||||
df = text_df(n_labels=1)
|
||||
df = text_df(n_labels=n_labels)
|
||||
lmdf = df#[["text"]]
|
||||
print(lmdf.head())
|
||||
lmdata = TextLMDataBunch.from_df(path, lmdf, lmdf, tokenizer=Tokenizer(BaseTokenizer),
|
||||
lm_type=contrib_data.LanguageModelType.BiLM)
|
||||
learn = bilm_learner(lmdata, emb_sz=100, nl=1, drop_mult=0.1, qrnn=False)
|
||||
learn = bilm_learner(lmdata, emb_sz=emb_sz, nl=nl, drop_mult=0.1, qrnn=False)
|
||||
learn.save_encoder("enc")
|
||||
data = TextClasDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=list(range(n_labels)), text_cols=["text"])
|
||||
classifier = bilm_text_classifier_learner(data, emb_sz=100, nl=1, drop_mult=0.1, qrnn=False)
|
||||
data = TextClasDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=list(range(n_labels)), text_cols=["text"], bs=8)
|
||||
classifier = bilm_text_classifier_learner(data, emb_sz=emb_sz, nl=nl, drop_mult=0.1, qrnn=False)
|
||||
print(last_layer(classifier.model), )
|
||||
classifier.load_encoder("enc")
|
||||
classifier.fit(1)
|
||||
|
||||
@@ -1,14 +1,26 @@
|
||||
import gc
|
||||
import shutil
|
||||
from functools import wraps
|
||||
|
||||
import fire
|
||||
from .pretrain_lm import LMHyperParams
|
||||
from .train_clas import CLSHyperParams
|
||||
from pathlib import Path
|
||||
|
||||
class FireView:
|
||||
def __init__(self, **kwargs):
|
||||
for k,v in kwargs.items():
|
||||
setattr(self, k, v)
|
||||
|
||||
def get_dataset_path(p):
|
||||
return [x for x in p.parents if x.name == "models"][0].parent
|
||||
|
||||
def get_lang_from_dataset_path(ds):
|
||||
lang,*_ = ds.name.split("-")
|
||||
if len(lang) == 2:
|
||||
return lang
|
||||
return "en"
|
||||
|
||||
class ULMFiT:
|
||||
@wraps(LMHyperParams)
|
||||
def lm(self, dataset_path, **changes):
|
||||
@@ -22,5 +34,23 @@ class ULMFiT:
|
||||
params = CLSHyperParams.from_lm(dataset_path, base_lm_path, **changes)
|
||||
return FireView(train=params.train_cls, validate_cls=params.validate_cls)
|
||||
|
||||
def eval(self, glob="mldoc/*-1/models/sp30k/lstm_nl4.m", name="tmp-100", cuda_id=0, **trn_params):
|
||||
results={}
|
||||
for base_model in Path("data").glob(glob):
|
||||
dataset_path = get_dataset_path(base_model)
|
||||
lang = get_lang_from_dataset_path(dataset_path)
|
||||
params = CLSHyperParams.from_lm(dataset_path, base_model, lang=lang, name=name, cuda_id=cuda_id)
|
||||
key = str(params.model_dir.relative_to(Path.cwd()))
|
||||
if params.model_dir.exists():
|
||||
print("Evaluating previously trained model")
|
||||
results[key] = params.validate_cls()[1]
|
||||
else:
|
||||
print("Training")
|
||||
results[key] = params.train_cls(num_lm_epochs=0, **trn_params)[1]
|
||||
params = None
|
||||
gc.collect()
|
||||
|
||||
print(list(sorted(results.items())))
|
||||
|
||||
if __name__ == '__main__':
|
||||
fire.Fire(ULMFiT())
|
||||
|
||||
@@ -53,7 +53,7 @@ def limit_vocab(unk_path, vocab):
|
||||
tokens = [''] + tokens
|
||||
line = ' '.join(tokens)
|
||||
f_out.write(line)
|
||||
print(f'{unk_path.name}. # of tokens: {total_num_tokens}')
|
||||
print(f'{unk_path.name}. # of tokens: {total_num_tokens}')
|
||||
temp_file_path.replace(unk_path)
|
||||
|
||||
|
||||
@@ -101,5 +101,6 @@ def postprocess_wikitext(path, lang):
|
||||
unk_path = dest_path / f'{lang}.wiki.{split}.tokens'
|
||||
limit_vocab(unk_path, vocab)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
fire.Fire(postprocess_wikitext)
|
||||
fire.Fire(postprocess_wikitext)
|
||||
|
||||
+28
-7
@@ -56,6 +56,7 @@ class LMHyperParams:
|
||||
dataset_path: str # data_dir
|
||||
|
||||
base_lm_path: str = None
|
||||
backwards: str = False
|
||||
bidir: bool =False
|
||||
qrnn: bool = True
|
||||
max_vocab: int = 60000
|
||||
@@ -71,12 +72,17 @@ class LMHyperParams:
|
||||
dps = dict(output_p=0.25, hidden_p=0.1, input_p=0.2, embed_p=0.02, weight_p=0.15) # consider removing dps & clip from the default hyperparams and put them to train
|
||||
clip: float = 0.12
|
||||
bptt: int = 70
|
||||
# alpha and beta - defaults like in fastai/text/learner.py:RNNLearner()
|
||||
rnn_alpha: float = 2 # activation regularization (AR)
|
||||
rnn_beta: float = 1 # temporal activation regularization (TAR)
|
||||
|
||||
lang: str = 'en'
|
||||
name: str = None
|
||||
cuda_id: InitVar[int] = 0
|
||||
|
||||
def __post_init__(self, cuda_id):
|
||||
if self.bidir and self.backwards:
|
||||
raise ValueError('Both "backwards" and "bidir" options cannot be enabled at the same time')
|
||||
if not torch.cuda.is_available():
|
||||
print('CUDA not available. Setting device=-1.')
|
||||
cuda_id = -1
|
||||
@@ -89,7 +95,6 @@ class LMHyperParams:
|
||||
self.cache_dir = self.dataset_path / 'models' / self.tokenizer_prefix
|
||||
self.model_dir = self.cache_dir / self.model_name
|
||||
|
||||
self.model_dir.mkdir(exist_ok=True, parents=True)
|
||||
print('Max vocab:', self.max_vocab)
|
||||
print('Cache dir:', self.cache_dir)
|
||||
print('Model dir:', self.model_dir)
|
||||
@@ -100,7 +105,16 @@ class LMHyperParams:
|
||||
def tokenizer_prefix(self): return f"{self.tokenizer.value}{self.max_vocab // 1000}k"
|
||||
|
||||
@property
|
||||
def model_prefix(self): return ('bi' if self.bidir else '') + ('qrnn' if self.qrnn else 'lstm')
|
||||
def model_direction(self):
|
||||
if self.bidir:
|
||||
return 'bi'
|
||||
if self.backwards:
|
||||
return 'bwd'
|
||||
else:
|
||||
return ''
|
||||
|
||||
@property
|
||||
def model_prefix(self): return self.model_direction + ('qrnn' if self.qrnn else 'lstm')
|
||||
|
||||
@property
|
||||
def model_name(self): return f"{self.model_prefix}_{self.name}.m"
|
||||
@@ -110,9 +124,14 @@ class LMHyperParams:
|
||||
|
||||
@property
|
||||
def lm_type(self):
|
||||
return contrib_data.LanguageModelType.BiLM if self.bidir else contrib_data.LanguageModelType.FwdLM
|
||||
if self.bidir:
|
||||
return contrib_data.LanguageModelType.BiLM
|
||||
if self.backwards:
|
||||
return contrib_data.LanguageModelType.BwdLM
|
||||
else:
|
||||
return contrib_data.LanguageModelType.FwdLM
|
||||
|
||||
def tokenzier_to_fastai_args(self, sp_data_func, use_moses):
|
||||
def tokenizer_to_fastai_args(self, sp_data_func, use_moses):
|
||||
tok_func = MosesTokenizerFunc if use_moses else BaseTokenizer
|
||||
if self.tokenizer is Tokenizers.SUBWORD:
|
||||
if self.base_lm_path and not(self.cache_dir/"spm.model").exists(): # ensure we are using the same sentence piece model
|
||||
@@ -146,6 +165,7 @@ class LMHyperParams:
|
||||
print("Saving info", self.model_dir / 'info.json')
|
||||
|
||||
def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, lr=5e-3):
|
||||
self.model_dir.mkdir(exist_ok=True, parents=True)
|
||||
data_lm = self.load_wiki_data(bs=bs) if data_lm is None else data_lm
|
||||
learn = self.create_lm_learner(data_lm, drop_mult=drop_mult)
|
||||
|
||||
@@ -168,7 +188,7 @@ class LMHyperParams:
|
||||
learn.unfreeze()
|
||||
if not learn.true_wd: learn.fit_one_cycle(num_epochs, lr, (0.8, 0.7), wd=1e-7)
|
||||
else: learn.fit_one_cycle(num_epochs, lr, (0.8, 0.7)) # TODO find proper values
|
||||
learn.save("lm_best_with_opt", with_opt=False)
|
||||
learn.save("lm_best_with_opt", with_opt=True)
|
||||
learn.save_encoder(ENC_BEST)
|
||||
learn.save(LM_BEST, with_opt=False)
|
||||
print(learn.path)
|
||||
@@ -182,7 +202,7 @@ class LMHyperParams:
|
||||
config = dict(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, pad_token=PAD_TOKEN_ID, qrnn=self.qrnn, bidir=self.bidir,
|
||||
tie_weights=True, out_bias=True)
|
||||
config.update(dps or self.dps)
|
||||
trn_args = dict(clip=self.clip)
|
||||
trn_args = dict(clip=self.clip, alpha=self.rnn_alpha, beta=self.rnn_beta)
|
||||
trn_args.update(kwargs)
|
||||
print ("Training args: ", trn_args, "dps: ", dps or self.dps)
|
||||
learn = language_model_learner(data_lm, AWD_LSTM, config=config, model_dir=self.model_dir.relative_to(data_lm.path), pretrained=False, **trn_args)
|
||||
@@ -210,13 +230,14 @@ class LMHyperParams:
|
||||
return [line.rstrip('\n') for line in f]
|
||||
|
||||
def load_wiki_data(self, bs=70):
|
||||
self.model_dir.mkdir(exist_ok=True, parents=True)
|
||||
trn_path = self.dataset_path / f'{self.lang}.wiki.train.tokens'
|
||||
val_path = self.dataset_path / f'{self.lang}.wiki.valid.tokens'
|
||||
tst_path = self.dataset_path / f'{self.lang}.wiki.test.tokens'
|
||||
for path_ in [trn_path, val_path, tst_path]:
|
||||
assert path_.exists(), f'Error: {path_} does not exist.'
|
||||
|
||||
args = self.tokenzier_to_fastai_args(sp_data_func=self.load_train_text, use_moses=False)
|
||||
args = self.tokenizer_to_fastai_args(sp_data_func=self.load_train_text, use_moses=False)
|
||||
try:
|
||||
data_lm = TextLMDataBunch.load(self.cache_dir, '.',
|
||||
bs=bs)
|
||||
|
||||
+12
-9
@@ -38,9 +38,10 @@ class CLSHyperParams(LMHyperParams):
|
||||
@property
|
||||
def need_fine_tune_lm(self): return not (self.model_dir/f"enc_best.pth").exists()
|
||||
|
||||
def train_cls(self, num_lm_epochs, unfreeze=True, bs=40, true_wd=True, drop_mul_lm=0.3, drop_mul_cls=0.5,
|
||||
def train_cls(self, num_lm_epochs, unfreeze=True, num_cls_frozen_epochs=1, bs=40, true_wd=True, drop_mul_lm=0.3, drop_mul_cls=0.5,
|
||||
use_test_for_validation=False, num_cls_epochs=2, limit=None, noise=0.0, cls_max_len=20*70):
|
||||
assert use_test_for_validation == False, "use_test_for_validation=True is not supported"
|
||||
self.model_dir.mkdir(exist_ok=True, parents=True)
|
||||
|
||||
data_clas, data_lm, data_tst = self.load_cls_data(bs, limit=limit, noise=noise)
|
||||
|
||||
@@ -55,7 +56,7 @@ class CLSHyperParams(LMHyperParams):
|
||||
learn.true_wd = True
|
||||
print("Starting classifier training")
|
||||
learn.freeze_to(-1)
|
||||
learn.fit_one_cycle(1, 2e-2, moms=(0.8, 0.7))
|
||||
learn.fit_one_cycle(num_cls_frozen_epochs, 2e-2, moms=(0.8, 0.7))
|
||||
if unfreeze:
|
||||
learn.freeze_to(-2)
|
||||
learn.fit_one_cycle(1, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7))
|
||||
@@ -66,7 +67,7 @@ class CLSHyperParams(LMHyperParams):
|
||||
else:
|
||||
learn.true_wd = False
|
||||
print("Starting classifier training")
|
||||
learn.fit_one_cycle(1, 5e-2, moms=(0.8, 0.7), wd=1e-7)
|
||||
learn.fit_one_cycle(num_cls_frozen_epochs, 5e-2, moms=(0.8, 0.7), wd=1e-7)
|
||||
if unfreeze:
|
||||
learn.freeze_to(-2)
|
||||
learn.fit_one_cycle(1, slice(5e-2 / (2.6 ** 4), 5e-2), moms=(0.8, 0.7), wd=1e-7)
|
||||
@@ -77,17 +78,18 @@ class CLSHyperParams(LMHyperParams):
|
||||
print(f"Saving models at {learn.path / learn.model_dir}")
|
||||
learn.save('cls_last', with_opt=False)
|
||||
|
||||
self.validate_cls('cls_best', bs=bs, limit=limit, data_tst=data_tst, learn=learn)
|
||||
return None
|
||||
return self.validate_cls('cls_best', bs=bs, data_tst=data_tst, learn=learn)
|
||||
|
||||
def validate_cls(self, save_name='cls_last', limit=None, bs=40, data_tst=None, learn=None):
|
||||
def validate_cls(self, save_name='cls_last', bs=40, data_tst=None, learn=None):
|
||||
if data_tst is None:
|
||||
_, _, data_tst = self.load_cls_data(bs, limit=limit)
|
||||
_, _, data_tst = self.load_cls_data(bs)
|
||||
if learn is None:
|
||||
learn = self.create_cls_learner(data_tst, drop_mult=0.3)
|
||||
learn.unfreeze()
|
||||
learn.load(save_name)
|
||||
print(f"Loss and accuracy using ({save_name}):", learn.validate(data_tst.valid_dl))
|
||||
results = learn.validate(data_tst.valid_dl)
|
||||
print(f"Loss and accuracy using ({save_name}):", results)
|
||||
return list(map(float, results))
|
||||
|
||||
def create_cls_learner(self, data_clas, dps=None, **kwargs):
|
||||
assert self.bidir == False, "bidirectional model is not yet supported"
|
||||
@@ -110,6 +112,7 @@ class CLSHyperParams(LMHyperParams):
|
||||
return learn
|
||||
|
||||
def load_cls_data(self, bs, **kwargs):
|
||||
self.model_dir.mkdir(exist_ok=True, parents=True)
|
||||
add_trn_to_lm = True
|
||||
lang = self.lang
|
||||
use_moses = True
|
||||
@@ -155,7 +158,7 @@ class CLSHyperParams(LMHyperParams):
|
||||
lm_trn_df = lm_trn_df[val_len:]
|
||||
lm_val_df = lm_trn_df[:val_len]
|
||||
|
||||
args = self.tokenzier_to_fastai_args(sp_data_func=lambda: trn_df[1], use_moses=use_moses)
|
||||
args = self.tokenizer_to_fastai_args(sp_data_func=lambda: trn_df[1], use_moses=use_moses)
|
||||
try:
|
||||
if force: raise FileNotFoundError("Forcing reloading of caches")
|
||||
data_lm = TextLMDataBunch.load(self.cache_dir, 'lm', bs=bs)
|
||||
|
||||
Reference in New Issue
Block a user