mirror of
https://github.com/wassname/multifit.git
synced 2026-09-09 11:27:26 +08:00
Merge branch 'master' of https://github.com/n-waves/ulmfit-multilingual into text_cols
This commit is contained in:
NAUSICAA\Julian
@@ -66,6 +66,8 @@ class SentencePieceTokenizer(Tokenizer):
|
||||
toks = super().process_text(t, tok)
|
||||
toks = tok.sp.EncodeAsPieces(" ".join(toks))
|
||||
return toks
|
||||
full_char_coverage_langs = ["bg", "cs", "da", "de", "el", "en", "es", "et", "fi", "fr", "ga", "hr", "hu",
|
||||
"it","lt","lv","mt","nl","pl","pt","ro","sk","sl","sv"] # all European langus
|
||||
|
||||
def get_sentencepiece(cache_dir:PathOrStr, load_text, pre_rules: ListRules=None, post_rules:ListRules=None,
|
||||
vocab_size:int=30000, model_type:str='unigram', input_sentence_size:int=1E7, lang='en'):
|
||||
@@ -93,9 +95,11 @@ def get_sentencepiece(cache_dir:PathOrStr, load_text, pre_rules: ListRules=None,
|
||||
raw_text_path = cache_dir / 'all_text.txt'
|
||||
with open(raw_text_path, 'w') as f: f.write("\n".join(text))
|
||||
|
||||
char_coverage = 1 if lang in full_char_coverage_langs else 0.99
|
||||
|
||||
sp_params = [
|
||||
f"--input={raw_text_path}",
|
||||
f"--character_coverage=1.0",
|
||||
f"--character_coverage={char_coverage}",
|
||||
f"--unk_id={len(defaults.text_spec_tok)}",
|
||||
f"--pad_id=-1",
|
||||
f"--bos_id=-1",
|
||||
|
||||
+13
-6
@@ -5,14 +5,21 @@
|
||||
|LASER 0 shot | 80.75 | 87.03 | 82.60 | 82.83 | 73.25 | 60.95 | 68.83 | 72.90 |
|
||||
|LASER | 90.73 | 92.70 | 88.75 | 90.80 | 85.93 | 85.15 | 84.65 | 88.98 |
|
||||
|MultiCCA | 92.2 | 93.70 | 94.45 | 92.05 | 85.55 | 85.35 | 85.65 | 87.30 |
|
||||
|ULMFiT 100 | | 91.35 | 83.32 | 88.77 | 77.99 | 71.12 | 72.20 | |
|
||||
|ULMFiT Zeroshot from Laser | | 94.48 | 86.93 | 88.78 | 79.35 | | 72.88 | 85.55 |
|
||||
|ULMFiT | | 95.4 | **95.15** | 93.67 | 88.42 | **89.20** | **87.27** | 90.20 |
|
||||
|ULMFiT sp-fixed | | **95.6** | 94.80 | **94.20** | **88.52** | 88.72 | 86.85 | 90.47 |
|
||||
|Bert Multi | 93.23% | 94.0% | **95.15** | 93.20 | 85.82 | 87.48 | 86.85 | **90.72** |
|
||||
|Bert Multi | 93.23 | 94.0 | 95.15 | 93.20 | 85.82 | 87.48 | 86.85 | 90.72 |
|
||||
|ULMFiT L30k-100 | | 91.35 | 83.32 | 88.77 | 77.99 | 71.12 | 72.20 | |
|
||||
|ULMFiT L30k | | 95.4 | 95.15 | 93.67 | 88.42 | 89.20 | **87.27** | 90.20 |
|
||||
|ULMFiT sp-fixed | | **95.6** | 94.80 | 94.20 | 88.52 | 88.72 | 86.85 | 90.47 |
|
||||
|ULMFIT Q15k 1cyc| **94.62** | **95.65** | 95.15 | **94.42** | **89.92** | 89.60 | | 90.78/89.82 |
|
||||
|ULMFIT Q15k 1c l| | | | | | | | **92.22** |
|
||||
|ULMFIT L30k 1cyc| | **95.85** | **96.32** | **94.82** | 89.87 | **90.45** | **87.94** | 92.02/91.64 |
|
||||
|
||||
- L30k - LSTM sp30k trained using gradual unfreezing
|
||||
- L30k-100 - --||-- **on 100 samples**
|
||||
- ULMFiT sp-fixed - --||-- with fixed tokenization
|
||||
- Q15k 1cyc - QRNN sp15k trained using 1cycle learning rate schedule
|
||||
- L30k 1cyc - LSTM sp30k trained using 1cycle learning rate schedule
|
||||
|
||||
## Zero shot approaches
|
||||
## Zero shot approaches - LSTM
|
||||
|
||||
| Model | de | es | fr | it | ru | zh |
|
||||
|----------------------|------------|------------|-----------|-----------|-----------|-----------|
|
||||
|
||||
@@ -0,0 +1,448 @@
|
||||
Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/de-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/de-100/models/sp15k/qrnn_nl4.m/../itos')]
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 3.467852 2.558666 0.525457
|
||||
Total time: 02:25
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 2.722157 2.387366 0.548566
|
||||
2 2.477095 2.170018 0.580988
|
||||
3 2.182971 1.981363 0.609205
|
||||
4 2.078041 1.836848 0.629900
|
||||
5 1.975613 1.744062 0.642769
|
||||
6 1.866875 1.656678 0.655799
|
||||
7 1.831995 1.595655 0.665479
|
||||
8 1.768020 1.540487 0.673880
|
||||
9 1.751569 1.488140 0.682557
|
||||
10 1.647143 1.441723 0.690275
|
||||
11 1.712795 1.399652 0.697534
|
||||
12 1.529405 1.350384 0.706170
|
||||
13 1.549134 1.313349 0.713210
|
||||
14 1.585015 1.278395 0.719908
|
||||
15 1.475010 1.248854 0.725591
|
||||
16 1.532636 1.221373 0.731053
|
||||
17 1.445181 1.203350 0.734503
|
||||
18 1.396236 1.191440 0.737102
|
||||
19 1.316587 1.186497 0.738052
|
||||
20 1.374460 1.185027 0.738290
|
||||
Total time: 1:11:26
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_nl4.m/info.json
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_nl4.m
|
||||
Loss and accuracy using (cls_best): [1.3879647, tensor(0.2595)]
|
||||
Processing data/wiki/en-100/models/sp15k/qrnn_nl4.m
|
||||
|
||||
|
||||
------
|
||||
|
||||
|
||||
|
||||
|
||||
$ python -m ulmfit eval --glob="wiki/*-100/models/sp15k/qrnn_nl4.m" --name nl4 --dataset-template='../mldoc/${lang}-1' --num-lm-epochs=20 --num-cls-epochs=8 --bs=18 --lr_sched=1cycle
|
||||
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁the', '▁,', 's', '▁.', '▁of', '▁and', '▁in', '▁to', '▁a', 'ed']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Loading pretrained model
|
||||
Unknown tokens 0, first 100: []
|
||||
Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/en-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/en-100/models/sp15k/qrnn_nl4.m/../itos')]
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 4.567319 3.820788 0.346054
|
||||
Total time: 02:26
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 3.889761 3.601505 0.374049
|
||||
2 3.570620 3.357854 0.406134
|
||||
3 3.389516 3.153452 0.432199
|
||||
4 3.217872 2.985234 0.452187
|
||||
5 3.063675 2.851744 0.468071
|
||||
6 3.023959 2.754062 0.480278
|
||||
7 2.907327 2.647027 0.493494
|
||||
8 2.786187 2.562560 0.505051
|
||||
9 2.737610 2.500068 0.513554
|
||||
10 2.696695 2.430095 0.523029
|
||||
11 2.658439 2.380829 0.530339
|
||||
12 2.598193 2.318927 0.539454
|
||||
13 2.558214 2.275014 0.546136
|
||||
14 2.520342 2.230543 0.553176
|
||||
15 2.475964 2.190341 0.559245
|
||||
16 2.370359 2.161100 0.564223
|
||||
17 2.430078 2.136685 0.568197
|
||||
18 2.383946 2.125458 0.569950
|
||||
19 2.389433 2.117541 0.571265
|
||||
20 2.297921 2.116168 0.571367
|
||||
Total time: 1:11:10
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k/qrnn_nl4.m/info.json
|
||||
Single training schedule
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.705876 0.241169 0.918000
|
||||
2 0.450546 0.239528 0.926000
|
||||
3 0.335179 0.221836 0.931000
|
||||
4 0.202048 0.208652 0.951000
|
||||
5 0.144956 0.223669 0.954000
|
||||
6 0.073117 0.277062 0.953000
|
||||
7 0.045186 0.258046 0.962000
|
||||
8 0.022987 0.265977 0.961000
|
||||
Total time: 02:33
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k/qrnn_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.29402012, tensor(0.9460)]
|
||||
Processing data/wiki/es-100/models/sp15k/qrnn_nl4.m
|
||||
../mldoc/es-1
|
||||
Max vocab: 15000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4.m
|
||||
Evaluating previously trained model
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv
|
||||
Data lm, trn: 13013, val: 1445
|
||||
Data cls, trn: 1000, val: 1000
|
||||
Data tst, trn: 1000, val: 4000
|
||||
Size of vocabulary: 15000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que']
|
||||
Loss and accuracy using (cls_last): [0.19222946, tensor(0.9515)]
|
||||
Processing data/wiki/fr-100/models/sp15k/qrnn_nl4.m
|
||||
../mldoc/fr-1
|
||||
Max vocab: 15000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_nl4.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/fr.dev.csv
|
||||
Running tokenization lm...
|
||||
Data lm, trn: 13500, val: 1500
|
||||
Running tokenization cls...
|
||||
Data cls, trn: 1000, val: 1000
|
||||
Running tokenization tst...
|
||||
Data tst, trn: 1000, val: 4000
|
||||
Size of vocabulary: 15000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', 's', '▁.', "'", '▁la', '▁le', '▁et', '▁l', '▁à']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Loading pretrained model
|
||||
Unknown tokens 0, first 100: []
|
||||
Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp15k/qrnn_nl4.m/../itos')]
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 3.375679 2.676224 0.454405
|
||||
Total time: 02:19
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 2.901917 2.540690 0.475910
|
||||
2 2.593614 2.370477 0.504601
|
||||
3 2.423170 2.205713 0.530328
|
||||
4 2.287688 2.083261 0.549087
|
||||
5 2.161118 1.984955 0.564804
|
||||
6 2.221017 1.912810 0.575434
|
||||
7 2.111272 1.837854 0.588076
|
||||
8 2.032289 1.775163 0.598341
|
||||
9 1.984161 1.720519 0.607980
|
||||
10 1.904775 1.668184 0.617407
|
||||
11 1.829098 1.621347 0.626292
|
||||
12 1.855409 1.577870 0.634512
|
||||
13 1.843696 1.536835 0.642584
|
||||
14 1.767968 1.496428 0.650317
|
||||
15 1.741591 1.463305 0.656908
|
||||
16 1.682118 1.438706 0.662601
|
||||
17 1.666425 1.418383 0.666283
|
||||
18 1.623713 1.406877 0.668710
|
||||
19 1.645482 1.401546 0.669716
|
||||
20 1.579352 1.399608 0.670167
|
||||
Total time: 1:06:50
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_nl4.m/info.json
|
||||
Single training schedule
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.614734 0.262120 0.906000
|
||||
2 0.377443 0.327852 0.917000
|
||||
3 0.296728 0.392655 0.903000
|
||||
4 0.179866 0.423420 0.928000
|
||||
5 0.114529 0.398973 0.935000
|
||||
6 0.082004 0.325470 0.944000
|
||||
7 0.047604 0.359636 0.945000
|
||||
8 0.032579 0.354014 0.944000
|
||||
Total time: 02:22
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.33020702, tensor(0.9450)]
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Loading pretrained model
|
||||
Unknown tokens 0, first 100: []
|
||||
Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp15k/qrnn_nl4.m/../itos')]
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 3.524609 2.582580 0.512131
|
||||
Total time: 02:34
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 2.656181 2.315530 0.550262
|
||||
2 2.265949 2.019140 0.598469
|
||||
3 1.985897 1.769565 0.638234
|
||||
4 1.831071 1.617888 0.660760
|
||||
5 1.735492 1.509642 0.677379
|
||||
6 1.637924 1.427618 0.690664
|
||||
7 1.564483 1.363384 0.700825
|
||||
8 1.508054 1.318165 0.708210
|
||||
9 1.471599 1.267787 0.716080
|
||||
10 1.398376 1.232899 0.722340
|
||||
11 1.311976 1.199602 0.728811
|
||||
12 1.401354 1.162299 0.735328
|
||||
13 1.385588 1.132408 0.740850
|
||||
14 1.256193 1.106556 0.745935
|
||||
15 1.289892 1.083529 0.750840
|
||||
16 1.220951 1.063360 0.754845
|
||||
17 1.259715 1.050884 0.757371
|
||||
18 1.165468 1.042870 0.759241
|
||||
19 1.242660 1.038160 0.760036
|
||||
20 1.194239 1.037506 0.760167
|
||||
Total time: 1:13:55
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k/qrnn_nl4.m/info.json
|
||||
Single training schedule
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.760780 0.391838 0.872000
|
||||
2 0.592674 0.427392 0.870000
|
||||
3 0.446265 0.593488 0.838000
|
||||
4 0.318780 0.605533 0.858000
|
||||
5 0.226914 0.665538 0.872000
|
||||
6 0.135525 0.742310 0.891000
|
||||
7 0.063984 0.778616 0.892000
|
||||
8 0.039366 0.827663 0.884000
|
||||
Total time: 02:49
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k/qrnn_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.70555997, tensor(0.8960)]
|
||||
Processing data/wiki/zh-100/models/sp15k/qrnn_nl4.m
|
||||
../mldoc/zh-1
|
||||
Max vocab: 15000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4.m
|
||||
Evaluating previously trained model
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv
|
||||
Data lm, trn: 13500, val: 1500
|
||||
Data cls, trn: 1000, val: 1000
|
||||
Data tst, trn: 1000, val: 4000
|
||||
Size of vocabulary: 15000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有']
|
||||
Loss and accuracy using (cls_last): [0.30052844, tensor(0.8982)]
|
||||
OrderedDict([('data/mldoc/de-1/models/sp15k/qrnn_nl4.m', 0.2592499852180481),
|
||||
('data/mldoc/en-1/models/sp15k/qrnn_nl4.m', 0.9462500214576721),
|
||||
('data/mldoc/es-1/models/sp15k/qrnn_nl4.m', 0.9514999985694885),
|
||||
('data/mldoc/fr-1/models/sp15k/qrnn_nl4.m', 0.9442499876022339),
|
||||
('data/mldoc/ja-1/models/sp15k/qrnn_nl4.m', 0.8960000276565552),
|
||||
('data/mldoc/zh-1/models/sp15k/qrnn_nl4.m', 0.8982499837875366)])
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
!! WARNING !!
|
||||
|
||||
warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler))
|
||||
Single training schedule
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.610785 0.239165 0.923000
|
||||
2 0.392899 0.281254 0.937000
|
||||
3 0.268695 0.444383 0.909000
|
||||
4 0.162150 0.427744 0.931000
|
||||
5 0.109248 0.422351 0.948000
|
||||
6 0.061984 0.411351 0.947000
|
||||
7 0.033645 0.413174 0.951000
|
||||
8 0.018704 0.404264 0.947000
|
||||
Total time: 02:16
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.29712877, tensor(0.9565)]
|
||||
Processing data/wiki/en-100/models/sp15k/qrnn_nl4.m
|
||||
../mldoc/en-1
|
||||
Max vocab: 15000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k/qrnn_nl4.m
|
||||
Evaluating previously trained model
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/en.dev.csv
|
||||
Data lm, trn: 13500, val: 1500
|
||||
Data cls, trn: 1000, val: 1000
|
||||
Data tst, trn: 1000, val: 4000
|
||||
Size of vocabulary: 15000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁the', '▁,', 's', '▁.', '▁of', '▁and', '▁in', '▁to', '▁a', 'ed']
|
||||
Loss and accuracy using (cls_last): [0.29526812, tensor(0.9463)]
|
||||
Processing data/wiki/es-100/models/sp15k/qrnn_nl4.m
|
||||
../mldoc/es-1
|
||||
Max vocab: 15000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4.m
|
||||
Evaluating previously trained model
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv
|
||||
Data lm, trn: 13013, val: 1445
|
||||
Data cls, trn: 1000, val: 1000
|
||||
Data tst, trn: 1000, val: 4000
|
||||
Size of vocabulary: 15000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que']
|
||||
Loss and accuracy using (cls_last): [0.19222946, tensor(0.9515)]
|
||||
Processing data/wiki/fr-100/models/sp15k/qrnn_nl4.m
|
||||
../mldoc/fr-1
|
||||
Max vocab: 15000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_nl4.m
|
||||
Evaluating previously trained model
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/fr.dev.csv
|
||||
Data lm, trn: 13500, val: 1500
|
||||
Data cls, trn: 1000, val: 1000
|
||||
Data tst, trn: 1000, val: 4000
|
||||
Size of vocabulary: 15000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', 's', '▁.', "'", '▁la', '▁le', '▁et', '▁l', '▁à']
|
||||
Loss and accuracy using (cls_last): [0.33129737, tensor(0.9442)]
|
||||
Processing data/wiki/it-100/models/sp15k/qrnn_nl4.m
|
||||
../mldoc/it-1
|
||||
Max vocab: 15000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k/qrnn_nl4.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/it.dev.csv
|
||||
Running tokenization lm...
|
||||
Data lm, trn: 13500, val: 1500
|
||||
Running tokenization cls...
|
||||
Data cls, trn: 1000, val: 1000
|
||||
Running tokenization tst...
|
||||
Data tst, trn: 1000, val: 4000
|
||||
Size of vocabulary: 15000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁di', '▁e', "▁&'", "'", '▁il', '▁la', '▁in', 'e']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Loading pretrained model
|
||||
Unknown tokens 0, first 100: []
|
||||
Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/it-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/it-100/models/sp15k/qrnn_nl4.m/../itos')]
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 3.929510 2.916437 0.434550
|
||||
Total time: 01:20
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 3.231523 2.723415 0.461407
|
||||
2 2.823881 2.498599 0.496812
|
||||
3 2.586648 2.283846 0.530524
|
||||
4 2.417038 2.125690 0.553137
|
||||
5 2.278636 1.995558 0.572757
|
||||
6 2.199877 1.887804 0.589102
|
||||
7 2.090629 1.799082 0.603201
|
||||
8 2.046975 1.725273 0.615247
|
||||
9 1.935966 1.654829 0.626968
|
||||
10 1.921190 1.590797 0.638228
|
||||
11 1.894758 1.528087 0.649369
|
||||
12 1.792718 1.477532 0.658754
|
||||
13 1.679359 1.428426 0.668648
|
||||
14 1.723383 1.377170 0.678987
|
||||
15 1.597491 1.339658 0.686348
|
||||
16 1.620966 1.307664 0.692993
|
||||
17 1.568962 1.284500 0.697923
|
||||
18 1.533934 1.271438 0.700628
|
||||
19 1.496832 1.264714 0.701968
|
||||
20 1.486198 1.262870 0.702333
|
||||
Total time: 39:33
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k/qrnn_nl4.m/info.json
|
||||
Single training schedule
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.784124 0.414312 0.847000
|
||||
2 0.550873 0.413405 0.861000
|
||||
3 0.445371 0.363693 0.877000
|
||||
4 0.271702 0.426771 0.899000
|
||||
5 0.165902 0.556069 0.881000
|
||||
6 0.091403 0.628809 0.897000
|
||||
7 0.065516 0.693292 0.893000
|
||||
8 0.033616 0.675199 0.897000
|
||||
Total time: 01:24
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k/qrnn_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.7380945, tensor(0.8992)]
|
||||
Processing data/wiki/ja-100/models/sp15k/qrnn_nl4.m
|
||||
../mldoc/ja-1
|
||||
Max vocab: 15000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k/qrnn_nl4.m
|
||||
Evaluating previously trained model
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv
|
||||
Data lm, trn: 13500, val: 1500
|
||||
Data cls, trn: 1000, val: 1000
|
||||
Data tst, trn: 1000, val: 4000
|
||||
Size of vocabulary: 15000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)']
|
||||
Loss and accuracy using (cls_last): [0.7049702, tensor(0.8953)]
|
||||
Processing data/wiki/zh-100/models/sp15k/qrnn_nl4.m
|
||||
../mldoc/zh-1
|
||||
Max vocab: 15000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4.m
|
||||
Evaluating previously trained model
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv
|
||||
Data lm, trn: 13500, val: 1500
|
||||
Data cls, trn: 1000, val: 1000
|
||||
Data tst, trn: 1000, val: 4000
|
||||
Size of vocabulary: 15000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有']
|
||||
Loss and accuracy using (cls_last): [0.30052844, tensor(0.8982)]
|
||||
OrderedDict([('data/mldoc/de-1/models/sp15k/qrnn_nl4.m', 0.9564999938011169),
|
||||
('data/mldoc/en-1/models/sp15k/qrnn_nl4.m', 0.9462500214576721),
|
||||
('data/mldoc/es-1/models/sp15k/qrnn_nl4.m', 0.9514999985694885),
|
||||
('data/mldoc/fr-1/models/sp15k/qrnn_nl4.m', 0.9442499876022339),
|
||||
('data/mldoc/it-1/models/sp15k/qrnn_nl4.m', 0.8992499709129333),
|
||||
('data/mldoc/ja-1/models/sp15k/qrnn_nl4.m', 0.8952500224113464),
|
||||
('data/mldoc/zh-1/models/sp15k/qrnn_nl4.m', 0.8982499837875366)])
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## DE
|
||||
----------------------------------------
|
||||
Training issues
|
||||
|
||||
|
||||
1/2nd -- That was without fine tuning !!! 0 shot:)
|
||||
```
|
||||
python -m ulmfit load_cls data/mldoc/de-1/models/sp15k/qrnn_nl4.m --lang=de - train 0 --num-cls-epochs 8 --bs=18 --lr-sched=1cycle ✘ 1
|
||||
Max vocab: 15000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_None.m
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/de.dev.csv
|
||||
Data lm, trn: 13500, val: 1500
|
||||
Data cls, trn: 1000, val: 1000
|
||||
Data tst, trn: 1000, val: 4000
|
||||
Size of vocabulary: 15000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"]
|
||||
/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning:
|
||||
|
||||
!! WARNING !!
|
||||
|
||||
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
Your compiler (c++) may be ABI-incompatible with PyTorch!
|
||||
Please use a compiler that is ABI-compatible with GCC 4.9 and above.
|
||||
See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html.
|
||||
|
||||
See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6
|
||||
for instructions on how to install GCC 4.9 or higher.
|
||||
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
|
||||
!! WARNING !!
|
||||
|
||||
warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler))
|
||||
Single training schedule
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 1.310368 1.177105 0.520000
|
||||
2 1.096284 0.899281 0.739000
|
||||
3 0.860910 0.668378 0.864000
|
||||
4 0.676764 0.733304 0.868000
|
||||
5 0.573360 0.590983 0.885000
|
||||
6 0.438448 0.446631 0.918000
|
||||
7 0.397323 0.531330 0.919000
|
||||
8 0.339557 0.437841 0.922000
|
||||
Total time: 02:25
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_None.m
|
||||
Loss and accuracy using (cls_best): [0.3380329, tensor(0.9295)]
|
||||
0.33803290128707886
|
||||
0.9294999837875366
|
||||
```
|
||||
|
||||
3rd aproach
|
||||
```
|
||||
Single training schedule
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.610785 0.239165 0.923000
|
||||
2 0.392899 0.281254 0.937000
|
||||
3 0.268695 0.444383 0.909000
|
||||
4 0.162150 0.427744 0.931000
|
||||
5 0.109248 0.422351 0.948000
|
||||
6 0.061984 0.411351 0.947000
|
||||
7 0.033645 0.413174 0.951000
|
||||
8 0.018704 0.404264 0.947000
|
||||
Total time: 02:16
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.29712877, tensor(0.9565)]
|
||||
```
|
||||
@@ -0,0 +1,57 @@
|
||||
|
||||
|
||||
|
||||
## QRNN sp15k
|
||||
```
|
||||
cd fastai # go to fast ai
|
||||
git checkout ulfit_multilingual
|
||||
git pull
|
||||
|
||||
cd ../ulmfit-multilingual # go to ulmfit
|
||||
git checkout master
|
||||
git pull
|
||||
|
||||
export CUDA_VISIBLE_DEVICES=1
|
||||
LANG=fr
|
||||
python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0
|
||||
|
||||
## Jeremy
|
||||
export CUDA_VISIBLE_DEVICES=2
|
||||
LANG=it
|
||||
python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0
|
||||
|
||||
export CUDA_VISIBLE_DEVICES=3
|
||||
LANG=ru
|
||||
python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0
|
||||
```
|
||||
|
||||
|
||||
|
||||
```
|
||||
#
|
||||
export CUDA_VISIBLE_DEVICES=0
|
||||
LANG=de
|
||||
python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0
|
||||
#
|
||||
|
||||
## Piotr
|
||||
export CUDA_VISIBLE_DEVICES=1
|
||||
LANG=es
|
||||
python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0
|
||||
|
||||
export CUDA_VISIBLE_DEVICES=0
|
||||
LANG=zh
|
||||
python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0
|
||||
```
|
||||
|
||||
# trained
|
||||
export CUDA_VISIBLE_DEVICES=0
|
||||
LANG=ja
|
||||
python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0
|
||||
|
||||
done V100
|
||||
```
|
||||
export CUDA_VISIBLE_DEVICES=0
|
||||
LANG=en
|
||||
python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0
|
||||
```
|
||||
@@ -0,0 +1,172 @@
|
||||
# without col merge
|
||||
````
|
||||
python -m ulmfit cls --dataset-path data/cls/${LANG}-books --base-lm-path data/wiki-m/${LANG}-100/models/sp30k/lstm_nl4.m --lang=${LANG} --name 'nl4' - train 20 --bs 20 --num-cls-epochs=8 --lr-sched=single
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k/lstm_nl4.m
|
||||
Validation set not found using 10% of trn
|
||||
Data lm, trn: 33183, val: 3687
|
||||
Data cls, trn: 1800, val: 200
|
||||
Running tokenization tst...
|
||||
Data tst, trn: 200, val: 2000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Loading pretrained model
|
||||
Unknown tokens 0, first 100: []
|
||||
Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki-m/fr-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki-m/fr-100/models/sp30k/lstm_nl4.m/../itos')]
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 3.356221 2.821012 0.518492
|
||||
Total time: 00:21
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 3.041214 2.734799 0.524577
|
||||
2 2.919576 2.648412 0.535661
|
||||
3 2.822292 2.542236 0.549206
|
||||
4 2.721790 2.414110 0.561852
|
||||
5 2.596515 2.276732 0.579841
|
||||
6 2.453715 2.140479 0.600370
|
||||
7 2.333764 2.000186 0.621349
|
||||
8 2.231092 1.873927 0.644259
|
||||
9 2.101130 1.765473 0.660529
|
||||
10 2.006949 1.666797 0.682196
|
||||
11 1.905025 1.584023 0.696058
|
||||
12 1.820798 1.513958 0.709841
|
||||
13 1.751217 1.456632 0.720846
|
||||
14 1.689076 1.410359 0.729947
|
||||
15 1.646113 1.371438 0.739868
|
||||
16 1.594153 1.346142 0.744577
|
||||
17 1.564375 1.332298 0.746693
|
||||
18 1.536557 1.322925 0.748995
|
||||
19 1.532926 1.319159 0.749444
|
||||
20 1.525449 1.318028 0.749815
|
||||
Total time: 08:51
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k/lstm_nl4.m/info.json
|
||||
Single training schedule
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.588118 0.581087 0.700000
|
||||
2 0.504373 0.583527 0.720000
|
||||
3 0.412651 0.538866 0.750000
|
||||
4 0.295401 0.658459 0.750000
|
||||
5 0.212442 1.054068 0.720000
|
||||
6 0.126090 1.302099 0.745000
|
||||
7 0.078312 1.307932 0.760000
|
||||
8 0.050346 1.339740 0.745000
|
||||
Total time: 00:35
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [1.3513571, tensor(0.7700)]
|
||||
1.351357102394104
|
||||
0.7699999809265137
|
||||
````
|
||||
|
||||
### FR books
|
||||
|
||||
````bash
|
||||
python -m ulmfit cls --dataset-path data/cls/${LANG}-books --base-lm-path data/wiki-m/${LANG}-100/models/sp30k/lstm_nl4.m --lang=${LANG} --name 'nl4' - train 20 --bs 20 --num-cls-epochs=8 --lr-sched=single
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k/lstm_nl4.m
|
||||
Validation set not found using 10% of trn
|
||||
Running tokenization lm...
|
||||
Data lm, trn: 33183, val: 3687
|
||||
Running tokenization cls...
|
||||
Data cls, trn: 1800, val: 200
|
||||
Running tokenization tst...
|
||||
Data tst, trn: 200, val: 2000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Loading pretrained model
|
||||
Unknown tokens 0, first 100: []
|
||||
Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki-m/fr-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki-m/fr-100/models/sp30k/lstm_nl4.m/../itos')]
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 3.790325 3.409294 0.367234
|
||||
Total time: 06:02
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 3.526303 3.326439 0.378936
|
||||
2 3.466923 3.226977 0.392378
|
||||
3 3.342312 3.111874 0.406997
|
||||
4 3.244619 2.992510 0.422330
|
||||
5 3.156150 2.877498 0.437467
|
||||
6 3.070326 2.762509 0.453874
|
||||
7 2.956969 2.651613 0.471552
|
||||
8 2.878008 2.535935 0.491058
|
||||
9 2.790110 2.438724 0.508560
|
||||
10 2.684145 2.323467 0.528415
|
||||
11 2.633781 2.231418 0.547093
|
||||
12 2.535126 2.143523 0.564889
|
||||
13 2.464436 2.055402 0.582077
|
||||
14 2.330094 1.989257 0.596582
|
||||
15 2.372371 1.924338 0.610048
|
||||
16 2.190224 1.866912 0.621738
|
||||
17 2.176868 1.834098 0.629221
|
||||
18 2.168293 1.809196 0.633879
|
||||
19 2.151132 1.797144 0.636382
|
||||
20 2.130476 1.793351 0.637044
|
||||
Total time: 2:30:05
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k/lstm_nl4.m/info.json
|
||||
Single training schedule
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.314315 0.530879 0.865000
|
||||
2 0.336746 0.468635 0.865000
|
||||
3 0.255810 0.324242 0.870000
|
||||
4 0.149121 0.480570 0.885000
|
||||
5 0.093909 0.613743 0.890000
|
||||
6 0.091678 0.660452 0.885000
|
||||
7 0.049993 0.649642 0.910000
|
||||
8 0.034218 0.640008 0.910000
|
||||
Total time: 04:19
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp30k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.5418505, tensor(0.9100)]
|
||||
0.5418505072593689
|
||||
0.9100000262260437
|
||||
````
|
||||
|
||||
```
|
||||
Loading pretrained model
|
||||
Unknown tokens 0, first 100: []
|
||||
Training lm from: [PosixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp15k/qrnn_nl4.m/../itos')]
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 4.037580 3.312217 0.364410
|
||||
Total time: 01:44
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 3.709982 3.256320 0.371825
|
||||
2 3.459413 3.150574 0.386972
|
||||
3 3.296628 3.037327 0.402039
|
||||
4 3.186458 2.914899 0.418413
|
||||
5 3.092632 2.817097 0.431216
|
||||
6 2.966957 2.726081 0.442906
|
||||
7 2.924824 2.647339 0.453871
|
||||
8 2.818279 2.561596 0.466795
|
||||
9 2.773893 2.501994 0.475877
|
||||
10 2.736084 2.438490 0.485978
|
||||
11 2.688937 2.370927 0.496899
|
||||
12 2.615245 2.314875 0.506508
|
||||
13 2.583292 2.260717 0.515725
|
||||
14 2.535631 2.220295 0.522666
|
||||
15 2.466035 2.179093 0.530148
|
||||
16 2.461427 2.151952 0.535315
|
||||
17 2.390641 2.131065 0.538749
|
||||
18 2.376235 2.116927 0.541430
|
||||
19 2.407630 2.115370 0.542039
|
||||
20 2.391378 2.112687 0.542522
|
||||
Total time: 46:33
|
||||
/home/n-waves/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp15k
|
||||
Saving info /home/n-waves/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp15k/qrnn_nl4.m/info.json
|
||||
Single training schedule
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.466671 0.534682 0.745000
|
||||
2 0.358965 0.372612 0.875000
|
||||
3 0.251557 0.311034 0.900000
|
||||
4 0.166484 0.585425 0.865000
|
||||
5 0.101803 0.726341 0.900000
|
||||
6 0.072025 0.587875 0.885000
|
||||
7 0.045328 0.760989 0.890000
|
||||
8 0.027765 0.727203 0.890000
|
||||
Total time: 01:17
|
||||
Saving models at /home/n-waves/workspace/ulmfit-multilingual/data/cls/fr-books/models/sp15k/qrnn_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.55982095, tensor(0.8970)]
|
||||
0.5598209500312805
|
||||
0.8970000147819519
|
||||
```
|
||||
@@ -1,4 +1,503 @@
|
||||
# MLDoc
|
||||
### Different training schedules
|
||||
|
||||
### 1cycle -lstm
|
||||
```
|
||||
(fastaiv1) pczapla@galatea ~/w/ulmfit-multilingual ❯❯❯ python -m ulmfit eval --glob="mldoc/*-1/models/sp30k/lstm_nl4.m" --name nl4-1cyc --num-cls-epochs=8 --bs=18 --lr_sched=1cycle ✘ 1
|
||||
Processing data/mldoc/de-1/models/sp30k/lstm_nl4.m
|
||||
de-1
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-1cyc.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/de.dev.csv
|
||||
Data lm, trn: 13500, val: 1500
|
||||
Data cls, trn: 1000, val: 1000
|
||||
Data tst, trn: 1000, val: 4000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Loading pretrained model
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-1cyc.m/info.json
|
||||
Single training schedule
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.610423 0.287707 0.920000
|
||||
2 0.390499 0.266688 0.948000
|
||||
3 0.366716 0.302463 0.933000
|
||||
4 0.248321 0.305547 0.937000
|
||||
5 0.166564 0.411075 0.948000
|
||||
6 0.083940 0.406182 0.950000
|
||||
7 0.033326 0.388105 0.949000
|
||||
8 0.014658 0.397507 0.948000
|
||||
Total time: 06:42
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-1cyc.m
|
||||
Loss and accuracy using (cls_best): [0.3040595, tensor(0.9585)]
|
||||
Processing data/mldoc/es-1/models/sp30k/lstm_nl4.m
|
||||
es-1
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-1cyc.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv
|
||||
Data lm, trn: 13013, val: 1445
|
||||
Data cls, trn: 1000, val: 1000
|
||||
Data tst, trn: 1000, val: 4000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Loading pretrained model
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-1cyc.m/info.json
|
||||
Single training schedule
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.610724 0.278892 0.925000
|
||||
2 0.372022 0.348428 0.937000
|
||||
3 0.310411 0.386958 0.927000
|
||||
4 0.215536 0.273834 0.958000
|
||||
5 0.163195 0.319600 0.958000
|
||||
6 0.085268 0.313287 0.961000
|
||||
7 0.037369 0.347500 0.961000
|
||||
8 0.016851 0.338436 0.963000
|
||||
Total time: 05:36
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-1cyc.m
|
||||
Loss and accuracy using (cls_best): [0.31034237, tensor(0.9632)]
|
||||
Processing data/mldoc/fr-1/models/sp30k/lstm_nl4.m
|
||||
fr-1
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4-1cyc.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/fr.dev.csv
|
||||
Data lm, trn: 13500, val: 1500
|
||||
Data cls, trn: 1000, val: 1000
|
||||
Data tst, trn: 1000, val: 4000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Loading pretrained model
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4-1cyc.m/info.json
|
||||
Single training schedule
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.642809 0.240702 0.928000
|
||||
2 0.420564 0.658542 0.852000
|
||||
3 0.443345 0.244053 0.927000
|
||||
4 0.338779 0.335634 0.914000
|
||||
5 0.224778 0.263748 0.928000
|
||||
6 0.116705 0.280655 0.944000
|
||||
7 0.072063 0.287557 0.945000
|
||||
8 0.048084 0.289200 0.946000
|
||||
Total time: 06:33
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4-1cyc.m
|
||||
Loss and accuracy using (cls_best): [0.29398218, tensor(0.9482)]
|
||||
Processing data/mldoc/it-1/models/sp30k/lstm_nl4.m
|
||||
it-1
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4-1cyc.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/it.dev.csv
|
||||
Data lm, trn: 13500, val: 1500
|
||||
Data cls, trn: 1000, val: 1000
|
||||
Data tst, trn: 1000, val: 4000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁di', "▁&'", "'", '▁e', '▁il', '▁la', 'e', '▁in']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Loading pretrained model
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4-1cyc.m/info.json
|
||||
Single training schedule
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.736070 0.391008 0.859000
|
||||
2 0.512531 0.614638 0.860000
|
||||
3 0.343422 0.594530 0.862000
|
||||
4 0.370786 0.540225 0.884000
|
||||
5 0.234727 0.591903 0.892000
|
||||
6 0.141539 0.589971 0.906000
|
||||
7 0.073315 0.544248 0.906000
|
||||
8 0.038477 0.580940 0.904000
|
||||
Total time: 03:48
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4-1cyc.m
|
||||
Loss and accuracy using (cls_best): [0.6642357, tensor(0.8988)]
|
||||
Processing data/mldoc/ja-1/models/sp30k/lstm_nl4.m
|
||||
ja-1
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-1cyc.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv
|
||||
Data lm, trn: 13500, val: 1500
|
||||
Data cls, trn: 1000, val: 1000
|
||||
Data tst, trn: 1000, val: 4000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁(']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Loading pretrained model
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-1cyc.m/info.json
|
||||
Single training schedule
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.804430 0.435780 0.837000
|
||||
2 0.576932 0.452903 0.839000
|
||||
3 0.499791 0.640789 0.806000
|
||||
4 0.418024 0.610898 0.839000
|
||||
5 0.259578 0.582953 0.868000
|
||||
6 0.188161 0.719131 0.888000
|
||||
7 0.101508 0.766175 0.877000
|
||||
8 0.072221 0.795415 0.883000
|
||||
Total time: 08:19
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-1cyc.m
|
||||
Loss and accuracy using (cls_best): [0.636261, tensor(0.9045)]
|
||||
Processing data/mldoc/ru-1/models/sp30k/lstm_nl4.m
|
||||
ru-1
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4-1cyc.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv
|
||||
Data lm, trn: 9195, val: 1021
|
||||
Data cls, trn: 1000, val: 1000
|
||||
Data tst, trn: 1000, val: 4000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁в', 'а', '▁и', 'е', 'и', 'й', '▁на', 'х']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Loading pretrained model
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4-1cyc.m/info.json
|
||||
Single training schedule
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.821141 0.532432 0.814000
|
||||
2 0.611059 0.457023 0.866000
|
||||
3 0.464408 0.484035 0.870000
|
||||
4 0.446560 0.477454 0.858000
|
||||
5 0.299095 0.906959 0.858000
|
||||
6 0.176480 0.709579 0.875000
|
||||
7 0.089683 0.781081 0.876000
|
||||
8 0.047031 0.772935 0.877000
|
||||
Total time: 08:58
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4-1cyc.m
|
||||
Loss and accuracy using (cls_best): [0.8528109, tensor(0.8795)]
|
||||
Processing data/mldoc/zh-1/models/sp30k/lstm_nl4.m
|
||||
zh-1
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv
|
||||
Data lm, trn: 13500, val: 1500
|
||||
Data cls, trn: 1000, val: 1000
|
||||
Data tst, trn: 1000, val: 4000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Loading pretrained model
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc.m/info.json
|
||||
Single training schedule
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.696346 0.335083 0.895000
|
||||
2 0.505075 0.360600 0.906000
|
||||
3 0.427076 0.462661 0.883000
|
||||
4 0.351177 0.489026 0.919000
|
||||
5 0.244958 0.415151 0.918000
|
||||
6 0.156452 0.494367 0.926000
|
||||
7 0.085807 0.471611 0.927000
|
||||
8 0.046887 0.484232 0.929000
|
||||
Total time: 06:41
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc.m
|
||||
Loss and accuracy using (cls_best): [0.50999963, tensor(0.9165)]
|
||||
OrderedDict([('data/mldoc/de-1/models/sp30k/lstm_nl4-1cyc.m',
|
||||
0.9585000276565552),
|
||||
('data/mldoc/es-1/models/sp30k/lstm_nl4-1cyc.m',
|
||||
0.9632499814033508),
|
||||
('data/mldoc/fr-1/models/sp30k/lstm_nl4-1cyc.m',
|
||||
0.9482499957084656),
|
||||
('data/mldoc/it-1/models/sp30k/lstm_nl4-1cyc.m',
|
||||
0.8987500071525574),
|
||||
('data/mldoc/ja-1/models/sp30k/lstm_nl4-1cyc.m',
|
||||
0.9045000076293945),
|
||||
('data/mldoc/ru-1/models/sp30k/lstm_nl4-1cyc.m',
|
||||
0.8794999718666077),
|
||||
('data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc.m',
|
||||
0.9164999723434448)])
|
||||
data/mldoc/de-1/models/sp30k/lstm_nl4-1cyc.m: 0.9585000276565552
|
||||
data/mldoc/es-1/models/sp30k/lstm_nl4-1cyc.m: 0.9632499814033508
|
||||
data/mldoc/fr-1/models/sp30k/lstm_nl4-1cyc.m: 0.9482499957084656
|
||||
data/mldoc/it-1/models/sp30k/lstm_nl4-1cyc.m: 0.8987500071525574
|
||||
data/mldoc/ja-1/models/sp30k/lstm_nl4-1cyc.m: 0.9045000076293945
|
||||
data/mldoc/ru-1/models/sp30k/lstm_nl4-1cyc.m: 0.8794999718666077
|
||||
data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc.m: 0.9164999723434448
|
||||
```
|
||||
|
||||
|
||||
### 2cycle
|
||||
```bash
|
||||
python -m ulmfit eval --glob="mldoc/*-1/models/sp15k/qrnn_nl4.m" --name nl4-8e-2cycle --num-cls-epochs=8 --bs=18 --lr_sched=2cycle
|
||||
Max vocab: 15000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-8e-2cycle.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv
|
||||
Data lm, trn: 13013, val: 1445
|
||||
Data cls, trn: 1000, val: 1000
|
||||
Data tst, trn: 1000, val: 4000
|
||||
Size of vocabulary: 15000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning:
|
||||
|
||||
!! WARNING !!
|
||||
|
||||
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
Your compiler (c++) may be ABI-incompatible with PyTorch!
|
||||
Please use a compiler that is ABI-compatible with GCC 4.9 and above.
|
||||
See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html.
|
||||
|
||||
See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6
|
||||
for instructions on how to install GCC 4.9 or higher.
|
||||
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
|
||||
!! WARNING !!
|
||||
|
||||
warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler))
|
||||
Loading pretrained model
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-8e-2cycle.m/info.json
|
||||
2cycle training schedule
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.600965 0.232749 0.937000
|
||||
Total time: 00:07
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.304946 0.202946 0.946000
|
||||
2 0.326092 0.207825 0.954000
|
||||
3 0.286274 0.290416 0.943000
|
||||
4 0.230937 0.263474 0.950000
|
||||
5 0.153293 0.293336 0.962000
|
||||
6 0.080219 0.328380 0.960000
|
||||
7 0.065156 0.343692 0.961000
|
||||
8 0.046342 0.367162 0.962000
|
||||
9 0.028884 0.396987 0.960000
|
||||
10 0.034997 0.366203 0.960000
|
||||
Total time: 02:27
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-8e-2cycle.m
|
||||
Loss and accuracy using (cls_best): [0.35007542, tensor(0.9528)]
|
||||
Max vocab: 15000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-8e-2cycle.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv
|
||||
Data lm, trn: 13500, val: 1500
|
||||
Data cls, trn: 1000, val: 1000
|
||||
Data tst, trn: 1000, val: 4000
|
||||
Size of vocabulary: 15000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Loading pretrained model
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-8e-2cycle.m/info.json
|
||||
2cycle training schedule
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.675992 0.460828 0.836000
|
||||
Total time: 00:09
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.439060 0.314642 0.888000
|
||||
2 0.382209 0.374305 0.893000
|
||||
3 0.338183 0.361669 0.911000
|
||||
4 0.260323 0.431681 0.901000
|
||||
5 0.146894 0.597865 0.899000
|
||||
6 0.090651 0.589435 0.910000
|
||||
7 0.079902 0.624589 0.918000
|
||||
8 0.043067 0.558498 0.918000
|
||||
9 0.022371 0.568702 0.921000
|
||||
10 0.022498 0.576052 0.922000
|
||||
Total time: 02:53
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-8e-2cycle.m
|
||||
Loss and accuracy using (cls_best): [0.6146808, tensor(0.9150)]
|
||||
OrderedDict([('data/mldoc/es-1/models/sp15k/qrnn_nl4-8e-2cycle.m',
|
||||
0.952750027179718),
|
||||
('data/mldoc/zh-1/models/sp15k/qrnn_nl4-8e-2cycle.m',
|
||||
0.9150000214576721)])
|
||||
```
|
||||
|
||||
### SIUNGLE 2epochs
|
||||
```
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv
|
||||
Data lm, trn: 13500, val: 1500
|
||||
Data cls, trn: 1000, val: 1000
|
||||
Data tst, trn: 1000, val: 4000
|
||||
Size of vocabulary: 15000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Loading pretrained model
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-2e-single.m/info.json
|
||||
Single training schedule
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.626836 0.318460 0.912000
|
||||
2 0.386851 0.327937 0.918000
|
||||
Total time: 00:34
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-2e-single.m
|
||||
Loss and accuracy using (cls_best): [0.32642558, tensor(0.9135)]
|
||||
OrderedDict([('data/mldoc/es-1/models/sp15k/qrnn_nl4-2e-single.m',
|
||||
0.9539999961853027),
|
||||
('data/mldoc/zh-1/models/sp15k/qrnn_nl4-2e-single.m',
|
||||
0.9135000109672546)])
|
||||
```
|
||||
### SINGLE 4epochs
|
||||
```
|
||||
OrderedDict([('data/mldoc/es-1/models/sp15k/qrnn_nl4-4e-single.m',
|
||||
0.9539999961853027),
|
||||
('data/mldoc/zh-1/models/sp15k/qrnn_nl4-4e-single.m',
|
||||
0.9210000038146973)])
|
||||
```
|
||||
|
||||
|
||||
### SINGLE 5 epochs
|
||||
```
|
||||
python -m ulmfit eval --glob="mldoc/*-1/models/sp15k/qrnn_nl4.m" --name nl4-5e-single --num-cls-epochs=5 --bs=18 --lr_sched=single
|
||||
Max vocab: 15000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-5e-single.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv
|
||||
Data lm, trn: 13013, val: 1445
|
||||
Data cls, trn: 1000, val: 1000
|
||||
Data tst, trn: 1000, val: 4000
|
||||
Size of vocabulary: 15000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning:
|
||||
|
||||
!! WARNING !!
|
||||
|
||||
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
Your compiler (c++) may be ABI-incompatible with PyTorch!
|
||||
Please use a compiler that is ABI-compatible with GCC 4.9 and above.
|
||||
See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html.
|
||||
|
||||
See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6
|
||||
for instructions on how to install GCC 4.9 or higher.
|
||||
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
|
||||
!! WARNING !!
|
||||
|
||||
warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler))
|
||||
Loading pretrained model
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-5e-single.m/info.json
|
||||
Single training schedule
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.559638 0.250022 0.931000
|
||||
2 0.366238 0.348553 0.932000
|
||||
3 0.246830 0.243392 0.954000
|
||||
4 0.136335 0.242888 0.960000
|
||||
5 0.106189 0.254603 0.965000
|
||||
Total time: 01:14
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-5e-single.m
|
||||
Loss and accuracy using (cls_best): [0.24189772, tensor(0.9588)]
|
||||
Max vocab: 15000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-5e-single.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv
|
||||
Data lm, trn: 13500, val: 1500
|
||||
Data cls, trn: 1000, val: 1000
|
||||
Data tst, trn: 1000, val: 4000
|
||||
Size of vocabulary: 15000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Loading pretrained model
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-5e-single.m/info.json
|
||||
Single training schedule
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.641053 0.334561 0.891000
|
||||
2 0.486526 0.374095 0.894000
|
||||
3 0.309409 0.359222 0.908000
|
||||
4 0.179104 0.403468 0.920000
|
||||
5 0.083530 0.414904 0.919000
|
||||
Total time: 01:24
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-5e-single.m
|
||||
Loss and accuracy using (cls_best): [0.44048822, tensor(0.9110)]
|
||||
OrderedDict([('data/mldoc/es-1/models/sp15k/qrnn_nl4-5e-single.m',
|
||||
0.9587500095367432),
|
||||
('data/mldoc/zh-1/models/sp15k/qrnn_nl4-5e-single.m',
|
||||
0.9110000133514404)])
|
||||
```
|
||||
|
||||
|
||||
#### SINGLE 11epochs
|
||||
```
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-8e-single.m/info.json
|
||||
Starting classifier training
|
||||
Single training schedule
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.676591 0.205210 0.947000
|
||||
2 0.402020 0.461279 0.912000
|
||||
3 0.287975 0.496294 0.921000
|
||||
4 0.258515 0.243489 0.954000
|
||||
5 0.219352 0.274136 0.949000
|
||||
6 0.149339 0.352294 0.956000
|
||||
7 0.092821 0.378696 0.962000
|
||||
8 0.055485 0.367379 0.963000
|
||||
9 0.042695 0.367151 0.964000
|
||||
10 0.034858 0.386749 0.961000
|
||||
11 0.021245 0.392899 0.963000
|
||||
Total time: 02:38
|
||||
|
||||
Loss and accuracy using (cls_last): [0.4098273, tensor(0.9595)]
|
||||
Max vocab: 15000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-8e-single.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv
|
||||
Data lm, trn: 13500, val: 1500
|
||||
Data cls, trn: 1000, val: 1000
|
||||
Data tst, trn: 1000, val: 4000
|
||||
Size of vocabulary: 15000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Loading pretrained model
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-8e-single.m/info.json
|
||||
Starting classifier training
|
||||
Single training schedule
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.710727 0.355552 0.895000
|
||||
2 0.523836 0.328691 0.895000
|
||||
3 0.408339 0.440722 0.894000
|
||||
4 0.326642 0.422076 0.909000
|
||||
5 0.217328 0.539624 0.906000
|
||||
6 0.156753 0.583433 0.912000
|
||||
7 0.102770 0.549827 0.921000
|
||||
8 0.053252 0.533528 0.928000
|
||||
9 0.032845 0.568053 0.927000
|
||||
10 0.022289 0.604236 0.926000
|
||||
11 0.015289 0.587667 0.929000
|
||||
Total time: 03:12
|
||||
OrderedDict([('data/mldoc/es-1/models/sp15k/qrnn_nl4-8e-single.m',
|
||||
0.9595000147819519),
|
||||
('data/mldoc/zh-1/models/sp15k/qrnn_nl4-8e-single.m',
|
||||
0.9202499985694885)])
|
||||
|
||||
```
|
||||
|
||||
## Limiit to 100 examples
|
||||
```
|
||||
python -m ulmfit eval --glob="mldoc/*-1/models/sp30k/lstm_nl4.m" --name nl4-100e8 --cuda-id=1 --limit=100 --num-cls-epochs=8
|
||||
|
||||
@@ -1,4 +1,28 @@
|
||||
# DE
|
||||
## SP15k LSTM nl4
|
||||
```
|
||||
$ python -m ulmfit lm --dataset-path data/wiki/de-100 --cuda-id=0 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang de --qrnn=False - train 10 --bs=100 --drop_mult=0
|
||||
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁.', '▁,', '▁der',
|
||||
'▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"]
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Training lm from random weights
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 2.519809 2.600072 0.529963
|
||||
2 2.436580 2.538897 0.534651
|
||||
3 2.402220 2.510569 0.537314
|
||||
4 2.305741 2.439347 0.546574
|
||||
5 2.265683 2.376482 0.553794
|
||||
6 2.210663 2.305362 0.562672
|
||||
7 2.134196 2.230041 0.572958
|
||||
8 2.085375 2.150917 0.584621
|
||||
9 2.037781 2.097170 0.593747
|
||||
10 1.986773 2.081469 0.595799
|
||||
Total time: 19:18:33
|
||||
data/wiki/de-100/models/sp15k
|
||||
Saving info data/wiki/de-100/models/sp15k/lstm_nl4.m/info.jso
|
||||
```
|
||||
|
||||
## VF60k LSTM nl 3
|
||||
### LM
|
||||
```
|
||||
|
||||
+187
-1
@@ -80,4 +80,190 @@ epoch train_loss valid_loss accuracy
|
||||
2 0.262074 0.202975 0.945000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.1749019, tensor(0.9515)]
|
||||
```
|
||||
```
|
||||
|
||||
|
||||
|
||||
## ES optimization
|
||||
|
||||
|
||||
|
||||
### Smaler vocab 15k
|
||||
#### LM
|
||||
```
|
||||
python -m ulmfit lm --dataset-path data/wiki-m/es-100 --cuda-id=0 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 \ ✘ 1
|
||||
--lang es --qrnn=False - train 10 --bs=50 --drop_mult=0
|
||||
Max vocab: 15000
|
||||
Cache dir: data/wiki-m/es-100/models/sp15k
|
||||
Model dir: data/wiki-m/es-100/models/sp15k/lstm_nl4.m
|
||||
Tokenized data loaded
|
||||
Size of vocabulary: 15000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Training lm from random weights
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 2.961702 3.187043 0.403149
|
||||
Better model found at epoch 1 with val_loss value: 3.1870434284210205.
|
||||
2 2.928991 3.170802 0.402026
|
||||
Better model found at epoch 2 with val_loss value: 3.170802354812622.
|
||||
3 2.931906 3.128328 0.407816
|
||||
Better model found at epoch 3 with val_loss value: 3.128328323364258.
|
||||
4 2.869332 3.072160 0.414345
|
||||
Better model found at epoch 4 with val_loss value: 3.072160243988037.
|
||||
5 2.803377 2.997071 0.424847
|
||||
Better model found at epoch 5 with val_loss value: 2.997070550918579.
|
||||
6 2.758087 2.927369 0.432256
|
||||
Better model found at epoch 6 with val_loss value: 2.927368640899658.
|
||||
7 2.657733 2.825029 0.446440
|
||||
Better model found at epoch 7 with val_loss value: 2.8250293731689453.
|
||||
8 2.563273 2.728652 0.459271
|
||||
Better model found at epoch 8 with val_loss value: 2.7286524772644043.
|
||||
9 2.475741 2.654844 0.470864
|
||||
Better model found at epoch 9 with val_loss value: 2.654844045639038.
|
||||
10 2.428898 2.634355 0.474821
|
||||
Better model found at epoch 10 with val_loss value: 2.634355306625366.
|
||||
Total time: 17:53:59
|
||||
data/wiki-m/es-100/models/sp15k
|
||||
Saving info data/wiki-m/es-100/models/sp15k/lstm_nl4.m/info.json
|
||||
```
|
||||
|
||||
#### MLDoc
|
||||
```
|
||||
python -m ulmfit cls --dataset-path data/mldoc/es-1 --base-lm-path data/wiki-m/es-100/models/sp15k/lstm_nl4.m --lang=es --name 'nl4' --cuda-id=0 - train 20 --bs 20 --num-cls-epochs=8
|
||||
Max vocab: 15000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/lstm_nl4.m
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 13013, cls.val 1445
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 15000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Loading pretrained model
|
||||
Unknown tokens 0, first 100: []
|
||||
Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki-m/es-100/models/sp15k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki-m/es-100/models/sp15k/lstm_nl4.m/../itos')]
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 2.355042 1.850417 0.589128
|
||||
Better model found at epoch 1 with val_loss value: 1.850416898727417.
|
||||
Total time: 03:25
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 2.087364 1.677666 0.619909
|
||||
Better model found at epoch 1 with val_loss value: 1.6776657104492188.
|
||||
2 1.880730 1.522996 0.648134
|
||||
Better model found at epoch 2 with val_loss value: 1.5229955911636353.
|
||||
3 1.767530 1.403644 0.668117
|
||||
Better model found at epoch 3 with val_loss value: 1.4036436080932617.
|
||||
4 1.659950 1.309353 0.684900
|
||||
Better model found at epoch 4 with val_loss value: 1.3093526363372803.
|
||||
5 1.546585 1.232220 0.699358
|
||||
Better model found at epoch 5 with val_loss value: 1.2322196960449219.
|
||||
6 1.592862 1.161846 0.713034
|
||||
Better model found at epoch 6 with val_loss value: 1.1618456840515137.
|
||||
7 1.444965 1.098108 0.726811
|
||||
Better model found at epoch 7 with val_loss value: 1.0981075763702393.
|
||||
8 1.340874 1.029193 0.741337
|
||||
Better model found at epoch 8 with val_loss value: 1.0291931629180908.
|
||||
9 1.351407 0.974317 0.753408
|
||||
Better model found at epoch 9 with val_loss value: 0.9743167757987976.
|
||||
10 1.231713 0.915328 0.767088
|
||||
Better model found at epoch 10 with val_loss value: 0.9153280854225159.
|
||||
11 1.151926 0.852391 0.782414
|
||||
Better model found at epoch 11 with val_loss value: 0.852391242980957.
|
||||
12 1.163565 0.794699 0.797228
|
||||
Better model found at epoch 12 with val_loss value: 0.7946987152099609.
|
||||
13 1.054929 0.743652 0.810518
|
||||
Better model found at epoch 13 with val_loss value: 0.74365234375.
|
||||
14 0.974651 0.695024 0.823344
|
||||
Better model found at epoch 14 with val_loss value: 0.6950243711471558.
|
||||
15 0.869718 0.651691 0.834510
|
||||
Better model found at epoch 15 with val_loss value: 0.6516908407211304.
|
||||
16 0.889763 0.615112 0.844947
|
||||
Better model found at epoch 16 with val_loss value: 0.6151121258735657.
|
||||
17 0.843503 0.590130 0.851694
|
||||
Better model found at epoch 17 with val_loss value: 0.5901297926902771.
|
||||
18 0.752870 0.575217 0.855496
|
||||
Better model found at epoch 18 with val_loss value: 0.5752172470092773.
|
||||
19 0.807087 0.567187 0.857605
|
||||
Better model found at epoch 19 with val_loss value: 0.5671872496604919.
|
||||
20 0.784531 0.566082 0.857827
|
||||
Better model found at epoch 20 with val_loss value: 0.5660821199417114.
|
||||
Total time: 1:26:48
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/lstm_nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.616963 0.275447 0.923000
|
||||
Better model found at epoch 1 with val_loss value: 0.27544698119163513.
|
||||
Total time: 00:24
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.425890 0.201535 0.932000
|
||||
Better model found at epoch 1 with val_loss value: 0.2015346735715866.
|
||||
Total time: 00:27
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.265563 0.186434 0.951000
|
||||
Better model found at epoch 1 with val_loss value: 0.18643426895141602.
|
||||
Total time: 00:32
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.162097 0.180026 0.955000
|
||||
Better model found at epoch 1 with val_loss value: 0.1800260841846466.
|
||||
2 0.161748 0.187014 0.957000
|
||||
3 0.142789 0.166486 0.961000
|
||||
Better model found at epoch 3 with val_loss value: 0.1664857715368271.
|
||||
4 0.105920 0.173207 0.963000
|
||||
5 0.078164 0.184849 0.962000
|
||||
6 0.070540 0.189451 0.962000
|
||||
7 0.051490 0.208019 0.959000
|
||||
8 0.047614 0.193699 0.962000
|
||||
Total time: 05:20
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.1623594, tensor(0.9538)]
|
||||
0.16235940158367157
|
||||
0.9537500143051147
|
||||
```
|
||||
|
||||
### Larger dropout - no luck
|
||||
```
|
||||
python -m ulmfit cls --dataset-path data/mldoc/es-1 --base-lm-path data/wiki-m/es-100/models/sp30k/lstm_nl4.m --lang=es --name 'nl4-drop' --cuda-id=0 - train 0 --bs 20 --num-cls-epochs=8 --drop-mul-lm=0.5 --drop-mul-cls=0.8
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-drop.m
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 13013, cls.val 1445
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que']
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 1.310594 0.903076 0.724000
|
||||
Better model found at epoch 1 with val_loss value: 0.9030755758285522.
|
||||
Total time: 00:22
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 1.209348 0.714140 0.756000
|
||||
Better model found at epoch 1 with val_loss value: 0.714139997959137.
|
||||
Total time: 00:23
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 1.122636 0.627526 0.797000
|
||||
Better model found at epoch 1 with val_loss value: 0.6275263428688049.
|
||||
Total time: 00:29
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 1.102433 0.593338 0.801000
|
||||
Better model found at epoch 1 with val_loss value: 0.5933384895324707.
|
||||
2 1.096480 0.543266 0.818000
|
||||
Better model found at epoch 2 with val_loss value: 0.5432664155960083.
|
||||
3 1.082919 0.501089 0.837000
|
||||
Better model found at epoch 3 with val_loss value: 0.5010889172554016.
|
||||
4 1.069694 0.518807 0.812000
|
||||
5 1.040208 0.508399 0.825000
|
||||
6 1.032841 0.512187 0.838000
|
||||
7 1.031225 0.504557 0.825000
|
||||
8 1.016486 0.502335 0.837000
|
||||
Total time: 04:56
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4-drop.m
|
||||
Loss and accuracy using (cls_best): [0.52473265, tensor(0.8160)]
|
||||
0.5247326493263245
|
||||
0.8159999847412109
|
||||
```
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
python -m ulmfit eval --glob="mldoc/*-1/models/sp15k/qrnn_nl4.m" --name nl4-8e-single --num-cls-epochs=8 --bs=18 --single=True
|
||||
Max vocab: 15000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-8e-single.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv
|
||||
Data lm, trn: 13013, val: 1445
|
||||
Data cls, trn: 1000, val: 1000
|
||||
Data tst, trn: 1000, val: 4000
|
||||
Size of vocabulary: 15000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning:
|
||||
|
||||
!! WARNING !!
|
||||
|
||||
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
Your compiler (c++) may be ABI-incompatible with PyTorch!
|
||||
Please use a compiler that is ABI-compatible with GCC 4.9 and above.
|
||||
See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html.
|
||||
|
||||
See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6
|
||||
for instructions on how to install GCC 4.9 or higher.
|
||||
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
|
||||
!! WARNING !!
|
||||
|
||||
warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler))
|
||||
Loading pretrained model
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-8e-single.m/info.json
|
||||
Starting classifier training
|
||||
Single training schedule
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.676591 0.205210 0.947000
|
||||
2 0.402020 0.461279 0.912000
|
||||
3 0.287975 0.496294 0.921000
|
||||
4 0.258515 0.243489 0.954000
|
||||
5 0.219352 0.274136 0.949000
|
||||
6 0.149339 0.352294 0.956000
|
||||
7 0.092821 0.378696 0.962000
|
||||
8 0.055485 0.367379 0.963000
|
||||
9 0.042695 0.367151 0.964000
|
||||
10 0.034858 0.386749 0.961000
|
||||
11 0.021245 0.392899 0.963000
|
||||
Total time: 02:38
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-8e-single.m
|
||||
Traceback (most recent call last):
|
||||
File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/runpy.py", line 193, in _run_module_as_main
|
||||
"__main__", mod_spec)
|
||||
File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/runpy.py", line 85, in _run_code
|
||||
exec(code, run_globals)
|
||||
File "/home/pczapla/workspace/ulmfit-multilingual/ulmfit/__main__.py", line 73, in <module>
|
||||
fire.Fire(ULMFiT())
|
||||
File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/fire/core.py", line 127, in Fire
|
||||
component_trace = _Fire(component, args, context, name)
|
||||
File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/fire/core.py", line 366, in _Fire
|
||||
component, remaining_args)
|
||||
File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/fire/core.py", line 542, in _CallCallable
|
||||
result = fn(*varargs, **kwargs)
|
||||
File "/home/pczapla/workspace/ulmfit-multilingual/ulmfit/__main__.py", line 55, in eval
|
||||
results[key] = params.train_cls(num_lm_epochs=num_lm_epochs, **trn_params)[1]
|
||||
File "/home/pczapla/workspace/ulmfit-multilingual/ulmfit/train_clas.py", line 76, in train_cls
|
||||
return self.validate_cls('cls_best', bs=bs, data_tst=data_tst, learn=learn)
|
||||
File "/home/pczapla/workspace/ulmfit-multilingual/ulmfit/train_clas.py", line 84, in validate_cls
|
||||
learn.load(save_name)
|
||||
File "/home/pczapla/workspace/_oss/fastai/fastai/fastai/basic_train.py", line 243, in load
|
||||
if purge: self.purge(clear_opt=ifnone(with_opt, False))
|
||||
File "/home/pczapla/workspace/_oss/fastai/fastai/fastai/basic_train.py", line 293, in purge
|
||||
self.opt = OptimWrapper.load_with_state_and_layer_group(state['opt'], self.layer_groups)
|
||||
File "/home/pczapla/workspace/_oss/fastai/fastai/fastai/callback.py", line 130, in load_with_state_and_layer_group
|
||||
res.load_state_dict(state['opt_state'])
|
||||
File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/optim/optimizer.py", line 108, in load_state_dict
|
||||
raise ValueError("loaded state dict contains a parameter group "
|
||||
ValueError: loaded state dict contains a parameter group that doesn't match the size of optimizer's group
|
||||
@@ -1,4 +1,37 @@
|
||||
# FR
|
||||
|
||||
## SP15k QRNN nl 4
|
||||
```
|
||||
python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0
|
||||
Max vocab: 15000
|
||||
Cache dir: data/wiki/fr-100/models/sp15k
|
||||
Model dir: data/wiki/fr-100/models/sp15k/qrnn_nl4.m
|
||||
Wiki text was split to 174227 articles
|
||||
Wiki text was split to 491 articles
|
||||
Running tokenization lm...
|
||||
Data lm, trn: 174227, val: 491
|
||||
Size of vocabulary: 15000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', 's', '▁.', "'", '▁la', '▁le',
|
||||
'▁et', '▁l', '▁à']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Training lm from random weights
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 2.881558 2.790402 0.465847
|
||||
2 2.824942 2.732005 0.471660
|
||||
3 2.758845 2.672040 0.478273
|
||||
4 2.715069 2.602380 0.489159
|
||||
5 2.677029 2.553575 0.494752
|
||||
6 2.602514 2.476142 0.507337
|
||||
7 2.564386 2.388670 0.518902
|
||||
8 2.470835 2.304033 0.532000
|
||||
9 2.366890 2.243269 0.542781
|
||||
10 2.390439 2.223538 0.546622
|
||||
Total time: 9:09:26
|
||||
data/wiki/fr-100/models/sp15k
|
||||
Saving info data/wiki/fr-100/models/sp15k/qrnn_nl4.m/info.json
|
||||
|
||||
```
|
||||
|
||||
## SP30k LSTM nl 4
|
||||
### LM
|
||||
```
|
||||
|
||||
+25
-1
@@ -1,4 +1,28 @@
|
||||
# FR
|
||||
## SP15k QRNN NL4
|
||||
### LM
|
||||
|
||||
```
|
||||
export CUDA_VISIBLE_DEVICES=0
|
||||
LANG=it
|
||||
python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 3.171145 3.516659 0.359233
|
||||
2 3.045057 3.472802 0.359628
|
||||
3 3.023009 3.401181 0.367101
|
||||
4 2.985105 3.351916 0.372709
|
||||
5 2.858441 3.280903 0.380848
|
||||
6 2.862504 3.210976 0.390263
|
||||
7 2.758775 3.122354 0.402106
|
||||
8 2.683234 3.035321 0.413798
|
||||
9 2.593757 2.964551 0.424886
|
||||
10 2.535500 2.947672 0.427958
|
||||
Total time: 11:30:03
|
||||
data/wiki/it-100/models/sp15k
|
||||
Saving info data/wiki/it-100/models/sp15k/qrnn_nl4.m/info.json
|
||||
```
|
||||
## xx
|
||||
|
||||
## SP30k LSTM nl 4
|
||||
### LM
|
||||
```
|
||||
@@ -79,4 +103,4 @@ epoch train_loss valid_loss accuracy
|
||||
2 0.340658 0.315946 0.877000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4bs100.m
|
||||
Loss and accuracy using (cls_best): [0.32998973, tensor(0.8842)]
|
||||
```
|
||||
```
|
||||
|
||||
+304
-1
@@ -30,6 +30,8 @@ Saving info data/wiki/ja-100/models/sp30k/lstm_nl4.m/info.json
|
||||
```
|
||||
|
||||
### MLDoc
|
||||
|
||||
#### CLS 1
|
||||
MultiCCA 85.35%, ULMFiT 89.20%
|
||||
```
|
||||
python -m ulmfit cls --dataset-path data/mldoc/ja-1 --base-lm-path data/wiki/ja-100/models/sp30k/lstm_nl4.m --lang=ja --name 'nl4' --cuda-id=1 - train 20 --bs 40 --num-cls-epochs=8
|
||||
@@ -89,6 +91,66 @@ epoch train_loss valid_loss accuracy
|
||||
8 0.278896 0.358145 0.877000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.29789856, tensor(0.8920)]
|
||||
|
||||
|
||||
$ mv /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4x8.m
|
||||
|
||||
$ python -m ulmfit eval --glob="mldoc/ja-1/models/sp30k/lstm_nl4x8.m" --name nl4 --cuda-id=0
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv
|
||||
Tokenized data loaded, lm.trn 13500, lm.val 1500
|
||||
Tokenized data loaded, cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁(']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4x8.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4x8.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.757615 0.562652 0.825000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.609298 0.382412 0.870000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.544682 0.379602 0.871000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.435453 0.360421 0.885000
|
||||
2 0.426099 0.350480 0.885000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.32738593, tensor(0.8905)]
|
||||
OrderedDict([('data/mldoc/ja-1/models/sp30k/lstm_nl4.m', 0.890500009059906)])
|
||||
|
||||
python -m ulmfit eval --glob="mldoc/ja-1/models/sp30k/lstm_nl4x8.m" --name nl4x2 --cuda-id=0
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4x2.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv
|
||||
Tokenized data loaded, lm.trn 13500, lm.val 1500
|
||||
Tokenized data loaded, cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁(']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4x8.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4x8.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4x2.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.778722 0.690746 0.805000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.574789 0.386483 0.862000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.518843 0.361983 0.869000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.435260 0.350808 0.869000
|
||||
2 0.386701 0.352221 0.875000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4x2.m
|
||||
Loss and accuracy using (cls_best): [0.31783763, tensor(0.8892)]
|
||||
OrderedDict([('data/mldoc/ja-1/models/sp30k/lstm_nl4x2.m', 0.8892499804496765)])
|
||||
|
||||
```
|
||||
|
||||
### JA on 100 elements
|
||||
@@ -150,4 +212,245 @@ epoch train_loss valid_loss accuracy
|
||||
8 0.668421 0.984848 0.590000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-100.m
|
||||
Loss and accuracy using (cls_best): [0.81621724, tensor(0.7437)]
|
||||
```
|
||||
```
|
||||
|
||||
|
||||
### Japanese fixed sentence piece
|
||||
```
|
||||
python -m ulmfit cls --dataset-path data/mldoc/ja-1 --base-lm-path data/mldoc/ja-1/models/bsp30k/lstm_nl4.m --lang=ja --name 'nl4' --cuda-id=0 - train 1 --bs 40 --num-cls-epochs=2
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv
|
||||
Tokenized data loaded, lm.trn 13500, lm.val 1500
|
||||
Tokenized data loaded, cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁(']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/../itos')]
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 1.222162 0.986234 0.765830
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 1.237291 0.983976 0.766659
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.759230 0.619306 0.826000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.599281 0.423162 0.841000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.485808 0.360609 0.869000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.415960 0.390202 0.872000
|
||||
2 0.371651 0.365374 0.876000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.330675, tensor(0.8873)]
|
||||
0.33067500591278076
|
||||
0.8872500061988831
|
||||
(fastaiv1) pczapla@galatea ~/w/ulmfit-multilingual ❯❯❯ python -m ulmfit eval --glob="mldoc/ja-1/models/sp30k/lstm_nl4.m" --name nl4-2nd --cuda-id=0
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-2nd.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv
|
||||
Tokenized data loaded, lm.trn 13500, lm.val 1500
|
||||
Tokenized data loaded, cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁(']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-2nd.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.844110 0.700549 0.743000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.610796 0.400912 0.853000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.449974 0.358793 0.870000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.407609 0.397678 0.868000
|
||||
2 0.367383 0.373168 0.869000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4-2nd.m
|
||||
Loss and accuracy using (cls_best): [0.33044776, tensor(0.8863)]
|
||||
OrderedDict([('data/mldoc/ja-1/models/sp30k/lstm_nl4-2nd.m',
|
||||
0.8862500190734863)])
|
||||
```
|
||||
```
|
||||
python -m ulmfit cls --dataset-path data/mldoc/ja-1 --base-lm-path data/mldoc/ja-1/models/bsp30k/lstm_nl4.m --lang=ja --name '2nd-nl4' --cuda-id=0 - train 1 --bs 40 --num-cls-epochs=2
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_2nd-nl4.m
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv
|
||||
Tokenized data loaded, lm.trn 13500, lm.val 1500
|
||||
Tokenized data loaded, cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁(']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/../itos')]
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 1.208774 0.984775 0.766183
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 1.198147 0.984786 0.766730
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_2nd-nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.735084 0.613895 0.803000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.550159 0.406097 0.867000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.468788 0.404081 0.862000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.395969 0.380797 0.870000
|
||||
2 0.349470 0.386497 0.866000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_2nd-nl4.m
|
||||
Loss and accuracy using (cls_best): [0.32550755, tensor(0.8857)]
|
||||
0.3255075514316559
|
||||
0.8857499957084656
|
||||
|
||||
```
|
||||
```bash
|
||||
python -m ulmfit cls --dataset-path data/mldoc/ja-1 --base-lm-path data/mldoc/ja-1/models/bsp30k/lstm_nl4.m --lang=ja --name '3nd-nl4' --cuda-id=0 - train 0 --bs 40 --num-cls-epochs=2
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_3nd-nl4.m
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv
|
||||
Tokenized data loaded, lm.trn 13500, lm.val 1500
|
||||
Tokenized data loaded, cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁(']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_3nd-nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.777661 0.617013 0.786000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.603897 0.388985 0.867000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.510845 0.374942 0.874000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.468642 0.379503 0.872000
|
||||
2 0.430415 0.365797 0.880000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_3nd-nl4.m
|
||||
Loss and accuracy using (cls_best): [0.33084384, tensor(0.8882)]
|
||||
0.33084383606910706
|
||||
0.8882499933242798
|
||||
|
||||
python -m ulmfit cls --dataset-path data/mldoc/ja-1 --base-lm-path data/mldoc/ja-1/models/bsp30k/lstm_nl4.m --lang=ja --tokenizer=bsp --name '3nd-nl4' --cuda-id=0 - train 0 --bs 40 --num-cls-epochs=2
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_3nd-nl4.m
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv
|
||||
Tokenized data loaded, lm.trn 13500, lm.val 1500
|
||||
Tokenized data loaded, cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁(']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_3nd-nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.824216 0.604706 0.825000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.606317 0.409647 0.854000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.500782 0.381826 0.862000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.403516 0.366863 0.866000
|
||||
2 0.394599 0.357580 0.874000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_3nd-nl4.m
|
||||
Loss and accuracy using (cls_best): [0.32903105, tensor(0.8848)]
|
||||
0.3290310502052307
|
||||
0.8847500085830688
|
||||
|
||||
python -m ulmfit cls --dataset-path data/mldoc/ja-1 --base-lm-path data/mldoc/ja-1/models/bsp30k/lstm_nl4.m --lang=ja --tokenizer=bsp --name '3nd-nl4' --cuda-id=0 - train 0 --bs 40 --num-cls-epochs=2
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_3nd-nl4.m
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv
|
||||
Tokenized data loaded, lm.trn 13500, lm.val 1500
|
||||
Tokenized data loaded, cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁(']
|
||||
Loading last classifier
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.460803 0.451998 0.855000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.460069 0.421900 0.867000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.361791 0.447982 0.859000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.301233 0.404477 0.868000
|
||||
2 0.269350 0.406427 0.870000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/bsp30k/lstm_3nd-nl4.m
|
||||
Loss and accuracy using (cls_best): [0.34159982, tensor(0.8925)]
|
||||
0.34159982204437256
|
||||
0.8924999833106995
|
||||
|
||||
```
|
||||
## SP60k
|
||||
```
|
||||
python -m ulmfit lm --dataset-path data/wiki/ja-100 --cuda-id=1 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 60000 \
|
||||
--lang ja --qrnn=False - train 10 --bs=50 --drop_mult=0
|
||||
|
||||
Running tokenization
|
||||
Wiki text was split to 98375 articles
|
||||
Wiki text was split to 138 articles
|
||||
Size of vocabulary: 60000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁(']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': None, 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Training lm from random weights
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 3.557822 3.682454 0.366108
|
||||
2 3.377493 3.614226 0.369889
|
||||
3 3.391634 3.562171 0.377114
|
||||
4 3.328160 3.497388 0.385236
|
||||
5 3.290285 3.424971 0.394655
|
||||
6 3.159867 3.337317 0.407095
|
||||
7 3.139091 3.250999 0.417750
|
||||
8 3.103923 3.153146 0.433443
|
||||
9 2.979789 3.092179 0.443405
|
||||
10 2.984099 3.077171 0.446887
|
||||
data/wiki/ja-100/models/sp60k
|
||||
Saving info data/wiki/ja-100/models/sp60k/lstm_nl4.m/info.json
|
||||
```
|
||||
## MLDoc
|
||||
````bash
|
||||
python -m ulmfit cls --dataset-path data/mldoc/ja-1 --base-lm-path data/wiki/ja-100/models/sp60k/lstm_nl4.m --lang=ja --name 'nl4' --cuda-id=1 - train 20 --bs 40 --num-cls-epochs=2
|
||||
Max vocab: 60000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp60k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp60k/lstm_nl4.m
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv
|
||||
Tokenized data loaded, lm.trn 13500, lm.val 1500
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 60000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁(']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp60k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp60k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp60k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp60k/lstm_nl4.m/../itos')]
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 3.019972 2.548868 0.503754
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 2.643698 2.363415 0.532341
|
||||
2 2.403588 2.149524 0.567359
|
||||
3 2.218298 1.969651 0.597484
|
||||
4 2.059648 1.829897 0.619758
|
||||
5 1.941803 1.722339 0.636215
|
||||
6 1.862969 1.630191 0.650293
|
||||
7 1.796515 1.551929 0.663782
|
||||
8 1.727768 1.481659 0.675489
|
||||
9 1.667709 1.417764 0.687287
|
||||
10 1.606343 1.357994 0.697264
|
||||
11 1.553344 1.303901 0.707811
|
||||
12 1.539182 1.251784 0.718038
|
||||
Traceback (most recent call last):
|
||||
````
|
||||
@@ -0,0 +1,223 @@
|
||||
# MLDoc
|
||||
|
||||
## QRNN 15k
|
||||
|
||||
Exec 1
|
||||
```
|
||||
python -m ulmfit eval --glob="mldoc/zh-1/models/sp15k/qrnn_nl4.m" --name nl4-1cyc-sl --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.1
|
||||
Processing data/mldoc/zh-1/models/sp15k/qrnn_nl4.m
|
||||
zh-1
|
||||
Max vocab: 15000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv
|
||||
Data lm, trn: 13500, val: 1500
|
||||
Data cls, trn: 1000, val: 1000
|
||||
Data tst, trn: 1000, val: 4000
|
||||
Size of vocabulary: 15000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning:
|
||||
Loading pretrained model
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl.m/info.json
|
||||
Single training schedule
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.859153 0.767961 0.864000
|
||||
2 0.768888 0.775161 0.904000
|
||||
3 0.658956 0.685653 0.902000
|
||||
4 0.589073 0.618438 0.923000
|
||||
5 0.540008 0.622157 0.915000
|
||||
6 0.508080 0.606979 0.914000
|
||||
7 0.487228 0.599491 0.918000
|
||||
8 0.477516 0.602196 0.923000
|
||||
Total time: 02:18
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl.m
|
||||
Loss and accuracy using (cls_best): [0.2829206, tensor(0.9205)]
|
||||
OrderedDict([('data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl.m',
|
||||
0.9204999804496765)])
|
||||
data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl.m: 0.9204999804496765
|
||||
```
|
||||
Exec 2
|
||||
````python -m ulmfit eval --glob="mldoc/zh-1/models/sp15k/qrnn_nl4.m" --name nl4-1cyc-sl1 --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.1
|
||||
Processing data/mldoc/zh-1/models/sp15k/qrnn_nl4.m
|
||||
zh-1
|
||||
Max vocab: 15000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl1.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv
|
||||
Data lm, trn: 13500, val: 1500
|
||||
Data cls, trn: 1000, val: 1000
|
||||
Data tst, trn: 1000, val: 4000
|
||||
Size of vocabulary: 15000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Loading pretrained model
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl1.m/info.json
|
||||
Single training schedule
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.881513 0.712176 0.865000
|
||||
2 0.743687 0.665091 0.906000
|
||||
3 0.677436 0.687689 0.873000
|
||||
4 0.595139 0.626483 0.920000
|
||||
5 0.542732 0.600652 0.914000
|
||||
6 0.512080 0.597546 0.916000
|
||||
7 0.487021 0.597065 0.912000
|
||||
8 0.476598 0.596792 0.914000
|
||||
Total time: 02:20
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl1.m
|
||||
Loss and accuracy using (cls_best): [0.29172945, tensor(0.9178)]
|
||||
OrderedDict([('data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl1.m',
|
||||
0.9177500009536743)])
|
||||
data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl1.m: 0.9177500009536743
|
||||
````
|
||||
Exec 4
|
||||
```bash
|
||||
python -m ulmfit eval --glob="mldoc/zh-1/models/sp15k/qrnn_nl4.m" --name nl4-1cyc-sl-e4 --num-cls-epochs=4 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.1 ✘ 130
|
||||
Processing data/mldoc/zh-1/models/sp15k/qrnn_nl4.m
|
||||
zh-1
|
||||
Max vocab: 15000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv
|
||||
Data lm, trn: 13500, val: 1500
|
||||
Data cls, trn: 1000, val: 1000
|
||||
Data tst, trn: 1000, val: 4000
|
||||
Size of vocabulary: 15000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Loading pretrained model
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m/info.json
|
||||
Single training schedule
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.880415 0.677291 0.901000
|
||||
2 0.729670 0.659975 0.911000
|
||||
3 0.624817 0.603056 0.921000
|
||||
4 0.542027 0.601961 0.921000
|
||||
Total time: 01:08
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m
|
||||
Loss and accuracy using (cls_best): [0.28558904, tensor(0.9222)]
|
||||
OrderedDict([('data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m',
|
||||
0.922249972820282)])
|
||||
data/mldoc/zh-1/models/sp15k/qrnn_nl4-1cyc-sl-e4.m: 0.922249972820282
|
||||
```
|
||||
## LSTM sp30k
|
||||
### 0.1
|
||||
```bash
|
||||
python -m ulmfit eval --glob="mldoc/zh-1/models/sp30k/lstm_nl4.m" --name nl4-1cyc-sl --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.1
|
||||
Processing data/mldoc/zh-1/models/sp30k/lstm_nl4.m
|
||||
zh-1
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv
|
||||
Data lm, trn: 13500, val: 1500
|
||||
Data cls, trn: 1000, val: 1000
|
||||
Data tst, trn: 1000, val: 4000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Loading pretrained model
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl.m/info.json
|
||||
Single training schedule
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.870432 0.670671 0.882000
|
||||
2 0.754248 0.824157 0.895000
|
||||
3 0.654601 0.727428 0.885000
|
||||
4 0.602772 0.668668 0.901000
|
||||
5 0.542110 0.625137 0.903000
|
||||
6 0.506150 0.617842 0.913000
|
||||
7 0.480944 0.616885 0.912000
|
||||
8 0.472876 0.614381 0.911000
|
||||
Total time: 06:38
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl.m
|
||||
Loss and accuracy using (cls_best): [0.2977172, tensor(0.9233)]
|
||||
OrderedDict([('data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl.m',
|
||||
0.9232500195503235)])
|
||||
data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl.m: 0.9232500195503235
|
||||
```
|
||||
### 0.2
|
||||
```bash
|
||||
python -m ulmfit eval --glob="mldoc/zh-1/models/sp30k/lstm_nl4.m" --name nl4-1cyc-sl2 --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.2
|
||||
Processing data/mldoc/zh-1/models/sp30k/lstm_nl4.m
|
||||
zh-1
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl2.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv
|
||||
Data lm, trn: 13500, val: 1500
|
||||
Data cls, trn: 1000, val: 1000
|
||||
Data tst, trn: 1000, val: 4000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Loading pretrained model
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl2.m/info.json
|
||||
Single training schedule
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 1.045619 0.908213 0.874000
|
||||
2 0.957379 0.857977 0.921000
|
||||
3 0.891791 0.852157 0.905000
|
||||
4 0.845289 0.849923 0.914000
|
||||
5 0.818228 0.848613 0.921000
|
||||
6 0.787021 0.840483 0.920000
|
||||
7 0.776123 0.844006 0.919000
|
||||
8 0.762384 0.857240 0.916000
|
||||
Total time: 06:33
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl2.m
|
||||
Loss and accuracy using (cls_best): [0.40299156, tensor(0.9170)]
|
||||
OrderedDict([('data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl2.m',
|
||||
0.9169999957084656)])
|
||||
data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl2.m: 0.9169999957084656
|
||||
```
|
||||
### 0.4
|
||||
```bash
|
||||
python -m ulmfit eval --glob="mldoc/zh-1/models/sp30k/lstm_nl4.m" --name nl4-1cyc-sl4 --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.4
|
||||
Processing data/mldoc/zh-1/models/sp30k/lstm_nl4.m
|
||||
zh-1
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl4.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv
|
||||
Data lm, trn: 13500, val: 1500
|
||||
Data cls, trn: 1000, val: 1000
|
||||
Data tst, trn: 1000, val: 4000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Loading pretrained model
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl4.m/info.json
|
||||
Single training schedule
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 1.251581 1.183341 0.898000
|
||||
2 1.214358 1.201266 0.834000
|
||||
3 1.190343 1.165525 0.919000
|
||||
4 1.168018 1.172510 0.903000
|
||||
5 1.149965 1.161660 0.914000
|
||||
6 1.140140 1.161689 0.915000
|
||||
7 1.135877 1.159853 0.912000
|
||||
8 1.134425 1.160039 0.911000
|
||||
Total time: 06:34
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl4.m
|
||||
Loss and accuracy using (cls_best): [0.64041936, tensor(0.9195)]
|
||||
OrderedDict([('data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl4.m',
|
||||
0.9194999933242798)])
|
||||
data/mldoc/zh-1/models/sp30k/lstm_nl4-1cyc-sl4.m: 0.9194999933242798
|
||||
```
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,4 +1,37 @@
|
||||
# QRNN EN
|
||||
|
||||
## SP15k nl 4
|
||||
```
|
||||
python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --m
|
||||
ax-vocab 15000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0
|
||||
Max vocab: 15000
|
||||
Cache dir: data/wiki/en-100/models/sp15k
|
||||
Model dir: data/wiki/en-100/models/sp15k/qrnn_nl4.m
|
||||
Wiki text was split to 28476 articles
|
||||
Wiki text was split to 60 articles
|
||||
Data lm, trn: 28476, val: 60
|
||||
Size of vocabulary: 15000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁the', '▁,', 's', '▁.', '▁of', '▁and', '▁in', '▁to', '▁a', 'ed']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Training lm from random weights
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 3.080874 3.197244 0.431796
|
||||
2 3.021043 3.147150 0.433593
|
||||
3 2.933366 3.125982 0.435766
|
||||
4 2.905764 3.103272 0.437356
|
||||
5 2.867981 3.032923 0.445030
|
||||
6 2.815294 2.958662 0.453979
|
||||
7 2.733671 2.869483 0.466015
|
||||
8 2.744779 2.785220 0.475833
|
||||
9 2.717722 2.704370 0.487687
|
||||
10 2.666089 2.675301 0.493602
|
||||
Total time: 9:07:27
|
||||
data/wiki/en-100/models/sp15k
|
||||
Saving info data/wiki/en-100/models/sp15k/qrnn_nl4.m/info.json
|
||||
```
|
||||
|
||||
|
||||
|
||||
## SP30k nl 4
|
||||
### LM
|
||||
|
||||
|
||||
@@ -1,5 +1,44 @@
|
||||
# QRNN ES
|
||||
|
||||
## SP15k nl 4
|
||||
``
|
||||
export CUDA_VISIBLE_DEVICES=1
|
||||
LANG=es
|
||||
python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0
|
||||
|
||||
Wiki text was split to 161509 articles
|
||||
Wiki text was split to 78 articles
|
||||
Running tokenization lm...
|
||||
Data lm, trn: 161509, val: 78
|
||||
Size of vocabulary: 15000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁.', '▁la', 's', '▁el', '▁en', '▁y', '▁a', "▁&'"]
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning:
|
||||
|
||||
Training lm from random weights
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 2.851575 3.398695 0.372940
|
||||
2 2.801543 3.353015 0.372648
|
||||
3 2.807216 3.290132 0.380787
|
||||
4 2.696361 3.220115 0.388937
|
||||
5 2.668488 3.132770 0.399528
|
||||
6 2.565685 3.062742 0.408880
|
||||
7 2.503054 2.985069 0.419262
|
||||
8 2.448338 2.895266 0.431797
|
||||
9 2.411213 2.829787 0.441973
|
||||
10 2.403536 2.811063 0.445468
|
||||
Total time: 11:52:32
|
||||
data/wiki/es-100/models/sp15k
|
||||
Saving info data/wiki/es-100/models/sp15k/qrnn_nl4.m/info.json
|
||||
``
|
||||
|
||||
```bash
|
||||
export CUDA_VISIBLE_DEVICES=1
|
||||
LANG=es
|
||||
python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/sp15k/qrnn_nl4.m --lang=${LANG} --name 'nl4' - train 20 --bs 20 --num-cls-epochs=8
|
||||
```
|
||||
|
||||
|
||||
## SP30k nl 4
|
||||
### LM
|
||||
```
|
||||
|
||||
@@ -1,4 +1,16 @@
|
||||
# QRNN RU
|
||||
## SP15k nl8
|
||||
data/wiki/ru-100/models/sp15k/qrnn_nl8.m
|
||||
|
||||
export CUDA_VISIBLE_DEVICES=0
|
||||
LANG=ru
|
||||
python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/sp15k/qrnn_nl8.m --lang=${LANG} --name 'nl8' - train 20 --bs 20 --num-cls-epochs=8 --lr_sched=1cycle
|
||||
|
||||
export CUDA_VISIBLE_DEVICES=0
|
||||
LANG=de
|
||||
python -m ulmfit cls --dataset-path data/mldoc-m/${LANG}-1 --base-lm-path data/wiki-m/${LANG}-100/models/sp15k/lstm_nl4.m --lang=${LANG} --name 'nl4' - train 20 --bs 20 --num-cls-epochs=8 --lr_sched=1cycle
|
||||
|
||||
|
||||
## SP30k nl4
|
||||
### LM
|
||||
```
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
|
||||
#
|
||||
```
|
||||
export CUDA_VISIBLE_DEVICES=0
|
||||
LANG=zh
|
||||
python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0
|
||||
|
||||
Wiki text was split to 103929 articles
|
||||
Wiki text was split to 113 articles
|
||||
Running tokenization lm...
|
||||
Data lm, trn: 103929, val: 113
|
||||
Size of vocabulary: 15000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning:
|
||||
|
||||
!! WARNING !!
|
||||
|
||||
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
Your compiler (c++) may be ABI-incompatible with PyTorch!
|
||||
Please use a compiler that is ABI-compatible with GCC 4.9 and above.
|
||||
See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html.
|
||||
|
||||
See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6
|
||||
for instructions on how to install GCC 4.9 or higher.
|
||||
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
|
||||
!! WARNING !!
|
||||
|
||||
warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler))
|
||||
Training lm from random weights
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 2.489521 2.734049 0.482433
|
||||
2 2.427567 2.662464 0.488089
|
||||
3 2.415744 2.613971 0.494118
|
||||
4 2.334062 2.560180 0.501209
|
||||
5 2.343723 2.503271 0.507307
|
||||
6 2.260171 2.444533 0.516768
|
||||
7 2.198721 2.367407 0.526631
|
||||
8 2.161857 2.308182 0.535856
|
||||
9 2.142125 2.252678 0.544535
|
||||
10 2.087831 2.234440 0.548529
|
||||
Total time: 11:01:47
|
||||
data/wiki/zh-100/models/sp15k
|
||||
Saving info data/wiki/zh-100/models/sp15k/qrnn_nl4.m/info.json
|
||||
```
|
||||
|
||||
## MLDoc
|
||||
```bash
|
||||
export CUDA_VISIBLE_DEVICES=0
|
||||
LANG=zh
|
||||
python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/sp15k/qrnn_nl4.m --lang=${LANG} --name 'nl4' - train 20 --bs 20 --num-cls-epochs=8
|
||||
|
||||
Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/zh-100/models/sp15k/qrnn_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/zh-100/models/sp15k/qrnn_nl4.m/../itos')]
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 2.723684 2.148748 0.571206
|
||||
Total time: 02:13
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 2.157829 1.937637 0.601026
|
||||
2 1.898958 1.712967 0.637379
|
||||
3 1.722818 1.547745 0.664276
|
||||
4 1.570266 1.427551 0.682546
|
||||
5 1.503477 1.344690 0.696379
|
||||
6 1.434701 1.289549 0.704813
|
||||
7 1.425267 1.217570 0.717714
|
||||
8 1.373606 1.174655 0.725217
|
||||
9 1.297397 1.116406 0.735997
|
||||
10 1.211259 1.062999 0.745848
|
||||
11 1.248108 1.024482 0.754134
|
||||
12 1.198918 0.980273 0.762664
|
||||
13 1.121848 0.937985 0.771961
|
||||
14 1.111386 0.898821 0.780796
|
||||
15 1.120596 0.866009 0.787908
|
||||
16 1.056925 0.836998 0.794833
|
||||
17 1.020636 0.816387 0.799694
|
||||
18 1.002068 0.802623 0.802859
|
||||
19 0.998480 0.796877 0.804212
|
||||
20 0.959919 0.794685 0.804594
|
||||
Total time: 1:02:57
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.666322 0.433893 0.855000
|
||||
Total time: 00:08
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.448371 0.317440 0.889000
|
||||
Total time: 00:09
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.336693 0.309876 0.900000
|
||||
Total time: 00:10
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.266735 0.302003 0.903000
|
||||
2 0.222821 0.294501 0.905000
|
||||
3 0.207295 0.293751 0.908000
|
||||
4 0.179668 0.296945 0.911000
|
||||
5 0.153803 0.293158 0.911000
|
||||
Traceback (most recent call last):
|
||||
File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/runpy.py", line 193, in _run_module_as_main
|
||||
"__main__", mod_spec)
|
||||
File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/runpy.py", line 85, in _run_code
|
||||
exec(code, run_globals)
|
||||
File "/home/pczapla/workspace/ulmfit-multilingual/ulmfit/__main__.py", line 73, in <module>
|
||||
fire.Fire(ULMFiT())
|
||||
File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/fire/core.py", line 127, in Fire
|
||||
component_trace = _Fire(component, args, context, name)
|
||||
File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/fire/core.py", line 366, in _Fire
|
||||
component, remaining_args)
|
||||
File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/fire/core.py", line 542, in _CallCallable
|
||||
result = fn(*varargs, **kwargs)
|
||||
File "/home/pczapla/workspace/ulmfit-multilingual/ulmfit/train_clas.py", line 54, in train_cls
|
||||
learn.fit_one_cycle(num_cls_epochs, slice(1e-3 / (2.6 ** 4), 1e-3), moms=(0.8, 0.7))
|
||||
File "/home/pczapla/workspace/_oss/fastai/fastai/fastai/train.py", line 22, in fit_one_cycle
|
||||
learn.fit(cyc_len, max_lr, wd=wd, callbacks=callbacks)
|
||||
File "/home/pczapla/workspace/_oss/fastai/fastai/fastai/basic_train.py", line 178, in fit
|
||||
callbacks=self.callbacks+callbacks)
|
||||
File "/home/pczapla/workspace/_oss/fastai/fastai/fastai/utils/mem.py", line 77, in wrapper
|
||||
return func(*args, **kwargs)
|
||||
File "/home/pczapla/workspace/_oss/fastai/fastai/fastai/basic_train.py", line 90, in fit
|
||||
loss = loss_batch(model, xb, yb, loss_func, opt, cb_handler)
|
||||
File "/home/pczapla/workspace/_oss/fastai/fastai/fastai/basic_train.py", line 20, in loss_batch
|
||||
out = model(*xb)
|
||||
File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/nn/modules/module.py", line 477, in __call__
|
||||
result = self.forward(*input, **kwargs)
|
||||
File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/nn/modules/container.py", line 92, in forward
|
||||
input = module(input)
|
||||
File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/nn/modules/module.py", line 477, in __call__
|
||||
result = self.forward(*input, **kwargs)
|
||||
File "/home/pczapla/workspace/_oss/fastai/fastai/fastai/text/learner.py", line 235, in forward
|
||||
return self.concat(raw_outputs), self.concat(outputs)
|
||||
File "/home/pczapla/workspace/_oss/fastai/fastai/fastai/text/learner.py", line 221, in concat
|
||||
return [torch.cat([l[si] for l in arrs], dim=1) for si in range_of(arrs[0])]
|
||||
File "/home/pczapla/workspace/_oss/fastai/fastai/fastai/text/learner.py", line 221, in <listcomp>
|
||||
return [torch.cat([l[si] for l in arrs], dim=1) for si in range_of(arrs[0])]
|
||||
RuntimeError: CUDA error: out of memory
|
||||
```
|
||||
+86
-1
@@ -1,4 +1,24 @@
|
||||
# RU
|
||||
## SP15k nl4
|
||||
```
|
||||
Training lm from random weights
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 3.053061 3.070487 0.450466
|
||||
2 2.874137 2.999093 0.455027
|
||||
3 2.864496 2.969308 0.458116
|
||||
4 2.890568 2.903564 0.466970
|
||||
5 2.746530 2.839789 0.474205
|
||||
6 2.683900 2.750476 0.486806
|
||||
7 2.674458 2.658535 0.499701
|
||||
8 2.595780 2.573735 0.512515
|
||||
9 2.530827 2.512999 0.522372
|
||||
10 2.505664 2.491850 0.526431
|
||||
Total time: 10:43:03
|
||||
data/wiki/ru-100/models/sp15k
|
||||
Saving info data/wiki/ru-100/models/sp15k/qrnn_nl4.m/info.json
|
||||
```
|
||||
|
||||
|
||||
## SP30k nl4
|
||||
### LM
|
||||
```
|
||||
@@ -21,7 +41,7 @@ epoch train_loss valid_loss accuracy
|
||||
data/wiki/ru-100/models/sp30k
|
||||
Saving info data/wiki/ru-100/models/sp30k/lstm_nl4.m/info.json
|
||||
```
|
||||
### MLDoc
|
||||
### MLDoc - bsp
|
||||
MultiCCA: 85.65% ulmfit: 87.27%
|
||||
```
|
||||
python -m ulmfit cls --dataset-path data/mldoc/ru-1 --base-lm-path data/wiki/ru-100/models/sp30k/lstm_nl4.m --lang=ru --name 'nl4-100' --cuda-id=1 - train 20 --bs 40 --num-cls-epochs=2
|
||||
@@ -75,4 +95,69 @@ epoch train_loss valid_loss accuracy
|
||||
2 0.417901 0.369961 0.882000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.38499942, tensor(0.8727)]
|
||||
```
|
||||
|
||||
### MLDoc run 2x sp
|
||||
```
|
||||
python -m ulmfit cls --dataset-path data/mldoc/ru-1 --base-lm-path data/wiki/ru-100/models/sp30k/lstm_nl4.m --lang=ru --name 'nl4' --cuda-id=1 - train 20 --bs 40 --num-cls-epochs=2
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 9195, cls.val 1021
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁в', 'а', '▁и', 'е', 'и', 'й', '▁на', 'х']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp30k/lstm_nl4.m/../itos')]
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 2.662225 2.284158 0.552927
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 2.436114 2.151187 0.574219
|
||||
2 2.260576 2.012279 0.595820
|
||||
3 2.067110 1.862512 0.620246
|
||||
4 2.000703 1.729883 0.641713
|
||||
5 1.860899 1.609955 0.661346
|
||||
6 1.751010 1.522195 0.676297
|
||||
7 1.705993 1.420628 0.694044
|
||||
8 1.592143 1.338552 0.708978
|
||||
9 1.524927 1.270614 0.722596
|
||||
10 1.475408 1.198585 0.736638
|
||||
11 1.438226 1.134858 0.749314
|
||||
12 1.408821 1.076875 0.761448
|
||||
13 1.345137 1.020660 0.773432
|
||||
14 1.321399 0.978076 0.783070
|
||||
15 1.235357 0.936674 0.791642
|
||||
16 1.204204 0.906822 0.798548
|
||||
17 1.198709 0.884949 0.803528
|
||||
18 1.176732 0.874523 0.805585
|
||||
19 1.111195 0.871806 0.806239
|
||||
20 1.031497 0.869280 0.806826
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.834704 0.615589 0.786000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.679823 0.418461 0.851000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.555612 0.426877 0.861000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.468084 0.391777 0.873000
|
||||
2 0.434714 0.388670 0.882000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.3987146, tensor(0.8680)]
|
||||
0.3987146019935608
|
||||
0.8679999709129333
|
||||
```
|
||||
|
||||
```
|
||||
Second execution
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 2.749340 2.284773 0.552775
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 2.418463 2.157943 0.572302
|
||||
```
|
||||
@@ -1,5 +1,7 @@
|
||||
# ZH
|
||||
|
||||
## SP15k QRNN
|
||||
|
||||
## SP30k LSTM nl 4
|
||||
### LM
|
||||
```
|
||||
@@ -79,6 +81,8 @@ Loss and accuracy using (cls_best): [0.28411642, tensor(0.9020)]
|
||||
0.9020000100135803
|
||||
```
|
||||
|
||||
|
||||
|
||||
## SP60k LSTM nl 4
|
||||
### LM
|
||||
```
|
||||
|
||||
@@ -39,7 +39,8 @@ def get_test_data():
|
||||
copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.valid.tokens', n=600*sz)
|
||||
copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.test.tokens', n=600*sz)
|
||||
copy_head(imdb / 'train.csv', test_imdb / 'train.csv', n=10*sz)
|
||||
copy_head(imdb / 'train.csv', test_imdb / 'test.csv', n=6*sz)
|
||||
copy_head(imdb / 'train.csv', test_imdb / 'test.csv', n=6 * sz)
|
||||
copy_head(imdb / 'train.csv', test_imdb / 'dev.csv', n=6 * sz)
|
||||
copy_head(imdb / 'train.csv', test_imdb / 'unsup.csv', n=1*sz)
|
||||
|
||||
return test_data, test_wt
|
||||
@@ -109,12 +110,33 @@ def test_ulmfit_fastai_end_to_end():
|
||||
qrnn=False,
|
||||
tokenizer='f',
|
||||
max_vocab=100,
|
||||
nl=1,
|
||||
name=lm_name,
|
||||
)
|
||||
exp.train_lm(num_epochs=1, bs=2)
|
||||
exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir)
|
||||
exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, )
|
||||
|
||||
def test_ulmfit_fastai_end_to_end_label_smoothing():
|
||||
""" Test ulmfit with sentencepiece tokenizer on small wikipedia dataset.
|
||||
"""
|
||||
test_data, wt2 = get_test_data()
|
||||
lm_name = 'end-to-end-test-fastai'
|
||||
|
||||
exp = ulmfit.pretrain_lm.LMHyperParams(
|
||||
dataset_path=wt2,
|
||||
lang='en',
|
||||
cuda_id=cuda_id,
|
||||
qrnn=False,
|
||||
tokenizer='f',
|
||||
max_vocab=100,
|
||||
name=lm_name,
|
||||
)
|
||||
exp.train_lm(num_epochs=1, bs=2, label_smoothing_eps=0.1)
|
||||
exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir)
|
||||
exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, label_smoothing_eps=0.1 )
|
||||
|
||||
|
||||
def test_ulmfit_fastai_bidir_end_to_end():
|
||||
""" Test ulmfit with sentencepiece tokenizer on small wikipedia dataset.
|
||||
"""
|
||||
|
||||
+57
-11
@@ -1,10 +1,11 @@
|
||||
import gc
|
||||
import os
|
||||
import pprint
|
||||
import tarfile
|
||||
import shutil
|
||||
from collections import OrderedDict
|
||||
from functools import wraps
|
||||
|
||||
import pandas as pd
|
||||
import fire
|
||||
from .pretrain_lm import LMHyperParams
|
||||
from .train_clas import CLSHyperParams
|
||||
@@ -25,7 +26,9 @@ def get_lang_from_dataset_path(ds):
|
||||
def get_dataset_path(p, dataset_template):
|
||||
ds = [x for x in p.parents if x.name == "models"][0].parent
|
||||
lang = get_lang_from_dataset_path(ds)
|
||||
for ds_path in ds.parent.glob(Template(dataset_template).substitute(lang=lang, ds_name=ds.name)):
|
||||
pattern = Template(dataset_template).substitute(lang=lang, ds_name=ds.name)
|
||||
print(pattern)
|
||||
for ds_path in ds.parent.glob(pattern):
|
||||
yield lang, ds_path
|
||||
|
||||
class ULMFiT:
|
||||
@@ -41,22 +44,65 @@ class ULMFiT:
|
||||
params = CLSHyperParams.from_lm(dataset_path, base_lm_path, **changes)
|
||||
return FireView(train=params.train_cls, validate_cls=params.validate_cls)
|
||||
|
||||
@wraps(CLSHyperParams)
|
||||
def load_cls(self, model_path, **changes):
|
||||
params = CLSHyperParams.from_json(model_path, **changes)
|
||||
return FireView(train=params.train_cls, validate_cls=params.validate_cls)
|
||||
|
||||
|
||||
def eval_noise_resistance(self, lang="de", size=1, prefix_name="", model="sp15k/qrnn_nl4.m"):
|
||||
def first_or_default(l, default=None):
|
||||
l = list(l)
|
||||
if l:
|
||||
return l[0]
|
||||
return default
|
||||
results= []
|
||||
for noise in range(0, 80, 5):
|
||||
print("Noise: ", noise)
|
||||
d = self.eval(glob=f"mldoc/{lang}-1/models/{model}",
|
||||
name=f"nl4_{prefix_name}{noise}",
|
||||
noise=noise/100,
|
||||
dataset_template='${lang}-'+str(size),
|
||||
num_cls_epochs=8,
|
||||
bs=18,
|
||||
lr_sched="1cycle")
|
||||
val = first_or_default(d.values(), default=-1)
|
||||
results.append((noise/100, val))
|
||||
df = pd.DataFrame(results, columns=["noise", "accuracy"])
|
||||
df.to_csv(f"noise_{lang}-{size}{prefix_name}.csv")
|
||||
print(df)
|
||||
|
||||
def tar(self, model_path):
|
||||
params = CLSHyperParams.from_json(model_path)
|
||||
tar_name = f"models/{params.lang}-{params.tokenizer_prefix}-{params.model_name}.tar"
|
||||
print("Storing model in", tar_name)
|
||||
with tarfile.open(tar_name, mode="w") as tar:
|
||||
for g in map(params.model_dir.glob, ['*_last.*', 'info.json', 'info.json', '../spm.*', '../itos.*',]):
|
||||
for f in g:
|
||||
print("Adding", f, f.relative_to("data"))
|
||||
tar.add(f, f.relative_to("data"))
|
||||
|
||||
def eval(self, glob="mldoc/*-1/models/sp30k/lstm_nl4.m", dataset_template='${lang}-1', name="tmp-100", num_lm_epochs=0, cuda_id=0, **trn_params):
|
||||
results = OrderedDict()
|
||||
for base_model in sorted(Path("data").glob(glob)):
|
||||
print("Processing", base_model)
|
||||
for lang, dataset_path in sorted(get_dataset_path(base_model, dataset_template)):
|
||||
params = CLSHyperParams.from_lm(dataset_path, base_model, lang=lang, name=name, cuda_id=cuda_id)
|
||||
key = str(params.model_dir.relative_to(Path.cwd()))
|
||||
if (params.model_dir/"cls_last.pth").exists():
|
||||
print("Evaluating previously trained model")
|
||||
results[key] = params.validate_cls()[1]
|
||||
else:
|
||||
print("Training")
|
||||
results[key] = params.train_cls(num_lm_epochs=num_lm_epochs, **trn_params)[1]
|
||||
del params
|
||||
try:
|
||||
params = CLSHyperParams.from_lm(dataset_path, base_model, lang=lang, name=name, cuda_id=cuda_id)
|
||||
key = str(params.model_dir.relative_to(Path.cwd()))
|
||||
if (params.model_dir/"cls_best.pth").exists():
|
||||
print("Evaluating previously trained model")
|
||||
results[key] = params.validate_cls()[1]
|
||||
else:
|
||||
print("Training")
|
||||
results[key] = params.train_cls(num_lm_epochs=num_lm_epochs, **trn_params)[1]
|
||||
del params
|
||||
except Exception as e:
|
||||
print("Error", e)
|
||||
gc.collect()
|
||||
|
||||
pprint.pprint(results)
|
||||
return results
|
||||
|
||||
def remove_lm_saves(self):
|
||||
for lm_save in Path("data").glob("**/lm_*.pth"):
|
||||
|
||||
+26
-4
@@ -171,10 +171,10 @@ class LMHyperParams:
|
||||
with (self.model_dir / 'info.json').open("w") as fp: json.dump(vals, fp)
|
||||
print("Saving info", self.model_dir / 'info.json')
|
||||
|
||||
def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, lr=5e-3):
|
||||
def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, lr=5e-3, label_smoothing_eps=0.0):
|
||||
self.model_dir.mkdir(exist_ok=True, parents=True)
|
||||
data_lm = self.load_wiki_data(bs=bs) if data_lm is None else data_lm
|
||||
learn = self.create_lm_learner(data_lm, drop_mult=drop_mult)
|
||||
learn = self.create_lm_learner(data_lm, drop_mult=drop_mult, label_smoothing_eps=label_smoothing_eps)
|
||||
|
||||
learn.true_wd = true_wd
|
||||
if num_epochs > 0:
|
||||
@@ -204,7 +204,7 @@ class LMHyperParams:
|
||||
# do we need to return `learn'? it adds noise to Fire output
|
||||
#return learn
|
||||
|
||||
def create_lm_learner(self, data_lm, dps=None, **kwargs):
|
||||
def create_lm_learner(self, data_lm, dps=None, label_smoothing_eps=0.0, **kwargs):
|
||||
assert self.bidir == False, "bidirectional model is not yet supported"
|
||||
config = dict(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, pad_token=PAD_TOKEN_ID, qrnn=self.qrnn,
|
||||
tie_weights=True, out_bias=True)
|
||||
@@ -230,6 +230,8 @@ class LMHyperParams:
|
||||
learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/lm-history"),
|
||||
# partial(SaveModelCallback, every='improvement', name='lm') disabled due to Memory issues
|
||||
]
|
||||
if label_smoothing_eps > 0.0:
|
||||
learn.loss_func = FlattenedLoss(LabelSmoothingCrossEntropy, eps=label_smoothing_eps)
|
||||
return learn
|
||||
|
||||
def load_train_text(self):
|
||||
@@ -297,8 +299,8 @@ class LMHyperParams:
|
||||
|
||||
@classmethod
|
||||
def from_lm(cls, dataset_path, base_lm_path, **kwargs) -> 'LMHyperParams':
|
||||
base_lm_path = Path(base_lm_path).resolve()
|
||||
dataset_path = Path(dataset_path).resolve()
|
||||
base_lm_path = Path(base_lm_path).resolve()
|
||||
with open(base_lm_path/'info.json', 'r') as f: d = json.load(f)
|
||||
d['dataset_path'] = dataset_path
|
||||
d['base_lm_path'] = base_lm_path
|
||||
@@ -315,6 +317,26 @@ class LMHyperParams:
|
||||
|
||||
d.update(kwargs)
|
||||
return cls(**d)
|
||||
@classmethod
|
||||
def from_json(cls, model_path:Path, **kwargs):
|
||||
model_path = Path(model_path).resolve()
|
||||
name = re.search(r"[a-z]+_(.+).m", model_path.name).group(1)
|
||||
with open(model_path / 'info.json', 'r') as f:
|
||||
d = json.load(f)
|
||||
d.update(kwargs)
|
||||
d['name'] = name
|
||||
dataset_path = path_strip(model_path, "data", "models").parent
|
||||
d['dataset_path'] = str(dataset_path)
|
||||
d['lang'] = infer_lang_from_dataset(dataset_path.name)
|
||||
return cls(**d)
|
||||
|
||||
def infer_lang_from_dataset(name:str):
|
||||
return name.split("-")[0]
|
||||
|
||||
def path_strip(path, from_folder, to_folder):
|
||||
to_p = [p for p in path.parents if p.name == to_folder][0]
|
||||
from_p = [p for p in path.parents if p.name == from_folder][0]
|
||||
return to_p.relative_to(from_p.parent)
|
||||
|
||||
def validate_lm(self):
|
||||
if not self.exp.subword and self.exp.max_vocab is None:
|
||||
|
||||
+85
-39
@@ -26,50 +26,75 @@ class CLSHyperParams(LMHyperParams):
|
||||
@property
|
||||
def need_fine_tune_lm(self): return not (self.model_dir/f"enc_best.pth").exists()
|
||||
|
||||
def train_cls(self, num_lm_epochs, unfreeze=True, num_cls_frozen_epochs=1, bs=40, true_wd=True, drop_mul_lm=0.3, drop_mul_cls=0.5,
|
||||
use_test_for_validation=False, num_cls_epochs=2, limit=None, noise=0.0, cls_max_len=20*70):
|
||||
def lr_schedule_layered(self, learn, num_cls_epochs):
|
||||
learn.freeze_to(-1)
|
||||
learn.fit_one_cycle(1, 2e-2, moms=(0.8, 0.7))
|
||||
if num_cls_epochs > 1:
|
||||
learn.freeze_to(-2)
|
||||
learn.fit_one_cycle(1, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7))
|
||||
learn.freeze_to(-3)
|
||||
learn.fit_one_cycle(1, slice(5e-3 / (2.6 ** 4), 5e-3), moms=(0.8, 0.7))
|
||||
learn.unfreeze()
|
||||
learn.fit_one_cycle(num_cls_epochs, slice(1e-3 / (2.6 ** 4), 1e-3), moms=(0.8, 0.7))
|
||||
|
||||
def lr_schedule_2cycle(self, learn, num_cls_epochs):
|
||||
print("2cycle training schedule")
|
||||
learn.freeze_to(-1)
|
||||
learn.fit_one_cycle(1, 2e-2, moms=(0.8, 0.7))
|
||||
learn.unfreeze()
|
||||
if num_cls_epochs > 1:
|
||||
learn.fit_one_cycle(num_cls_epochs -1, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7))
|
||||
|
||||
def lr_schedule_1cycle(self, learn, num_cls_epochs):
|
||||
print("Single training schedule")
|
||||
learn.unfreeze()
|
||||
learn.fit_one_cycle(num_cls_epochs, slice(1e-2 / (2.6 ** 4), 2e-2), moms=(0.8, 0.7))
|
||||
|
||||
def lr_schedule_false_wd(self, learn, num_cls_epochs):
|
||||
learn.true_wd = False
|
||||
print("Starting classifier training")
|
||||
learn.fit_one_cycle(1, 5e-2, moms=(0.8, 0.7), wd=1e-7)
|
||||
if num_cls_epochs > 1:
|
||||
learn.freeze_to(-2)
|
||||
learn.fit_one_cycle(1, slice(5e-2 / (2.6 ** 4), 5e-2), moms=(0.8, 0.7), wd=1e-7)
|
||||
learn.freeze_to(-3)
|
||||
learn.fit_one_cycle(1, slice(5e-4 / (2.6 ** 4), 5e-4), moms=(0.8, 0.7), wd=1e-7)
|
||||
learn.unfreeze()
|
||||
if num_cls_epochs > 5:
|
||||
learn.fit_one_cycle(num_cls_epochs-4, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7), wd=1e-7)
|
||||
|
||||
def train_cls(self, num_lm_epochs, unfreeze=True, num_cls_frozen_epochs=1, bs=40, drop_mul_lm=0.3, drop_mul_cls=0.5,
|
||||
use_test_for_validation=False, num_cls_epochs=2, limit=None, noise=0.0, cls_max_len=20*70, lr_sched='layered',
|
||||
label_smoothing_eps=0.0):
|
||||
assert use_test_for_validation == False, "use_test_for_validation=True is not supported"
|
||||
self.model_dir.mkdir(exist_ok=True, parents=True)
|
||||
|
||||
if not unfreeze:
|
||||
num_cls_epochs = 1
|
||||
|
||||
data_clas, data_lm, data_tst = self.load_cls_data(bs, limit=limit, noise=noise)
|
||||
|
||||
if self.need_fine_tune_lm: self.train_lm(num_lm_epochs, data_lm=data_lm, true_wd=true_wd, drop_mult=drop_mul_lm)
|
||||
learn = self.create_cls_learner(data_clas, drop_mult=drop_mul_cls, max_len=cls_max_len)
|
||||
if self.need_fine_tune_lm: self.train_lm(num_lm_epochs, data_lm=data_lm, drop_mult=drop_mul_lm, label_smoothing_eps=label_smoothing_eps)
|
||||
learn = self.create_cls_learner(data_clas, drop_mult=drop_mul_cls, max_len=cls_max_len, label_smoothing_eps=label_smoothing_eps)
|
||||
try:
|
||||
learn.load('cls_last')
|
||||
learn.load('cls_best')
|
||||
print("Loading last classifier")
|
||||
except FileNotFoundError:
|
||||
learn.load_encoder(ENC_BEST)
|
||||
if true_wd:
|
||||
|
||||
if hasattr(self, 'lr_schedule_'+lr_sched):
|
||||
learn.true_wd = True
|
||||
print("Starting classifier training")
|
||||
learn.freeze_to(-1)
|
||||
learn.fit_one_cycle(num_cls_frozen_epochs, 2e-2, moms=(0.8, 0.7))
|
||||
if unfreeze:
|
||||
learn.freeze_to(-2)
|
||||
learn.fit_one_cycle(1, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7))
|
||||
learn.freeze_to(-3)
|
||||
learn.fit_one_cycle(1, slice(5e-3 / (2.6 ** 4), 5e-3), moms=(0.8, 0.7))
|
||||
learn.unfreeze()
|
||||
learn.fit_one_cycle(num_cls_epochs, slice(1e-3 / (2.6 ** 4), 1e-3), moms=(0.8, 0.7))
|
||||
getattr(self, 'lr_schedule_'+lr_sched)(learn, num_cls_epochs)
|
||||
else:
|
||||
learn.true_wd = False
|
||||
print("Starting classifier training")
|
||||
learn.fit_one_cycle(num_cls_frozen_epochs, 5e-2, moms=(0.8, 0.7), wd=1e-7)
|
||||
if unfreeze:
|
||||
learn.freeze_to(-2)
|
||||
learn.fit_one_cycle(1, slice(5e-2 / (2.6 ** 4), 5e-2), moms=(0.8, 0.7), wd=1e-7)
|
||||
learn.freeze_to(-3)
|
||||
learn.fit_one_cycle(1, slice(5e-4 / (2.6 ** 4), 5e-4), moms=(0.8, 0.7), wd=1e-7)
|
||||
learn.unfreeze()
|
||||
learn.fit_one_cycle(num_cls_epochs, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7), wd=1e-7)
|
||||
raise ValueError(f"Wrong lr_sched: {lr_sched}")
|
||||
|
||||
print(f"Saving models at {learn.path / learn.model_dir}")
|
||||
learn.save('cls_last', with_opt=False)
|
||||
learn.save('cls_best', with_opt=False) # we don't use early stopping for the time being
|
||||
del learn
|
||||
return self.validate_cls('cls_best', bs=bs, data_tst=data_tst, learn=None)
|
||||
|
||||
return self.validate_cls('cls_best', bs=bs, data_tst=data_tst, learn=learn)
|
||||
|
||||
def validate_cls(self, save_name='cls_last', bs=40, data_tst=None, learn=None):
|
||||
def validate_cls(self, save_name='cls_best', bs=40, data_tst=None, learn=None):
|
||||
if data_tst is None:
|
||||
_, _, data_tst = self.load_cls_data(bs)
|
||||
if learn is None:
|
||||
@@ -80,7 +105,7 @@ class CLSHyperParams(LMHyperParams):
|
||||
print(f"Loss and accuracy using ({save_name}):", results)
|
||||
return list(map(float, results))
|
||||
|
||||
def create_cls_learner(self, data_clas, dps=None, **kwargs):
|
||||
def create_cls_learner(self, data_clas, dps=None, label_smoothing_eps=0.0, **kwargs):
|
||||
assert self.bidir == False, "bidirectional model is not yet supported"
|
||||
config = dict(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, pad_token=PAD_TOKEN_ID, qrnn=self.qrnn)
|
||||
config.update(dps or self.dps)
|
||||
@@ -99,6 +124,8 @@ class CLSHyperParams(LMHyperParams):
|
||||
learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/cls-history"),
|
||||
#partial(SaveModelCallback, every='improvement', name='cls_best') disabled due to memory issues
|
||||
]
|
||||
if label_smoothing_eps > 0.0:
|
||||
learn.loss_func = FlattenedLoss(LabelSmoothingCrossEntropy, eps=label_smoothing_eps)
|
||||
return learn
|
||||
|
||||
def load_cls_data(self, bs, **kwargs):
|
||||
@@ -121,6 +148,16 @@ class CLSHyperParams(LMHyperParams):
|
||||
**kwargs)
|
||||
return self.databunches(bs, **data)
|
||||
|
||||
def merge_cols(self, df):
|
||||
if len(df.columns) <= 2:
|
||||
return df
|
||||
ndf = df[[0,1]].copy()
|
||||
for i in range(2, len(df.columns)):
|
||||
ndf[1] += ("\n" + FLD + "\n") + df[i].fillna(" ")
|
||||
|
||||
assert ndf[1].isna().sum().sum() == 0, f"You have NaN values in column(s) of your dataframe, please fix it."
|
||||
return ndf
|
||||
|
||||
def load_data(self, lang='', **kwargs):
|
||||
prefix = '' if lang == '' else lang+'.'
|
||||
trn_df = pd.read_csv(self.dataset_path / f'{prefix}train.csv', header=None)
|
||||
@@ -139,10 +176,24 @@ class CLSHyperParams(LMHyperParams):
|
||||
val_len = max(int(len(trn_df) * 0.1), 2)
|
||||
trn_len = len(trn_df) - val_len
|
||||
trn_df, val_df = trn_df[:trn_len], trn_df[trn_len:]
|
||||
|
||||
trn_df = self.merge_cols(trn_df)
|
||||
val_df = self.merge_cols(val_df)
|
||||
tst_df = self.merge_cols(tst_df)
|
||||
unsup_df = self.merge_cols(unsup_df)
|
||||
kwargs.update(dict(trn_df=trn_df, val_df=val_df, tst_df=tst_df, unsup_df=unsup_df))
|
||||
return kwargs
|
||||
|
||||
def add_noise(self, trn_df, noise):
|
||||
count = len(trn_df)
|
||||
labels = trn_df[0].unique()
|
||||
assert np.issubdtype(labels.dtype, np.integer), "noise only works on numerical numbers"
|
||||
modulo = labels.max() + 1
|
||||
idx_to_distrub = np.random.permutation(count)[:int(count * noise)]
|
||||
trn_df.loc[idx_to_distrub, [0]] = (np.random.randint(1, modulo - 1, size=len(idx_to_distrub)) +
|
||||
trn_df.loc[idx_to_distrub][0]) % modulo
|
||||
print(f"Added noise to {len(idx_to_distrub)} examples, only {(count - len(idx_to_distrub)) / count} have correct labels")
|
||||
return trn_df
|
||||
|
||||
def databunches(self, bs, trn_df, val_df, tst_df, unsup_df, add_trn_to_lm=True, use_moses=False, force=False, limit=None, noise=0.0):
|
||||
lm_trn_df = pd.concat([unsup_df, val_df, tst_df] + ([trn_df] if add_trn_to_lm else []))
|
||||
val_len = max(int(len(lm_trn_df) * 0.1), 2)
|
||||
@@ -157,14 +208,9 @@ class CLSHyperParams(LMHyperParams):
|
||||
cls_name=f'{cls_name}limit{limit}'
|
||||
|
||||
if noise > 0.0:
|
||||
count = len(trn_df)
|
||||
labels = trn_df[0].unique()
|
||||
assert np.issubdtype(labels.dtype, np.integer), "noise only works on numerical numbers"
|
||||
modulo = labels.max()+1
|
||||
idx_to_distrub = np.random.permutation(count)[:int(count * noise)]
|
||||
trn_df.loc[idx_to_distrub, [0]] = (trn_df.loc[idx_to_distrub, [0]] + 1) % modulo
|
||||
print(f"Added noise to {len(idx_to_distrub)} examples, only {(count-len(idx_to_distrub))/count} have correct labels")
|
||||
cls_name = f'{cls_name}noise{noise}'
|
||||
trn_df = self.add_noise(trn_df, noise)
|
||||
val_df = self.add_noise(val_df, noise)
|
||||
cls_name = f'{cls_name}noise{noise}tv'
|
||||
|
||||
args = self.tokenizer_to_fastai_args(sp_data_func=lambda: trn_df[1], use_moses=use_moses)
|
||||
args['text_cols'] = list(trn_df.columns.values)[1:]
|
||||
|
||||
Reference in New Issue
Block a user