mirror of
https://github.com/wassname/multifit.git
synced 2026-08-25 11:21:38 +08:00
Update to fastai v1.0.57 - use new sentence piece implementaiton & sizes of hidden layers
This commit is contained in:
Piotr Czapla
+25
-2
@@ -80,8 +80,7 @@ def get_sentencepiece(cache_dir:PathOrStr, load_text, pre_rules: ListRules=None,
|
||||
pre_rules = pre_rules if pre_rules is not None else defaults.text_pre_rules
|
||||
post_rules = post_rules if post_rules is not None else defaults.text_post_rules
|
||||
|
||||
#special_cases = defaults.text_spec_tok + ['<link>', '<user>', '<number>', '<emoji>', '</emoji>']
|
||||
special_cases = defaults.text_spec_tok + ['xxlink', 'xxuser', 'xxnumber', 'xxemoji', 'yyemoji']
|
||||
special_cases = defaults.text_spec_tok # + ['xxlink', 'xxuser', 'xxnumber', 'xxemoji', 'yyemoji']
|
||||
if not os.path.isfile(cache_dir / 'spm.model') or not os.path.isfile(cache_dir / f'itos.pkl'):
|
||||
# load the text from the train tokens file
|
||||
text = load_text()
|
||||
@@ -129,6 +128,30 @@ def get_sentencepiece(cache_dir:PathOrStr, load_text, pre_rules: ListRules=None,
|
||||
post_rules=post_rules)
|
||||
return {'tokenizer': tokenizer, 'vocab': vocab}
|
||||
|
||||
def get_sentencepiece_fastai(cache_dir: PathOrStr, pre_rules: ListRules = None,
|
||||
post_rules: ListRules = None,
|
||||
vocab_size: int = 30000, lang='en'):
|
||||
cache_dir = pathlib.Path(cache_dir)
|
||||
|
||||
sp_model = cache_dir / 'spm.model'
|
||||
if not sp_model.is_file():
|
||||
sp_model = None
|
||||
|
||||
sp_vocab = cache_dir / 'spm.vocab'
|
||||
if not sp_vocab.is_file():
|
||||
sp_vocab = None
|
||||
|
||||
processor = SPProcessor(
|
||||
pre_rules=pre_rules,
|
||||
post_rules=post_rules,
|
||||
mark_fields=True,
|
||||
vocab_sz=vocab_size,
|
||||
sp_model=sp_model,
|
||||
sp_vocab=sp_vocab,
|
||||
lang=lang,
|
||||
tmp_dir=cache_dir.absolute() # absolute make sure that dataset path is not added as prefix
|
||||
)
|
||||
return {'processor': processor}
|
||||
|
||||
def clear_cache_directory(path:PathOrStr, cache_name:str='tmp'):
|
||||
path = pathlib.Path(path)
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
```
|
||||
$ python -m ulmfit ensemble --glob="data/mldoc/*laser*" --file_template='${dataset_path}/${lang}.train.csv' --gold_labels_template='data/mldoc/${lang}-1/${lang}.train.csv' --key_template='${lang}' --out_template='data/mldoc/${key}-1-ensemble/${key}.train.csv' --exclude_re=".*([a-z][a-z])-1-laser-probs-\1.*"
|
||||
Skipping /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-probs-es1
|
||||
Skipping /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1-laser-probs-ja1
|
||||
Skipping /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-probs-fr1
|
||||
Skipping /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-probs-zh1
|
||||
Skipping /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1-laser-probs-en1
|
||||
Skipping /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-probs-it1
|
||||
Skipping /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-probs-de1
|
||||
Skipping /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-probs-ru1
|
||||
{'Key': 'ru', 'Test Accuracy': 0.682, 'on': PosixPath('data/mldoc/ru-1/ru.train.csv'), 'files_count': 7}
|
||||
{'File saved to': PosixPath('data/mldoc/ru-1-ensemble/ru.train.csv')}
|
||||
{'Key': 'en', 'Test Accuracy': 0.82, 'on': PosixPath('data/mldoc/en-1/en.train.csv'), 'files_count': 7}
|
||||
{'File saved to': PosixPath('data/mldoc/en-1-ensemble/en.train.csv')}
|
||||
{'Key': 'es', 'Test Accuracy': 0.821, 'on': PosixPath('data/mldoc/es-1/es.train.csv'), 'files_count': 7}
|
||||
{'File saved to': PosixPath('data/mldoc/es-1-ensemble/es.train.csv')}
|
||||
{'Key': 'it', 'Test Accuracy': 0.782, 'on': PosixPath('data/mldoc/it-1/it.train.csv'), 'files_count': 7}
|
||||
{'File saved to': PosixPath('data/mldoc/it-1-ensemble/it.train.csv')}
|
||||
{'Key': 'ja', 'Test Accuracy': 0.685, 'on': PosixPath('data/mldoc/ja-1/ja.train.csv'), 'files_count': 7}
|
||||
{'File saved to': PosixPath('data/mldoc/ja-1-ensemble/ja.train.csv')}
|
||||
{'Key': 'zh', 'Test Accuracy': 0.789, 'on': PosixPath('data/mldoc/zh-1/zh.train.csv'), 'files_count': 7}
|
||||
{'File saved to': PosixPath('data/mldoc/zh-1-ensemble/zh.train.csv')}
|
||||
{'Key': 'de', 'Test Accuracy': 0.905, 'on': PosixPath('data/mldoc/de-1/de.train.csv'), 'files_count': 7}
|
||||
{'File saved to': PosixPath('data/mldoc/de-1-ensemble/de.train.csv')}
|
||||
{'Key': 'fr', 'Test Accuracy': 0.86, 'on': PosixPath('data/mldoc/fr-1/fr.train.csv'), 'files_count': 7}
|
||||
{'File saved to': PosixPath('data/mldoc/fr-1-ensemble/fr.train.csv')}
|
||||
```
|
||||
|
||||
```
|
||||
|
||||
|
||||
|
||||
ln -s /home/pczapla/workspace/ulmfit-multilingual/data-archive/mldoc/de-1/models/sp15k/qrnn_nl4_0.m data-archive/mldoc/de-1/models/sp15k/qrnn_base.m
|
||||
ln -s /home/pczapla/workspace/ulmfit-multilingual/data-archive/mldoc/en-1/models/sp15k/qrnn_nl4_tls.m data-archive/mldoc/en-1/models/sp15k/qrnn_base.m
|
||||
ln -s /home/pczapla/workspace/ulmfit-multilingual/data-archive/mldoc/es-1/models/sp15k/qrnn_nl4_0.m data-archive/mldoc/es-1/models/sp15k/qrnn_base.m
|
||||
ln -s /home/pczapla/workspace/ulmfit-multilingual/data-archive/mldoc/fr-1/models/sp15k/qrnn_nl4_tls.m data-archive/mldoc/fr-1/models/sp15k/qrnn_base.m
|
||||
ln -s /home/pczapla/workspace/ulmfit-multilingual/data-archive/mldoc/it-1/models/sp15k/qrnn_nl4_tls.m data-archive/mldoc/it-1/models/sp15k/qrnn_base.m
|
||||
ln -s /home/pczapla/workspace/ulmfit-multilingual/data-archive/mldoc/ja-1/models/sp15k/qrnn_nl4_tls.m data-archive/mldoc/ja-1/models/sp15k/qrnn_base.m
|
||||
ln -s /home/pczapla/workspace/ulmfit-multilingual/data-archive/mldoc/ru-1/models/sp15k/qrnn_nl4_tls.m data-archive/mldoc/ru-1/models/sp15k/qrnn_base.m
|
||||
ln -s /home/pczapla/workspace/ulmfit-multilingual/data-archive/mldoc/zh-1/models/sp15k/qrnn_nl4_tls.m data-archive/mldoc/zh-1/models/sp15k/qrnn_base.m
|
||||
|
||||
|
||||
for a in data-archive/mldoc/*-1; do cp $a/*unsup.csv $a/*test.csv $a/*dev.csv ${a/-archive/}-ensemble; done
|
||||
python -m ulmfit ls --glob 'data-archive/mldoc/*-1/models/sp15k/qrnn_base.m' --dataset_template='data/mldoc/${lang}-ensemble'
|
||||
|
||||
|
||||
python -m ulmfit eval --glob 'data-archive/mldoc/*-1/models/sp15k/qrnn_base.m' --dataset_template='../../data/mldoc/${lang}-ensemble' --num_lm_epochs=0 --num_cls_epochs=8 --early_stopping=False --bs=20 --label-smoothing-eps=0.1 --lr_sched=1cycle --skip_on_error=False
|
||||
|
||||
```
|
||||
@@ -0,0 +1,32 @@
|
||||
```
|
||||
LANG=de
|
||||
python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='fsp' --nl 4 --name 'nl4' --max-vocab 30000 --lang ${LANG} --qrnn=True --lmseed=1 --nh=1552 - train 10 --bs=50 --drop_mult=0 --label-smoothing-eps=0.1
|
||||
Training lm
|
||||
Max vocab: 30000
|
||||
Cache dir: data/wiki/de-100/models/fsp30k
|
||||
Model dir: data/wiki/de-100/models/fsp30k/qrnn_nl4_lmseed-1.m
|
||||
Setting LM seed to 1
|
||||
Wiki text was split to 191112 articles
|
||||
Wiki text was split to 431 articles
|
||||
Data lm, trn: 191112, val: 431
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['▁xxunk', '▁xxpad', '▁xxbos', '▁xxeos', '▁xxfld', '▁xxmaj', '▁xxup', '▁xxrep', '▁xxwrep', '<unk>', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', "▁&'", 's', '-']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'input_p': 0.25, 'output_p': 0.1, 'weight_p': 0.2, 'embed_p': 0.02, 'hidden_p': 0.15}
|
||||
Bptt 70
|
||||
Training lm from random weights
|
||||
epoch train_loss valid_loss accuracy time
|
||||
0 4.403168 4.469939 0.403841 1:30:29
|
||||
1 4.312332 4.432437 0.404200 1:30:30
|
||||
2 4.334051 4.423142 0.405613 1:30:03
|
||||
3 4.239668 4.376138 0.411162 1:30:05
|
||||
4 4.193250 4.324453 0.416959 1:30:04
|
||||
5 4.151386 4.248287 0.427130 1:30:16
|
||||
6 4.103295 4.160756 0.438965 1:30:32
|
||||
7 4.033971 4.086159 0.450292 1:30:16
|
||||
8 3.967596 4.029943 0.459819 1:31:20
|
||||
9 3.954203 4.013039 0.463228 1:31:14
|
||||
Total time: 15:04:52
|
||||
data/wiki/de-100/models/fsp30k
|
||||
Saving info data/wiki/de-100/models/fsp30k/qrnn_nl4_lmseed-1.m/info.json
|
||||
```
|
||||
-------
|
||||
@@ -0,0 +1,34 @@
|
||||
```
|
||||
LANG=ja ✘ 130
|
||||
python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='fsp' --nl 4 --name 'nl4' --max-vocab 30000 --lang ${LANG} --qrnn=True --lmseed=1 --nh=1552 - train 10 --bs=50 --drop_mult=0 --label-smoothing-eps=0.1
|
||||
Training lm
|
||||
Max vocab: 30000
|
||||
Cache dir: data/wiki/ja-100/models/fsp30k
|
||||
Model dir: data/wiki/ja-100/models/fsp30k/qrnn_nl4_lmseed-1.m
|
||||
Setting LM seed to 1
|
||||
Wiki text was split to 120037 articles
|
||||
Wiki text was split to 63 articles
|
||||
Data lm, trn: 120037, val: 63
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['▁xxunk', '▁xxpad', '▁xxbos', '▁xxeos', '▁xxfld', '▁xxmaj', '▁xxup', '▁xxrep', '▁xxwrep', '<unk>', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', '▁は', '▁・', '▁(']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'input_p': 0.25, 'output_p': 0.1, 'weight_p': 0.2, 'embed_p': 0.02, 'hidden_p': 0.15}
|
||||
Bptt 70
|
||||
Training lm from random weights
|
||||
epoch train_loss valid_loss accuracy time
|
||||
0 4.346260 4.408598 0.370535 1:12:49
|
||||
1 4.253693 4.355113 0.372354 1:12:40
|
||||
2 4.190918 4.288729 0.383211 1:12:15
|
||||
3 4.148739 4.242265 0.389964 1:12:12
|
||||
4 4.136361 4.190885 0.398423 1:12:23
|
||||
5 4.051008 4.119476 0.409002 1:12:15
|
||||
6 3.966213 4.052222 0.419292 1:12:20
|
||||
7 3.928247 3.979634 0.431336 1:12:18
|
||||
8 3.840935 3.929402 0.442688 1:12:39
|
||||
9 3.909105 3.911067 0.446342 1:12:51
|
||||
Total time: 12:04:47
|
||||
data/wiki/ja-100/models/fsp30k
|
||||
Saving info data/wiki/ja-100/models/fsp30k/qrnn_nl4_lmseed-1.m/info.json
|
||||
```
|
||||
-------
|
||||
|
||||
python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/fsp30k/qrnn_nl4_lmseed-1.m --lang=${LANG} --name 'nl4' --clsweightseed=0 - train 20 --bs 20 --lr_sched=1cycle --label-smoothing-eps=0.1
|
||||
@@ -0,0 +1,69 @@
|
||||
LANG=ja
|
||||
python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 30000 --lang ${LANG} --qrnn=True --lmseed=1 - train 10 --bs=50 --drop_mult=0
|
||||
|
||||
LANG=ja
|
||||
python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang ${LANG} --qrnn=True "--tokenizer-mod=-fix" --lmseed=1 - train 10 --bs=50 --drop_mult=0
|
||||
|
||||
python -m ulmfit cls --dataset-path data/mldoc/ja-1 --base-lm-path data/wiki/ja-100/models/sp15k-fix/qrnn_nl4_lmseed-1.m --lang=ja --name 'nl4' - train 20 --bs 20 --lr_sched=1cycle --label-smoothing-eps=0.1set-path data/mldoc/es-1 --base-lm-path data/wiki/ja-100/models/sp30k-fix/qrnn_nl4.m --lang=ja --name 'nl4' - train 20 --bs 40
|
||||
|
||||
0 4.875985 3.940408 0.479504 02:40
|
||||
Total time: 02:40
|
||||
epoch train_loss valid_loss accuracy time
|
||||
0 3.814340 3.574515 0.524143 03:43
|
||||
1 3.344660 3.174727 0.591801 03:42
|
||||
2 3.062797 2.909801 0.638397 03:42
|
||||
3 2.924287 2.753593 0.662699 03:42
|
||||
4 2.832728 2.648505 0.679922 03:41
|
||||
5 2.673981 2.575237 0.692270 03:41
|
||||
6 2.727100 2.521090 0.702180 03:42
|
||||
7 2.647422 2.474463 0.710956 03:42
|
||||
8 2.557694 2.437784 0.717445 03:41
|
||||
9 2.619366 2.398306 0.725727 03:42
|
||||
10 2.501441 2.368656 0.731127 03:41
|
||||
11 2.501446 2.340000 0.737203 03:41
|
||||
12 2.539080 2.316010 0.742567 03:42
|
||||
13 2.441686 2.290955 0.748064 03:41
|
||||
14 2.406307 2.273114 0.752770 03:41
|
||||
15 2.421776 2.256135 0.756699 03:42
|
||||
16 2.399470 2.245539 0.758821 03:42
|
||||
17 2.336457 2.237878 0.760780 03:42
|
||||
18 2.383474 2.234359 0.761501 03:41
|
||||
19 2.407631 2.233689 0.761739 03:42
|
||||
Total time: 1:14:01
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k-fix
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k-fix/qrnn_nl4_lmseed-1.m/info.json
|
||||
Single training schedule
|
||||
epoch train_loss valid_loss f_beta precision recall kappa_score matthews_correff accuracy time
|
||||
/home/pczapla/workspace/_oss/fastai/fastai/fastai/metrics.py:179: UserWarning: average=`binary` was selected for a non binary case. Value for average has now been set to `macro` instead.
|
||||
def _precision(self):
|
||||
0 0.872342 0.839480 0.739856 0.836258 0.741572 0.649270 0.682293 0.737000 00:19
|
||||
Better model found at epoch 0 with f_beta value: 0.739856481552124.
|
||||
/home/pczapla/workspace/_oss/fastai/fastai/fastai/metrics.py:179: UserWarning: average=`binary` was selected for a non binary case. Value for average has now been set to `macro` instead.
|
||||
def _precision(self):
|
||||
1 0.678405 0.587759 0.886923 0.888504 0.885348 0.845242 0.846027 0.884000 00:19
|
||||
Better model found at epoch 1 with f_beta value: 0.8869231939315796.
|
||||
Total time: 00:39
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k-fix/qrnn_nl4_lmseed-1.m
|
||||
/home/pczapla/workspace/_oss/fastai/fastai/fastai/metrics.py:179: UserWarning: average=`binary` was selected for a non binary case. Value for average has now been set to `macro` instead.
|
||||
def _precision(self):
|
||||
Model: nl4
|
||||
Validation on: test
|
||||
F1 score bin: 0.9002974033355713
|
||||
Loss: 0.3307778239250183
|
||||
Precision: 0.9020541906356812
|
||||
Recall: 0.9002037048339844
|
||||
Accuracy: 0.9007499814033508
|
||||
test F1 score bin: 0.9002974033355713
|
||||
test Loss: 0.33077782
|
||||
test Precision: 0.9020541906356812
|
||||
test Recall: 0.9002037048339844
|
||||
test Kappa Linear: 0.8676621913909912
|
||||
test Matthews Correff: 0.868194043636322
|
||||
test Accuracy: 0.9007499814033508
|
||||
|
||||
|
||||
|
||||
eval --glob="mldoc/ja-1/models/sp15k/qrnn_nl4.m" --name nl4-1cyc-sl --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.1
|
||||
|
||||
|
||||
|
||||
@@ -133,4 +133,8 @@ Traceback (most recent call last):
|
||||
File "/home/pczapla/workspace/_oss/fastai/fastai/fastai/text/learner.py", line 221, in <listcomp>
|
||||
return [torch.cat([l[si] for l in arrs], dim=1) for si in range_of(arrs[0])]
|
||||
RuntimeError: CUDA error: out of memory
|
||||
```
|
||||
```
|
||||
|
||||
## Fixed sentence piece
|
||||
LANG=zh
|
||||
python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang ${LANG} --qrnn=True "--tokenizer-mod=-fix" --lmseed=1 - train 10 --bs=50 --drop_mult=0
|
||||
|
||||
+35
-52
@@ -34,10 +34,10 @@ def get_test_data():
|
||||
test_imdb.mkdir(exist_ok=True, parents=True)
|
||||
|
||||
sz=1
|
||||
# we use the same text to see if models can overfit
|
||||
copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.train.tokens', n=1000*sz)
|
||||
copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.valid.tokens', n=600*sz)
|
||||
copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.test.tokens', n=600*sz)
|
||||
# we use the same text to see if models overfits
|
||||
copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.train.tokens', n=100*sz)
|
||||
copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.valid.tokens', n=60*sz)
|
||||
copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.test.tokens', n=60*sz)
|
||||
copy_head(imdb / 'train.csv', test_imdb / 'train.csv', n=10*sz)
|
||||
copy_head(imdb / 'train.csv', test_imdb / 'test.csv', n=6 * sz)
|
||||
copy_head(imdb / 'train.csv', test_imdb / 'dev.csv', n=6 * sz)
|
||||
@@ -46,12 +46,21 @@ def get_test_data():
|
||||
return test_data, test_wt
|
||||
|
||||
|
||||
def test_evaluate():
|
||||
""" Test ulmfit with (default) Moses tokenizer on small wikipedia dataset.
|
||||
"""
|
||||
os.chdir(get_data_folder()/"..")
|
||||
fastai.core.defaults.cpus=0
|
||||
test_data, wt2 = get_test_data()
|
||||
exp = ulmfit.train_clas.CLSHyperParams(test_data / 'imdb', lang='en', qrnn=False, max_vocab=1000, name="tst")
|
||||
exp.evaluate_cls(save_name=None, bs=2)
|
||||
|
||||
|
||||
def test_ulmfit_works_with_relative_paths():
|
||||
""" Test ulmfit with (default) Moses tokenizer on small wikipedia dataset.
|
||||
"""
|
||||
os.chdir(get_data_folder()/"..")
|
||||
|
||||
|
||||
test_data, wt2 = get_test_data()
|
||||
lm_name = 'end-to-end-test-default'
|
||||
cuda_id = 0
|
||||
@@ -121,7 +130,7 @@ def test_ulmfit_fastai_end_to_end_label_smoothing():
|
||||
""" Test ulmfit with sentencepiece tokenizer on small wikipedia dataset.
|
||||
"""
|
||||
test_data, wt2 = get_test_data()
|
||||
lm_name = 'end-to-end-test-fastai'
|
||||
lm_name = 'end-to-end-test-fastai-lablel-smoothing'
|
||||
|
||||
exp = ulmfit.pretrain_lm.LMHyperParams(
|
||||
dataset_path=wt2,
|
||||
@@ -133,54 +142,9 @@ def test_ulmfit_fastai_end_to_end_label_smoothing():
|
||||
name=lm_name,
|
||||
)
|
||||
exp.train_lm(num_epochs=1, bs=2, label_smoothing_eps=0.1)
|
||||
exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir)
|
||||
exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir, name=lm_name)
|
||||
exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, label_smoothing_eps=0.1 )
|
||||
|
||||
|
||||
def test_ulmfit_fastai_bidir_end_to_end():
|
||||
""" Test ulmfit with sentencepiece tokenizer on small wikipedia dataset.
|
||||
"""
|
||||
test_data, wt2 = get_test_data()
|
||||
lm_name = 'end-to-end-test-fastai'
|
||||
|
||||
exp = ulmfit.pretrain_lm.LMHyperParams(
|
||||
dataset_path=wt2,
|
||||
lang='en',
|
||||
cuda_id=cuda_id,
|
||||
qrnn=False,
|
||||
bidir=True,
|
||||
tokenizer='f',
|
||||
max_vocab=100,
|
||||
name=lm_name,
|
||||
)
|
||||
exp.train_lm(num_epochs=1, bs=2)
|
||||
exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(str(test_data / 'imdb'), str(exp.model_dir))
|
||||
exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, )
|
||||
|
||||
def test_ulmfit_moses_fa_bidir_end_to_end():
|
||||
""" Test ulmfit with sentencepiece tokenizer on small wikipedia dataset.
|
||||
"""
|
||||
test_data, wt2 = get_test_data()
|
||||
lm_name = 'end-to-end-test-fastai'
|
||||
|
||||
exp = ulmfit.pretrain_lm.LMHyperParams(
|
||||
dataset_path=wt2,
|
||||
lang='en',
|
||||
cuda_id=cuda_id,
|
||||
qrnn=False,
|
||||
bidir=True,
|
||||
tokenizer='vf',
|
||||
max_vocab=100,
|
||||
name=lm_name,
|
||||
)
|
||||
exp.train_lm(num_epochs=1, bs=2)
|
||||
exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir)
|
||||
exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, )
|
||||
|
||||
# def test_classification_model_work_with_different_dropmul():
|
||||
# learn = self.create_cls_learner(data_clas, drop_mult=0.1)
|
||||
# learn = self.create_cls_learner(data_clas, drop_mult=0.0)
|
||||
|
||||
def test_ulmfit_sentencepiece_end_to_end():
|
||||
""" Test ulmfit with sentencepiece tokenizer on small wikipedia dataset.
|
||||
"""
|
||||
@@ -201,6 +165,25 @@ def test_ulmfit_sentencepiece_end_to_end():
|
||||
exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir)
|
||||
exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, )
|
||||
|
||||
def test_ulmfit_sentencepiece_fastai_impl_end_to_end():
|
||||
""" Test ulmfit with sentencepiece tokenizer on small wikipedia dataset.
|
||||
"""
|
||||
test_data, wt2 = get_test_data()
|
||||
lm_name = 'end-to-end-test-spm-fa'
|
||||
|
||||
exp = ulmfit.pretrain_lm.LMHyperParams(
|
||||
dataset_path=wt2,
|
||||
lang='en',
|
||||
cuda_id=cuda_id,
|
||||
qrnn=False,
|
||||
tokenizer=ulmfit.pretrain_lm.Tokenizers.FASTAI_SUBWORD,
|
||||
max_vocab=200,
|
||||
name=lm_name,
|
||||
)
|
||||
exp.train_lm(num_epochs=1, bs=2)
|
||||
# not supported yet
|
||||
exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir)
|
||||
exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, )
|
||||
|
||||
if __name__ == "__main__":
|
||||
fire.Fire() # allows using all functions via CLI
|
||||
|
||||
+4
-4
@@ -49,13 +49,13 @@ class ULMFiT:
|
||||
params = CLSHyperParams.from_lm(dataset_path, base_lm_path, **changes)
|
||||
else:
|
||||
params = CLSHyperParams(dataset_path=dataset_path, **changes)
|
||||
return FireView(train=params.train_cls, validate_cls=params.validate_cls)
|
||||
return FireView(train=params.train_cls, evaluate_cls=params.evaluate_cls)
|
||||
|
||||
|
||||
@wraps(CLSHyperParams)
|
||||
def load_cls(self, model_path, **changes):
|
||||
params = CLSHyperParams.from_json(model_path, **changes)
|
||||
return FireView(train=params.train_cls, validate_cls=params.validate_cls)
|
||||
return FireView(train=params.train_cls, evaluate_cls=params.evaluate_cls)
|
||||
|
||||
|
||||
def eval_noise_resistance(self, lang="de", size=1, prefix_name="", model="sp15k/qrnn_nl4.m",
|
||||
@@ -255,8 +255,8 @@ class ULMFiT:
|
||||
last_model_dir = params.model_dir.relative_to(data_dir.parent)
|
||||
if (params.model_dir/"cls_best.pth").exists():
|
||||
print("Evaluating previously trained model")
|
||||
d_tst = params.validate_cls(save_name=save_name, label_smoothing_eps=label_smoothing_eps, use_cache=True, mode="test")
|
||||
d_val = params.validate_cls(save_name=save_name, label_smoothing_eps=label_smoothing_eps, use_cache=True, mode="valid")
|
||||
d_tst = params.evaluate_cls(save_name=save_name, label_smoothing_eps=label_smoothing_eps, use_cache=True, mode="test")
|
||||
d_val = params.evaluate_cls(save_name=save_name, label_smoothing_eps=label_smoothing_eps, use_cache=True, mode="valid")
|
||||
d={}
|
||||
d.update(d_val)
|
||||
d.update(d_tst)
|
||||
|
||||
+51
-6
@@ -17,7 +17,7 @@ from fastai.text import *
|
||||
import torch
|
||||
from fastai_contrib.utils import read_file, read_whitespace_file, \
|
||||
validate, PAD, UNK, get_sentencepiece, read_clas_data, TRN, VAL, TST, PAD_TOKEN_ID, \
|
||||
replace_std_toks, MosesPreprocessingFunc
|
||||
replace_std_toks, MosesPreprocessingFunc, get_sentencepiece_fastai
|
||||
from fastai_contrib.learner import bilm_learner, accuracy_fwd, accuracy_bwd, bilm_text_classifier_learner
|
||||
import pickle
|
||||
|
||||
@@ -31,15 +31,17 @@ ENC_BEST = "enc_best"
|
||||
|
||||
|
||||
class Tokenizers(Enum):
|
||||
FASTAI_SUBWORD = 'fsp'
|
||||
SUBWORD='sp'
|
||||
BROKENSUBWORD = 'bsp'
|
||||
MOSES='v'
|
||||
MOSES_FA='vf'
|
||||
FASTAI='f'
|
||||
|
||||
|
||||
def istitle(line):
|
||||
return len(re.findall(r'^ ?= [^=]* = ?$', line)) != 0
|
||||
|
||||
|
||||
def read_wiki_articles(filename):
|
||||
if "reddit" in str(filename): # Temporary hack to handle poleval reddit dataset
|
||||
return pd.read_csv(filename, header=None, names=["texts"]).fillna("")
|
||||
@@ -58,6 +60,7 @@ def read_wiki_articles(filename):
|
||||
print(f"Wiki text was split to {len(articles)} articles")
|
||||
return pd.DataFrame({'texts': np.array(articles, dtype=np.object)})
|
||||
|
||||
|
||||
name_re = re.compile("(bwd)?(lstm|qrnn)_(.*)_(lmseed-)?.*\.m")
|
||||
def folder_name_to_model_name(folder_name):
|
||||
if hasattr(folder_name, 'name'):
|
||||
@@ -67,6 +70,7 @@ def folder_name_to_model_name(folder_name):
|
||||
return match.group(3)
|
||||
return None
|
||||
|
||||
|
||||
@dataclass
|
||||
class DataSetParams:
|
||||
dataset_path: str # data_dir
|
||||
@@ -85,11 +89,40 @@ class DataSetParams:
|
||||
except KeyError as e:
|
||||
raise KeyError(f"{e} , options:{repr(list(params.keys()))}")
|
||||
|
||||
# temporary loading function as from_df does not support processors
|
||||
def make_data_bunch_from_df(cls, path: PathOrStr, train_df: DataFrame, valid_df: DataFrame,
|
||||
tokenizer: Tokenizer = None, vocab: Vocab = None, classes: Collection[str] = None,
|
||||
text_cols: IntsOrStrs = 1,
|
||||
label_cols: IntsOrStrs = 0, label_delim: str = None, chunksize: int = 10000,
|
||||
max_vocab: int = 60000,
|
||||
min_freq: int = 2, mark_fields: bool = False, include_bos: bool = True,
|
||||
include_eos: bool = False, processor=None, **kwargs) -> DataBunch:
|
||||
"Create a `TextDataBunch` from DataFrames. `kwargs` are passed to the dataloader creation."
|
||||
assert processor is None or tokenizer is None, "Processor and tokenizer are mutually exclusive."
|
||||
|
||||
if processor is None:
|
||||
processor = fastai.text.data._get_processor(tokenizer=tokenizer, vocab=vocab, chunksize=chunksize, max_vocab=max_vocab,
|
||||
min_freq=min_freq, mark_fields=mark_fields,
|
||||
include_bos=include_bos, include_eos=include_eos)
|
||||
|
||||
if classes is None and is_listy(label_cols) and len(label_cols) > 1: classes = label_cols
|
||||
src = ItemLists(path, TextList.from_df(train_df, path, cols=text_cols, processor=processor),
|
||||
TextList.from_df(valid_df, path, cols=text_cols, processor=processor))
|
||||
if cls == TextLMDataBunch:
|
||||
src = src.label_for_lm()
|
||||
else:
|
||||
if label_delim is not None:
|
||||
src = src.label_from_df(cols=label_cols, classes=classes, label_delim=label_delim)
|
||||
else:
|
||||
src = src.label_from_df(cols=label_cols, classes=classes)
|
||||
return src.databunch(**kwargs)
|
||||
|
||||
|
||||
@dataclass
|
||||
class LMHyperParams(DataSetParams):
|
||||
base_lm_path: str = None
|
||||
backwards: str = False
|
||||
bidir: bool =False
|
||||
bidir: bool = False
|
||||
qrnn: bool = True
|
||||
max_vocab: int = 60000
|
||||
tokenizer: Tokenizers = Tokenizers.MOSES
|
||||
@@ -131,7 +164,7 @@ class LMHyperParams(DataSetParams):
|
||||
self.cache_dir = self.dataset_path / 'models' / self.tokenizer_prefix
|
||||
self.model_dir = self.cache_dir / self.model_name
|
||||
|
||||
if self.nh is None: self.nh = 1550 if self.qrnn else 1150
|
||||
if self.nh is None: self.nh = 1552 if self.qrnn else 1152
|
||||
if self.name is None: self.name = self.lang
|
||||
|
||||
@property
|
||||
@@ -169,7 +202,17 @@ class LMHyperParams(DataSetParams):
|
||||
|
||||
def tokenizer_to_fastai_args(self, sp_data_func, use_moses):
|
||||
moses_preproc = [MosesPreprocessingFunc(self.lang)] if use_moses else []
|
||||
if self.tokenizer is Tokenizers.SUBWORD or self.tokenizer is Tokenizers.BROKENSUBWORD:
|
||||
if self.tokenizer is Tokenizers.FASTAI_SUBWORD:
|
||||
if self.base_lm_path and not (self.cache_dir / "spm.model").exists(): # ensure we are using the same sentence piece model
|
||||
shutil.copy(self.base_lm_path / '..' / 'spm.model', self.cache_dir)
|
||||
shutil.copy(self.base_lm_path / '..' / 'spm.vocab', self.cache_dir)
|
||||
args = get_sentencepiece_fastai(
|
||||
cache_dir=self.cache_dir,
|
||||
vocab_size=self.max_vocab,
|
||||
lang=self.lang,
|
||||
pre_rules=moses_preproc + defaults.text_pre_rules)
|
||||
|
||||
elif self.tokenizer is Tokenizers.SUBWORD:
|
||||
if self.base_lm_path and not(self.cache_dir/"spm.model").exists(): # ensure we are using the same sentence piece model
|
||||
shutil.copy(self.base_lm_path / '..' / 'itos.pkl', self.cache_dir)
|
||||
shutil.copy(self.base_lm_path / '..' / 'spm.model', self.cache_dir)
|
||||
@@ -230,6 +273,7 @@ class LMHyperParams(DataSetParams):
|
||||
learn = self.create_lm_learner(data_lm, drop_mult=drop_mult, label_smoothing_eps=label_smoothing_eps)
|
||||
print("Bptt", data_lm.bptt)
|
||||
learn.true_wd = true_wd
|
||||
#learn = learn.to_fp16()
|
||||
if num_epochs > 0:
|
||||
if self.pretrained_fnames or self.pretrained_model:
|
||||
print("Training lm from: ", self.pretrained_fnames or self.pretrained_model)
|
||||
@@ -337,12 +381,13 @@ class LMHyperParams(DataSetParams):
|
||||
data = load_data(self.cache_dir, name, bs=bs)
|
||||
else:
|
||||
print(f"Running tokenization {name}...")
|
||||
data = bunch_class.from_df(path=self.cache_dir,
|
||||
data = make_data_bunch_from_df(cls=bunch_class, path=self.cache_dir,
|
||||
train_df=train_df,
|
||||
valid_df=valid_df,
|
||||
max_vocab=self.max_vocab,
|
||||
bs=bs,
|
||||
**args)
|
||||
|
||||
data.save(name)
|
||||
with open(self.cache_dir/"itos.pkl", 'wb') as f:
|
||||
pickle.dump(data.vocab.itos, f)
|
||||
|
||||
+15
-9
@@ -102,9 +102,9 @@ class CLSHyperParams(LMHyperParams):
|
||||
kappa_lin = KappaScore()
|
||||
matthews_correff = MatthewsCorreff()
|
||||
metrics = [f1_score, precision, recall, kappa_lin, matthews_correff]
|
||||
# TODO: fix this in fast.ai
|
||||
if init:
|
||||
for metric in metrics: metric.on_train_begin()
|
||||
# # TODO: fix this in fast.ai
|
||||
# if init:
|
||||
# for metric in metrics: metric.on_train_begin()
|
||||
metrics.append(accuracy)
|
||||
return metrics
|
||||
|
||||
@@ -176,11 +176,11 @@ class CLSHyperParams(LMHyperParams):
|
||||
learn.save('cls_best', with_opt=False)
|
||||
#learn.save('cls_best', with_opt=False) # we don't use early stopping for the time being
|
||||
del learn
|
||||
return self.validate_cls('cls_best', bs=bs, data_tst=data_tst, learn=None)
|
||||
return self.evaluate_cls('cls_best', bs=bs, data_tst=data_tst, learn=None)
|
||||
|
||||
def validate_cls(self, save_name='cls_best', bs=40, data_tst=None, learn=None,
|
||||
def evaluate_cls(self, save_name='cls_best', bs=40, data_tst=None, learn=None,
|
||||
dump_preds=None, mode="test", label_smoothing_eps=None, use_cache=False):
|
||||
cache_file = (self.model_dir / f'results_{mode+("" if save_name == "cls_best" else save_name)}.json')
|
||||
cache_file = (self.model_dir / f'results_{mode+("" if save_name == "cls_best" else str(save_name))}.json')
|
||||
if use_cache and cache_file.exists():
|
||||
with cache_file.open("r") as fp:
|
||||
return json.load(fp)
|
||||
@@ -191,9 +191,13 @@ class CLSHyperParams(LMHyperParams):
|
||||
else:
|
||||
dt = data_tst
|
||||
if learn is None:
|
||||
learn = self.create_cls_learner(dt, drop_mult=0.3, metrics=self.get_metrics(True))
|
||||
learn = self.create_cls_learner(dt, drop_mult=0.3, metrics=self.get_metrics(True), silent=True, early_stopping=False)
|
||||
learn.unfreeze()
|
||||
learn.load(save_name)
|
||||
if save_name is not None:
|
||||
learn.load(save_name)
|
||||
else:
|
||||
print("Using random weights!")
|
||||
|
||||
if mode == "test":
|
||||
ds = data_tst.valid_dl
|
||||
elif mode == "valid" or mode == "dev":
|
||||
@@ -211,7 +215,7 @@ class CLSHyperParams(LMHyperParams):
|
||||
np.save(self.model_dir / f"preds-on-{mode}.npy", probs.cpu().numpy())
|
||||
results = learn.validate(ds)
|
||||
print(f"Model: {self.name}")
|
||||
print(f"Validation on: {mode}")
|
||||
print(f"Evaluation on: {mode}")
|
||||
labeled_results = self.output_metrics(results, mode=mode)
|
||||
|
||||
with cache_file.open("w") as fp:
|
||||
@@ -243,6 +247,8 @@ class CLSHyperParams(LMHyperParams):
|
||||
|
||||
if label_smoothing_eps > 0.0:
|
||||
learn.loss_func = FlattenedLoss(LabelSmoothingCrossEntropy, eps=label_smoothing_eps)
|
||||
|
||||
#learn = learn.to_fp16()
|
||||
return learn
|
||||
|
||||
def load_cls_data(self, bs, **kwargs):
|
||||
|
||||
Reference in New Issue
Block a user