diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py
index 2c814ce..6583e25 100644
--- a/fastai_contrib/utils.py
+++ b/fastai_contrib/utils.py
@@ -80,8 +80,7 @@ def get_sentencepiece(cache_dir:PathOrStr, load_text, pre_rules: ListRules=None,
pre_rules = pre_rules if pre_rules is not None else defaults.text_pre_rules
post_rules = post_rules if post_rules is not None else defaults.text_post_rules
- #special_cases = defaults.text_spec_tok + ['', '', '', '', '']
- special_cases = defaults.text_spec_tok + ['xxlink', 'xxuser', 'xxnumber', 'xxemoji', 'yyemoji']
+ special_cases = defaults.text_spec_tok # + ['xxlink', 'xxuser', 'xxnumber', 'xxemoji', 'yyemoji']
if not os.path.isfile(cache_dir / 'spm.model') or not os.path.isfile(cache_dir / f'itos.pkl'):
# load the text from the train tokens file
text = load_text()
@@ -129,6 +128,30 @@ def get_sentencepiece(cache_dir:PathOrStr, load_text, pre_rules: ListRules=None,
post_rules=post_rules)
return {'tokenizer': tokenizer, 'vocab': vocab}
+def get_sentencepiece_fastai(cache_dir: PathOrStr, pre_rules: ListRules = None,
+ post_rules: ListRules = None,
+ vocab_size: int = 30000, lang='en'):
+ cache_dir = pathlib.Path(cache_dir)
+
+ sp_model = cache_dir / 'spm.model'
+ if not sp_model.is_file():
+ sp_model = None
+
+ sp_vocab = cache_dir / 'spm.vocab'
+ if not sp_vocab.is_file():
+ sp_vocab = None
+
+ processor = SPProcessor(
+ pre_rules=pre_rules,
+ post_rules=post_rules,
+ mark_fields=True,
+ vocab_sz=vocab_size,
+ sp_model=sp_model,
+ sp_vocab=sp_vocab,
+ lang=lang,
+ tmp_dir=cache_dir.absolute() # absolute make sure that dataset path is not added as prefix
+ )
+ return {'processor': processor}
def clear_cache_directory(path:PathOrStr, cache_name:str='tmp'):
path = pathlib.Path(path)
diff --git a/results/ensemble-laser.md b/results/ensemble-laser.md
new file mode 100644
index 0000000..171be0b
--- /dev/null
+++ b/results/ensemble-laser.md
@@ -0,0 +1,49 @@
+```
+$ python -m ulmfit ensemble --glob="data/mldoc/*laser*" --file_template='${dataset_path}/${lang}.train.csv' --gold_labels_template='data/mldoc/${lang}-1/${lang}.train.csv' --key_template='${lang}' --out_template='data/mldoc/${key}-1-ensemble/${key}.train.csv' --exclude_re=".*([a-z][a-z])-1-laser-probs-\1.*"
+Skipping /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-probs-es1
+Skipping /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1-laser-probs-ja1
+Skipping /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-probs-fr1
+Skipping /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-probs-zh1
+Skipping /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1-laser-probs-en1
+Skipping /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-probs-it1
+Skipping /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-probs-de1
+Skipping /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-probs-ru1
+{'Key': 'ru', 'Test Accuracy': 0.682, 'on': PosixPath('data/mldoc/ru-1/ru.train.csv'), 'files_count': 7}
+{'File saved to': PosixPath('data/mldoc/ru-1-ensemble/ru.train.csv')}
+{'Key': 'en', 'Test Accuracy': 0.82, 'on': PosixPath('data/mldoc/en-1/en.train.csv'), 'files_count': 7}
+{'File saved to': PosixPath('data/mldoc/en-1-ensemble/en.train.csv')}
+{'Key': 'es', 'Test Accuracy': 0.821, 'on': PosixPath('data/mldoc/es-1/es.train.csv'), 'files_count': 7}
+{'File saved to': PosixPath('data/mldoc/es-1-ensemble/es.train.csv')}
+{'Key': 'it', 'Test Accuracy': 0.782, 'on': PosixPath('data/mldoc/it-1/it.train.csv'), 'files_count': 7}
+{'File saved to': PosixPath('data/mldoc/it-1-ensemble/it.train.csv')}
+{'Key': 'ja', 'Test Accuracy': 0.685, 'on': PosixPath('data/mldoc/ja-1/ja.train.csv'), 'files_count': 7}
+{'File saved to': PosixPath('data/mldoc/ja-1-ensemble/ja.train.csv')}
+{'Key': 'zh', 'Test Accuracy': 0.789, 'on': PosixPath('data/mldoc/zh-1/zh.train.csv'), 'files_count': 7}
+{'File saved to': PosixPath('data/mldoc/zh-1-ensemble/zh.train.csv')}
+{'Key': 'de', 'Test Accuracy': 0.905, 'on': PosixPath('data/mldoc/de-1/de.train.csv'), 'files_count': 7}
+{'File saved to': PosixPath('data/mldoc/de-1-ensemble/de.train.csv')}
+{'Key': 'fr', 'Test Accuracy': 0.86, 'on': PosixPath('data/mldoc/fr-1/fr.train.csv'), 'files_count': 7}
+{'File saved to': PosixPath('data/mldoc/fr-1-ensemble/fr.train.csv')}
+```
+
+```
+
+
+
+ln -s /home/pczapla/workspace/ulmfit-multilingual/data-archive/mldoc/de-1/models/sp15k/qrnn_nl4_0.m data-archive/mldoc/de-1/models/sp15k/qrnn_base.m
+ln -s /home/pczapla/workspace/ulmfit-multilingual/data-archive/mldoc/en-1/models/sp15k/qrnn_nl4_tls.m data-archive/mldoc/en-1/models/sp15k/qrnn_base.m
+ln -s /home/pczapla/workspace/ulmfit-multilingual/data-archive/mldoc/es-1/models/sp15k/qrnn_nl4_0.m data-archive/mldoc/es-1/models/sp15k/qrnn_base.m
+ln -s /home/pczapla/workspace/ulmfit-multilingual/data-archive/mldoc/fr-1/models/sp15k/qrnn_nl4_tls.m data-archive/mldoc/fr-1/models/sp15k/qrnn_base.m
+ln -s /home/pczapla/workspace/ulmfit-multilingual/data-archive/mldoc/it-1/models/sp15k/qrnn_nl4_tls.m data-archive/mldoc/it-1/models/sp15k/qrnn_base.m
+ln -s /home/pczapla/workspace/ulmfit-multilingual/data-archive/mldoc/ja-1/models/sp15k/qrnn_nl4_tls.m data-archive/mldoc/ja-1/models/sp15k/qrnn_base.m
+ln -s /home/pczapla/workspace/ulmfit-multilingual/data-archive/mldoc/ru-1/models/sp15k/qrnn_nl4_tls.m data-archive/mldoc/ru-1/models/sp15k/qrnn_base.m
+ln -s /home/pczapla/workspace/ulmfit-multilingual/data-archive/mldoc/zh-1/models/sp15k/qrnn_nl4_tls.m data-archive/mldoc/zh-1/models/sp15k/qrnn_base.m
+
+
+for a in data-archive/mldoc/*-1; do cp $a/*unsup.csv $a/*test.csv $a/*dev.csv ${a/-archive/}-ensemble; done
+python -m ulmfit ls --glob 'data-archive/mldoc/*-1/models/sp15k/qrnn_base.m' --dataset_template='data/mldoc/${lang}-ensemble'
+
+
+python -m ulmfit eval --glob 'data-archive/mldoc/*-1/models/sp15k/qrnn_base.m' --dataset_template='../../data/mldoc/${lang}-ensemble' --num_lm_epochs=0 --num_cls_epochs=8 --early_stopping=False --bs=20 --label-smoothing-eps=0.1 --lr_sched=1cycle --skip_on_error=False
+
+```
\ No newline at end of file
diff --git a/results/logs/new-runs/qrnn-de.md b/results/logs/new-runs/qrnn-de.md
new file mode 100644
index 0000000..0080691
--- /dev/null
+++ b/results/logs/new-runs/qrnn-de.md
@@ -0,0 +1,32 @@
+```
+LANG=de
+python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='fsp' --nl 4 --name 'nl4' --max-vocab 30000 --lang ${LANG} --qrnn=True --lmseed=1 --nh=1552 - train 10 --bs=50 --drop_mult=0 --label-smoothing-eps=0.1
+Training lm
+Max vocab: 30000
+Cache dir: data/wiki/de-100/models/fsp30k
+Model dir: data/wiki/de-100/models/fsp30k/qrnn_nl4_lmseed-1.m
+Setting LM seed to 1
+Wiki text was split to 191112 articles
+Wiki text was split to 431 articles
+Data lm, trn: 191112, val: 431
+Size of vocabulary: 30000
+First 20 words in vocab: ['▁xxunk', '▁xxpad', '▁xxbos', '▁xxeos', '▁xxfld', '▁xxmaj', '▁xxup', '▁xxrep', '▁xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', "▁&'", 's', '-']
+Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'input_p': 0.25, 'output_p': 0.1, 'weight_p': 0.2, 'embed_p': 0.02, 'hidden_p': 0.15}
+Bptt 70
+Training lm from random weights
+epoch train_loss valid_loss accuracy time
+0 4.403168 4.469939 0.403841 1:30:29
+1 4.312332 4.432437 0.404200 1:30:30
+2 4.334051 4.423142 0.405613 1:30:03
+3 4.239668 4.376138 0.411162 1:30:05
+4 4.193250 4.324453 0.416959 1:30:04
+5 4.151386 4.248287 0.427130 1:30:16
+6 4.103295 4.160756 0.438965 1:30:32
+7 4.033971 4.086159 0.450292 1:30:16
+8 3.967596 4.029943 0.459819 1:31:20
+9 3.954203 4.013039 0.463228 1:31:14
+Total time: 15:04:52
+data/wiki/de-100/models/fsp30k
+Saving info data/wiki/de-100/models/fsp30k/qrnn_nl4_lmseed-1.m/info.json
+```
+-------
diff --git a/results/logs/new-runs/qrrn-ja.md b/results/logs/new-runs/qrrn-ja.md
new file mode 100644
index 0000000..b2b729a
--- /dev/null
+++ b/results/logs/new-runs/qrrn-ja.md
@@ -0,0 +1,34 @@
+```
+LANG=ja ✘ 130
+python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='fsp' --nl 4 --name 'nl4' --max-vocab 30000 --lang ${LANG} --qrnn=True --lmseed=1 --nh=1552 - train 10 --bs=50 --drop_mult=0 --label-smoothing-eps=0.1
+Training lm
+Max vocab: 30000
+Cache dir: data/wiki/ja-100/models/fsp30k
+Model dir: data/wiki/ja-100/models/fsp30k/qrnn_nl4_lmseed-1.m
+Setting LM seed to 1
+Wiki text was split to 120037 articles
+Wiki text was split to 63 articles
+Data lm, trn: 120037, val: 63
+Size of vocabulary: 30000
+First 20 words in vocab: ['▁xxunk', '▁xxpad', '▁xxbos', '▁xxeos', '▁xxfld', '▁xxmaj', '▁xxup', '▁xxrep', '▁xxwrep', '', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', '▁は', '▁・', '▁(']
+Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'input_p': 0.25, 'output_p': 0.1, 'weight_p': 0.2, 'embed_p': 0.02, 'hidden_p': 0.15}
+Bptt 70
+Training lm from random weights
+epoch train_loss valid_loss accuracy time
+0 4.346260 4.408598 0.370535 1:12:49
+1 4.253693 4.355113 0.372354 1:12:40
+2 4.190918 4.288729 0.383211 1:12:15
+3 4.148739 4.242265 0.389964 1:12:12
+4 4.136361 4.190885 0.398423 1:12:23
+5 4.051008 4.119476 0.409002 1:12:15
+6 3.966213 4.052222 0.419292 1:12:20
+7 3.928247 3.979634 0.431336 1:12:18
+8 3.840935 3.929402 0.442688 1:12:39
+9 3.909105 3.911067 0.446342 1:12:51
+Total time: 12:04:47
+data/wiki/ja-100/models/fsp30k
+Saving info data/wiki/ja-100/models/fsp30k/qrnn_nl4_lmseed-1.m/info.json
+```
+-------
+
+python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/fsp30k/qrnn_nl4_lmseed-1.m --lang=${LANG} --name 'nl4' --clsweightseed=0 - train 20 --bs 20 --lr_sched=1cycle --label-smoothing-eps=0.1
\ No newline at end of file
diff --git a/results/logs/qrnn-ja.md b/results/logs/qrnn-ja.md
new file mode 100644
index 0000000..ffc1046
--- /dev/null
+++ b/results/logs/qrnn-ja.md
@@ -0,0 +1,69 @@
+LANG=ja
+python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 30000 --lang ${LANG} --qrnn=True --lmseed=1 - train 10 --bs=50 --drop_mult=0
+
+LANG=ja
+python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang ${LANG} --qrnn=True "--tokenizer-mod=-fix" --lmseed=1 - train 10 --bs=50 --drop_mult=0
+
+python -m ulmfit cls --dataset-path data/mldoc/ja-1 --base-lm-path data/wiki/ja-100/models/sp15k-fix/qrnn_nl4_lmseed-1.m --lang=ja --name 'nl4' - train 20 --bs 20 --lr_sched=1cycle --label-smoothing-eps=0.1set-path data/mldoc/es-1 --base-lm-path data/wiki/ja-100/models/sp30k-fix/qrnn_nl4.m --lang=ja --name 'nl4' - train 20 --bs 40
+
+0 4.875985 3.940408 0.479504 02:40
+Total time: 02:40
+epoch train_loss valid_loss accuracy time
+0 3.814340 3.574515 0.524143 03:43
+1 3.344660 3.174727 0.591801 03:42
+2 3.062797 2.909801 0.638397 03:42
+3 2.924287 2.753593 0.662699 03:42
+4 2.832728 2.648505 0.679922 03:41
+5 2.673981 2.575237 0.692270 03:41
+6 2.727100 2.521090 0.702180 03:42
+7 2.647422 2.474463 0.710956 03:42
+8 2.557694 2.437784 0.717445 03:41
+9 2.619366 2.398306 0.725727 03:42
+10 2.501441 2.368656 0.731127 03:41
+11 2.501446 2.340000 0.737203 03:41
+12 2.539080 2.316010 0.742567 03:42
+13 2.441686 2.290955 0.748064 03:41
+14 2.406307 2.273114 0.752770 03:41
+15 2.421776 2.256135 0.756699 03:42
+16 2.399470 2.245539 0.758821 03:42
+17 2.336457 2.237878 0.760780 03:42
+18 2.383474 2.234359 0.761501 03:41
+19 2.407631 2.233689 0.761739 03:42
+Total time: 1:14:01
+/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k-fix
+Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k-fix/qrnn_nl4_lmseed-1.m/info.json
+Single training schedule
+epoch train_loss valid_loss f_beta precision recall kappa_score matthews_correff accuracy time
+/home/pczapla/workspace/_oss/fastai/fastai/fastai/metrics.py:179: UserWarning: average=`binary` was selected for a non binary case. Value for average has now been set to `macro` instead.
+ def _precision(self):
+0 0.872342 0.839480 0.739856 0.836258 0.741572 0.649270 0.682293 0.737000 00:19
+Better model found at epoch 0 with f_beta value: 0.739856481552124.
+/home/pczapla/workspace/_oss/fastai/fastai/fastai/metrics.py:179: UserWarning: average=`binary` was selected for a non binary case. Value for average has now been set to `macro` instead.
+ def _precision(self):
+1 0.678405 0.587759 0.886923 0.888504 0.885348 0.845242 0.846027 0.884000 00:19
+Better model found at epoch 1 with f_beta value: 0.8869231939315796.
+Total time: 00:39
+Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k-fix/qrnn_nl4_lmseed-1.m
+/home/pczapla/workspace/_oss/fastai/fastai/fastai/metrics.py:179: UserWarning: average=`binary` was selected for a non binary case. Value for average has now been set to `macro` instead.
+ def _precision(self):
+Model: nl4
+Validation on: test
+F1 score bin: 0.9002974033355713
+Loss: 0.3307778239250183
+Precision: 0.9020541906356812
+Recall: 0.9002037048339844
+Accuracy: 0.9007499814033508
+test F1 score bin: 0.9002974033355713
+test Loss: 0.33077782
+test Precision: 0.9020541906356812
+test Recall: 0.9002037048339844
+test Kappa Linear: 0.8676621913909912
+test Matthews Correff: 0.868194043636322
+test Accuracy: 0.9007499814033508
+
+
+
+eval --glob="mldoc/ja-1/models/sp15k/qrnn_nl4.m" --name nl4-1cyc-sl --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.1
+
+
+
diff --git a/results/logs/qrnn-zh.md b/results/logs/qrnn-zh.md
index ce7b125..dcb9e15 100644
--- a/results/logs/qrnn-zh.md
+++ b/results/logs/qrnn-zh.md
@@ -133,4 +133,8 @@ Traceback (most recent call last):
File "/home/pczapla/workspace/_oss/fastai/fastai/fastai/text/learner.py", line 221, in
return [torch.cat([l[si] for l in arrs], dim=1) for si in range_of(arrs[0])]
RuntimeError: CUDA error: out of memory
-```
\ No newline at end of file
+```
+
+## Fixed sentence piece
+LANG=zh
+python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 15000 --lang ${LANG} --qrnn=True "--tokenizer-mod=-fix" --lmseed=1 - train 10 --bs=50 --drop_mult=0
diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py
index 127c8fa..d28c546 100644
--- a/tests/test_end_to_end.py
+++ b/tests/test_end_to_end.py
@@ -34,10 +34,10 @@ def get_test_data():
test_imdb.mkdir(exist_ok=True, parents=True)
sz=1
- # we use the same text to see if models can overfit
- copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.train.tokens', n=1000*sz)
- copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.valid.tokens', n=600*sz)
- copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.test.tokens', n=600*sz)
+ # we use the same text to see if models overfits
+ copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.train.tokens', n=100*sz)
+ copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.valid.tokens', n=60*sz)
+ copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.test.tokens', n=60*sz)
copy_head(imdb / 'train.csv', test_imdb / 'train.csv', n=10*sz)
copy_head(imdb / 'train.csv', test_imdb / 'test.csv', n=6 * sz)
copy_head(imdb / 'train.csv', test_imdb / 'dev.csv', n=6 * sz)
@@ -46,12 +46,21 @@ def get_test_data():
return test_data, test_wt
+def test_evaluate():
+ """ Test ulmfit with (default) Moses tokenizer on small wikipedia dataset.
+ """
+ os.chdir(get_data_folder()/"..")
+ fastai.core.defaults.cpus=0
+ test_data, wt2 = get_test_data()
+ exp = ulmfit.train_clas.CLSHyperParams(test_data / 'imdb', lang='en', qrnn=False, max_vocab=1000, name="tst")
+ exp.evaluate_cls(save_name=None, bs=2)
+
+
def test_ulmfit_works_with_relative_paths():
""" Test ulmfit with (default) Moses tokenizer on small wikipedia dataset.
"""
os.chdir(get_data_folder()/"..")
-
test_data, wt2 = get_test_data()
lm_name = 'end-to-end-test-default'
cuda_id = 0
@@ -121,7 +130,7 @@ def test_ulmfit_fastai_end_to_end_label_smoothing():
""" Test ulmfit with sentencepiece tokenizer on small wikipedia dataset.
"""
test_data, wt2 = get_test_data()
- lm_name = 'end-to-end-test-fastai'
+ lm_name = 'end-to-end-test-fastai-lablel-smoothing'
exp = ulmfit.pretrain_lm.LMHyperParams(
dataset_path=wt2,
@@ -133,54 +142,9 @@ def test_ulmfit_fastai_end_to_end_label_smoothing():
name=lm_name,
)
exp.train_lm(num_epochs=1, bs=2, label_smoothing_eps=0.1)
- exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir)
+ exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir, name=lm_name)
exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, label_smoothing_eps=0.1 )
-
-def test_ulmfit_fastai_bidir_end_to_end():
- """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset.
- """
- test_data, wt2 = get_test_data()
- lm_name = 'end-to-end-test-fastai'
-
- exp = ulmfit.pretrain_lm.LMHyperParams(
- dataset_path=wt2,
- lang='en',
- cuda_id=cuda_id,
- qrnn=False,
- bidir=True,
- tokenizer='f',
- max_vocab=100,
- name=lm_name,
- )
- exp.train_lm(num_epochs=1, bs=2)
- exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(str(test_data / 'imdb'), str(exp.model_dir))
- exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, )
-
-def test_ulmfit_moses_fa_bidir_end_to_end():
- """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset.
- """
- test_data, wt2 = get_test_data()
- lm_name = 'end-to-end-test-fastai'
-
- exp = ulmfit.pretrain_lm.LMHyperParams(
- dataset_path=wt2,
- lang='en',
- cuda_id=cuda_id,
- qrnn=False,
- bidir=True,
- tokenizer='vf',
- max_vocab=100,
- name=lm_name,
- )
- exp.train_lm(num_epochs=1, bs=2)
- exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir)
- exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, )
-
-# def test_classification_model_work_with_different_dropmul():
-# learn = self.create_cls_learner(data_clas, drop_mult=0.1)
-# learn = self.create_cls_learner(data_clas, drop_mult=0.0)
-
def test_ulmfit_sentencepiece_end_to_end():
""" Test ulmfit with sentencepiece tokenizer on small wikipedia dataset.
"""
@@ -201,6 +165,25 @@ def test_ulmfit_sentencepiece_end_to_end():
exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir)
exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, )
+def test_ulmfit_sentencepiece_fastai_impl_end_to_end():
+ """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset.
+ """
+ test_data, wt2 = get_test_data()
+ lm_name = 'end-to-end-test-spm-fa'
+
+ exp = ulmfit.pretrain_lm.LMHyperParams(
+ dataset_path=wt2,
+ lang='en',
+ cuda_id=cuda_id,
+ qrnn=False,
+ tokenizer=ulmfit.pretrain_lm.Tokenizers.FASTAI_SUBWORD,
+ max_vocab=200,
+ name=lm_name,
+ )
+ exp.train_lm(num_epochs=1, bs=2)
+ # not supported yet
+ exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir)
+ exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, )
if __name__ == "__main__":
fire.Fire() # allows using all functions via CLI
diff --git a/ulmfit/__main__.py b/ulmfit/__main__.py
index b6c35c6..9ffa500 100644
--- a/ulmfit/__main__.py
+++ b/ulmfit/__main__.py
@@ -49,13 +49,13 @@ class ULMFiT:
params = CLSHyperParams.from_lm(dataset_path, base_lm_path, **changes)
else:
params = CLSHyperParams(dataset_path=dataset_path, **changes)
- return FireView(train=params.train_cls, validate_cls=params.validate_cls)
+ return FireView(train=params.train_cls, evaluate_cls=params.evaluate_cls)
@wraps(CLSHyperParams)
def load_cls(self, model_path, **changes):
params = CLSHyperParams.from_json(model_path, **changes)
- return FireView(train=params.train_cls, validate_cls=params.validate_cls)
+ return FireView(train=params.train_cls, evaluate_cls=params.evaluate_cls)
def eval_noise_resistance(self, lang="de", size=1, prefix_name="", model="sp15k/qrnn_nl4.m",
@@ -255,8 +255,8 @@ class ULMFiT:
last_model_dir = params.model_dir.relative_to(data_dir.parent)
if (params.model_dir/"cls_best.pth").exists():
print("Evaluating previously trained model")
- d_tst = params.validate_cls(save_name=save_name, label_smoothing_eps=label_smoothing_eps, use_cache=True, mode="test")
- d_val = params.validate_cls(save_name=save_name, label_smoothing_eps=label_smoothing_eps, use_cache=True, mode="valid")
+ d_tst = params.evaluate_cls(save_name=save_name, label_smoothing_eps=label_smoothing_eps, use_cache=True, mode="test")
+ d_val = params.evaluate_cls(save_name=save_name, label_smoothing_eps=label_smoothing_eps, use_cache=True, mode="valid")
d={}
d.update(d_val)
d.update(d_tst)
diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py
index 07da1ce..67f49c9 100644
--- a/ulmfit/pretrain_lm.py
+++ b/ulmfit/pretrain_lm.py
@@ -17,7 +17,7 @@ from fastai.text import *
import torch
from fastai_contrib.utils import read_file, read_whitespace_file, \
validate, PAD, UNK, get_sentencepiece, read_clas_data, TRN, VAL, TST, PAD_TOKEN_ID, \
- replace_std_toks, MosesPreprocessingFunc
+ replace_std_toks, MosesPreprocessingFunc, get_sentencepiece_fastai
from fastai_contrib.learner import bilm_learner, accuracy_fwd, accuracy_bwd, bilm_text_classifier_learner
import pickle
@@ -31,15 +31,17 @@ ENC_BEST = "enc_best"
class Tokenizers(Enum):
+ FASTAI_SUBWORD = 'fsp'
SUBWORD='sp'
- BROKENSUBWORD = 'bsp'
MOSES='v'
MOSES_FA='vf'
FASTAI='f'
+
def istitle(line):
return len(re.findall(r'^ ?= [^=]* = ?$', line)) != 0
+
def read_wiki_articles(filename):
if "reddit" in str(filename): # Temporary hack to handle poleval reddit dataset
return pd.read_csv(filename, header=None, names=["texts"]).fillna("")
@@ -58,6 +60,7 @@ def read_wiki_articles(filename):
print(f"Wiki text was split to {len(articles)} articles")
return pd.DataFrame({'texts': np.array(articles, dtype=np.object)})
+
name_re = re.compile("(bwd)?(lstm|qrnn)_(.*)_(lmseed-)?.*\.m")
def folder_name_to_model_name(folder_name):
if hasattr(folder_name, 'name'):
@@ -67,6 +70,7 @@ def folder_name_to_model_name(folder_name):
return match.group(3)
return None
+
@dataclass
class DataSetParams:
dataset_path: str # data_dir
@@ -85,11 +89,40 @@ class DataSetParams:
except KeyError as e:
raise KeyError(f"{e} , options:{repr(list(params.keys()))}")
+# temporary loading function as from_df does not support processors
+def make_data_bunch_from_df(cls, path: PathOrStr, train_df: DataFrame, valid_df: DataFrame,
+ tokenizer: Tokenizer = None, vocab: Vocab = None, classes: Collection[str] = None,
+ text_cols: IntsOrStrs = 1,
+ label_cols: IntsOrStrs = 0, label_delim: str = None, chunksize: int = 10000,
+ max_vocab: int = 60000,
+ min_freq: int = 2, mark_fields: bool = False, include_bos: bool = True,
+ include_eos: bool = False, processor=None, **kwargs) -> DataBunch:
+ "Create a `TextDataBunch` from DataFrames. `kwargs` are passed to the dataloader creation."
+ assert processor is None or tokenizer is None, "Processor and tokenizer are mutually exclusive."
+
+ if processor is None:
+ processor = fastai.text.data._get_processor(tokenizer=tokenizer, vocab=vocab, chunksize=chunksize, max_vocab=max_vocab,
+ min_freq=min_freq, mark_fields=mark_fields,
+ include_bos=include_bos, include_eos=include_eos)
+
+ if classes is None and is_listy(label_cols) and len(label_cols) > 1: classes = label_cols
+ src = ItemLists(path, TextList.from_df(train_df, path, cols=text_cols, processor=processor),
+ TextList.from_df(valid_df, path, cols=text_cols, processor=processor))
+ if cls == TextLMDataBunch:
+ src = src.label_for_lm()
+ else:
+ if label_delim is not None:
+ src = src.label_from_df(cols=label_cols, classes=classes, label_delim=label_delim)
+ else:
+ src = src.label_from_df(cols=label_cols, classes=classes)
+ return src.databunch(**kwargs)
+
+
@dataclass
class LMHyperParams(DataSetParams):
base_lm_path: str = None
backwards: str = False
- bidir: bool =False
+ bidir: bool = False
qrnn: bool = True
max_vocab: int = 60000
tokenizer: Tokenizers = Tokenizers.MOSES
@@ -131,7 +164,7 @@ class LMHyperParams(DataSetParams):
self.cache_dir = self.dataset_path / 'models' / self.tokenizer_prefix
self.model_dir = self.cache_dir / self.model_name
- if self.nh is None: self.nh = 1550 if self.qrnn else 1150
+ if self.nh is None: self.nh = 1552 if self.qrnn else 1152
if self.name is None: self.name = self.lang
@property
@@ -169,7 +202,17 @@ class LMHyperParams(DataSetParams):
def tokenizer_to_fastai_args(self, sp_data_func, use_moses):
moses_preproc = [MosesPreprocessingFunc(self.lang)] if use_moses else []
- if self.tokenizer is Tokenizers.SUBWORD or self.tokenizer is Tokenizers.BROKENSUBWORD:
+ if self.tokenizer is Tokenizers.FASTAI_SUBWORD:
+ if self.base_lm_path and not (self.cache_dir / "spm.model").exists(): # ensure we are using the same sentence piece model
+ shutil.copy(self.base_lm_path / '..' / 'spm.model', self.cache_dir)
+ shutil.copy(self.base_lm_path / '..' / 'spm.vocab', self.cache_dir)
+ args = get_sentencepiece_fastai(
+ cache_dir=self.cache_dir,
+ vocab_size=self.max_vocab,
+ lang=self.lang,
+ pre_rules=moses_preproc + defaults.text_pre_rules)
+
+ elif self.tokenizer is Tokenizers.SUBWORD:
if self.base_lm_path and not(self.cache_dir/"spm.model").exists(): # ensure we are using the same sentence piece model
shutil.copy(self.base_lm_path / '..' / 'itos.pkl', self.cache_dir)
shutil.copy(self.base_lm_path / '..' / 'spm.model', self.cache_dir)
@@ -230,6 +273,7 @@ class LMHyperParams(DataSetParams):
learn = self.create_lm_learner(data_lm, drop_mult=drop_mult, label_smoothing_eps=label_smoothing_eps)
print("Bptt", data_lm.bptt)
learn.true_wd = true_wd
+ #learn = learn.to_fp16()
if num_epochs > 0:
if self.pretrained_fnames or self.pretrained_model:
print("Training lm from: ", self.pretrained_fnames or self.pretrained_model)
@@ -337,12 +381,13 @@ class LMHyperParams(DataSetParams):
data = load_data(self.cache_dir, name, bs=bs)
else:
print(f"Running tokenization {name}...")
- data = bunch_class.from_df(path=self.cache_dir,
+ data = make_data_bunch_from_df(cls=bunch_class, path=self.cache_dir,
train_df=train_df,
valid_df=valid_df,
max_vocab=self.max_vocab,
bs=bs,
**args)
+
data.save(name)
with open(self.cache_dir/"itos.pkl", 'wb') as f:
pickle.dump(data.vocab.itos, f)
diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py
index 417628f..36220fc 100644
--- a/ulmfit/train_clas.py
+++ b/ulmfit/train_clas.py
@@ -102,9 +102,9 @@ class CLSHyperParams(LMHyperParams):
kappa_lin = KappaScore()
matthews_correff = MatthewsCorreff()
metrics = [f1_score, precision, recall, kappa_lin, matthews_correff]
- # TODO: fix this in fast.ai
- if init:
- for metric in metrics: metric.on_train_begin()
+ # # TODO: fix this in fast.ai
+ # if init:
+ # for metric in metrics: metric.on_train_begin()
metrics.append(accuracy)
return metrics
@@ -176,11 +176,11 @@ class CLSHyperParams(LMHyperParams):
learn.save('cls_best', with_opt=False)
#learn.save('cls_best', with_opt=False) # we don't use early stopping for the time being
del learn
- return self.validate_cls('cls_best', bs=bs, data_tst=data_tst, learn=None)
+ return self.evaluate_cls('cls_best', bs=bs, data_tst=data_tst, learn=None)
- def validate_cls(self, save_name='cls_best', bs=40, data_tst=None, learn=None,
+ def evaluate_cls(self, save_name='cls_best', bs=40, data_tst=None, learn=None,
dump_preds=None, mode="test", label_smoothing_eps=None, use_cache=False):
- cache_file = (self.model_dir / f'results_{mode+("" if save_name == "cls_best" else save_name)}.json')
+ cache_file = (self.model_dir / f'results_{mode+("" if save_name == "cls_best" else str(save_name))}.json')
if use_cache and cache_file.exists():
with cache_file.open("r") as fp:
return json.load(fp)
@@ -191,9 +191,13 @@ class CLSHyperParams(LMHyperParams):
else:
dt = data_tst
if learn is None:
- learn = self.create_cls_learner(dt, drop_mult=0.3, metrics=self.get_metrics(True))
+ learn = self.create_cls_learner(dt, drop_mult=0.3, metrics=self.get_metrics(True), silent=True, early_stopping=False)
learn.unfreeze()
- learn.load(save_name)
+ if save_name is not None:
+ learn.load(save_name)
+ else:
+ print("Using random weights!")
+
if mode == "test":
ds = data_tst.valid_dl
elif mode == "valid" or mode == "dev":
@@ -211,7 +215,7 @@ class CLSHyperParams(LMHyperParams):
np.save(self.model_dir / f"preds-on-{mode}.npy", probs.cpu().numpy())
results = learn.validate(ds)
print(f"Model: {self.name}")
- print(f"Validation on: {mode}")
+ print(f"Evaluation on: {mode}")
labeled_results = self.output_metrics(results, mode=mode)
with cache_file.open("w") as fp:
@@ -243,6 +247,8 @@ class CLSHyperParams(LMHyperParams):
if label_smoothing_eps > 0.0:
learn.loss_func = FlattenedLoss(LabelSmoothingCrossEntropy, eps=label_smoothing_eps)
+
+ #learn = learn.to_fp16()
return learn
def load_cls_data(self, bs, **kwargs):