diff --git a/prepare_imdb.sh b/prepare_imdb.sh index 89815b7..4c2105b 100755 --- a/prepare_imdb.sh +++ b/prepare_imdb.sh @@ -7,5 +7,5 @@ echo "Saving data in $DATA_DIR" wget -c "http://files.fast.ai/data/aclImdb.tgz" -P "${DATA_DIR}" echo "Imdb is raw text no preparation is done" -python -m fastai_contrib.utils prepare_imdb "${DATA_DIR}/aclImdb.tgz" +python -m ulmfit.datasets.utils prepare_imdb "${DATA_DIR}/aclImdb.tgz" diff --git a/results/logs/new-runs/qrnn-de-1152.md b/results/logs/new-runs/qrnn-de-1152.md new file mode 100644 index 0000000..639775d --- /dev/null +++ b/results/logs/new-runs/qrnn-de-1152.md @@ -0,0 +1,31 @@ +``` +LANG=de +python -m multifit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='fsp' --nl 4 --name '1152' --max-vocab 30000 --lang ${LANG} --qrnn=True --lmseed=1 --nh=1152 - train 10 --bs=50 --drop_mult=0 --label-smoothing-eps=0.1 +Training lm +Max vocab: 30000 +Cache dir: data/wiki/de-100/models/fsp30k +Model dir: data/wiki/de-100/models/fsp30k/qrnn_1152_lmseed-1.m +Setting LM seed to 1 +Wiki text was split to 191112 articles +Wiki text was split to 431 articles +Data lm, trn: 191112, val: 431 +Size of vocabulary: 30000 +First 20 words in vocab: ['▁xxunk', '▁xxpad', '▁xxbos', '▁xxeos', '▁xxfld', '▁xxmaj', '▁xxup', '▁xxrep', '▁xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', "▁&'", 's', '-'] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} config: {'emb_sz': 400, 'n_hid': 1152, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.1, 'hidden_p': 0.15, 'input_p': 0.25, 'embed_p': 0.02, 'weight_p': 0.2, 'tie_weights': True, 'out_bias': True} +Bptt 70 +Training lm from random weights +epoch train_loss valid_loss accuracy time +0 4.452324 4.517938 0.397063 1:12:10 +1 4.354969 4.473063 0.399108 1:12:08 +2 4.375927 4.460905 0.400854 1:11:47 +3 4.279962 4.406254 0.406279 1:11:47 +4 4.229251 4.358620 0.413019 1:11:51 +5 4.194514 4.380852 0.409330 1:11:54 +6 4.151179 4.204870 0.431857 1:11:57 +7 4.085239 4.131142 0.442781 1:12:28 +8 4.026665 4.080124 0.451819 1:13:08 +9 4.015738 4.063066 0.454933 1:13:28 +Total time: 12:02:42 +data/wiki/de-100/models/fsp30k +Saving info data/wiki/de-100/models/fsp30k/qrnn_1152_lmseed-1.m/info.json +``` \ No newline at end of file diff --git a/results/logs/new-runs/qrnn-ja-1152.md b/results/logs/new-runs/qrnn-ja-1152.md new file mode 100644 index 0000000..faf9ea8 --- /dev/null +++ b/results/logs/new-runs/qrnn-ja-1152.md @@ -0,0 +1,139 @@ +```` +LANG=ja ✘ 130 +python -m multifit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='fsp' --nl 4 --name 'nl4-1152' --max-vocab 15000 --lang ${LANG} --qrnn=True --lmseed=1 --nh=1152 - train 10 --bs=50 --drop_mult=0 --label-smoothing-eps=0.1 +Training lm +Max vocab: 15000 +Cache dir: data/wiki/ja-100/models/fsp15k +Model dir: data/wiki/ja-100/models/fsp15k/qrnn_nl4-1152_lmseed-1.m +Setting LM seed to 1 +Wiki text was split to 120037 articles +Wiki text was split to 63 articles +Running tokenization lm... +sentencepiece_trainer.cc(116) LOG(INFO) Running command: --input=/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/fsp15k/all_text.out --max_sentence_length=20480 --character_coverage=0.9998 --unk_id=9 --pad_id=-1 --bos_id=-1 --eos_id=-1 --user_defined_symbols=▁xxunk,▁xxpad,▁xxbos,▁xxeos,▁xxfld,▁xxmaj,▁xxup,▁xxrep,▁xxwrep --model_prefix=/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/fsp15k/spm --vocab_size=15000 --model_type=unigram +sentencepiece_trainer.cc(49) LOG(INFO) Starts training with : +TrainerSpec { + input: /home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/fsp15k/all_text.out + input_format: + model_prefix: /home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/fsp15k/spm + model_type: UNIGRAM + vocab_size: 15000 + self_test_sample_size: 0 + character_coverage: 0.9998 + input_sentence_size: 0 + shuffle_input_sentence: 1 + seed_sentencepiece_size: 1000000 + shrinking_factor: 0.75 + max_sentence_length: 20480 + num_threads: 16 + num_sub_iterations: 2 + max_sentencepiece_length: 16 + split_by_unicode_script: 1 + split_by_number: 1 + split_by_whitespace: 1 + treat_whitespace_as_suffix: 0 + user_defined_symbols: ▁xxunk + user_defined_symbols: ▁xxpad + user_defined_symbols: ▁xxbos + user_defined_symbols: ▁xxeos + user_defined_symbols: ▁xxfld + user_defined_symbols: ▁xxmaj + user_defined_symbols: ▁xxup + user_defined_symbols: ▁xxrep + user_defined_symbols: ▁xxwrep + hard_vocab_limit: 1 + use_all_vocab: 0 + unk_id: 9 + bos_id: -1 + eos_id: -1 + pad_id: -1 + unk_piece: + bos_piece: + eos_piece: + pad_piece: + unk_surface: ⁇ +} +NormalizerSpec { + name: nmt_nfkc + add_dummy_prefix: 1 + remove_extra_whitespaces: 1 + escape_whitespaces: 1 + normalization_rule_tsv: +} + +trainer_interface.cc(267) LOG(INFO) Loading corpus: /home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/fsp15k/all_text.out +trainer_interface.cc(287) LOG(WARNING) Found too long line (74468 > 20480). +trainer_interface.cc(289) LOG(WARNING) Too long lines are skipped in the training. +trainer_interface.cc(290) LOG(WARNING) The maximum length can be changed with --max_sentence_length= flag. +trainer_interface.cc(315) LOG(INFO) Loaded all 115812 sentences +trainer_interface.cc(321) LOG(INFO) Skipped 4225 too long sentences. +trainer_interface.cc(330) LOG(INFO) Adding meta_piece: ▁xxunk +trainer_interface.cc(330) LOG(INFO) Adding meta_piece: ▁xxpad +trainer_interface.cc(330) LOG(INFO) Adding meta_piece: ▁xxbos +trainer_interface.cc(330) LOG(INFO) Adding meta_piece: ▁xxeos +trainer_interface.cc(330) LOG(INFO) Adding meta_piece: ▁xxfld +trainer_interface.cc(330) LOG(INFO) Adding meta_piece: ▁xxmaj +trainer_interface.cc(330) LOG(INFO) Adding meta_piece: ▁xxup +trainer_interface.cc(330) LOG(INFO) Adding meta_piece: ▁xxrep +trainer_interface.cc(330) LOG(INFO) Adding meta_piece: ▁xxwrep +trainer_interface.cc(330) LOG(INFO) Adding meta_piece: +trainer_interface.cc(335) LOG(INFO) Normalizing sentences... +trainer_interface.cc(385) LOG(INFO) all chars count=179924674 +trainer_interface.cc(393) LOG(INFO) Done: 99.98% characters are covered. +trainer_interface.cc(403) LOG(INFO) Alphabet size=4440 +trainer_interface.cc(404) LOG(INFO) Final character coverage=0.9998 +trainer_interface.cc(436) LOG(INFO) Done! preprocessed 115812 sentences. +unigram_model_trainer.cc(129) LOG(INFO) Making suffix array... +unigram_model_trainer.cc(133) LOG(INFO) Extracting frequent sub strings... +unigram_model_trainer.cc(184) LOG(INFO) Initialized 1000000 seed sentencepieces +trainer_interface.cc(442) LOG(INFO) Tokenizing input sentences with whitespace: 115812 +trainer_interface.cc(452) LOG(INFO) Done! 2256013 +unigram_model_trainer.cc(470) LOG(INFO) Using 2256013 sentences for EM training +unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=0 size=589366 obj=9.67447 num_tokens=5565112 num_tokens/piece=9.44254 +unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=1 size=499262 obj=8.54312 num_tokens=5565195 num_tokens/piece=11.1468 +unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=0 size=374209 obj=8.48394 num_tokens=5655360 num_tokens/piece=15.1128 +unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=1 size=373594 obj=8.47578 num_tokens=5655268 num_tokens/piece=15.1375 +unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=0 size=280179 obj=8.57339 num_tokens=5837695 num_tokens/piece=20.8356 +unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=1 size=280142 obj=8.57282 num_tokens=5839303 num_tokens/piece=20.8441 +unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=0 size=210103 obj=8.62784 num_tokens=6061221 num_tokens/piece=28.8488 +unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=1 size=210098 obj=8.62292 num_tokens=6062624 num_tokens/piece=28.8562 +unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=0 size=157573 obj=8.72461 num_tokens=6293760 num_tokens/piece=39.9419 +unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=1 size=157573 obj=8.71413 num_tokens=6295112 num_tokens/piece=39.9504 +unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=0 size=118179 obj=8.87183 num_tokens=6531351 num_tokens/piece=55.2666 +unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=1 size=118178 obj=8.86018 num_tokens=6532542 num_tokens/piece=55.2771 +unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=0 size=88632 obj=9.09891 num_tokens=6780015 num_tokens/piece=76.4962 +unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=1 size=88632 obj=9.08523 num_tokens=6781876 num_tokens/piece=76.5172 +unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=0 size=66474 obj=9.39972 num_tokens=7048669 num_tokens/piece=106.036 +unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=1 size=66474 obj=9.38439 num_tokens=7050941 num_tokens/piece=106.071 +unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=0 size=49855 obj=9.80014 num_tokens=7334836 num_tokens/piece=147.123 +unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=1 size=49855 obj=9.78241 num_tokens=7339902 num_tokens/piece=147.225 +unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=0 size=37391 obj=10.5579 num_tokens=7654233 num_tokens/piece=204.708 +unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=1 size=37391 obj=10.5324 num_tokens=7672187 num_tokens/piece=205.188 +unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=0 size=28043 obj=11.6071 num_tokens=8021953 num_tokens/piece=286.059 +unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=1 size=28043 obj=11.5755 num_tokens=8063782 num_tokens/piece=287.551 +unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=0 size=21032 obj=13.1808 num_tokens=8464260 num_tokens/piece=402.447 +unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=1 size=21032 obj=13.1392 num_tokens=8549719 num_tokens/piece=406.51 +unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=0 size=16500 obj=14.8618 num_tokens=8932725 num_tokens/piece=541.377 +unigram_model_trainer.cc(486) LOG(INFO) EM sub_iter=1 size=16500 obj=14.8185 num_tokens=8996465 num_tokens/piece=545.24 +trainer_interface.cc(508) LOG(INFO) Saving model: /home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/fsp15k/spm.model +trainer_interface.cc(532) LOG(INFO) Saving vocabs: /home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/fsp15k/spm.vocab +Data lm, trn: 120037, val: 63 +Size of vocabulary: 15000 +First 20 words in vocab: ['▁xxunk', '▁xxpad', '▁xxbos', '▁xxeos', '▁xxfld', '▁xxmaj', '▁xxup', '▁xxrep', '▁xxwrep', '', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', '▁は', '▁・', '▁('] +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} config: {'emb_sz': 400, 'n_hid': 1152, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.1, 'hidden_p': 0.15, 'input_p': 0.25, 'embed_p': 0.02, 'weight_p': 0.2, 'tie_weights': True, 'out_bias': True} +Bptt 70 +Training lm from random weights +epoch train_loss valid_loss accuracy time +0 3.886147 3.948560 0.430394 47:41 +1 3.835572 3.931224 0.429429 47:37 +2 3.830497 3.871733 0.439411 47:25 +3 3.740645 3.824856 0.446847 47:25 +4 3.715881 3.770223 0.455526 47:25 +5 3.698531 3.708276 0.463582 47:25 +6 3.600051 3.639733 0.476275 47:25 +7 3.540591 3.576993 0.487240 47:25 +8 3.487257 3.533188 0.496038 47:25 +9 3.503373 3.515034 0.499575 47:31 +Total time: 7:54:45 +data/wiki/ja-100/models/fsp15k +Saving info data/wiki/ja-100/models/fsp15k/qrnn_nl4-1152_lmseed-1.m/info.json +```` \ No newline at end of file diff --git a/results/logs/new-runs/qrrn-ja.md b/results/logs/new-runs/qrrn-ja.md index b2b729a..3cb33cd 100644 --- a/results/logs/new-runs/qrrn-ja.md +++ b/results/logs/new-runs/qrrn-ja.md @@ -31,4 +31,91 @@ Saving info data/wiki/ja-100/models/fsp30k/qrnn_nl4_lmseed-1.m/info.json ``` ------- -python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/fsp30k/qrnn_nl4_lmseed-1.m --lang=${LANG} --name 'nl4' --clsweightseed=0 - train 20 --bs 20 --lr_sched=1cycle --label-smoothing-eps=0.1 \ No newline at end of file +``` +python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/fsp30k/qrnn_nl4_lmseed-1.m --lang=${LANG} --name 'nl4' --clsweightseed=0 - train 20 --bs 20 --lr_sched=1cycle --label-smoothing-eps=0.1 +data/mldoc/ja-1 'data/wiki/ja-100/models/fsp30k/qrnn_nl4_lmseed-1.m' +Training CLS +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/fsp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/fsp30k/qrnn_nl4_lmseed-1-clsweightseed-0.m +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv +Running tokenization lm-notst... +Data lm-notst, trn: 13500, val: 1500 +Running tokenization cls... +Data cls, trn: 1000, val: 1000 +Running tokenization tst... +Data tst, trn: 1000, val: 4000 +Size of vocabulary: 30000 +First 20 words in vocab: ['▁xxunk', '▁xxpad', '▁xxbos', '▁xxeos', '▁xxfld', '▁xxmaj', '▁xxup', '▁xxrep', '▁xxwrep', '', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', '▁は', '▁・', '▁('] +Training lm +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/fsp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/fsp30k/qrnn_nl4_lmseed-1-clsweightseed-0.m +Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'input_p': 0.25, 'output_p': 0.1, 'weight_p': 0.2, 'embed_p': 0.02, 'hidden_p': 0.15} +Loading pretrained model +Bptt 70 +Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/fsp30k/qrnn_nl4_lmseed-1.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/fsp30k/qrnn_nl4_lmseed-1.m/../itos')] +epoch train_loss valid_loss accuracy time +0 4.921511 4.071093 0.449957 02:49 +Total time: 02:49 +epoch train_loss valid_loss accuracy time +0 4.078950 3.772346 0.490980 03:45 +1 3.727190 3.439734 0.550465 03:47 +2 3.359830 3.201281 0.590776 03:47 +3 3.319000 3.039330 0.618913 03:48 +4 3.127082 2.923124 0.637883 03:49 +5 3.004416 2.842367 0.652876 03:49 +6 2.940764 2.775950 0.664768 03:49 +7 2.961673 2.723808 0.672876 03:48 +8 2.895331 2.680753 0.681351 03:49 +9 2.804141 2.642330 0.688681 03:48 +10 2.867590 2.607323 0.695910 03:49 +11 2.755013 2.570714 0.703812 03:48 +12 2.747998 2.539443 0.710536 03:49 +13 2.710947 2.512321 0.716777 03:49 +14 2.675630 2.486792 0.721944 03:49 +15 2.654088 2.466209 0.726944 03:50 +16 2.668594 2.452018 0.730059 03:49 +17 2.640188 2.444180 0.731973 03:47 +18 2.562034 2.439116 0.732988 03:43 +19 2.641966 2.438401 0.733259 03:42 +Total time: 1:16:04 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/fsp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/fsp30k/qrnn_nl4_lmseed-1-clsweightseed-0.m/info.json +Setting classifier weights seed to 0 +Single training schedule +epoch train_loss valid_loss f_beta precision recall kappa_score matthews_correff accuracy time +/home/pczapla/workspace/_oss/fastai/fastai/fastai/metrics.py:189: UserWarning: average=`binary` was selected for a non binary case. Value for average has now been set to `macro` instead. + warn("average=`binary` was selected for a non binary case. Value for average has now been set to `macro` instead.") +0 0.866883 0.690708 0.826320 0.859738 0.829345 0.770440 0.781947 0.828000 00:13 +Better model found at epoch 0 with f_beta value: 0.826319694519043. +/home/pczapla/workspace/_oss/fastai/fastai/fastai/metrics.py:189: UserWarning: average=`binary` was selected for a non binary case. Value for average has now been set to `macro` instead. + warn("average=`binary` was selected for a non binary case. Value for average has now been set to `macro` instead.") +1 0.676495 0.609408 0.881395 0.888526 0.874378 0.831782 0.835147 0.874000 00:13 +Better model found at epoch 1 with f_beta value: 0.8813954591751099. +Total time: 00:27 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/fsp30k/qrnn_nl4_lmseed-1-clsweightseed-0.m +/home/pczapla/workspace/_oss/fastai/fastai/fastai/metrics.py:189: UserWarning: average=`binary` was selected for a non binary case. Value for average has now been set to `macro` instead. + warn("average=`binary` was selected for a non binary case. Value for average has now been set to `macro` instead.") +Model: nl4 +Evaluation on: test +F1 score bin: 0.8911986351013184 +Loss: 0.32485464215278625 +Precision: 0.8969309329986572 +Recall: 0.8911643028259277 +Accuracy: 0.8914999961853027 +test F1 score bin: 0.8911986351013184 +test Loss: 0.32485464 +test Precision: 0.8969309329986572 +test Recall: 0.8911643028259277 +test Kappa Linear: 0.8553637266159058 +test Matthews Correff: 0.8571781516075134 +test Accuracy: 0.8914999961853027 +``` + +---- +``` +for seed in 1 2 3 4 5 ; do + python -m multifit eval --glob="mldoc/${LANG}-1/models/fsp30k/qrnn_nl4_lmseed-1-clsweightseed-0.m" --name nl4 --clsweightseed=$seed --num-cls-epochs=8 --bs=18 --lr_sched=1cycle --label-smoothing-eps=0.1 +done +``` \ No newline at end of file diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index d28c546..6ef9ba5 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -1,19 +1,12 @@ -import os -import glob -import fire -import ulmfit.pretrain_lm -import ulmfit.train_clas -from fastai import * -from fastai.text import * -from fastai_contrib.utils import * - -""" -It is a mixture of a pytest unit test and woven together to compose an end to end functional test. -""" +import multifit.pretrain_lm +import multifit.train_clas +from multifit.datasets.utils import * +from multifit.pretrain_lm import get_data_folder import fastai.core fastai.core.defaults.cpus = 1 cuda_id=0 + def copy_head(src_fn, dst_fn, n=1000): with src_fn.open("r") as s, dst_fn.open("w") as d: for i in range(n): @@ -52,7 +45,7 @@ def test_evaluate(): os.chdir(get_data_folder()/"..") fastai.core.defaults.cpus=0 test_data, wt2 = get_test_data() - exp = ulmfit.train_clas.CLSHyperParams(test_data / 'imdb', lang='en', qrnn=False, max_vocab=1000, name="tst") + exp = multifit.train_clas.CLSHyperParams(test_data / 'imdb', lang='en', qrnn=False, max_vocab=1000, name="tst") exp.evaluate_cls(save_name=None, bs=2) @@ -64,7 +57,7 @@ def test_ulmfit_works_with_relative_paths(): test_data, wt2 = get_test_data() lm_name = 'end-to-end-test-default' cuda_id = 0 - exp = ulmfit.pretrain_lm.LMHyperParams( + exp = multifit.pretrain_lm.LMHyperParams( dataset_path=wt2.relative_to(Path.cwd()), lang='en', qrnn=False, @@ -76,12 +69,12 @@ def test_ulmfit_works_with_relative_paths(): #assert exp.results['accuracy'] > 0.02 - exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) + exp2 = multifit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) exp2.train_cls(num_lm_epochs=1, unfreeze=False, bs=4,) # should work for the second time as well - exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) + exp2 = multifit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, ) @@ -91,7 +84,7 @@ def test_ulmfit_default_end_to_end(): test_data, wt2 = get_test_data() lm_name = 'end-to-end-test-default' cuda_id = 0 - exp = ulmfit.pretrain_lm.LMHyperParams( + exp = multifit.pretrain_lm.LMHyperParams( dataset_path=wt2, lang='en', qrnn=False, @@ -103,7 +96,7 @@ def test_ulmfit_default_end_to_end(): #assert exp.results['accuracy'] > 0.02 - exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) + exp2 = multifit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4,) def test_ulmfit_fastai_end_to_end(): @@ -112,7 +105,7 @@ def test_ulmfit_fastai_end_to_end(): test_data, wt2 = get_test_data() lm_name = 'end-to-end-test-fastai' - exp = ulmfit.pretrain_lm.LMHyperParams( + exp = multifit.pretrain_lm.LMHyperParams( dataset_path=wt2, lang='en', cuda_id=cuda_id, @@ -123,7 +116,7 @@ def test_ulmfit_fastai_end_to_end(): name=lm_name, ) exp.train_lm(num_epochs=1, bs=2) - exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) + exp2 = multifit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, ) def test_ulmfit_fastai_end_to_end_label_smoothing(): @@ -132,7 +125,7 @@ def test_ulmfit_fastai_end_to_end_label_smoothing(): test_data, wt2 = get_test_data() lm_name = 'end-to-end-test-fastai-lablel-smoothing' - exp = ulmfit.pretrain_lm.LMHyperParams( + exp = multifit.pretrain_lm.LMHyperParams( dataset_path=wt2, lang='en', cuda_id=cuda_id, @@ -142,7 +135,7 @@ def test_ulmfit_fastai_end_to_end_label_smoothing(): name=lm_name, ) exp.train_lm(num_epochs=1, bs=2, label_smoothing_eps=0.1) - exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir, name=lm_name) + exp2 = multifit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir, name=lm_name) exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, label_smoothing_eps=0.1 ) def test_ulmfit_sentencepiece_end_to_end(): @@ -151,18 +144,18 @@ def test_ulmfit_sentencepiece_end_to_end(): test_data, wt2 = get_test_data() lm_name = 'end-to-end-test-spm' - exp = ulmfit.pretrain_lm.LMHyperParams( + exp = multifit.pretrain_lm.LMHyperParams( dataset_path=wt2, lang='en', cuda_id=cuda_id, qrnn=False, - tokenizer=ulmfit.pretrain_lm.Tokenizers.SUBWORD, + tokenizer=multifit.pretrain_lm.Tokenizers.SUBWORD, max_vocab=200, name=lm_name, ) exp.train_lm(num_epochs=1, bs=2) # not supported yet - exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) + exp2 = multifit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, ) def test_ulmfit_sentencepiece_fastai_impl_end_to_end(): @@ -171,18 +164,18 @@ def test_ulmfit_sentencepiece_fastai_impl_end_to_end(): test_data, wt2 = get_test_data() lm_name = 'end-to-end-test-spm-fa' - exp = ulmfit.pretrain_lm.LMHyperParams( + exp = multifit.pretrain_lm.LMHyperParams( dataset_path=wt2, lang='en', cuda_id=cuda_id, qrnn=False, - tokenizer=ulmfit.pretrain_lm.Tokenizers.FASTAI_SUBWORD, + tokenizer=multifit.pretrain_lm.Tokenizers.FASTAI_SUBWORD, max_vocab=200, name=lm_name, ) exp.train_lm(num_epochs=1, bs=2) # not supported yet - exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) + exp2 = multifit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, ) if __name__ == "__main__": diff --git a/tests/test_text_data.py b/tests/test_text_data.py deleted file mode 100644 index 1b96079..0000000 --- a/tests/test_text_data.py +++ /dev/null @@ -1,56 +0,0 @@ -import pytest -import fastai.text - -from fastai import * -from fastai.text import * - -import fastai_contrib.data as contrib_data - -def text_df(labels): - data = [] - texts = ["fast ai is a cool project", "hello world"] * 20 - for ind, text in enumerate(texts): - sample = {} - sample["label"] = labels[ind%len(labels)] - sample["text"] = text - data.append(sample) - return pd.DataFrame(data) - -###################### UPDATED CODE -def test_should_load_backwards_lm(): - path = untar_data(URLs.IMDB_SAMPLE) - df = text_df(['neg','pos']) - - data = TextLMDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=0, text_cols=["text"], bs=2, - lm_type=contrib_data.LanguageModelType.BwdLM) - lml = data.train_dl.dl - lml.data = lml.batchify(np.concatenate([lml.dataset.x.items[i] for i in range(len(lml.dataset))])) - batch = lml.get_batch(lml.data, 0, 70) - - assert batch[0].shape == (lml.bs, 70) - assert batch[1].shape == (70*lml.bs,) - - as_text = [lml.dataset.vocab.itos[x] for x in batch[0][0]] - np.testing.assert_array_equal(as_text[:5], ["world", "hello", 'xxbos', 'project', 'cool']) - -def test_should_load_bi_lm(): - path = untar_data(URLs.IMDB_SAMPLE) - df = text_df(['neg', 'pos']) - - data = TextLMDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=0, text_cols=["text"], bs=2, - lm_type=contrib_data.LanguageModelType.BiLM) - lml = data.train_dl.dl - lml.data = lml.batchify(np.concatenate([lml.dataset.x.items[i] for i in range(len(lml.dataset))])) - batch = lml.get_batch(lml.data, 0, 70) - - assert batch[0].shape == (lml.bs, 70, 2) - assert batch[1].shape == (70*lml.bs, 2) - - as_text = [lml.dataset.vocab.itos[x] for x in batch[0][0, :, 0]] - np.testing.assert_array_equal(as_text[:7], "xxbos fast ai is a cool project".split()) - - as_text = [lml.dataset.vocab.itos[x] for x in batch[0][0, :, 1]] - np.testing.assert_array_equal(as_text[:5], ["world", "hello", 'xxbos', 'project', 'cool']) - -###################### NEW CODE - diff --git a/tests/test_text_train.py b/tests/test_text_train.py deleted file mode 100644 index 7c3d894..0000000 --- a/tests/test_text_train.py +++ /dev/null @@ -1,103 +0,0 @@ -import pytest -from fastai import * -from fastai.text import * - -pytestmark = pytest.mark.integration - -print(sys.path) -import fastai_contrib.data as contrib_data - -from fastai_contrib.learner import bilm_learner, accuracy_fwd, bilm_text_classifier_learner - - -def read_file(fname): - texts = [] - with open(fname, 'r') as f: - texts = f.readlines() - labels = [0] * len(texts) - df = pd.DataFrame({'labels':labels, 'texts':texts}, columns = ['labels', 'texts']) - return df - -def prep_human_numbers(): - path = untar_data(URLs.HUMAN_NUMBERS) - df_trn = read_file(path/'train.txt') - df_val = read_file(path/'valid.txt') - return path, df_trn, df_val - -def manual_seed(seed=42): - torch.manual_seed(seed) - np.random.seed(seed) - if torch.cuda.is_available(): - torch.cuda.manual_seed_all(seed) - torch.backends.cudnn.deterministic = True - torch.backends.cudnn.benchmark = False - -@pytest.fixture(scope="module") -def learn(): - path, df_trn, df_val = prep_human_numbers() - data = TextLMDataBunch.from_df(path, df_trn, df_val, tokenizer=Tokenizer(BaseTokenizer)) - learn = language_model_learner(data, emb_sz=100, nl=1, drop_mult=0.1) - learn.fit_one_cycle(4, 5e-3) - return learn - -def text_df(n_labels): - data = [] - texts = ["fast ai is a cool project", "hello world"] * 20 - for ind, text in enumerate(texts): - sample = {} - for label in range(n_labels): sample[label] = ind%2 - sample["text"] = text - data.append(sample) - df = pd.DataFrame(data) - return df - -###################### NEW CODE - -def test_val_loss(learn): - assert learn.validate()[1] > 0.5 - - -def test_bilm_classifier_loads_encoder(): - n_labels=1 - nl = 1 - emb_sz = 100 - path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data', 'tmp') - os.makedirs(path) - try: - df = text_df(n_labels=n_labels) - lmdf = df#[["text"]] - print(lmdf.head()) - lmdata = TextLMDataBunch.from_df(path, lmdf, lmdf, tokenizer=Tokenizer(BaseTokenizer), - lm_type=contrib_data.LanguageModelType.BiLM) - learn = bilm_learner(lmdata, emb_sz=emb_sz, nl=nl, drop_mult=0.1, qrnn=False) - learn.save_encoder("enc") - data = TextClasDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=list(range(n_labels)), text_cols=["text"], bs=8) - classifier = bilm_text_classifier_learner(data, emb_sz=emb_sz, nl=nl, drop_mult=0.1, qrnn=False) - print(last_layer(classifier.model), ) - classifier.load_encoder("enc") - classifier.fit(1) - finally: - shutil.rmtree(path) - - -def test_bilm_lstm_can_be_trained(): - manual_seed() - path, df_trn, df_val = prep_human_numbers() - data = TextLMDataBunch.from_df(path, df_trn, df_val, tokenizer=Tokenizer(BaseTokenizer), - lm_type = contrib_data.LanguageModelType.BiLM) - - learn = bilm_learner(data, emb_sz=100, nl=1, drop_mult=0.1, qrnn=False) - learn.metrics = [accuracy_fwd] - learn.fit_one_cycle(2, 5e-3) - assert learn.validate()[1] > 0.3 - - -def test_bwdlm_lstm_can_be_trained(): - manual_seed() - path, df_trn, df_val = prep_human_numbers() - data = TextLMDataBunch.from_df(path, df_trn, df_val, tokenizer=Tokenizer(BaseTokenizer), - lm_type = contrib_data.LanguageModelType.BwdLM) - - learn = language_model_learner(data, emb_sz=100, nl=1, drop_mult=0.1, qrnn=False) - learn.fit_one_cycle(2, 5e-3) - assert learn.validate()[1] > 0.3 diff --git a/ulmfit/__init__.py b/ulmfit/__init__.py index 8b13789..10c1209 100644 --- a/ulmfit/__init__.py +++ b/ulmfit/__init__.py @@ -1 +1,3 @@ - +from .datasets import Dataset, ULMFiTDataset +from .training import ULMFiT +from .configurations import * \ No newline at end of file diff --git a/ulmfit/configurations.py b/ulmfit/configurations.py new file mode 100644 index 0000000..4f82561 --- /dev/null +++ b/ulmfit/configurations.py @@ -0,0 +1,39 @@ +from .training import * + +def ulmfit_orig(): + raise NotImplementedError("TODO move hyper params") + +def multifit_paper_version(): + raise NotImplementedError("TODO move hyper params") + +def multifit_fp32(bs=64): + self = ULMFiT() + self.replace_( + label_smoothing_eps=0.0, + true_wd=True, + wd=0.1, + seed=0, + fp16=False, + bs=bs, + use_adam_08=False, + name=multifit_fp32.__name__ + ) + self.arch.replace_( + tokenizer='fsp', + max_vocab=15000, + qrnn=True, + n_layers=4, + n_hid=1552 + ) + self.pretrain_lm.replace_(num_epochs=10, drop_mult=0.5, lr=(1e-2 * bs / 48)) + self.finetune_lm.replace_(num_epochs=10, drop_mult=1.0, lr=(1e-3 * bs / 48)) + self.classifier.replace_(num_epochs=8, drop_mult=0.5, bs=20, label_smoothing_eps=0.1) + return self + +def multifit_fp16(): + return multifit_fp32(bs=128).replace_(fp16=True, name=multifit_fp16.__name__) + +def multifit_lstm(): + return multifit_fp32(bs=128).replace_(qrnn=False, n_hid=1552, name=multifit_lstm.__name__) + + diff --git a/ulmfit/datasets/__init__.py b/ulmfit/datasets/__init__.py index 8b13789..e4c18c8 100644 --- a/ulmfit/datasets/__init__.py +++ b/ulmfit/datasets/__init__.py @@ -1 +1,312 @@ +import pathlib +from dataclasses import asdict +from string import Template +import fastai + +from fastai import * +from fastai.callbacks import CSVLogger +import fastai.text +from fastai.text import * +import torch +from ulmfit.datasets.utils import read_whitespace_file, \ + validate, UNK +from fastai_contrib.text_data import MosesPreprocessingFunc, get_sentencepiece, get_sentencepiece_fastai, \ + make_data_bunch_from_df +import pickle + +from pathlib import Path + + +def istitle(line): + return len(re.findall(r'^ ?= [^=]* = ?$', line)) != 0 + + +def read_wiki_articles(filename): + articles = [] + + with open(filename, encoding='utf8') as f: + lines = f.readlines() + current_article = [] + for i, line in enumerate(lines): + current_article.append(line) + if i < len(lines) - 2 and lines[i + 1].strip() == "" and istitle(lines[i + 2]): + articles.append("".join(current_article)) + current_article = [] + articles.append("".join(current_article)) + print(f"Wiki text was split to {len(articles)} articles") + df = pd.DataFrame({'0': np.zeros(len(articles)), 'texts': np.array(articles, dtype=np.object)}) + if len(df.columns) == 1: + df.insert(0, 'label', 0) + return df + + +def read_clas_csv(fn): + df = pd.read_csv(fn, header=None).fillna("na") + if len(df.columns) == 1: + df.insert(0, 'label', 0) + return df + + +@dataclass +class Dataset: + dataset_path: Path + use_tst_for_lm: bool = True + noise: float = 0.0 + limit: int = None + + def __post_init__(self): + self.add_trn_to_lm = True + self._trn_df = None + self._tst_df = None + self._val_df = None + + if 'wiki' in str(self.dataset_path) and len(list(self.dataset_path.glob('*.wiki.*.tokens'))) >= 2: + self._post_init_tokenized_wiki() + elif 'reddit' in str(self.dataset_path): + self._post_init_default_csv( + lang='en', + uses_moses=False, + add_trn_to_lm=True, + use_lang_as_prefix=False) + + elif 'xnli' in str(self.dataset_path): + raise NotImplementedError("Support for XNLI is not implemented yet") + elif 'imdb' in self.dataset_path.name: + self._post_init_default_csv( + lang='en', + uses_moses=True, + add_trn_to_lm=True, + use_lang_as_prefix=False) + elif 'mldoc' in str(self.dataset_path): + self._post_init_default_csv( + lang=self._language_from_dataset_path(), + uses_moses=False, + add_trn_to_lm=False, + use_lang_as_prefix=True) + elif 'hate' in str(self.dataset_path): + self._post_init_default_csv( + lang=self._language_from_dataset_path(), + uses_moses=False, + add_trn_to_lm=True, + use_lang_as_prefix=True) + else: + raise NotImplementedError(f"Not supported dataset {self.dataset_path}") + + def _post_init_default_csv(self, lang, uses_moses, add_trn_to_lm, use_lang_as_prefix): + self.lang = lang + self.uses_moses = uses_moses + self.add_trn_to_lm = add_trn_to_lm + self.use_tst_for_lm = False + self.label_column = 0 + + prefix = f"{self.lang}." if use_lang_as_prefix else "" + + self.trn_path = self.dataset_path / f'{prefix}train.csv' + self.val_path = self.dataset_path / f'{prefix}dev.csv' + + self._read_data = read_clas_csv + + self.trn_path = self.dataset_path / f'{prefix}train.csv' + self.val_path = self.dataset_path / f'{prefix}dev.csv' + self.tst_path = self.dataset_path / f'{prefix}test.csv' + self.unsup_path = self.dataset_path / f'{prefix}unsup.csv' + + def _post_init_tokenized_wiki(self): + self.uses_moses = True + self.use_tst_for_lm = False + self.add_trn_to_lm = True + self.lang = self._language_from_dataset_path() + + self._read_data = read_wiki_articles + + self.trn_path = self.dataset_path / f'{self.lang}.wiki.train.tokens' + self.val_path = self.dataset_path / f'{self.lang}.wiki.valid.tokens' + self.tst_path = self.dataset_path / f'{self.lang}.wiki.test.tokens' + self.unsup_path = self.dataset_path / f'{self.lang}.wiki.unsup.tokens' + + def _language_from_dataset_path(self): + lang, size = self.dataset_path.name.split('-') + return lang + + def _load_n_cache_supervised_data(self): + if not self._trn_df is not None or not self._tst_df is not None or not self._val_df is not None: + trn_df = self._read_data(self.trn_path) + tst_df = self._read_data(self.tst_path) + val_df = self._read_data(self.val_path) if self.val_path.exists() else None + + if val_df is None: + print("Validation set not found using 10% of trn") + val_len = max(int(len(trn_df) * 0.1), 2) + trn_len = len(trn_df) - val_len + trn_df, val_df = trn_df[:trn_len], trn_df[trn_len:] + + self._trn_df, self._val_df, self._tst_df = trn_df, val_df, tst_df + + return self._trn_df, self._val_df, self._tst_df + + def load_supervised_data(self): + trn_df, val_df, tst_df = self._load_n_cache_supervised_data() + if self.noise > 0.0 is not None: + trn_df = self._add_noise(trn_df, self.noise) + val_df = self._add_noise(val_df, self.noise) + + if self.limit is not None: + print("Limiting data set to:", self.limit) + trn_df = trn_df[:self.limit] + val_df = val_df[:self.limit] + + return trn_df, val_df, tst_df + + def load_unsupervised_data(self): + trn_df, val_df, tst_df = self._load_n_cache_supervised_data() + unsup_df = self._read_data(self.unsup_path) if self.unsup_path.exists() else None + lm_trn_df = pd.concat( + ([trn_df] if self.add_trn_to_lm else []) + + ([unsup_df] if unsup_df is not None else []) + + ([tst_df] if self.use_tst_for_lm else [])) + + # val_len = max(int(len(lm_trn_df) * 0.1), 2) + # lm_trn_df = lm_trn_df[val_len:] + # lm_val_df = lm_trn_df[:val_len] + + return lm_trn_df, val_df + + def _add_noise(self, trn_df, noise): + count = len(trn_df) + labels = trn_df[0].unique() + assert np.issubdtype(labels.dtype, np.integer), "noise only works on numerical numbers" + modulo = labels.max() + 1 + idx_to_distrub = np.random.permutation(count)[:int(count * noise)] + trn_df.loc[idx_to_distrub, [0]] = (np.random.randint(1, modulo - 1, size=len(idx_to_distrub)) + + trn_df.loc[idx_to_distrub][0]) % modulo + print( + f"Added noise to {len(idx_to_distrub)} examples, only {(count - len(idx_to_distrub)) / count} have correct labels") + return trn_df + + +@dataclass +class ULMFiTDataset(Dataset): + tokenizer: str = 'f' + max_vocab: int = 60000 + + def __post_init__(self): + super().__post_init__() + tokenizer_prefix = f"{self.tokenizer}{self.max_vocab // 1000}k" + self.cache_path = self.dataset_path / "models" / tokenizer_prefix + self._vocab = None + + def use_base_model_subword_vocabulary(self, base_lm_path: Path): + """ + In case of subwoard vocabularies reuse the base model vocabulary during tokenization. + For word tokenization we still generate new vocabulary for each dataset, + and we expect finetuning to handle the conversion + """ + # reuse base model sentencepiece vocabulary + self.cache_path.mkdir(exist_ok=True) + if (base_lm_path / '..' / 'spm.vocab').exists() and (base_lm_path.parent.resolve() != self.cache_path.resolve()): + shutil.copy(str(base_lm_path / '..' / 'itos.pkl'), str(self.cache_path)) + shutil.copy(str(base_lm_path / '..' / 'spm.model'), str(self.cache_path)) + shutil.copy(str(base_lm_path / '..' / 'spm.vocab'), str(self.cache_path)) + + # TODO: implement / maybe put the vocabulary md5 to the file names and keep spm models together? + # sp12k/7599013a8ce538b2e3d4405684221ecaf26bcba1.lm + # sp12k/7599013a8ce538b2e3d4405684221ecaf26bcba1-vocab.link + # then we don't need the use_vocabulary, we can just use load_lm_databunch(using_vacab=XXX) + # we could put spm.model and spm.vocab to the model folder it self then, and copy /link it when we use the orignal model + # for the time being we can simply compy the spm.model on the right spot and raise an error ir the two are different? + + def load_lm_databunch(self, bs, bptt): + lm_suffix = bptt if bptt != 70 else "" + lm_suffix += self.use_tst_for_lm if "" else "-notst" + data_lm = self._databunch(f"lm{lm_suffix}", + bunch_class=TextLMDataBunch, + data_loader=self.load_unsupervised_data, + bptt=bptt, + bs=bs) + + with (self.cache_path / "itos.pkl").open('wb') as f: + pickle.dump(data_lm.vocab.itos, f) + self._vocab = data_lm.vocab + + print('Size of vocabulary:', len(data_lm.vocab.itos)) + print('First 20 words in vocab:', data_lm.vocab.itos[:20]) + + return data_lm + + def load_vocab(self): + if self._vocab is None: + self._vocab = self.load_lm_databunch(bs=20, bptt=70).vocab + return self._vocab + + def load_clas_databunch(self, bs): + vocab = self.load_vocab() + + cls_name = "cls" + if self.limit is not None: + cls_name = f'{cls_name}limit{self.limit}' + if self.noise > 0.0: + cls_name = f'{cls_name}noise{self.noise}' + + args = dict(vocab=vocab, bunch_class=TextClasDataBunch, bs=bs) + data_cls = self._databunch(cls_name, data_loader=lambda: self.load_supervised_data()[:2], **args) + # Hack to load test dataset with labels + data_tst = self._databunch('tst', data_loader=lambda: self.load_supervised_data()[1:], **args) + return data_cls, data_tst + + def _databunch(self, name, bunch_class, data_loader, bs, **args): + bunch_path = self.cache_path / name + if bunch_path.exists(): + databunch = load_data(self.cache_path, name, bs=bs) + else: + print(f"Running tokenization {name}...") + args.update(**self._get_processor(ds_need_moses=not self.uses_moses)) #TODO depends on the previous model + train_df, valid_df = data_loader() + databunch = make_data_bunch_from_df(cls=bunch_class, + path=self.cache_path, + train_df=train_df, + valid_df=valid_df, + max_vocab=self.max_vocab, + mark_fields=True, + text_cols=list(train_df.columns.values)[1:], + **args) + databunch.save(name) + print(f"Data {name}, trn: {len(databunch.train_ds)}, val: {len(databunch.valid_ds)}") + return databunch + + def _get_processor(self, ds_need_moses): + return { + 'fsp': self._get_processor_sentence_piece, + 'f': self._get_processor_pure_fastai, + 'm': self._get_processor_pure_moses, + 'mf': self._get_processor_moses_fastai, + + 'sp': self._get_processor_sentence_piece, # deprecated + 'v': self._get_processor_pure_moses, # deprecated + 'vf': self._get_processor_moses_fastai, # deprecated + }.get(self.tokenizer)(ds_need_moses) + + def _get_processor_sentence_piece(self, ds_need_moses): + moses_preproc = [MosesPreprocessingFunc(self.lang)] if ds_need_moses else [] + return get_sentencepiece_fastai( + cache_dir=self.cache_path, + vocab_size=self.max_vocab, + lang=self.lang, + pre_rules=moses_preproc + defaults.text_pre_rules) + + def _get_processor_pure_moses(self, ds_need_moses): + moses_preproc = [MosesPreprocessingFunc(self.lang)] if ds_need_moses else [] + return dict(tokenizer=Tokenizer(tok_func=BaseTokenizer, + lang=self.lang, + pre_rules=moses_preproc, + post_rules=[])) + + def _get_processor_moses_fastai(self, ds_need_moses): + moses_preproc = [MosesPreprocessingFunc(self.lang)] if ds_need_moses else [] + return dict(tokenizer=Tokenizer(tok_func=BaseTokenizer, + lang=self.lang, + pre_rules=moses_preproc + defaults.text_pre_rules, + post_rules=defaults.text_post_rules)) + + def _get_processor_pure_fastai(self, ds_need_moses): + return dict() diff --git a/ulmfit/datasets/create_wikitext.py b/ulmfit/datasets/create_wikitext.py index 6ba156c..14b5f11 100644 --- a/ulmfit/datasets/create_wikitext.py +++ b/ulmfit/datasets/create_wikitext.py @@ -102,7 +102,7 @@ def wiki2csv(file_path, text_iter, num_tokens): print('Processed {:,} documents. Total # tokens: {:,}.'.format(i, total_num_tokens)) -def main2(args): +def main(args): input_path = Path(args.input) output = Path(args.output) @@ -139,19 +139,19 @@ def main2(args): copyfile(lrg_wiki_train, all_wiki_train) write_wikitext(all_wiki_train, text_iter, mt, None, mode='a') -def main(args): - - input_path = Path(args.input) - output = Path(args.output) - assert input_path.exists(), f'Error: {input_path} does not exist.' - output.mkdir(exist_ok=True) - - lrg_wiki = output / f'{args.lang}-100' - lrg_wiki.mkdir(exist_ok=True) - - text_iter = get_texts(input_path) - - wiki2csv(lrg_wiki / "rawtexts.csv", text_iter, int(2e7)) +# def main(args): +# +# input_path = Path(args.input) +# output = Path(args.output) +# assert input_path.exists(), f'Error: {input_path} does not exist.' +# output.mkdir(exist_ok=True) +# +# lrg_wiki = output / f'{args.lang}-100' +# lrg_wiki.mkdir(exist_ok=True) +# +# text_iter = get_texts(input_path) +# +# wiki2csv(lrg_wiki / "rawtexts.csv", text_iter, int(2e7)) if __name__ == '__main__': diff --git a/ulmfit/datasets/postprocess_wikitext.py b/ulmfit/datasets/postprocess_wikitext.py index 7ae2c67..6bac2c4 100644 --- a/ulmfit/datasets/postprocess_wikitext.py +++ b/ulmfit/datasets/postprocess_wikitext.py @@ -10,7 +10,7 @@ from pathlib import Path import fire -from multifit.datasets.utils import replace_number, UNK +from .utils import replace_number, UNK def build_vocab(file_path, cutoff=3): diff --git a/ulmfit/training.py b/ulmfit/training.py new file mode 100644 index 0000000..3639302 --- /dev/null +++ b/ulmfit/training.py @@ -0,0 +1,440 @@ +import dataclasses +from fastai.callbacks import CSVLogger, SaveModelCallback +from fastai.text import * +import torch + +from ulmfit.datasets import ULMFiTDataset +from pathlib import Path + +CLS_BEST = 'cls_best' +LM_BEST = "lm_best" +ENC_BEST = "enc_best" + + +@dataclass +class Params: + def replace_(self, **changes): + for f in dataclasses.fields(self): + if f.name in changes: + v = changes[f.name] + if f.type == Path and v is not None: + v = Path(v) + setattr(self, f.name, v) + return self + + +@dataclass +class ULMFITArchitecture(Params): + tokenizer: str = "f" + max_vocab: int = 60000 + + emb_sz: int = awd_lstm_lm_config['emb_sz'] + n_hid: int = awd_lstm_lm_config['n_hid'] + n_layers: int = awd_lstm_lm_config['n_layers'] + qrnn: bool = awd_lstm_lm_config['qrnn'] + + def model_name(self, name): + model_suffix = '' # if self.lmseed is None else f'_lmseed-{self.lmseed}' + model_prefix = 'qrnn' if self.qrnn else 'lstm' + + model_name = f"{model_prefix}_{name}{model_suffix}.m" + return model_name + + def dataset_cache_suffix(self): + tokenizer_prefix = f"{self.tokenizer}{self.max_vocab // 1000}k" + return f'models/{tokenizer_prefix}' + + def dataset(self, dataset_path, **args): + return ULMFiTDataset(dataset_path=dataset_path, tokenizer=self.tokenizer, max_vocab=self.max_vocab, **args) + + +def set_seed(seed, name): + if seed is not None: + print(f"Setting {name} seed to {seed}") + torch.manual_seed(seed) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + np.random.seed(seed) + + +def to_json_serializable(d): + n = {} + for k, v in d.items(): + if isinstance(v, dict): + n[k] = to_json_serializable(v) + elif isinstance(v, (float, int, str, list, tuple)): + n[k] = v + elif v is None: + n[k] = v + else: + n[k] = str(v) + return n + + +@dataclass +class ULMFiTTrainingCommand(Params): + seed: int = 0 + name: str = None + arch: ULMFITArchitecture = field(repr=False, default=None) + experiment_path: Path = None + + @property + def model_name(self): + return (self.name or self.arch.model_name()) + ( + "" if self.seed == 0 or "seed" in self.name else f"seed{self.seed}") + + @property + def info_json(self): + return self.__class__.__name__.lower().replace("ulmfit", "") + ".json" + + def save_paramters(self): + params = dataclasses.asdict(self) + base_exp_path = params.pop('base', {}).pop('experiment_path', None) + params['base'] = base_exp_path + exp_path = params.get('experiment_path', None) + if exp_path: + fn = self.info_json + print("Saving dump to", exp_path / fn) + json_str = json.dumps(to_json_serializable(params), indent=2) + with (exp_path / fn).open("w") as f: + f.write(json_str) + return json_str + + def load_(self, experiment_path, tantetive=True): + fn = experiment_path / self.info_json + if not fn.exists(): + if not tantetive: + warn(f"Unable to load experiment_path {experiment_path}") + return False + print(f"Loading {fn}") + with fn.open('r') as f: + d = json.load(f) + base = d.pop('base', None) + arch = d.pop('arch') + self.arch.replace_(**arch) + self.replace_(**d) + if base is not None: + other_arch = getattr(self, 'base').load_(Path(base), tantetive=True) + if not other_arch == arch: + warn(f"architecuture does not match {arch}, {other_arch}") + self.name = experiment_path.name + return arch + + +@dataclass +class ULMFiTPretraining(ULMFiTTrainingCommand): + num_epochs: int = 10 + bs: int = 20 + bptt: int = 70 + drop_mult: float = 1.0 + label_smoothing_eps: float = 0.0 + use_adam_08: bool = False + true_wd: bool = True + wd: bool = 0.1 + fp16: bool = False + lr: float = 5e-3 + + def _learner(self, dataset, **additional_trn_args): + config = awd_lstm_lm_config.copy() + config.update(emb_sz=self.arch.emb_sz, n_hid=self.arch.n_hid, n_layers=self.arch.n_layers, qrnn=self.arch.qrnn) + + trn_args = dict(drop_mult=self.drop_mult, true_wd=self.true_wd, wd=self.wd, + pretrained=False) + trn_args.update(**additional_trn_args) + print("Training args: ", trn_args, "config: ", config) + data_lm = dataset.load_lm_databunch(bs=self.bs, bptt=self.bptt) + learn = language_model_learner(data_lm, + AWD_LSTM, + config=config, + model_dir=self.model_name, + **trn_args) + + # compared to standard Adam, we set beta_1 to 0.8 + if self.use_adam_08: + learn.opt_func = partial(optim.Adam, betas=(0.8, 0.99)) + + learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/lm-history")] + if self.label_smoothing_eps > 0.0: + learn.loss_func = FlattenedLoss(LabelSmoothingCrossEntropy, eps=self.label_smoothing_eps) + + set_seed(self.seed, "LM training seed") + if self.fp16: + learn.to_fp16() + return learn + + def _fit_schedule(self, learn): + print("Training lm from random weights") + learn.unfreeze() + learn.fit_one_cycle(self.num_epochs, self.lr, (0.8, 0.7)) + + def train_(self, dataset, **train_config): + self.replace_(**train_config, _strict=True) + set_seed(self.seed, "LM weights seed") + if hasattr(self, 'base'): + dataset.use_base_model_subword_vocabulary(self.base.experiment_path) + learn = self._learner(dataset) + experiment_path = learn.path / learn.model_dir + print("Experiment", experiment_path) + if self.num_epochs > 0: + self._fit_schedule(learn) + self.experiment_path = experiment_path + learn.to_fp32() + learn.save_encoder(ENC_BEST) + learn.save(LM_BEST, with_opt=False) + learn.destroy() + print("Language model saved to", self.experiment_path) + self.save_paramters() + + def validate(self): + raise NotImplementedError("The validation on the language model is not implemented.") + + @property + def model_fnames(self): + if self.experiment_path: + return [self.experiment_path.absolute() / LM_BEST, self.experiment_path.parent.absolute() / 'itos'] + return None + + @property + def encoder_fname(self): + if self.experiment_path: + return (self.experiment_path / ENC_BEST).absolute() + return None + + +@dataclass +class ULMFiTFinetuining(ULMFiTPretraining): + base: ULMFiTPretraining = field(repr=False, default=None) + pretrained: bool = True + + def __post_init__(self): + self.lr = 1e-3 + + def _learner(self, dataset, **additional_trn_args): + pretrained_fnames = None if self.base is None else self.base.model_fnames + if self.pretrained and pretrained_fnames is None and dataset.lang != 'en': + warn( + "You are using fastai english langauge model for {data_lm.lang}, you might be better off with just random weights.") + return super()._learner(dataset, pretrained=self.pretrained, pretrained_fnames=pretrained_fnames, + **additional_trn_args) + + def _fit_schedule(self, learn): + if self.pretrained: + print("Fitting using 2 cycle fit schedule") + learn.freeze_to(-1) + learn.fit_one_cycle(1, self.lr * 10, moms=(0.8, 0.7)) + learn.unfreeze() + learn.fit_one_cycle(self.num_epochs, self.lr, moms=(0.8, 0.7)) + else: + super()._fit_schedule(learn) + + +@dataclass +class ULMFiTClassifier(ULMFiTTrainingCommand): + bs: int = 20 + num_epochs: int = 10 + drop_mult: float = 0.5 + wd: float = 0.1 + label_smoothing_eps: float = 0.0 + weighted_cross_entropy: tuple = None + early_stopping: str = 'accuracy' + fit_schedule: str = '1cycle' + base: ULMFiTFinetuining = field(repr=False, default=None) + random_init: bool = False + seed: int = 0 + bptt: int = 70 + arch: ULMFITArchitecture = None + dataset_path: Path = None + fp16: bool = False + + def _learner(self, dataset, eval_only=False, **additional_trn_args): + assert self.weighted_cross_entropy is None or self.label_smoothing_eps == 0, "Label smoohting not implemented with weighted_cross_entropy" + if self.weighted_cross_entropy is not None: + loss_func = CrossEntropyFlat(weight=torch.tensor(self.weighted_cross_entropy, dtype=torch.float32).cuda()) + elif self.label_smoothing_eps > 0.0: + loss_func = FlattenedLoss(LabelSmoothingCrossEntropy, eps=self.label_smoothing_eps) + else: + loss_func = None + + set_seed(self.seed, "Classifier weights seed") + data_clas, data_tst = dataset.load_clas_databunch(bs=self.bs) + config = awd_lstm_clas_config.copy() + config.update(emb_sz=self.arch.emb_sz, n_hid=self.arch.n_hid, n_layers=self.arch.n_layers, qrnn=self.arch.qrnn) + + trn_args = dict(drop_mult=self.drop_mult, wd=self.wd, pretrained=False, bptt=self.bptt, + loss_func=loss_func) + + trn_args.update(**additional_trn_args) + print("Training args: ", trn_args, "config: ", config) + learn = text_classifier_learner(data_clas, + AWD_LSTM, + config=config, + model_dir=self.model_name, + silent=eval_only, + **trn_args) + learn.data.test_dl = data_tst.valid_dl + if self.base and not self.random_init: + print("Loading pretrained model", self.base.encoder_fname) + learn.load_encoder(self.base.encoder_fname) + learn.freeze() + + set_seed(self.seed, "Classifier training seed") + if not eval_only: + learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/cls-history")] + if self.early_stopping: + learn.callback_fns += [partial(SaveModelCallback, every='improvement', + name='cls_best_tmp', + monitor=self.early_stopping)] + if self.fp16: + learn.to_fp16() + return learn + + def train_(self, dataset, **train_config): + self.replace_(**train_config, _strict=True) + dataset.use_base_model_subword_vocabulary(self.base.experiment_path) + learn = self._learner(dataset) + + self._fit_schedule(learn) + + self.experiment_path = learn.path / learn.model_dir + learn.to_fp32() + learn.save(CLS_BEST, with_opt=False) + print("Classifier model saved to", self.experiment_path) + self.save_paramters() + learn.destroy() + + def _validate(self, learn, ds_type): + ds_name = ds_type.name.lower() + print(f"Model: {self.name}, ds_name: {ds_name}") + results_dict = dict(zip( + [f'{ds_name} loss'] + [f"{ds_name} {getattr(m, '__name__', m.__class__.__name__)}" for m in learn.metrics], + map(float, learn.validate(learn.data.dl(ds_type))))) + results_dict['name'] = self.name + return results_dict + + def validate(self, dataset, save_name=CLS_BEST, use_cache=True): + cache_file = (self.experiment_path / f'results{"" if save_name == CLS_BEST else "-" + save_name}.json') + if use_cache and cache_file.exists(): + with cache_file.open("r") as fp: + return json.load(fp) + + learn = self._learner(dataset, eval_only=True) + avg = 'binary' if learn.data.c == 2 else 'macro' + learn.metrics = [FBeta(beta=1.0, average=avg), Precision(average=avg), Recall(average=avg), accuracy] + learn.load(save_name) + + probs, targets = learn.get_preds(ordered=True, ds_type=DatasetType.Test, activ=partial(F.softmax, dim=-1)) + np.save(str(self.experiment_path / f"preds-on-test.npy"), probs.cpu().numpy()) + + results_dict = self._validate(learn, DatasetType.Test) + results_dict.update(self._validate(learn, DatasetType.Valid)) + results_dict.update(self._validate(learn, DatasetType.Train)) + print(results_dict) + with cache_file.open("w") as fp: + json.dump(results_dict, fp) + return results_dict + + def _fit_schedule(self, learn): + getattr(self, '_fit_schedule_' + self.fit_schedule)(learn) + + def _fit_schedule_1cycle(self, learn): + print("Single training schedule") + learn.unfreeze() + learn.fit_one_cycle(self.num_epochs, slice(1e-2 / (2.6 ** 4), 2e-2), moms=(0.8, 0.7)) + + def _fit_schedule_layered(self, learn): + learn.freeze_to(-1) + learn.fit_one_cycle(1, 2e-2, moms=(0.8, 0.7)) + if self.num_epochs > 1: + learn.freeze_to(-2) + learn.fit_one_cycle(1, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7)) + learn.freeze_to(-3) + learn.fit_one_cycle(1, slice(5e-3 / (2.6 ** 4), 5e-3), moms=(0.8, 0.7)) + learn.unfreeze() + if self.num_epochs > 5: + learn.fit_one_cycle(self.num_epochs - 4, slice(1e-3 / (2.6 ** 4), 1e-3), moms=(0.8, 0.7)) + + def _fit_schedule_2cycle(self, learn): + print("2cycle training schedule") + learn.freeze_to(-1) + learn.fit_one_cycle(1, 2e-2, moms=(0.8, 0.7)) + learn.unfreeze() + if self.num_epochs > 1: + learn.fit_one_cycle(self.num_epochs - 1, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7)) + + def _fit_schedule_reverse_2cycle(self, learn): + print("Reverse 2cycle ") + learn.unfreeze() + for g in learn.layer_groups[-1:]: + for l in g: + if not learn.train_bn or not isinstance(l, bn_types): requires_grad(l, False) + learn.create_opt(defaults.lr) + print("training LM") + learn.fit_one_cycle(self.num_epochs, slice(1e-2 / (2.6 ** 4), 2e-2), moms=(0.8, 0.7)) + learn.unfreeze() + print("training ALL") + learn.fit_one_cycle(self.num_epochs, slice(1e-3 / (2.6 ** 4), 2e-3), moms=(0.8, 0.7)) + + def _fit_schedule_false_wd(self, learn): + learn.true_wd = False + print("Starting classifier training") + learn.fit_one_cycle(1, 5e-2, moms=(0.8, 0.7), wd=1e-7) + if self.num_epochs > 1: + learn.freeze_to(-2) + learn.fit_one_cycle(1, slice(5e-2 / (2.6 ** 4), 5e-2), moms=(0.8, 0.7), wd=1e-7) + learn.freeze_to(-3) + learn.fit_one_cycle(1, slice(5e-4 / (2.6 ** 4), 5e-4), moms=(0.8, 0.7), wd=1e-7) + learn.unfreeze() + if self.num_epochs > 5: + learn.fit_one_cycle(self.num_epochs - 4, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7), wd=1e-7) + + +def path_if_model_exists(path, weights_name): + """Return path to model if it exists""" + model_path = path / (weights_name + ".pth") + return path if model_path.exists() else None + + +@dataclass +class ULMFiT: + arch: ULMFITArchitecture = None + pretrain_lm: ULMFiTPretraining = None + finetune_lm: ULMFiTFinetuining = None + classifier: ULMFiTClassifier = None + + def __post_init__(self): + self.arch = ULMFITArchitecture() + self.pretrain_lm = ULMFiTPretraining(arch=self.arch) + self.finetune_lm = ULMFiTFinetuining(arch=self.arch, base=self.pretrain_lm) + self.classifier = ULMFiTClassifier(arch=self.arch, base=self.finetune_lm) + + def load_(self, experiment_path): + success = (self.classifier.load_(experiment_path) or + self.finetune_lm.load_(experiment_path) or + self.pretrain_lm.load_(experiment_path) or + self.load_legacy_(experiment_path)) + if not success: + warn('Unable to load experiment') + return self + + def load_legacy_(self, experiment_path): + if not (experiment_path / "info.json").exists(): + return False + with (experiment_path / "info.json").open('r') as f: + d = json.load(f) + d.pop('dataset_path', None) + d['n_hid'] = d['nh'] + d['n_layers'] = d['nl'] + self.replace_(**d) + if "wiki" in str(experiment_path): + self.pretrain_lm.experiment_path = path_if_model_exists(experiment_path, LM_BEST) + else: + self.finetune_lm.experiment_path = path_if_model_exists(experiment_path, ENC_BEST) + self.classifier.experiment_path = path_if_model_exists(experiment_path, CLS_BEST) + return True + + def replace_(self, **kwargs): + self.arch.replace_(**kwargs) + self.pretrain_lm.replace_(**kwargs) + self.finetune_lm.replace_(**kwargs) + self.classifier.replace_(**kwargs) + return self