mirror of
https://github.com/wassname/multifit.git
synced 2026-09-09 11:27:26 +08:00
53 KiB
53 KiB
In [2]:
%env CUDA_VISIBLE_DEVICES=1env: CUDA_VISIBLE_DEVICES=1
In [3]:
%reload_ext autoreload
%autoreload 2
%matplotlib inline
%cd ../home/pczapla/workspace/ulmfit-multilingual
In [4]:
from fastai.text import *
from multifit.datasets import ULMFiTDataset, Dataset
import multifitIn [5]:
import warnings
warnings.filterwarnings("ignore") # to ignore pytorch 1.3 warnings triggered by older fastaiIn [6]:
lang='de'In [7]:
exp = multifit.from_pretrained(f'{lang}_multifit_paper_version')In [8]:
exp.replace_(name='multifit_paper_version20')
exp.archOut [8]:
ULMFiTArchitecture(tokenizer_type='sp', max_vocab=15000, lang='de', emb_sz=400, n_hid=1550, n_layers=4, qrnn=True)
In [9]:
cls_dataset = exp.arch.dataset(Path(f'data/cls/{lang}-music'), exp.pretrain_lm.tokenizer)Copy sp model from /home/pczapla/.fastai/models/de_multifit_paper_version to data/cls/de-music/models/sp15k
In [10]:
cls_dataset.langOut [10]:
'de'
In [11]:
cls_dataset.load_clas_databunch(bs=exp.finetune_lm.bs).show_batch()Data lm-notst, trn: 31800, val: 200 Size of vocabulary: 15000 First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"] Data cls, trn: 1800, val: 200 Data tst, trn: 200, val: 2000
| text | target |
|---|---|
| ▁ xxbos ▁ xxfld ▁1 ▁ xxmaj ▁in ▁der ▁ xxmaj ▁ special ▁ xxmaj ▁ edition ▁mit ▁fünf ▁zusätzlich en ▁ xxmaj ▁ track s ▁noch ▁viel ▁besser ▁, ▁als ▁ohne hin ▁schon ▁: ▁ xxmaj ▁alan is ▁ xxmaj ▁mo riss ette s ▁neues ▁ xxmaj ▁album ▁ xxfld ▁2 ▁ xxmaj ▁nach ▁vier ▁ xxmaj ▁jahren ▁ist ▁ xxmaj ▁alan is ▁ xxmaj ▁mo riss ette ▁endlich | 1 |
| ▁ xxbos ▁ xxfld ▁1 ▁und ▁ es ▁geht ▁immer ▁noch ▁schlecht er ▁ xxrep ▁5 ▁. ▁ xxfld ▁2 ▁ xxmaj ▁irgend wi e ▁ist ▁ es ▁bei ▁ xxmaj ▁ ram m stein ▁mittlerweil e ▁ein ▁ xxmaj ▁ phänomen ▁im ▁negative n ▁ xxmaj ▁sinne ▁ - ▁&' ▁ xxmaj ▁reise ▁, ▁ xxmaj ▁reise ▁&' ▁war ▁nach ▁&' ▁ xxmaj ▁herz e leid ▁&' ▁, ▁&' | 0 |
| ▁ xxbos ▁ xxfld ▁1 ▁ xxmaj ▁ tol les ▁ debüt - album ▁ ! ▁ xxfld ▁2 ▁ xxmaj ▁die s ▁ist ▁das ▁1995 ▁erschienen e ▁, ▁erste ▁ xxmaj ▁album ▁der ▁ xxmaj ▁bremer ▁ xxmaj ▁stadt musik anten ▁&' ▁ xxmaj ▁mr . ▁ xxmaj ▁ pres ident ▁&' ▁gewesen ▁. ▁ xxmaj ▁nachdem ▁ ich ▁zuvor ▁schon ▁von ▁den ▁ xxmaj ▁single s ▁&' ▁ | 1 |
| ▁ xxbos ▁ xxfld ▁1 ▁ xxmaj ▁hoch mut ▁kommt ▁vor ▁dem ▁ xxmaj ▁ fall ▁ xxfld ▁2 ▁ xxmaj ▁wie ▁lange ▁habe ▁ ich ▁doch ▁ge warte t ▁bis ▁ xxmaj ▁ ti es to ▁seit ▁seiner ▁letzten ▁ xxup ▁cd ▁( ▁ xxmaj ▁in ▁ xxmaj ▁ se arch ▁of ▁ xxmaj ▁ s un ris e ▁ xxmaj ▁ asia ▁) ▁eine ▁neue ▁rau s bringt | 0 |
| ▁ xxbos ▁ xxfld ▁1 ▁5 ▁ xxmaj ▁ sterne ▁ reichen ▁nicht ▁aus ▁ xxfld ▁2 ▁ xxmaj ▁ ig n ite ▁ - ▁ xxmaj ▁ our ▁ xxmaj ▁dar ke st ▁ day s ▁ - ▁cd - re vie w ▁1. in tro ▁( ▁ xxmaj ▁ our ▁ xxmaj ▁dar ke st ▁ xxmaj ▁ day s ▁) ▁ xxmaj ▁das ▁ist ▁kein ▁in tro | 1 |
In [12]:
exp.finetune_lm.train_(cls_dataset, num_epochs=20)Setting LM weights seed seed to 0
Data lm-notst, trn: 31800, val: 200
Size of vocabulary: 15000
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"]
Training args: {'drop_mult': 0.3, 'true_wd': False, 'wd': 1e-07, 'pretrained': False, 'clip': 0.12} config: {'emb_sz': 400, 'n_hid': 1550, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15, 'tie_weights': True, 'out_bias': True}
Setting LM training seed seed to 0
Loading pretrained weights: [PosixPath('/home/pczapla/.fastai/models/de_multifit_paper_version/lm_best'), PosixPath('/home/pczapla/.fastai/models/de_multifit_paper_version/itos')]
Experiment data/cls/de-music/models/sp15k/multifit_paper_version20
Fitting using 2 cycle fit schedule
Total time: 04:09
| epoch | train_loss | valid_loss | accuracy | time |
|---|---|---|---|---|
| 0 | 3.866541 | 3.233327 | 0.435623 | 04:09 |
Total time: 2:00:30
| epoch | train_loss | valid_loss | accuracy | time |
|---|---|---|---|---|
| 0 | 3.277351 | 3.070962 | 0.455366 | 06:01 |
| 1 | 3.079563 | 2.893713 | 0.477454 | 06:01 |
| 2 | 2.897532 | 2.768432 | 0.492125 | 06:01 |
| 3 | 2.809160 | 2.678615 | 0.503370 | 06:01 |
| 4 | 2.703292 | 2.628708 | 0.509084 | 06:01 |
| 5 | 2.673165 | 2.586905 | 0.512564 | 06:01 |
| 6 | 2.589240 | 2.558473 | 0.516227 | 06:01 |
| 7 | 2.564356 | 2.546148 | 0.519396 | 06:01 |
| 8 | 2.469183 | 2.526894 | 0.521648 | 06:01 |
| 9 | 2.457526 | 2.516867 | 0.523095 | 06:01 |
| 10 | 2.439664 | 2.505525 | 0.525311 | 06:01 |
| 11 | 2.425264 | 2.503526 | 0.526886 | 06:01 |
| 12 | 2.327446 | 2.494336 | 0.528297 | 06:01 |
| 13 | 2.311902 | 2.497768 | 0.528187 | 06:01 |
| 14 | 2.265695 | 2.497650 | 0.528791 | 06:01 |
| 15 | 2.223954 | 2.503026 | 0.528187 | 06:01 |
| 16 | 2.232797 | 2.505526 | 0.527912 | 06:01 |
| 17 | 2.204011 | 2.505398 | 0.528553 | 06:01 |
| 18 | 2.183692 | 2.508069 | 0.528663 | 06:01 |
| 19 | 2.185323 | 2.508334 | 0.528736 | 06:01 |
Copy sp model from /home/pczapla/.fastai/models/de_multifit_paper_version to data/cls/de-music/models/sp15k/multifit_paper_version20 this Learner object self-destroyed - it still exists, but no longer usable Saving dump to data/cls/de-music/models/sp15k/multifit_paper_version20/finetuning.json Language model saved to data/cls/de-music/models/sp15k/multifit_paper_version20
In [13]:
exp.load_(cls_dataset.cache_path/exp.pretrain_lm.name).finetune_lmOut [13]:
Loading data/cls/de-music/models/sp15k/multifit_paper_version20/classifier.json Loading data/cls/de-music/models/sp15k/multifit_paper_version20/finetuning.json ULMFiTClassifier Replacing experiment_path 'None' with 'data/cls/de-music/models/sp15k/multifit_paper_version20 ULMFiTClassifier Replacing dataset_path 'None' with 'data/cls/de-music
ULMFiTFinetuning(seed=0, name='multifit_paper_version20', experiment_path=PosixPath('data/cls/de-music/models/sp15k/multifit_paper_version20'), dataset_path=PosixPath('data/cls/de-music'), num_epochs=20, bs=20, bptt=70, drop_mult=0.3, dropout_values={'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}, label_smoothing_eps=0.0, label_smoothing_eps_norm_by_classes=True, use_adam_08=False, true_wd=False, wd=1e-07, clip=0.12, fp16=False, lr=0.001)In [14]:
exp.classifier.train_(seed=0)Copy sp model from data/cls/de-music/models/sp15k/multifit_paper_version20 to data/cls/de-music/models/sp15k
Data lm-notst, trn: 31800, val: 200
Size of vocabulary: 15000
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"]
Data cls, trn: 1800, val: 200
Data tst, trn: 200, val: 2000
Using Label smoothing with eps = 0.05
Setting Classifier weights seed seed to 0
Training args: {'drop_mult': 0.5, 'wd': 0.01, 'pretrained': False, 'bptt': 70, 'loss_func': FlattenedLoss of LabelSmoothingCrossEntropy(), 'clip': 0.12} config: {'emb_sz': 400, 'n_hid': 1550, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
Loading pretrained model /home/pczapla/workspace/ulmfit-multilingual/data/cls/de-music/models/sp15k/multifit_paper_version20/enc_best
Setting Classifier training seed seed to 0
Training: data/cls/de-music/models/sp15k/multifit_paper_version20
Total time: 02:50
| epoch | train_loss | valid_loss | accuracy | time |
|---|---|---|---|---|
| 0 | 0.482573 | 0.453916 | 0.865000 | 00:21 |
| 1 | 0.425821 | 0.410971 | 0.855000 | 00:21 |
| 2 | 0.349141 | 0.482879 | 0.835000 | 00:21 |
| 3 | 0.283344 | 0.442784 | 0.850000 | 00:21 |
| 4 | 0.241310 | 0.372520 | 0.895000 | 00:21 |
| 5 | 0.231902 | 0.398809 | 0.885000 | 00:21 |
| 6 | 0.219850 | 0.380606 | 0.890000 | 00:21 |
| 7 | 0.219838 | 0.381202 | 0.870000 | 00:21 |
Classifier model saved to data/cls/de-music/models/sp15k/multifit_paper_version20 Saving dump to data/cls/de-music/models/sp15k/multifit_paper_version20/classifier.json this Learner object self-destroyed - it still exists, but no longer usable
In [15]:
exp.classifier.train_(seed=1)Copy sp model from data/cls/de-music/models/sp15k/multifit_paper_version20 to data/cls/de-music/models/sp15k
Data lm-notst, trn: 31800, val: 200
Size of vocabulary: 15000
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"]
Data cls, trn: 1800, val: 200
Data tst, trn: 200, val: 2000
Using Label smoothing with eps = 0.05
Setting Classifier weights seed seed to 1
Training args: {'drop_mult': 0.5, 'wd': 0.01, 'pretrained': False, 'bptt': 70, 'loss_func': FlattenedLoss of LabelSmoothingCrossEntropy(), 'clip': 0.12} config: {'emb_sz': 400, 'n_hid': 1550, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
Loading pretrained model /home/pczapla/workspace/ulmfit-multilingual/data/cls/de-music/models/sp15k/multifit_paper_version20/enc_best
Setting Classifier training seed seed to 1
Training: data/cls/de-music/models/sp15k/multifit_paper_version20seed1
Total time: 02:52
| epoch | train_loss | valid_loss | accuracy | time |
|---|---|---|---|---|
| 0 | 0.478058 | 0.420762 | 0.850000 | 00:21 |
| 1 | 0.417134 | 0.439587 | 0.835000 | 00:21 |
| 2 | 0.354391 | 0.474661 | 0.830000 | 00:21 |
| 3 | 0.283689 | 0.365501 | 0.900000 | 00:21 |
| 4 | 0.246739 | 0.367108 | 0.895000 | 00:21 |
| 5 | 0.232322 | 0.366319 | 0.895000 | 00:21 |
| 6 | 0.224353 | 0.356722 | 0.880000 | 00:21 |
| 7 | 0.214791 | 0.355131 | 0.885000 | 00:21 |
Copy sp model from data/cls/de-music/models/sp15k/multifit_paper_version20 to data/cls/de-music/models/sp15k/multifit_paper_version20seed1 Classifier model saved to data/cls/de-music/models/sp15k/multifit_paper_version20seed1 Saving dump to data/cls/de-music/models/sp15k/multifit_paper_version20seed1/classifier.json this Learner object self-destroyed - it still exists, but no longer usable
In [16]:
exp.classifier.train_(seed=2)Copy sp model from data/cls/de-music/models/sp15k/multifit_paper_version20 to data/cls/de-music/models/sp15k
Data lm-notst, trn: 31800, val: 200
Size of vocabulary: 15000
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"]
Data cls, trn: 1800, val: 200
Data tst, trn: 200, val: 2000
Using Label smoothing with eps = 0.05
Setting Classifier weights seed seed to 2
Training args: {'drop_mult': 0.5, 'wd': 0.01, 'pretrained': False, 'bptt': 70, 'loss_func': FlattenedLoss of LabelSmoothingCrossEntropy(), 'clip': 0.12} config: {'emb_sz': 400, 'n_hid': 1550, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
Loading pretrained model /home/pczapla/workspace/ulmfit-multilingual/data/cls/de-music/models/sp15k/multifit_paper_version20/enc_best
Setting Classifier training seed seed to 2
Training: data/cls/de-music/models/sp15k/multifit_paper_version20seed2
Total time: 02:51
| epoch | train_loss | valid_loss | accuracy | time |
|---|---|---|---|---|
| 0 | 0.482315 | 0.439644 | 0.850000 | 00:21 |
| 1 | 0.428751 | 0.445248 | 0.860000 | 00:21 |
| 2 | 0.361243 | 0.384282 | 0.885000 | 00:21 |
| 3 | 0.275155 | 0.572610 | 0.790000 | 00:21 |
| 4 | 0.250888 | 0.368725 | 0.915000 | 00:21 |
| 5 | 0.237372 | 0.347252 | 0.905000 | 00:21 |
| 6 | 0.230649 | 0.338449 | 0.910000 | 00:21 |
| 7 | 0.219903 | 0.336891 | 0.910000 | 00:21 |
Copy sp model from data/cls/de-music/models/sp15k/multifit_paper_version20 to data/cls/de-music/models/sp15k/multifit_paper_version20seed2 Classifier model saved to data/cls/de-music/models/sp15k/multifit_paper_version20seed2 Saving dump to data/cls/de-music/models/sp15k/multifit_paper_version20seed2/classifier.json this Learner object self-destroyed - it still exists, but no longer usable
In [17]:
exp.classifier.train_(seed=3)Copy sp model from data/cls/de-music/models/sp15k/multifit_paper_version20 to data/cls/de-music/models/sp15k
Data lm-notst, trn: 31800, val: 200
Size of vocabulary: 15000
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"]
Data cls, trn: 1800, val: 200
Data tst, trn: 200, val: 2000
Using Label smoothing with eps = 0.05
Setting Classifier weights seed seed to 3
Training args: {'drop_mult': 0.5, 'wd': 0.01, 'pretrained': False, 'bptt': 70, 'loss_func': FlattenedLoss of LabelSmoothingCrossEntropy(), 'clip': 0.12} config: {'emb_sz': 400, 'n_hid': 1550, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
Loading pretrained model /home/pczapla/workspace/ulmfit-multilingual/data/cls/de-music/models/sp15k/multifit_paper_version20/enc_best
Setting Classifier training seed seed to 3
Training: data/cls/de-music/models/sp15k/multifit_paper_version20seed3
Total time: 02:51
| epoch | train_loss | valid_loss | accuracy | time |
|---|---|---|---|---|
| 0 | 0.484043 | 0.420790 | 0.870000 | 00:21 |
| 1 | 0.408424 | 0.537166 | 0.820000 | 00:21 |
| 2 | 0.340057 | 0.399938 | 0.860000 | 00:21 |
| 3 | 0.297742 | 0.435766 | 0.845000 | 00:21 |
| 4 | 0.246439 | 0.395233 | 0.865000 | 00:21 |
| 5 | 0.232959 | 0.381182 | 0.860000 | 00:21 |
| 6 | 0.222137 | 0.370006 | 0.875000 | 00:21 |
| 7 | 0.218951 | 0.367611 | 0.870000 | 00:21 |
Copy sp model from data/cls/de-music/models/sp15k/multifit_paper_version20 to data/cls/de-music/models/sp15k/multifit_paper_version20seed3 Classifier model saved to data/cls/de-music/models/sp15k/multifit_paper_version20seed3 Saving dump to data/cls/de-music/models/sp15k/multifit_paper_version20seed3/classifier.json this Learner object self-destroyed - it still exists, but no longer usable
In [18]:
exp.classifier.train_(seed=4)Copy sp model from data/cls/de-music/models/sp15k/multifit_paper_version20 to data/cls/de-music/models/sp15k
Data lm-notst, trn: 31800, val: 200
Size of vocabulary: 15000
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"]
Data cls, trn: 1800, val: 200
Data tst, trn: 200, val: 2000
Using Label smoothing with eps = 0.05
Setting Classifier weights seed seed to 4
Training args: {'drop_mult': 0.5, 'wd': 0.01, 'pretrained': False, 'bptt': 70, 'loss_func': FlattenedLoss of LabelSmoothingCrossEntropy(), 'clip': 0.12} config: {'emb_sz': 400, 'n_hid': 1550, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
Loading pretrained model /home/pczapla/workspace/ulmfit-multilingual/data/cls/de-music/models/sp15k/multifit_paper_version20/enc_best
Setting Classifier training seed seed to 4
Training: data/cls/de-music/models/sp15k/multifit_paper_version20seed4
Total time: 02:53
| epoch | train_loss | valid_loss | accuracy | time |
|---|---|---|---|---|
| 0 | 0.481196 | 0.459419 | 0.845000 | 00:21 |
| 1 | 0.432749 | 0.458411 | 0.845000 | 00:21 |
| 2 | 0.330244 | 0.439864 | 0.840000 | 00:21 |
| 3 | 0.286105 | 0.366315 | 0.890000 | 00:21 |
| 4 | 0.261380 | 0.438214 | 0.835000 | 00:21 |
| 5 | 0.241455 | 0.362442 | 0.870000 | 00:21 |
| 6 | 0.230032 | 0.372623 | 0.880000 | 00:22 |
| 7 | 0.220430 | 0.372761 | 0.880000 | 00:21 |
Copy sp model from data/cls/de-music/models/sp15k/multifit_paper_version20 to data/cls/de-music/models/sp15k/multifit_paper_version20seed4 Classifier model saved to data/cls/de-music/models/sp15k/multifit_paper_version20seed4 Saving dump to data/cls/de-music/models/sp15k/multifit_paper_version20seed4/classifier.json this Learner object self-destroyed - it still exists, but no longer usable
In [19]:
exp.classifier.train_(seed=5)Copy sp model from data/cls/de-music/models/sp15k/multifit_paper_version20 to data/cls/de-music/models/sp15k
Data lm-notst, trn: 31800, val: 200
Size of vocabulary: 15000
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"]
Data cls, trn: 1800, val: 200
Data tst, trn: 200, val: 2000
Using Label smoothing with eps = 0.05
Setting Classifier weights seed seed to 5
Training args: {'drop_mult': 0.5, 'wd': 0.01, 'pretrained': False, 'bptt': 70, 'loss_func': FlattenedLoss of LabelSmoothingCrossEntropy(), 'clip': 0.12} config: {'emb_sz': 400, 'n_hid': 1550, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
Loading pretrained model /home/pczapla/workspace/ulmfit-multilingual/data/cls/de-music/models/sp15k/multifit_paper_version20/enc_best
Setting Classifier training seed seed to 5
Training: data/cls/de-music/models/sp15k/multifit_paper_version20seed5
Total time: 02:52
| epoch | train_loss | valid_loss | accuracy | time |
|---|---|---|---|---|
| 0 | 0.469831 | 0.412838 | 0.880000 | 00:21 |
| 1 | 0.403946 | 0.478476 | 0.860000 | 00:21 |
| 2 | 0.352868 | 0.606190 | 0.770000 | 00:22 |
| 3 | 0.284212 | 0.430632 | 0.865000 | 00:20 |
| 4 | 0.254504 | 0.370566 | 0.895000 | 00:21 |
| 5 | 0.233122 | 0.367492 | 0.900000 | 00:21 |
| 6 | 0.222597 | 0.370217 | 0.890000 | 00:21 |
| 7 | 0.223786 | 0.372048 | 0.890000 | 00:21 |
Copy sp model from data/cls/de-music/models/sp15k/multifit_paper_version20 to data/cls/de-music/models/sp15k/multifit_paper_version20seed5 Classifier model saved to data/cls/de-music/models/sp15k/multifit_paper_version20seed5 Saving dump to data/cls/de-music/models/sp15k/multifit_paper_version20seed5/classifier.json this Learner object self-destroyed - it still exists, but no longer usable
In [20]:
exp.classifier.train_(seed=6)Copy sp model from data/cls/de-music/models/sp15k/multifit_paper_version20 to data/cls/de-music/models/sp15k
Data lm-notst, trn: 31800, val: 200
Size of vocabulary: 15000
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"]
Data cls, trn: 1800, val: 200
Data tst, trn: 200, val: 2000
Using Label smoothing with eps = 0.05
Setting Classifier weights seed seed to 6
Training args: {'drop_mult': 0.5, 'wd': 0.01, 'pretrained': False, 'bptt': 70, 'loss_func': FlattenedLoss of LabelSmoothingCrossEntropy(), 'clip': 0.12} config: {'emb_sz': 400, 'n_hid': 1550, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
Loading pretrained model /home/pczapla/workspace/ulmfit-multilingual/data/cls/de-music/models/sp15k/multifit_paper_version20/enc_best
Setting Classifier training seed seed to 6
Training: data/cls/de-music/models/sp15k/multifit_paper_version20seed6
Total time: 02:53
| epoch | train_loss | valid_loss | accuracy | time |
|---|---|---|---|---|
| 0 | 0.476341 | 0.412699 | 0.855000 | 00:21 |
| 1 | 0.414260 | 0.465001 | 0.855000 | 00:21 |
| 2 | 0.345892 | 0.381468 | 0.880000 | 00:21 |
| 3 | 0.282183 | 0.471623 | 0.845000 | 00:21 |
| 4 | 0.254331 | 0.364531 | 0.885000 | 00:21 |
| 5 | 0.230084 | 0.352639 | 0.900000 | 00:22 |
| 6 | 0.226763 | 0.354634 | 0.905000 | 00:21 |
| 7 | 0.225935 | 0.345909 | 0.920000 | 00:21 |
Copy sp model from data/cls/de-music/models/sp15k/multifit_paper_version20 to data/cls/de-music/models/sp15k/multifit_paper_version20seed6 Classifier model saved to data/cls/de-music/models/sp15k/multifit_paper_version20seed6 Saving dump to data/cls/de-music/models/sp15k/multifit_paper_version20seed6/classifier.json this Learner object self-destroyed - it still exists, but no longer usable
In [21]:
exp.classifier.train_(seed=7)Copy sp model from data/cls/de-music/models/sp15k/multifit_paper_version20 to data/cls/de-music/models/sp15k
Data lm-notst, trn: 31800, val: 200
Size of vocabulary: 15000
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"]
Data cls, trn: 1800, val: 200
Data tst, trn: 200, val: 2000
Using Label smoothing with eps = 0.05
Setting Classifier weights seed seed to 7
Training args: {'drop_mult': 0.5, 'wd': 0.01, 'pretrained': False, 'bptt': 70, 'loss_func': FlattenedLoss of LabelSmoothingCrossEntropy(), 'clip': 0.12} config: {'emb_sz': 400, 'n_hid': 1550, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
Loading pretrained model /home/pczapla/workspace/ulmfit-multilingual/data/cls/de-music/models/sp15k/multifit_paper_version20/enc_best
Setting Classifier training seed seed to 7
Training: data/cls/de-music/models/sp15k/multifit_paper_version20seed7
Total time: 02:52
| epoch | train_loss | valid_loss | accuracy | time |
|---|---|---|---|---|
| 0 | 0.504087 | 0.451805 | 0.825000 | 00:21 |
| 1 | 0.431187 | 0.464305 | 0.825000 | 00:21 |
| 2 | 0.362195 | 0.409977 | 0.860000 | 00:21 |
| 3 | 0.289505 | 0.386984 | 0.880000 | 00:21 |
| 4 | 0.246833 | 0.402201 | 0.865000 | 00:21 |
| 5 | 0.230346 | 0.385452 | 0.855000 | 00:21 |
| 6 | 0.221595 | 0.381722 | 0.880000 | 00:21 |
| 7 | 0.227807 | 0.384149 | 0.870000 | 00:21 |
Copy sp model from data/cls/de-music/models/sp15k/multifit_paper_version20 to data/cls/de-music/models/sp15k/multifit_paper_version20seed7 Classifier model saved to data/cls/de-music/models/sp15k/multifit_paper_version20seed7 Saving dump to data/cls/de-music/models/sp15k/multifit_paper_version20seed7/classifier.json this Learner object self-destroyed - it still exists, but no longer usable
In [22]:
def get_results(exp_path):
exp = multifit.ULMFiT().load_(exp_path, silent=True).classifier
results = exp.validate(use_cache=False)
results.update(seed=exp.seed, fp16=exp.fp16)
return results
results = [get_results(exp_path) for exp_path in cls_dataset.cache_path.glob(exp.pretrain_lm.name+"seed*")]
results_df = pd.DataFrame.from_records(results)
results_df.sort_values(["valid accuracy"])[["name", "seed", "test accuracy", "valid accuracy"]]Out [22]:
{'test loss': 0.3395114839076996, 'test accuracy': 0.921500027179718, 'name': 'multifit_paper_version20', 'valid loss': 0.3720475435256958, 'valid accuracy': 0.8899999856948853, 'train loss': 0.20120832324028015, 'train accuracy': 1.0}
| name | seed | test accuracy | valid accuracy | |
|---|---|---|---|---|
| 0 | multifit_paper_version20 | 7 | 0.9170 | 0.870 |
| 2 | multifit_paper_version20 | 3 | 0.9085 | 0.870 |
| 5 | multifit_paper_version20 | 4 | 0.9085 | 0.880 |
| 1 | multifit_paper_version20 | 1 | 0.9080 | 0.885 |
| 6 | multifit_paper_version20 | 5 | 0.9215 | 0.890 |
| 3 | multifit_paper_version20 | 2 | 0.9160 | 0.910 |
| 4 | multifit_paper_version20 | 6 | 0.9130 | 0.920 |
In [ ]: