From 22e40d81f33f0377294168630f0cf7ff0b0f9a7e Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Mon, 11 Feb 2019 10:51:42 +0100 Subject: [PATCH] Additional DE results and first results on FR and JA --- results/logs/de.md | 391 ++++++++++++++++++++++++++++++++++++++++++++- results/logs/fr.md | 153 ++++++++++++++++-- results/logs/ja.md | 90 +++++++++++ 3 files changed, 617 insertions(+), 17 deletions(-) create mode 100644 results/logs/ja.md diff --git a/results/logs/de.md b/results/logs/de.md index fb28696..89b88b2 100644 --- a/results/logs/de.md +++ b/results/logs/de.md @@ -28,7 +28,61 @@ data/wiki/de-100/models/vf60k Saving info data/wiki/de-100/models/vf60k/lstm_nl3.m/info.json ``` ### MLDocs -... +``` +python -m ulmfit cls --dataset-path data/mldoc/de-1 --base-lm-path data/wiki/de-100/models/vf60k/lstm_nl3.m --lang=de --name 'nl3' - train 20 --bs 40 +Max vocab: 60000 +Cache dir: data/mldoc/de-1/models/vf60k +Model dir: data/mldoc/de-1/models/vf60k/lstm_nl3.m +Loading validation data/mldoc/de-1/de.dev.csv +Running tokenization... +Saving tokenized: cls.trn 13500, cls.val 1500 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 39171 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '.', 'der', ',', 'die', ')', '(', 'in', 'und', 'auf', 'von', 'den', 'im'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('data/wiki/de-100/models/vf60k/lstm_nl3.m/lm_best'), PosixPath('data/wiki/de-100/models/vf60k/lstm_nl3.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 20582, first 100: ['"', 'vh', 'ös', 'brs', '&', 'geg', 'lpo', 'vormonat', 'fgc', 'waigel', 'tcs', 'mic', 'bund-future', 'ajs', 'brn', 'dih', 'analysten', 'mrd', 'rpk', 'notierten', 'dividende', 'feb', 'aktienmarkt', 'rev', 'rentenmarkt', 'basispunkte', 'müßten', 'gewinnmitnahmen', 'aktienbörse', 'rußland', 'volkswirte', 'fls', 'steuerreform', 'kontrakte', 'kps', 'mge', 'zählern', 'vortagesschluß', 'umsätzen', 'prozent.', 'snb', 'dow-jones-index', 'reingewinn', 'notierungen', "\\'", 'gesamtmarkt', 'industrieproduktion', 'akr', 'kjf', '49-69-7565', 'abl', 'hoh', 'finanzdienst', 'atx', 'feinunze', 'zinserhöhung', 'zugelegt', 'netanjahu', 'verbraucherpreise', 'pence', 'ticks', 'arafat', 'kursgewinne', 'ker', 'aktienindex', 'rlb', 'smi', 'vorbörslich', 'dst', 'mkl', 'kontrakten', 'calls', 'veraenderung', 'gwa', 'gesamtjahr', 'auftragseingang', 'überschuß', 'erwarte', 'verlautete', 'eju', 'tms', 'jahresvergleich', 'vorjahreszeitraum', 'werden.', 'betriebsergebnis', 'rin', 'bobl-future', 'puts', 'fri', '4.50', 'schluß', 'ewu', 'standardwerte', 'jahresüberschuß', 'rechne', '49-69-756525', '16.00', 'peh', 'hmh', 'dtb'] +Training lm from: [PosixPath('data/wiki/de-100/models/vf60k/lstm_nl3.m/lm_best'), PosixPath('data/wiki/de-100/models/vf60k/lstm_nl3.m/../itos')] +epoch train_loss valid_loss accuracy +1 3.532079 3.059487 0.465283 +epoch train_loss valid_loss accuracy +1 3.219148 2.945357 0.475736 +2 3.014822 2.804256 0.494567 +3 2.896143 2.652700 0.513166 +4 2.756027 2.516747 0.528836 +5 2.629735 2.383480 0.543956 +6 2.515785 2.281831 0.556083 +7 2.422463 2.178855 0.567950 +8 2.351060 2.091266 0.579531 +9 2.297676 2.017783 0.590206 +10 2.205688 1.937085 0.601936 +11 2.155664 1.871271 0.612579 +12 2.065812 1.806647 0.623888 +13 2.038635 1.748420 0.634389 +14 1.957434 1.696571 0.643807 +15 1.895242 1.653865 0.651743 +16 1.910458 1.618776 0.658140 +17 1.843909 1.598143 0.662129 +18 1.837299 1.583182 0.664999 +19 1.788718 1.573136 0.666785 +20 1.780236 1.574308 0.666625 +data/mldoc/de-1/models/vf60k +Saving info data/mldoc/de-1/models/vf60k/lstm_nl3.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.526303 0.328480 0.892000 +epoch train_loss valid_loss accuracy +1 0.346665 0.238605 0.920000 +epoch train_loss valid_loss accuracy +1 0.266841 0.285444 0.921000 +epoch train_loss valid_loss accuracy +1 0.175013 0.280545 0.921000 +2 0.178333 0.286059 0.923000 +Saving models at data/mldoc/de-1/models/vf60k/lstm_nl3.m +Loss and accuracy using (cls_last): [0.28054512, tensor(0.9210)] +Loss and accuracy using (cls_best): [0.28054512, tensor(0.9210)] +``` +MultiCCA: 93.7% , ulmfit: 92.1% ## SP30k LSTM nl 4 ### LM ``` @@ -95,8 +149,337 @@ epoch train_loss valid_loss accuracy 1 0.159555 0.230822 0.947000 2 0.144418 0.226450 0.943000 Saving models at data/mldoc/de-1/models/sp30k/lstm_nl4.m -Loss and accuracy using (cls_last): [0.22645034, tensor(0.9430)] -Loss and accuracy using (cls_best): [0.22645034, tensor(0.9430)] +Loss and accuracy using (cls_last): [0.16306259, tensor(0.9540)] ``` -MultiCCA: 93.7% , ulmfit: 94.3% +MultiCCA: 93.7% , ulmfit: 95.4% + ``` + Max vocab: 30000 +Cache dir: data/mldoc/de-1/models/sp30k +Model dir: data/mldoc/de-1/models/sp30k/lstm_nl4-2nd.m +Loading validation data/mldoc/de-1/de.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Tokenized data loaded, cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +data/mldoc/de-1/models/sp30k +Saving info data/mldoc/de-1/models/sp30k/lstm_nl4-2nd.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.464957 0.258905 0.928000 +epoch train_loss valid_loss accuracy +1 0.284900 0.243053 0.937000 +epoch train_loss valid_loss accuracy +1 0.298546 0.204188 0.948000 +epoch train_loss valid_loss accuracy +1 0.159097 0.199651 0.952000 +2 0.112476 0.203827 0.953000 +Saving models at data/mldoc/de-1/models/sp30k/lstm_nl4-2nd.m +Loss and accuracy using (cls_last): [0.1689675, tensor(0.9550)] + ``` +### examples limited to 100 +#### 2x run +first run +``` +python -m ulmfit cls --dataset-path data/mldoc/de-1 --base-lm-path data/mldoc/de-1/models/sp30k/lstm_nl4.m --lang=de --name 'nl4-100-2x' --cuda-id=1 - train 0 --bs 40 --limit=100 --drop-mult-cls=0.3 +Max vocab: 30000 +Cache dir: data/mldoc/de-1/models/sp30k +Model dir: data/mldoc/de-1/models/sp30k/lstm_nl4-100-2x.m +Loading validation data/mldoc/de-1/de.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Limiting data set to: 100 +Running tokenization... +Saving tokenized: cls.trn 100, cls.val 100 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +data/mldoc/de-1/models/sp30k +Saving info data/mldoc/de-1/models/sp30k/lstm_nl4-100-2x.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 1.181207 1.315258 0.300000 +epoch train_loss valid_loss accuracy +1 0.749909 1.204297 0.660000 +epoch train_loss valid_loss accuracy +1 0.558658 1.083666 0.830000 +epoch train_loss valid_loss accuracy +1 0.486175 1.020435 0.850000 +2 0.485117 0.958238 0.880000 +Saving models at data/mldoc/de-1/models/sp30k/lstm_nl4-100-2x.m + ..? .. +``` +2nd run +``` +python -m ulmfit cls --dataset-path data/mldoc/de-1 --base-lm-path data/mldoc/de-1/models/sp30k/lstm_nl4.m --lang=de --name 'nl4-100-2x' --cuda-id=1 - train 0 --bs 40 --limit=100 +Max vocab: 30000 +Cache dir: data/mldoc/de-1/models/sp30k +Model dir: data/mldoc/de-1/models/sp30k/lstm_nl4-100-2x.m +Loading validation data/mldoc/de-1/de.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Limiting data set to: 100 +Tokenized data loaded, cls.trn 100, cls.val 100 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] +Loading last classifier +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.441340 0.716396 0.840000 +epoch train_loss valid_loss accuracy +1 0.312035 0.532610 0.910000 +epoch train_loss valid_loss accuracy +1 0.267714 0.462694 0.920000 +epoch train_loss valid_loss accuracy +1 0.242031 0.430018 0.930000 +2 0.231161 0.398335 0.930000 +Saving models at data/mldoc/de-1/models/sp30k/lstm_nl4-100-2x.m +Loss and accuracy using (cls_last): [0.33284584, tensor(0.9252)] +``` +#### 8 epoches at the end +``` +python -m ulmfit cls --dataset-path data/mldoc/de-1 --base-lm-path data/mldoc/de-1/models/sp30k/lstm_nl4.m --lang=de --name 'nl4-100-e8' --cuda-id=1 - train 0 --bs 40 --limit=100 --num-cls-epochs=8 --drop-mult-cls=0.3 +Max vocab: 30000 +Cache dir: data/mldoc/de-1/models/sp30k +Model dir: data/mldoc/de-1/models/sp30k/lstm_nl4-100-e8.m +Loading validation data/mldoc/de-1/de.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Limiting data set to: 100 +Tokenized data loaded, cls.trn 100, cls.val 100 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +data/mldoc/de-1/models/sp30k +Saving info data/mldoc/de-1/models/sp30k/lstm_nl4-100-e8.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 1.208816 1.324359 0.280000 +epoch train_loss valid_loss accuracy +1 0.716811 1.195012 0.440000 +epoch train_loss valid_loss accuracy +1 0.535809 1.075753 0.590000 +epoch train_loss valid_loss accuracy +1 0.499198 1.018431 0.760000 +2 0.480971 0.948658 0.880000 +3 0.468659 0.866477 0.860000 +4 0.460322 0.770794 0.880000 +5 0.461138 0.704613 0.900000 +6 0.442423 0.623944 0.900000 +7 0.422423 0.568031 0.920000 +8 0.417041 0.527571 0.930000 +Saving models at data/mldoc/de-1/models/sp30k/lstm_nl4-100-e8.m +Loss and accuracy using (cls_last): [0.47343642, tensor(0.9070)] +``` +Dropout 0.6 +``` +python -m ulmfit cls --dataset-path data/mldoc/de-1 --base-lm-path data/mldoc/de-1/models/sp30k/lstm_nl4.m --lang=de --name 'nl4-100-e8dp6' --cuda-id=1 - train 0 --bs 40 --limit=100 --num-cls-epochs=8 --drop-mult-cls=0.6 +Max vocab: 30000 +Cache dir: data/mldoc/de-1/models/sp30k +Model dir: data/mldoc/de-1/models/sp30k/lstm_nl4-100-e8dp6.m +Loading validation data/mldoc/de-1/de.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Limiting data set to: 100 +Tokenized data loaded, cls.trn 100, cls.val 100 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +data/mldoc/de-1/models/sp30k +Saving info data/mldoc/de-1/models/sp30k/lstm_nl4-100-e8dp6.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 1.151162 1.323036 0.280000 +epoch train_loss valid_loss accuracy +1 0.745338 1.160084 0.610000 +epoch train_loss valid_loss accuracy +1 0.535118 1.041519 0.770000 +epoch train_loss valid_loss accuracy +1 0.459913 0.995187 0.860000 +2 0.451289 0.949036 0.830000 +3 0.460395 0.885940 0.800000 +4 0.454847 0.848194 0.770000 +5 0.447404 0.788741 0.810000 +6 0.428524 0.748181 0.760000 +7 0.419571 0.696069 0.760000 +8 0.408938 0.661937 0.770000 +Saving models at data/mldoc/de-1/models/sp30k/lstm_nl4-100-e8dp6.m +Loss and accuracy using (cls_last): [0.53202456, tensor(0.8830)] +``` +``` +python -m ulmfit cls --dataset-path data/mldoc/de-1 --base-lm-path data/mldoc/de-1/models/sp30k/lstm_nl4.m --lang=de --name 'nl4-100-e8dp6x2' --cuda-id=1 - train 0 --bs 40 --limit=100 --num-cls-epochs=8 --drop-mult-cls=0.6 +Max vocab: 30000 +Cache dir: data/mldoc/de-1/models/sp30k +Model dir: data/mldoc/de-1/models/sp30k/lstm_nl4-100-e8dp6x2.m +Loading validation data/mldoc/de-1/de.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Limiting data set to: 100 +Tokenized data loaded, cls.trn 100, cls.val 100 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +data/mldoc/de-1/models/sp30k +Saving info data/mldoc/de-1/models/sp30k/lstm_nl4-100-e8dp6x2.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 1.126586 1.338514 0.470000 +epoch train_loss valid_loss accuracy +1 0.751379 1.181071 0.550000 +epoch train_loss valid_loss accuracy +1 0.559534 1.083532 0.810000 +epoch train_loss valid_loss accuracy +1 0.444137 1.034607 0.870000 +2 0.438850 0.983929 0.830000 +3 0.436560 0.906958 0.840000 +4 0.447400 0.847952 0.840000 +5 0.431961 0.783818 0.850000 +6 0.422364 0.713126 0.850000 +7 0.414145 0.662799 0.840000 +8 0.407066 0.630168 0.840000 +Saving models at data/mldoc/de-1/models/sp30k/lstm_nl4-100-e8dp6x2.m +Loss and accuracy using (cls_last): [0.46259913, tensor(0.9147)] +``` + +``` +python -m ulmfit cls --dataset-path data/mldoc/de-1 --base-lm-path data/mldoc/de-1/models/sp30k/lstm_nl4.m --lang=de --name 'nl4-100-e8dp2' --cuda-id=1 - train 0 --bs 40 --limit=100 --num-cls-epochs=8 --drop-mult-cls=0.2 +Max vocab: 30000 +Cache dir: data/mldoc/de-1/models/sp30k +Model dir: data/mldoc/de-1/models/sp30k/lstm_nl4-100-e8dp2.m +Loading validation data/mldoc/de-1/de.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Limiting data set to: 100 +Tokenized data loaded, cls.trn 100, cls.val 100 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +data/mldoc/de-1/models/sp30k +Saving info data/mldoc/de-1/models/sp30k/lstm_nl4-100-e8dp2.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 1.172059 1.311985 0.280000 +epoch train_loss valid_loss accuracy +1 0.721259 1.180611 0.720000 +epoch train_loss valid_loss accuracy +1 0.495393 1.051538 0.770000 +epoch train_loss valid_loss accuracy +1 0.445929 0.984670 0.830000 +2 0.430556 0.897431 0.870000 +3 0.442683 0.800808 0.900000 +4 0.427033 0.711604 0.880000 +5 0.411931 0.624835 0.890000 +6 0.397705 0.560819 0.900000 +7 0.387848 0.506201 0.900000 +8 0.380063 0.459507 0.900000 +Saving models at data/mldoc/de-1/models/sp30k/lstm_nl4-100-e8dp2.m +Loss and accuracy using (cls_last): [0.41103342, tensor(0.9105)] +``` + +#### 2x e8 +``` +python -m ulmfit cls --dataset-path data/mldoc/de-1 --base-lm-path data/mldoc/de-1/models/sp30k/lstm_nl4.m --lang=de --name 'nl4-100-2nd-2x' --cuda-id=1 - train 0 --bs 40 --limit=100 --num-cls-epochs=8 --drop-mult-cls=0.2 +Max vocab: 30000 +Cache dir: data/mldoc/de-1/models/sp30k +Model dir: data/mldoc/de-1/models/sp30k/lstm_nl4-100-2nd-2x.m +Loading validation data/mldoc/de-1/de.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Limiting data set to: 100 +Tokenized data loaded, cls.trn 100, cls.val 100 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +data/mldoc/de-1/models/sp30k +Saving info data/mldoc/de-1/models/sp30k/lstm_nl4-100-2nd-2x.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 1.122616 1.290291 0.300000 +epoch train_loss valid_loss accuracy +1 0.746805 1.166377 0.730000 +epoch train_loss valid_loss accuracy +1 0.535163 1.058924 0.900000 +epoch train_loss valid_loss accuracy +1 0.437773 1.022764 0.850000 +2 0.449220 0.949963 0.820000 +3 0.443732 0.854293 0.860000 +4 0.432721 0.746918 0.900000 +5 0.423113 0.718745 0.840000 +6 0.403492 0.671295 0.820000 +7 0.399091 0.539798 0.900000 +8 0.394950 0.508265 0.900000 +Saving models at data/mldoc/de-1/models/sp30k/lstm_nl4-100-2nd-2x.m +Loading validation data/mldoc/de-1/de.dev.csv +Loss and accuracy using (cls_last): [0.44688165, tensor(0.9062)] +Loss and accuracy using (cls_best): [0.44688165, tensor(0.9062)] +``` +``` +python -m ulmfit cls --dataset-path data/mldoc/de-1 --base-lm-path data/mldoc/de-1/models/sp30k/lstm_nl4.m --lang=de --name 'nl4-100-2nd-2x' --cuda-id=1 - train 0 --bs 40 --limit=100 --num-cls-epochs=8 --drop-mult-cls=0.2 +Max vocab: 30000 +Cache dir: data/mldoc/de-1/models/sp30k +Model dir: data/mldoc/de-1/models/sp30k/lstm_nl4-100-2nd-2x.m +Loading validation data/mldoc/de-1/de.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Limiting data set to: 100 +Tokenized data loaded, cls.trn 100, cls.val 100 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] +Loading last classifier +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.422151 0.797943 0.720000 +epoch train_loss valid_loss accuracy +1 0.357166 0.736997 0.780000 +epoch train_loss valid_loss accuracy +1 0.238496 1.497305 0.660000 +epoch train_loss valid_loss accuracy +1 0.312356 1.522862 0.660000 +2 0.267801 1.518249 0.660000 +3 0.244077 1.110030 0.680000 +4 0.264972 0.798898 0.770000 +5 0.236816 0.398245 0.860000 +6 0.251284 0.415783 0.860000 +7 0.244988 0.417737 0.860000 +8 0.240362 0.415114 0.860000 +Saving models at data/mldoc/de-1/models/sp30k/lstm_nl4-100-2nd-2x.m +Loss and accuracy using (cls_last): [0.27954015, tensor(0.9125)] +Loss and accuracy using (cls_best): [0.27954015, tensor(0.9125)] +``` +### Adding noise +#### 40% +``` +python -m ulmfit cls --dataset-path data/mldoc/de-1 --base-lm-path data/mldoc/de-1/models/sp30k/lstm_nl4.m --lang=de --name 'nl4-noise0.4' --cuda-id=1 - train 0 --bs 40 --noise=0.4 --num-cls-epochs=8 --drop-mult-cls=0.2 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-noise0.4.m +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/de.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Added noise to 400 examples, only 0.6 have correct labels +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-noise0.4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 1.053928 0.938391 0.535000 +epoch train_loss valid_loss accuracy +1 0.941778 0.599400 0.836000 +epoch train_loss valid_loss accuracy +1 0.858363 0.675211 0.760000 +epoch train_loss valid_loss accuracy +1 0.768678 0.645293 0.788000 +2 0.758538 0.636551 0.780000 +3 0.753799 0.673323 0.708000 +4 0.731245 0.638630 0.736000 +5 0.691206 0.659491 0.717000 +6 0.691426 0.682510 0.696000 +7 0.672320 0.668610 0.702000 +8 0.653569 0.669633 0.694000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4-noise0.4.m +Loss and accuracy using (cls_last): [0.62477165, tensor(0.7717)] +``` diff --git a/results/logs/fr.md b/results/logs/fr.md index 757cd26..7feda12 100644 --- a/results/logs/fr.md +++ b/results/logs/fr.md @@ -1,20 +1,147 @@ -= FR = -== VF60k LSTM nl 3 == -=== LM === +# FR +## SP30k LSTM nl 4 +### LM ``` -``` -=== MLDocs === -``` -``` - -== SP30k LSTM nl 4 == -=== LM === -``` -python -m ulmfit lm --dataset-path data/wiki/fr-100 --cuda-id=1 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 30000 \ +python -m ulmfit lm --dataset-path data/wiki/fr-100 --cuda-id=1 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 30000 \ ✘ 130 --lang fr --qrnn=False - train 10 --bs=50 --drop_mult=0 +Max vocab: 30000 +Cache dir: data/wiki/fr-100/models/sp30k +Model dir: data/wiki/fr-100/models/sp30k/lstm_nl4.m +Running tokenization +Wiki text was split to 113288 articles +Wiki text was split to 88 articles +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': None, 'pretrained_model': None, 'drop_mult': 0} dps: [0.25 0.1 0.2 0.02 0.15] +Training lm from random weights +epoch train_loss valid_loss accuracy +1 3.120035 3.449028 0.383274 +2 3.070353 3.431372 0.382520 +3 3.092097 3.406521 0.384604 +4 3.035016 3.356502 0.391155 +5 2.936505 3.297572 0.396365 +6 2.926953 3.192980 0.407546 +7 2.841542 3.115280 0.417741 +8 2.805254 3.008793 0.429512 +9 2.681713 2.944207 0.439959 +10 2.644920 2.923765 0.442415 +data/wiki/fr-100/models/sp30k +Saving info data/wiki/fr-100/models/sp30k/lstm_nl4.m/info.json ``` -=== MLDocs === +## MLDocs +### First run +MultiCCA 92.05, ulmfit 93.90 ``` +python -m ulmfit cls --dataset-path data/mldoc/fr-1 --base-lm-path data/wiki/fr-100/models/sp30k/lstm_nl4.m --lang=fr --name 'nl4' --cuda-id=1 - train 20 --bs 40 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/fr.dev.csv +Running tokenization... +Saving tokenized: cls.trn 13500, cls.val 1500 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp30k/lstm_nl4.m/../itos')] +epoch train_loss valid_loss accuracy +1 3.072937 2.621444 0.468314 +epoch train_loss valid_loss accuracy +1 2.737065 2.485359 0.486546 +2 2.625827 2.353188 0.507669 +3 2.408049 2.224600 0.527609 +4 2.332804 2.113603 0.544255 +5 2.242967 2.016229 0.560002 +6 2.162170 1.925214 0.574050 +7 2.094163 1.843778 0.587944 +8 2.011285 1.773228 0.599802 +9 1.931492 1.708201 0.611245 +10 1.883735 1.643842 0.623145 +11 1.793858 1.583394 0.635366 +12 1.759305 1.526640 0.646132 +13 1.741412 1.474198 0.657485 +14 1.675670 1.430597 0.666407 +15 1.624235 1.390453 0.674829 +16 1.588415 1.359892 0.681364 +17 1.594124 1.336594 0.686985 +18 1.567758 1.322139 0.689745 +19 1.536472 1.315883 0.690974 +20 1.530144 1.314872 0.691084 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.528480 0.428756 0.853000 +epoch train_loss valid_loss accuracy +1 0.365323 0.224117 0.928000 +epoch train_loss valid_loss accuracy +1 0.300881 0.199623 0.936000 +epoch train_loss valid_loss accuracy +1 0.217855 0.198016 0.937000 +2 0.206357 0.212208 0.938000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m +Loss and accuracy using (cls_best): [0.18914989, tensor(0.9390)] ``` +## Second run +MultiCCA 92.05, ulmfit 93.67 +``` +python -m ulmfit cls --dataset-path data/mldoc/fr-1 --base-lm-path data/wiki/fr-100/models/sp30k/lstm_nl4.m --lang=fr --name 'nl4-2nd' --cuda-id=1 - train 20 --bs 40 --num-cls-epochs=8 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4-2nd.m +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/fr.dev.csv +Tokenized data loaded, lm.trn 13500, lm.val 1500 +Tokenized data loaded, cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à'] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/fr-100/models/sp30k/lstm_nl4.m/../itos')] +epoch train_loss valid_loss accuracy +1 3.080331 2.625329 0.467306 +epoch train_loss valid_loss accuracy +1 2.750554 2.485271 0.486776 +2 2.578659 2.353940 0.507637 +3 2.422981 2.224983 0.527749 +4 2.342781 2.113364 0.545006 +5 2.254575 2.007775 0.560709 +6 2.124016 1.920536 0.575680 +7 2.068470 1.847463 0.586699 +8 2.013289 1.775580 0.599840 +9 1.929649 1.705369 0.612201 +10 1.916013 1.646228 0.623175 +11 1.825515 1.586714 0.634298 +12 1.795780 1.529771 0.645840 +13 1.725532 1.476651 0.656197 +14 1.673942 1.429790 0.666030 +15 1.639384 1.392116 0.674128 +16 1.605681 1.359316 0.681356 +17 1.560283 1.337794 0.686116 +18 1.543926 1.323153 0.689276 +19 1.531950 1.318164 0.690415 +20 1.494068 1.316459 0.690586 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4-2nd.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.537720 0.395818 0.886000 +epoch train_loss valid_loss accuracy +1 0.332603 0.232112 0.930000 +epoch train_loss valid_loss accuracy +1 0.267323 0.230307 0.927000 +epoch train_loss valid_loss accuracy +1 0.216402 0.226042 0.930000 +2 0.231040 0.232696 0.936000 +3 0.182048 0.217882 0.934000 +4 0.170389 0.212531 0.937000 +5 0.148332 0.214293 0.937000 +6 0.124968 0.210322 0.936000 +7 0.117591 0.234207 0.936000 +8 0.109146 0.218597 0.938000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4-2nd.m +Loss and accuracy using (cls_best): [0.21502711, tensor(0.9367)] +``` \ No newline at end of file diff --git a/results/logs/ja.md b/results/logs/ja.md new file mode 100644 index 0000000..6e0872f --- /dev/null +++ b/results/logs/ja.md @@ -0,0 +1,90 @@ +## +``` +python -m ulmfit lm --dataset-path data/wiki/ja-100 --cuda-id=0 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 30000 \ +--lang ja --qrnn=False - train 10 --bs=50 --drop_mult=0 +Max vocab: 30000 +Cache dir: data/wiki/ja-100/models/sp30k +Model dir: data/wiki/ja-100/models/sp30k/lstm_nl4.m +Running tokenization +Wiki text was split to 98375 articles +Wiki text was split to 138 articles +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': None, 'pretrained_model': None, 'drop_mult': 0} dps: [0.25 0.1 0.2 0.02 0.15] +Training lm from random weights +epoch train_loss valid_loss accuracy +1 3.211025 3.328014 0.396197 +2 3.119410 3.286294 0.395946 +3 3.042064 3.247161 0.403915 +4 3.023840 3.161323 0.413816 +5 2.944752 3.102044 0.423163 +6 2.907167 3.015610 0.434095 +7 2.796073 2.927566 0.447088 +8 2.715568 2.828766 0.461556 +9 2.717255 2.747889 0.473289 +10 2.619846 2.731164 0.477403 +data/wiki/ja-100/models/sp30k +Saving info data/wiki/ja-100/models/sp30k/lstm_nl4.m/info.json +``` + +## MLDoc +MultiCCA 85.35%, ULMFiT 89.20% +``` +python -m ulmfit cls --dataset-path data/mldoc/ja-1 --base-lm-path data/wiki/ja-100/models/sp30k/lstm_nl4.m --lang=ja --name 'nl4' --cuda-id=1 - train 20 --bs 40 --num-cls-epochs=8 +Max vocab: 30000 +Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k +Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m +Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv +Running tokenization... +Saving tokenized: cls.trn 13500, cls.val 1500 +Running tokenization... +Saving tokenized: cls.trn 1000, cls.val 1000 +Size of vocabulary: 30000 +First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '', '▁', '▁、', '▁の', '▁。', '▁に', '▁を', '▁は', '▁年', '▁が', '▁)', '▁('] +Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15] +Unknown tokens 0, first 100: [] +Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ja-100/models/sp30k/lstm_nl4.m/../itos')] +epoch train_loss valid_loss accuracy +1 2.828645 2.386208 0.518716 +epoch train_loss valid_loss accuracy +1 2.462274 2.191761 0.549783 +2 2.229925 1.982564 0.586238 +3 2.043816 1.805435 0.616238 +4 1.885779 1.674736 0.637964 +5 1.773445 1.575366 0.653925 +6 1.713029 1.490263 0.667570 +7 1.660558 1.419641 0.680072 +8 1.579792 1.357093 0.690826 +9 1.459628 1.298609 0.701452 +10 1.433604 1.251296 0.710232 +11 1.439143 1.202794 0.719104 +12 1.399083 1.158469 0.728430 +13 1.310390 1.120877 0.736382 +14 1.322389 1.085479 0.744013 +15 1.272924 1.056051 0.750401 +16 1.235312 1.034233 0.755225 +17 1.227864 1.016682 0.759288 +18 1.209589 1.007038 0.761234 +19 1.173158 1.001694 0.762281 +20 1.189994 1.000854 0.762526 +/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k +Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m/info.json +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.745803 0.554439 0.819000 +epoch train_loss valid_loss accuracy +1 0.620647 0.392026 0.856000 +epoch train_loss valid_loss accuracy +1 0.489173 0.369560 0.869000 +epoch train_loss valid_loss accuracy +1 0.406491 0.365988 0.872000 +2 0.392645 0.351823 0.876000 +3 0.386403 0.331737 0.880000 +4 0.361338 0.333245 0.882000 +5 0.319456 0.347253 0.879000 +6 0.295419 0.350348 0.885000 +7 0.286144 0.348592 0.879000 +8 0.278896 0.358145 0.877000 +Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp30k/lstm_nl4.m +Loss and accuracy using (cls_best): [0.29789856, tensor(0.8920)] +``` \ No newline at end of file