mirror of
https://github.com/wassname/multifit.git
synced 2026-09-10 12:12:50 +08:00
Merge branch 'master' of https://github.com/n-waves/ulmfit-multilingual into cls
This commit is contained in:
NAUSICAA\Julian
+14
-27
@@ -35,33 +35,28 @@ CLASSES = ['neg', 'pos', 'unsup']
|
||||
number_match_re = re.compile(r'^([0-9]+[,.]?)+$')
|
||||
number_split_re = re.compile(r'([,.])')
|
||||
|
||||
class MosesTokenizerFunc(BaseTokenizer):
|
||||
"Wrapper around a MosesTokenizer to make it a `BaseTokenizer`."
|
||||
def __init__(self, lang:str):
|
||||
super().__init__(lang=lang)
|
||||
self.tok = MosesTokenizer(lang)
|
||||
class MosesPreprocessingFunc():
|
||||
|
||||
def tokenizer(self, t:str) -> List[str]:
|
||||
return self.tok.tokenize(t, return_str=False, escape=False)
|
||||
def __init__(self, lang: str):
|
||||
self.mt = MosesTokenizer(lang)
|
||||
|
||||
def add_special_cases(self, toks:Collection[str]):
|
||||
for w in toks:
|
||||
assert len(self.tokenizer(w))==1, f"Tokenizer is unable to keep {w} as one token!"
|
||||
def __call__(self, t: str) -> str:
|
||||
return self.mt.tokenize(t, return_str=True, escape=True)
|
||||
|
||||
class SentencePieceTokenizer(Tokenizer):
|
||||
"Put together rules and a tokenizer function to tokenize text with multiprocessing."
|
||||
def __init__(self, spm_model, lang:str='en', pre_rules:ListRules=None,
|
||||
post_rules:ListRules=None, special_cases:Collection[str]=None, n_cpus:int=None, use_moses=False):
|
||||
post_rules:ListRules=None, special_cases:Collection[str]=None, n_cpus:int=None):
|
||||
# moses is added to preprocessing functions
|
||||
super().__init__(self.tok_fun_with_sp, lang, pre_rules, post_rules, special_cases, n_cpus)
|
||||
self.spm_model = spm_model
|
||||
self.use_moses = use_moses
|
||||
|
||||
def tok_fun_with_sp(self, lang):
|
||||
try:
|
||||
import sentencepiece as spm
|
||||
except ImportError:
|
||||
raise Exception('sentencepiece module is missing: run `pip install sentencepiece`')
|
||||
tok = MosesTokenizerFunc(lang) if self.use_moses else BaseTokenizer(lang)
|
||||
tok = BaseTokenizer(lang)
|
||||
tok.sp = spm.SentencePieceProcessor()
|
||||
tok.sp.Load(str(self.spm_model))
|
||||
return tok
|
||||
@@ -72,9 +67,8 @@ class SentencePieceTokenizer(Tokenizer):
|
||||
toks = tok.sp.EncodeAsPieces(" ".join(toks))
|
||||
return toks
|
||||
|
||||
def get_sentencepiece(cache_dir:PathOrStr, load_text,pre_rules:ListRules=None, post_rules:ListRules=None,
|
||||
vocab_size:int=30000, model_type:str='unigram', input_sentence_size:int=1E7,
|
||||
use_moses=False, lang='en'):
|
||||
def get_sentencepiece(cache_dir:PathOrStr, load_text, pre_rules: ListRules=None, post_rules:ListRules=None,
|
||||
vocab_size:int=30000, model_type:str='unigram', input_sentence_size:int=1E7, lang='en'):
|
||||
try:
|
||||
import sentencepiece as spm
|
||||
except ImportError:
|
||||
@@ -85,19 +79,13 @@ def get_sentencepiece(cache_dir:PathOrStr, load_text,pre_rules:ListRules=None, p
|
||||
post_rules = post_rules if post_rules is not None else defaults.text_post_rules
|
||||
|
||||
special_cases = defaults.text_spec_tok
|
||||
|
||||
if not os.path.isfile(cache_dir / 'spm.model') or not os.path.isfile(cache_dir / f'itos.pkl'):
|
||||
# load the text from the train tokens file
|
||||
text = load_text()
|
||||
text = filter(lambda x: len(x.rstrip(" ")), text)
|
||||
text = (reduce(lambda t, rule: rule(t), pre_rules, line) for line in text)
|
||||
if use_moses:
|
||||
mt = MosesTokenizer(lang)
|
||||
splitter = lambda t: mt.tokenize(t, return_str=False, escape=False)
|
||||
else:
|
||||
splitter = lambda t: t.split()
|
||||
def cleanup_n_postprocess(t):
|
||||
t = splitter(t)
|
||||
t = t.split()
|
||||
for r in post_rules:
|
||||
t = r(t)
|
||||
return ' '.join(t)
|
||||
@@ -128,10 +116,9 @@ def get_sentencepiece(cache_dir:PathOrStr, load_text,pre_rules:ListRules=None, p
|
||||
# We cannot use lambdas or local methods here, since `tok_func` needs to be
|
||||
# pickle-able in order to be called in subprocesses when multithread tokenizing
|
||||
tokenizer = SentencePieceTokenizer(cache_dir/'spm.model',
|
||||
use_moses=use_moses,
|
||||
lang=lang,
|
||||
pre_rules=pre_rules,
|
||||
post_rules=post_rules)
|
||||
lang=lang,
|
||||
pre_rules=pre_rules,
|
||||
post_rules=post_rules)
|
||||
return {'tokenizer': tokenizer, 'vocab': vocab}
|
||||
|
||||
|
||||
|
||||
Executable
+18
@@ -0,0 +1,18 @@
|
||||
#!/usr/bin/env bash
|
||||
set -x
|
||||
ZIPNAME="preprocessed_wiki_8langs.zip"
|
||||
OUTDIR="data/wiki"
|
||||
wget -nc 'https://www.dropbox.com/sh/srfwvur6orq0cre/AAAQc36bcD17C1KM1mneXN7fa/data/wiki?dl=1' -O "${ZIPNAME}"
|
||||
mkdir -p "${OUTDIR}"
|
||||
unzip "${ZIPNAME}" -d "${OUTDIR}"
|
||||
|
||||
for archive in "${OUTDIR}"/??-100.tar.gz; do tar xvf "${archive}" -C "${OUTDIR}" && rm "${archive}"; done
|
||||
|
||||
#optionally download models
|
||||
MODELS="pretrained_lm_models.zip"
|
||||
read -r -p "Download pretrained lm models? [y/N] " response
|
||||
if [[ "$response" =~ ^[yY]$ ]]
|
||||
then
|
||||
wget -nc 'https://www.dropbox.com/sh/srfwvur6orq0cre/AAABRFdrCNHmbpf4nNcMiJwJa/models/data/wiki?dl=1' -O "${MODELS}"
|
||||
unzip "${MODELS}" -d "${OUTDIR}"
|
||||
fi
|
||||
+33
-17
@@ -1,27 +1,43 @@
|
||||
|
||||
# non-zeroshot
|
||||
## Supervised classification results on MLDoc
|
||||
| Model | en | de | es | fr | it | ja | ru | zh |
|
||||
|----------------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|------------|
|
||||
|LASER | 90.73 | 92.70 | 88.75 | 90.80 | 85.93 | 85.15 | 84.65 | 88.98 |
|
||||
|MultiCCA | 92.2 | 93.70 | 94.45 | 92.05 | 85.55 | 85.35 | 85.65 | 87.30 |
|
||||
|ULMFiT | | **95.4** | **95.15** | **93.67** | **88.42** | **89.20** | **87.27** | |
|
||||
|ULMFiT 100 | | 91.35 | 83.32 | 88.77 | 77.99 | 71.12 | 72.20 | |
|
||||
|ULMFiT | | **95.4** | **95.15** | **93.67** | **88.42** | **89.20** | **87.27** | 90.20 |
|
||||
|ULMFiT sp-fixed | | 95.6 | 94.80 | 94.20 | 88.52 | 88.72 | 86.85 | 90.47 |
|
||||
|ULMFiT 100 | | 91.35 | 83.32 | 88.77 | 77.99 | 71.12 | 72.20 | |
|
||||
|Bert Multi | 93.23% | 94.0% | **95.15** | 93.20 | 85.82 | 87.48 | 86.85 | **90.72** |
|
||||
|
||||
# Zero shot approaches
|
||||
^ - sp60k lstm nl 4
|
||||
|
||||
| Model | en | de | es | fr | it | ja | ru | zh |
|
||||
|----------------------|------------|-----------|-----------|-----------|-----------|-----------|-----------|------------|
|
||||
|LASER 0 shot | 80.75 (en) | 87.03 (fr)| 82.60 (it)| 82.83 (de)| 73.25 (de)| 60.95 (en)| 68.83 (it)| 72.90 (de) |
|
||||
|LASER base 0 shot | | 86.48 | 79.23 | 76.73 |
|
||||
|ULMFiT 0 shot | | **91.97**| **85.35** | 85.54 |
|
||||
|ULMFiT 100 for comp. | | 91.35 | 83.32 | 88.77 | 77.99 | 71.12 | 72.20 | |
|
||||
## Zero shot approaches
|
||||
|
||||
| Model | de | es | fr | it | ru | zh |
|
||||
|----------------------|------------|------------|-----------|-----------|-----------|-----------|
|
||||
| LASER-de | | 81.40 | 81.50 | 74.53 | 64.58 | 73.20 |
|
||||
| LASER-fr | 88.75 | 80.12 | | 72.58 | 67.35 | 79.40 |
|
||||
| LASER-en | 87.65 | 75.48 | 84.00 | 71.18 | 66.58 | 76.65 |
|
||||
| | | | | | | |
|
||||
| ULMFiT on LASER-de | | **85.50** | 87.37 | **78.75** | 66.95 | 72.32 |
|
||||
| ULMFiT on LASER-fr | 92.22 | 81.00 | | 76.88 | 68.33 | **84.65** |
|
||||
| ULMFiT on LASER-en | **92.95** | 80.50 | **88.78** | 76.20 | **70.05** | 80.45 |
|
||||
| | | | | | | |
|
||||
| % impr over LASER-de | | 22% | 32% | 17% | 7% | *-3%* |
|
||||
| % impr over LASER-fr | 31% | 4% | | 16% | 3% | 25% |
|
||||
| % impr over LASER-en | 43% | 20% | 30% | 17% | 10% | 16% |
|
||||
| ULMFiT 100 for comp. | 91.35 | 83.32 | 88.77 | 77.99 | 71.12 | |
|
||||
| | | | | | | |
|
||||
| Bert Multilingual-EN | 74.50 | 61.85 | 69.77 | 57.73 | 51.10 | 64.08 |
|
||||
|
||||
To simulate ulmfit zero shot we add noise to the training labels to simulate training from Laser labels
|
||||
All ULMFiT examples above were trained on 1k training data generated by a LASER classification model
|
||||
|
||||
## Noise resistance
|
||||
|
||||
| Model | en | de | es | fr | it | ja | ru | zh |
|
||||
|---------------------------------|------------|-----------|-----------|-----------|-----------|-----------|-----------|------------|
|
||||
|LASER 0 shot | 80.75 (en) | 87.03 (fr)| 82.60 (it)| 82.83 (de)| 73.25 (de)| 60.95 (en)| 68.83 (it)| 72.90 (de) |
|
||||
|ULMFiT | | **95.4** | **95.15** | **93.67** | **88.42** | **89.20** | **87.27** | |
|
||||
| % of noise | 20% | 13% | 18% | 18% | 27% | 40% | 32% | 28% |
|
||||
|ULMFiT trained on 1k noisy exmp. | | 94.49 | 93.12 | 90.49 | 83.72 | 74.72 | 75.67 | |
|
||||
|
||||
| Model | en | de | es | fr | it | ja | ru | zh |
|
||||
|----------------------|------------|-----------|-----------|-----------|-----------|-----------|-----------|------------|
|
||||
|LASER 0 shot | 80.75 (en) | 87.03 (fr)| 82.60 (it)| 82.83 (de)| 73.25 (de)| 60.95 (en)| 68.83 (it)| 72.90 (de) |
|
||||
|ULMFiT | | **95.4** | **95.15** | **93.67** | **88.42** | **89.20** | **87.27** | |
|
||||
| Noise | 20% | 13% | 18% | 18% | 27% | 40% | 32% | 28% |
|
||||
|ULMFiT noise ~ 0 shot | | 94.49 | 93.12 | 90.49 | 83.72 | 74.72 | 75.67 | |
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
# EN
|
||||
## SP30k LSTM nl 4
|
||||
### LM
|
||||
|
||||
### MLDoc
|
||||
|
||||
+4
-2
@@ -1,6 +1,8 @@
|
||||
# ES
|
||||
|
||||
## SP30k LSTM nl 4
|
||||
### LM
|
||||
````
|
||||
|
||||
|
||||
python -m ulmfit lm --dataset-path data/wiki/es-100 --cuda-id=0 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 30000 --lang es --qrnn=False - train 10 --bs=50 --drop_mult=0
|
||||
Running tokenization
|
||||
Wiki text was split to 96224 articles
|
||||
|
||||
+3
-3
@@ -29,8 +29,8 @@ data/wiki/fr-100/models/sp30k
|
||||
Saving info data/wiki/fr-100/models/sp30k/lstm_nl4.m/info.json
|
||||
```
|
||||
|
||||
## MLDocs
|
||||
### First run
|
||||
### MLDocs
|
||||
#### First run
|
||||
MultiCCA 92.05, ulmfit 93.90
|
||||
```
|
||||
python -m ulmfit cls --dataset-path data/mldoc/fr-1 --base-lm-path data/wiki/fr-100/models/sp30k/lstm_nl4.m --lang=fr --name 'nl4' --cuda-id=1 - train 20 --bs 40
|
||||
@@ -86,7 +86,7 @@ Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/mod
|
||||
Loss and accuracy using (cls_best): [0.18914989, tensor(0.9390)]
|
||||
```
|
||||
|
||||
## Second run
|
||||
#### Second run
|
||||
MultiCCA 92.05, ulmfit 93.67
|
||||
```
|
||||
python -m ulmfit cls --dataset-path data/mldoc/fr-1 --base-lm-path data/wiki/fr-100/models/sp30k/lstm_nl4.m --lang=fr --name 'nl4-2nd' --cuda-id=1 - train 20 --bs 40 --num-cls-epochs=8
|
||||
|
||||
+4
-2
@@ -1,4 +1,6 @@
|
||||
##
|
||||
# JA
|
||||
## SP30k LSTM nl 4
|
||||
### LM
|
||||
```
|
||||
python -m ulmfit lm --dataset-path data/wiki/ja-100 --cuda-id=0 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 30000 \
|
||||
--lang ja --qrnn=False - train 10 --bs=50 --drop_mult=0
|
||||
@@ -27,7 +29,7 @@ data/wiki/ja-100/models/sp30k
|
||||
Saving info data/wiki/ja-100/models/sp30k/lstm_nl4.m/info.json
|
||||
```
|
||||
|
||||
## MLDoc
|
||||
### MLDoc
|
||||
MultiCCA 85.35%, ULMFiT 89.20%
|
||||
```
|
||||
python -m ulmfit cls --dataset-path data/mldoc/ja-1 --base-lm-path data/wiki/ja-100/models/sp30k/lstm_nl4.m --lang=ja --name 'nl4' --cuda-id=1 - train 20 --bs 40 --num-cls-epochs=8
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
# QRNN DE
|
||||
## SP30k nl
|
||||
### LM
|
||||
```
|
||||
python -m ulmfit lm --dataset-path data/wiki/de-100 --bidir=False --qrnn=True --nl 4 --tokenizer='sp' --max-vocab 30000 --lang de --name 'nl4' --cuda-id=0 - train 10 --drop-mult=0 --bs=50
|
||||
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', "▁&'", 'en', 's', '-']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Training lm from random weights
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 2.790653 2.867094 0.511392
|
||||
2 2.742032 2.843288 0.510885
|
||||
3 2.696114 2.833874 0.512062
|
||||
4 2.671780 2.786312 0.516448
|
||||
5 2.611292 2.725993 0.522723
|
||||
6 2.542737 2.655713 0.530968
|
||||
7 2.572076 2.582141 0.539928
|
||||
8 2.465960 2.509654 0.549987
|
||||
9 2.405682 2.448580 0.558674
|
||||
10 2.339395 2.428111 0.562502
|
||||
```
|
||||
|
||||
### MLDocs
|
||||
```
|
||||
python -m ulmfit cls --dataset-path data/mldoc/de-1 --cuda-id=0 --base-lm-path data-filtered/data/wiki/de-100/models/sp30k/qrnn_nl4.m --lang=de --name 'nl4' - train 20 --bs 40 --cls-max-len 700
|
||||
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/de-1/models/sp30k
|
||||
Model dir: /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/qrnn_nl4.m
|
||||
Loading validation /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/de-1/de.dev.csv
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 13500, cls.val 1500
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', "▁&'", 'en', 's', '-']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Loading pretrained model
|
||||
Unknown tokens 0, first 100: []
|
||||
Training lm from: [PosixPath('/home/marcin/github/n-waves/ulmfit-multilingual/data-filtered/data/wiki/de-100/models/sp30k/qrnn_nl4.m/lm_best'), PosixPath('/home/marcin/github/n-waves/ulmfit-multilingual/data-filtered/data/wiki/de-100/models/sp30k/qrnn_nl4.m/../itos')]
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 3.450698 2.601732 0.527671
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 2.888087 2.477170 0.542949
|
||||
2 2.621279 2.300024 0.568743
|
||||
3 2.313220 2.120824 0.592728
|
||||
4 2.176746 1.973596 0.613343
|
||||
5 2.114441 1.857317 0.628628
|
||||
6 2.022593 1.765069 0.642017
|
||||
7 1.936942 1.696150 0.651549
|
||||
8 1.860200 1.622848 0.661923
|
||||
9 1.795039 1.549579 0.673416
|
||||
10 1.740739 1.500053 0.681305
|
||||
11 1.695835 1.448141 0.689201
|
||||
12 1.605702 1.402924 0.697096
|
||||
13 1.582328 1.354327 0.706123
|
||||
14 1.548034 1.316290 0.712870
|
||||
15 1.496170 1.282155 0.719413
|
||||
16 1.514243 1.255556 0.724801
|
||||
17 1.482411 1.236461 0.728380
|
||||
18 1.458308 1.223498 0.730708
|
||||
19 1.422691 1.218288 0.731713
|
||||
20 1.380592 1.217068 0.731893
|
||||
/home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/de-1/models/sp30k
|
||||
Saving info /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/qrnn_nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.529402 0.376163 0.900000
|
||||
Better model found at epoch 1 with val_loss value: 0.3761630356311798.
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.290838 0.252989 0.916000
|
||||
Better model found at epoch 1 with val_loss value: 0.25298893451690674.
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.184352 0.204892 0.941000
|
||||
Better model found at epoch 1 with val_loss value: 0.20489171147346497.
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.113328 0.204136 0.947000
|
||||
Better model found at epoch 1 with val_loss value: 0.20413607358932495.
|
||||
2 0.106220 0.200674 0.949000
|
||||
Better model found at epoch 2 with val_loss value: 0.20067360997200012.
|
||||
Saving models at /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/qrnn_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.15208693, tensor(0.9532)]
|
||||
0.15208692848682404
|
||||
0.953249990940094
|
||||
```
|
||||
@@ -0,0 +1,108 @@
|
||||
# QRNN EN
|
||||
## SP30k nl 4
|
||||
### LM
|
||||
|
||||
```
|
||||
python -m ulmfit lm --dataset-path data/wiki/wikitext-103 --bidir=False --qrnn=True --nl 4 --tokenizer='sp' --max-vocab 30000 --lang en --name 'nl4' --cuda-id=1 - train 10 --drop-mult=0 --bs=50
|
||||
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁the', '▁,', '▁.', 's', '▁of', '▁and', '▁in', '▁to', '▁a', 'ed']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.5} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Training lm from random weights
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 3.184221 3.256314 0.438527
|
||||
2 3.084555 3.241498 0.435628
|
||||
3 3.099060 3.258447 0.435060
|
||||
4 3.119621 3.220939 0.437597
|
||||
5 3.073662 3.165012 0.445108
|
||||
6 2.938047 3.086962 0.452921
|
||||
7 2.920506 2.998151 0.462940
|
||||
8 2.920506 2.899240 0.474378
|
||||
9 2.862836 2.835098 0.485305
|
||||
10 2.891070 2.810929 0.489867
|
||||
```
|
||||
|
||||
### LM, BS=128, drop-mult=0.5
|
||||
```
|
||||
python -m ulmfit lm --dataset-path data/wiki/wikitext-103 --bidir=False --qrnn=True --nl 4 --tokenizer='sp' --max-vocab 30000 --lang en --name 'nl4-bs128' --cuda-id=1 - train 10 --drop-mult=0.5 --bs=128
|
||||
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁the', '▁,', '▁.', 's', '▁of', '▁and', '▁in', '▁to', '▁a', 'ed']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.5} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Training lm from random weights
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 3.413345 3.280860 0.433011
|
||||
2 3.219606 3.129479 0.444172
|
||||
3 3.136091 3.094905 0.448493
|
||||
4 3.145281 3.033001 0.452830
|
||||
5 3.100366 2.980189 0.458984
|
||||
6 3.062894 2.923044 0.464841
|
||||
7 3.001627 2.834753 0.475316
|
||||
8 2.979051 2.792044 0.480915
|
||||
9 2.933140 2.733279 0.488346
|
||||
10 2.964397 2.720861 0.490423
|
||||
```
|
||||
|
||||
### MLDocs
|
||||
```
|
||||
python -m ulmfit cls --dataset-path data/mldoc/en-1 --cuda-id=0 --base-lm-path data-filtered/data/wiki/wikitext-103/models/sp30k/qrnn_nl4.m --lang=en --name 'nl4' - train 20 --bs 40 --cls-max-len 700
|
||||
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/en-1/models/sp30k
|
||||
Model dir: /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/en-1/models/sp30k/qrnn_nl4.m
|
||||
Loading validation /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/en-1/en.dev.csv
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 13500, cls.val 1500
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁the', '▁,', '▁.', 's', '▁of', '▁and', '▁in', '▁to', '▁a', 'ed']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Loading pretrained model
|
||||
Unknown tokens 0, first 100: []
|
||||
Training lm from: [PosixPath('/home/marcin/github/n-waves/ulmfit-multilingual/data-filtered/data/wiki/wikitext-103/models/sp30k/qrnn_nl4.m/lm_best'), PosixPath('/home/marcin/github/n-waves/ulmfit-multilingual/data-filtered/data/wiki/wikitext-103/models/sp30k/qrnn_nl4.m/.
|
||||
./itos')]
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 4.459886 3.692770 0.364677
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 3.962907 3.560222 0.379027
|
||||
2 3.673292 3.378484 0.402066
|
||||
3 3.460093 3.191662 0.424295
|
||||
4 3.296515 3.030681 0.442995
|
||||
5 3.161650 2.891829 0.459052
|
||||
6 3.022674 2.776469 0.473280
|
||||
7 2.974365 2.686321 0.484403
|
||||
8 2.869587 2.593854 0.496297
|
||||
9 2.785321 2.509093 0.506853
|
||||
10 2.677728 2.440328 0.516178
|
||||
11 2.641243 2.371950 0.525810
|
||||
12 2.652385 2.320008 0.533105
|
||||
13 2.547195 2.261057 0.542046
|
||||
14 2.491570 2.216933 0.548810
|
||||
15 2.454437 2.179364 0.555077
|
||||
16 2.414449 2.147612 0.559972
|
||||
17 2.358593 2.125351 0.563405
|
||||
18 2.362696 2.111580 0.565614
|
||||
19 2.341626 2.104268 0.566749
|
||||
20 2.342680 2.102918 0.566966
|
||||
/home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/en-1/models/sp30k
|
||||
Saving info /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/en-1/models/sp30k/qrnn_nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.622379 0.422002 0.882000
|
||||
Better model found at epoch 1 with val_loss value: 0.42200201749801636.
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.313018 0.275563 0.908000
|
||||
Better model found at epoch 1 with val_loss value: 0.27556276321411133.
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.241521 0.174606 0.933000
|
||||
Better model found at epoch 1 with val_loss value: 0.1746061146259308.
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.125556 0.170286 0.940000
|
||||
Better model found at epoch 1 with val_loss value: 0.17028628289699554.
|
||||
2 0.107322 0.181366 0.939000
|
||||
Saving models at /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/en-1/models/sp30k/qrnn_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.18917121, tensor(0.9388)]
|
||||
0.1891712099313736
|
||||
0.9387500286102295
|
||||
```
|
||||
@@ -0,0 +1,88 @@
|
||||
# QRNN ES
|
||||
|
||||
## SP30k nl 4
|
||||
### LM
|
||||
```
|
||||
python -m ulmfit lm --dataset-path data/wiki/es-100 --bidir=False --qrnn=True --nl 4 --tokenizer='sp' --max-vocab 30000 --lang es --name 'nl4' --cuda-id=0 - train 10 --drop-mult=0 --bs=50
|
||||
|
||||
Wiki text was split to 161509 articles
|
||||
Wiki text was split to 78 articles
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁.', '▁la', '▁el', '▁en', '▁y', 's', '▁a', "▁&'"]
|
||||
Training args: {'clip': 0.12, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Training lm from random weights
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 3.067289 3.640350 0.357276
|
||||
2 2.958243 3.619773 0.358111
|
||||
3 3.033412 3.587700 0.359495
|
||||
4 2.933573 3.525202 0.367685
|
||||
5 2.904549 3.467990 0.372583
|
||||
6 2.798806 3.409506 0.380045
|
||||
7 2.733132 3.303108 0.391922
|
||||
8 2.675272 3.224150 0.401143
|
||||
9 2.635299 3.166430 0.410160
|
||||
10 2.656724 3.145599 0.413176
|
||||
```
|
||||
|
||||
### MLDocs
|
||||
```
|
||||
python -m ulmfit cls --dataset-path data/mldoc/es-1 --cuda-id=0 --base-lm-path data-filtered/data/wiki/es-100/models/sp30k/qrnn_nl4.m --lang=es --name 'nl4' - train 20 --bs 40 --cls-max-len 700
|
||||
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/es-1/models/sp30k
|
||||
Model dir: /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/qrnn_nl4.m
|
||||
Loading validation /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 13013, cls.val 1445
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁.', '▁la', '▁el', '▁en', '▁y', 's', '▁a', "▁&'"]
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Loading pretrained model
|
||||
Unknown tokens 0, first 100: []
|
||||
Training lm from: [PosixPath('/home/marcin/github/n-waves/ulmfit-multilingual/data-filtered/data/wiki/es-100/models/sp30k/qrnn_nl4.m/lm_best'), PosixPath('/home/marcin/github/n-waves/ulmfit-multilingual/data-filtered/data/wiki/es-100/models/sp30k/qrnn_nl4.m/../itos')]
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 3.352874 2.367255 0.514858
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 2.796090 2.203233 0.536513
|
||||
2 2.515840 1.970145 0.576640
|
||||
3 2.198857 1.774013 0.610990
|
||||
4 2.035614 1.633484 0.633450
|
||||
5 1.944539 1.535505 0.649110
|
||||
6 1.848854 1.451618 0.661764
|
||||
7 1.788579 1.382675 0.673166
|
||||
8 1.675414 1.320675 0.683617
|
||||
9 1.614536 1.264944 0.694086
|
||||
10 1.618723 1.215493 0.702936
|
||||
11 1.504875 1.164356 0.712921
|
||||
12 1.411316 1.126858 0.721374
|
||||
13 1.421174 1.079897 0.731196
|
||||
14 1.352116 1.044965 0.738148
|
||||
15 1.318876 1.013755 0.745312
|
||||
16 1.268569 0.986391 0.751383
|
||||
17 1.273424 0.971129 0.754643
|
||||
18 1.256196 0.960661 0.757439
|
||||
19 1.233202 0.955790 0.758405
|
||||
20 1.230536 0.955070 0.758496
|
||||
/home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/es-1/models/sp30k
|
||||
Saving info /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/qrnn_nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.739119 0.438338 0.867000
|
||||
Better model found at epoch 1 with val_loss value: 0.438338041305542.
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.425376 0.207067 0.950000
|
||||
Better model found at epoch 1 with val_loss value: 0.20706671476364136.
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.311269 0.172416 0.956000
|
||||
Better model found at epoch 1 with val_loss value: 0.17241604626178741.
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.226164 0.166543 0.958000
|
||||
Better model found at epoch 1 with val_loss value: 0.1665433794260025.
|
||||
2 0.199775 0.167683 0.956000
|
||||
Saving models at /home/marcin/github/n-waves/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/qrnn_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.18184493, tensor(0.9448)]
|
||||
0.18184493482112885
|
||||
0.9447500109672546
|
||||
```
|
||||
@@ -0,0 +1,30 @@
|
||||
# IT
|
||||
## SP30k QRNN nl 4
|
||||
### LM
|
||||
```
|
||||
python -m ulmfit lm --dataset-path data/wiki/it-100/ --cuda-id=0 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 30000 --lang it --qrnn=True - train 10 --bs=50 --drop_mult=0
|
||||
Max vocab: 30000
|
||||
Cache dir: data/wiki/it-100/models/sp30k
|
||||
Model dir: data/wiki/it-100/models/sp30k/qrnn_nl4.m
|
||||
Tokenized data loaded
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁di', "▁&'", "'", '▁e', '▁il', '▁la', 'e', '▁in']
|
||||
Training args: {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Training lm from random weights
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 3.354224 3.749085 0.350236
|
||||
2 3.274838 3.697026 0.351104
|
||||
3 3.222462 3.680071 0.352152
|
||||
4 3.217652 3.628976 0.357922
|
||||
5 3.117965 3.563592 0.364370
|
||||
6 3.075397 3.483997 0.372794
|
||||
7 3.002098 3.394749 0.383217
|
||||
8 2.936974 3.316284 0.393616
|
||||
9 2.843549 3.258448 0.401605
|
||||
10 2.818070 3.240303 0.404684
|
||||
Total time: 10:49:44
|
||||
data/wiki/it-100/models/sp30k
|
||||
Saving info data/wiki/it-100/models/sp30k/qrnn_nl4.m/info.json
|
||||
```
|
||||
|
||||
### MLDoc
|
||||
@@ -0,0 +1,24 @@
|
||||
# QRNN RU
|
||||
## SP30k nl4
|
||||
### LM
|
||||
```
|
||||
python -m ulmfit lm --dataset-path data/wiki/ru-100 --bidir=False --qrnn=True --nl 4 --tokenizer='sp' --max-vocab 30000 --lang ru --name 'nl4' --cuda-id=0 - train 10 --drop-mult=0 --bs=50
|
||||
|
||||
Wiki text was split to 193047 articles
|
||||
Wiki text was split to 460 articles
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁в', 'а', '▁и', 'е', 'и', 'й', '▁на', '▁с']
|
||||
Training args: {'clip': 0.12, 'drop_mult': 0} dps: {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
|
||||
Training lm from random weights
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 3.273207 3.350111 0.429702
|
||||
2 3.169897 3.274238 0.433682
|
||||
3 3.162197 3.247077 0.435900
|
||||
4 3.131630 3.168798 0.445252
|
||||
5 3.042942 3.096774 0.453532
|
||||
6 2.950550 3.002989 0.465113
|
||||
7 2.833593 2.902871 0.478954
|
||||
8 2.829737 2.805592 0.492138
|
||||
9 2.746991 2.733609 0.503711
|
||||
10 2.687201 2.708546 0.508050
|
||||
```
|
||||
+714
-168
@@ -1,215 +1,761 @@
|
||||
# Laser Performance
|
||||
## Laser Perforamnce
|
||||
|
||||
Accuracy matrix:
|
||||
|
||||
| Train | en | de | es | fr | it | ru | zh |
|
||||
| Train | en | de | es | fr | it | ru | zh |
|
||||
|-------|-------|-------|-------|-------|-------|-------|-------|
|
||||
| en: | 90.88 | 86.48 | 67.62 | 61.98 | 69.95 | 22.95 | 11.65 |
|
||||
| de: | 73.23 | 92.90 | 77.23 | 74.05 | 72.30 | 24.80 | 9.93 |
|
||||
| es: | 65.62 | 80.58 | 92.03 | 73.28 | 69.03 | 34.10 | 12.58 |
|
||||
| fr: | 78.35 | 85.45 | 78.20 | 89.68 | 69.85 | 33.88 | 9.68 |
|
||||
| it: | 73.93 | 84.58 | 79.23 | 76.73 | 84.03 | 34.48 | 11.83 |
|
||||
| ru: | 57.33 | 63.78 | 45.80 | 52.78 | 51.15 | 66.08 | 36.28 |
|
||||
| zh: | 26.15 | 28.13 | 21.88 | 29.33 | 30.58 | 34.38 | 75.62 |
|
||||
|
||||
# DE
|
||||
Laser 0shot: 86.48, ULMFiT 0shot: 91.97
|
||||
```
|
||||
python ../../source/classify.py embed-2019-02-12/mldoc.en-en.h5 ~/workspace/ulmfit-multilingual/data/mldoc/de-1
|
||||
| Test: 86.48% | classes: 24.30 22.77 28.90 24.02
|
||||
Making train set
|
||||
| Train: 85.70% | classes: 27.00 21.40 27.60 24.00
|
||||
Accuracy 0.857
|
||||
0 1
|
||||
0 3 Tokio (Reuter) - Der Dollar ist am Donnerstag ...
|
||||
1 3 Kairo (Reuter) - Die ägyptische Zentralbank se...
|
||||
2 2 Bonn (Reuter) - Wegen einer Bombendrohung ist ...
|
||||
3 0 Berlin (Reuter) - Die Bahn AG will mit Hilfe p...
|
||||
4 3 08.15 Uhr MEZ - Deutsche Aktien nach den Rekor...
|
||||
| en: | 91.48 | 87.65 | 75.48 | 84.00 | 71.18 | 66.58 | 76.65 |
|
||||
| de: | 78.23 | 93.50 | 81.40 | 81.50 | 74.53 | 64.58 | 73.20 |
|
||||
| es: | 71.62 | 84.00 | 93.73 | 78.90 | 73.38 | 53.33 | 55.83 |
|
||||
| fr: | 81.30 | 88.75 | 80.12 | 90.85 | 72.58 | 67.35 | 79.40 |
|
||||
| it: | 74.33 | 83.53 | 80.58 | 79.78 | 84.48 | 66.45 | 63.35 |
|
||||
| ru: | 72.38 | 81.65 | 65.73 | 71.30 | 63.33 | 85.45 | 59.58 |
|
||||
| zh: | 74.98 | 81.35 | 72.20 | 73.28 | 70.08 | 66.23 | 88.30 |
|
||||
|
||||
Making dev set
|
||||
| Train: 85.60% | classes: 23.70 22.30 30.60 23.40
|
||||
Accuracy 0.856
|
||||
0 1
|
||||
0 1 New York (Reuter) - Das Vertrauen der US-Verbr...
|
||||
1 2 Tokio (Reuter) - Russische Patrouillenboote ha...
|
||||
2 2 Paris (Reuter) - Bei der Volksabstimmung in Al...
|
||||
3 2 Belgrad (Reuter) - Die serbische Polizei hat n...
|
||||
4 0 München (Reuter) - Der Stuttgarter Bosch-Konze...
|
||||
|
||||
## Evaluation of Laser Performance
|
||||
```
|
||||
```
|
||||
python -m ulmfit cls --dataset-path data/mldoc/de-1-laser --base-lm-path data/mldoc/de-1/models/sp30k/lstm_nl4.m --lang=de --name 'nl4' --cuda-id=1 - train 0 --bs 40 --num-cls-epochs=2
|
||||
python -m ulmfit eval --glob="mldoc/*-1/models/sp60k/lstm_nl4.m" --dataset_template="{}-laser-*" --name nl4 --cuda-id=0 ✘ 130
|
||||
Max vocab: 60000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp60k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp60k/lstm_nl4.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/zh.dev.csv
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 13500, cls.val 1500
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 60000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁是', '▁人']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp60k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp60k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp60k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp60k/lstm_nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.789124 0.620514 0.781000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.621348 0.524669 0.828000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.497774 0.467979 0.842000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.445851 0.479755 0.833000
|
||||
2 0.424097 0.468968 0.826000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp60k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.53502685, tensor(0.8235)]
|
||||
[('data/mldoc/zh-1-laser-fr/models/sp60k/lstm_nl4.m', 0.8234999775886536)]
|
||||
python -m ulmfit eval --glob="mldoc/*-1/models/sp60k/lstm_nl4.m" --dataset_template="{}-laser-*" --name nl4 --cuda-id=0 ✘ 130
|
||||
Max vocab: 60000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp60k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp60k/lstm_nl4.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/zh.dev.csv
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 13500, cls.val 1500
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 60000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁是', '▁人']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp60k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp60k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp60k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp60k/lstm_nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.789124 0.620514 0.781000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.621348 0.524669 0.828000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.497774 0.467979 0.842000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.445851 0.479755 0.833000
|
||||
2 0.424097 0.468968 0.826000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp60k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.53502685, tensor(0.8235)]
|
||||
[('data/mldoc/zh-1-laser-fr/models/sp60k/lstm_nl4.m', 0.8234999775886536)]
|
||||
(fastaiv1) pczapla@galatea ~/w/ulmfit-multilingual ❯❯❯ python -m ulmfit eval --glob="mldoc/*-1/models/sp30k/lstm_nl4.m" --dataset_template="{}-laser-*" --name nl4 --cuda-id=0
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser/models/sp30k/lstm_nl4.m
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser/de.dev.csv
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/models/sp30k/lstm_nl4.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/it.dev.csv
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 13500, cls.val 1500
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁di', "▁&'", "'", '▁e', '▁il', '▁la', 'e', '▁in']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/models/sp30k/lstm_nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.823176 0.588192 0.802000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.654395 0.465622 0.846000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.536948 0.453061 0.847000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.488410 0.454361 0.845000
|
||||
2 0.450684 0.448873 0.849000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/models/sp30k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.6332891, tensor(0.7875)]
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/models/sp30k/lstm_nl4.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/de.dev.csv
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 13500, cls.val 1500
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser/models/sp30k/lstm_nl4.m/info.json
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/models/sp30k/lstm_nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.671869 0.466408 0.863000
|
||||
1 0.566941 0.389549 0.882000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.518045 0.388151 0.887000
|
||||
1 0.399470 0.302616 0.898000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.375156 0.370652 0.893000
|
||||
1 0.349054 0.336955 0.900000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.339284 0.367223 0.891000
|
||||
2 0.314325 0.369492 0.891000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser/models/sp30k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.25416428, tensor(0.9197)]
|
||||
0.25416427850723267
|
||||
0.9197499752044678
|
||||
```
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# ES from IT
|
||||
```
|
||||
python ../../source/classify.py embed-2019-02-12/mldoc.it-it.h5 ~/workspace/ulmfit-multilingual/data/mldoc/es-1 ✘ 130
|
||||
| Test: 79.23% | classes: 25.48 16.45 24.18 33.90
|
||||
Making train set
|
||||
| Train: 80.30% | classes: 27.10 19.20 22.60 31.10
|
||||
Accuracy 0.803
|
||||
0 1
|
||||
0 3 LONDRES, 5 sep (Reuter) - El dólar se mantenía...
|
||||
1 1 MADRID, 30 dic (Reuter) - La Generalitat de Va...
|
||||
2 3 PARIS, 30 jun (Reuter) - La Bolsa de París neg...
|
||||
3 0 MADRID, 23 dic (Reuter) - La agencia de valore...
|
||||
4 0 MADRID, 4 Feb (Reuter) - El Banco Bilbao Vizca...
|
||||
|
||||
Making dev set
|
||||
| Train: 79.70% | classes: 25.40 17.50 26.20 30.90
|
||||
Accuracy 0.797
|
||||
0 1
|
||||
0 0 NUEVA YORK, 11 abr (Reuter) - MCI Communicatio...
|
||||
1 3 FRANCFORT, 17 jun (Reuter) - La Bolsa de Franc...
|
||||
2 1 BONN, 3 jun (Reuter) - Un destacado miembro de...
|
||||
3 2 LONDRES, 3 sep (Reuter) - El secretario de Def...
|
||||
4 2 MADRID, 3 oct (Reuter) - Las acciones de Pryca...
|
||||
```
|
||||
|
||||
```
|
||||
python -m ulmfit cls --dataset-path data/mldoc/es-1-laser-it --base-lm-path data/mldoc/es-1/models/sp30k/lstm_nl4.m --lang=es --name 'nl4' --cuda-id=1 - train 0 --bs 40 --num-cls-epochs=2
|
||||
```
|
||||
|
||||
# FR from IT
|
||||
```
|
||||
python ../../source/classify.py embed-2019-02-12/mldoc.it-it.h5 ~/workspace/ulmfit-multilingual/data/mldoc/fr-1
|
||||
| Test: 76.73% | classes: 21.65 21.98 31.77 24.60
|
||||
Making train set
|
||||
| Train: 79.20% | classes: 22.20 22.40 31.40 24.00
|
||||
Accuracy 0.792
|
||||
0 1
|
||||
0 2 WASHINGTON, 13 septembre, Reuter - Les Etats-U...
|
||||
1 1 PARIS, 10 juillet, Reuter - L'audit des financ...
|
||||
2 2 MOSCOU, 29 mai, Reuter - Après l'accord interv...
|
||||
3 2 PARIS, 1er octobre, Reuter - Le groupe communi...
|
||||
4 0 LONDRES, 3 juin, Reuter - National Grid Group ...
|
||||
|
||||
Making dev set
|
||||
| Train: 76.60% | classes: 23.30 20.10 33.00 23.60
|
||||
Accuracy 0.766
|
||||
0 1
|
||||
0 0 PARIS, 30 décembre, Reuter - Zodiac . Chiffre ...
|
||||
1 0 AJACCIO, 11 décembre, Reuter - Une charge de 7...
|
||||
2 0 BRUXELLES, 26 décembre, Reuter - 1997 s'annonc...
|
||||
3 0 PARIS, 26 septembre, Reuter - Alcatel Alsthom ...
|
||||
4 0 NEW YORK, 25 octobre, Reuter - La hausse plus ...
|
||||
```
|
||||
|
||||
```
|
||||
python -m ulmfit cls --dataset-path data/mldoc/fr-1-laser-it --base-lm-path data/mldoc/fr-1/models/sp30k/lstm_nl4.m --lang=fr --name 'nl4' --cuda-id=1 - train 0 --bs 40 --num-cls-epochs=2
|
||||
1 0.278230 0.333488 0.896000
|
||||
2 0.275510 0.343370 0.899000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/models/sp30k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.26227093, tensor(0.9222)]
|
||||
Traceback (most recent call last):
|
||||
File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/runpy.py", line 193, in _run_module_as_main
|
||||
"__main__", mod_spec)
|
||||
File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/runpy.py", line 85, in _run_code
|
||||
exec(code, run_globals)
|
||||
File "/home/pczapla/workspace/ulmfit-multilingual/ulmfit/__main__.py", line 58, in <module>
|
||||
fire.Fire(ULMFiT())
|
||||
File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/fire/core.py", line 127, in Fire
|
||||
component_trace = _Fire(component, args, context, name)
|
||||
File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/fire/core.py", line 366, in _Fire
|
||||
component, remaining_args)
|
||||
File "/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/fire/core.py", line 542, in _CallCallable
|
||||
result = fn(*varargs, **kwargs)
|
||||
File "/home/pczapla/workspace/ulmfit-multilingual/ulmfit/__main__.py", line 41, in eval
|
||||
dataset_path = get_dataset_path(base_model, dataset_template)
|
||||
File "/home/pczapla/workspace/ulmfit-multilingual/ulmfit/__main__.py", line 17, in get_dataset_path
|
||||
return list(ds.parent.glob(dataset_template.format(ds.name)))[0]
|
||||
IndexError: list index out of range
|
||||
(fastaiv1) pczapla@galatea ~/w/ulmfit-multilingual ❯❯❯ python -m ulmfit eval --glob="mldoc/*-1/models/sp30k/lstm_nl4.m" --dataset_template="{}-laser-*" --name nl4 --cuda-id=0 ✘ 1
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-it/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-it/models/sp30k/lstm_nl4.m
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-it/fr.dev.csv
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/models/sp30k/lstm_nl4.m
|
||||
Evaluating previously trained model
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/it.dev.csv
|
||||
Tokenized data loaded, lm.trn 13500, lm.val 1500
|
||||
Tokenized data loaded, cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁di', "▁&'", "'", '▁e', '▁il', '▁la', 'e', '▁in']
|
||||
Loss and accuracy using (cls_last): [0.6332891, tensor(0.7875)]
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/models/sp30k/lstm_nl4.m
|
||||
Evaluating previously trained model
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/de.dev.csv
|
||||
Tokenized data loaded, lm.trn 13500, lm.val 1500
|
||||
Tokenized data loaded, cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-']
|
||||
Loss and accuracy using (cls_last): [0.26227093, tensor(0.9222)]
|
||||
Skipping data/mldoc/ja-1/models/sp30k/lstm_nl4.m as template {}-laser-* was not found
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp30k/lstm_nl4.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/zh.dev.csv
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 13500, cls.val 1500
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp30k/lstm_nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.745636 0.601781 0.812000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.564749 0.435314 0.851000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.485875 0.428803 0.850000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.405431 0.439304 0.847000
|
||||
2 0.418333 0.442639 0.845000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp30k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.5289812, tensor(0.8465)]
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en/models/sp30k/lstm_nl4.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en/fr.dev.csv
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 13500, cls.val 1500
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-it/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-it/models/sp30k/lstm_nl4.m/info.json
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en/models/sp30k/lstm_nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.737947 0.627607 0.793000
|
||||
1 0.669493 0.510190 0.852000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.603060 0.513449 0.831000
|
||||
1 0.464863 0.349456 0.888000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.481312 0.499689 0.828000
|
||||
1 0.396977 0.335358 0.879000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.422958 0.508330 0.825000
|
||||
2 0.408061 0.493875 0.839000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-it/models/sp30k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.4295174, tensor(0.8555)]
|
||||
0.42951738834381104
|
||||
0.8554999828338623
|
||||
```
|
||||
# FR From EN
|
||||
```
|
||||
python ../../source/classify.py embed-2019-02-12/mldoc.en-en.h5 ~/workspace/ulmfit-multilingual/data/mldoc/fr-1
|
||||
| Test: 61.98% | classes: 11.85 41.10 40.05 7.00
|
||||
Making train set
|
||||
| Train: 63.70% | classes: 11.70 43.80 38.40 6.10
|
||||
Accuracy 0.637
|
||||
0 1
|
||||
0 2 WASHINGTON, 13 septembre, Reuter - Les Etats-U...
|
||||
1 1 PARIS, 10 juillet, Reuter - L'audit des financ...
|
||||
2 2 MOSCOU, 29 mai, Reuter - Après l'accord interv...
|
||||
3 2 PARIS, 1er octobre, Reuter - Le groupe communi...
|
||||
4 0 LONDRES, 3 juin, Reuter - National Grid Group ...
|
||||
|
||||
Making dev set
|
||||
| Train: 61.60% | classes: 11.90 40.90 39.70 7.50
|
||||
Accuracy 0.616
|
||||
0 1
|
||||
0 1 PARIS, 30 décembre, Reuter - Zodiac . Chiffre ...
|
||||
1 0 AJACCIO, 11 décembre, Reuter - Une charge de 7...
|
||||
2 1 BRUXELLES, 26 décembre, Reuter - 1997 s'annonc...
|
||||
3 1 PARIS, 26 septembre, Reuter - Alcatel Alsthom ...
|
||||
4 1 NEW YORK, 25 octobre, Reuter - La hausse plus ...
|
||||
```
|
||||
```
|
||||
|
||||
python -m ulmfit cls --dataset-path data/mldoc/fr-1-laser --base-lm-path data/mldoc/fr-1/models/sp30k/lstm_nl4.m --lang=fr --name 'nl4-laser' --cuda-id=1 - train 0 --bs 40 --num-cls-epochs=2
|
||||
1 0.316100 0.326822 0.882000
|
||||
2 0.292052 0.326660 0.874000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en/models/sp30k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.3416499, tensor(0.8878)]
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-laser.m
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/fr.dev.csv
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-fr/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-fr/models/sp30k/lstm_nl4.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-fr/ru.dev.csv
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 9195, cls.val 1021
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁в', 'а', '▁и', 'е', 'и', 'й', '▁на', 'х']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-fr/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-fr/models/sp30k/lstm_nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.906124 0.592495 0.797000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.751562 0.440800 0.842000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.631221 0.393381 0.860000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.582251 0.376320 0.867000
|
||||
2 0.543821 0.374095 0.860000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-fr/models/sp30k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [1.0429544, tensor(0.6833)]
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-de/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-de/models/sp30k/lstm_nl4.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-de/es.dev.csv
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 13013, cls.val 1445
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-de/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-de/models/sp30k/lstm_nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.667080 0.471187 0.884000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.553853 0.329840 0.904000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.463647 0.309136 0.907000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.396284 0.282263 0.911000
|
||||
2 0.368159 0.287222 0.916000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-de/models/sp30k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.5038375, tensor(0.8550)]
|
||||
[('data/mldoc/de-1-laser-fr/models/sp30k/lstm_nl4.m', 0.922249972820282), ('data/mldoc/es-1-laser-de/models/sp30k/lstm_nl4.m', 0.8550000190734863), ('data/mldoc/fr-1-laser-en/models/sp30k/lstm_nl4.m', 0.8877500295639038), ('data/mldoc/it-1-laser-de/models/sp30k/lstm_nl4.m', 0.7875000238418579), ('data/mldoc/ru-1-laser-fr/models/sp30k/lstm_nl4.m', 0.6832500100135803), ('data/mldoc/zh-1-laser-fr/models/sp30k/lstm_nl4.m', 0.8464999794960022)]
|
||||
```
|
||||
second run
|
||||
```
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-de/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-de/models/sp30k/lstm_nl4.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-de/de.dev.csv
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 13500, cls.val 1500
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-de/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-de/models/sp30k/lstm_nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.467292 0.243158 0.919000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.270090 0.207252 0.941000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.201597 0.219442 0.934000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.193163 0.199092 0.943000
|
||||
2 0.169631 0.199501 0.940000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-de/models/sp30k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.16265252, tensor(0.9545)]
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-en/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-en/models/sp30k/lstm_nl4.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-en/de.dev.csv
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 13500, cls.val 1500
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-en/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-en/models/sp30k/lstm_nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.575276 0.419917 0.879000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.475003 0.263138 0.909000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.345987 0.260215 0.911000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.305776 0.268171 0.906000
|
||||
2 0.289134 0.267642 0.911000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-en/models/sp30k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.23464507, tensor(0.9295)]
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/models/sp30k/lstm_nl4.m
|
||||
Evaluating previously trained model
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-fr/de.dev.csv
|
||||
Tokenized data loaded, lm.trn 13500, lm.val 1500
|
||||
Tokenized data loaded, cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁.', '▁,', '▁der', '▁die', '▁und', '▁in', 'en', "▁&'", 's', '-']
|
||||
Loss and accuracy using (cls_last): [0.26227093, tensor(0.9222)]
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-de/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-de/models/sp30k/lstm_nl4.m
|
||||
Evaluating previously trained model
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-de/es.dev.csv
|
||||
Tokenized data loaded, lm.trn 13013, lm.val 1445
|
||||
Tokenized data loaded, cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que']
|
||||
Loss and accuracy using (cls_last): [0.5038375, tensor(0.8550)]
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en/models/sp30k/lstm_nl4.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en/es.dev.csv
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 13013, cls.val 1445
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en/models/sp30k/lstm_nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.784271 0.667321 0.741000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.601108 0.471457 0.854000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.489287 0.428631 0.854000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.434144 0.413409 0.864000
|
||||
2 0.443724 0.385349 0.869000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en/models/sp30k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.82167965, tensor(0.8050)]
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-fr/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-fr/models/sp30k/lstm_nl4.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-fr/es.dev.csv
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 13013, cls.val 1445
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁la', '▁.', '▁en', '▁el', '▁y', 's', '▁a', '▁que']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-fr/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-fr/models/sp30k/lstm_nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.752788 0.615142 0.786000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.566108 0.403893 0.870000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.503008 0.468810 0.865000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.413641 0.448900 0.873000
|
||||
2 0.381155 0.413034 0.879000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-fr/models/sp30k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.7937071, tensor(0.8100)]
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-de/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-de/models/sp30k/lstm_nl4.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-de/fr.dev.csv
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 13500, cls.val 1500
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-laser.m/info.json
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-de/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-de/models/sp30k/lstm_nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.797327 0.697984 0.730000
|
||||
1 0.674638 0.524605 0.796000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.639780 0.582377 0.763000
|
||||
1 0.493693 0.401442 0.851000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.585295 0.582596 0.762000
|
||||
1 0.418525 0.394886 0.859000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.482629 0.582803 0.765000
|
||||
2 0.470849 0.582416 0.771000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser/models/sp30k/lstm_nl4-laser.m
|
||||
Loss and accuracy using (cls_best): [0.80327946, tensor(0.6920)]
|
||||
1 0.343561 0.402565 0.862000
|
||||
2 0.335855 0.418237 0.851000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-de/models/sp30k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.44778627, tensor(0.8737)]
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en/models/sp30k/lstm_nl4.m
|
||||
Evaluating previously trained model
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en/fr.dev.csv
|
||||
Tokenized data loaded, lm.trn 13500, lm.val 1500
|
||||
Tokenized data loaded, cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à']
|
||||
Loss and accuracy using (cls_last): [0.3416499, tensor(0.8878)]
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-fr/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-fr/models/sp30k/lstm_nl4.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-fr/fr.dev.csv
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 13500, cls.val 1500
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁.', "'", 's', '▁la', '▁le', '▁et', '▁l', '▁à']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-fr/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-fr/models/sp30k/lstm_nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.477812 0.332947 0.894000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.305868 0.201659 0.937000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.208116 0.224481 0.931000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.146847 0.214640 0.941000
|
||||
2 0.129603 0.227498 0.929000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-fr/models/sp30k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.19940722, tensor(0.9358)]
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/models/sp30k/lstm_nl4.m
|
||||
Evaluating previously trained model
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-de/it.dev.csv
|
||||
Tokenized data loaded, lm.trn 13500, lm.val 1500
|
||||
Tokenized data loaded, cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁di', "▁&'", "'", '▁e', '▁il', '▁la', 'e', '▁in']
|
||||
Loss and accuracy using (cls_last): [0.6332891, tensor(0.7875)]
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-en/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-en/models/sp30k/lstm_nl4.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-en/it.dev.csv
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 13500, cls.val 1500
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁di', "▁&'", "'", '▁e', '▁il', '▁la', 'e', '▁in']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-en/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-en/models/sp30k/lstm_nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.845312 0.660645 0.769000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.699314 0.584146 0.786000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.556744 0.531658 0.801000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.503091 0.529716 0.805000
|
||||
2 0.474142 0.520058 0.806000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-en/models/sp30k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.7639212, tensor(0.7620)]
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-fr/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-fr/models/sp30k/lstm_nl4.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-fr/it.dev.csv
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 13500, cls.val 1500
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁di', "▁&'", "'", '▁e', '▁il', '▁la', 'e', '▁in']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-fr/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-fr/models/sp30k/lstm_nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.773207 0.568426 0.803000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.570457 0.516704 0.821000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.527280 0.460192 0.840000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.469201 0.461563 0.841000
|
||||
2 0.458892 0.443310 0.836000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-fr/models/sp30k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.80693215, tensor(0.7688)]
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-de/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-de/models/sp30k/lstm_nl4.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-de/ru.dev.csv
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 9195, cls.val 1021
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁в', 'а', '▁и', 'е', 'и', 'й', '▁на', 'х']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-de/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-de/models/sp30k/lstm_nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.878202 0.549530 0.815000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.747663 0.439798 0.860000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.610381 0.391122 0.878000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.563902 0.393633 0.880000
|
||||
2 0.515117 0.403987 0.878000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-de/models/sp30k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [1.3181443, tensor(0.6695)]
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-en/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-en/models/sp30k/lstm_nl4.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-en/ru.dev.csv
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 9195, cls.val 1021
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁в', 'а', '▁и', 'е', 'и', 'й', '▁на', 'х']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-en/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-en/models/sp30k/lstm_nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.897468 0.570228 0.801000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.704874 0.560132 0.812000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.595008 0.507041 0.816000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.484754 0.479213 0.825000
|
||||
2 0.454896 0.501114 0.824000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-en/models/sp30k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [1.1765001, tensor(0.7005)]
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-fr/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-fr/models/sp30k/lstm_nl4.m
|
||||
Evaluating previously trained model
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-fr/ru.dev.csv
|
||||
Tokenized data loaded, lm.trn 9195, lm.val 1021
|
||||
Tokenized data loaded, cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁в', 'а', '▁и', 'е', 'и', 'й', '▁на', 'х']
|
||||
Loss and accuracy using (cls_last): [1.0429544, tensor(0.6833)]
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-de/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-de/models/sp30k/lstm_nl4.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-de/zh.dev.csv
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 13500, cls.val 1500
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-de/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-de/models/sp30k/lstm_nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.759435 0.762386 0.707000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.631534 0.591862 0.786000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.534237 0.589429 0.801000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.454291 0.589220 0.799000
|
||||
2 0.446990 0.586956 0.804000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-de/models/sp30k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.8401224, tensor(0.7232)]
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-en/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-en/models/sp30k/lstm_nl4.m
|
||||
Training
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-en/zh.dev.csv
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 13500, cls.val 1500
|
||||
Running tokenization...
|
||||
Saving tokenized: cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-en/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-en/models/sp30k/lstm_nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.827517 0.821250 0.712000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.636761 0.656195 0.772000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.582199 0.675501 0.769000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.511542 0.634232 0.764000
|
||||
2 0.508244 0.647197 0.771000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-en/models/sp30k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.5421255, tensor(0.8045)]
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/models/sp30k/lstm_nl4.m
|
||||
Evaluating previously trained model
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-fr/zh.dev.csv
|
||||
Tokenized data loaded, lm.trn 13500, lm.val 1500
|
||||
Tokenized data loaded, cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是']
|
||||
Loss and accuracy using (cls_last): [0.5289812, tensor(0.8465)]
|
||||
OrderedDict([('data/mldoc/de-1-laser-de/models/sp30k/lstm_nl4.m',
|
||||
0.9545000195503235),
|
||||
('data/mldoc/de-1-laser-en/models/sp30k/lstm_nl4.m',
|
||||
0.9294999837875366),
|
||||
('data/mldoc/de-1-laser-fr/models/sp30k/lstm_nl4.m',
|
||||
0.922249972820282),
|
||||
('data/mldoc/es-1-laser-de/models/sp30k/lstm_nl4.m',
|
||||
0.8550000190734863),
|
||||
('data/mldoc/es-1-laser-en/models/sp30k/lstm_nl4.m',
|
||||
0.8050000071525574),
|
||||
('data/mldoc/es-1-laser-fr/models/sp30k/lstm_nl4.m',
|
||||
0.8100000023841858),
|
||||
('data/mldoc/fr-1-laser-de/models/sp30k/lstm_nl4.m',
|
||||
0.8737499713897705),
|
||||
('data/mldoc/fr-1-laser-en/models/sp30k/lstm_nl4.m',
|
||||
0.8877500295639038),
|
||||
('data/mldoc/fr-1-laser-fr/models/sp30k/lstm_nl4.m',
|
||||
0.9357500076293945),
|
||||
('data/mldoc/it-1-laser-de/models/sp30k/lstm_nl4.m',
|
||||
0.7875000238418579),
|
||||
('data/mldoc/it-1-laser-en/models/sp30k/lstm_nl4.m',
|
||||
0.7620000243186951),
|
||||
('data/mldoc/it-1-laser-fr/models/sp30k/lstm_nl4.m',
|
||||
0.768750011920929),
|
||||
('data/mldoc/ru-1-laser-de/models/sp30k/lstm_nl4.m',
|
||||
0.6694999933242798),
|
||||
('data/mldoc/ru-1-laser-en/models/sp30k/lstm_nl4.m',
|
||||
0.7005000114440918),
|
||||
('data/mldoc/ru-1-laser-fr/models/sp30k/lstm_nl4.m',
|
||||
0.6832500100135803),
|
||||
('data/mldoc/zh-1-laser-de/models/sp30k/lstm_nl4.m',
|
||||
0.7232499718666077),
|
||||
('data/mldoc/zh-1-laser-en/models/sp30k/lstm_nl4.m',
|
||||
0.8044999837875366),
|
||||
('data/mldoc/zh-1-laser-fr/models/sp30k/lstm_nl4.m',
|
||||
0.8464999794960022)])
|
||||
```
|
||||
|
||||
|
||||
|
||||
### Building dataset
|
||||
|
||||
```
|
||||
for SRC_LANG in en de fr; do ✘ 130
|
||||
for LANG in en de es fr it ru zh; do
|
||||
echo $LANG from $SRC_LANG
|
||||
python ../../source/classify.py embed/mldoc.${SRC_LANG}-${SRC_LANG}.h5 ~/workspace/ulmfit-multilingual/data/mldoc/${LANG}-1 | grep Test:
|
||||
done
|
||||
done
|
||||
|
||||
en from en
|
||||
| Test: 91.48% | classes: 23.77 24.90 26.25 25.07
|
||||
de from en
|
||||
| Test: 87.65% | classes: 21.98 24.45 27.65 25.93
|
||||
es from en
|
||||
| Test: 75.48% | classes: 21.60 15.82 22.10 40.48
|
||||
fr from en
|
||||
| Test: 84.00% | classes: 23.18 29.12 27.90 19.80
|
||||
it from en
|
||||
| Test: 71.18% | classes: 23.65 22.88 25.68 27.80
|
||||
ru from en
|
||||
| Test: 66.58% | classes: 29.48 13.78 34.52 22.23
|
||||
zh from en
|
||||
| Test: 76.65% | classes: 30.25 31.30 13.93 24.52
|
||||
en from de
|
||||
| Test: 78.23% | classes: 31.80 17.73 30.15 20.32
|
||||
de from de
|
||||
| Test: 93.50% | classes: 24.45 25.45 26.00 24.10
|
||||
es from de
|
||||
| Test: 81.40% | classes: 24.15 25.77 20.12 29.95
|
||||
fr from de
|
||||
| Test: 81.50% | classes: 25.52 29.45 27.45 17.57
|
||||
it from de
|
||||
| Test: 74.53% | classes: 24.70 27.25 22.43 25.62
|
||||
ru from de
|
||||
| Test: 64.58% | classes: 45.62 9.12 26.73 18.52
|
||||
zh from de
|
||||
| Test: 73.20% | classes: 31.20 43.38 7.60 17.82
|
||||
en from fr
|
||||
| Test: 81.30% | classes: 28.95 18.02 24.98 28.05
|
||||
de from fr
|
||||
| Test: 88.75% | classes: 24.00 23.75 24.85 27.40
|
||||
es from fr
|
||||
| Test: 80.12% | classes: 24.50 14.82 18.40 42.27
|
||||
fr from fr
|
||||
| Test: 90.85% | classes: 24.50 24.75 24.68 26.07
|
||||
it from fr
|
||||
| Test: 72.58% | classes: 25.45 24.10 17.50 32.95
|
||||
ru from fr
|
||||
| Test: 67.35% | classes: 47.15 13.62 16.68 22.55
|
||||
zh from fr
|
||||
| Test: 79.40% | classes: 33.60 31.12 9.07 26.20
|
||||
```
|
||||
|
||||
|
||||
|
||||
+156
-3
@@ -1,6 +1,159 @@
|
||||
# ZH
|
||||
|
||||
|
||||
|
||||
## SP30k LSTM nl 4
|
||||
### LM
|
||||
```
|
||||
python -m ulmfit lm --dataset-path data/wiki/zh-100 --cuda-id=0 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 60000 --lang zh --qrnn=False - train 10 --bs=50 --drop_mult=0
|
||||
python -m ulmfit lm --dataset-path data/wiki/zh-100 --cuda-id=0 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 30000 --lang zh --qrnn=False - train 10 --bs=50 --drop_mult=0
|
||||
Max vocab: 30000
|
||||
Cache dir: data/wiki/zh-100/models/sp30k
|
||||
Model dir: data/wiki/zh-100/models/sp30k/lstm_nl4.m
|
||||
Tokenized data loaded
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': None, 'pretrained_model': None, 'drop_mult': 0} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Training lm from random weights
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 2.736679 3.050473 0.428462
|
||||
2 2.664505 3.011505 0.432414
|
||||
3 2.607435 2.942389 0.439985
|
||||
4 2.561503 2.851523 0.451965
|
||||
5 2.499060 2.798222 0.459438
|
||||
6 2.387191 2.720054 0.471021
|
||||
7 2.356725 2.648299 0.479029
|
||||
8 2.301895 2.553860 0.493597
|
||||
9 2.275601 2.481724 0.505979
|
||||
10 2.187606 2.465159 0.509590
|
||||
```
|
||||
### MLDoc
|
||||
```
|
||||
python -m ulmfit cls --dataset-path data/mldoc/zh-1 --base-lm-path data/wiki/zh-100/models/sp30k/lstm_nl4.m --lang=zh --name 'nl4' --cuda-id=0 - train 20 --bs 40 --num-cls-epochs=2
|
||||
Max vocab: 30000
|
||||
Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k
|
||||
Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4.m
|
||||
Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv
|
||||
Tokenized data loaded, lm.trn 13500, lm.val 1500
|
||||
Tokenized data loaded, cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 30000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁人', '▁是']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/zh-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/zh-100/models/sp30k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
Training lm from: [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/zh-100/models/sp30k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/zh-100/models/sp30k/lstm_nl4.m/../itos')]
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 2.604460 2.225315 0.546099
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 2.240892 2.020697 0.578796
|
||||
2 2.025043 1.816424 0.613192
|
||||
3 1.832658 1.646025 0.640532
|
||||
4 1.746628 1.530125 0.659058
|
||||
5 1.621672 1.425179 0.675305
|
||||
6 1.544814 1.345650 0.689195
|
||||
7 1.464704 1.271710 0.702200
|
||||
8 1.412583 1.204830 0.714764
|
||||
9 1.332440 1.147108 0.725389
|
||||
10 1.327941 1.092910 0.736447
|
||||
11 1.227284 1.039441 0.747662
|
||||
12 1.200814 0.991910 0.758105
|
||||
13 1.161579 0.947898 0.768121
|
||||
14 1.100010 0.908599 0.776732
|
||||
15 1.059006 0.872309 0.785161
|
||||
16 1.045412 0.844972 0.791998
|
||||
17 1.026688 0.824872 0.796891
|
||||
18 1.013831 0.812786 0.799699
|
||||
19 0.978586 0.807678 0.800954
|
||||
20 0.982473 0.805671 0.801201
|
||||
/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k
|
||||
Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.637427 0.505143 0.836000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.471189 0.317678 0.887000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.384985 0.288901 0.904000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.316358 0.275456 0.906000
|
||||
2 0.295534 0.278589 0.907000
|
||||
Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp30k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_best): [0.28411642, tensor(0.9020)]
|
||||
0.2841164171695709
|
||||
0.9020000100135803
|
||||
```
|
||||
|
||||
## SP60k LSTM nl 4
|
||||
### LM
|
||||
```
|
||||
Wiki text was split to 153503 articles
|
||||
Wiki text was split to 145 articles
|
||||
Size of vocabulary: 60000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁是', '▁人']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': None, 'pretrained_model': None, 'drop_mult': 0} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Training lm from random weights
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 3.312704 3.701317 0.334740
|
||||
2 3.212988 3.648671 0.336709
|
||||
3 3.060103 3.584413 0.344427
|
||||
4 3.108131 3.477978 0.356738
|
||||
5 2.952951 3.410785 0.365901
|
||||
6 2.919397 3.325265 0.376316
|
||||
7 2.839392 3.224750 0.391707
|
||||
8 2.750095 3.132644 0.404416
|
||||
9 2.805704 3.066595 0.415245
|
||||
10 2.653435 3.055314 0.417736
|
||||
data/wiki/zh-100/models/sp60k
|
||||
Saving info data/wiki/zh-100/models/sp60k/lstm_nl4.m/info.json
|
||||
```
|
||||
### MLDoc
|
||||
```
|
||||
python -m ulmfit cls --dataset-path data/mldoc/zh-1 --base-lm-path data/wiki/zh-100/models/sp60k/lstm_nl4.m --lang=zh --name 'nl4' --cu
|
||||
da-id=0 - train 20 --bs 40 --num-cls-epochs=2
|
||||
Max vocab: 60000
|
||||
Cache dir: /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp60k
|
||||
Model dir: /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp60k/lstm_nl4.m
|
||||
Loading validation /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv
|
||||
Tokenized data loaded, lm.trn 13500, lm.val 1500
|
||||
Tokenized data loaded, cls.trn 1000, cls.val 1000
|
||||
Size of vocabulary: 60000
|
||||
First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁中', '▁是', '▁人']
|
||||
Training args: {'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': [PosixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/zh-100/models/sp60k/lstm_nl4.m/lm_best'), Po
|
||||
sixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/zh-100/models/sp60k/lstm_nl4.m/../itos')], 'pretrained_model': None, 'drop_mult': 0.3} dps: [0.25 0.1 0.2 0.02 0.15]
|
||||
Unknown tokens 0, first 100: []
|
||||
Training lm from: [PosixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/zh-100/models/sp60k/lstm_nl4.m/lm_best'), PosixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/zh-
|
||||
100/models/sp60k/lstm_nl4.m/../itos')]
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 3.055914 2.690310 0.467917
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 2.713421 2.464873 0.503386
|
||||
2 2.429520 2.215309 0.543961
|
||||
3 2.247576 2.010849 0.578106
|
||||
4 2.083628 1.853473 0.602419
|
||||
5 1.969939 1.734762 0.621440
|
||||
6 1.904438 1.624005 0.640240
|
||||
7 1.783416 1.526202 0.656981
|
||||
8 1.719215 1.445780 0.671753
|
||||
9 1.621891 1.366912 0.687187
|
||||
10 1.589463 1.295759 0.701207
|
||||
11 1.510032 1.223578 0.716387
|
||||
12 1.404720 1.160607 0.729603
|
||||
13 1.414636 1.107378 0.741273
|
||||
14 1.364716 1.056422 0.753112
|
||||
15 1.327804 1.011525 0.763934
|
||||
16 1.255990 0.976447 0.771864
|
||||
17 1.181438 0.951213 0.778309
|
||||
18 1.192709 0.936060 0.781858
|
||||
19 1.190164 0.928613 0.783513
|
||||
20 1.172130 0.927612 0.783722
|
||||
/home/n-waves/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp60k
|
||||
Saving info /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp60k/lstm_nl4.m/info.json
|
||||
Starting classifier training
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.646537 0.516221 0.836000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.441884 0.361802 0.873000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.376583 0.318426 0.893000
|
||||
epoch train_loss valid_loss accuracy
|
||||
1 0.280910 0.314279 0.889000
|
||||
2 0.308887 0.309718 0.903000
|
||||
Saving models at /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp60k/lstm_nl4.m
|
||||
Loss and accuracy using (cls_last): [0.30276635, tensor(0.8978)]
|
||||
```
|
||||
+37
-20
@@ -1,26 +1,33 @@
|
||||
import gc
|
||||
import os
|
||||
import pprint
|
||||
import shutil
|
||||
from collections import OrderedDict
|
||||
from functools import wraps
|
||||
|
||||
import fire
|
||||
from .pretrain_lm import LMHyperParams
|
||||
from .train_clas import CLSHyperParams
|
||||
from pathlib import Path
|
||||
from string import Template
|
||||
|
||||
class FireView:
|
||||
def __init__(self, **kwargs):
|
||||
for k,v in kwargs.items():
|
||||
setattr(self, k, v)
|
||||
|
||||
def get_dataset_path(p):
|
||||
return [x for x in p.parents if x.name == "models"][0].parent
|
||||
|
||||
def get_lang_from_dataset_path(ds):
|
||||
lang,*_ = ds.name.split("-")
|
||||
if len(lang) == 2:
|
||||
return lang
|
||||
return "en"
|
||||
|
||||
def get_dataset_path(p, dataset_template):
|
||||
ds = [x for x in p.parents if x.name == "models"][0].parent
|
||||
lang = get_lang_from_dataset_path(ds)
|
||||
for ds_path in ds.parent.glob(Template(dataset_template).substitute(lang=lang, ds_name=ds.name)):
|
||||
yield lang, ds_path
|
||||
|
||||
class ULMFiT:
|
||||
@wraps(LMHyperParams)
|
||||
def lm(self, dataset_path, **changes):
|
||||
@@ -34,23 +41,33 @@ class ULMFiT:
|
||||
params = CLSHyperParams.from_lm(dataset_path, base_lm_path, **changes)
|
||||
return FireView(train=params.train_cls, validate_cls=params.validate_cls)
|
||||
|
||||
def eval(self, glob="mldoc/*-1/models/sp30k/lstm_nl4.m", name="tmp-100", cuda_id=0, **trn_params):
|
||||
results={}
|
||||
for base_model in Path("data").glob(glob):
|
||||
dataset_path = get_dataset_path(base_model)
|
||||
lang = get_lang_from_dataset_path(dataset_path)
|
||||
params = CLSHyperParams.from_lm(dataset_path, base_model, lang=lang, name=name, cuda_id=cuda_id)
|
||||
key = str(params.model_dir.relative_to(Path.cwd()))
|
||||
if params.model_dir.exists():
|
||||
print("Evaluating previously trained model")
|
||||
results[key] = params.validate_cls()[1]
|
||||
else:
|
||||
print("Training")
|
||||
results[key] = params.train_cls(num_lm_epochs=0, **trn_params)[1]
|
||||
params = None
|
||||
gc.collect()
|
||||
def eval(self, glob="mldoc/*-1/models/sp30k/lstm_nl4.m", dataset_template='${lang}-1', name="tmp-100", num_lm_epochs=0, cuda_id=0, **trn_params):
|
||||
results = OrderedDict()
|
||||
for base_model in sorted(Path("data").glob(glob)):
|
||||
for lang, dataset_path in sorted(get_dataset_path(base_model, dataset_template)):
|
||||
params = CLSHyperParams.from_lm(dataset_path, base_model, lang=lang, name=name, cuda_id=cuda_id)
|
||||
key = str(params.model_dir.relative_to(Path.cwd()))
|
||||
if (params.model_dir/"cls_last.pth").exists():
|
||||
print("Evaluating previously trained model")
|
||||
results[key] = params.validate_cls()[1]
|
||||
else:
|
||||
print("Training")
|
||||
results[key] = params.train_cls(num_lm_epochs=num_lm_epochs, **trn_params)[1]
|
||||
del params
|
||||
gc.collect()
|
||||
|
||||
print(list(sorted(results.items())))
|
||||
pprint.pprint(results)
|
||||
|
||||
def remove_lm_saves(self):
|
||||
for lm_save in Path("data").glob("**/lm_*.pth"):
|
||||
num = lm_save.stem.split("_")[-1]
|
||||
if not num.isdigit():
|
||||
continue
|
||||
if int(num) not in [5, 10, 15]:
|
||||
print("rm ", lm_save)
|
||||
os.remove(lm_save)
|
||||
|
||||
# python -m ulmfit cls --dataset-path data/mldoc/de-1-laser --base-lm-path data/mldoc/de-1/models/sp30k/lstm_nl4.m --lang=de --name 'nl4' --cuda-id=1 - train 0 --bs 40 --num-cls-epochs=2
|
||||
|
||||
if __name__ == '__main__':
|
||||
fire.Fire(ULMFiT())
|
||||
fire.Fire(ULMFiT())
|
||||
|
||||
+42
-25
@@ -14,8 +14,8 @@ from fastai.callbacks import CSVLogger, SaveModelCallback
|
||||
from fastai.text import *
|
||||
import torch
|
||||
from fastai_contrib.utils import read_file, read_whitespace_file, \
|
||||
validate, PAD, UNK, get_sentencepiece, read_clas_data, TRN, VAL, TST, PAD_TOKEN_ID, MosesTokenizerFunc, \
|
||||
replace_std_toks
|
||||
validate, PAD, UNK, get_sentencepiece, read_clas_data, TRN, VAL, TST, PAD_TOKEN_ID, \
|
||||
replace_std_toks, MosesPreprocessingFunc
|
||||
from fastai_contrib.learner import bilm_learner, accuracy_fwd, accuracy_bwd, bilm_text_classifier_learner
|
||||
import pickle
|
||||
|
||||
@@ -30,6 +30,7 @@ ENC_BEST = "enc_best"
|
||||
|
||||
class Tokenizers(Enum):
|
||||
SUBWORD='sp'
|
||||
BROKENSUBWORD = 'bsp'
|
||||
MOSES='v'
|
||||
MOSES_FA='vf'
|
||||
FASTAI='f'
|
||||
@@ -69,7 +70,7 @@ class LMHyperParams:
|
||||
|
||||
# these hyperparameters are for training on ~100M tokens (e.g. WikiText-103)
|
||||
# for training on smaller datasets, more dropout is necessary
|
||||
dps = (0.25, 0.1, 0.2, 0.02, 0.15) # consider removing dps & clip from the default hyperparams and put them to train
|
||||
dps = dict(output_p=0.25, hidden_p=0.1, input_p=0.2, embed_p=0.02, weight_p=0.15) # consider removing dps & clip from the default hyperparams and put them to train
|
||||
clip: float = 0.12
|
||||
bptt: int = 70
|
||||
# alpha and beta - defaults like in fastai/text/learner.py:RNNLearner()
|
||||
@@ -98,7 +99,6 @@ class LMHyperParams:
|
||||
print('Max vocab:', self.max_vocab)
|
||||
print('Cache dir:', self.cache_dir)
|
||||
print('Model dir:', self.model_dir)
|
||||
self.dps = np.array(self.dps)
|
||||
if self.nh is None: self.nh = 1550 if self.qrnn else 1150
|
||||
if self.name is None: self.name = self.lang
|
||||
|
||||
@@ -121,7 +121,7 @@ class LMHyperParams:
|
||||
def model_name(self): return f"{self.model_prefix}_{self.name}.m"
|
||||
|
||||
@property
|
||||
def pretrained_fnames(self): return [self.base_lm_path / 'lm_best', self.base_lm_path / '../itos'] if self.base_lm_path else None
|
||||
def pretrained_fnames(self): return [self.base_lm_path / LM_BEST, self.base_lm_path / '../itos'] if self.base_lm_path else None
|
||||
|
||||
@property
|
||||
def lm_type(self):
|
||||
@@ -133,8 +133,8 @@ class LMHyperParams:
|
||||
return contrib_data.LanguageModelType.FwdLM
|
||||
|
||||
def tokenizer_to_fastai_args(self, sp_data_func, use_moses):
|
||||
tok_func = MosesTokenizerFunc if use_moses else BaseTokenizer
|
||||
if self.tokenizer is Tokenizers.SUBWORD:
|
||||
moses_preproc = [MosesPreprocessingFunc(self.lang)] if use_moses else []
|
||||
if self.tokenizer is Tokenizers.SUBWORD or self.tokenizer is Tokenizers.BROKENSUBWORD:
|
||||
if self.base_lm_path and not(self.cache_dir/"spm.model").exists(): # ensure we are using the same sentence piece model
|
||||
shutil.copy(self.base_lm_path / '..' / 'itos.pkl', self.cache_dir)
|
||||
shutil.copy(self.base_lm_path / '..' / 'spm.model', self.cache_dir)
|
||||
@@ -142,13 +142,19 @@ class LMHyperParams:
|
||||
args = get_sentencepiece(self.cache_dir,
|
||||
sp_data_func,
|
||||
vocab_size=self.max_vocab,
|
||||
use_moses=use_moses,
|
||||
lang=self.lang)
|
||||
|
||||
lang=self.lang,
|
||||
pre_rules=moses_preproc + defaults.text_pre_rules,
|
||||
post_rules=defaults.text_post_rules)
|
||||
elif self.tokenizer is Tokenizers.MOSES:
|
||||
args = dict(tokenizer=Tokenizer(tok_func=tok_func, lang=self.lang, pre_rules=[replace_std_toks], post_rules=[]))
|
||||
args = dict(tokenizer=Tokenizer(tok_func=BaseTokenizer,
|
||||
lang=self.lang,
|
||||
pre_rules=moses_preproc + [replace_std_toks],
|
||||
post_rules=[]))
|
||||
elif self.tokenizer is Tokenizers.MOSES_FA:
|
||||
args = dict(tokenizer=Tokenizer(tok_func=tok_func, lang=self.lang)) # use default pre/post rules
|
||||
args = dict(tokenizer=Tokenizer(tok_func=BaseTokenizer,
|
||||
lang=self.lang,
|
||||
pre_rules=moses_preproc + defaults.text_pre_rules,
|
||||
post_rules=defaults.text_post_rules))
|
||||
elif self.tokenizer is Tokenizers.FASTAI:
|
||||
args = dict()
|
||||
else:
|
||||
@@ -195,25 +201,34 @@ class LMHyperParams:
|
||||
print(learn.path)
|
||||
|
||||
self.save_info()
|
||||
return learn
|
||||
# do we need to return `learn'? it adds noise to Fire output
|
||||
#return learn
|
||||
|
||||
def create_lm_learner(self, data_lm, dps=None, **kwargs):
|
||||
fastai.text.learner.default_dropout['language'] = dps or self.dps
|
||||
lm_learner = bilm_learner if self.bidir else language_model_learner
|
||||
|
||||
trn_args = dict(tie_weights=True, clip=self.clip, bptt=self.bptt,
|
||||
pretrained_fnames=self.pretrained_fnames,
|
||||
pretrained_model=self.pretrained_model,
|
||||
alpha=self.rnn_alpha, beta=self.rnn_beta)
|
||||
assert self.bidir == False, "bidirectional model is not yet supported"
|
||||
config = dict(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, pad_token=PAD_TOKEN_ID, qrnn=self.qrnn,
|
||||
tie_weights=True, out_bias=True)
|
||||
config.update(dps or self.dps)
|
||||
trn_args = dict(clip=self.clip, alpha=self.rnn_alpha, beta=self.rnn_beta)
|
||||
trn_args.update(kwargs)
|
||||
print ("Training args: ", trn_args, "dps: ", dps or self.dps)
|
||||
learn = lm_learner(data_lm, emb_sz=self.emb_sz, nh=self.nh, nl=self.nl, pad_token=PAD_TOKEN_ID,
|
||||
bias=True, qrnn=self.qrnn, model_dir=self.model_dir.relative_to(data_lm.path), **trn_args)
|
||||
learn = language_model_learner(data_lm, AWD_LSTM, config=config, model_dir=self.model_dir.relative_to(data_lm.path), pretrained=False, **trn_args)
|
||||
if self.pretrained_model is not None:
|
||||
print("Loading pretrained model")
|
||||
model_path = untar_data(self.pretrained_model, data=False)
|
||||
fnames = [list(model_path.glob(f'*.{ext}'))[0] for ext in ['pth', 'pkl']]
|
||||
learn.load_pretrained(*fnames)
|
||||
learn.freeze()
|
||||
if self.pretrained_fnames is not None:
|
||||
print("Loading pretrained model")
|
||||
fnames = [f'{fn}.{ext}' for fn,ext in zip(self.pretrained_fnames, ['pth', 'pkl'])]
|
||||
learn.load_pretrained(*fnames)
|
||||
learn.freeze()
|
||||
# compared to standard Adam, we set beta_1 to 0.8
|
||||
learn.opt_fn = partial(optim.Adam, betas=(0.8, 0.99))
|
||||
learn.metrics = [accuracy_fwd, accuracy_bwd] if self.bidir else [accuracy]
|
||||
learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/lm-history"),
|
||||
partial(SaveModelCallback, every='epoch', name='lm')]
|
||||
partial(SaveModelCallback, every='improvement', name='lm')]
|
||||
return learn
|
||||
|
||||
def load_train_text(self):
|
||||
@@ -231,13 +246,15 @@ class LMHyperParams:
|
||||
|
||||
args = self.tokenizer_to_fastai_args(sp_data_func=self.load_train_text, use_moses=False)
|
||||
try:
|
||||
data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type, bs=bs)
|
||||
data_lm = TextLMDataBunch.load(self.cache_dir, '.',
|
||||
bs=bs)
|
||||
print("Tokenized data loaded")
|
||||
except FileNotFoundError:
|
||||
print("Running tokenization")
|
||||
data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=read_wiki_articles(trn_path),
|
||||
valid_df=read_wiki_articles(val_path),
|
||||
classes=None, lm_type=self.lm_type, max_vocab=self.max_vocab,
|
||||
classes=None,
|
||||
max_vocab=self.max_vocab,
|
||||
bs=bs, text_cols='texts', **args)
|
||||
data_lm.save('.')
|
||||
|
||||
|
||||
+20
-14
@@ -15,7 +15,7 @@ from fastai_contrib import utils
|
||||
from fastai_contrib.data import LanguageModelType
|
||||
from fastai_contrib.learner import bilm_text_classifier_learner, bilm_learner, accuracy_fwd, accuracy_bwd
|
||||
from fastai_contrib.utils import PAD, UNK, read_clas_data, PAD_TOKEN_ID, DATASETS, TRN, VAL, TST, ensure_paths_exists, \
|
||||
get_sentencepiece, MosesTokenizerFunc
|
||||
get_sentencepiece
|
||||
from fastai.text.transform import Vocab
|
||||
|
||||
import fire
|
||||
@@ -24,6 +24,7 @@ from pathlib import Path
|
||||
|
||||
from ulmfit.pretrain_lm import LMHyperParams, Tokenizers, ENC_BEST
|
||||
|
||||
|
||||
class CLSHyperParams(LMHyperParams):
|
||||
# dir_path -> data/imdb/
|
||||
use_test_for_validation=False
|
||||
@@ -38,14 +39,14 @@ class CLSHyperParams(LMHyperParams):
|
||||
def need_fine_tune_lm(self): return not (self.model_dir/f"enc_best.pth").exists()
|
||||
|
||||
def train_cls(self, num_lm_epochs, unfreeze=True, num_cls_frozen_epochs=1, bs=40, true_wd=True, drop_mul_lm=0.3, drop_mul_cls=0.5,
|
||||
use_test_for_validation=False, num_cls_epochs=2, limit=None, noise=0.0):
|
||||
use_test_for_validation=False, num_cls_epochs=2, limit=None, noise=0.0, cls_max_len=20*70):
|
||||
assert use_test_for_validation == False, "use_test_for_validation=True is not supported"
|
||||
self.model_dir.mkdir(exist_ok=True, parents=True)
|
||||
|
||||
data_clas, data_lm, data_tst = self.load_cls_data(bs, limit=limit, noise=noise)
|
||||
|
||||
if self.need_fine_tune_lm: self.train_lm(num_lm_epochs, data_lm=data_lm, true_wd=true_wd, drop_mult=drop_mul_lm)
|
||||
learn = self.create_cls_learner(data_clas, drop_mult=drop_mul_cls)
|
||||
learn = self.create_cls_learner(data_clas, drop_mult=drop_mul_cls, max_len=cls_max_len)
|
||||
try:
|
||||
learn.load('cls_last')
|
||||
print("Loading last classifier")
|
||||
@@ -91,16 +92,21 @@ class CLSHyperParams(LMHyperParams):
|
||||
return list(map(float, results))
|
||||
|
||||
def create_cls_learner(self, data_clas, dps=None, **kwargs):
|
||||
fastai.text.learner.default_dropout['language'] = dps or self.dps
|
||||
trn_args=dict(bptt=self.bptt, clip=self.clip,)
|
||||
assert self.bidir == False, "bidirectional model is not yet supported"
|
||||
config = dict(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, pad_token=PAD_TOKEN_ID, qrnn=self.qrnn)
|
||||
config.update(dps or self.dps)
|
||||
trn_args=dict(bptt=self.bptt, clip=self.clip)
|
||||
trn_args.update(kwargs)
|
||||
classifier_learner = text_classifier_learner
|
||||
if self.bidir:
|
||||
classifier_learner = bilm_text_classifier_learner
|
||||
trn_args['bicls_head'] = self.bicls_head
|
||||
learn = classifier_learner(data_clas, pad_token=PAD_TOKEN_ID,
|
||||
path=self.model_dir.parent, model_dir=self.model_dir.name,
|
||||
qrnn=self.qrnn, emb_sz=self.emb_sz, nh=self.nh, nl=self.nl, **trn_args)
|
||||
learn = text_classifier_learner(data_clas, AWD_LSTM, config=config,
|
||||
pretrained=False, path=self.model_dir.parent, model_dir=self.model_dir.name, **trn_args)
|
||||
|
||||
if self.pretrained_model is not None:
|
||||
print("Loading pretrained model")
|
||||
model_path = untar_data(self.pretrained_model, data=False)
|
||||
fnames = [list(model_path.glob(f'*.{ext}'))[0] for ext in ['pth', 'pkl']]
|
||||
learn.load_pretrained(*fnames, strict=False)
|
||||
learn.freeze()
|
||||
|
||||
learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/cls-history"),
|
||||
partial(SaveModelCallback, every='improvement', name='cls_best')]
|
||||
return learn
|
||||
@@ -155,12 +161,12 @@ class CLSHyperParams(LMHyperParams):
|
||||
args = self.tokenizer_to_fastai_args(sp_data_func=lambda: trn_df[1], use_moses=use_moses)
|
||||
try:
|
||||
if force: raise FileNotFoundError("Forcing reloading of caches")
|
||||
data_lm = TextLMDataBunch.load(self.cache_dir, 'lm', lm_type=self.lm_type, bs=bs)
|
||||
data_lm = TextLMDataBunch.load(self.cache_dir, 'lm', bs=bs)
|
||||
print(f"Tokenized data loaded, lm.trn {len(data_lm.train_ds)}, lm.val {len(data_lm.valid_ds)}")
|
||||
except FileNotFoundError:
|
||||
print(f"Running tokenization...")
|
||||
data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=lm_trn_df, valid_df=lm_val_df,
|
||||
max_vocab=self.max_vocab, bs=bs, lm_type=self.lm_type, **args)
|
||||
max_vocab=self.max_vocab, bs=bs, **args)
|
||||
print(f"Saving tokenized: cls.trn {len(data_lm.train_ds)}, cls.val {len(data_lm.valid_ds)}")
|
||||
data_lm.save('lm')
|
||||
|
||||
|
||||
Reference in New Issue
Block a user