From c4e0b7b3d652e8f788343b80f5f9df41a5afe0b0 Mon Sep 17 00:00:00 2001
From: "NAUSICAA\\Julian" <eisenjulian@gmail.com>
Date: Sun, 25 Nov 2018 23:47:24 -0300
Subject: [PATCH 01/32] Lower memory footprint

---
 fastai_contrib/utils.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py
index 2dd21f9..8ae01cd 100644
--- a/fastai_contrib/utils.py
+++ b/fastai_contrib/utils.py
@@ -70,11 +70,11 @@ def get_sentencepiece(path:PathOrStr, trn_path:Path, name:str, pre_rules:ListRul
     pre_rules = pre_rules if pre_rules is not None else []
     post_rules = post_rules if post_rules is not None else []
     
-    # load the text frmo the train tokens file
-    text = [line.rstrip('\n') for line in open(trn_path)]
-    text = list(filter(None, text))
 
     if not os.path.isfile(path / 'models' / 'spm.model') or not os.path.isfile(path / 'models' / f'itos_{name}.pkl'):
+        # load the text frmo the train tokens file
+        text = [line.rstrip('\n') for line in open(trn_path)]
+        text = list(filter(None, text))
         raw_text = reduce(lambda t, rule: rule(t), pre_rules, '\n'.join(text))
         raw_text_path = path / cache_name / 'all_text.txt'
         with open(raw_text_path, 'w') as f:
@@ -337,7 +337,7 @@ def read_file(file_path, outname):
     with open(file_path, encoding='utf8') as f:
         text = f.readlines()
     df = pd.DataFrame(
-        {'text': np.array(text), 'labels': np.zeros(len(text))},
+        {'text': text, 'labels': np.zeros(len(text))},
         columns=['labels', 'text'])
     df.to_csv(file_path.parent / f'{outname}.csv', header=False, index=False)
 

From 699093725f743b89f26055cd1f24d4dd89ce5432 Mon Sep 17 00:00:00 2001
From: "NAUSICAA\\Julian" <eisenjulian@gmail.com>
Date: Mon, 26 Nov 2018 14:49:38 -0300
Subject: [PATCH 02/32] Script to merge two language token files

---
 ulmfit/merge_langs.py | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)
 create mode 100644 ulmfit/merge_langs.py

diff --git a/ulmfit/merge_langs.py b/ulmfit/merge_langs.py
new file mode 100644
index 0000000..7fee7ed
--- /dev/null
+++ b/ulmfit/merge_langs.py
@@ -0,0 +1,35 @@
+"""
+Script to merge WikiText files created with `create_wikitext.py`.
+"""
+import fire
+from pathlib import Path
+from contextlib import ExitStack
+
+def merge_wikitext(paths, langs, dest_path, num_sentences):
+    wiki_paths = [Path(path) for path in paths]
+    for wiki_path in wiki_paths:
+        assert wiki_path.exists(), f'Error: {wiki_path} does not exist.'
+    dest_path = Path(dest_path)
+    dest_path.mkdir(exist_ok=True)
+    splits = ['train', 'valid', 'test']
+    concat_langs = '-'.join(langs)
+    for split in splits:
+        with ExitStack() as stack:
+            files = [stack.enter_context(open(
+                wiki_path / f'{lang}.wiki.{split}.tokens', 'r', encoding='utf-8')) 
+                for lang, wiki_path in zip(langs, wiki_paths)]
+            
+            output = stack.enter_context(open(dest_path / f'{concat_langs}.wiki.{split}.tokens', 'w', encoding='utf-8'))
+            done = False
+            while not done:
+                for file in files:
+                    lines = [file.readline() for x in range(num_sentences)]
+                    size = len(lines)
+                    lines = [line for line in lines if line]
+                    if len(lines) < size:
+                        done = True
+                    for line in lines:
+                        output.write(line)
+
+if __name__ == '__main__':
+    fire.Fire(merge_wikitext)
\ No newline at end of file

From 807404196520054634f219910a991871c6e7b69a Mon Sep 17 00:00:00 2001
From: "NAUSICAA\\Julian" <eisenjulian@gmail.com>
Date: Mon, 26 Nov 2018 21:24:19 -0300
Subject: [PATCH 03/32] Cache intermediate results

---
 ulmfit/pretrain_lm.py | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py
index aaca9e7..4d6ab9f 100644
--- a/ulmfit/pretrain_lm.py
+++ b/ulmfit/pretrain_lm.py
@@ -9,6 +9,7 @@ import fire
 
 from fastai import *
 from fastai.text import *
+from fastai.callbacks.tracker import SaveModelCallback
 import torch
 from fastai_contrib.utils import read_file, read_whitespace_file, \
     validate, PAD, UNK, get_sentencepiece
@@ -69,14 +70,16 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo
         trn_path = dir_path / f'{lang}.wiki.train.tokens'
         val_path = dir_path / f'{lang}.wiki.valid.tokens'
 
-        read_file(trn_path, 'train')
-        read_file(val_path, 'valid')
-
-        sp = get_sentencepiece(dir_path, trn_path, name, vocab_size=max_vocab)
-
         lm_type = contrib_data.LanguageModelType.BiLM if bidir else  contrib_data.LanguageModelType.FwdLM
-
-        data_lm = TextLMDataBunch.from_csv(dir_path, 'train.csv', **sp, bs=bs, bptt=bptt, lm_type=lm_type)
+        try:
+            data_lm = TextLMDataBunch.load(dir_path, bs=bs, bptt=bptt, lm_type=lm_type)
+            print("Saved DataBunch loaded")
+        except FileNotFoundError:
+            read_file(trn_path, 'train')
+            read_file(val_path, 'valid')
+            sp = get_sentencepiece(dir_path, trn_path, name, vocab_size=max_vocab)
+            data_lm = TextLMDataBunch.from_csv(dir_path, 'train.csv', **sp, bs=bs, bptt=bptt, lm_type=lm_type)
+            data_lm.save();
         itos = data_lm.train_ds.vocab.itos
         stoi = data_lm.train_ds.vocab.stoi
     else:
@@ -139,7 +142,8 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo
     lm_learner = bilm_learner if bidir else language_model_learner
     learn = lm_learner(data_lm, bptt=bptt, emb_sz=emb_sz, nh=nh, nl=nl, pad_token=1,
                        drop_mult=drop_mult, tie_weights=True, model_dir=model_dir.name,
-                       bias=True, qrnn=qrnn, clip=0.12)
+                       bias=True, qrnn=qrnn, clip=0.12, 
+                       callbacks=[SaveModelCallback(every='epoch')])
     # compared to standard Adam, we set beta_1 to 0.8
     learn.opt_fn = partial(optim.Adam, betas=(0.8, 0.99))
 

From 4d4ef1ae95e3dae2e126ca12931037f26cf11f38 Mon Sep 17 00:00:00 2001
From: "NAUSICAA\\Julian" <eisenjulian@gmail.com>
Date: Mon, 26 Nov 2018 22:43:39 -0300
Subject: [PATCH 04/32] Change callback_fns

---
 ulmfit/pretrain_lm.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py
index 4d6ab9f..2396eaf 100644
--- a/ulmfit/pretrain_lm.py
+++ b/ulmfit/pretrain_lm.py
@@ -79,7 +79,7 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo
             read_file(val_path, 'valid')
             sp = get_sentencepiece(dir_path, trn_path, name, vocab_size=max_vocab)
             data_lm = TextLMDataBunch.from_csv(dir_path, 'train.csv', **sp, bs=bs, bptt=bptt, lm_type=lm_type)
-            data_lm.save();
+            data_lm.save()
         itos = data_lm.train_ds.vocab.itos
         stoi = data_lm.train_ds.vocab.stoi
     else:
@@ -143,7 +143,7 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo
     learn = lm_learner(data_lm, bptt=bptt, emb_sz=emb_sz, nh=nh, nl=nl, pad_token=1,
                        drop_mult=drop_mult, tie_weights=True, model_dir=model_dir.name,
                        bias=True, qrnn=qrnn, clip=0.12, 
-                       callbacks=[SaveModelCallback(every='epoch')])
+                       callbacks=[lambda lrn: SaveModelCallback(lrn, every='epoch')])
     # compared to standard Adam, we set beta_1 to 0.8
     learn.opt_fn = partial(optim.Adam, betas=(0.8, 0.99))
 

From bf33f222daaa2ae73a6aea08a1e4f614e34e1401 Mon Sep 17 00:00:00 2001
From: "NAUSICAA\\Julian" <eisenjulian@gmail.com>
Date: Mon, 26 Nov 2018 22:48:42 -0300
Subject: [PATCH 05/32] callback > callback_fns

---
 ulmfit/pretrain_lm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py
index 2396eaf..44dfa70 100644
--- a/ulmfit/pretrain_lm.py
+++ b/ulmfit/pretrain_lm.py
@@ -143,7 +143,7 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo
     learn = lm_learner(data_lm, bptt=bptt, emb_sz=emb_sz, nh=nh, nl=nl, pad_token=1,
                        drop_mult=drop_mult, tie_weights=True, model_dir=model_dir.name,
                        bias=True, qrnn=qrnn, clip=0.12, 
-                       callbacks=[lambda lrn: SaveModelCallback(lrn, every='epoch')])
+                       callback_fns=[lambda lrn: SaveModelCallback(lrn, every='epoch')])
     # compared to standard Adam, we set beta_1 to 0.8
     learn.opt_fn = partial(optim.Adam, betas=(0.8, 0.99))
 

From 3ef3be1b93d1d2f93349e8cb429111d0c440fbfb Mon Sep 17 00:00:00 2001
From: Julian Eisenschlos <eisenjulian@users.noreply.github.com>
Date: Mon, 24 Dec 2018 17:02:37 -0300
Subject: [PATCH 06/32] Create train_xlingual_cls.py

---
 ulmfit/train_xlingual_cls.py | 79 ++++++++++++++++++++++++++++++++++++
 1 file changed, 79 insertions(+)
 create mode 100644 ulmfit/train_xlingual_cls.py

diff --git a/ulmfit/train_xlingual_cls.py b/ulmfit/train_xlingual_cls.py
new file mode 100644
index 0000000..f383411
--- /dev/null
+++ b/ulmfit/train_xlingual_cls.py
@@ -0,0 +1,79 @@
+from dataclasses import dataclass
+from ulmfit.train_clas import CLSHyperParams, MosesTokenizerFunc
+from ulmfit.pretrain_lm import LMHyperParams, Tokenizers, ENC_BEST
+from fastai.text import TextLMDataBunch, TextClasDataBunch, language_model_learner, text_classifier_learner
+from fastai_contrib.utils import PAD, UNK, read_clas_data, PAD_TOKEN_ID, DATASETS, TRN, VAL, TST, ensure_paths_exists, \
+    get_sentencepiece
+
+from typing import List
+from pathlib import Path
+import pandas as pd
+import fire
+
+@dataclass
+class XLingualCLSHyperParams(CLSHyperParams):
+    csv_name: str='train.csv'
+    target_paths: List[str] = None
+    
+    def __post_init__(self, *args, **kwargs):
+        super().__post_init__(*args, **kwargs)
+        self.target_paths = [] if self.target_paths is None else self.target_paths
+
+    def load_cls_data(self, bs, force=False, use_test_for_validation=False, **kwargs):
+        if self.tokenizer is Tokenizers.SUBWORD:
+            args = get_sentencepiece(self.base_lm_path.parent, self.base_lm_path.parent / 'train.csv',
+                                     self.name, vocab_size=self.max_vocab, pre_rules=[], post_rules=[])
+        elif self.tokenizer is Tokenizers.MOSES:
+            args = dict(tokenizer=Tokenizer(tok_func=MosesTokenizerFunc, lang='en', pre_rules=[], post_rules=[]))
+        elif self.tokenizer is Tokenizers.MOSES_FA:
+            args = dict(tokenizer=Tokenizer(tok_func=MosesTokenizerFunc, lang='en')) # use default pre/post rules
+        elif self.tokenizer is Tokenizers.FASTAI:
+            args = dict()
+        else:
+            raise ValueError(
+                f"self.tokenizer has wrong value {self.tokenizer}, Allowed values are taken from {Tokenizers}")
+
+        src_path = self.dataset_path
+        csv_name = self.csv_name
+        tgt_paths = [Path(tgt_path) for tgt_path in self.target_paths]
+        mixed_csv = pd.read_csv(src_path / csv_name, header=None)
+        for tgt_path in tgt_paths:
+            mixed_csv = pd.concat([mixed_csv, pd.read_csv(tgt_path / csv_name, header=None)])
+
+        xcvs_name = ('x_' + csv_name)
+        mixed_csv.to_csv(src_path / xcvs_name, header=None, index=False)
+        
+        data_eval = [
+            TextClasDataBunch.from_csv(path=tgt_path, csv_name=csv_name, **kwargs)
+            for tgt_path in tgt_paths
+        ]
+
+        try:
+            if force: raise FileNotFoundError("Forcing reloading of caches")
+            data_lm = TextLMDataBunch.load(src_path, 'xlm', lm_type=self.lm_type, bs=bs)
+            print(f"Tokenized data loaded, xlm.trn {len(data_lm.train_ds)}, lm.val {len(data_lm.valid_ds)}")
+        except FileNotFoundError:
+            print(f"Running tokenization...")
+            data_lm = TextLMDataBunch.from_csv(path=src_path, csv_name=xcvs_name, bs=bs, lm_type=self.lm_type, **kwargs, **args)
+            print(f"Saving tokenized: cls.trn {len(data_lm.train_ds)}, cls.val {len(data_lm.valid_ds)}")
+            data_lm.save('xlm')
+
+        try:
+            if force: raise FileNotFoundError("Forcing reloading of caches")
+            data_cls = TextClasDataBunch.load(src_path, 'cls', bs=bs)
+            print(f"Tokenized data loaded, cls.trn {len(data_cls.train_ds)}, cls.val {len(data_cls.valid_ds)}")
+        except FileNotFoundError:
+            args['vocab'] = data_lm.vocab  # make sure we use the same vocab for classifcation
+            print(f"Running tokenization...")
+            data_cls = TextClasDataBunch.from_csv(path=src_path, csv_name=csv_name, bs=bs, **kwargs, **args)
+            
+            print(f"Saving tokenized: cls.trn {len(data_cls.train_ds)}, cls.val {len(data_cls.valid_ds)}")
+            data_cls.save('cls')
+
+        print('Size of vocabulary:', len(data_lm.vocab.itos))
+        print('First 20 words in vocab:', data_lm.vocab.itos[:20])
+        return data_cls, data_lm # , data_eval
+
+
+if __name__ == '__main__':
+    fire.Fire(XLingualCLSHyperParams)

From 0436ba7ca1aee296ea4555435c0d644bc578c925 Mon Sep 17 00:00:00 2001
From: Julian Eisenschlos <eisenjulian@users.noreply.github.com>
Date: Mon, 24 Dec 2018 17:09:22 -0300
Subject: [PATCH 07/32] Rename train_xlingual_cls.py to train_xlingual_clas.py

---
 ulmfit/{train_xlingual_cls.py => train_xlingual_clas.py} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename ulmfit/{train_xlingual_cls.py => train_xlingual_clas.py} (100%)

diff --git a/ulmfit/train_xlingual_cls.py b/ulmfit/train_xlingual_clas.py
similarity index 100%
rename from ulmfit/train_xlingual_cls.py
rename to ulmfit/train_xlingual_clas.py

From d439f814d38fbd3c99e01bf8951a16db18c4f076 Mon Sep 17 00:00:00 2001
From: "NAUSICAA\\Julian" <eisenjulian@gmail.com>
Date: Sat, 29 Dec 2018 17:02:14 -0300
Subject: [PATCH 08/32] Validate other languages

---
 ulmfit/train_xlingual_clas.py | 26 +++++++++++++++++++-------
 1 file changed, 19 insertions(+), 7 deletions(-)

diff --git a/ulmfit/train_xlingual_clas.py b/ulmfit/train_xlingual_clas.py
index f383411..6d94ddf 100644
--- a/ulmfit/train_xlingual_clas.py
+++ b/ulmfit/train_xlingual_clas.py
@@ -19,7 +19,7 @@ class XLingualCLSHyperParams(CLSHyperParams):
         super().__post_init__(*args, **kwargs)
         self.target_paths = [] if self.target_paths is None else self.target_paths
 
-    def load_cls_data(self, bs, force=False, use_test_for_validation=False, **kwargs):
+    def get_tokenizer_args(self):
         if self.tokenizer is Tokenizers.SUBWORD:
             args = get_sentencepiece(self.base_lm_path.parent, self.base_lm_path.parent / 'train.csv',
                                      self.name, vocab_size=self.max_vocab, pre_rules=[], post_rules=[])
@@ -33,6 +33,10 @@ class XLingualCLSHyperParams(CLSHyperParams):
             raise ValueError(
                 f"self.tokenizer has wrong value {self.tokenizer}, Allowed values are taken from {Tokenizers}")
 
+        return args
+
+    def load_cls_data(self, bs, force=False, use_test_for_validation=False, **kwargs):
+        args = self.get_tokenizer_args()
         src_path = self.dataset_path
         csv_name = self.csv_name
         tgt_paths = [Path(tgt_path) for tgt_path in self.target_paths]
@@ -42,11 +46,6 @@ class XLingualCLSHyperParams(CLSHyperParams):
 
         xcvs_name = ('x_' + csv_name)
         mixed_csv.to_csv(src_path / xcvs_name, header=None, index=False)
-        
-        data_eval = [
-            TextClasDataBunch.from_csv(path=tgt_path, csv_name=csv_name, **kwargs)
-            for tgt_path in tgt_paths
-        ]
 
         try:
             if force: raise FileNotFoundError("Forcing reloading of caches")
@@ -72,7 +71,20 @@ class XLingualCLSHyperParams(CLSHyperParams):
 
         print('Size of vocabulary:', len(data_lm.vocab.itos))
         print('First 20 words in vocab:', data_lm.vocab.itos[:20])
-        return data_cls, data_lm # , data_eval
+        return data_cls, data_lm
+
+    def validate_cls(self, save_name='cls_last', bs=40):
+        args = self.get_tokenizer_args()
+        data_clas, data_lm = self.load_cls_data_full(bs, use_test_for_validation=True)
+        data_eval = [
+            TextClasDataBunch.from_csv(path=Path(tgt_path), csv_name=self.csv_name, **args)
+            for tgt_path in self.target_paths
+        ]
+
+        for data in [data_clas] + data_eval:
+            learn = self.create_cls_learner(data, drop_mult=0.1)
+            learn.load(save_name)
+            print(f"Loss and accuracy using ({save_name}) for dataset at {data.path}:", learn.validate())
 
 
 if __name__ == '__main__':

From 30565a2148b4cc0e3563e795d2588dea393ff544 Mon Sep 17 00:00:00 2001
From: Julian Eisenschlos <eisenjulian@gmail.com>
Date: Sat, 29 Dec 2018 20:09:46 +0000
Subject: [PATCH 09/32] Remove bad imports

---
 fastai_contrib/learner.py | 4 ++--
 fastai_contrib/utils.py   | 1 -
 ulmfit/train_clas.py      | 1 -
 3 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/fastai_contrib/learner.py b/fastai_contrib/learner.py
index 9699c41..6775e00 100644
--- a/fastai_contrib/learner.py
+++ b/fastai_contrib/learner.py
@@ -1,6 +1,6 @@
 from torch.nn import CrossEntropyLoss
 
-from fastai import GradientClipping, accuracy
+from fastai.metrics import accuracy
 from fastai.callbacks import *
 from fastai.basic_data import *
 from fastai.datasets import untar_data
@@ -108,4 +108,4 @@ def convert_weights_with_prefix(wgts:Weights, stoi_wgts:Dict[str,int], itos_new:
 import fastai.text.learner
 fastai.text.learner.convert_weights = convert_weights
 
-#endregion
\ No newline at end of file
+#endregion
diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py
index ce5cb31..62cc97e 100644
--- a/fastai_contrib/utils.py
+++ b/fastai_contrib/utils.py
@@ -4,7 +4,6 @@ Utility methods for data processing.
 import pandas as pd
 import numpy as np
 import fire
-from fastai import F, to_device
 import torch
 from tqdm import tqdm
 import re
diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py
index 627cb09..fafb573 100644
--- a/ulmfit/train_clas.py
+++ b/ulmfit/train_clas.py
@@ -14,7 +14,6 @@ from fastai.text import *
 
 import torch
 from fastai.text import TextLMDataBunch, TextClasDataBunch, language_model_learner, text_classifier_learner
-from fastai import fit_one_cycle, accuracy
 from fastai_contrib.data import LanguageModelType
 from fastai_contrib.learner import bilm_text_classifier_learner, bilm_learner, accuracy_fwd, accuracy_bwd
 from fastai_contrib.utils import PAD, UNK, read_clas_data, PAD_TOKEN_ID, DATASETS, TRN, VAL, TST, ensure_paths_exists, \

From b78a2ebaada287d5191c71df89bcedd2cac90428 Mon Sep 17 00:00:00 2001
From: "NAUSICAA\\Julian" <eisenjulian@gmail.com>
Date: Sat, 29 Dec 2018 17:12:38 -0300
Subject: [PATCH 10/32] Fixing typo

---
 ulmfit/train_xlingual_clas.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ulmfit/train_xlingual_clas.py b/ulmfit/train_xlingual_clas.py
index 6d94ddf..12a7ea7 100644
--- a/ulmfit/train_xlingual_clas.py
+++ b/ulmfit/train_xlingual_clas.py
@@ -75,7 +75,7 @@ class XLingualCLSHyperParams(CLSHyperParams):
 
     def validate_cls(self, save_name='cls_last', bs=40):
         args = self.get_tokenizer_args()
-        data_clas, data_lm = self.load_cls_data_full(bs, use_test_for_validation=True)
+        data_clas, data_lm = self.load_cls_data(bs, use_test_for_validation=True)
         data_eval = [
             TextClasDataBunch.from_csv(path=Path(tgt_path), csv_name=self.csv_name, **args)
             for tgt_path in self.target_paths

From 6780559b46b76aa8eac19658309f0b1d9e2ed332 Mon Sep 17 00:00:00 2001
From: "NAUSICAA\\Julian" <eisenjulian@gmail.com>
Date: Sat, 29 Dec 2018 17:13:38 -0300
Subject: [PATCH 11/32] Dataset name

---
 ulmfit/train_xlingual_clas.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ulmfit/train_xlingual_clas.py b/ulmfit/train_xlingual_clas.py
index 12a7ea7..b32a715 100644
--- a/ulmfit/train_xlingual_clas.py
+++ b/ulmfit/train_xlingual_clas.py
@@ -50,7 +50,7 @@ class XLingualCLSHyperParams(CLSHyperParams):
         try:
             if force: raise FileNotFoundError("Forcing reloading of caches")
             data_lm = TextLMDataBunch.load(src_path, 'xlm', lm_type=self.lm_type, bs=bs)
-            print(f"Tokenized data loaded, xlm.trn {len(data_lm.train_ds)}, lm.val {len(data_lm.valid_ds)}")
+            print(f"Tokenized data loaded, xlm.trn {len(data_lm.train_ds)}, xlm.val {len(data_lm.valid_ds)}")
         except FileNotFoundError:
             print(f"Running tokenization...")
             data_lm = TextLMDataBunch.from_csv(path=src_path, csv_name=xcvs_name, bs=bs, lm_type=self.lm_type, **kwargs, **args)

From 0856d33fa854fa1b15738e0175758931eaebc16a Mon Sep 17 00:00:00 2001
From: "NAUSICAA\\Julian" <eisenjulian@gmail.com>
Date: Thu, 3 Jan 2019 09:35:17 -0300
Subject: [PATCH 12/32] Merge issues

---
 ulmfit/pretrain_lm.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py
index 6495155..c9d3b47 100644
--- a/ulmfit/pretrain_lm.py
+++ b/ulmfit/pretrain_lm.py
@@ -12,7 +12,6 @@ import fire
 from fastai import *
 from fastai.callbacks import CSVLogger, SaveModelCallback
 from fastai.text import *
-from fastai.callbacks.tracker import SaveModelCallback
 import torch
 from fastai_contrib.utils import read_file, read_whitespace_file, \
     validate, PAD, UNK, get_sentencepiece, read_clas_data, TRN, VAL, TST, PAD_TOKEN_ID, MosesTokenizerFunc, \

From 39d38f0f46070726482ee7edbda71b9abd212f17 Mon Sep 17 00:00:00 2001
From: "NAUSICAA\\Julian" <eisenjulian@gmail.com>
Date: Thu, 3 Jan 2019 09:40:31 -0300
Subject: [PATCH 13/32] Restore newline

---
 fastai_contrib/learner.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fastai_contrib/learner.py b/fastai_contrib/learner.py
index 93d51b1..67e79cb 100644
--- a/fastai_contrib/learner.py
+++ b/fastai_contrib/learner.py
@@ -110,4 +110,4 @@ def convert_weights_with_prefix(wgts:Weights, stoi_wgts:Dict[str,int], itos_new:
 import fastai.text.learner
 fastai.text.learner.convert_weights = convert_weights
 
-#endregion
+#endregion
\ No newline at end of file

From 0e2e058b827b23c91999a48dfe105c8ffe2a57bf Mon Sep 17 00:00:00 2001
From: "NAUSICAA\\Julian" <eisenjulian@gmail.com>
Date: Thu, 3 Jan 2019 17:02:56 -0300
Subject: [PATCH 14/32] Simplify finetuning using new helper methods

---
 ulmfit/train_xlingual_clas.py | 27 ++++-----------------------
 1 file changed, 4 insertions(+), 23 deletions(-)

diff --git a/ulmfit/train_xlingual_clas.py b/ulmfit/train_xlingual_clas.py
index b32a715..0381973 100644
--- a/ulmfit/train_xlingual_clas.py
+++ b/ulmfit/train_xlingual_clas.py
@@ -1,9 +1,6 @@
 from dataclasses import dataclass
-from ulmfit.train_clas import CLSHyperParams, MosesTokenizerFunc
-from ulmfit.pretrain_lm import LMHyperParams, Tokenizers, ENC_BEST
-from fastai.text import TextLMDataBunch, TextClasDataBunch, language_model_learner, text_classifier_learner
-from fastai_contrib.utils import PAD, UNK, read_clas_data, PAD_TOKEN_ID, DATASETS, TRN, VAL, TST, ensure_paths_exists, \
-    get_sentencepiece
+from ulmfit.train_clas import CLSHyperParams
+from fastai.text import TextLMDataBunch, TextClasDataBunch
 
 from typing import List
 from pathlib import Path
@@ -19,24 +16,8 @@ class XLingualCLSHyperParams(CLSHyperParams):
         super().__post_init__(*args, **kwargs)
         self.target_paths = [] if self.target_paths is None else self.target_paths
 
-    def get_tokenizer_args(self):
-        if self.tokenizer is Tokenizers.SUBWORD:
-            args = get_sentencepiece(self.base_lm_path.parent, self.base_lm_path.parent / 'train.csv',
-                                     self.name, vocab_size=self.max_vocab, pre_rules=[], post_rules=[])
-        elif self.tokenizer is Tokenizers.MOSES:
-            args = dict(tokenizer=Tokenizer(tok_func=MosesTokenizerFunc, lang='en', pre_rules=[], post_rules=[]))
-        elif self.tokenizer is Tokenizers.MOSES_FA:
-            args = dict(tokenizer=Tokenizer(tok_func=MosesTokenizerFunc, lang='en')) # use default pre/post rules
-        elif self.tokenizer is Tokenizers.FASTAI:
-            args = dict()
-        else:
-            raise ValueError(
-                f"self.tokenizer has wrong value {self.tokenizer}, Allowed values are taken from {Tokenizers}")
-
-        return args
-
     def load_cls_data(self, bs, force=False, use_test_for_validation=False, **kwargs):
-        args = self.get_tokenizer_args()
+        args = self.tokenzier_to_fastai_args(trn_data_loading_func=lambda: trn_df[1], add_moses=True)
         src_path = self.dataset_path
         csv_name = self.csv_name
         tgt_paths = [Path(tgt_path) for tgt_path in self.target_paths]
@@ -74,7 +55,7 @@ class XLingualCLSHyperParams(CLSHyperParams):
         return data_cls, data_lm
 
     def validate_cls(self, save_name='cls_last', bs=40):
-        args = self.get_tokenizer_args()
+        args = self.tokenzier_to_fastai_args(trn_data_loading_func=lambda: trn_df[1], add_moses=True)
         data_clas, data_lm = self.load_cls_data(bs, use_test_for_validation=True)
         data_eval = [
             TextClasDataBunch.from_csv(path=Path(tgt_path), csv_name=self.csv_name, **args)

From 866d0c0bed8bb4ef82f6f84d7a3f14f4a7802fdf Mon Sep 17 00:00:00 2001
From: "NAUSICAA\\Julian" <eisenjulian@gmail.com>
Date: Tue, 12 Feb 2019 12:52:12 -0300
Subject: [PATCH 15/32] Addiing XLingual LM

---
 ulmfit/pretrain_xlingual_lm.py | 109 +++++++++++++++++++++++++++++++++
 ulmfit/train_xlingual_clas.py  |   3 +-
 2 files changed, 111 insertions(+), 1 deletion(-)
 create mode 100644 ulmfit/pretrain_xlingual_lm.py

diff --git a/ulmfit/pretrain_xlingual_lm.py b/ulmfit/pretrain_xlingual_lm.py
new file mode 100644
index 0000000..803989b
--- /dev/null
+++ b/ulmfit/pretrain_xlingual_lm.py
@@ -0,0 +1,109 @@
+from dataclasses import dataclass
+from ulmfit.train_clas import LMHyperParams
+from fastai.text import TextLMDataBunch, TextClasDataBunch
+from fastai.basic_train import LearnerCallback
+from fastai.torch_core import PBar, Rank0Tensor
+from torch import nn, Tensor
+
+from typing import List, Collection, Any
+from pathlib import Path
+import pandas as pd
+import fire
+import random
+
+@dataclass
+class ParallelAlignmentCallback(LearnerCallback):
+    "A `LearnerCallback` that adds parallel alignment between sentences."
+
+    data_src:TextClasDataBunch
+    data_tgt:TextClasDataBunch
+    alpha:float=0.1
+
+    def __post_init__(self):
+        self.bs = self.data_src.bs
+        self.loss = nn.CosineEmbeddingLoss(margin=0.5)
+        self.ones = torch.cat((torch.ones(self.bs), -torch.ones(self.bs)))
+
+    def pool(self, x:Tensor, bs:int, is_max:bool):
+        "Pool the tensor along the seq_len dimension."
+        f = F.adaptive_max_pool1d if is_max else F.adaptive_avg_pool1d
+        return f(x.transpose(1,2), (1,)).view(bs,-1)
+
+    def get_representation(batch):
+        last_output = self.learn.model(batch)
+        output = last_output[1][-1]
+        bs,sl,_ = output.size()
+        avgpool = self.pool(output, bs, False)
+        mxpool = self.pool(output, bs, True)
+        return torch.cat([output[:,-1], mxpool, avgpool], 1)
+
+    def on_train_begin(self, pbar:PBar, metrics_names:Collection[str], **kwargs:Any)->None:
+        self.counter = 0
+
+    def on_backward_begin(self, last_loss:Rank0Tensor, last_input:Tensor, **kwargs):
+        "Adjust the loss by adding similarity of parallel sentences"
+        src_rep = self.get_representation(data_src.train_ds[self.counter])
+        tgt_rep = self.get_representation(data_tgt.train_ds[self.counter])
+
+        offset = -random.randrange(1, self.bs)
+
+        src_rep = torch.cat((src_rep, src_rep))
+        tgt_rep = torch.cat((tgt_rep, tgt_rep[range(offset, self.bs + offset)]))
+
+        parallel_loss = self.alpha * self.loss(src_rep, tgt_rep, self.y)
+        
+        self.counter += 1
+        self.counter %= len(data_src.train_ds)
+        return last_loss + parallel_loss
+
+
+@dataclass
+class XLingualLMHyperParams(LMHyperParams):
+    
+    parallel_data_path: str=None
+    parallel_data_bs: int=32
+    src_lang: str=None
+    tgt_lang: str=None
+
+    def create_lm_learner(self, data_lm, dps=None, **kwargs):
+        learner = super().create_lm_learner(data_lm, dps, **kwargs)
+        if self.parallel_data_path is not None:
+            src_trn_df = pd.read_csv(self.parallel_data_path / self.src_lang / 'train.csv', header=None)
+            tgt_trn_df = pd.read_csv(self.parallel_data_path / self.tgt_lang / 'train.csv', header=None)
+            bs = self.parallel_data_bs
+            data_src = TextClasDataBunch.from_df(path=self.cache_dir, train_df=src_trn_df, lm_type=self.lm_type, bs=bs)
+            data_tgt = TextClasDataBunch.from_df(path=self.cache_dir, train_df=tgt_trn_df, lm_type=self.lm_type, bs=bs)
+            learner.callback_fns = [
+                partial(ParallelAlignmentCallback, data_src=data_src, data_tgt=data_tgt)
+            ] + learner.callback_fns
+
+    def load_wiki_data(self, bs=70):
+        trn_path = self.dataset_path / f'{self.lang}.wiki.train.tokens'
+        val_path = self.dataset_path / f'{self.lang}.wiki.valid.tokens'
+        tst_path = self.dataset_path / f'{self.lang}.wiki.test.tokens'
+        for path_ in [trn_path, val_path, tst_path]:
+            assert path_.exists(), f'Error: {path_} does not exist.'
+
+        args = self.tokenzier_to_fastai_args(trn_data_loading_func=self.load_train_text, add_moses=False)
+        try:
+            data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type, bs=bs)
+            print("Tokenized data loaded")
+        except FileNotFoundError:
+            print("Running tokenization")
+            data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=read_wiki_articles(trn_path),
+                                              valid_df=read_wiki_articles(val_path),
+                                              classes=None, lm_type=self.lm_type, max_vocab=self.max_vocab,
+                                              bs=bs, text_cols='texts', **args)
+            data_lm.save('.')
+
+        itos, stoi, trn_path = data_lm.vocab.itos, data_lm.vocab.stoi, data_lm.path
+        print('Size of vocabulary:', len(itos))
+        print('First 20 words in vocab:', data_lm.vocab.itos[:20])
+        return data_lm
+
+if __name__ == '__main__':
+    fire.Fire(XLingualLMHyperParams)
+
+
+# python -m ulmfit.XLingualLMHyperParams --dataset-path data/wiki/wikitext-103 --bidir=True --qrnn=True --nl=4 --tokenizer=sp --name 'nl4' --bs 120 --cuda-id 0 - train 10 --drop-mult=0 --bs 40
+
diff --git a/ulmfit/train_xlingual_clas.py b/ulmfit/train_xlingual_clas.py
index 0381973..d30f89e 100644
--- a/ulmfit/train_xlingual_clas.py
+++ b/ulmfit/train_xlingual_clas.py
@@ -10,7 +10,8 @@ import fire
 @dataclass
 class XLingualCLSHyperParams(CLSHyperParams):
     csv_name: str='train.csv'
-    target_paths: List[str] = None
+    target_paths: List[str]=None
+    parallel_data_path: str=None
     
     def __post_init__(self, *args, **kwargs):
         super().__post_init__(*args, **kwargs)

From 837925ff5307da970373bc974124797512381a51 Mon Sep 17 00:00:00 2001
From: Piotr Czapla <github@piotrczapla.com>
Date: Sun, 3 Mar 2019 13:29:49 +0100
Subject: [PATCH 16/32] Imporved validate_cls & eval to pick the best model
 based on val accuracy

---
 ulmfit/__main__.py    | 66 ++++++++++++++++++++++++++++++++++---------
 ulmfit/pretrain_lm.py | 14 +++++++--
 ulmfit/train_clas.py  | 20 +++++++------
 3 files changed, 75 insertions(+), 25 deletions(-)

diff --git a/ulmfit/__main__.py b/ulmfit/__main__.py
index 38b7f76..cec29f7 100644
--- a/ulmfit/__main__.py
+++ b/ulmfit/__main__.py
@@ -7,7 +7,7 @@ from collections import OrderedDict
 from functools import wraps
 import pandas as pd
 import fire
-from .pretrain_lm import LMHyperParams
+from .pretrain_lm import LMHyperParams, json_save, json_load, np
 from .train_clas import CLSHyperParams
 from pathlib import Path
 from string import Template
@@ -40,8 +40,11 @@ class ULMFiT:
 
     lm2 = LMHyperParams
     @wraps(CLSHyperParams)
-    def cls(self, dataset_path, base_lm_path, **changes):
-        params = CLSHyperParams.from_lm(dataset_path, base_lm_path, **changes)
+    def cls(self, dataset_path, base_lm_path=None, **changes):
+        if base_lm_path is not None:
+            params = CLSHyperParams.from_lm(dataset_path, base_lm_path, **changes)
+        else:
+            params = CLSHyperParams(dataset_path=dataset_path, **changes)
         return FireView(train=params.train_cls, validate_cls=params.validate_cls)
 
     @wraps(CLSHyperParams)
@@ -63,8 +66,9 @@ class ULMFiT:
                           bs=bs,
                           lr_sched=lr_sched,
                           label_smoothing_eps=label_smoothing_eps,
+                          return_df=True,
                           **kwargs)
-            val = next(iter(d.values()), -1)
+            val = d['tst_accuracy'][0]
             results.append((noise/100, val))
         df = pd.DataFrame(results, columns=["noise", "accuracy"])
         df.to_csv(f"noise_{lang}-{size}{prefix_name}.csv")
@@ -84,27 +88,61 @@ class ULMFiT:
                     print("Adding", f, dest)
                     tar.add(f, dest)
 
-    def eval(self, glob="mldoc/*-1/models/sp30k/lstm_nl4.m", dataset_template='${lang}-1', name="tmp-100", num_lm_epochs=0, cuda_id=0, **trn_params):
-        results = OrderedDict()
+    def eval(self, glob="mldoc/*-1/models/sp30k/lstm_nl4.m", dataset_template='${ds_name}', name=None,
+             num_lm_epochs=0, cuda_id=0, train=True, to_csv=None, return_df=False, label_smoothing_eps=0.0,
+             **trn_params):
+        results = []
+
+
+        def extract_agg(group):
+            best = group.loc[group["val_accuracy"].idxmax()]["tst_accuracy"]
+            best_name = group.loc[group["val_accuracy"].idxmax()]["n"]
+            return pd.Series({'best': best* 100,
+                              'max': group['tst_accuracy'].max()* 100,
+                              'avg': group['tst_accuracy'].mean()* 100})
+        def pivot_to_lang(df):
+            df['ds'] = df['name'].str.extract(r'data/[a-z]*/([^/]*)/models')
+            df['n'] = df['name'].str.extract(r'models/[^/]*/([^/]*).m')
+            best = df.groupby('ds').apply(extract_agg)
+            best = best.round(2)
+            return best.T
         for base_model in sorted(Path("data").glob(glob)):
             print("Processing", base_model)
             for lang, dataset_path in sorted(get_dataset_path(base_model, dataset_template)):
                 try:
-                    params = CLSHyperParams.from_lm(dataset_path, base_model, lang=lang, name=name, cuda_id=cuda_id)
+                    _name = name
+                    if name is None:
+                        _name = base_model.name.replace(".m","").replace("lstm_","").replace("qrnn_","")
+                    params = CLSHyperParams.from_lm(dataset_path, base_model, lang=lang, name=_name, cuda_id=cuda_id)
                     key = str(params.model_dir.relative_to(Path.cwd()))
-                    if (params.model_dir/"cls_best.pth").exists():
+                    if (params.model_dir / "results.npy").exists():
+                        d = np.load(params.model_dir / "results.npy")
+                        d = d.tolist() # magiacally convert to dict
+                    elif (params.model_dir/"cls_best.pth").exists():
                         print("Evaluating previously trained model")
-                        results[key] = params.validate_cls()[1]
-                    else:
+                        d = params.validate_cls(label_smoothing_eps=label_smoothing_eps)
+                    elif train:
                         print("Training")
-                        results[key] = params.train_cls(num_lm_epochs=num_lm_epochs, **trn_params)[1]
+                        d = params.train_cls(num_lm_epochs=num_lm_epochs, label_smoothing_eps=label_smoothing_eps, **trn_params)
+                    else:
+                        print("Skipping", (params.model_dir/"cls_best.pth"))
+                        d  = None
+                    if d is not None:
+                        d['name']=key
+                        np.save(params.model_dir / "results.npy", d)
+                        results.append(d)
                     del params
                 except Exception as e:
                     print("Error", e)
                 gc.collect()
-
-        pprint.pprint(results)
-        return results
+        df = pd.DataFrame.from_records(results)
+        print(df)
+        print(pivot_to_lang(df))
+        if to_csv is not None:
+            print(f"Saving result to: {to_csv}")
+            df.to_csv(to_csv)
+        if return_df:
+            return df
 
     def remove_lm_saves(self):
         for lm_save in Path("data").glob("**/lm_*.pth"):
diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py
index 2a6f59f..94efd8a 100644
--- a/ulmfit/pretrain_lm.py
+++ b/ulmfit/pretrain_lm.py
@@ -52,6 +52,14 @@ def read_wiki_articles(filename):
     print(f"Wiki text was split to {len(articles)} articles")
     return pd.DataFrame({'texts': np.array(articles, dtype=np.object)})
 
+def json_save(f, d):
+    with Path(f).open("w") as fp:
+        json.dump(d, fp)
+
+def json_load(f):
+    with open(f, 'r') as f:
+        return json.load(f)
+
 @dataclass
 class LMHyperParams:
     dataset_path: str # data_dir
@@ -168,7 +176,7 @@ class LMHyperParams:
         vals.pop('name', None)
         vals.pop('lang', None)
         vals['tokenizer'] = self.tokenizer.value
-        with (self.model_dir / 'info.json').open("w") as fp: json.dump(vals, fp)
+        json_save(self.model_dir/'info.json', vals)
         print("Saving info", self.model_dir / 'info.json')
 
     def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, lr=5e-3, label_smoothing_eps=0.0):
@@ -249,7 +257,7 @@ class LMHyperParams:
 
         args = self.tokenizer_to_fastai_args(sp_data_func=self.load_train_text, use_moses=False)
 
-        data_lm = self.lm_databunch("lm",
+        data_lm = self.lm_databunch(f"lm{self.bptt if self.bptt != 70 else ''}",
                           train_df=read_wiki_articles(trn_path),
                           valid_df=read_wiki_articles(val_path),
                           classes=None,
@@ -302,7 +310,7 @@ class LMHyperParams:
     def from_lm(cls, dataset_path, base_lm_path, **kwargs) -> 'LMHyperParams':
         dataset_path = Path(dataset_path).resolve()
         base_lm_path = Path(base_lm_path).resolve()
-        with open(base_lm_path/'info.json', 'r') as f: d = json.load(f)
+        d = json_load(base_lm_path/'info.json')
         d['dataset_path'] = dataset_path
         d['base_lm_path'] = base_lm_path
         d.pop('bs', None)
diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py
index b989989..965df41 100644
--- a/ulmfit/train_clas.py
+++ b/ulmfit/train_clas.py
@@ -10,7 +10,7 @@ from fastai_contrib.utils import PAD_TOKEN_ID
 
 import fire
 
-from ulmfit.pretrain_lm import LMHyperParams, ENC_BEST
+from ulmfit.pretrain_lm import LMHyperParams, ENC_BEST, json_save
 
 
 class CLSHyperParams(LMHyperParams):
@@ -101,18 +101,22 @@ class CLSHyperParams(LMHyperParams):
         learn.save('cls_last', with_opt=False)
         learn.save('cls_best', with_opt=False) # we don't use early stopping for the time being
         del learn
-        return self.validate_cls('cls_best', bs=bs, data_tst=data_tst, learn=None)
+        return self.validate_cls('cls_best', bs=bs, data_cls=data_clas, data_tst=data_tst, learn=None)
 
-    def validate_cls(self, save_name='cls_best', bs=40, data_tst=None, learn=None):
+    def validate_cls(self, save_name='cls_best', bs=40, data_cls=None, data_tst=None, learn=None, label_smoothing_eps=0.0):
         if data_tst is None:
-            _, _, data_tst = self.load_cls_data(bs)
+            data_cls, _, data_tst = self.load_cls_data(bs)
         if learn is None:
-            learn = self.create_cls_learner(data_tst, drop_mult=0.3)
+            learn = self.create_cls_learner(data_tst, drop_mult=0.3, label_smoothing_eps=label_smoothing_eps)
             learn.unfreeze()
         learn.load(save_name)
-        results = learn.validate(data_tst.valid_dl)
-        print(f"Loss and accuracy using ({save_name}):", results)
-        return list(map(float, results))
+        val_res=[-1, -1]
+        if data_cls:
+            val_res = learn.validate(data_cls.valid_dl)
+        tst_res = learn.validate(data_tst.valid_dl)
+        print(f"Loss and accuracy using ({save_name}):", tst_res, val_res)
+        results = {'val_loss': val_res[0], 'val_accuracy': float(val_res[1]), 'tst_loss':tst_res[0], 'tst_accuracy': float(tst_res[1]) }
+        return results
 
     def create_cls_learner(self, data_clas, dps=None, label_smoothing_eps=0.0, random_init=False, **kwargs):
         assert self.bidir == False, "bidirectional model is not yet supported"

From 66ec30a1228eb133a29d04bd5816aaa521a51e34 Mon Sep 17 00:00:00 2001
From: Piotr Czapla <github@piotrczapla.com>
Date: Tue, 26 Mar 2019 21:27:18 +0100
Subject: [PATCH 17/32] Add more results

---
 fastai_contrib/models.py                   |   4 +-
 fastai_contrib/utils.py                    |   2 +-
 results/CLS.md                             |   6 +
 results/MLDoc.md                           |  20 +-
 results/logs/100examples.md                | 289 ++++++++++++++++++++-
 results/logs/de.md                         |  92 +++++++
 results/logs/mldoc/label_smoothing.md      |  39 +++
 results/logs/mldoc/random-init.md          | 242 ++++++++++++++++-
 results/logs/pseudo-labeling.md            | 174 +++++++++++++
 results/logs/ru.md                         | 157 ++++++++++-
 results/logs/ru/bptt.md                    | 132 ++++++++++
 results/logs/ru/nowiki-sp16k.md            | 115 ++++++++
 results/logs/ru/results.md                 |  13 +
 results/logs/ru/sp25.md                    |  28 ++
 results/logs/tokenization/average10runs.md | 280 ++++++++++++++++++++
 results/logs/tokenization/fr.md            | 241 +++++++++++++++++
 results/logs/tokenization/ru.md            | 149 +++++++++++
 results/logs/zeroshot.md                   | 266 +++++++++++++++++++
 18 files changed, 2224 insertions(+), 25 deletions(-)
 create mode 100644 results/CLS.md
 create mode 100644 results/logs/mldoc/label_smoothing.md
 create mode 100644 results/logs/pseudo-labeling.md
 create mode 100644 results/logs/ru/bptt.md
 create mode 100644 results/logs/ru/nowiki-sp16k.md
 create mode 100644 results/logs/ru/results.md
 create mode 100644 results/logs/ru/sp25.md
 create mode 100644 results/logs/tokenization/average10runs.md
 create mode 100644 results/logs/tokenization/fr.md

diff --git a/fastai_contrib/models.py b/fastai_contrib/models.py
index 92ac7e4..fbed679 100644
--- a/fastai_contrib/models.py
+++ b/fastai_contrib/models.py
@@ -69,8 +69,8 @@ class MultiBatchBiLMModel(BiLMModel):
                 raw_outputs.append(r)
                 outputs.append(o)
         return self.concat(raw_outputs), self.concat(outputs)
-
-class BiPoolingLinearClassifier(PoolingLinearClassifier):
+#PoolingLinearClassifier
+class BiPoolingLinearClassifier(nn.Module):
     "Create a linear classifier with pooling."
 
     def forward(self, input:Tuple[Tensor,Tensor])->Tuple[Tensor,Tensor,Tensor]:
diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py
index ebd9674..d0c93ea 100644
--- a/fastai_contrib/utils.py
+++ b/fastai_contrib/utils.py
@@ -67,7 +67,7 @@ class SentencePieceTokenizer(Tokenizer):
         toks = tok.sp.EncodeAsPieces(" ".join(toks))
         return toks
 full_char_coverage_langs = ["bg", "cs", "da", "de", "el", "en", "es", "et", "fi", "fr", "ga", "hr", "hu",
-                       "it","lt","lv","mt","nl","pl","pt","ro","sk","sl","sv"] # all European langus
+                       "it","lt","lv","mt","nl","pl","pt","ro","sk","sl","sv"] # all European langs
 
 def get_sentencepiece(cache_dir:PathOrStr, load_text, pre_rules: ListRules=None, post_rules:ListRules=None,
                       vocab_size:int=30000, model_type:str='unigram', input_sentence_size:int=1E7, lang='en'):
diff --git a/results/CLS.md b/results/CLS.md
new file mode 100644
index 0000000..87a070e
--- /dev/null
+++ b/results/CLS.md
@@ -0,0 +1,6 @@
+# Zero shot from CLS
+
+
+
+
+### zeroshoot 
\ No newline at end of file
diff --git a/results/MLDoc.md b/results/MLDoc.md
index b8a9ff1..93ecad6 100644
--- a/results/MLDoc.md
+++ b/results/MLDoc.md
@@ -11,7 +11,7 @@
 |ULMFiT sp-fixed |           |  **95.6** |   94.80   |   94.20   |   88.52   |   88.72   |   86.85   |  90.47     |
 |ULMFIT Q15k 1cyc|   94.62   | **95.65** |   95.15   | **94.42** |   89.92   |   89.60   |           |  90.78/89.82  |
 |ULMFIT Q15k 1c l| **94.99** |           |   95.64   |   94.34   | **90.32** |   89.67   |   87.67^  | **92.22**  |
-|ULMFIT Q15k 1cfl| **95.55** | **96.10** |   95.82   |   94.80   | **90.04** |   89.87   |   87.17   | **91.90**  |
+|ULMFIT Q15k 1cfl| **95.55** | **96.10** |   95.97   |   94.77   | **90.07** |   89.87   |   87.17   | **92.40**  |
 |ULMFIT L30k 1cyc|           | **95.85** | **96.32** | **94.82** |   89.87   | **90.45** | **87.94** |  92.02/91.64  |
 
 - L30k  -  LSTM sp30k trained using gradual unfreezing
@@ -22,8 +22,18 @@
 - We checked LSTM on sp15k on DE and got 95.53% accuracy which is comparable to QRNN sp15k
 - ^ - 16 epochs qrnn_nl4sl-bs500
 
-## Zero shot approaches - LSTM
 
+## Zeroshot 
+
+| Model                |    de      |    es      |   fr      |   it      |   ru      |   zh      |
+|----------------------|------------|------------|-----------|-----------|-----------|-----------|
+| LASER-en             |    87.65   |    75.48   |   84.00   |   71.18   |   66.58   |   76.65   |
+| ULMFiT L on LASER-en |  **92.95** |  **80.50** |   88.78   |   76.20   | **70.05** |   80.45   |
+| ULMFiT Q on LASER-en |    91.34   |    78.92   | **89.45** |   76.00   |   68.19   | **82.45** |
+- L - 1k LSTM sp30k
+- Q - 1k QRNN sp15k
+
+#### LSTM results
 | Model                |    de      |    es      |   fr      |   it      |   ru      |   zh      |
 |----------------------|------------|------------|-----------|-----------|-----------|-----------|
 | LASER-de             |            |    81.40   |   81.50   |   74.53   |   64.58   |   73.20   |
@@ -42,7 +52,7 @@
 | Bert Multilingual-EN |    74.50   |   61.85    |   69.77   |   57.73   |   51.10   |  64.08    |
 
 
-### From Laser trained on French data
+#### From Laser trained on French data
 | Model Name                | de    | es    | fr | it    | ru    | zh    |
 |---------------------------|-------|-------|----|-------|-------|-------|
 | LASER fr 10k              | 91.65 | 81.05 |    | 75.08 | 70.73 | 76.33 |
@@ -55,7 +65,7 @@
 | Impr 10k over 1k          | 32%   | 10%   |    | 11%   | 7%    | 30%   |
 | Impr 1k over 1k           | 31%   | 4%    |    | 16%   | 3%    | 25%   |
 
-### From Laser trained on German data
+#### From Laser trained on German data
 | Model Name                | de | es    | fr    | it    | ru    | zh    |
 |---------------------------|----|-------|-------|-------|-------|-------|
 | LASER de 10k              |    | 83.5  | 82.85 | 76.6  | 68.8  | 73.12 |
@@ -68,7 +78,7 @@
 | Impr 10k over 1k          |    | 17%   | 32%   | 16%   | 9%    | 16%   |
 | Impr 1k over 1k           |    | 22%   | 32%   | 17%   | 7%    | -3%   |
 
-### From Laser trained on English data 
+#### From Laser trained on English data 
 | Model Name                | de    | es    | fr    | it    | ru    | zh    |
 |---------------------------|-------|-------|-------|-------|-------|-------|
 | LASER en 10k              | 87.43 | 77.38 | 78.7  | 72.53 | 67.7  | 75.18 |
diff --git a/results/logs/100examples.md b/results/logs/100examples.md
index 602ec9b..d39a64d 100644
--- a/results/logs/100examples.md
+++ b/results/logs/100examples.md
@@ -1,22 +1,290 @@
 # MLDoc classification using 100 examples
 
-| Language    | 8 epochs (1)      |  4 epochs (1)      |  4 epochs (2)     | 8 epochs (2) | 
-|-------------|--------------------|---------------------|------------------|----------|
-| de          |          92.37     |        91.79        |   84.60          |  91.27   |
-| en          |          77.14     |        66.02        |   70.85          |  87.00   |
-| es          |          89.52     |        87.55        |   80.17          |  89.57   |
-| fr          |          81.44     |        74.25        |   79.97          |  88.15   |
-| it          |          81.15     |        69.24        |   74.17          |  77.54   |
-| ja          |          78.87     |        70.30        |   69.74          |  78.64   |
-| zh          |          83.57     |        70.47        |   77.39          |  87.17   |
+| Language    | 8 epochs (1)       |  4 epochs (1)       |  4 epochs (2)   | 8 epochs (2) | 8 epochs (3)| 
+|-------------|--------------------|---------------------|------------------|----------|------------|
+| en          |          77.14     |        66.02        |   70.85          |  87.00   |    83.07   |
+| de          |          92.37     |        91.79        |   84.60          |  91.27   |    90.90   |
+| es          |          89.52     |        87.55        |   80.17          |  89.57   |    89.00   |
+| fr          |          81.44     |        74.25        |   79.97          |  88.15   |    85.03   |
+| it          |          81.15     |        69.24        |   74.17          |  77.54   |    80.12   |
+| ja          |          78.87     |        70.30        |   69.74          |  78.64   |    80.55   |
+| ru          |                     |                     |                 |          |    73.55   |
+| zh          |          83.57     |        70.47        |   77.39          |  87.17   |    88.02   |
+
 
 - (1) - a larger dropout value for output_p=0.7 instead of output_p=0.2, and wd=1e-1
 - (2) - normal dropout but still wd=1e-1
 - (3) - normal dropout but and normal wd=1e-2
 
 ## QRNN sp15k - normal dropout, normal wd
+Russian
+```
+python -m ulmfit eval --glob="mldoc/*-1/models/sp15k/qrnn_nl4.m" --name nl4-100-e8-normal-dp-wd  --limit=100 --num-cls-epochs=8 --lr_sched=1cycle --label-smoothing-eps=0.1 --bs=18
+ru-1
+Max vocab: 15000
+Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k
+Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m
+Training
+Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv
+Limiting data set to: 100
+Data lm, trn: 9195, val: 1021
+Running tokenization clslimit100...
+Data clslimit100, trn: 100, val: 100
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 15000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', '▁на', 'х']
+Training args:  {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps:  {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
+Loading pretrained model
+Unknown tokens 0, first 100: []
+Bptt 70
+/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k
+Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m/info.json
+Single training schedule
+epoch     train_loss  valid_loss  accuracy
+1         1.327627    1.381715    0.230000
+2         1.285086    1.290231    0.450000
+3         1.131082    1.246561    0.370000
+4         1.005522    1.124416    0.530000
+5         0.942438    1.133643    0.540000
+6         0.858336    1.063248    0.620000
+7         0.802710    1.037930    0.640000
+8         0.760177    1.013346    0.640000
+Total time: 00:26
+Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m
+Loss and accuracy using (cls_best): [0.7943169, tensor(0.7355)]
+Processing data/mldoc/zh-1/models/sp15k/qrnn_nl4.m
 
+data/mldoc/de-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m: 0.9089999794960022
+data/mldoc/en-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m: 0.8307499885559082
+data/mldoc/es-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m: 0.8889999985694885
+data/mldoc/fr-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m: 0.8502500057220459
+data/mldoc/it-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m: 0.7987499833106995
+data/mldoc/ja-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m: 0.8015000224113464
+data/mldoc/ru-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m: 0.7354999780654907
+data/mldoc/zh-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m: 0.8774999976158142
+```
+```bash
+python -m ulmfit eval --glob="mldoc/*-1/models/sp15k/qrnn_nl4.m" --name nl4-100-e8-normal-dp-wd  --limit=100 --num-cls-epochs=8 --lr_sched=1cycle --label-smoothing-eps=0.1 --bs=18
+Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m
+de-1
+Max vocab: 15000
+Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k
+Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m
+Training
+Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/de.dev.csv
+Limiting data set to: 100
+Data lm, trn: 13500, val: 1500
+Data clslimit100, trn: 100, val: 100
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 15000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"]
+/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning:
 
+                               !! WARNING !!
+
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+Your compiler (c++) may be ABI-incompatible with PyTorch!
+Please use a compiler that is ABI-compatible with GCC 4.9 and above.
+See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html.
+
+See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6
+for instructions on how to install GCC 4.9 or higher.
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+                              !! WARNING !!
+
+  warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler))
+Single training schedule
+epoch     train_loss  valid_loss  accuracy
+1         1.299891    1.361820    0.280000
+2         1.073693    1.173532    0.520000
+3         0.901732    0.802865    0.850000
+4         0.814596    0.769840    0.940000
+5         0.751278    0.741906    0.920000
+6         0.701402    0.704007    0.930000
+7         0.660147    0.702021    0.930000
+8         0.626870    0.683630    0.920000
+Total time: 00:23
+Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m
+Loss and accuracy using (cls_best): [0.5047863, tensor(0.9090)]
+Processing data/mldoc/en-1/models/sp15k/qrnn_nl4.m
+en-1
+Max vocab: 15000
+Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k
+Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m
+Training
+Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/en.dev.csv
+Limiting data set to: 100
+Data lm, trn: 13500, val: 1500
+Data clslimit100, trn: 100, val: 100
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 15000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁the', '▁,', 's', '▁.', '▁of', '▁and', '▁in', '▁to', '▁a', 'ed']
+Single training schedule
+epoch     train_loss  valid_loss  accuracy
+1         1.349818    1.378012    0.340000
+2         1.126246    1.169077    0.700000
+3         0.975751    1.123801    0.580000
+4         0.873535    0.803686    0.880000
+5         0.808903    0.850648    0.860000
+6         0.751683    0.858902    0.810000
+7         0.714687    0.868248    0.760000
+8         0.673551    0.842978    0.790000
+Total time: 00:21
+Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m
+Loss and accuracy using (cls_best): [0.62346387, tensor(0.8307)]
+Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m
+es-1
+Max vocab: 15000
+Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k
+Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m
+Training
+Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv
+Limiting data set to: 100
+Data lm, trn: 13013, val: 1445
+Data clslimit100, trn: 100, val: 100
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 15000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que']
+Single training schedule
+epoch     train_loss  valid_loss  accuracy
+1         1.301228    1.357104    0.220000
+2         1.085064    1.102389    0.480000
+3         0.917675    0.933821    0.700000
+4         0.819897    0.867835    0.770000
+5         0.754799    0.838266    0.800000
+6         0.705542    0.753635    0.860000
+7         0.664083    0.705510    0.900000
+8         0.631440    0.695309    0.900000
+Total time: 00:19
+Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m
+Loss and accuracy using (cls_best): [0.47515148, tensor(0.8900)]
+Processing data/mldoc/fr-1/models/sp15k/qrnn_nl4.m
+fr-1
+Max vocab: 15000
+Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k
+Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m
+Training
+Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/fr.dev.csv
+Limiting data set to: 100
+Data lm, trn: 13500, val: 1500
+Data clslimit100, trn: 100, val: 100
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 15000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', 's', '▁.', "'", '▁la', '▁le', '▁et', '▁l', '▁à']
+Single training schedule
+epoch     train_loss  valid_loss  accuracy
+1         1.342008    1.382562    0.230000
+2         1.098667    1.072769    0.720000
+3         0.954929    1.140843    0.580000
+4         0.848720    0.878910    0.740000
+5         0.769108    0.851176    0.770000
+6         0.710496    0.773629    0.870000
+7         0.666720    0.768422    0.850000
+8         0.632068    0.754751    0.870000
+Total time: 00:22
+Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m
+Loss and accuracy using (cls_best): [0.5851091, tensor(0.8503)]
+Processing data/mldoc/it-1/models/sp15k/qrnn_nl4.m
+it-1
+Max vocab: 15000
+Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k
+Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m
+Training
+Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/it.dev.csv
+Limiting data set to: 100
+Data lm, trn: 13500, val: 1500
+Data clslimit100, trn: 100, val: 100
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 15000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁di', '▁e', "▁&'", "'", '▁il', '▁la', '▁in', 'e']
+Single training schedule
+epoch     train_loss  valid_loss  accuracy
+1         1.422726    1.367270    0.420000
+2         1.209058    1.242881    0.410000
+3         1.057216    1.163817    0.420000
+4         0.933097    0.889416    0.800000
+5         0.867725    0.970408    0.720000
+6         0.796040    0.905692    0.790000
+7         0.748742    0.887287    0.760000
+8         0.705950    0.869270    0.790000
+Total time: 00:15
+Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m
+Loss and accuracy using (cls_best): [0.67111975, tensor(0.8012)]
+Processing data/mldoc/ja-1/models/sp15k/qrnn_nl4.m
+ja-1
+Max vocab: 15000
+Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k
+Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m
+Training
+Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/ja.dev.csv
+Limiting data set to: 100
+Data lm, trn: 13500, val: 1500
+Data clslimit100, trn: 100, val: 100
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 15000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)']
+Single training schedule
+epoch     train_loss  valid_loss  accuracy
+1         1.255629    1.371983    0.350000
+2         1.092927    1.205145    0.630000
+3         1.048904    1.064253    0.600000
+4         0.943902    0.894827    0.810000
+5         0.849500    0.947021    0.710000
+6         0.797143    0.908895    0.730000
+7         0.746244    0.853216    0.770000
+8         0.705181    0.823923    0.800000
+Total time: 00:24
+Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ja-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m
+Loss and accuracy using (cls_best): [0.6462945, tensor(0.8055)]
+Processing data/mldoc/zh-1/models/sp15k/qrnn_nl4.m
+zh-1
+Max vocab: 15000
+Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k
+Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m
+Training
+Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/zh.dev.csv
+Limiting data set to: 100
+Data lm, trn: 13500, val: 1500
+Data clslimit100, trn: 100, val: 100
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 15000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有']
+Single training schedule
+epoch     train_loss  valid_loss  accuracy
+1         1.262576    1.341136    0.330000
+2         1.030734    1.104332    0.800000
+3         0.893319    1.021920    0.700000
+4         0.791810    1.040577    0.630000
+5         0.719387    0.888048    0.760000
+6         0.670949    0.809457    0.880000
+7         0.634845    0.786899    0.870000
+8         0.606350    0.774056    0.860000
+Total time: 00:19
+Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m
+Loss and accuracy using (cls_best): [0.59781086, tensor(0.8802)]
+OrderedDict([('data/mldoc/de-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m',
+              0.9089999794960022),
+             ('data/mldoc/en-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m',
+              0.8307499885559082),
+             ('data/mldoc/es-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m',
+              0.8899999856948853),
+             ('data/mldoc/fr-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m',
+              0.8502500057220459),
+             ('data/mldoc/it-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m',
+              0.8012499809265137),
+             ('data/mldoc/ja-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m',
+              0.8054999709129333),
+             ('data/mldoc/zh-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m',
+              0.8802499771118164)])
+data/mldoc/de-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m: 0.9089999794960022
+data/mldoc/en-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m: 0.8307499885559082
+data/mldoc/es-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m: 0.8899999856948853
+data/mldoc/fr-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m: 0.8502500057220459
+data/mldoc/it-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m: 0.8012499809265137
+data/mldoc/ja-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m: 0.8054999709129333
+data/mldoc/zh-1/models/sp15k/qrnn_nl4-100-e8-normal-dp-wd.m: 0.8802499771118164
+```
 
 ## QRNN sp15k - normal dropout, wd=1e-1
 
@@ -1014,4 +1282,5 @@ data/mldoc/fr-1/models/sp15k/qrnn_nl4-100e4.m: 0.7425000071525574
 data/mldoc/it-1/models/sp15k/qrnn_nl4-100e4.m: 0.6924999952316284
 data/mldoc/ja-1/models/sp15k/qrnn_nl4-100e4.m: 0.703000009059906
 data/mldoc/zh-1/models/sp15k/qrnn_nl4-100e4.m: 0.7047500014305115
-````
\ No newline at end of file
+````
+
diff --git a/results/logs/de.md b/results/logs/de.md
index 5142dbb..922d776 100644
--- a/results/logs/de.md
+++ b/results/logs/de.md
@@ -173,6 +173,98 @@ Loss and accuracy using (cls_best): [0.16954255, tensor(0.9475)]
 OrderedDict([('data/mldoc/de-1/models/vf60k/lstm_nl3.m', 0.9474999904632568)])
 ```
 MultiCCA: 93.7% , ulmfit: 94.74%
+## VF60k QRNN nl 4
+```
+Bptt 70
+Training lm from random weights
+epoch     train_loss  valid_loss  accuracy
+1         4.131590    4.123428    0.460519
+2         4.073408    4.077561    0.461808
+3         4.038208    4.056053    0.464489
+4         4.007055    4.012294    0.469722
+5         3.992992    3.977304    0.473496
+6         3.903659    3.934043    0.480102
+7         3.897762    3.894066    0.484782
+8         3.877661    3.854888    0.492338
+9         3.831059    3.829723    0.497970
+10        3.810376    3.823137    0.499966
+Total time: 18:44:08
+data/wiki/de-100/models/vf60k
+Saving info data/wiki/de-100/models/vf60k/qrnn_nl4.m/info.json
+```
+
+```bash
+python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path  data/wiki/de-100/models/vf60k/qrnn_nl4.m  --lang=${LANG} --name nl4 - train 20 --bs 18 --num-cls-epochs=4 --lr_sched=1cycle --label-smoothing-eps=0.1
+Max vocab: 60000
+Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/vf60k
+Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/vf60k/qrnn_nl4.m
+Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/de.dev.csv
+Data lm, trn: 13500, val: 1500
+Data cls, trn: 1000, val: 1000
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 39171
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '.', 'der', ',', 'die', ')', '(', 'in', 'und', 'auf', 'von', 'den', 'im']
+Training args:  {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps:  {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
+/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning:
+
+                               !! WARNING !!
+
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+Your compiler (c++) may be ABI-incompatible with PyTorch!
+Please use a compiler that is ABI-compatible with GCC 4.9 and above.
+See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html.
+
+See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6
+for instructions on how to install GCC 4.9 or higher.
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+                              !! WARNING !!
+
+  warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler))
+Loading pretrained model
+Unknown tokens 20300, first 100: ['"', 'vh', 'ös', 'brs', '&', 'geg', 'lpo', 'vormonat', 'fgc', 'mesz', 'waigel', 'tcs', 'bund-future', 'ajs', 'brn', 'dih', 'analysten', 'mrd', 'rpk', 'emu', 'notierten', 'feb', 'aktienmarkt', 'dor', 'rentenmarkt', 'basispunkte', 'müßten', 'gewinnmitnahmen', 'aktienbörse', 'jelzin', 'rußland', 'volkswirte', 'fls', 'steuerreform', 'kontrakte', 'kps', 'mge', 'vortagesschluß', 'umsätzen', 'prozent.', 'dow-jones-index', 'reingewinn', 'notierungen', "\\'", 'gesamtmarkt', 'akr', 'kjf', '49-69-7565', 'abl', 'hoh', 'finanzdienst', 'atx', 'feinunze', 'zinserhöhung', 'zugelegt', 'verbraucherpreise', 'ticks', 'kursgewinne', 'ker', 'rlb', 'smi', 'vorbörslich', 'dst', 'mkl', 'ale', 'kontrakten', 'calls', 'veraenderung', 'gwa', 'gesamtjahr', 'auftragseingang', 'überschuß', 'verlautete', 'eju', 'tms', 'jahresvergleich', 'vorjahreszeitraum', 'werden.', 'betriebsergebnis', 'bobl-future', 'puts', 'fri', '4.50', 'schluß', 'ewu', 'spanne', 'standardwerte', 'jahresüberschuß', 'rechne', 'lire', '49-69-756525', '16.00', 'peh', 'hmh', 'dtb', 'tagesgeld', 'us-notenbank', 'corp', 'vorstandschef', 'greenspan']
+Bptt 70
+Training lm from:  [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/de-100/models/vf60k/qrnn_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/de-100/models/vf60k/qrnn_nl4.m/../itos')]
+epoch     train_loss  valid_loss  accuracy
+1         4.724948    4.178421    0.449862
+Total time: 02:19
+epoch     train_loss  valid_loss  accuracy
+1         4.312489    4.049916    0.466992
+2         4.197414    3.919862    0.488602
+3         4.000882    3.793147    0.510018
+4         3.960565    3.691311    0.524944
+5         3.841827    3.590782    0.539775
+6         3.756638    3.515933    0.551585
+7         3.738561    3.439131    0.563536
+8         3.623295    3.371250    0.575563
+9         3.585063    3.307532    0.586810
+10        3.523384    3.256143    0.596964
+11        3.484239    3.195987    0.610036
+12        3.439287    3.140971    0.622494
+13        3.385262    3.087693    0.634652
+14        3.308803    3.050615    0.644066
+15        3.242234    2.999897    0.656247
+16        3.229038    2.966996    0.664862
+17        3.203192    2.946324    0.670038
+18        3.169675    2.930080    0.674204
+19        3.140696    2.920569    0.676475
+20        3.207376    2.919055    0.676769
+Total time: 1:00:09
+/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/vf60k
+Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/vf60k/qrnn_nl4.m/info.json
+Single training schedule
+epoch     train_loss  valid_loss  accuracy
+1         0.779320    0.638282    0.925000
+2         0.653313    0.592119    0.940000
+3         0.569095    0.577936    0.939000
+4         0.519593    0.568577    0.947000
+Total time: 00:50
+Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1/models/vf60k/qrnn_nl4.m
+Loss and accuracy using (cls_best): [0.19424936, tensor(0.9528)]
+0.19424936175346375
+0.952750027179718
+```
+
 ## SP30k LSTM nl 4
 ### LM
 ```
diff --git a/results/logs/mldoc/label_smoothing.md b/results/logs/mldoc/label_smoothing.md
new file mode 100644
index 0000000..b7ba9ca
--- /dev/null
+++ b/results/logs/mldoc/label_smoothing.md
@@ -0,0 +1,39 @@
+# Multifit Best results after label smoothing
+ 
+|     |  de-1 |  en-1 |  es-1 |  fr-1 |  it-1 |  ja-1 |  ru-1 |  zh-1|
+|-----|-------|-------|-------|-------|-------|-------|-------|------|
+|best | 95.90 | 95.17 | 96.07 | 94.75 | 90.25 | 90.03 | 87.65 | 92.52|
+|max  | 95.90 | 95.55 | 96.07 | 94.75 | 90.38 | 90.03 | 87.65 | 92.52|
+|avg  | 95.77 | 95.27 | 95.92 | 94.75 | 90.24 | 89.89 | 87.28 | 92.31|
+ 
+ 
+## Log
+```
+python -m ulmfit eval --glob="mldoc/ru-1/models/sp15k/qrnn_nl4.m" --lr_sched=1cycle --bs=18 --num-cls-epochs=8 --name "nl4_tls4" --label-smoothing-eps=0.1
+
+                                            name  tst_accuracy  tst_loss  val_accuracy  val_loss
+0    data/mldoc/de-1/models/sp15k/qrnn_nl4_tls.m       0.95850  0.254842         0.946  0.320358
+1   data/mldoc/de-1/models/sp15k/qrnn_nl4_tls2.m       0.95900  0.245983         0.947  0.303949
+2   data/mldoc/de-1/models/sp15k/qrnn_nl4_tls3.m       0.95550  0.270527         0.938  0.323216
+3    data/mldoc/en-1/models/sp15k/qrnn_nl4_tls.m       0.95550  0.246017         0.959  0.237861
+4   data/mldoc/en-1/models/sp15k/qrnn_nl4_tls2.m       0.95075  0.258219         0.959  0.235698
+5   data/mldoc/en-1/models/sp15k/qrnn_nl4_tls3.m       0.95175  0.249414         0.960  0.245007
+6    data/mldoc/es-1/models/sp15k/qrnn_nl4_tls.m       0.95875  0.258491         0.961  0.255865
+7   data/mldoc/es-1/models/sp15k/qrnn_nl4_tls2.m       0.95825  0.263527         0.959  0.274785
+8   data/mldoc/es-1/models/sp15k/qrnn_nl4_tls3.m       0.96075  0.253370         0.965  0.254268
+9    data/mldoc/fr-1/models/sp15k/qrnn_nl4_tls.m       0.94750  0.277039         0.942  0.295544
+10  data/mldoc/fr-1/models/sp15k/qrnn_nl4_tls2.m       0.94750  0.284394         0.943  0.288495
+11  data/mldoc/fr-1/models/sp15k/qrnn_nl4_tls3.m       0.94750  0.268739         0.938  0.274793
+12   data/mldoc/it-1/models/sp15k/qrnn_nl4_tls.m       0.90100  0.424416         0.899  0.386466
+13  data/mldoc/it-1/models/sp15k/qrnn_nl4_tls2.m       0.90375  0.410442         0.913  0.381761
+14  data/mldoc/it-1/models/sp15k/qrnn_nl4_tls3.m       0.90250  0.416314         0.917  0.378864
+15   data/mldoc/ja-1/models/sp15k/qrnn_nl4_tls.m       0.89850  0.456913         0.887  0.507895
+16  data/mldoc/ja-1/models/sp15k/qrnn_nl4_tls2.m       0.90025  0.426836         0.897  0.469335
+17  data/mldoc/ja-1/models/sp15k/qrnn_nl4_tls3.m       0.89800  0.449715         0.890  0.502422
+18   data/mldoc/ru-1/models/sp15k/qrnn_nl4_tls.m       0.86550  0.571294         0.870  0.548535
+19  data/mldoc/ru-1/models/sp15k/qrnn_nl4_tls2.m       0.87650  0.587116         0.877  0.585862
+20  data/mldoc/ru-1/models/sp15k/qrnn_nl4_tls3.m       0.87625  0.550317         0.866  0.574534
+21   data/mldoc/zh-1/models/sp15k/qrnn_nl4_tls.m       0.92525  0.347967         0.921  0.350878
+22  data/mldoc/zh-1/models/sp15k/qrnn_nl4_tls2.m       0.92175  0.377572         0.917  0.380295
+23  data/mldoc/zh-1/models/sp15k/qrnn_nl4_tls3.m       0.92225  0.350547         0.916  0.362135
+```
\ No newline at end of file
diff --git a/results/logs/mldoc/random-init.md b/results/logs/mldoc/random-init.md
index f5805fd..713f9ec 100644
--- a/results/logs/mldoc/random-init.md
+++ b/results/logs/mldoc/random-init.md
@@ -1,11 +1,36 @@
 
 
 ### MLDoc laser zero shoot 10k
-data/mldoc/de-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m: 0.9052500128746033
-data/mldoc/es-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m: 0.6974999904632568
-data/mldoc/fr-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m: 0.8740000128746033
-data/mldoc/it-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m: 0.7272499799728394
-data/mldoc/zh-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m: 0.8144999742507935
+Loss and accuracy using (cls_best): [0.58419716, tensor(0.8150)] [0.6386394, tensor(0.7850)]
+                                                name  tst_accuracy  tst_loss  val_accuracy  val_loss
+0  data/mldoc/de-10-laser-en1/models/sp15k/qrnn_r...       0.90550  0.407956         0.917  0.378366
+1  data/mldoc/es-10-laser-en1/models/sp15k/qrnn_r...       0.69725  1.371628         0.747  1.040883
+2  data/mldoc/fr-10-laser-en1/models/sp15k/qrnn_r...       0.87350  0.533766         0.882  0.488216
+3  data/mldoc/it-10-laser-en1/models/sp15k/qrnn_r...       0.72750  1.168527         0.804  1.180750
+4  data/mldoc/ja-10-laser-en1/models/sp15k/qrnn_r...       0.67550  1.941633         0.780  0.991396
+5  data/mldoc/ru-10-laser-en1/models/sp15k/qrnn_r...       0.63675  2.139875         0.822  0.765918
+6  data/mldoc/zh-10-laser-en1/models/sp15k/qrnn_r...       0.81500  0.584197         0.785  0.638639
+ds    de-10-laser-  es-10-laser-  fr-10-laser-  it-10-laser-  ja-10-laser-  ru-10-laser-  zh-10-laser-
+best         90.55         69.73         87.35         72.75         67.55         63.67          81.5
+max          90.55         69.73         87.35         72.75         67.55         63.67          81.5
+avg          90.55         69.73         87.35         72.75         67.55         63.67          81.5
+
+```
+
+Starting classifier from random weights
+Single training schedule
+epoch     train_loss  valid_loss  accuracy
+1         1.087360    1.237505    0.515000
+2         0.810132    1.667991    0.545000
+3         0.779427    1.003253    0.679000
+4         0.662206    2.510274    0.800000
+5         0.604669    2.394876    0.718000
+6         0.501023    0.866812    0.810000
+7         0.398415    0.639844    0.818000
+8         0.448071    0.922495    0.822000
+Total time: 11:52
+Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m
+```
 
 ```
 python -m ulmfit eval --glob="mldoc/*-1/models/sp15k/qrnn_nl4.m" --dataset_template='${lang}-10-laser-en1' --name rnd_nl4 --num-cls-epochs=8 --label-smoothing-eps=0.1 --lr_sched=1cycle --bs=18 --random-init=True
@@ -177,6 +202,213 @@ data/mldoc/it-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m: 0.7272499799728394
 data/mldoc/zh-10-laser-en1/models/sp15k/qrnn_rnd_nl4.m: 0.8144999742507935
 ```
 
+
+### MLDoc laser zero shoot 1k
+
+data/mldoc/de-1-laser-en1/models/sp15k/qrnn_nl4-rnd.m: 0.777999997138977
+data/mldoc/es-1-laser-en1/models/sp15k/qrnn_nl4-rnd.m: 0.7049999833106995
+data/mldoc/fr-1-laser-en1/models/sp15k/qrnn_nl4-rnd.m: 0.7565000057220459
+data/mldoc/it-1-laser-en1/models/sp15k/qrnn_nl4-rnd.m: 0.6852499842643738
+data/mldoc/ru-1-laser-en1/models/sp15k/qrnn_nl4-rnd.m: 0.6137499809265137
+data/mldoc/zh-1-laser-en1/models/sp15k/qrnn_nl4-rnd.m: 0.7919999957084656
+```
+python -m ulmfit eval --glob="mldoc/*-1/models/sp15k/qrnn_nl4.m" --dataset_template='${lang}-1-laser-en1' --name nl4-rnd --num-cls-epochs=8 --label-smoothing-eps=0.1 --lr_sched=1cycle --bs=18 --random-init=True
+Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m
+de-1-laser-en1
+Max vocab: 15000
+Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-en1/models/sp15k
+Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-en1/models/sp15k/qrnn_nl4-rnd.m
+Training
+Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-en1/de.dev.csv
+Data lm, trn: 13500, val: 1500
+Data cls, trn: 1000, val: 1000
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 15000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"]
+/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning:
+
+                               !! WARNING !!
+
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+Your compiler (c++) may be ABI-incompatible with PyTorch!
+Please use a compiler that is ABI-compatible with GCC 4.9 and above.
+See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html.
+
+See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6
+for instructions on how to install GCC 4.9 or higher.
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+                              !! WARNING !!
+
+  warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler))
+Starting classifier from random weights
+Single training schedule
+epoch     train_loss  valid_loss  accuracy
+1         1.409097    1.357696    0.412000
+2         1.270668    1.920674    0.259000
+3         1.121176    1.099281    0.539000
+4         1.034688    2.448050    0.263000
+5         0.893729    1.306312    0.560000
+6         0.794611    0.945334    0.742000
+7         0.711580    0.997155    0.703000
+8         0.668383    0.877867    0.784000
+Total time: 02:21
+Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-en1/models/sp15k/qrnn_nl4-rnd.m
+Loss and accuracy using (cls_best): [0.64768696, tensor(0.7780)]
+Processing data/mldoc/en-1/models/sp15k/qrnn_nl4.m
+en-1-laser-en1
+Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m
+es-1-laser-en1
+Max vocab: 15000
+Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en1/models/sp15k
+Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en1/models/sp15k/qrnn_nl4-rnd.m
+Training
+Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en1/es.dev.csv
+Data lm, trn: 13013, val: 1445
+Data cls, trn: 1000, val: 1000
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 15000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que']
+Starting classifier from random weights
+Single training schedule
+epoch     train_loss  valid_loss  accuracy
+1         1.390623    1.330758    0.365000
+2         1.226550    1.615878    0.511000
+3         1.097214    1.439728    0.532000
+4         0.988972    1.093623    0.688000
+5         0.872939    1.278118    0.623000
+6         0.811549    0.894074    0.779000
+7         0.703965    0.821635    0.818000
+8         0.630001    0.782405    0.823000
+Total time: 02:00
+Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en1/models/sp15k/qrnn_nl4-rnd.m
+Loss and accuracy using (cls_best): [0.9085049, tensor(0.7050)]
+Processing data/mldoc/fr-1/models/sp15k/qrnn_nl4.m
+fr-1-laser-en1
+Max vocab: 15000
+Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en1/models/sp15k
+Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en1/models/sp15k/qrnn_nl4-rnd.m
+Training
+Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en1/fr.dev.csv
+Data lm, trn: 13500, val: 1500
+Data cls, trn: 1000, val: 1000
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 15000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', 's', '▁.', "'", '▁la', '▁le', '▁et', '▁l', '▁à']
+Starting classifier from random weights
+Single training schedule
+epoch     train_loss  valid_loss  accuracy
+1         1.331783    1.457629    0.270000
+2         1.206117    1.568209    0.442000
+3         1.085947    1.397149    0.477000
+4         0.965919    1.025710    0.668000
+5         0.854985    0.915386    0.732000
+6         0.784643    0.922064    0.725000
+7         0.691292    0.896307    0.761000
+8         0.641204    0.867914    0.785000
+Total time: 02:22
+Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en1/models/sp15k/qrnn_nl4-rnd.m
+Loss and accuracy using (cls_best): [0.66897815, tensor(0.7565)]
+Processing data/mldoc/it-1/models/sp15k/qrnn_nl4.m
+it-1-laser-en1
+Max vocab: 15000
+Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-en1/models/sp15k
+Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-en1/models/sp15k/qrnn_nl4-rnd.m
+Training
+Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-en1/it.dev.csv
+Data lm, trn: 13500, val: 1500
+Data cls, trn: 1000, val: 1000
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 15000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁di', '▁e', "▁&'", "'", '▁il', '▁la', '▁in', 'e']
+Starting classifier from random weights
+Single training schedule
+epoch     train_loss  valid_loss  accuracy
+1         1.365793    1.365611    0.388000
+2         1.262174    2.729846    0.271000
+3         1.109978    1.754144    0.415000
+4         0.961007    0.922328    0.731000
+5         0.822061    0.961720    0.721000
+6         0.724208    0.979705    0.707000
+7         0.637849    0.910123    0.754000
+8         0.588057    0.915819    0.744000
+Total time: 01:25
+Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-en1/models/sp15k/qrnn_nl4-rnd.m
+Loss and accuracy using (cls_best): [0.9051443, tensor(0.6852)]
+Processing data/mldoc/ja-1/models/sp15k/qrnn_nl4.m
+ja-1-laser-en1
+Processing data/mldoc/ru-1/models/sp15k/qrnn_nl4.m
+ru-1-laser-en1
+Max vocab: 15000
+Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-en1/models/sp15k
+Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-en1/models/sp15k/qrnn_nl4-rnd.m
+Training
+Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-en1/ru.dev.csv
+Data lm, trn: 9195, val: 1021
+Data cls, trn: 1000, val: 1000
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 15000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', '▁на', 'х']
+Starting classifier from random weights
+Single training schedule
+epoch     train_loss  valid_loss  accuracy
+1         1.381033    1.389893    0.231000
+2         1.327294    1.474365    0.396000
+3         1.190571    2.075783    0.501000
+4         1.070966    1.025509    0.611000
+5         0.950332    0.956073    0.718000
+6         0.812717    1.165698    0.706000
+7         0.737636    0.924283    0.778000
+8         0.697346    0.947041    0.779000
+Total time: 03:16
+Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-en1/models/sp15k/qrnn_nl4-rnd.m
+Loss and accuracy using (cls_best): [1.2389272, tensor(0.6137)]
+Processing data/mldoc/zh-1/models/sp15k/qrnn_nl4.m
+zh-1-laser-en1
+Max vocab: 15000
+Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-en1/models/sp15k
+Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-en1/models/sp15k/qrnn_nl4-rnd.m
+Training
+Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-en1/zh.dev.csv
+Data lm, trn: 13500, val: 1500
+Data cls, trn: 1000, val: 1000
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 15000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有']
+Starting classifier from random weights
+Single training schedule
+epoch     train_loss  valid_loss  accuracy
+1         1.382525    1.435203    0.328000
+2         1.171075    3.494349    0.342000
+3         1.044511    1.895226    0.475000
+4         0.954990    1.675222    0.473000
+5         0.880456    0.954913    0.747000
+6         0.808870    1.084140    0.669000
+7         0.762522    0.920228    0.770000
+8         0.710014    0.894134    0.769000
+Total time: 02:22
+Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-en1/models/sp15k/qrnn_nl4-rnd.m
+Loss and accuracy using (cls_best): [0.6264392, tensor(0.7920)]
+OrderedDict([('data/mldoc/de-1-laser-en1/models/sp15k/qrnn_nl4-rnd.m',
+              0.777999997138977),
+             ('data/mldoc/es-1-laser-en1/models/sp15k/qrnn_nl4-rnd.m',
+              0.7049999833106995),
+             ('data/mldoc/fr-1-laser-en1/models/sp15k/qrnn_nl4-rnd.m',
+              0.7565000057220459),
+             ('data/mldoc/it-1-laser-en1/models/sp15k/qrnn_nl4-rnd.m',
+              0.6852499842643738),
+             ('data/mldoc/ru-1-laser-en1/models/sp15k/qrnn_nl4-rnd.m',
+              0.6137499809265137),
+             ('data/mldoc/zh-1-laser-en1/models/sp15k/qrnn_nl4-rnd.m',
+              0.7919999957084656)])
+data/mldoc/de-1-laser-en1/models/sp15k/qrnn_nl4-rnd.m: 0.777999997138977
+data/mldoc/es-1-laser-en1/models/sp15k/qrnn_nl4-rnd.m: 0.7049999833106995
+data/mldoc/fr-1-laser-en1/models/sp15k/qrnn_nl4-rnd.m: 0.7565000057220459
+data/mldoc/it-1-laser-en1/models/sp15k/qrnn_nl4-rnd.m: 0.6852499842643738
+data/mldoc/ru-1-laser-en1/models/sp15k/qrnn_nl4-rnd.m: 0.6137499809265137
+data/mldoc/zh-1-laser-en1/models/sp15k/qrnn_nl4-rnd.m: 0.7919999957084656
+```
+
 ### MLDoc Classification on 1k
 
 data/mldoc/de-1/models/sp15k/qrnn_rnd-nl4.m: 0.9024999737739563
diff --git a/results/logs/pseudo-labeling.md b/results/logs/pseudo-labeling.md
new file mode 100644
index 0000000..750109e
--- /dev/null
+++ b/results/logs/pseudo-labeling.md
@@ -0,0 +1,174 @@
+## 100 ex. pseudo labeling bootstrapping
+
+
+## Laser pseudo labeling bootstrapping
+```
+Processing data/mldoc/de-1-laser-en1/models/sp15k/qrnn_nl4.m
+Max vocab: 15000
+Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-en1/models/sp15k
+Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-en1/models/sp15k/qrnn_nl4.m
+Generating pseduolabels data/mldoc/de-1-laser-en1-ps
+Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-en1/de.dev.csv
+Data lm, trn: 13500, val: 1500
+Data cls, trn: 1000, val: 1000
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 15000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"]
+/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning: 
+
+                               !! WARNING !!
+
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+Your compiler (c++) may be ABI-incompatible with PyTorch!
+Please use a compiler that is ABI-compatible with GCC 4.9 and above.
+See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html.
+
+See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6
+for instructions on how to install GCC 4.9 or higher.
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+                              !! WARNING !!
+
+  warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler))
+Generating train dataset of size 1000, the accuracy is 0.997
+   0                                                  1  preds
+0  3  Tokio (Reuter) - Der Dollar ist am Donnerstag ...      3
+1  3  Kairo (Reuter) - Die ägyptische Zentralbank se...      3
+2  2  Bonn (Reuter) - Wegen einer Bombendrohung ist ...      2
+3  0  Berlin (Reuter) - Die Bahn AG will mit Hilfe p...      0
+4  3  08.15 Uhr MEZ - Deutsche Aktien nach den Rekor...      3
+Generating dev dataset of size 1000, the accuracy is 0.91
+   0                                                  1  preds
+0  1  New York (Reuter) - Das Vertrauen der US-Verbr...      1
+1  2  Tokio (Reuter) - Russische Patrouillenboote ha...      2
+2  2  Paris (Reuter) - Bei der Volksabstimmung in Al...      2
+3  2  Belgrad (Reuter) - Die serbische Polizei hat n...      2
+4  0  München (Reuter) - Der Stuttgarter Bosch-Konze...      0
+Processing data/mldoc/es-1-laser-en1/models/sp15k/qrnn_nl4.m
+Max vocab: 15000
+Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en1/models/sp15k
+Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en1/models/sp15k/qrnn_nl4.m
+Generating pseduolabels data/mldoc/es-1-laser-en1-ps
+Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en1/es.dev.csv
+Data lm, trn: 13013, val: 1445
+Data cls, trn: 1000, val: 1000
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 15000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que']
+Generating train dataset of size 1000, the accuracy is 0.988
+   0                                                  1  preds
+0  3  LONDRES, 5 sep (Reuter) - El dólar se mantenía...      3
+1  2  MADRID, 30 dic (Reuter) - La Generalitat de Va...      2
+2  3  PARIS, 30 jun (Reuter) - La Bolsa de París neg...      3
+3  0  MADRID, 23 dic (Reuter) - La agencia de valore...      0
+4  0  MADRID, 4 Feb (Reuter) - El Banco Bilbao Vizca...      0
+Generating dev dataset of size 1000, the accuracy is 0.879
+   0                                                  1  preds
+0  0  NUEVA YORK, 11 abr (Reuter) - MCI Communicatio...      0
+1  3  FRANCFORT, 17 jun (Reuter) - La Bolsa de Franc...      3
+2  2  BONN, 3 jun (Reuter) - Un destacado miembro de...      1
+3  2  LONDRES, 3 sep (Reuter) - El secretario de Def...      2
+4  3  MADRID, 3 oct (Reuter) - Las acciones de Pryca...      3
+Processing data/mldoc/fr-1-laser-en1/models/sp15k/qrnn_nl4.m
+Max vocab: 15000
+Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en1/models/sp15k
+Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en1/models/sp15k/qrnn_nl4.m
+Generating pseduolabels data/mldoc/fr-1-laser-en1-ps
+Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en1/fr.dev.csv
+Data lm, trn: 13500, val: 1500
+Data cls, trn: 1000, val: 1000
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 15000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', 's', '▁.', "'", '▁la', '▁le', '▁et', '▁l', '▁à']
+Generating train dataset of size 1000, the accuracy is 0.993
+   0                                                  1  preds
+0  2  WASHINGTON, 13 septembre, Reuter - Les Etats-U...      2
+1  1  PARIS, 10 juillet, Reuter - L'audit des financ...      1
+2  2  MOSCOU, 29 mai, Reuter - Après l'accord interv...      2
+3  2  PARIS, 1er octobre, Reuter - Le groupe communi...      2
+4  0  LONDRES, 3 juin, Reuter - National Grid Group ...      0
+Generating dev dataset of size 1000, the accuracy is 0.887
+   0                                                  1  preds
+0  1  PARIS, 30 décembre, Reuter - Zodiac . Chiffre ...      0
+1  0  AJACCIO, 11 décembre, Reuter - Une charge de 7...      2
+2  0  BRUXELLES, 26 décembre, Reuter - 1997 s'annonc...      0
+3  0  PARIS, 26 septembre, Reuter - Alcatel Alsthom ...      0
+4  1  NEW YORK, 25 octobre, Reuter - La hausse plus ...      1
+Processing data/mldoc/it-1-laser-en1/models/sp15k/qrnn_nl4.m
+Max vocab: 15000
+Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-en1/models/sp15k
+Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-en1/models/sp15k/qrnn_nl4.m
+Generating pseduolabels data/mldoc/it-1-laser-en1-ps
+Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-en1/it.dev.csv
+Data lm, trn: 13500, val: 1500
+Data cls, trn: 1000, val: 1000
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 15000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁di', '▁e', "▁&'", "'", '▁il', '▁la', '▁in', 'e']
+Generating train dataset of size 1000, the accuracy is 0.987
+   0                                                  1  preds
+0  3  MILANO, 6 nov (Reuter) - La lira recupera ai p...      3
+1  1  MILANO, 20 giugno (Reuter) - Lo stacco dividen...      1
+2  3  MILANO, 20 set (Reuter) - Olivetti entra nel t...      3
+3  1  LONDRA, 2 aprile (Reuter) - L'aggregato moneta...      1
+4  3  Oro Londra fix ore 10,30 - 4 nov - $378,65. (c...      3
+Generating dev dataset of size 1000, the accuracy is 0.819
+   0                                                  1  preds
+0  0  L'istituto prevede un aumento dell'utile opera...      0
+1  1  FRANCOFORTE, 18 dic (Reuter) - La Bundesbank a...      1
+2  1  TOKIO, 28 agosto (Reuter) - Il ministro delle ...      1
+3  1  ROMA, 23 luglio (Reuter) - Il presidente del C...      1
+4  1  MONACO, 19 marzo (Reuter) - Il ministro delle ...      1
+Processing data/mldoc/ru-1-laser-en1/models/sp15k/qrnn_nl4.m
+Max vocab: 15000
+Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-en1/models/sp15k
+Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-en1/models/sp15k/qrnn_nl4.m
+Generating pseduolabels data/mldoc/ru-1-laser-en1-ps
+Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-en1/ru.dev.csv
+Data lm, trn: 9195, val: 1021
+Data cls, trn: 1000, val: 1000
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 15000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', '▁на', 'х']
+Generating train dataset of size 1000, the accuracy is 0.996
+   0                                                  1  preds
+0  0  КИЕВ, 20 июн (Рейтер) - Нацбанк Украины планир...      0
+1  3  МИНСК, 13 фев (Рейтер) - Курс белорусского руб...      3
+2  0  САНКТ-ПЕТЕРБУРГ, 25 авг (Рейтер) - Астробанк (...      0
+3  0  MOSCOW, Feb 7 (Reuter) - U.S. plane-maker Boei...      0
+4  2  В данном обзоре казахстанской прессы приводитс...      2
+Generating dev dataset of size 1000, the accuracy is 0.837
+   0                                                  1  preds
+0  0  ТБИЛИСИ, 25 мар (Рейтер) - Партнерский Фонд, с...      0
+1  3  МОСКВА, 3 ноя (Рейтер) - Казахстанская Межбанк...      3
+2  1  КИЕВ, 25 июл (Рейтер) - Нацбанк Украины рассмо...      1
+3  0  МОСКВА, 2 дек (Рейтер) - АО Уралсвязьинформ пр...      0
+4  2  В данном обзоре киргизской прессы приводится к...      2
+Processing data/mldoc/zh-1-laser-en1/models/sp15k/qrnn_nl4.m
+Max vocab: 15000
+Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-en1/models/sp15k
+Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-en1/models/sp15k/qrnn_nl4.m
+Generating pseduolabels data/mldoc/zh-1-laser-en1-ps
+Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-en1/zh.dev.csv
+Data lm, trn: 13500, val: 1500
+Data cls, trn: 1000, val: 1000
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 15000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有']
+Generating train dataset of size 1000, the accuracy is 0.992
+   0                                                  1  preds
+0  1  〔路透社紐約10日電〕　　芝加哥聯邦準備銀行總裁墨斯克週四表示,他預期1997年國內生產總值...      1
+1  0  〔路透社台北14日電〕台灣合作金庫週四將2週､1個月及2個月內的附條件交易利率全開在5.20...      0
+2  1  〔路透社倫敦6日電〕　　在英國工黨政府賦予央行利率自主權後,英國央行在其新的首次貨幣政策委員...      1
+3  3  〔路透社東京4日電〕　　東京股市週一收盤下跌,但在短暫跌破關鍵支撐19,500點後縮減跌幅....      3
+4  2  美國總統克林頓接受明報訪問時表示,美國是貫徹始終地支持中英''聯合聲明''作為香港未來的基石...      2
+Generating dev dataset of size 1000, the accuracy is 0.817
+   0                                                  1  preds
+0  0  〔路透社台北20日電〕　　台灣塑膠類週一早盤上漲,經紀商表示,主要是因為近期原物料價格上漲及...      0
+1  2  〔路透社華盛頓2日電〕比利時央行總裁弗沛雷茲週三表示,義大利里拉被低估,但美元可望攀升.  ...      1
+2  0  〔路透社吉隆坡29日電〕　　吉隆坡股市周二收市微升.分析師指二線股有散戶吸納,助長市場升勢,...      3
+3  2  〔路透社香港26日電〕　　香港明報周四報導,面對台灣當局的"務實外交",和"台獨"傾向,中國...      2
+4  0  [路透社上海6日電]    據上海証券報周五報導,有關專家就滬市四家上市公司法人股通過拍賣進...      0
+Python 3.7.0 (default, Oct  9 2018, 10:31:47) 
+Type 'copyright', 'credits' or 'license' for more information
+```
\ No newline at end of file
diff --git a/results/logs/ru.md b/results/logs/ru.md
index dc469aa..b0cea3e 100644
--- a/results/logs/ru.md
+++ b/results/logs/ru.md
@@ -1,5 +1,5 @@
 # RU
-## SP15k nl4
+## SP15k nl4 QRNN
 ```
 Training lm from random weights
 epoch     train_loss  valid_loss  accuracy
@@ -22,7 +22,160 @@ python -m ulmfit cls --dataset-path data/mldoc/ru-1  --base-lm-path data/wiki/ru
 
 ```
 
-## SP30k nl4 
+## SP25k qrnn
+### LM
+```bash
+ python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name
+'nl4' --max-vocab 25000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0  --label-smoothing-eps=0.10 --tokenizer='sp
+Max vocab: 25000
+Cache dir: data/wiki/ru-100/models/sp25k
+Model dir: data/wiki/ru-100/models/sp25k/qrnn_nl4.m
+Wiki text was split to 193047 articles
+Wiki text was split to 460 articles
+Data lm, trn: 193047, val: 460
+Size of vocabulary: 25000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', 'х', '▁на']
+Training args:  {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps:  {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
+Training lm from random weights
+epoch     train_loss  valid_loss  accuracy
+1         4.154972    4.198218    0.447508
+2         4.030367    4.159642    0.449420
+3         4.138530    4.146010    0.451526
+4         3.997120    4.097048    0.457177
+5         3.999151    4.036350    0.465117
+6         3.935380    3.955517    0.476446
+7         3.912357    3.875987    0.487591
+8         3.785693    3.789099    0.501560
+9         3.743162    3.725730    0.512294
+10        3.690226    3.706929    0.516769
+Total time: 12:10:03
+data/wiki/ru-100/models/sp25k
+```
+
+```bash
+python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG
+}-100/models/sp25k/qrnn_${NAME}.m  --lang=${LANG} --name ${NAME} - train 20 --bs 18 --num-cls-epochs=4 --lr_sched=1cycle --label-smoothing-eps=0.1
+
+Max vocab: 25000
+Cache dir: /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp25k
+Model dir: /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp25k/qrnn_nl4.m
+Loading validation /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv
+Running tokenization lm...
+Data lm, trn: 9195, val: 1021
+Running tokenization cls...
+Data cls, trn: 1000, val: 1000
+Running tokenization tst...
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 25000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', 'х', '▁на']
+Training args:  {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps:  {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
+Loading pretrained model
+Unknown tokens 0, first 100: []
+Training lm from:  [PosixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp25k/qrnn_nl4.m/lm_best'), PosixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp25k/qrnn_nl4.m/../itos')]
+epoch     train_loss  valid_loss  accuracy
+1         4.626971    3.868075    0.474742
+Total time: 01:58
+epoch     train_loss  valid_loss  accuracy
+1         3.821786    3.625366    0.519506
+2         3.570115    3.379288    0.566803
+3         3.517294    3.179166    0.599955
+4         3.160131    3.028985    0.626484
+5         3.135806    2.923198    0.644557
+6         3.055160    2.840300    0.659376
+7         3.005086    2.770163    0.672080
+8         2.811366    2.708846    0.684065
+9         2.818394    2.658951    0.694358
+10        2.881018    2.605373    0.705269
+11        2.793422    2.560091    0.715893
+12        2.708385    2.516373    0.725908
+13        2.690258    2.471159    0.735673
+14        2.748342    2.436113    0.744533
+15        2.601220    2.394404    0.754131
+16        2.616882    2.372301    0.760451
+17        2.602902    2.349164    0.766014
+18        2.560349    2.336217    0.769222
+19        2.549936    2.332076    0.770150
+20        2.546798    2.331103    0.770472
+Total time: 53:22
+/home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp25k
+Saving info /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp25k/qrnn_nl4.m/info.json
+Single training schedule
+epoch     train_loss  valid_loss  accuracy
+1         1.043533    0.961182    0.731000
+2         0.859086    0.837210    0.824000
+3         0.735276    0.724173    0.871000
+4         0.612012    0.711034    0.857000
+Total time: 01:15
+Saving models at /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp25k/qrnn_nl4.m
+Loss and accuracy using (cls_best): [0.3957597, tensor(0.8720)]
+0.3957597017288208
+0.871999979019165
+```
+
+## VF60k QRNN
+### LM
+
+### MLDoc
+```bash
+Max vocab: 60000
+Cache dir: /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/vf60k
+Model dir: /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/vf60k/qrnn_nl4.m
+Loading validation /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv
+Running tokenization lm...
+Data lm, trn: 9195, val: 1021
+Running tokenization cls...
+Data cls, trn: 1000, val: 1000
+Running tokenization tst...
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 55567
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', ',', '.', '-', 'в', ')', '(', 'на', "&'", 'и', 'по', 'с', 'the']
+Training args:  {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps:  {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
+Loading pretrained model
+Unknown tokens 34364, first 100: ['рейтер', '941', '8520', '095', 'said', '\x7f', 'доллару', 'янв', 'погашение', '3272', 'reuter', 'xd0', 'фев', '509410', 'уставный', 'which', 'percent', 'объективность', 'торгах', 'купона', 'million', 'its', 'июл', '044', 'алма-атинское', 'валютной', 'триллиона', 'межбанковской', 'would', 'авг', 'government', 'котировки', 'балансовая', 'ртс', 'выплата', 'прц', '8832', 'yeltsin', '983', 'средневзвешенная', '961', 'president', 'дек', 'minister', '2264', 'нацбанка', 'цбр', 'июн', 'newsroom', 'ммвб', 'гособлигаций', 'стр.1', 'also', 'foreign', 'офз', 'заявленный', 'шестимесячных', 'дисконтных', '-сказал', 'предыдущему', 'тбилисское', 'размещенный', 'told', 'riga', 'лари', 'стр.2', 'kroons', 'окт', 'сиданко', '--московское', 'adr', 'мосэнерго', 'shares', 'пресс-релизе', 'дилеры', 'триллионов', 'акциям', 'billion', 'демченко', 'тнк', 'litas', 'lats', 'дилеров', '--алма-атинское', 'щелкните', 'tuesday', 'зинец', 'friday', 'умвб', 'thursday', 'онэксим', 'трейдеры', 'nato', 'feb', 'дивиденды', 'former', 'could', 'нацбанк', 'стр.6', 'economic']
+Bptt 70
+Training lm from:  [PosixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/ru-100/models/vf60k/qrnn_nl4.m/lm_best'), PosixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/ru-100/models/vf60k/qrnn_nl4.m/../itos')]
+epoch     train_loss  valid_loss  accuracy
+1         5.637876    4.853484    0.379844
+Total time: 01:28
+epoch     train_loss  valid_loss  accuracy
+1         4.906714    4.683807    0.405109
+2         4.850066    4.490903    0.434562
+3         4.591409    4.284740    0.464436
+4         4.379681    4.103634    0.490118
+5         4.079576    3.954377    0.511206
+6         4.199800    3.811692    0.531036
+7         4.004812    3.694871    0.548372
+8         3.995378    3.584868    0.567285
+9         3.884090    3.499729    0.583162
+10        3.897333    3.416602    0.598120
+11        3.726276    3.338907    0.613920
+12        3.690300    3.263694    0.629643
+13        3.614015    3.192474    0.646335
+14        3.530548    3.136064    0.659729
+15        3.451486    3.100320    0.668686
+16        3.444497    3.058001    0.678824
+17        3.407755    3.024943    0.686764
+18        3.383617    3.008939    0.690451
+19        3.342304    2.999911    0.692378
+20        3.339514    2.998623    0.692671
+Total time: 36:01
+/home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/vf60k
+Saving info /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/vf60k/qrnn_nl4.m/info.json
+Single training schedule
+epoch     train_loss  valid_loss  accuracy
+1         0.946690    0.855268    0.805000
+2         0.808650    0.750561    0.866000
+3         0.701750    0.712251    0.884000
+4         0.596392    0.687266    0.884000
+Total time: 00:44
+Saving models at /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/vf60k/qrnn_nl4.m
+Loss and accuracy using (cls_best): [0.37472174, tensor(0.8802)]
+0.3747217357158661
+0.8802499771118164
+```
+
+
+## SP30k LSTM nl4 
 ### LM
 ```
 python -m ulmfit lm --dataset-path data/wiki/ru-100 --cuda-id=0 --tokenizer='sp' --nl 4 --name 'nl4' --max-vocab 30000 --lang ru --qrnn=False - train 10 --bs=50 --drop_mult=0
diff --git a/results/logs/ru/bptt.md b/results/logs/ru/bptt.md
new file mode 100644
index 0000000..e42ce87
--- /dev/null
+++ b/results/logs/ru/bptt.md
@@ -0,0 +1,132 @@
+## bptt140
+### CLS
+```
+LANG=ru
+python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name 'nl4-bptt140' --max-vocab 15000 --lang ${LANG} --qrnn=True - train 14 --bs=50 --drop_mult=0  --label-smoothing-eps=0.1
+
+Max vocab: 15000
+Cache dir: data/wiki/ru-100/models/sp15k
+Model dir: data/wiki/ru-100/models/sp15k/qrnn_nl4-bptt140.m
+Wiki text was split to 193047 articles
+Wiki text was split to 460 articles
+Data lm, trn: 193047, val: 460
+Size of vocabulary: 15000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', '▁на', 'х']
+Training args:  {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps:  {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
+/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning:
+
+                               !! WARNING !!
+
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+Your compiler (c++) may be ABI-incompatible with PyTorch!
+Please use a compiler that is ABI-compatible with GCC 4.9 and above.
+See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html.
+
+See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6
+for instructions on how to install GCC 4.9 or higher.
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+                              !! WARNING !!
+
+  warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler))
+Bptt 70
+Training lm from random weights
+epoch     train_loss  valid_loss  accuracy
+1         4.022324    4.046203    0.451453
+2         3.840025    3.935647    0.462081
+3         3.873172    3.940451    0.459741
+4         3.850415    3.918466    0.462763
+5         3.814188    3.898976    0.465359
+6         3.771836    3.857443    0.472302
+7         3.761032    3.801748    0.479811
+8         3.712323    3.755207    0.486181
+9         3.706044    3.707724    0.493604
+10        3.693287    3.650429    0.502407
+11        3.563701    3.588871    0.513251
+12        3.477192    3.538018    0.522175
+13        3.486541    3.504327    0.528571
+14        3.484132    3.495028    0.530480
+Total time: 19:53:42
+data/wiki/ru-100/models/sp15k
+Saving info data/wiki/ru-100/models/sp15k/qrnn_nl4-bptt140.m/info.json
+```
+
+### MLDoc
+```
+python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path  data/wiki/${LANG}-100/models/sp15k/qrnn_nl4-bptt140.m  --lang=${LANG} --name nl4-bptt140 --bptt=140 - train 20 --bs 18 --num-cls-epochs=8 --lr_sched=1cycle --label-smoothing-eps=0.1
+Max vocab: 15000
+Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k
+Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4-bptt140.m
+Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv
+Running tokenization lm140...
+Data lm140, trn: 9195, val: 1021
+Data cls, trn: 1000, val: 1000
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 15000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', '▁на', 'х']
+Training args:  {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps:  {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
+/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning:
+
+                               !! WARNING !!
+
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+Your compiler (c++) may be ABI-incompatible with PyTorch!
+Please use a compiler that is ABI-compatible with GCC 4.9 and above.
+See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html.
+
+See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6
+for instructions on how to install GCC 4.9 or higher.
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+                              !! WARNING !!
+
+  warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler))
+Loading pretrained model
+Unknown tokens 0, first 100: []
+Bptt 140
+Training lm from:  [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp15k/qrnn_nl4-bptt140.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp15k/qrnn_nl4-bptt140.m/../itos')]
+epoch     train_loss  valid_loss  accuracy
+1         3.855653    2.852676    0.452966
+Total time: 01:56
+epoch     train_loss  valid_loss  accuracy
+1         3.052491    2.572216    0.504417
+2         2.565436    2.252638    0.557341
+3         2.238792    1.980807    0.599827
+4         1.990266    1.784574    0.629615
+5         1.851867    1.647570    0.651466
+6         1.800950    1.539561    0.668753
+7         1.692110    1.447140    0.684268
+8         1.546868    1.380541    0.696082
+9         1.618451    1.312476    0.708090
+10        1.478336    1.255234    0.718722
+11        1.477739    1.197032    0.729453
+12        1.418238    1.151929    0.738932
+13        1.384237    1.103246    0.748681
+14        1.245625    1.061356    0.757009
+15        1.289399    1.028937    0.763857
+16        1.280893    1.006447    0.768844
+17        1.268177    0.985106    0.773329
+18        1.251713    0.975138    0.775565
+19        1.288352    0.968812    0.776884
+20        1.164147    0.967133    0.777174
+Total time: 52:46
+/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k
+Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4-bptt140.m/info.json
+Single training schedule
+epoch     train_loss  valid_loss  accuracy
+1         0.905054    0.572781    0.811000
+2         0.747270    0.606469    0.806000
+3         0.644590    0.682804    0.810000
+4         0.457427    0.605931    0.863000
+5         0.351969    0.652187    0.842000
+6         0.286099    0.589351    0.860000
+7         0.218377    0.622760    0.857000
+8         0.185043    0.597372    0.860000
+Total time: 03:05
+Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/qrnn_nl4-bptt140.m
+Loss and accuracy using (cls_best): [0.47860995, tensor(0.8737)] [0.48851612, tensor(0.8600)]
+val_loss:     0.48851612
+val_accuracy: 0.8600000143051147
+tst_loss:     0.47860995
+tst_accuracy: 0.8737499713897705
+```
\ No newline at end of file
diff --git a/results/logs/ru/nowiki-sp16k.md b/results/logs/ru/nowiki-sp16k.md
new file mode 100644
index 0000000..e39abe7
--- /dev/null
+++ b/results/logs/ru/nowiki-sp16k.md
@@ -0,0 +1,115 @@
+```
+python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --tokenizer sp --max-vocab 16000 --qrnn True --lang=${LANG} --name nl4 - train 20 --bs 18 --num-cls-epochs=4 --lr_sched=1cycle --label-smoothing-eps=0.1
+Max vocab: 16000
+Cache dir: data/mldoc/ru-1/models/sp16k
+Model dir: data/mldoc/ru-1/models/sp16k/qrnn_nl4.m
+Loading validation data/mldoc/ru-1/ru.dev.csv
+/sentencepiece/src/sentencepiece_trainer.cc(185) LOG(INFO) Running command: --input=data/mldoc/ru-1/models/sp16k/all_text.txt --character_coverage=0.99 --unk_id=8 --pad_id=-1 --bos_id=-1 --eos_id=-1 --max_sentence_length=20480 --input_sentence_size=10000000 --user_defined_symbols=xxunk,xxpad,xxbos,xxfld,xxmaj,xxup,xxrep,xxwrep --model_prefix=data/mldoc/ru-1/models/sp16k/spm --vocab_size=16000 --model_type=unigram
+/sentencepiece/src/unigram_model_trainer.cc(481) LOG(INFO) Starts training with :
+input: "data/mldoc/ru-1/models/sp16k/all_text.txt"
+model_prefix: "data/mldoc/ru-1/models/sp16k/spm"
+model_type: UNIGRAM
+vocab_size: 16000
+character_coverage: 0.99
+input_sentence_size: 10000000
+max_sentence_length: 20480
+user_defined_symbols: "xxunk"
+user_defined_symbols: "xxpad"
+user_defined_symbols: "xxbos"
+user_defined_symbols: "xxfld"
+user_defined_symbols: "xxmaj"
+user_defined_symbols: "xxup"
+user_defined_symbols: "xxrep"
+user_defined_symbols: "xxwrep"
+unk_id: 8
+bos_id: -1
+eos_id: -1
+pad_id: -1
+
+/sentencepiece/src/trainer_interface.cc(183) LOG(INFO) Loading corpus: data/mldoc/ru-1/models/sp16k/all_text.txt
+/sentencepiece/src/trainer_interface.cc(216) LOG(INFO) Loading: ▁	▁киев▁,▁20▁июн▁(▁	▁рейтер▁)▁-▁	▁нацбанк▁	▁украины▁планирует▁постепенно▁отказаться▁от▁кредитных▁аукционов▁и▁использовать▁для▁рефинансирования▁банков▁только▁операции▁репо▁и▁ломбардное▁кредитование▁,▁сказала▁директор▁департамента▁	▁нбу▁	▁наталия▁	▁гребеник▁.▁&'▁	▁от▁кредитных▁аукционов▁	▁нбу▁будет▁в▁дальнейшем▁отказываться▁,▁используя▁репо▁и▁ломбардное▁кредитование▁&'▁,▁-▁сказала▁директор▁кредитно-эмиссионного▁департамента▁.▁	▁по▁ее▁словам▁,▁в▁настоящее▁время▁	▁нацбанк▁использует▁все▁три▁канала▁рефинансирования▁банков▁.▁	▁удельный▁вес▁рефинансирования▁через▁операции▁репо▁составляет▁50▁процентов▁,▁через▁кредитные▁аукционы▁и▁ломбардное▁кредитование▁под▁залог▁гособлигаций▁по▁25▁процентов▁.▁в▁частности▁,▁с▁начала▁года▁были▁проведены▁четыре▁кредитных▁аукционах▁на▁которых▁банкам▁было▁продано▁560▁миллионов▁гривен▁кредитов▁,▁сказала▁	▁гребеник▁.▁	▁по▁ее▁словам▁,▁средняя▁ставка▁продажи▁ресурсов▁на▁кредитных▁аукционах▁на▁3-4▁процента▁превышала▁ставку▁рефинансирования▁,▁действующую▁на▁день▁проведения▁аукциона▁.▁	▁действующая▁в▁настоящее▁время▁ставка▁рефинансирования▁	▁нбу▁составляет▁21▁процент▁годовых▁,▁ломбардная▁ставка▁-▁31▁процент▁.▁	▁по▁соглашениям▁репо▁ставка▁может▁быть▁ниже▁ставки▁рефинансирования▁,▁но▁не▁более▁,▁чем▁на▁5▁процентных▁пунктов▁,▁сказал▁	▁гребеник▁.▁	▁по▁ее▁словам▁,▁в▁будущем▁	▁нбу▁также▁планирует▁освоить▁инструмент▁векселей▁при▁рефинансировании▁коммерческих▁банков▁.▁&'▁	▁мы▁будем▁переходить▁к▁использованию▁векселей▁как▁залога▁,▁что▁даст▁нам▁возможность▁более▁четко▁определять▁стоимость▁денежных▁ресурсов▁&'▁,▁-▁сказала▁	▁гребеник▁.▁-▁	▁наталия▁	▁зинец▁,▁	▁киевское▁бюро▁,▁(▁044▁)▁244▁9150▁.▁(▁c▁)▁	▁reuters▁	▁limited▁1997▁.	size=0
+/sentencepiece/src/trainer_interface.cc(200) LOG(INFO) Too long lines (>=20480 bytes (it can be changed with --max_sentence_length flag). Skipped.
+/sentencepiece/src/trainer_interface.cc(200) LOG(INFO) Too long lines (>=20480 bytes (it can be changed with --max_sentence_length flag). Skipped.
+/sentencepiece/src/trainer_interface.cc(240) LOG(INFO) Loaded 998 sentences
+/sentencepiece/src/trainer_interface.cc(241) LOG(INFO) Loaded 0 test sentences
+/sentencepiece/src/trainer_interface.cc(265) LOG(INFO) all chars count=1565524
+/sentencepiece/src/trainer_interface.cc(273) LOG(INFO) Done: 99.1426% characters are covered.
+/sentencepiece/src/trainer_interface.cc(283) LOG(INFO) Alphabet size=68
+/sentencepiece/src/trainer_interface.cc(284) LOG(INFO) Final character coverage=0.991426
+/sentencepiece/src/trainer_interface.cc(316) LOG(INFO) Done! 998 sentences are loaded
+/sentencepiece/src/unigram_model_trainer.cc(127) LOG(INFO) Using 998 sentences for making seed sentencepieces
+/sentencepiece/src/unigram_model_trainer.cc(155) LOG(INFO) Making suffix array...
+/sentencepiece/src/unigram_model_trainer.cc(159) LOG(INFO) Extracting frequent sub strings...
+/sentencepiece/src/unigram_model_trainer.cc(210) LOG(INFO) Initialized 67755 seed sentencepieces
+/sentencepiece/src/trainer_interface.cc(322) LOG(INFO) Tokenizing input sentences with whitespace: 998
+/sentencepiece/src/trainer_interface.cc(331) LOG(INFO) Done! 31975
+/sentencepiece/src/unigram_model_trainer.cc(502) LOG(INFO) Using 31975 sentences for EM training
+/sentencepiece/src/unigram_model_trainer.cc(518) LOG(INFO) EM sub_iter=0 size=22877 obj=16.6184 num_tokens=70560 num_tokens/piece=3.08432
+/sentencepiece/src/unigram_model_trainer.cc(518) LOG(INFO) EM sub_iter=1 size=19595 obj=14.256 num_tokens=71915 num_tokens/piece=3.67007
+/sentencepiece/src/unigram_model_trainer.cc(518) LOG(INFO) EM sub_iter=0 size=17579 obj=14.2013 num_tokens=73054 num_tokens/piece=4.15575
+/sentencepiece/src/unigram_model_trainer.cc(518) LOG(INFO) EM sub_iter=1 size=17469 obj=14.1655 num_tokens=73390 num_tokens/piece=4.20116
+/sentencepiece/src/trainer_interface.cc(387) LOG(INFO) Saving model: data/mldoc/ru-1/models/sp16k/spm.model
+/sentencepiece/src/trainer_interface.cc(411) LOG(INFO) Saving vocabs: data/mldoc/ru-1/models/sp16k/spm.vocab
+Running tokenization lm...
+Data lm, trn: 9195, val: 1021
+Running tokenization cls...
+Data cls, trn: 1000, val: 1000
+Running tokenization tst...
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 16000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', ',', '▁.', 'и', 'е', '▁в', 'й', '▁-', 'а', ')', '(']
+Training args:  {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps:  {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
+/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning:
+
+                               !! WARNING !!
+
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+Your compiler (c++) may be ABI-incompatible with PyTorch!
+Please use a compiler that is ABI-compatible with GCC 4.9 and above.
+See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html.
+
+See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6
+for instructions on how to install GCC 4.9 or higher.
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+                              !! WARNING !!
+
+  warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler))
+Bptt 70
+Training lm from random weights
+epoch     train_loss  valid_loss  accuracy
+1         4.311670    4.217185    0.471835
+2         3.735446    3.595414    0.552916
+3         3.553164    3.354127    0.581794
+4         3.363475    3.259169    0.593828
+5         3.514256    3.261860    0.590224
+6         3.413725    3.223500    0.597156
+7         3.453391    3.182702    0.601941
+8         3.317564    3.131130    0.610511
+9         3.398653    3.092810    0.616117
+10        3.276093    3.037282    0.624851
+11        3.207109    2.980038    0.634575
+12        3.141415    2.928465    0.643130
+13        3.164837    2.878245    0.653095
+14        3.093078    2.823911    0.662821
+15        3.026668    2.770853    0.673216
+16        2.968236    2.723534    0.682577
+17        2.983422    2.690081    0.689747
+18        2.862256    2.666973    0.694282
+19        2.876733    2.656204    0.696821
+20        2.853209    2.654935    0.696994
+Total time: 39:43
+data/mldoc/ru-1/models/sp16k
+Saving info data/mldoc/ru-1/models/sp16k/qrnn_nl4.m/info.json
+Single training schedule
+epoch     train_loss  valid_loss  accuracy
+1         0.993379    0.788644    0.807000
+2         0.832733    0.773031    0.864000
+3         0.706515    0.715565    0.864000
+4         0.618606    0.720445    0.868000
+Total time: 00:56
+Saving models at data/mldoc/ru-1/models/sp16k/qrnn_nl4.m
+Loss and accuracy using (cls_best): [0.39357555, tensor(0.8685)]
+0.3935755491256714
+0.8684999942779541
+```
\ No newline at end of file
diff --git a/results/logs/ru/results.md b/results/logs/ru/results.md
new file mode 100644
index 0000000..1d37e86
--- /dev/null
+++ b/results/logs/ru/results.md
@@ -0,0 +1,13 @@
+\toprule
+RNN type & Vocabluary Size & Tokenization     & Language & MLDoc Accuracy\\
+\midrule
+LSTM 3   & 60k             & moses            & DE       &   94.74      \\
+LSTM 4   & 30k             & sentence piece   & DE       &   95.40      \\
+QRNN 4   & 60k             & moses            & DE       &   95.28     \\
+QRNN 4   & 15k             & sentence piece   & DE       &   96.10      \\
+\midrule
+LSTM 4   & 30k             & sentence piece   & RU       &   87.27    \\
+LSTM 4   & 15k             & sentence piece   & RU       &   86.47    \\
+QRNN 4   & 60k             & moses            & RU       &   87.60    \\
+QRNN 4   & 25k             & sentence piece   & RU       &   87.20    \\
+QRNN 4   & 15k             & sentence piece   & RU       &   87.17    \\
diff --git a/results/logs/ru/sp25.md b/results/logs/ru/sp25.md
new file mode 100644
index 0000000..25742a2
--- /dev/null
+++ b/results/logs/ru/sp25.md
@@ -0,0 +1,28 @@
+```
+% python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name
+'nl4' --max-vocab 25000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0  --label-smoothing-eps=0.10 --tokenizer='sp
+Max vocab: 25000
+Cache dir: data/wiki/ru-100/models/sp25k
+Model dir: data/wiki/ru-100/models/sp25k/qrnn_nl4.m
+Wiki text was split to 193047 articles
+Wiki text was split to 460 articles
+Data lm, trn: 193047, val: 460
+Size of vocabulary: 25000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', 'х', '▁на']
+Training args:  {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps:  {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
+Training lm from random weights
+epoch     train_loss  valid_loss  accuracy
+1         4.154972    4.198218    0.447508
+2         4.030367    4.159642    0.449420
+3         4.138530    4.146010    0.451526
+4         3.997120    4.097048    0.457177
+5         3.999151    4.036350    0.465117
+6         3.935380    3.955517    0.476446
+7         3.912357    3.875987    0.487591
+8         3.785693    3.789099    0.501560
+9         3.743162    3.725730    0.512294
+10        3.690226    3.706929    0.516769
+Total time: 12:10:03
+data/wiki/ru-100/models/sp25k
+Saving info data/wiki/ru-100/models/sp25k/qrnn_nl4.m/info.json
+```
\ No newline at end of file
diff --git a/results/logs/tokenization/average10runs.md b/results/logs/tokenization/average10runs.md
new file mode 100644
index 0000000..acb0112
--- /dev/null
+++ b/results/logs/tokenization/average10runs.md
@@ -0,0 +1,280 @@
+
+TOK=sp15k
+NAME=e8avg
+for LANG in ru fr; do 
+    for i in 1 2 3 4 5 6 7 8 9; do python -m ulmfit eval --glob="mldoc/${LANG}-1/models/${TOK}/qrnn_nl4.m" --name ${NAME}_$i --num-cls-epochs=8 --label-smoothing-eps=0.1 --lr_sched=1cycle --bs=18; done
+done
+TOK=sp15k
+NAME=avg
+for LANG in ru fr; do 
+    for i in 1 2 3 4 5 6 7 8 9; do python -m ulmfit eval --glob="mldoc/${LANG}-1/models/${TOK}/qrnn_nl4.m" --name ${NAME}_$i --num-cls-epochs=4 --label-smoothing-eps=0.1 --lr_sched=1cycle --bs=18; done
+done
+
+NAME=e8avg
+TOK=vf60k
+for LANG in ru fr; do 
+    for i in 1 2 3 4 5 6 7 8 9; do python -m ulmfit eval --glob="mldoc/${LANG}-1/models/${TOK}/qrnn_nl4.m" --name ${NAME}_$i --num-cls-epochs=8 --label-smoothing-eps=0.1 --lr_sched=1cycle --bs=18; done
+done
+NAME=avg
+TOK=vf60k
+for LANG in ru fr; do 
+    for i in 1 2 3 4 5 6 7 8 9; do python -m ulmfit eval --glob="mldoc/${LANG}-1/models/${TOK}/qrnn_nl4.m" --name ${NAME}_$i --num-cls-epochs=4 --label-smoothing-eps=0.1 --lr_sched=1cycle --bs=18; done
+done
+
+
+for TOK in vf60k sp15k; do
+for LANG in ru fr; do 
+    NAME=e8avg 
+    for i in 1 2 3 4 5 6 7 8 9; do python -m ulmfit eval --glob="mldoc/${LANG}-1/models/${TOK}/qrnn_nl4.m" --name ${NAME}_$i --num-cls-epochs=8 --label-smoothing-eps=0.1 --lr_sched=1cycle --bs=18; done
+done
+done
+
+for TOK in vf60k sp15k; do
+for LANG in ru fr; do
+    NAME=avg  
+    for i in 1 2 3 4 5 6 7 8 9; do python -m ulmfit eval --glob="mldoc/${LANG}-1/models/${TOK}/qrnn_nl4.m" --name ${NAME}_$i --num-cls-epochs=4 --label-smoothing-eps=0.1 --lr_sched=1cycle --bs=18; done
+done
+done
+LANG=es
+for i in 1 2 3 4 5 6 7 9; do python -m ulmfit eval --glob="mldoc/${LANG}-1/models/sp15k/qrnn_nl4.m" --name avg_$i --num-cls-epochs=4 --label-smoothing-eps=0.1 --lr_sched=1cycle --bs=18; done
+
+LANG=de
+for i in 1 2 3 4 5 6 7 9; do python -m ulmfit eval --glob="mldoc/${LANG}-1/models/sp15k/qrnn_nl4.m" --name avg_$i --num-cls-epochs=4 --label-smoothing-eps=0.1 --lr_sched=1cycle --bs=18; done
+
+## epoch 8
+vf60k
+ds     de-1   es-1   fr-1   it-1   ru-1 
+best  95.45  96.17  94.77  90.72  87.85
+max   95.63  96.43  95.32  91.05  88.30
+avg   95.42  96.05  95.07  90.59  87.80
+
+sp15k
+ds     de-1   es-1   fr-1   it-1   ru-1
+best  96.17  95.92  94.55  90.45  86.95
+max   96.28  96.03  95.10  90.72  87.45
+avg   96.01  95.72  94.63  90.37  86.95
+
+
+-0--
+0   data/mldoc/fr-1/models/vf60k/qrnn_e8avg_1.m       0.95325  0.211328         0.951  0.211664
+1   data/mldoc/fr-1/models/vf60k/qrnn_e8avg_2.m       0.95075  0.199939         0.947  0.198606
+2   data/mldoc/fr-1/models/vf60k/qrnn_e8avg_3.m       0.95125  0.217569         0.952  0.215529
+3   data/mldoc/fr-1/models/vf60k/qrnn_e8avg_4.m       0.95225  0.208047         0.951  0.203784
+4   data/mldoc/fr-1/models/vf60k/qrnn_e8avg_5.m       0.95025  0.206937         0.946  0.206194
+5   data/mldoc/fr-1/models/vf60k/qrnn_e8avg_6.m       0.95075  0.203967         0.951  0.204809
+6   data/mldoc/fr-1/models/vf60k/qrnn_e8avg_7.m       0.94775  0.211408         0.954  0.201543
+7   data/mldoc/fr-1/models/vf60k/qrnn_e8avg_8.m       0.95075  0.202703         0.952  0.197218
+8   data/mldoc/fr-1/models/vf60k/qrnn_e8avg_9.m       0.94925  0.207698         0.947  0.204896
+9     data/mldoc/ru-1/models/vf60k/qrnn_avg_1.m       0.88300  0.375823         0.876  0.364029
+10    data/mldoc/ru-1/models/vf60k/qrnn_avg_2.m       0.87675  0.386695         0.883  0.356660
+11    data/mldoc/ru-1/models/vf60k/qrnn_avg_3.m       0.87750  0.372321         0.879  0.374368
+12    data/mldoc/ru-1/models/vf60k/qrnn_avg_4.m       0.87400  0.379490         0.875  0.370343
+13    data/mldoc/ru-1/models/vf60k/qrnn_avg_5.m       0.87725  0.380067         0.877  0.367522
+14    data/mldoc/ru-1/models/vf60k/qrnn_avg_6.m       0.87525  0.393280         0.874  0.368825
+15    data/mldoc/ru-1/models/vf60k/qrnn_avg_7.m       0.87900  0.380393         0.882  0.373376
+16    data/mldoc/ru-1/models/vf60k/qrnn_avg_8.m       0.88025  0.376825         0.875  0.375059
+17    data/mldoc/ru-1/models/vf60k/qrnn_avg_9.m       0.88125  0.380887         0.884  0.367705
+18  data/mldoc/ru-1/models/vf60k/qrnn_e8avg_1.m       0.87850  0.385976         0.888  0.385302
+19  data/mldoc/ru-1/models/vf60k/qrnn_e8avg_2.m       0.87600  0.384469         0.879  0.384878
+20  data/mldoc/ru-1/models/vf60k/qrnn_e8avg_4.m       0.87600  0.386223         0.870  0.391646
+21  data/mldoc/ru-1/models/vf60k/qrnn_e8avg_5.m       0.87950  0.385152         0.885  0.371122
+22  data/mldoc/ru-1/models/vf60k/qrnn_e8avg_6.m       0.88175  0.383746         0.875  0.391054
+23  data/mldoc/ru-1/models/vf60k/qrnn_e8avg_7.m       0.87850  0.392120         0.874  0.382000
+24  data/mldoc/ru-1/models/vf60k/qrnn_e8avg_8.m       0.87250  0.394342         0.881  0.378663
+25  data/mldoc/ru-1/models/vf60k/qrnn_e8avg_9.m       0.87950  0.387625         0.881  0.380548
+ds     fr-1   ru-1
+best  94.77  87.85
+max   95.32  88.30
+avg   95.07  87.80
+---
+
+# epoch 4
+## SP15k
+                                         name  tst_accuracy  tst_loss  val_accuracy  val_loss
+0   data/mldoc/de-1/models/sp15k/qrnn_avg_1.m       0.95700  0.142444         0.945  0.206329
+1   data/mldoc/de-1/models/sp15k/qrnn_avg_2.m       0.95975  0.141225         0.955  0.189209
+2   data/mldoc/de-1/models/sp15k/qrnn_avg_3.m       0.95925  0.140172         0.946  0.198159
+3   data/mldoc/de-1/models/sp15k/qrnn_avg_4.m       0.95650  0.145889         0.942  0.193202
+4   data/mldoc/de-1/models/sp15k/qrnn_avg_5.m       0.96000  0.137710         0.953  0.192368
+5   data/mldoc/de-1/models/sp15k/qrnn_avg_6.m       0.95975  0.145318         0.954  0.196413
+6   data/mldoc/de-1/models/sp15k/qrnn_avg_7.m       0.95925  0.141719         0.943  0.199226
+7   data/mldoc/de-1/models/sp15k/qrnn_avg_8.m       0.96100  0.140644         0.949  0.204398
+8   data/mldoc/de-1/models/sp15k/qrnn_avg_9.m       0.96050  0.143330         0.949  0.189472
+9   data/mldoc/es-1/models/sp15k/qrnn_avg_1.m       0.95700  0.149081         0.963  0.162808
+10  data/mldoc/es-1/models/sp15k/qrnn_avg_2.m       0.95875  0.141572         0.963  0.150970
+11  data/mldoc/es-1/models/sp15k/qrnn_avg_3.m       0.95900  0.151387         0.962  0.157047
+12  data/mldoc/es-1/models/sp15k/qrnn_avg_4.m       0.95375  0.165935         0.956  0.182161
+13  data/mldoc/es-1/models/sp15k/qrnn_avg_5.m       0.95850  0.151109         0.960  0.156376
+14  data/mldoc/es-1/models/sp15k/qrnn_avg_6.m       0.95800  0.150724         0.961  0.152761
+15  data/mldoc/es-1/models/sp15k/qrnn_avg_7.m       0.95875  0.142476         0.963  0.151585
+16  data/mldoc/es-1/models/sp15k/qrnn_avg_8.m       0.95525  0.165120         0.957  0.164723
+17  data/mldoc/es-1/models/sp15k/qrnn_avg_9.m       0.95725  0.151323         0.960  0.156123
+18  data/mldoc/it-1/models/sp15k/qrnn_avg_1.m       0.90175  0.312996         0.900  0.297118
+19  data/mldoc/it-1/models/sp15k/qrnn_avg_2.m       0.90250  0.316763         0.903  0.274423
+20  data/mldoc/it-1/models/sp15k/qrnn_avg_3.m       0.89900  0.329157         0.915  0.290098
+21  data/mldoc/it-1/models/sp15k/qrnn_avg_4.m       0.90100  0.322112         0.907  0.285727
+22  data/mldoc/it-1/models/sp15k/qrnn_avg_5.m       0.90100  0.308545         0.910  0.276683
+23  data/mldoc/it-1/models/sp15k/qrnn_avg_6.m       0.90275  0.323594         0.915  0.287004
+24  data/mldoc/it-1/models/sp15k/qrnn_avg_7.m       0.89925  0.295158         0.910  0.269167
+25  data/mldoc/it-1/models/sp15k/qrnn_avg_9.m       0.90325  0.312664         0.908  0.297471
+ds     de-1   es-1   it-1
+best  95.97  95.70  89.90
+max   96.10  95.90  90.32
+avg   95.92  95.74  90.13
+
+## VF60k
+                                         name  tst_accuracy  tst_loss  val_accuracy  val_loss
+0     data/mldoc/de-1/models/vf60k/qrnn_avg.m       0.95250  0.193797         0.946  0.225316
+1   data/mldoc/de-1/models/vf60k/qrnn_avg_1.m       0.95575  0.157327         0.947  0.189885
+2   data/mldoc/de-1/models/vf60k/qrnn_avg_2.m       0.95400  0.174519         0.947  0.201792
+3   data/mldoc/de-1/models/vf60k/qrnn_avg_3.m       0.95325  0.180489         0.947  0.208106
+4   data/mldoc/de-1/models/vf60k/qrnn_avg_4.m       0.95425  0.161056         0.949  0.199169
+5   data/mldoc/de-1/models/vf60k/qrnn_avg_5.m       0.94775  0.182012         0.941  0.210262
+6   data/mldoc/de-1/models/vf60k/qrnn_avg_6.m       0.95375  0.164578         0.947  0.198632
+7   data/mldoc/de-1/models/vf60k/qrnn_avg_7.m       0.95575  0.152596         0.947  0.196844
+8   data/mldoc/de-1/models/vf60k/qrnn_avg_8.m       0.95350  0.167661         0.942  0.203538
+9     data/mldoc/es-1/models/vf60k/qrnn_avg.m       0.95950  0.146121         0.961  0.161852
+10  data/mldoc/es-1/models/vf60k/qrnn_avg_1.m       0.95500  0.154836         0.960  0.176217
+11  data/mldoc/es-1/models/vf60k/qrnn_avg_2.m       0.95850  0.154539         0.961  0.163008
+12  data/mldoc/es-1/models/vf60k/qrnn_avg_3.m       0.96100  0.151916         0.966  0.169869
+13  data/mldoc/es-1/models/vf60k/qrnn_avg_4.m       0.95825  0.144630         0.962  0.144410
+14  data/mldoc/es-1/models/vf60k/qrnn_avg_5.m       0.95675  0.155685         0.960  0.175439
+15  data/mldoc/es-1/models/vf60k/qrnn_avg_6.m       0.95900  0.143995         0.959  0.164156
+16  data/mldoc/es-1/models/vf60k/qrnn_avg_7.m       0.95800  0.144662         0.962  0.162957
+17  data/mldoc/es-1/models/vf60k/qrnn_avg_8.m       0.95850  0.149185         0.962  0.163159
+18    data/mldoc/it-1/models/vf60k/qrnn_avg.m       0.89925  0.320389         0.912  0.272104
+19  data/mldoc/it-1/models/vf60k/qrnn_avg_1.m       0.90525  0.305978         0.920  0.255507
+20  data/mldoc/it-1/models/vf60k/qrnn_avg_2.m       0.90725  0.287647         0.917  0.245568
+21  data/mldoc/it-1/models/vf60k/qrnn_avg_3.m       0.89925  0.313870         0.910  0.271480
+22  data/mldoc/it-1/models/vf60k/qrnn_avg_4.m       0.91125  0.285618         0.915  0.255942
+23  data/mldoc/it-1/models/vf60k/qrnn_avg_5.m       0.91100  0.288841         0.911  0.255724
+24  data/mldoc/it-1/models/vf60k/qrnn_avg_6.m       0.90525  0.287412         0.914  0.253394
+25  data/mldoc/it-1/models/vf60k/qrnn_avg_7.m       0.90000  0.308104         0.910  0.256991
+26  data/mldoc/it-1/models/vf60k/qrnn_avg_8.m       0.90450  0.301262         0.918  0.251368
+ds     de-1   es-1   it-1
+best  95.42  96.10  90.53
+max   95.57  96.10  91.12
+avg   95.34  95.83  90.48
+
+
+
+# IT 
+## VF60k - 9 runs eval 
+for i in 1 2 3 4 5 6 7 8; do python -m ulmfit eval --glob="mldoc/${LANG}-1/models/vf60k/qrnn_nl4.m" --name avg_$i --num-cls-epochs=4 --label-smoothing-eps=0.1 --lr_sched=1cycle --bs=10; done
+python -m ulmfit eval --glob="mldoc/${LANG}-1/models/vf60k/qrnn_nl4*.m" --train=False
+
+                                        name  tst_accuracy  tst_loss  val_accuracy  val_loss
+0    data/mldoc/it-1/models/vf60k/qrnn_nl4.m       0.89925  0.320389         0.912  0.272104
+1  data/mldoc/it-1/models/vf60k/qrnn_nl4_1.m       0.90525  0.305978         0.920  0.255507
+2  data/mldoc/it-1/models/vf60k/qrnn_nl4_2.m       0.90725  0.287647         0.917  0.245568
+3  data/mldoc/it-1/models/vf60k/qrnn_nl4_3.m       0.89925  0.313870         0.910  0.271480
+4  data/mldoc/it-1/models/vf60k/qrnn_nl4_4.m       0.91125  0.285618         0.915  0.255942
+5  data/mldoc/it-1/models/vf60k/qrnn_nl4_5.m       0.91100  0.288841         0.911  0.255724
+6  data/mldoc/it-1/models/vf60k/qrnn_nl4_6.m       0.90525  0.287412         0.914  0.253394
+7  data/mldoc/it-1/models/vf60k/qrnn_nl4_7.m       0.90000  0.308104         0.910  0.256991
+8  data/mldoc/it-1/models/vf60k/qrnn_nl4_8.m       0.90450  0.301262         0.918  0.251368
+ds     it-1
+best  90.53
+max   91.12
+avg   90.48
+
+## sp15k - 9 runs eval
+LANG=it
+for i in 1 2 3 4 5 6 7 9; do python -m ulmfit eval --glob="mldoc/${LANG}-1/models/sp15k/qrnn_nl4.m" --name avg_$i --num-cls-epochs=4 --label-smoothing-eps=0.1 --lr_sched=1cycle --bs=18; done
+python -m ulmfit eval --glob="mldoc/${LANG}-1/models/sp15k/qrnn_nl4_a*.m" --train=False
+                                        name  tst_accuracy  tst_loss  val_accuracy  val_loss
+0  data/mldoc/it-1/models/sp15k/qrnn_avg_1.m       0.90175  0.312996         0.900  0.297118
+1  data/mldoc/it-1/models/sp15k/qrnn_avg_2.m       0.90250  0.316763         0.903  0.274423
+2  data/mldoc/it-1/models/sp15k/qrnn_avg_3.m       0.89900  0.329157         0.915  0.290098
+3  data/mldoc/it-1/models/sp15k/qrnn_avg_4.m       0.90100  0.322112         0.907  0.285727
+4  data/mldoc/it-1/models/sp15k/qrnn_avg_5.m       0.90100  0.308545         0.910  0.276683
+5  data/mldoc/it-1/models/sp15k/qrnn_avg_6.m       0.90275  0.323594         0.915  0.287004
+6  data/mldoc/it-1/models/sp15k/qrnn_avg_7.m       0.89925  0.295158         0.910  0.269167
+7  data/mldoc/it-1/models/sp15k/qrnn_avg_9.m       0.90325  0.312664         0.908  0.297471
+ds     it-1
+best  89.90
+max   90.32
+avg   90.13
+
+
+# ES
+## VF60k - 8 runs eval
+for i in 1 2 3 4 5 6 7 8; do python -m ulmfit eval --glob="mldoc/${LANG}-1/models/vf60k/qrnn_nl4.m" --name avg_$i --num-cls-epochs=4 --label-smoothing-eps=0.1 --lr_sched=1cycle --bs=10; done
+python -m ulmfit eval --glob="mldoc/${LANG}-1/models/vf60k/qrnn_nl4*.m" --train=False
+
+                                        name  tst_accuracy  tst_loss  val_accuracy  val_loss
+0    data/mldoc/es-1/models/vf60k/qrnn_nl4.m       0.95950  0.146121         0.961  0.161852
+1  data/mldoc/es-1/models/vf60k/qrnn_nl4_1.m       0.95500  0.154836         0.960  0.176217
+2  data/mldoc/es-1/models/vf60k/qrnn_nl4_2.m       0.95850  0.154539         0.961  0.163008
+3  data/mldoc/es-1/models/vf60k/qrnn_nl4_3.m       0.96100  0.151916         0.966  0.169869
+4  data/mldoc/es-1/models/vf60k/qrnn_nl4_4.m       0.95825  0.144630         0.962  0.144410
+5  data/mldoc/es-1/models/vf60k/qrnn_nl4_5.m       0.95675  0.155685         0.960  0.175439
+6  data/mldoc/es-1/models/vf60k/qrnn_nl4_6.m       0.95900  0.143995         0.959  0.164156
+7  data/mldoc/es-1/models/vf60k/qrnn_nl4_7.m       0.95800  0.144662         0.962  0.162957
+8  data/mldoc/es-1/models/vf60k/qrnn_nl4_8.m       0.95850  0.149185         0.962  0.163159
+ds     es-1
+best  96.10
+max   96.10
+avg   95.83
+
+
+## sp15k - 8 runs eval 
+LANG=es
+for i in 1 2 3 4 5 6 7 9; do python -m ulmfit eval --glob="mldoc/${LANG}-1/models/sp15k/qrnn_nl4.m" --name avg_$i --num-cls-epochs=4 --label-smoothing-eps=0.1 --lr_sched=1cycle --bs=18; done
+python -m ulmfit eval --glob="mldoc/${LANG}-1/models/sp15k/qrnn_avg*.m" --train=False
+                                        name  tst_accuracy  tst_loss  val_accuracy  val_loss
+0  data/mldoc/es-1/models/sp15k/qrnn_avg_1.m       0.95700  0.149081         0.963  0.162808
+1  data/mldoc/es-1/models/sp15k/qrnn_avg_2.m       0.95875  0.141572         0.963  0.150970
+2  data/mldoc/es-1/models/sp15k/qrnn_avg_3.m       0.95900  0.151387         0.962  0.157047
+3  data/mldoc/es-1/models/sp15k/qrnn_avg_4.m       0.95375  0.165935         0.956  0.182161
+4  data/mldoc/es-1/models/sp15k/qrnn_avg_5.m       0.95850  0.151109         0.960  0.156376
+5  data/mldoc/es-1/models/sp15k/qrnn_avg_6.m       0.95800  0.150724         0.961  0.152761
+6  data/mldoc/es-1/models/sp15k/qrnn_avg_7.m       0.95875  0.142476         0.963  0.151585
+7  data/mldoc/es-1/models/sp15k/qrnn_avg_8.m       0.95525  0.165120         0.957  0.164723
+8  data/mldoc/es-1/models/sp15k/qrnn_avg_9.m       0.95725  0.151323         0.960  0.156123
+ds     es-1
+best  95.70
+max   95.90
+avg   95.74
+
+# DE
+## VF60k - 9 runs eval
+for i in 1 2 3 4 5 6 7 8; do python -m ulmfit eval --glob="mldoc/${LANG}-1/models/vf60k/qrnn_nl4.m" --name avg_$i --num-cls-epochs=4 --label-smoothing-eps=0.1 --lr_sched=1cycle --bs=10; done
+python -m ulmfit eval --glob="mldoc/${LANG}-1/models/vf60k/qrnn_nl4*.m" --train=False
+                                        name  tst_accuracy  tst_loss  val_accuracy  val_loss
+0    data/mldoc/de-1/models/vf60k/qrnn_nl4.m       0.95250  0.193797         0.946  0.225316
+1  data/mldoc/de-1/models/vf60k/qrnn_nl4_1.m       0.95575  0.157327         0.947  0.189885
+2  data/mldoc/de-1/models/vf60k/qrnn_nl4_2.m       0.95400  0.174519         0.947  0.201792
+3  data/mldoc/de-1/models/vf60k/qrnn_nl4_3.m       0.95325  0.180489         0.947  0.208106
+4  data/mldoc/de-1/models/vf60k/qrnn_nl4_4.m       0.95425  0.161056         0.949  0.199169
+5  data/mldoc/de-1/models/vf60k/qrnn_nl4_5.m       0.94775  0.182012         0.941  0.210262
+6  data/mldoc/de-1/models/vf60k/qrnn_nl4_6.m       0.95375  0.164578         0.947  0.198632
+7  data/mldoc/de-1/models/vf60k/qrnn_nl4_7.m       0.95575  0.152596         0.947  0.196844
+8  data/mldoc/de-1/models/vf60k/qrnn_nl4_8.m       0.95350  0.167661         0.942  0.203538
+ds     de-1
+best  95.42
+max   95.57
+avg   95.34
+
+## sp15k - 8 runs eval 
+for i in 1 2 3 4 5 6 7 9; do python -m ulmfit eval --glob="mldoc/${LANG}-1/models/sp15k/qrnn_nl4.m" --name avg_$i --num-cls-epochs=4 --label-smoothing-eps=0.1 --lr_sched=1cycle --bs=18; done
+python -m ulmfit eval --glob="mldoc/${LANG}-1/models/sp15k/qrnn_avg*.m" --train=False
+                                        name  tst_accuracy  tst_loss  val_accuracy  val_loss
+0  data/mldoc/de-1/models/sp15k/qrnn_avg_1.m       0.95700  0.142444         0.945  0.206329
+1  data/mldoc/de-1/models/sp15k/qrnn_avg_2.m       0.95975  0.141225         0.955  0.189209
+2  data/mldoc/de-1/models/sp15k/qrnn_avg_3.m       0.95925  0.140172         0.946  0.198159
+3  data/mldoc/de-1/models/sp15k/qrnn_avg_4.m       0.95650  0.145889         0.942  0.193202
+4  data/mldoc/de-1/models/sp15k/qrnn_avg_5.m       0.96000  0.137710         0.953  0.192368
+5  data/mldoc/de-1/models/sp15k/qrnn_avg_6.m       0.95975  0.145318         0.954  0.196413
+6  data/mldoc/de-1/models/sp15k/qrnn_avg_7.m       0.95925  0.141719         0.943  0.199226
+7  data/mldoc/de-1/models/sp15k/qrnn_avg_8.m       0.96100  0.140644         0.949  0.204398
+8  data/mldoc/de-1/models/sp15k/qrnn_avg_9.m       0.96050  0.143330         0.949  0.189472
+ds     de-1
+best  95.97
+max   96.10
+avg   95.92
+
+# RU
+## VF60k - 9 runs eval
+for i in 1 2 3 4 5 6 7 8; do python -m ulmfit eval --glob="mldoc/${LANG}-1/models/vf60k/qrnn_nl4.m" --name nl4_$i --num-cls-epochs=4 --label-smoothing-eps=0.1 --lr_sched=1cycle --bs=10; done
+python -m ulmfit eval --glob="mldoc/${LANG}-1/models/vf60k/qrnn_nl4*.m" --train=False
diff --git a/results/logs/tokenization/fr.md b/results/logs/tokenization/fr.md
new file mode 100644
index 0000000..68d209a
--- /dev/null
+++ b/results/logs/tokenization/fr.md
@@ -0,0 +1,241 @@
+# FR
+## LM
+```
+LANG=fr
+python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='vf' --nl 4 --name 'nl4' --max-vocab 60000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0  --label-smoothing-eps=0.0
+Max vocab: 60000
+Cache dir: data/wiki/fr-100/models/vf60k
+Model dir: data/wiki/fr-100/models/vf60k/qrnn_nl4.m
+Wiki text was split to 174227 articles
+Wiki text was split to 491 articles
+Running tokenization lm...
+Data lm, trn: 174227, val: 491
+Size of vocabulary: 60003
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', 'de', ',', '\n', '.', 'la', 'le', 'et', 'à', 'en', "l'", "&'", 'les']
+Training args:  {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps:  {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
+Bptt 70
+Training lm from random weights
+epoch     train_loss  valid_loss  accuracy
+1         3.359852    3.022507    0.434433
+2         3.253006    2.955765    0.435078
+3         3.274156    2.917242    0.442870
+4         3.181276    2.850124    0.451273
+5         3.169587    2.813115    0.456411
+6         3.075235    2.773676    0.462836
+7         3.054632    2.723182    0.469485
+8         2.964262    2.661821    0.479831
+9         3.019209    2.631244    0.487013
+10        2.899521    2.618838    0.489004
+Total time: 10:48:33
+data/wiki/fr-100/models/vf60k
+Saving info data/wiki/fr-100/models/vf60k/qrnn_nl4.m/info.json
+```
+## CLS
+
+
+
+# ES
+## LM
+```
+LANG=es
+python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='vf' --nl 4 --name 'nl4' --max-vocab 60000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0  --label-smoothing-eps=0.0
+Max vocab: 60000
+Cache dir: data/wiki/es-100/models/vf60k
+Model dir: data/wiki/es-100/models/vf60k/qrnn_nl4.m
+Wiki text was split to 161509 articles
+Wiki text was split to 78 articles
+Running tokenization lm...
+Data lm, trn: 161509, val: 78
+Size of vocabulary: 60003
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', 'de', ',', '\n', '.', 'la', 'el', 'en', 'y', 'a', "&'", 'que', 'los']
+Training args:  {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps:  {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
+Bptt 70
+Training lm from random weights
+epoch     train_loss  valid_loss  accuracy
+1         3.285345    3.884676    0.312458
+2         3.157721    3.832607    0.313905
+3         3.193605    3.800210    0.316862
+4         3.152273    3.747068    0.319891
+5         3.028921    3.713120    0.324912
+6         3.067516    3.652925    0.330345
+7         3.006576    3.571537    0.339488
+8         2.922181    3.529282    0.345483
+9         2.871947    3.497736    0.352535
+10        2.862057    3.491642    0.354063
+Total time: 14:46:42
+data/wiki/es-100/models/vf60k
+Saving info data/wiki/es-100/models/vf60k/qrnn_nl4.m/info.json
+```
+## MLDoc
+
+```
+python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path  data/wiki/${LANG}-100/models/vf60k/qrnn_nl4.m  --lang=${LANG} --name nl4 - train 20 --bs 18 --num-cls-epochs=4 --lr_sched=1cycle --label-smoothing-eps=0.1
+Max vocab: 60000
+Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/vf60k
+Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/vf60k/qrnn_nl4.m
+Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/es.dev.csv
+Running tokenization lm...
+Data lm, trn: 13013, val: 1445
+Running tokenization cls...
+Data cls, trn: 1000, val: 1000
+Running tokenization tst...
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 34317
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', 'de', ',', '.', 'el', 'la', 'a', 'en', ')', '(', 'y', 'los', 'que']
+Training args:  {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps:  {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
+Loading pretrained model
+Unknown tokens 17152, first 100: ['pct', 'reuter', 'corresponsalía', 'mln', 'indice', 'cotizaba', 'mlns', '585-8308', 'francfort', 'oct', 'jul', 'abr', '585-2154', 'ibex-35', 'feb', 'ibex', 'ago', '585-2152', 'bundesbank', 'ftse', '585-2196', 'interanual', '585-2159', 'cac-40', 'cotizaban', 'uem', 'm.m', '10a', 'alcista', 'bbv', 'anoche', 'argentaria', 'pagarés', 'btp', 'transferibles', 'c.l.p', 'bch', '8,80', '585-8315', 'corros', 'retevisión', '7,35', 'spread', 'bln', 'cnmv', 'decenal', 'opv', 'vespertina', 'greenspan', 'alzas', 'nikkei', 'cambista', 'tir', 'preapertura', 'mibtel', 'tabacalera', 'ptas', 'día-día', 'diff', '18-26', '6-12', 'dif.d.ant', 'max.año', 'min.año', 'spi', 'inem', 'indust', 'fecsa', 'securities', 'repos', 'fomc', 'obligs', 'mibor', 'descartaban', 'sepi', 'interbancario', 'tietmeyer', '5,50', 'piqué', '6,75', 'aprobacion', 'moscu', 'brutas', 'deficit', '0830', 'buba', 'g-7', 'waigel', 'stet', 'petróleo-químicas', '.ibex', '5,25', '6,00', '3m', '5,30', 'trimestrales', 'cauto', 'smi', 'ant-', 'facilitadas']
+Bptt 70
+Training lm from:  [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/es-100/models/vf60k/qrnn_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/es-100/models/vf60k/qrnn_nl4.m/../itos')]
+epoch     train_loss  valid_loss  accuracy
+1         3.714449    2.774868    0.469673
+Total time: 01:42
+epoch     train_loss  valid_loss  accuracy
+1         3.239635    2.591123    0.496131
+2         2.935826    2.367645    0.535486
+3         2.631979    2.196012    0.564117
+4         2.640709    2.058490    0.582902
+5         2.434918    1.949251    0.599310
+6         2.293211    1.855961    0.613708
+7         2.224960    1.773834    0.626423
+8         2.188689    1.698404    0.639268
+9         2.024225    1.623230    0.653119
+10        2.041964    1.555204    0.665692
+11        1.925207    1.492332    0.677868
+12        1.864637    1.421467    0.693237
+13        1.779024    1.361629    0.706401
+14        1.817028    1.301509    0.719889
+15        1.719223    1.261717    0.730797
+16        1.573684    1.221963    0.740282
+17        1.583578    1.192796    0.747645
+18        1.590957    1.174528    0.751411
+19        1.546806    1.167247    0.753300
+20        1.514999    1.165146    0.753615
+Total time: 37:16
+/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/vf60k
+Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/vf60k/qrnn_nl4.m/info.json
+Single training schedule
+epoch     train_loss  valid_loss  accuracy
+1         0.654116    0.368231    0.907000
+2         0.447137    0.287264    0.961000
+3         0.308758    0.285717    0.958000
+4         0.216707    0.275839    0.962000
+Total time: 00:42
+Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1/models/vf60k/qrnn_nl4.m
+Loss and accuracy using (cls_best): [0.14618756, tensor(0.9597)] [0.16216491, tensor(0.9620)]
+val_loss:     0.16216491
+val_accuracy: 0.9620000123977661
+tst_loss:     0.14618756
+tst_accuracy: 0.9597499966621399
+```
+
+
+# IT
+## LM
+```
+LANG=it
+python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='vf' --nl 4 --name 'nl4' --max-vocab 60000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0  --label-smoothing-eps=0.0
+
+Max vocab: 60000
+Cache dir: data/wiki/it-100/models/vf60k
+Model dir: data/wiki/it-100/models/vf60k/qrnn_nl4.m
+Wiki text was split to 164583 articles
+Wiki text was split to 98 articles
+Data lm, trn: 164583, val: 98
+Size of vocabulary: 60003
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', ',', '\n', '.', 'di', 'e', "&'", 'il', 'la', 'in', 'a', 'del', 'che']
+Training args:  {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps:  {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
+Bptt 70
+Training lm from random weights
+epoch     train_loss  valid_loss  accuracy
+1         3.629171    4.075579    0.290754
+2         3.496484    4.007234    0.291424
+3         3.541803    3.973911    0.294861
+4         3.431979    3.926369    0.299076
+5         3.432869    3.880250    0.303598
+6         3.356332    3.823208    0.309304
+7         3.256672    3.760301    0.316393
+8         3.312303    3.708765    0.323862
+9         3.240380    3.670833    0.329326
+10        3.240536    3.661237    0.331286
+Total time: 15:32:22
+data/wiki/it-100/models/vf60k
+Saving info data/wiki/it-100/models/vf60k/qrnn_nl4.m/info.json
+```
+
+
+```bash
+python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path  data/wiki/${LANG}-100/models/vf60k/qrnn_nl4.m  --lang=${LANG} --name nl4 - train 20 --bs 18 --num-cls-epochs=4 --lr_sched=1cycle --label-smoothing-eps=0.1
+Max vocab: 60000
+Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/vf60k
+Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/vf60k/qrnn_nl4.m
+Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/it.dev.csv
+Data lm, trn: 13500, val: 1500
+Running tokenization cls...
+Data cls, trn: 1000, val: 1000
+Running tokenization tst...
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 29600
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '.', ',', 'di', 'e', ')', '(', 'il', "'", 'a', 'in', 'la', 'del']
+Training args:  {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps:  {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
+Loading pretrained model
+Unknown tokens 13370, first 100: ['pct', 'reuter', 'mld', 'mln', 'societa', 'dealer', 'btp', 'ott', 'dlr', 'attivita', 'venerdi', 'nov', 'feb', 'dic', 'stet', 'mibtel', 'bundesbank', 'bankitalia', 'mib30', 'perche', 'ipsoa', 'comit', 'cct', 'nil', 'cedola', 'puo', 'possibilita', 'lunedi', 'tranche', 'stg', 'warrant', 'stamane', 'ctz', 'giovedi', 'citta', 'ord', 'consob', 'uem', 'martedi', 'spread', 'verra', 't-bond', 'mercoledi', 'risp', 'viv', 'ffr', 'avra', 'compart', 'gmn', 'dovra', 'potra', 'fib30', 'contrattazioni', 'gemina', 'frf', 'controvalore', 'overnight', 'cir', 'apr', 'consensus', 'tendenziale', 'nikkei', 'autorita', 'tus', 'pretasse', 'fib', 'rialzi', 'fomc', 'gilt', 'circ', 'destagionalizzati', 'prec', 'liquidita', 'ecu', 'destagionalizzato', 'cariplo', 'stamani', 'obbligazionario', 'bur', 'imi', 'aggiudicazione', 'treu', 'ambroveneto', 'fixing', 'hpi', 'rnc', 'capacita', 'dietimi', 'greenspan', 'tietmeyer', 'waigel', 'nasdaq', 'eltsin', 'redditivita', 'liffe', 'telematico', 'ifil', 'interpellati', '6,25', 'visco']
+Bptt 70
+Training lm from:  [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/it-100/models/vf60k/qrnn_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/it-100/models/vf60k/qrnn_nl4.m/../itos')]
+epoch     train_loss  valid_loss  accuracy
+1         4.406445    3.675336    0.338066
+Total time: 01:10
+epoch     train_loss  valid_loss  accuracy
+1         3.870676    3.516882    0.355481
+2         3.633525    3.322235    0.383076
+3         3.454955    3.121748    0.408930
+4         3.210115    2.935245    0.433205
+5         3.112426    2.775076    0.452784
+6         2.991053    2.638768    0.471221
+7         2.904022    2.533667    0.485577
+8         2.808465    2.426029    0.501932
+9         2.713658    2.320023    0.518699
+10        2.580141    2.226892    0.533786
+11        2.532727    2.133867    0.549680
+12        2.449591    2.034733    0.567797
+13        2.387805    1.963019    0.583013
+14        2.337399    1.880745    0.598986
+15        2.217255    1.818780    0.612503
+16        2.175724    1.764977    0.623581
+17        2.057536    1.726874    0.631422
+18        2.093975    1.705599    0.635835
+19        2.030292    1.694430    0.637838
+20        2.057254    1.691360    0.638669
+Total time: 32:28
+/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/vf60k
+Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/vf60k/qrnn_nl4.m/info.json
+
+***OOTM**
+```
+
+```
+python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path  data/wiki/${LANG}-100/models/vf60k/qrnn_nl4.m  --lang=${LANG} --name nl4 - train 20 --bs 18 --num-cls-epochs=4 --lr_sched=1cycle --label-smoothing-eps=0.1
+python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path  data/wiki/${LANG}-100/models/vf60k/qrnn_nl4.m  --lang=${LANG} --name nl4 - train 20 --bs 10 --num-cls-epochs=4 --lr_sched=1cycle --label-smoothing-eps=0.1
+Max vocab: 60000
+Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/vf60k
+Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/vf60k/qrnn_nl4.m
+Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/it.dev.csv
+Data lm, trn: 13500, val: 1500
+Data cls, trn: 1000, val: 1000
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 29600
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '.', ',', 'di', 'e', ')', '(', 'il', "'", 'a', 'in', 'la', 'del']
+Single training schedule
+epoch     train_loss  valid_loss  accuracy
+1         0.736275    0.717692    0.837000
+2         0.593485    0.444027    0.876000
+3         0.376322    0.411704    0.907000
+4         0.244267    0.370927    0.915000
+Total time: 00:33
+Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1/models/vf60k/qrnn_nl4.m
+Loss and accuracy using (cls_best): [0.3200554, tensor(0.8997)] [0.27118126, tensor(0.9150)]
+val_loss:     0.27118126
+val_accuracy: 0.9150000214576721
+tst_loss:     0.3200554
+tst_accuracy: 0.8997499942779541
+```
\ No newline at end of file
diff --git a/results/logs/tokenization/ru.md b/results/logs/tokenization/ru.md
index 6adaa2e..9985367 100644
--- a/results/logs/tokenization/ru.md
+++ b/results/logs/tokenization/ru.md
@@ -1,3 +1,96 @@
+
+## SP25k
+```bash
+python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='sp' --nl 4 --name
+'nl4' --max-vocab 25000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0  --label-smoothing-eps=0.10 --tokenizer='sp
+Max vocab: 25000
+Cache dir: data/wiki/ru-100/models/sp25k
+Model dir: data/wiki/ru-100/models/sp25k/qrnn_nl4.m
+Wiki text was split to 193047 articles
+Wiki text was split to 460 articles
+Data lm, trn: 193047, val: 460
+Size of vocabulary: 25000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', 'х', '▁на']
+Training args:  {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0} dps:  {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
+Training lm from random weights
+epoch     train_loss  valid_loss  accuracy
+1         4.154972    4.198218    0.447508
+2         4.030367    4.159642    0.449420
+3         4.138530    4.146010    0.451526
+4         3.997120    4.097048    0.457177
+5         3.999151    4.036350    0.465117
+6         3.935380    3.955517    0.476446
+7         3.912357    3.875987    0.487591
+8         3.785693    3.789099    0.501560
+9         3.743162    3.725730    0.512294
+10        3.690226    3.706929    0.516769
+Total time: 12:10:03
+data/wiki/ru-100/models/sp25k
+Saving info data/wiki/ru-100/models/sp25k/qrnn_nl4.m/info.json
+```
+
+```
+python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path data/wiki/${LANG}-100/models/sp25k/qrnn_${NAME}.m  --lang=${LANG} --name ${NAME} - train 20 --bs 18 --num-cls-epochs=4 --lr_sched=1cycle --label-smoothing-eps=0.1
+
+Max vocab: 25000
+Cache dir: /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp25k
+Model dir: /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp25k/qrnn_nl4.m
+Loading validation /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv
+Running tokenization lm...
+Data lm, trn: 9195, val: 1021
+Running tokenization cls...
+Data cls, trn: 1000, val: 1000
+Running tokenization tst...
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 25000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', 'х', '▁на']
+Training args:  {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps:  {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
+Loading pretrained model
+Unknown tokens 0, first 100: []
+Training lm from:  [PosixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp25k/qrnn_nl4.m/lm_best'), PosixPath('/home/n-waves/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp25k/qrnn_nl4.m/../itos')]
+epoch     train_loss  valid_loss  accuracy
+1         4.626971    3.868075    0.474742
+Total time: 01:58
+epoch     train_loss  valid_loss  accuracy
+1         3.821786    3.625366    0.519506
+2         3.570115    3.379288    0.566803
+3         3.517294    3.179166    0.599955
+4         3.160131    3.028985    0.626484
+5         3.135806    2.923198    0.644557
+6         3.055160    2.840300    0.659376
+7         3.005086    2.770163    0.672080
+8         2.811366    2.708846    0.684065
+9         2.818394    2.658951    0.694358
+10        2.881018    2.605373    0.705269
+11        2.793422    2.560091    0.715893
+12        2.708385    2.516373    0.725908
+13        2.690258    2.471159    0.735673
+14        2.748342    2.436113    0.744533
+15        2.601220    2.394404    0.754131
+16        2.616882    2.372301    0.760451
+17        2.602902    2.349164    0.766014
+18        2.560349    2.336217    0.769222
+19        2.549936    2.332076    0.770150
+20        2.546798    2.331103    0.770472
+Total time: 53:22
+/home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp25k
+Saving info /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp25k/qrnn_nl4.m/info.json
+Single training schedule
+epoch     train_loss  valid_loss  accuracy
+1         1.043533    0.961182    0.731000
+2         0.859086    0.837210    0.824000
+3         0.735276    0.724173    0.871000
+4         0.612012    0.711034    0.857000
+Total time: 01:15
+Saving models at /home/n-waves/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp25k/qrnn_nl4.m
+Loss and accuracy using (cls_best): [0.3957597, tensor(0.8720)]
+0.3957597017288208
+0.871999979019165
+```
+
+## V60k
+## VF60k
+
 ```
 LANG=ru
 python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='vf' --nl 4 --name 'nl4' --max-vocab 60000 --lang ${LANG} --qrnn=True - train 10 --bs=50 --drop_mult=0  --label-smoothing-eps=0.1
@@ -27,3 +120,59 @@ Total time: 11:24:03
 data/wiki/ru-100/models/vf60k
 ```
 
+## SP15k LSTM nl 3
+```bash
+python -m ulmfit cls --dataset-path data/mldoc/${LANG}-1 --base-lm-path  data/wiki/${LANG}-100/models/sp15k/lstm_nl4.m  --lang=${LANG} --name nl4 - train 20 --bs 18 --num-cls-epochs=4 --lr_sched=1cycle --label-smoothing-eps=0.1
+Max vocab: 15000
+Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k
+Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/lstm_nl4.m
+Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/ru.dev.csv
+Data lm, trn: 9195, val: 1021
+Data cls, trn: 1000, val: 1000
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 15000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', '▁на', 'х']
+Training args:  {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps:  {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
+Loading pretrained model
+Unknown tokens 0, first 100: []
+Bptt 70
+Training lm from:  [PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp15k/lstm_nl4.m/lm_best'), PosixPath('/home/pczapla/workspace/ulmfit-multilingual/data/wiki/ru-100/models/sp15k/lstm_nl4.m/../itos')]
+epoch     train_loss  valid_loss  accuracy
+1         3.705343    3.261906    0.558008
+Total time: 05:27
+epoch     train_loss  valid_loss  accuracy
+1         3.243956    3.073661    0.594862
+2         3.139877    2.917376    0.625388
+3         2.941367    2.786331    0.650792
+4         2.846027    2.682831    0.671712
+5         2.796714    2.600119    0.687167
+6         2.841771    2.527643    0.702408
+7         2.726931    2.459425    0.717738
+8         2.619217    2.402231    0.729743
+9         2.626002    2.349137    0.742474
+10        2.535362    2.299844    0.753796
+11        2.501980    2.257779    0.764137
+12        2.427705    2.209901    0.776203
+13        2.393852    2.167961    0.787562
+14        2.340693    2.129181    0.797972
+15        2.307895    2.094267    0.807763
+16        2.330075    2.069201    0.814278
+17        2.232444    2.049109    0.820321
+18        2.306738    2.038069    0.823257
+19        2.232783    2.031799    0.825218
+20        2.227589    2.030583    0.825465
+Total time: 2:13:57
+/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k
+Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/lstm_nl4.m/info.json
+Single training schedule
+epoch     train_loss  valid_loss  accuracy
+1         1.019235    0.894525    0.820000
+2         0.885900    0.831892    0.772000
+3         0.714437    0.711899    0.865000
+4         0.608688    0.706948    0.868000
+Total time: 05:07
+Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1/models/sp15k/lstm_nl4.m
+Loss and accuracy using (cls_best): [0.42021805, tensor(0.8648)]
+0.4202180504798889
+0.8647500276565552
+```
diff --git a/results/logs/zeroshot.md b/results/logs/zeroshot.md
index fbb6199..cc77d2d 100644
--- a/results/logs/zeroshot.md
+++ b/results/logs/zeroshot.md
@@ -89,8 +89,274 @@ ru from fr
 zh from fr
  | Test: 79.40% | classes: 33.60 31.12  9.07 26.20
 ```
+#### Fixed label smoohting
 
+```
+                                                 name  tst_accuracy  tst_loss  val_accuracy  val_loss
+0   data/mldoc/de-1-laser-en1/models/sp15k/qrnn_nl...       0.91625  0.295122         0.922  0.256934
+1   data/mldoc/de-1-laser-en1/models/sp15k/qrnn_nl4.m       0.91325  0.375667         0.910  0.357531
+2   data/mldoc/es-1-laser-en1/models/sp15k/qrnn_nl...       0.76725  1.266754         0.872  0.482154
+3   data/mldoc/es-1-laser-en1/models/sp15k/qrnn_nl...       0.78425  1.213045         0.876  0.478503
+4   data/mldoc/es-1-laser-en1/models/sp15k/qrnn_nl4.m       0.79100  0.828614         0.878  0.439474
+5   data/mldoc/fr-1-laser-en1/models/sp15k/qrnn_nl...       0.87125  0.406994         0.877  0.358970
+6   data/mldoc/fr-1-laser-en1/models/sp15k/qrnn_nl4.m       0.89425  0.384984         0.888  0.405739
+7   data/mldoc/it-1-laser-en1/models/sp15k/qrnn_nl...       0.75850  1.014424         0.815  0.579695
+8   data/mldoc/it-1-laser-en1/models/sp15k/qrnn_nl4.m       0.76025  0.808285         0.818  0.555738
+9   data/mldoc/ru-1-laser-en1/models/sp15k/qrnn_nl...       0.67925  1.588188         0.841  0.541397
+10  data/mldoc/ru-1-laser-en1/models/sp15k/qrnn_nl4.m       0.68125  1.069047         0.838  0.536006
+11  data/mldoc/zh-1-laser-en1/models/sp15k/qrnn_nl...       0.81450  0.624627         0.815  0.643505
+12  data/mldoc/zh-1-laser-en1/models/sp15k/qrnn_nl4.m       0.82475  0.582873         0.820  0.570754
+ds    de-1-laser-en1  es-1-laser-en1  fr-1-laser-en1  it-1-laser-en1  ru-1-laser-en1  zh-1-laser-en1
+best           91.62           79.10           89.42           76.02           67.93           82.48
+max            91.62           79.10           89.42           76.02           68.12           82.48
+avg            91.48           78.08           88.27           75.94           68.02           81.96
+Saving result to: laser-en1-results.csv
+```
+#### JA 
+```bash
+                                                name  tst_accuracy  tst_loss  val_accuracy  val_loss
+0  data/mldoc/ja-1-laser-en1/models/sp15k/qrnn_nl...       0.68500  1.000403         0.722  0.847306
+1  data/mldoc/ja-1-laser-en1/models/sp15k/qrnn_tls.m       0.65625  1.421663         0.788  0.749413
+2  data/mldoc/ja-1-laser-en1/models/sp15k/qrnn_tl...       0.68125  1.126172         0.786  0.650364
+3  data/mldoc/ja-1-laser-en1/models/sp15k/qrnn_tl...       0.68625  1.229256         0.784  0.674508
+4  data/mldoc/ja-1-laser-en1/models/sp15k/qrnn_tl...       0.69575  1.110130         0.800  0.644522
+5  data/mldoc/ja-1-laser-en1/models/sp15k/qrnn_tl...       0.69650  1.082014         0.767  0.672668
+ds    ja-1-laser-e
+best         69.57
+max          69.65
+avg          68.35
+```
+##### ES labels from laser-EN10k
+```
+python -m ulmfit eval --glob="mldoc/es-1/models/sp15k/qrnn_nl4.m" --dataset_template='${lang}-1-laser-en1' --name nl4 --num-cls-epochs=8 --label-smoothing-eps=0.1 --lr_sched=1cycle --bs=18
+Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m
+es-1-laser-en1
+Max vocab: 15000
+Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en1/models/sp15k
+Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en1/models/sp15k/qrnn_nl4.m
+Training
+Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en1/es.dev.csv
+Running tokenization lm...
+Data lm, trn: 13013, val: 1445
+Running tokenization cls...
+Data cls, trn: 1000, val: 1000
+Running tokenization tst...
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 15000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que']
+Training args:  {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps:  {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
+/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning:
+
+                               !! WARNING !!
+
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+Your compiler (c++) may be ABI-incompatible with PyTorch!
+Please use a compiler that is ABI-compatible with GCC 4.9 and above.
+See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html.
+
+See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6
+for instructions on how to install GCC 4.9 or higher.
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+                              !! WARNING !!
+
+  warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler))
+Loading pretrained model
+Unknown tokens 0, first 100: []
+Bptt 70
+/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en1/models/sp15k
+Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en1/models/sp15k/qrnn_nl4.m/info.json
+Single training schedule
+epoch     train_loss  valid_loss  accuracy
+1         0.755567    0.673722    0.822000
+2         0.641650    0.623495    0.854000
+3         0.526631    0.626180    0.857000
+4         0.442634    0.789478    0.837000
+5         0.341413    0.623467    0.859000
+6         0.254876    0.599503    0.875000
+7         0.202478    0.573548    0.870000
+8         0.179376    0.564544    0.870000
+Total time: 01:56
+Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en1/models/sp15k/qrnn_nl4.m
+Loss and accuracy using (cls_best): [1.0270673, tensor(0.7983)] [0.4585123, tensor(0.8700)]
+                                                name  tst_accuracy  tst_loss  val_accuracy  val_loss
+0  data/mldoc/es-1-laser-en1/models/sp15k/qrnn_nl4.m       0.79825  1.027067          0.87  0.458512
+ds    es-1-laser-en1
+best           79.83
+max            79.83
+avg            79.83
+```
 #### ULMFit zershot on laser-en1k 4 epochs
+##### missing RU
+```
+ python -m ulmfit eval --glob="mldoc/*-1/models/sp15k/qrnn_nl4.m" --dataset_template='${lang}-1-laser-en1' --name nl4 --num-cls-epochs=4 --label-smoothing-eps=0.1 --lr_sched=1cycle --bs=18
+Processing data/mldoc/de-1/models/sp15k/qrnn_nl4.m
+de-1-laser-en1
+Max vocab: 15000
+Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-en1/models/sp15k
+Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-en1/models/sp15k/qrnn_nl4.m
+Evaluating previously trained model
+Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/de-1-laser-en1/de.dev.csv
+Data lm, trn: 13500, val: 1500
+Data cls, trn: 1000, val: 1000
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 15000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁.', '▁,', '▁der', '▁die', 'en', '▁und', 's', '▁in', 'er', "▁&'"]
+/home/pczapla/anaconda3/envs/fastaiv1/lib/python3.7/site-packages/torch/utils/cpp_extension.py:152: UserWarning:
+
+                               !! WARNING !!
+
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+Your compiler (c++) may be ABI-incompatible with PyTorch!
+Please use a compiler that is ABI-compatible with GCC 4.9 and above.
+See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html.
+
+See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6
+for instructions on how to install GCC 4.9 or higher.
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+                              !! WARNING !!
+
+  warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler))
+Loss and accuracy using (cls_best): [0.3181207, tensor(0.9133)]
+Processing data/mldoc/en-1/models/sp15k/qrnn_nl4.m
+en-1-laser-en1
+Max vocab: 15000
+Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1-laser-en1/models/sp15k
+Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1-laser-en1/models/sp15k/qrnn_nl4.m
+Training
+Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1-laser-en1/en.dev.csv
+Running tokenization lm...
+Data lm, trn: 13500, val: 1500
+Running tokenization cls...
+Data cls, trn: 1000, val: 1000
+Running tokenization tst...
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 15000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁the', '▁,', 's', '▁.', '▁of', '▁and', '▁in', '▁to', '▁a', 'ed']
+Training args:  {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps:  {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
+Loading pretrained model
+Unknown tokens 0, first 100: []
+Bptt 70
+/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1-laser-en1/models/sp15k
+Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1-laser-en1/models/sp15k/qrnn_nl4.m/info.json
+Single training schedule
+epoch     train_loss  valid_loss  accuracy
+1         0.850595    0.667213    0.919000
+2         0.698336    0.648402    0.927000
+3         0.588279    0.600404    0.936000
+4         0.529861    0.581380    0.937000
+Total time: 01:10
+Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/en-1-laser-en1/models/sp15k/qrnn_nl4.m
+Loss and accuracy using (cls_best): [0.21109423, tensor(0.9490)]
+Processing data/mldoc/es-1/models/sp15k/qrnn_nl4.m
+es-1-laser-en1
+Max vocab: 15000
+Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en1/models/sp15k
+Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en1/models/sp15k/qrnn_nl4.m
+Evaluating previously trained model
+Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/es-1-laser-en1/es.dev.csv
+Data lm, trn: 13013, val: 1445
+Data cls, trn: 1000, val: 1000
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 15000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', '▁la', '▁.', 's', '▁en', '▁el', '▁y', '▁a', '▁que']
+Loss and accuracy using (cls_best): [0.7726423, tensor(0.7910)]
+Processing data/mldoc/fr-1/models/sp15k/qrnn_nl4.m
+fr-1-laser-en1
+Max vocab: 15000
+Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en1/models/sp15k
+Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en1/models/sp15k/qrnn_nl4.m
+Evaluating previously trained model
+Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/fr-1-laser-en1/fr.dev.csv
+Data lm, trn: 13500, val: 1500
+Data cls, trn: 1000, val: 1000
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 15000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁de', '▁,', 's', '▁.', "'", '▁la', '▁le', '▁et', '▁l', '▁à']
+Loss and accuracy using (cls_best): [0.3214729, tensor(0.8942)]
+Processing data/mldoc/it-1/models/sp15k/qrnn_nl4.m
+it-1-laser-en1
+Max vocab: 15000
+Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-en1/models/sp15k
+Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-en1/models/sp15k/qrnn_nl4.m
+Evaluating previously trained model
+Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/it-1-laser-en1/it.dev.csv
+Data lm, trn: 13500, val: 1500
+Data cls, trn: 1000, val: 1000
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 15000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁di', '▁e', "▁&'", "'", '▁il', '▁la', '▁in', 'e']
+Loss and accuracy using (cls_best): [0.75963426, tensor(0.7602)]
+Processing data/mldoc/ja-1/models/sp15k/qrnn_nl4.m
+ja-1-laser-en1
+Processing data/mldoc/ru-1/models/sp15k/qrnn_nl4.m
+ru-1-laser-en1
+Max vocab: 15000
+Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-en1/models/sp15k
+Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-en1/models/sp15k/qrnn_nl4.m
+Training
+Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-en1/ru.dev.csv
+Running tokenization lm...
+Data lm, trn: 9195, val: 1021
+Running tokenization cls...
+Data cls, trn: 1000, val: 1000
+Running tokenization tst...
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 15000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁.', '▁в', 'а', 'и', 'е', '▁и', 'й', '▁на', 'х']
+Training args:  {'clip': 0.12, 'alpha': 2, 'beta': 1, 'drop_mult': 0.3} dps:  {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
+Loading pretrained model
+Unknown tokens 0, first 100: []
+Bptt 70
+/home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-en1/models/sp15k
+Saving info /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-en1/models/sp15k/qrnn_nl4.m/info.json
+Single training schedule
+epoch     train_loss  valid_loss  accuracy
+1         1.053947    0.859605    0.785000
+2         0.882559    0.793324    0.836000
+3         0.713290    0.812535    0.834000
+4         0.620544    0.801801    0.837000
+Total time: 01:35
+Saving models at /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/ru-1-laser-en1/models/sp15k/qrnn_nl4.m
+Loss and accuracy using (cls_best): [1.0211968, tensor(0.6820)]
+Processing data/mldoc/zh-1/models/sp15k/qrnn_nl4.m
+zh-1-laser-en1
+Max vocab: 15000
+Cache dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-en1/models/sp15k
+Model dir: /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-en1/models/sp15k/qrnn_nl4.m
+Evaluating previously trained model
+Loading validation /home/pczapla/workspace/ulmfit-multilingual/data/mldoc/zh-1-laser-en1/zh.dev.csv
+Data lm, trn: 13500, val: 1500
+Data cls, trn: 1000, val: 1000
+Data tst, trn: 1000, val: 4000
+Size of vocabulary: 15000
+First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁,', '▁的', '▁。', '▁年', '▁、', '▁在', '▁一', '▁是', '▁中', '▁有']
+Loss and accuracy using (cls_best): [0.52925, tensor(0.8248)]
+OrderedDict([('data/mldoc/de-1-laser-en1/models/sp15k/qrnn_nl4.m',
+              0.9132500290870667),
+             ('data/mldoc/en-1-laser-en1/models/sp15k/qrnn_nl4.m',
+              0.9490000009536743),
+             ('data/mldoc/es-1-laser-en1/models/sp15k/qrnn_nl4.m',
+              0.7910000085830688),
+             ('data/mldoc/fr-1-laser-en1/models/sp15k/qrnn_nl4.m',
+              0.8942499756813049),
+             ('data/mldoc/it-1-laser-en1/models/sp15k/qrnn_nl4.m',
+              0.7602499723434448),
+             ('data/mldoc/ru-1-laser-en1/models/sp15k/qrnn_nl4.m',
+              0.6819999814033508),
+             ('data/mldoc/zh-1-laser-en1/models/sp15k/qrnn_nl4.m',
+              0.8247500061988831)])
+data/mldoc/de-1-laser-en1/models/sp15k/qrnn_nl4.m: 0.9132500290870667
+data/mldoc/en-1-laser-en1/models/sp15k/qrnn_nl4.m: 0.9490000009536743
+data/mldoc/es-1-laser-en1/models/sp15k/qrnn_nl4.m: 0.7910000085830688
+data/mldoc/fr-1-laser-en1/models/sp15k/qrnn_nl4.m: 0.8942499756813049
+data/mldoc/it-1-laser-en1/models/sp15k/qrnn_nl4.m: 0.7602499723434448
+data/mldoc/ru-1-laser-en1/models/sp15k/qrnn_nl4.m: 0.6819999814033508
+data/mldoc/zh-1-laser-en1/models/sp15k/qrnn_nl4.m: 0.8247500061988831
+```
+##### Other LAngs
+
 ```bash
 python -m ulmfit eval --glob="mldoc/*-1/models/sp15k/qrnn_nl4.m" --dataset_template="{lang}-1*-laser-en1" --name nl4 --num-cls-epochs=4 --label-smoothing-eps=0.1 --lr_sched=1cycle --bs=18
  python -m ulmfit eval --glob="mldoc/*-1/models/sp15k/qrnn_nl4.m" --dataset_template='${lang}-1*-laser-en1' --name nl4 --num-cls-epochs=4 --label-smoothing-eps=0.1 --lr_sched=1cycle --bs=18

From 24f1d741f0fa8a2a6663f82226c7de306d4d077f Mon Sep 17 00:00:00 2001
From: Piotr Czapla <github@piotrczapla.com>
Date: Tue, 26 Mar 2019 21:27:56 +0100
Subject: [PATCH 18/32] Add generating of pseudo labels

---
 ulmfit/__main__.py   | 37 +++++++++++++++++++++++++++++++++----
 ulmfit/train_clas.py | 36 ++++++++++++++++++++++++++++++++++++
 2 files changed, 69 insertions(+), 4 deletions(-)

diff --git a/ulmfit/__main__.py b/ulmfit/__main__.py
index cec29f7..01e109d 100644
--- a/ulmfit/__main__.py
+++ b/ulmfit/__main__.py
@@ -88,21 +88,50 @@ class ULMFiT:
                     print("Adding", f, dest)
                     tar.add(f, dest)
 
+    def generate_pseudo_labels(self, glob="mldoc/*-1-laser-en1/models/sp15k/qrnn_nl4.m", bs=20, dest_dataset_template='${ds_name}-ps'):
+        for base_model in sorted(Path("data").glob(glob)):
+            print("Processing", base_model)
+
+            dataset_path = [x for x in base_model.parents if x.name == "models"][0].parent
+            lang = get_lang_from_dataset_path(dataset_path)
+            dest_dataset_path = dataset_path.parent/Template(dest_dataset_template).substitute(ds_name=dataset_path.name)
+            try:
+                _name = base_model.name.replace(".m", "").replace("lstm_", "").replace("qrnn_", "")
+                params = CLSHyperParams.from_lm(dataset_path, base_model, lang=lang, name=_name, cuda_id=0)
+                key = str(params.model_dir.relative_to(Path.cwd()))
+                if (params.model_dir / "results.npy").exists():
+                    d = np.load(params.model_dir / "results.npy")
+                    d = d.tolist()  # magiacally convert to dict
+                elif (params.model_dir / "cls_best.pth").exists():
+                    print("Evaluating previously trained model")
+                    d = params.validate_cls(label_smoothing_eps=0.1)
+                else:
+                    print("The model is not trained ignoring")
+                    continue
+                print("Generating pseduolabels", dest_dataset_path)
+                params.generate_pseudo_labels(dest_dataset_path, bs=bs)
+                del params
+            except Exception as e:
+                print("Error", e)
+                raise e
+            gc.collect()
+
+    def ls(self, glob="mldoc/*-1/models/sp30k/lstm_nl4.m"):
+        for i, name in enumerate(sorted(Path("data").glob(glob))):
+            print(i, name, "cls:", (name/"cls_best.pth").exists())
+
     def eval(self, glob="mldoc/*-1/models/sp30k/lstm_nl4.m", dataset_template='${ds_name}', name=None,
              num_lm_epochs=0, cuda_id=0, train=True, to_csv=None, return_df=False, label_smoothing_eps=0.0,
              **trn_params):
         results = []
 
-
         def extract_agg(group):
             best = group.loc[group["val_accuracy"].idxmax()]["tst_accuracy"]
-            best_name = group.loc[group["val_accuracy"].idxmax()]["n"]
             return pd.Series({'best': best* 100,
                               'max': group['tst_accuracy'].max()* 100,
                               'avg': group['tst_accuracy'].mean()* 100})
         def pivot_to_lang(df):
-            df['ds'] = df['name'].str.extract(r'data/[a-z]*/([^/]*)/models')
-            df['n'] = df['name'].str.extract(r'models/[^/]*/([^/]*).m')
+            df['ds'] = df['name'].str.extract(r'data/[a-z]*/([^/]{1,12})[^/]*/models')
             best = df.groupby('ds').apply(extract_agg)
             best = best.round(2)
             return best.T
diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py
index 965df41..b0c6a5f 100644
--- a/ulmfit/train_clas.py
+++ b/ulmfit/train_clas.py
@@ -118,6 +118,42 @@ class CLSHyperParams(LMHyperParams):
         results = {'val_loss': val_res[0], 'val_accuracy': float(val_res[1]), 'tst_loss':tst_res[0], 'tst_accuracy': float(tst_res[1]) }
         return results
 
+    def generate_pseudo_labels(self, dest_folder,  save_name='cls_best', bs=40, data_cls=None,  learn=None, label_smoothing_eps=0.0):
+        if data_cls is None:
+            data_cls, _, _ = self.load_cls_data(bs)
+        if learn is None:
+            learn = self.create_cls_learner(data_cls, drop_mult=0.3, label_smoothing_eps=label_smoothing_eps)
+            learn.unfreeze()
+        learn.load(save_name)
+
+        def make_data_set(ds_type, name):
+            probs, lbls = learn.get_preds(ds_type=ds_type, ordered=True)
+            preds = torch.argmax(probs, 1)
+            preds = to_np(preds)
+            fn = self.dataset_path / f"{self.lang}.{name}.csv"
+            if fn.exists():
+                df = pd.read_csv(fn, header=None)
+                df = df.iloc[(len(df) - len(preds)):]  # account for the training files where first 10% elements were taken as validation
+            else:
+                df = pd.read_csv(self.dataset_path / f"{self.lang}.dev.csv", header=None)
+                df = df.iloc[:len(preds)]  # if using training only get first n for validatation
+
+            accuracy = (df[0] == preds).sum() / len(preds)
+            print(f"Generating {name} dataset of size {len(preds)}, the accuracy is {accuracy}")
+            df['preds'] = preds
+            print(df.head())
+            del df['preds']
+            assert accuracy > 0.7, f"Accuracy is smaller than 0.7 {accuracy}"
+
+            df[0] = preds
+            dest_folder.mkdir(parents=True, exist_ok=True)
+            df.to_csv(dest_folder / f"{self.lang}.{name}.csv", index=None, header=None)
+
+        make_data_set(DatasetType.Train, "train")
+        make_data_set(DatasetType.Valid, "dev")
+        shutil.copy(self.dataset_path / f"{self.lang}.test.csv", dest_folder)
+        shutil.copy(self.dataset_path / f"{self.lang}.unsup.csv", dest_folder)
+
     def create_cls_learner(self, data_clas, dps=None, label_smoothing_eps=0.0, random_init=False, **kwargs):
         assert self.bidir == False, "bidirectional model is not yet supported"
         config = dict(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, pad_token=PAD_TOKEN_ID, qrnn=self.qrnn)

From 0a823be17bdbb788b33e5fcbd63d0cede94fa23f Mon Sep 17 00:00:00 2001
From: Piotr Czapla <github@piotrczapla.com>
Date: Tue, 26 Mar 2019 21:29:23 +0100
Subject: [PATCH 19/32] Maki ti possible to not load unusp

---
 ulmfit/train_clas.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py
index b0c6a5f..479e8ba 100644
--- a/ulmfit/train_clas.py
+++ b/ulmfit/train_clas.py
@@ -217,8 +217,8 @@ class CLSHyperParams(LMHyperParams):
             val_df = pd.read_csv(val_fn, header=None)
         else:
             val_df = None
-
-        unsup_df = pd.read_csv(self.dataset_path / f'{prefix}unsup.csv', header=None)
+        unsup_fn = self.dataset_path / f'{prefix}unsup.csv'
+        unsup_df = pd.read_csv(unsup_fn, header=None) if unsup_fn.exists() else trn_df[:0]
 
         if val_df is None:
             print("Validation set not found using 10% of trn")

From edf3b8d76eebf051b89812060f9bd8f22f02af36 Mon Sep 17 00:00:00 2001
From: "NAUSICAA\\Julian" <eisenjulian@gmail.com>
Date: Sat, 13 Apr 2019 12:32:19 -0300
Subject: [PATCH 20/32] Adding missing param arch

---
 tests/test_text_train.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_text_train.py b/tests/test_text_train.py
index 7c3d894..0f41de5 100644
--- a/tests/test_text_train.py
+++ b/tests/test_text_train.py
@@ -36,7 +36,7 @@ def manual_seed(seed=42):
 def learn():
     path, df_trn, df_val = prep_human_numbers()
     data = TextLMDataBunch.from_df(path, df_trn, df_val, tokenizer=Tokenizer(BaseTokenizer))
-    learn = language_model_learner(data, emb_sz=100, nl=1, drop_mult=0.1)
+    learn = language_model_learner(data, AWD_LSTM, emb_sz=100, nl=1, drop_mult=0.1)
     learn.fit_one_cycle(4, 5e-3)
     return learn
 
@@ -98,6 +98,6 @@ def test_bwdlm_lstm_can_be_trained():
     data = TextLMDataBunch.from_df(path, df_trn, df_val, tokenizer=Tokenizer(BaseTokenizer),
                                    lm_type = contrib_data.LanguageModelType.BwdLM)
 
-    learn = language_model_learner(data, emb_sz=100, nl=1, drop_mult=0.1, qrnn=False)
+    learn = language_model_learner(data, AWD_LSTM, emb_sz=100, nl=1, drop_mult=0.1, qrnn=False)
     learn.fit_one_cycle(2, 5e-3)
     assert learn.validate()[1] > 0.3

From 7e0a8517b6f577635bf1a0aab7774ae439a0fa04 Mon Sep 17 00:00:00 2001
From: "NAUSICAA\\Julian" <eisenjulian@gmail.com>
Date: Sat, 13 Apr 2019 16:08:01 -0300
Subject: [PATCH 21/32] Remove config params from language_model_learner
 funcion

---
 tests/test_text_train.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_text_train.py b/tests/test_text_train.py
index 0f41de5..ccc9bef 100644
--- a/tests/test_text_train.py
+++ b/tests/test_text_train.py
@@ -36,7 +36,7 @@ def manual_seed(seed=42):
 def learn():
     path, df_trn, df_val = prep_human_numbers()
     data = TextLMDataBunch.from_df(path, df_trn, df_val, tokenizer=Tokenizer(BaseTokenizer))
-    learn = language_model_learner(data, AWD_LSTM, emb_sz=100, nl=1, drop_mult=0.1)
+    learn = language_model_learner(data, AWD_LSTM)
     learn.fit_one_cycle(4, 5e-3)
     return learn
 

From 65397d7ae40baeb754bdc93b3c78e7efb429c4d5 Mon Sep 17 00:00:00 2001
From: "NAUSICAA\\Julian" <eisenjulian@gmail.com>
Date: Sat, 13 Apr 2019 16:38:58 -0300
Subject: [PATCH 22/32] Bi-dir not supported yet

---
 tests/test_end_to_end.py | 72 ++++++++++++++++++++--------------------
 1 file changed, 36 insertions(+), 36 deletions(-)

diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py
index 127c8fa..4798122 100644
--- a/tests/test_end_to_end.py
+++ b/tests/test_end_to_end.py
@@ -137,45 +137,45 @@ def test_ulmfit_fastai_end_to_end_label_smoothing():
     exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, label_smoothing_eps=0.1 )
 
 
-def test_ulmfit_fastai_bidir_end_to_end():
-    """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset.
-    """
-    test_data, wt2 = get_test_data()
-    lm_name = 'end-to-end-test-fastai'
+# def test_ulmfit_fastai_bidir_end_to_end():
+#     """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset.
+#     """
+#     test_data, wt2 = get_test_data()
+#     lm_name = 'end-to-end-test-fastai'
 
-    exp = ulmfit.pretrain_lm.LMHyperParams(
-        dataset_path=wt2,
-        lang='en',
-        cuda_id=cuda_id,
-        qrnn=False,
-        bidir=True,
-        tokenizer='f',
-        max_vocab=100,
-        name=lm_name,
-    )
-    exp.train_lm(num_epochs=1, bs=2)
-    exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(str(test_data / 'imdb'), str(exp.model_dir))
-    exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, )
+#     exp = ulmfit.pretrain_lm.LMHyperParams(
+#         dataset_path=wt2,
+#         lang='en',
+#         cuda_id=cuda_id,
+#         qrnn=False,
+#         bidir=True,
+#         tokenizer='f',
+#         max_vocab=100,
+#         name=lm_name,
+#     )
+#     exp.train_lm(num_epochs=1, bs=2)
+#     exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(str(test_data / 'imdb'), str(exp.model_dir))
+#     exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, )
 
-def test_ulmfit_moses_fa_bidir_end_to_end():
-    """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset.
-    """
-    test_data, wt2 = get_test_data()
-    lm_name = 'end-to-end-test-fastai'
+# def test_ulmfit_moses_fa_bidir_end_to_end():
+#     """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset.
+#     """
+#     test_data, wt2 = get_test_data()
+#     lm_name = 'end-to-end-test-fastai'
 
-    exp = ulmfit.pretrain_lm.LMHyperParams(
-        dataset_path=wt2,
-        lang='en',
-        cuda_id=cuda_id,
-        qrnn=False,
-        bidir=True,
-        tokenizer='vf',
-        max_vocab=100,
-        name=lm_name,
-    )
-    exp.train_lm(num_epochs=1, bs=2)
-    exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir)
-    exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, )
+#     exp = ulmfit.pretrain_lm.LMHyperParams(
+#         dataset_path=wt2,
+#         lang='en',
+#         cuda_id=cuda_id,
+#         qrnn=False,
+#         bidir=True,
+#         tokenizer='vf',
+#         max_vocab=100,
+#         name=lm_name,
+#     )
+#     exp.train_lm(num_epochs=1, bs=2)
+#     exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir)
+#     exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, )
 
 # def test_classification_model_work_with_different_dropmul():
 #     learn = self.create_cls_learner(data_clas, drop_mult=0.1)

From 8facc21cfa2beed7fb895f1b8eb240b7ea3d994b Mon Sep 17 00:00:00 2001
From: Piotr Czapla <github@piotrczapla.com>
Date: Sun, 14 Apr 2019 19:13:43 +0200
Subject: [PATCH 23/32] Add F1 metrics to evaluate

---
 calc_100.sh          | 4 ----
 ulmfit/train_clas.py | 3 +++
 2 files changed, 3 insertions(+), 4 deletions(-)
 delete mode 100755 calc_100.sh

diff --git a/calc_100.sh b/calc_100.sh
deleted file mode 100755
index 158de54..0000000
--- a/calc_100.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-
-LANGS
-for
\ No newline at end of file
diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py
index 417f791..b230387 100644
--- a/ulmfit/train_clas.py
+++ b/ulmfit/train_clas.py
@@ -110,6 +110,9 @@ class CLSHyperParams(LMHyperParams):
             learn = self.create_cls_learner(data_tst, drop_mult=0.3, label_smoothing_eps=label_smoothing_eps)
             learn.unfreeze()
         learn.load(save_name)
+        f1 = FBeta(beta=1, average='binary')
+        f1.on_train_begin()
+        learn.metrics += [f1]
         val_res=[-1, -1]
         if data_cls:
             val_res = learn.validate(data_cls.valid_dl)

From 94be5e6e0fe864997b43ffb0d24c05d8a8a1c871 Mon Sep 17 00:00:00 2001
From: Piotr Czapla <github@piotrczapla.com>
Date: Sun, 14 Apr 2019 19:19:12 +0200
Subject: [PATCH 24/32] Remove unused tests

Skip the test that were developed to support BiLM training that due to the changes in fastai stopped compiling
---
 tests/test_text_data.py  |  56 ---------------------
 tests/test_text_train.py | 103 ---------------------------------------
 2 files changed, 159 deletions(-)
 delete mode 100644 tests/test_text_data.py
 delete mode 100644 tests/test_text_train.py

diff --git a/tests/test_text_data.py b/tests/test_text_data.py
deleted file mode 100644
index 1b96079..0000000
--- a/tests/test_text_data.py
+++ /dev/null
@@ -1,56 +0,0 @@
-import pytest
-import fastai.text
-
-from fastai import *
-from fastai.text import *
-
-import fastai_contrib.data as contrib_data
-
-def text_df(labels):
-    data = []
-    texts = ["fast ai is a cool project", "hello world"] * 20
-    for ind, text in enumerate(texts):
-        sample = {}
-        sample["label"] = labels[ind%len(labels)]
-        sample["text"] = text
-        data.append(sample)
-    return pd.DataFrame(data)
-
-###################### UPDATED CODE
-def test_should_load_backwards_lm():
-    path = untar_data(URLs.IMDB_SAMPLE)
-    df = text_df(['neg','pos'])
-
-    data = TextLMDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=0, text_cols=["text"], bs=2,
-                                   lm_type=contrib_data.LanguageModelType.BwdLM)
-    lml = data.train_dl.dl
-    lml.data = lml.batchify(np.concatenate([lml.dataset.x.items[i] for i in range(len(lml.dataset))]))
-    batch = lml.get_batch(lml.data, 0, 70)
-
-    assert batch[0].shape == (lml.bs, 70)
-    assert batch[1].shape == (70*lml.bs,)
-
-    as_text = [lml.dataset.vocab.itos[x] for x in batch[0][0]]
-    np.testing.assert_array_equal(as_text[:5], ["world", "hello", 'xxbos', 'project', 'cool'])
-
-def test_should_load_bi_lm():
-    path = untar_data(URLs.IMDB_SAMPLE)
-    df = text_df(['neg', 'pos'])
-
-    data = TextLMDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=0, text_cols=["text"], bs=2,
-                                   lm_type=contrib_data.LanguageModelType.BiLM)
-    lml = data.train_dl.dl
-    lml.data = lml.batchify(np.concatenate([lml.dataset.x.items[i] for i in range(len(lml.dataset))]))
-    batch = lml.get_batch(lml.data, 0, 70)
-
-    assert batch[0].shape == (lml.bs, 70, 2)
-    assert batch[1].shape == (70*lml.bs, 2)
-
-    as_text = [lml.dataset.vocab.itos[x] for x in batch[0][0, :, 0]]
-    np.testing.assert_array_equal(as_text[:7], "xxbos fast ai is a cool project".split())
-
-    as_text = [lml.dataset.vocab.itos[x] for x in batch[0][0, :, 1]]
-    np.testing.assert_array_equal(as_text[:5], ["world", "hello", 'xxbos', 'project', 'cool'])
-
-###################### NEW CODE
-
diff --git a/tests/test_text_train.py b/tests/test_text_train.py
deleted file mode 100644
index 7c3d894..0000000
--- a/tests/test_text_train.py
+++ /dev/null
@@ -1,103 +0,0 @@
-import pytest
-from fastai import *
-from fastai.text import *
-
-pytestmark = pytest.mark.integration
-
-print(sys.path)
-import fastai_contrib.data as contrib_data
-
-from fastai_contrib.learner import bilm_learner, accuracy_fwd, bilm_text_classifier_learner
-
-
-def read_file(fname):
-    texts = []
-    with open(fname, 'r') as f:
-        texts = f.readlines()
-    labels = [0] * len(texts)
-    df = pd.DataFrame({'labels':labels, 'texts':texts}, columns = ['labels', 'texts'])
-    return df
-
-def prep_human_numbers():
-    path = untar_data(URLs.HUMAN_NUMBERS)
-    df_trn = read_file(path/'train.txt')
-    df_val = read_file(path/'valid.txt')
-    return path, df_trn, df_val
-
-def manual_seed(seed=42):
-    torch.manual_seed(seed)
-    np.random.seed(seed)
-    if torch.cuda.is_available():
-        torch.cuda.manual_seed_all(seed)
-        torch.backends.cudnn.deterministic = True
-        torch.backends.cudnn.benchmark = False
-
-@pytest.fixture(scope="module")
-def learn():
-    path, df_trn, df_val = prep_human_numbers()
-    data = TextLMDataBunch.from_df(path, df_trn, df_val, tokenizer=Tokenizer(BaseTokenizer))
-    learn = language_model_learner(data, emb_sz=100, nl=1, drop_mult=0.1)
-    learn.fit_one_cycle(4, 5e-3)
-    return learn
-
-def text_df(n_labels):
-    data = []
-    texts = ["fast ai is a cool project", "hello world"] * 20
-    for ind, text in enumerate(texts):
-        sample = {}
-        for label in range(n_labels): sample[label] = ind%2
-        sample["text"] = text
-        data.append(sample)
-    df = pd.DataFrame(data)
-    return df
-
-###################### NEW CODE
-
-def test_val_loss(learn):
-    assert learn.validate()[1] > 0.5
-
-
-def test_bilm_classifier_loads_encoder():
-    n_labels=1
-    nl = 1
-    emb_sz = 100
-    path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data', 'tmp')
-    os.makedirs(path)
-    try:
-        df = text_df(n_labels=n_labels)
-        lmdf = df#[["text"]]
-        print(lmdf.head())
-        lmdata = TextLMDataBunch.from_df(path, lmdf, lmdf, tokenizer=Tokenizer(BaseTokenizer),
-                                       lm_type=contrib_data.LanguageModelType.BiLM)
-        learn = bilm_learner(lmdata, emb_sz=emb_sz, nl=nl, drop_mult=0.1, qrnn=False)
-        learn.save_encoder("enc")
-        data = TextClasDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=list(range(n_labels)), text_cols=["text"], bs=8)
-        classifier = bilm_text_classifier_learner(data, emb_sz=emb_sz, nl=nl, drop_mult=0.1, qrnn=False)
-        print(last_layer(classifier.model), )
-        classifier.load_encoder("enc")
-        classifier.fit(1)
-    finally:
-        shutil.rmtree(path)
-
-
-def test_bilm_lstm_can_be_trained():
-    manual_seed()
-    path, df_trn, df_val = prep_human_numbers()
-    data = TextLMDataBunch.from_df(path, df_trn, df_val, tokenizer=Tokenizer(BaseTokenizer),
-                                   lm_type = contrib_data.LanguageModelType.BiLM)
-
-    learn = bilm_learner(data, emb_sz=100, nl=1, drop_mult=0.1, qrnn=False)
-    learn.metrics = [accuracy_fwd]
-    learn.fit_one_cycle(2, 5e-3)
-    assert learn.validate()[1] > 0.3
-
-
-def test_bwdlm_lstm_can_be_trained():
-    manual_seed()
-    path, df_trn, df_val = prep_human_numbers()
-    data = TextLMDataBunch.from_df(path, df_trn, df_val, tokenizer=Tokenizer(BaseTokenizer),
-                                   lm_type = contrib_data.LanguageModelType.BwdLM)
-
-    learn = language_model_learner(data, emb_sz=100, nl=1, drop_mult=0.1, qrnn=False)
-    learn.fit_one_cycle(2, 5e-3)
-    assert learn.validate()[1] > 0.3

From e4d0529a3d076e5f8565781a61ee3ec8cdee21a7 Mon Sep 17 00:00:00 2001
From: "NAUSICAA\\Julian" <eisenjulian@gmail.com>
Date: Mon, 15 Apr 2019 14:41:08 -0300
Subject: [PATCH 25/32] Removing more bi-directional test

---
 tests/test_text_data.py  | 56 ----------------------------------------
 tests/test_text_train.py | 46 ---------------------------------
 2 files changed, 102 deletions(-)
 delete mode 100644 tests/test_text_data.py

diff --git a/tests/test_text_data.py b/tests/test_text_data.py
deleted file mode 100644
index 1b96079..0000000
--- a/tests/test_text_data.py
+++ /dev/null
@@ -1,56 +0,0 @@
-import pytest
-import fastai.text
-
-from fastai import *
-from fastai.text import *
-
-import fastai_contrib.data as contrib_data
-
-def text_df(labels):
-    data = []
-    texts = ["fast ai is a cool project", "hello world"] * 20
-    for ind, text in enumerate(texts):
-        sample = {}
-        sample["label"] = labels[ind%len(labels)]
-        sample["text"] = text
-        data.append(sample)
-    return pd.DataFrame(data)
-
-###################### UPDATED CODE
-def test_should_load_backwards_lm():
-    path = untar_data(URLs.IMDB_SAMPLE)
-    df = text_df(['neg','pos'])
-
-    data = TextLMDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=0, text_cols=["text"], bs=2,
-                                   lm_type=contrib_data.LanguageModelType.BwdLM)
-    lml = data.train_dl.dl
-    lml.data = lml.batchify(np.concatenate([lml.dataset.x.items[i] for i in range(len(lml.dataset))]))
-    batch = lml.get_batch(lml.data, 0, 70)
-
-    assert batch[0].shape == (lml.bs, 70)
-    assert batch[1].shape == (70*lml.bs,)
-
-    as_text = [lml.dataset.vocab.itos[x] for x in batch[0][0]]
-    np.testing.assert_array_equal(as_text[:5], ["world", "hello", 'xxbos', 'project', 'cool'])
-
-def test_should_load_bi_lm():
-    path = untar_data(URLs.IMDB_SAMPLE)
-    df = text_df(['neg', 'pos'])
-
-    data = TextLMDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=0, text_cols=["text"], bs=2,
-                                   lm_type=contrib_data.LanguageModelType.BiLM)
-    lml = data.train_dl.dl
-    lml.data = lml.batchify(np.concatenate([lml.dataset.x.items[i] for i in range(len(lml.dataset))]))
-    batch = lml.get_batch(lml.data, 0, 70)
-
-    assert batch[0].shape == (lml.bs, 70, 2)
-    assert batch[1].shape == (70*lml.bs, 2)
-
-    as_text = [lml.dataset.vocab.itos[x] for x in batch[0][0, :, 0]]
-    np.testing.assert_array_equal(as_text[:7], "xxbos fast ai is a cool project".split())
-
-    as_text = [lml.dataset.vocab.itos[x] for x in batch[0][0, :, 1]]
-    np.testing.assert_array_equal(as_text[:5], ["world", "hello", 'xxbos', 'project', 'cool'])
-
-###################### NEW CODE
-
diff --git a/tests/test_text_train.py b/tests/test_text_train.py
index ccc9bef..60fb1f9 100644
--- a/tests/test_text_train.py
+++ b/tests/test_text_train.py
@@ -55,49 +55,3 @@ def text_df(n_labels):
 
 def test_val_loss(learn):
     assert learn.validate()[1] > 0.5
-
-
-def test_bilm_classifier_loads_encoder():
-    n_labels=1
-    nl = 1
-    emb_sz = 100
-    path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data', 'tmp')
-    os.makedirs(path)
-    try:
-        df = text_df(n_labels=n_labels)
-        lmdf = df#[["text"]]
-        print(lmdf.head())
-        lmdata = TextLMDataBunch.from_df(path, lmdf, lmdf, tokenizer=Tokenizer(BaseTokenizer),
-                                       lm_type=contrib_data.LanguageModelType.BiLM)
-        learn = bilm_learner(lmdata, emb_sz=emb_sz, nl=nl, drop_mult=0.1, qrnn=False)
-        learn.save_encoder("enc")
-        data = TextClasDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=list(range(n_labels)), text_cols=["text"], bs=8)
-        classifier = bilm_text_classifier_learner(data, emb_sz=emb_sz, nl=nl, drop_mult=0.1, qrnn=False)
-        print(last_layer(classifier.model), )
-        classifier.load_encoder("enc")
-        classifier.fit(1)
-    finally:
-        shutil.rmtree(path)
-
-
-def test_bilm_lstm_can_be_trained():
-    manual_seed()
-    path, df_trn, df_val = prep_human_numbers()
-    data = TextLMDataBunch.from_df(path, df_trn, df_val, tokenizer=Tokenizer(BaseTokenizer),
-                                   lm_type = contrib_data.LanguageModelType.BiLM)
-
-    learn = bilm_learner(data, emb_sz=100, nl=1, drop_mult=0.1, qrnn=False)
-    learn.metrics = [accuracy_fwd]
-    learn.fit_one_cycle(2, 5e-3)
-    assert learn.validate()[1] > 0.3
-
-
-def test_bwdlm_lstm_can_be_trained():
-    manual_seed()
-    path, df_trn, df_val = prep_human_numbers()
-    data = TextLMDataBunch.from_df(path, df_trn, df_val, tokenizer=Tokenizer(BaseTokenizer),
-                                   lm_type = contrib_data.LanguageModelType.BwdLM)
-
-    learn = language_model_learner(data, AWD_LSTM, emb_sz=100, nl=1, drop_mult=0.1, qrnn=False)
-    learn.fit_one_cycle(2, 5e-3)
-    assert learn.validate()[1] > 0.3

From 2b7ec26f8b17b99321e28da2cec548f59d5179a0 Mon Sep 17 00:00:00 2001
From: "NAUSICAA\\Julian" <eisenjulian@gmail.com>
Date: Mon, 15 Apr 2019 15:00:46 -0300
Subject: [PATCH 26/32] Lower LM accuracy test threshold

---
 tests/test_text_train.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tests/test_text_train.py b/tests/test_text_train.py
index 60fb1f9..10ec42e 100644
--- a/tests/test_text_train.py
+++ b/tests/test_text_train.py
@@ -51,7 +51,5 @@ def text_df(n_labels):
     df = pd.DataFrame(data)
     return df
 
-###################### NEW CODE
-
 def test_val_loss(learn):
-    assert learn.validate()[1] > 0.5
+    assert learn.validate()[1] > 0.4

From 9f494b3ec6be72df06395ef4ea10dc9e0f0540b2 Mon Sep 17 00:00:00 2001
From: Piotr Czapla <github@piotrczapla.com>
Date: Fri, 19 Apr 2019 10:32:35 +0200
Subject: [PATCH 27/32] Remove old bilm code that wasn't working

---
 fastai_contrib/data.py    |  71 ----------------
 fastai_contrib/learner.py | 113 --------------------------
 fastai_contrib/models.py  | 166 --------------------------------------
 fastai_contrib/utils.py   |  13 ++-
 tests/test_end_to_end.py  |  45 -----------
 ulmfit/pretrain_lm.py     |  31 ++-----
 ulmfit/train_clas.py      |  10 +--
 7 files changed, 15 insertions(+), 434 deletions(-)
 delete mode 100644 fastai_contrib/data.py
 delete mode 100644 fastai_contrib/learner.py
 delete mode 100644 fastai_contrib/models.py

diff --git a/fastai_contrib/data.py b/fastai_contrib/data.py
deleted file mode 100644
index 5f6e732..0000000
--- a/fastai_contrib/data.py
+++ /dev/null
@@ -1,71 +0,0 @@
-"NLP data loading pipeline. Supports csv, folders, and preprocessed data."
-from fastai.text import *
-from fastai.torch_core import *
-from fastai.text.transform import *
-from fastai.basic_data import *
-from fastai.data_block import *
-
-#region Modified fastai classes
-
-LanguageModelType=Enum('LanguageModelType', 'FwdLM BwdLM BiLM')
-
-class LanguageModelLoader(): # copy of the original LanguageModelLoader
-    "Create a dataloader with bptt slightly changing."
-    def __init__(self, dataset:LabelList, bs:int=64, bptt:int=70,
-                 lm_type:LanguageModelType=LanguageModelType.FwdLM, shuffle:bool=False,
-                 max_len:int=25, p_bptt:int=0.95):
-        self.dataset,self.bs,self.bptt,self.lm_type,self.shuffle, self.p_bptt = dataset,bs,bptt,lm_type,shuffle,p_bptt
-        self.first,self.i,self.iter = True,0,0
-        self.n = len(np.concatenate(dataset.x.items)) // self.bs if len(dataset.x.items) > 0 else 0
-        self.max_len,self.num_workers = max_len,0
-        self.init_kwargs = dict(bs=bs, bptt=bptt, lm_type=lm_type, shuffle=shuffle, max_len=max_len, p_bptt=p_bptt)
-
-    def __iter__(self):
-        if getattr(self.dataset, 'item', None) is not None:
-            yield LongTensor(getattr(self.dataset, 'item'))[None],LongTensor([0])
-        idx = np.random.permutation(len(self.dataset)) if self.shuffle else range(len(self.dataset))
-        data = self.batchify(np.concatenate([self.dataset.x.items[i] for i in idx]))
-
-        pos, itr = 0,0
-        while pos < self.n-1 and itr<len(self):
-            if self.first and pos == 0: self.first,seq_len = False,self.bptt + self.max_len
-            else:
-                bptt = self.bptt if np.random.random() < self.p_bptt else self.bptt / 2.
-                seq_len = max(5, int(np.random.normal(bptt, 5)))
-                seq_len = min(seq_len, self.bptt + self.max_len)
-            res = self.get_batch(data, pos, seq_len)
-            pos += seq_len
-            itr += 1
-            yield res
-
-    def __len__(self) -> int: return int(math.ceil((self.n-1) / self.bptt)) # so that it is always at least 1
-    def __getattr__(self,k:str)->Any: return getattr(self.dataset, k)
-
-    @property
-    def batch_size(self): return self.bs
-    @batch_size.setter
-    def batch_size(self, v): self.bs = v
-
-    def batchify(self, data:np.ndarray) -> LongTensor:
-        "Split the corpus `data` in batches."
-        nb = data.shape[0] // self.bs
-        data = np.array(data[:nb*self.bs]).reshape(self.bs, -1)
-        if self.lm_type == LanguageModelType.BwdLM: data = data[:,::-1].copy()
-        elif self.lm_type == LanguageModelType.BiLM: data = np.stack([data, data[:,::-1].copy()], axis=2)
-        return LongTensor(data)
-
-    def get_batch(self, data:LongTensor,  i:int, seq_len:int) -> Tuple[LongTensor, LongTensor]:
-        "Create a batch at `i` of a given `seq_len`."
-        seq_len = min(seq_len, data.shape[1] - 1 - i)
-        x = data[:,i:i+seq_len]
-        y = data[:,i+1:i+1+seq_len]
-        y = y.contiguous().view(-1, 2) if self.lm_type == LanguageModelType.BiLM else y.contiguous().view(-1)
-        return x,y
-
-#endregion
-#region Replaces fastai classes
-
-import fastai.text.data
-fastai.text.data.LanguageModelLoader = LanguageModelLoader # Replace original LanguageModelLoader with new verion
-
-#endregion
\ No newline at end of file
diff --git a/fastai_contrib/learner.py b/fastai_contrib/learner.py
deleted file mode 100644
index 67e79cb..0000000
--- a/fastai_contrib/learner.py
+++ /dev/null
@@ -1,113 +0,0 @@
-from torch.nn import CrossEntropyLoss
-
-from fastai import *
-from fastai.text import *
-
-#region New code
-from fastai_contrib.models import *
-
-
-def bilm_learner(data:DataBunch, bptt:int=70, emb_sz:int=400, nh:int=1150, nl:int=3, pad_token:int=1,
-                  drop_mult:float=1., tie_weights:bool=True, bias:bool=True, qrnn:bool=False, pretrained_model=None,
-                  pretrained_fnames:OptStrTuple=None, **kwargs) -> 'LanguageLearner':
-    "Create a `Learner` with a language model."
-    dps = default_dropout['language'] * drop_mult
-    vocab_size = len(data.vocab.itos)
-    model = get_bilm(vocab_size, emb_sz, nh, nl, pad_token, input_p=dps[0], output_p=dps[1],
-                weight_p=dps[2], embed_p=dps[3], hidden_p=dps[4], tie_weights=tie_weights, bias=bias, qrnn=qrnn)
-    learn = LanguageLearner(data, model, bptt, split_func=bilm_split, **kwargs)
-    if pretrained_model is not None:
-        model_path = untar_data(pretrained_model, data=False)
-        fnames = [list(model_path.glob(f'*.{ext}'))[0] for ext in ['pth', 'pkl']]
-        learn.load_pretrained(*fnames)
-        learn.freeze()
-    if pretrained_fnames is not None:
-        fnames = [learn.path/learn.model_dir/f'{fn}.{ext}' for fn,ext in zip(pretrained_fnames, ['pth', 'pkl'])]
-        learn.load_pretrained(*fnames)
-        learn.freeze()
-    learn.loss_func = CrossEntropyLoss()  # I'm not sure why fast ai is using CrossEntropyFlat but it breaks bilm
-    return learn
-
-def bilm_text_classifier_learner(data: DataBunch, bptt: int = 70, max_len: int = 70 * 20, emb_sz: int = 400,
-                            nh: int = 1150, nl: int = 3,
-                            lin_ftrs: Collection[int] = None, ps: Collection[float] = None, pad_token: int = 1,
-                            drop_mult: float = 1., qrnn: bool = False, bicls_head:str='BiPoolingLinearClassifier',  **kwargs) -> 'TextClassifierLearner':
-    "Create a RNN classifier."
-    dps = default_dropout['classifier'] * drop_mult
-    if lin_ftrs is None: lin_ftrs = [50]
-    if ps is None:  ps = [0.1]
-    ds = data.train_ds
-    vocab_size, n_class = len(data.vocab.itos), data.c
-    if bicls_head == 'BiPoolingLinearClassifier':
-        count = 3*2
-    else:
-        count = 3
-    layers = [emb_sz * count] + lin_ftrs + [n_class]
-    ps = [dps[4]] + ps
-    model = get_birnn_classifier(bptt, max_len, n_class, vocab_size, emb_sz, nh, nl, pad_token,
-                               layers, ps, input_p=dps[0], weight_p=dps[1], embed_p=dps[2], hidden_p=dps[3],
-                               qrnn=qrnn, bicls_head=bicls_head)
-    learn = RNNLearner(data, model, bptt, split_func=birnn_classifier_split, **kwargs)
-    return learn
-
-def bilm_split(model:nn.Module) -> List[nn.Module]:
-    "Split a RNN `model` in groups for differential learning rates."
-
-    return [f+b for f,b in zip(lm_split(model.fwd_lm),lm_split(model.bwd_lm))]
-
-def birnn_classifier_split(model:nn.Module) -> List[nn.Module]:
-    "Split a RNN `model` in groups for differential learning rates."
-    f_rnn,b_rnn = model[0].fwd_lm,model[0].bwd_lm
-    groups = [[f_rnn.encoder, f_rnn.encoder_dp,b_rnn.encoder, b_rnn.encoder_dp]]
-    groups += [a for a in zip(f_rnn.rnns, f_rnn.hidden_dps, b_rnn.rnns, b_rnn.hidden_dps, )]
-    groups.append([model[1]])
-    return groups
-
-def accuracy_fwd(input, targs):
-    return accuracy(input[...,0], targs[...,0])
-
-def accuracy_bwd(input, targs):
-    return accuracy(input[...,1], targs[...,1])
-
-
-#endregion
-#region Modified fastai code
-
-def convert_weights(wgts:Weights, stoi_wgts:Dict[str,int], itos_new:Collection[str]) -> Weights:
-    "Convert the model weights to go with a new vocabulary."
-    if 'fwd_lm.0.encoder.weight' in wgts: #todo share embedding matrix computation
-        wgts = convert_weights_with_prefix(wgts, stoi_wgts, itos_new, prefix='fwd_lm.')
-        return convert_weights_with_prefix(wgts, stoi_wgts, itos_new, prefix='bwd_lm.')
-    else:
-        return convert_weights_with_prefix(wgts, stoi_wgts, itos_new, prefix='')
-
-def convert_weights_with_prefix(wgts:Weights, stoi_wgts:Dict[str,int], itos_new:Collection[str], prefix='') -> Weights:
-    "Convert the model weights to go with a new vocabulary."
-    if 'model' in wgts:
-        wgts['model'] = convert_weights_with_prefix(wgts['model'], stoi_wgts, itos_new, prefix)
-    else:
-        dec_bias, enc_wgts = wgts[prefix+'1.decoder.bias'], wgts[prefix+'0.encoder.weight']
-        bias_m, wgts_m = dec_bias.mean(0), enc_wgts.mean(0)
-        new_w = enc_wgts.new_zeros((len(itos_new),enc_wgts.size(1))).zero_()
-        new_b = dec_bias.new_zeros((len(itos_new),)).zero_()
-        unk_tokens=[]
-        for i,w in enumerate(itos_new):
-            r = stoi_wgts[w] if w in stoi_wgts else -1
-            if r < 0:
-                unk_tokens.append(w)
-            new_w[i] = enc_wgts[r] if r>=0 else wgts_m
-            new_b[i] = dec_bias[r] if r>=0 else bias_m
-        print(f"Unknown tokens {len(unk_tokens)}, first 100: {unk_tokens[:100]}")
-        wgts[prefix+'0.encoder.weight'] = new_w
-        wgts[prefix+'0.encoder_dp.emb.weight'] = new_w.clone()
-        wgts[prefix+'1.decoder.weight'] = new_w.clone()
-        wgts[prefix+'1.decoder.bias'] = new_b
-    return wgts
-
-#endregion
-#region Replace code in fastai
-
-import fastai.text.learner
-fastai.text.learner.convert_weights = convert_weights
-
-#endregion
\ No newline at end of file
diff --git a/fastai_contrib/models.py b/fastai_contrib/models.py
deleted file mode 100644
index fbed679..0000000
--- a/fastai_contrib/models.py
+++ /dev/null
@@ -1,166 +0,0 @@
-from fastai.torch_core import *
-from fastai.layers import *
-from fastai.text.models import *
-
-#region New code
-
-class BiLMModel(nn.Module):
-
-    def __init__(self, fwd_lm:nn.Module, bwd_lm:nn.Module, squash_bs_sl=False):
-        super().__init__()
-        self.fwd_lm = fwd_lm
-        self.bwd_lm = bwd_lm
-        self.squash_bs_sl = squash_bs_sl
-
-    def __getitem__(self, idx):
-        return BiLMModel(self.fwd_lm[idx], self.bwd_lm[idx])
-
-    def __len__(self):
-        return len(self.fwd_lm)
-
-    def stack(self, fwd_o, bwd_o):
-        if is_listy(fwd_o):
-            return [self.stack(f, b) for f,b in zip(fwd_o,bwd_o)]
-        else:
-            return torch.stack([fwd_o, bwd_o], dim=len(fwd_o.shape))
-
-    def forward(self, input):
-        if len(input.shape) == 3: # sl, bs, tracks
-            f = input[..., 0]
-            b = input[..., 1]
-        elif len(input.shape) == 2: # sl, bs - support during classification mode
-            f = input
-            b = torch.flip(input, [1]) # todo test if we are duplicating the backward pass correctly
-        else:
-            raise AttributeError(f"Inorrect size of input, {input.shape}")
-        fwd_o = self.fwd_lm(f)
-        bwd_o = self.bwd_lm(b)
-
-        outs = self.stack(fwd_o, bwd_o)
-        if self.squash_bs_sl:
-            o = outs[0]
-            o = o.view(o.shape[0]*o.shape[1],o.shape[2],o.shape[3])
-            outs[0] = o
-        return outs
-
-    def reset(self):
-        "Reset the hidden states of underlaying lms."
-        self.fwd_lm.reset()
-        self.bwd_lm.reset()
-
-class MultiBatchBiLMModel(BiLMModel):
-    "Create a RNNCore module that can process a full sentence."
-
-    def __init__(self, bptt:int, max_seq:int, *args, **kwargs):
-        self.max_seq,self.bptt = max_seq,bptt
-        super().__init__(*args, **kwargs)
-
-    def concat(self, arrs:Collection[Tensor])->Tensor:
-        "Concatenate the `arrs` along the batch dimension."
-        return [torch.cat([l[si] for l in arrs], dim=1) for si in range_of(arrs[0])]
-
-    def forward(self, input:LongTensor)->Tuple[Tensor,Tensor]:
-        bs,sl = input.size()
-        self.reset()
-        raw_outputs, outputs = [],[]
-        for i in range(0, sl, self.bptt):
-            r, o = super().forward(input[:,i: min(i+self.bptt, sl)])
-            if i>(sl-self.max_seq):
-                raw_outputs.append(r)
-                outputs.append(o)
-        return self.concat(raw_outputs), self.concat(outputs)
-#PoolingLinearClassifier
-class BiPoolingLinearClassifier(nn.Module):
-    "Create a linear classifier with pooling."
-
-    def forward(self, input:Tuple[Tensor,Tensor])->Tuple[Tensor,Tensor,Tensor]:
-        raw_outputs, outputs = input
-        output = outputs[-1]
-        if len(output.size()) == 3:
-            return super().forward(input)
-        elif len(output.size()) == 4:
-            bs, sl, em_sz, passes = output.size()
-
-            f_avgpool = self.pool(output[...,0], bs, False)
-            f_mxpool = self.pool(output[...,0], bs, True)
-            b_avgpool = self.pool(output[..., 1], bs, False)
-            b_mxpool = self.pool(output[..., 1], bs, True)
-            x = torch.cat([output[:,-1,..., 0], f_mxpool, f_avgpool,
-                           output[:,-1,..., 1], b_mxpool, b_avgpool,], 1)
-            x = self.layers(x)
-            return x, raw_outputs, outputs
-
-
-class AvgPoolingLinearClassifier(nn.Module):
-    "Create a linear classifier with pooling."
-
-    def __init__(self, layers:Collection[int], drops:Collection[float]):
-        super().__init__()
-        mod_layers = []
-        activs = [nn.ReLU(inplace=True)] * (len(layers) - 2) + [None]
-        for n_in,n_out,p,actn in zip(layers[:-1],layers[1:], drops, activs):
-            mod_layers += bn_drop_lin(n_in, n_out, p=p, actn=actn)
-        self.layers = nn.Sequential(*mod_layers)
-
-    def pool(self, x:Tensor, bs:int, is_max:bool):
-        "Pool the tensor along the seq_len dimension."
-        f = F.adaptive_max_pool1d if is_max else F.adaptive_avg_pool1d
-        return f(x.permute(1,2,0), (1,)).view(bs,-1)
-
-    def forward(self, input:Tuple[Tensor,Tensor])->Tuple[Tensor,Tensor,Tensor]:
-        raw_outputs, outputs = input
-        output = outputs[-1]
-        if len(output.size()) == 3:
-            sl,bs,_ = output.size()
-            avgpool = self.pool(output, bs, False)
-            mxpool = self.pool(output, bs, True)
-            x = torch.cat([output[-1], mxpool, avgpool], 1)
-            x = self.layers(x)
-            return x, raw_outputs, outputs
-        elif len(output.size()) == 4:
-            sl, bs, em_sz, passes = output.size()
-
-            avgpool = (self.pool(output[...,0], bs, False) + self.pool(output[..., 1], bs, False))/2
-            mxpool = (self.pool(output[...,0], bs, True) +self.pool(output[..., 1], bs, True))/2
-            x = torch.cat([(output[-1][..., 0]+output[-1][..., 1])/2, mxpool, avgpool], 1)
-            x = self.layers(x)
-            return x, raw_outputs, outputs
-
-
-def get_bilm(vocab_sz:int, emb_sz:int, n_hid:int, n_layers:int, pad_token:int, tie_weights:bool=True,
-                       qrnn:bool=False, bias:bool=True, bidir:bool=False, output_p:float=0.4, hidden_p:float=0.2, input_p:float=0.6,
-                       embed_p:float=0.1, weight_p:float=0.5)->nn.Module:
-    "Create a two AWD-LSTM one for each direction "
-    fwd_rnn_enc = RNNCore(vocab_sz, emb_sz, n_hid=n_hid, n_layers=n_layers, pad_token=pad_token, qrnn=qrnn, bidir=bidir,
-                          hidden_p=hidden_p, input_p=input_p, embed_p=embed_p, weight_p=weight_p)
-    bwd_rnn_enc = RNNCore(vocab_sz, emb_sz, n_hid=n_hid, n_layers=n_layers, pad_token=pad_token, qrnn=qrnn, bidir=bidir,
-                          hidden_p=hidden_p, input_p=input_p, embed_p=embed_p, weight_p=weight_p)
-    enc = None
-    if tie_weights:
-        enc = fwd_rnn_enc.encoder
-        fwd_rnn_enc.encoder.weight = enc.weight
-        bwd_rnn_enc.encoder.weight = enc.weight
-
-    return BiLMModel(
-        fwd_lm=SequentialRNN(fwd_rnn_enc, LinearDecoder(vocab_sz, emb_sz, output_p, tie_encoder=enc, bias=bias)),
-        bwd_lm=SequentialRNN(bwd_rnn_enc, LinearDecoder(vocab_sz, emb_sz, output_p, tie_encoder=enc, bias=bias)),
-        squash_bs_sl=True)
-
-def get_birnn_classifier(bptt:int, max_seq:int, n_class:int, vocab_sz:int, emb_sz:int, n_hid:int, n_layers:int,
-                       pad_token:int, layers:Collection[int], drops:Collection[float], bidir:bool=False, qrnn:bool=False,
-                       hidden_p:float=0.2, input_p:float=0.6, embed_p:float=0.1, weight_p:float=0.5, bicls_head:str='BiPoolingLinearClassifier')->nn.Module:
-    "Create a RNN classifier model."
-    fwd_rnn_enc = MultiBatchRNNCore(bptt, max_seq, vocab_sz, emb_sz, n_hid, n_layers, pad_token=pad_token, bidir=bidir,
-                      qrnn=qrnn, hidden_p=hidden_p, input_p=input_p, embed_p=embed_p, weight_p=weight_p)
-    bwd_rnn_enc = MultiBatchRNNCore(bptt, max_seq, vocab_sz, emb_sz, n_hid, n_layers, pad_token=pad_token, bidir=bidir,
-                                qrnn=qrnn, hidden_p=hidden_p, input_p=input_p, embed_p=embed_p, weight_p=weight_p)
-
-    head = BiPoolingLinearClassifier
-    if bicls_head == 'BiPoolingLinearClassifier': head = BiPoolingLinearClassifier
-    elif bicls_head == 'AvgPoolingLinearClassifier': head = AvgPoolingLinearClassifier
-
-    model = SequentialRNN(BiLMModel(fwd_rnn_enc, bwd_rnn_enc), head(layers, drops))
-    model.reset()
-    return model
-
-#endregion
\ No newline at end of file
diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py
index d0c93ea..1441ab0 100644
--- a/fastai_contrib/utils.py
+++ b/fastai_contrib/utils.py
@@ -1,17 +1,16 @@
 """
 Utility methods for data processing.
 """
-import fire
-from fastai import *
-from fastai.text import *
-
-import shutil
 import pathlib
+import shutil
 import tarfile
-from sklearn import model_selection
-from sacremoses import MosesTokenizer
 from typing import Dict, Tuple, List
 
+import fire
+from sacremoses import MosesTokenizer
+
+from fastai.text import *
+
 EOS = 'xxeos' # fastai does not use eos, but we do
 SEP = 'xxsep' # special separator token for NLI
 
diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py
index 127c8fa..5c25258 100644
--- a/tests/test_end_to_end.py
+++ b/tests/test_end_to_end.py
@@ -136,51 +136,6 @@ def test_ulmfit_fastai_end_to_end_label_smoothing():
     exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir)
     exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, label_smoothing_eps=0.1 )
 
-
-def test_ulmfit_fastai_bidir_end_to_end():
-    """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset.
-    """
-    test_data, wt2 = get_test_data()
-    lm_name = 'end-to-end-test-fastai'
-
-    exp = ulmfit.pretrain_lm.LMHyperParams(
-        dataset_path=wt2,
-        lang='en',
-        cuda_id=cuda_id,
-        qrnn=False,
-        bidir=True,
-        tokenizer='f',
-        max_vocab=100,
-        name=lm_name,
-    )
-    exp.train_lm(num_epochs=1, bs=2)
-    exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(str(test_data / 'imdb'), str(exp.model_dir))
-    exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, )
-
-def test_ulmfit_moses_fa_bidir_end_to_end():
-    """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset.
-    """
-    test_data, wt2 = get_test_data()
-    lm_name = 'end-to-end-test-fastai'
-
-    exp = ulmfit.pretrain_lm.LMHyperParams(
-        dataset_path=wt2,
-        lang='en',
-        cuda_id=cuda_id,
-        qrnn=False,
-        bidir=True,
-        tokenizer='vf',
-        max_vocab=100,
-        name=lm_name,
-    )
-    exp.train_lm(num_epochs=1, bs=2)
-    exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir)
-    exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, )
-
-# def test_classification_model_work_with_different_dropmul():
-#     learn = self.create_cls_learner(data_clas, drop_mult=0.1)
-#     learn = self.create_cls_learner(data_clas, drop_mult=0.0)
-
 def test_ulmfit_sentencepiece_end_to_end():
     """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset.
     """
diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py
index 94efd8a..6b0b760 100644
--- a/ulmfit/pretrain_lm.py
+++ b/ulmfit/pretrain_lm.py
@@ -4,25 +4,14 @@ expected to have been tokenized with Moses and processed with `postprocess_wikit
 That is, the data is expected to be white-space separated and numbers are expected
 to be split.
 """
-from dataclasses import InitVar
 
-import fastai
 import fire
 
-from fastai import *
-from fastai.callbacks import CSVLogger, SaveModelCallback
+from fastai.callbacks import CSVLogger
 from fastai.text import *
-import torch
-from fastai_contrib.utils import read_file, read_whitespace_file, \
-    validate, PAD, UNK, get_sentencepiece, read_clas_data, TRN, VAL, TST, PAD_TOKEN_ID, \
+from fastai_contrib.utils import read_whitespace_file, \
+    validate, UNK, get_sentencepiece, PAD_TOKEN_ID, \
     replace_std_toks, MosesPreprocessingFunc
-from fastai_contrib.learner import bilm_learner, accuracy_fwd, accuracy_bwd, bilm_text_classifier_learner
-import pickle
-
-from pathlib import Path
-
-from collections import Counter
-import fastai_contrib.data as contrib_data
 
 LM_BEST = "lm_best"
 ENC_BEST = "enc_best"
@@ -62,9 +51,9 @@ def json_load(f):
 
 @dataclass
 class LMHyperParams:
-    dataset_path: str # data_dir
+    dataset_path: Union[str, Path] # data_dir
 
-    base_lm_path: str = None
+    base_lm_path: Union[str, Path] = None
     backwards: str = False
     bidir: bool =False
     qrnn: bool = True
@@ -131,15 +120,6 @@ class LMHyperParams:
     @property
     def pretrained_fnames(self): return [self.base_lm_path / LM_BEST, self.base_lm_path / '../itos'] if self.base_lm_path else None
 
-    @property
-    def lm_type(self):
-        if self.bidir:
-            return contrib_data.LanguageModelType.BiLM
-        if self.backwards:
-            return contrib_data.LanguageModelType.BwdLM
-        else:
-            return contrib_data.LanguageModelType.FwdLM
-
     def tokenizer_to_fastai_args(self, sp_data_func, use_moses):
         moses_preproc = [MosesPreprocessingFunc(self.lang)] if use_moses else []
         if self.tokenizer is Tokenizers.SUBWORD or self.tokenizer is Tokenizers.BROKENSUBWORD:
@@ -234,7 +214,6 @@ class LMHyperParams:
             learn.freeze()
         # compared to standard Adam, we set beta_1 to 0.8
         learn.opt_fn = partial(optim.Adam, betas=(0.8, 0.99))
-        learn.metrics = [accuracy_fwd, accuracy_bwd] if self.bidir else [accuracy]
         learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/lm-history"),
                                # partial(SaveModelCallback, every='improvement', name='lm') disabled due to Memory issues
                                ]
diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py
index b230387..2fac573 100644
--- a/ulmfit/train_clas.py
+++ b/ulmfit/train_clas.py
@@ -3,14 +3,12 @@ Train a classifier on top of a language model trained with `pretrain_lm.py`.
 Optionally fine-tune LM before.
 """
 
-from fastai.callbacks import CSVLogger
-from fastai.text import *
-
-from fastai_contrib.utils import PAD_TOKEN_ID
-
 import fire
 
-from ulmfit.pretrain_lm import LMHyperParams, ENC_BEST, json_save
+from fastai.callbacks import CSVLogger
+from fastai.text import *
+from fastai_contrib.utils import PAD_TOKEN_ID
+from ulmfit.pretrain_lm import LMHyperParams, ENC_BEST
 
 
 class CLSHyperParams(LMHyperParams):

From f9032ad84654a89189f68176938efe456dda4b12 Mon Sep 17 00:00:00 2001
From: Piotr Czapla <github@piotrczapla.com>
Date: Sat, 20 Apr 2019 14:28:45 +0200
Subject: [PATCH 28/32] Update to README.md

to show the newest changes to the command line.
---
 README.md | 23 +++++++++--------------
 1 file changed, 9 insertions(+), 14 deletions(-)

diff --git a/README.md b/README.md
index 41cd8b6..c29e00f 100644
--- a/README.md
+++ b/README.md
@@ -1,14 +1,19 @@
 # ulmfit-multilingual
-Temporary repository used for collaboration on application of for multiple languages.
+Repository used for collaboration on application of ulmfit for multiple languages, it helps with pertraining and uses the 
+fastai v1 . (The version in n-waves/fastai:ulmfit_multilingual)
 
 # How to train classifier
 
 ```
-$ python -m ulmfit lm --dataset-path data/wiki/wikitext-103 --bidir=False --qrnn=False --tokenizer=vf --name 'bs40' --bs=40 --cuda-id=0  -  train 20 --drop-mult=0.9
+$ LANG=en
+$ python -m ulmfit lm --dataset-path data/wiki/${LANG}-100 --tokenizer='f' --nl 3 --name 'orig' --max-vocab 60000 \ 
+        --lang ${LANG} --qrnn=False - train 10 --bs=50 --drop_mult=0  --label-smoothing-eps=0.0
 ...
-Model dir: data/wiki/wikitext-103/models/vf60k/lstm_bs40.m
+Model name: data/wiki/en-100/models/f60k/lstm_orig.m
 ...
-$ python -m ulmfit cls --dataset-path data/imdb --base-lm-path data/wiki/wikitext-103/models/vf60k/lstm_bs40.m - train 20   
+
+$ python -m ulmfit cls --dataset-path data/imdb --base-lm-path  data/wiki/${LANG}-100/models/f60k/lstm_orig.m  \
+        --lang=${LANG} --name orig - train 20 --bs 18 --num-cls-epochs=4 --lr_sched=1cycle --label-smoothing-eps=0.1   
 ```
  
 
@@ -79,16 +84,6 @@ Switched to a new branch 'ulmfit_multilingual'
 $ git push --set-upstream n-waves ulmfit_multilingual  # to automatically push ulmfit_multilingual branch to the n-waves repo
 ```
 
-## Repo structure
-
-- `fastai_contrib`  -- anything that can be ported to fastai once we finish the project like:  NLI models, Sentence Piece tok.,
-- `ulmfit`  
-    - `data`  -- scripts to fetch and prepare data: wikipedia, xnli, classification data sets  
-    - `lm` -- scripts to train language models
-    - `bilm` -- scripts to train biLM ELMo style, Bert style
-    - `class`  -- scripts to test classifiers on multiple languages
-    - `xnli` -- scripts to test nli 
-
 
 ## Running tests
 

From f5306b406426530e4df2c250a6b5598b80ba621c Mon Sep 17 00:00:00 2001
From: Piotr Czapla <github@piotrczapla.com>
Date: Sat, 20 Apr 2019 14:34:26 +0200
Subject: [PATCH 29/32] Add a few examples of commands avaliable in
 ulmfit-multilingual

---
 README.md          | 15 ++++++++++++++-
 prepare_poleval.py |  2 ++
 2 files changed, 16 insertions(+), 1 deletion(-)
 create mode 100644 prepare_poleval.py

diff --git a/README.md b/README.md
index c29e00f..bd205d4 100644
--- a/README.md
+++ b/README.md
@@ -15,8 +15,21 @@ Model name: data/wiki/en-100/models/f60k/lstm_orig.m
 $ python -m ulmfit cls --dataset-path data/imdb --base-lm-path  data/wiki/${LANG}-100/models/f60k/lstm_orig.m  \
         --lang=${LANG} --name orig - train 20 --bs 18 --num-cls-epochs=4 --lr_sched=1cycle --label-smoothing-eps=0.1   
 ```
- 
 
+You can re-evaluate classifiers by running
+```
+python -m ulmfit eval --glob="imdb/models/*/lstm_*.m"
+```
+
+The same command can be used to quickly trian multiple classifiers, by adding the `--name` parameter:
+```
+python -m ulmfit eval --glob="imdb/models/*/lstm_nl3.m" --name "nl3-my-test1" --num-cls-epochs 4 --label-smoothing-eps=0.1 --lr_sched=1cycle
+``` 
+
+To create a tar with model simply run 
+```
+python -m ulmfit tar data/imdb/models/f60k/lstm_nl3.m
+```
 
 ## data directory strucutre
 
diff --git a/prepare_poleval.py b/prepare_poleval.py
new file mode 100644
index 0000000..6706e59
--- /dev/null
+++ b/prepare_poleval.py
@@ -0,0 +1,2 @@
+
+# https://storage.googleapis.com/reddit_comments_polish/comments.csv.gz

From 616bec18dc341090d57f2bb0a419af04584a7564 Mon Sep 17 00:00:00 2001
From: Cahya Wirawan <Cahya.Wirawan@gmail.com>
Date: Tue, 30 Apr 2019 11:48:14 +0200
Subject: [PATCH 30/32] Added an option to set the minimal limit of tokens per
 article

---
 ulmfit/create_wikitext.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/ulmfit/create_wikitext.py b/ulmfit/create_wikitext.py
index 30073d4..172bec4 100644
--- a/ulmfit/create_wikitext.py
+++ b/ulmfit/create_wikitext.py
@@ -27,7 +27,7 @@ def get_texts(root):
                     yield (f"={title}=\n"+text)
 
 
-def write_wikitext(file_path, text_iter, mt, num_tokens, mode='w'):
+def write_wikitext(file_path, text_iter, mt, num_tokens, mode='w', num_tokens_article_min=100):
     total_num_tokens = 0
     print(f'Writing to {file_path}...')
     i = 0
@@ -49,8 +49,8 @@ def write_wikitext(file_path, text_iter, mt, num_tokens, mode='w'):
                 # calculate length based on tokens; add 1 for newline
                 num_tokens_article += len(tokens) + 1
 
-            if num_tokens_article < 100:
-                # only use articles that have at least 100 tokens
+            if num_tokens_article < num_tokens_article_min:
+                # only use articles that have at least num_tokens_article_min tokens
                 continue
 
             for tokenized in tokenized_paragraphs:
@@ -87,7 +87,7 @@ def main(args):
     token_nums = [2000000, 200000, 200000]
     for split, token_num in zip(splits, token_nums):
         sml_file_path = sml_wiki / f'{args.lang}.wiki.{split}.tokens'
-        write_wikitext(sml_file_path, text_iter, mt, token_num)
+        write_wikitext(sml_file_path, text_iter, mt, token_num, num_tokens_article_min=args.tokens_min)
         lrg_file_path = lrg_wiki / f'{args.lang}.wiki.{split}.tokens'
         all_file_path = all_wiki / f'{args.lang}.wiki.{split}.tokens'
         # copy the content of the small file to the large file
@@ -97,10 +97,10 @@ def main(args):
 
     # add the new articles to the existing ones
     lrg_wiki_train = lrg_wiki / f'{args.lang}.wiki.train.tokens'
-    write_wikitext(lrg_wiki_train, text_iter, mt, 98000000, mode='a')
+    write_wikitext(lrg_wiki_train, text_iter, mt, 98000000, mode='a', num_tokens_article_min=args.tokens_min)
     all_wiki_train = all_wiki / f'{args.lang}.wiki.train.tokens'
     copyfile(lrg_wiki_train, all_wiki_train)
-    write_wikitext(all_wiki_train, text_iter, mt,  None, mode='a')
+    write_wikitext(all_wiki_train, text_iter, mt,  None, mode='a', num_tokens_article_min=args.tokens_min)
 
 if __name__ == '__main__':
 
@@ -115,5 +115,8 @@ if __name__ == '__main__':
     parser.add_argument('-l', '--lang', required=True,
                         help='the iso code of the language of the Wikipedia '
                              'documents, e.g. en, fr, de, etc.')
+    parser.add_argument('-t', '--tokens_min', required=False, type=int, default=100,
+                        help='the minimal number of tokens in an article')
     args = parser.parse_args()
     main(args)
+

From 66341801c0ee5f1249cf46e400907a0fb756fae8 Mon Sep 17 00:00:00 2001
From: Cahya Wirawan <Cahya.Wirawan@gmail.com>
Date: Wed, 11 Sep 2019 12:49:58 +0200
Subject: [PATCH 31/32] added question of minimal tokens/article in to
 prepare_wiki.sh changed 'fname' to 'file' in load_data function to match the
 current fastai version

---
 prepare_wiki.sh       | 9 ++++++++-
 ulmfit/pretrain_lm.py | 2 +-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/prepare_wiki.sh b/prepare_wiki.sh
index a43a868..d579275 100755
--- a/prepare_wiki.sh
+++ b/prepare_wiki.sh
@@ -13,6 +13,13 @@ else
 fi
 echo "Chosen language: ""$LANG"
 
+if [ "$2" == "" ] ; then
+    read -p "Enter the minimal tokens per articles [100]: " tokens_min
+    TOKENS_MIN=${tokens_min:-100}
+else
+    TOKENS_MIN="$2"
+fi
+
 DUMP_DIR="${ROOT}/wiki_dumps"
 EXTR_DIR="${ROOT}/wiki_extr"
 WIKI_DIR="${ROOT}/wiki"
@@ -46,7 +53,7 @@ else
   echo "${EXTR_PATH} already exists. Skipping extraction."
 fi
 
-python -m ulmfit.create_wikitext -i "${EXTR_PATH}"  -l "${LANG}" -o "${WIKI_DIR}"
+python -m ulmfit.create_wikitext -i "${EXTR_PATH}"  -l "${LANG}" -o "${WIKI_DIR}" -t "${TOKENS_MIN}"
 
 python -m ulmfit.postprocess_wikitext "${WIKI_DIR}/${LANG}-2" $LANG
 python -m ulmfit.postprocess_wikitext "${WIKI_DIR}/${LANG}-100" $LANG
diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py
index 6b0b760..5bd55cf 100644
--- a/ulmfit/pretrain_lm.py
+++ b/ulmfit/pretrain_lm.py
@@ -268,7 +268,7 @@ class LMHyperParams:
         if (bunch_path / 'itos.pkl').exists():
             data = bunch_class.load(self.cache_dir, name, bs=bs)
         elif bunch_path.exists():
-            data = load_data(self.cache_dir, fname=name, bs=bs)
+            data = load_data(self.cache_dir, file=name, bs=bs)
         else:
             print(f"Running tokenization {name}...")
             data = bunch_class.from_df(path=self.cache_dir,

From c9cdf09d5ba783ff11cf7af362e401ea265240b6 Mon Sep 17 00:00:00 2001
From: Julian Eisenschlos <eisenjulian@users.noreply.github.com>
Date: Tue, 15 Oct 2019 19:59:50 +0200
Subject: [PATCH 32/32] Create LICENCE

---
 LICENCE | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 LICENCE

diff --git a/LICENCE b/LICENCE
new file mode 100644
index 0000000..5285f42
--- /dev/null
+++ b/LICENCE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.