Fix use_moses=True for mldoc so that it is identical to wiki with uses_moses=False

The issue was that Moses was executed after pre_rules when use_moses = True, But when data set was pre tokenized with Moses (use_moses=False) the pre_rules were executed  after.
So our wikipedia had the following processing:
- raw text
- Moses
- pre_rules
- split(' ') # fastai BaseTokenizer
- post_rules
- sentence piece

While mldoc had the following tokenziation
- raw text
- pre_rules
- Moses
- post_rules
- sentence piece

After fix I've retrained the classfiers (without finetuning) and I haven't notice huge changes in the performance. 4 languages received slight improvment 4 got a slight decrease in performance.

fastai_contrib/utils.py

@@ -35,33 +35,28 @@ CLASSES = ['neg', 'pos', 'unsup']
 number_match_re = re.compile(r'^([0-9]+[,.]?)+$')
 number_split_re = re.compile(r'([,.])')
 
-class MosesTokenizerFunc(BaseTokenizer):
-    "Wrapper around a MosesTokenizer to make it a `BaseTokenizer`."
-    def __init__(self, lang:str):
-        super().__init__(lang=lang)
-        self.tok = MosesTokenizer(lang)
+class MosesPreprocessingFunc():
 
-    def tokenizer(self, t:str) -> List[str]:
-        return self.tok.tokenize(t, return_str=False, escape=False)
+    def __init__(self, lang: str):
+        self.mt = MosesTokenizer(lang)
 
-    def add_special_cases(self, toks:Collection[str]):
-        for w in toks:
-            assert len(self.tokenizer(w))==1, f"Tokenizer is unable to keep {w} as one token!"
+    def __call__(self, t: str) -> str:
+        return self.mt.tokenize(t, return_str=True, escape=True)
 
 class SentencePieceTokenizer(Tokenizer):
     "Put together rules and a tokenizer function to tokenize text with multiprocessing."
     def __init__(self, spm_model, lang:str='en', pre_rules:ListRules=None,
-                 post_rules:ListRules=None, special_cases:Collection[str]=None, n_cpus:int=None, use_moses=False):
+                 post_rules:ListRules=None, special_cases:Collection[str]=None, n_cpus:int=None):
+        # moses is added to preprocessing functions
         super().__init__(self.tok_fun_with_sp, lang, pre_rules, post_rules, special_cases, n_cpus)
         self.spm_model = spm_model
-        self.use_moses = use_moses
 
     def tok_fun_with_sp(self, lang):
         try:
             import sentencepiece as spm
         except ImportError:
             raise Exception('sentencepiece module is missing: run `pip install sentencepiece`')
-        tok = MosesTokenizerFunc(lang) if self.use_moses else BaseTokenizer(lang)
+        tok = BaseTokenizer(lang)
         tok.sp = spm.SentencePieceProcessor()
         tok.sp.Load(str(self.spm_model))
         return tok
@@ -72,9 +67,8 @@ class SentencePieceTokenizer(Tokenizer):
         toks = tok.sp.EncodeAsPieces(" ".join(toks))
         return toks
 
-def get_sentencepiece(cache_dir:PathOrStr, load_text,pre_rules:ListRules=None, post_rules:ListRules=None,
-                      vocab_size:int=30000, model_type:str='unigram', input_sentence_size:int=1E7, 
-                      use_moses=False, lang='en'):
+def get_sentencepiece(cache_dir:PathOrStr, load_text, pre_rules: ListRules=None, post_rules:ListRules=None,
+                      vocab_size:int=30000, model_type:str='unigram', input_sentence_size:int=1E7, lang='en'):
     try:
         import sentencepiece as spm
     except ImportError:
@@ -85,19 +79,13 @@ def get_sentencepiece(cache_dir:PathOrStr, load_text,pre_rules:ListRules=None, p
     post_rules = post_rules if post_rules is not None else defaults.text_post_rules
 
     special_cases = defaults.text_spec_tok
-
     if not os.path.isfile(cache_dir / 'spm.model') or not os.path.isfile(cache_dir / f'itos.pkl'):
         # load the text from the train tokens file
         text = load_text()
         text = filter(lambda x: len(x.rstrip(" ")), text)
         text = (reduce(lambda t, rule: rule(t), pre_rules, line) for line in text)
-        if use_moses:
-            mt = MosesTokenizer(lang)
-            splitter = lambda t: mt.tokenize(t, return_str=False, escape=False)
-        else:
-            splitter = lambda t: t.split()
         def cleanup_n_postprocess(t):
-            t = splitter(t)
+            t = t.split()
             for r in post_rules:
                 t = r(t)
             return ' '.join(t)
@@ -128,10 +116,9 @@ def get_sentencepiece(cache_dir:PathOrStr, load_text,pre_rules:ListRules=None, p
     # We cannot use lambdas or local methods here, since `tok_func` needs to be
     # pickle-able in order to be called in subprocesses when multithread tokenizing
     tokenizer = SentencePieceTokenizer(cache_dir/'spm.model',
-                                       use_moses=use_moses,
-                                       lang=lang,
-                                       pre_rules=pre_rules,
-                                       post_rules=post_rules)
+                                lang=lang,
+                                pre_rules=pre_rules,
+                                post_rules=post_rules)
     return {'tokenizer': tokenizer, 'vocab': vocab}
 
 

results/MLDoc.md

@@ -4,7 +4,8 @@
 |----------------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|------------|
 |LASER           |   90.73   |   92.70   |   88.75   |   90.80   |   85.93   |   85.15   |   84.65   |  88.98     |
 |MultiCCA        |   92.2    |   93.70   |   94.45   |   92.05   |   85.55   |   85.35   |   85.65   |  87.30     |
-|ULMFiT          |           |  **95.4** | **95.15** | **93.67** | **88.42** | **89.20** | **87.27** | **90.20** | 
+|ULMFiT          |           |  **95.4** | **95.15** | **93.67** | **88.42** | **89.20** | **87.27** | **90.20**  |
+|ULMFiT sp-fixed |           |    95.6   |   94.80   |   94.20   |   88.52   |   88.72   |   86.85   |   90.47    | 
 |ULMFiT 100      |           |   91.35   |  83.32    | 88.77     | 77.99     | 71.12     | 72.20     | |
 
 ^ - sp60k lstm nl 4

ulmfit/pretrain_lm.py

@@ -14,8 +14,8 @@ from fastai.callbacks import CSVLogger, SaveModelCallback
 from fastai.text import *
 import torch
 from fastai_contrib.utils import read_file, read_whitespace_file, \
-    validate, PAD, UNK, get_sentencepiece, read_clas_data, TRN, VAL, TST, PAD_TOKEN_ID, MosesTokenizerFunc, \
-    replace_std_toks
+    validate, PAD, UNK, get_sentencepiece, read_clas_data, TRN, VAL, TST, PAD_TOKEN_ID, \
+    replace_std_toks, MosesPreprocessingFunc
 from fastai_contrib.learner import bilm_learner, accuracy_fwd, accuracy_bwd, bilm_text_classifier_learner
 import pickle
 
@@ -30,6 +30,7 @@ ENC_BEST = "enc_best"
 
 class Tokenizers(Enum):
     SUBWORD='sp'
+    BROKENSUBWORD = 'bsp'
     MOSES='v'
     MOSES_FA='vf'
     FASTAI='f'
@@ -121,7 +122,7 @@ class LMHyperParams:
     def model_name(self): return f"{self.model_prefix}_{self.name}.m"
 
     @property
-    def pretrained_fnames(self): return [self.base_lm_path / 'lm_best', self.base_lm_path / '../itos'] if self.base_lm_path else None
+    def pretrained_fnames(self): return [self.base_lm_path / LM_BEST, self.base_lm_path / '../itos'] if self.base_lm_path else None
 
     @property
     def lm_type(self):
@@ -133,8 +134,8 @@ class LMHyperParams:
             return contrib_data.LanguageModelType.FwdLM
 
     def tokenizer_to_fastai_args(self, sp_data_func, use_moses):
-        tok_func = MosesTokenizerFunc if use_moses else BaseTokenizer
-        if self.tokenizer is Tokenizers.SUBWORD:
+        moses_preproc = [MosesPreprocessingFunc(self.lang)] if use_moses else []
+        if self.tokenizer is Tokenizers.SUBWORD or self.tokenizer is Tokenizers.BROKENSUBWORD:
             if self.base_lm_path and not(self.cache_dir/"spm.model").exists(): # ensure we are using the same sentence piece model
                 shutil.copy(self.base_lm_path / '..' / 'itos.pkl', self.cache_dir)
                 shutil.copy(self.base_lm_path / '..' / 'spm.model', self.cache_dir)
@@ -142,13 +143,19 @@ class LMHyperParams:
             args = get_sentencepiece(self.cache_dir,
                                      sp_data_func,
                                      vocab_size=self.max_vocab,
-                                     use_moses=use_moses,
-                                     lang=self.lang)
-
+                                     lang=self.lang,
+                                     pre_rules=moses_preproc + defaults.text_pre_rules,
+                                     post_rules=defaults.text_post_rules)
         elif self.tokenizer is Tokenizers.MOSES:
-            args = dict(tokenizer=Tokenizer(tok_func=tok_func, lang=self.lang, pre_rules=[replace_std_toks], post_rules=[]))
+            args = dict(tokenizer=Tokenizer(tok_func=BaseTokenizer,
+                                            lang=self.lang,
+                                            pre_rules=moses_preproc + [replace_std_toks],
+                                            post_rules=[]))
         elif self.tokenizer is Tokenizers.MOSES_FA:
-            args = dict(tokenizer=Tokenizer(tok_func=tok_func, lang=self.lang))  # use default pre/post rules
+            args = dict(tokenizer=Tokenizer(tok_func=BaseTokenizer,
+                                            lang=self.lang,
+                                            pre_rules=moses_preproc + defaults.text_pre_rules,
+                                            post_rules=defaults.text_post_rules))
         elif self.tokenizer is Tokenizers.FASTAI:
             args = dict()
         else:

ulmfit/train_clas.py

@@ -15,7 +15,7 @@ from fastai_contrib import utils
 from fastai_contrib.data import LanguageModelType
 from fastai_contrib.learner import bilm_text_classifier_learner, bilm_learner, accuracy_fwd, accuracy_bwd
 from fastai_contrib.utils import PAD, UNK, read_clas_data, PAD_TOKEN_ID, DATASETS, TRN, VAL, TST, ensure_paths_exists, \
-    get_sentencepiece, MosesTokenizerFunc
+    get_sentencepiece
 from fastai.text.transform import Vocab
 
 import fire