diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py index 1cde194..dce29d3 100644 --- a/fastai_contrib/utils.py +++ b/fastai_contrib/utils.py @@ -35,33 +35,28 @@ CLASSES = ['neg', 'pos', 'unsup'] number_match_re = re.compile(r'^([0-9]+[,.]?)+$') number_split_re = re.compile(r'([,.])') -class MosesTokenizerFunc(BaseTokenizer): - "Wrapper around a MosesTokenizer to make it a `BaseTokenizer`." - def __init__(self, lang:str): - super().__init__(lang=lang) - self.tok = MosesTokenizer(lang) +class MosesPreprocessingFunc(): - def tokenizer(self, t:str) -> List[str]: - return self.tok.tokenize(t, return_str=False, escape=False) + def __init__(self, lang: str): + self.mt = MosesTokenizer(lang) - def add_special_cases(self, toks:Collection[str]): - for w in toks: - assert len(self.tokenizer(w))==1, f"Tokenizer is unable to keep {w} as one token!" + def __call__(self, t: str) -> str: + return self.mt.tokenize(t, return_str=True, escape=True) class SentencePieceTokenizer(Tokenizer): "Put together rules and a tokenizer function to tokenize text with multiprocessing." def __init__(self, spm_model, lang:str='en', pre_rules:ListRules=None, - post_rules:ListRules=None, special_cases:Collection[str]=None, n_cpus:int=None, use_moses=False): + post_rules:ListRules=None, special_cases:Collection[str]=None, n_cpus:int=None): + # moses is added to preprocessing functions super().__init__(self.tok_fun_with_sp, lang, pre_rules, post_rules, special_cases, n_cpus) self.spm_model = spm_model - self.use_moses = use_moses def tok_fun_with_sp(self, lang): try: import sentencepiece as spm except ImportError: raise Exception('sentencepiece module is missing: run `pip install sentencepiece`') - tok = MosesTokenizerFunc(lang) if self.use_moses else BaseTokenizer(lang) + tok = BaseTokenizer(lang) tok.sp = spm.SentencePieceProcessor() tok.sp.Load(str(self.spm_model)) return tok @@ -72,9 +67,8 @@ class SentencePieceTokenizer(Tokenizer): toks = tok.sp.EncodeAsPieces(" ".join(toks)) return toks -def get_sentencepiece(cache_dir:PathOrStr, load_text,pre_rules:ListRules=None, post_rules:ListRules=None, - vocab_size:int=30000, model_type:str='unigram', input_sentence_size:int=1E7, - use_moses=False, lang='en'): +def get_sentencepiece(cache_dir:PathOrStr, load_text, pre_rules: ListRules=None, post_rules:ListRules=None, + vocab_size:int=30000, model_type:str='unigram', input_sentence_size:int=1E7, lang='en'): try: import sentencepiece as spm except ImportError: @@ -85,19 +79,13 @@ def get_sentencepiece(cache_dir:PathOrStr, load_text,pre_rules:ListRules=None, p post_rules = post_rules if post_rules is not None else defaults.text_post_rules special_cases = defaults.text_spec_tok - if not os.path.isfile(cache_dir / 'spm.model') or not os.path.isfile(cache_dir / f'itos.pkl'): # load the text from the train tokens file text = load_text() text = filter(lambda x: len(x.rstrip(" ")), text) text = (reduce(lambda t, rule: rule(t), pre_rules, line) for line in text) - if use_moses: - mt = MosesTokenizer(lang) - splitter = lambda t: mt.tokenize(t, return_str=False, escape=False) - else: - splitter = lambda t: t.split() def cleanup_n_postprocess(t): - t = splitter(t) + t = t.split() for r in post_rules: t = r(t) return ' '.join(t) @@ -128,10 +116,9 @@ def get_sentencepiece(cache_dir:PathOrStr, load_text,pre_rules:ListRules=None, p # We cannot use lambdas or local methods here, since `tok_func` needs to be # pickle-able in order to be called in subprocesses when multithread tokenizing tokenizer = SentencePieceTokenizer(cache_dir/'spm.model', - use_moses=use_moses, - lang=lang, - pre_rules=pre_rules, - post_rules=post_rules) + lang=lang, + pre_rules=pre_rules, + post_rules=post_rules) return {'tokenizer': tokenizer, 'vocab': vocab} diff --git a/results/MLDoc.md b/results/MLDoc.md index 58e9009..7dd706c 100644 --- a/results/MLDoc.md +++ b/results/MLDoc.md @@ -4,7 +4,8 @@ |----------------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|------------| |LASER | 90.73 | 92.70 | 88.75 | 90.80 | 85.93 | 85.15 | 84.65 | 88.98 | |MultiCCA | 92.2 | 93.70 | 94.45 | 92.05 | 85.55 | 85.35 | 85.65 | 87.30 | -|ULMFiT | | **95.4** | **95.15** | **93.67** | **88.42** | **89.20** | **87.27** | **90.20** | +|ULMFiT | | **95.4** | **95.15** | **93.67** | **88.42** | **89.20** | **87.27** | **90.20** | +|ULMFiT sp-fixed | | 95.6 | 94.80 | 94.20 | 88.52 | 88.72 | 86.85 | 90.47 | |ULMFiT 100 | | 91.35 | 83.32 | 88.77 | 77.99 | 71.12 | 72.20 | | ^ - sp60k lstm nl 4 diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 3df9213..5959f7e 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -14,8 +14,8 @@ from fastai.callbacks import CSVLogger, SaveModelCallback from fastai.text import * import torch from fastai_contrib.utils import read_file, read_whitespace_file, \ - validate, PAD, UNK, get_sentencepiece, read_clas_data, TRN, VAL, TST, PAD_TOKEN_ID, MosesTokenizerFunc, \ - replace_std_toks + validate, PAD, UNK, get_sentencepiece, read_clas_data, TRN, VAL, TST, PAD_TOKEN_ID, \ + replace_std_toks, MosesPreprocessingFunc from fastai_contrib.learner import bilm_learner, accuracy_fwd, accuracy_bwd, bilm_text_classifier_learner import pickle @@ -30,6 +30,7 @@ ENC_BEST = "enc_best" class Tokenizers(Enum): SUBWORD='sp' + BROKENSUBWORD = 'bsp' MOSES='v' MOSES_FA='vf' FASTAI='f' @@ -121,7 +122,7 @@ class LMHyperParams: def model_name(self): return f"{self.model_prefix}_{self.name}.m" @property - def pretrained_fnames(self): return [self.base_lm_path / 'lm_best', self.base_lm_path / '../itos'] if self.base_lm_path else None + def pretrained_fnames(self): return [self.base_lm_path / LM_BEST, self.base_lm_path / '../itos'] if self.base_lm_path else None @property def lm_type(self): @@ -133,8 +134,8 @@ class LMHyperParams: return contrib_data.LanguageModelType.FwdLM def tokenizer_to_fastai_args(self, sp_data_func, use_moses): - tok_func = MosesTokenizerFunc if use_moses else BaseTokenizer - if self.tokenizer is Tokenizers.SUBWORD: + moses_preproc = [MosesPreprocessingFunc(self.lang)] if use_moses else [] + if self.tokenizer is Tokenizers.SUBWORD or self.tokenizer is Tokenizers.BROKENSUBWORD: if self.base_lm_path and not(self.cache_dir/"spm.model").exists(): # ensure we are using the same sentence piece model shutil.copy(self.base_lm_path / '..' / 'itos.pkl', self.cache_dir) shutil.copy(self.base_lm_path / '..' / 'spm.model', self.cache_dir) @@ -142,13 +143,19 @@ class LMHyperParams: args = get_sentencepiece(self.cache_dir, sp_data_func, vocab_size=self.max_vocab, - use_moses=use_moses, - lang=self.lang) - + lang=self.lang, + pre_rules=moses_preproc + defaults.text_pre_rules, + post_rules=defaults.text_post_rules) elif self.tokenizer is Tokenizers.MOSES: - args = dict(tokenizer=Tokenizer(tok_func=tok_func, lang=self.lang, pre_rules=[replace_std_toks], post_rules=[])) + args = dict(tokenizer=Tokenizer(tok_func=BaseTokenizer, + lang=self.lang, + pre_rules=moses_preproc + [replace_std_toks], + post_rules=[])) elif self.tokenizer is Tokenizers.MOSES_FA: - args = dict(tokenizer=Tokenizer(tok_func=tok_func, lang=self.lang)) # use default pre/post rules + args = dict(tokenizer=Tokenizer(tok_func=BaseTokenizer, + lang=self.lang, + pre_rules=moses_preproc + defaults.text_pre_rules, + post_rules=defaults.text_post_rules)) elif self.tokenizer is Tokenizers.FASTAI: args = dict() else: diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 5cff5d3..0641790 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -15,7 +15,7 @@ from fastai_contrib import utils from fastai_contrib.data import LanguageModelType from fastai_contrib.learner import bilm_text_classifier_learner, bilm_learner, accuracy_fwd, accuracy_bwd from fastai_contrib.utils import PAD, UNK, read_clas_data, PAD_TOKEN_ID, DATASETS, TRN, VAL, TST, ensure_paths_exists, \ - get_sentencepiece, MosesTokenizerFunc + get_sentencepiece from fastai.text.transform import Vocab import fire