From 5529ec385b56feb7eb4fa9cd4db16062646a3533 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Mon, 18 Feb 2019 21:50:46 +0100 Subject: [PATCH] Add sentence piece char_coverage 0.99 for non european languages Otherwise Zh won't train on 15k --- fastai_contrib/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py index dce29d3..ebd9674 100644 --- a/fastai_contrib/utils.py +++ b/fastai_contrib/utils.py @@ -66,6 +66,8 @@ class SentencePieceTokenizer(Tokenizer): toks = super().process_text(t, tok) toks = tok.sp.EncodeAsPieces(" ".join(toks)) return toks +full_char_coverage_langs = ["bg", "cs", "da", "de", "el", "en", "es", "et", "fi", "fr", "ga", "hr", "hu", + "it","lt","lv","mt","nl","pl","pt","ro","sk","sl","sv"] # all European langus def get_sentencepiece(cache_dir:PathOrStr, load_text, pre_rules: ListRules=None, post_rules:ListRules=None, vocab_size:int=30000, model_type:str='unigram', input_sentence_size:int=1E7, lang='en'): @@ -93,9 +95,11 @@ def get_sentencepiece(cache_dir:PathOrStr, load_text, pre_rules: ListRules=None, raw_text_path = cache_dir / 'all_text.txt' with open(raw_text_path, 'w') as f: f.write("\n".join(text)) + char_coverage = 1 if lang in full_char_coverage_langs else 0.99 + sp_params = [ f"--input={raw_text_path}", - f"--character_coverage=1.0", + f"--character_coverage={char_coverage}", f"--unk_id={len(defaults.text_spec_tok)}", f"--pad_id=-1", f"--bos_id=-1",