diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py index c7eb129..2c814ce 100644 --- a/fastai_contrib/utils.py +++ b/fastai_contrib/utils.py @@ -70,7 +70,7 @@ full_char_coverage_langs = ["bg", "cs", "da", "de", "el", "en", "es", "et", "fi" "it","lt","lv","mt","nl","pl","pt","ro","sk","sl","sv"] # all European langus def get_sentencepiece(cache_dir:PathOrStr, load_text, pre_rules: ListRules=None, post_rules:ListRules=None, - vocab_size:int=30000, model_type:str='unigram', input_sentence_size:int=1E7, lang='en'): + vocab_size:int=30000, model_type:str='unigram', input_sentence_size:int=1E7, lang='en', fixed_character_coverage=False): try: import sentencepiece as spm except ImportError: @@ -96,7 +96,10 @@ def get_sentencepiece(cache_dir:PathOrStr, load_text, pre_rules: ListRules=None, raw_text_path = cache_dir / 'all_text.txt' with open(raw_text_path, 'w') as f: f.write("\n".join(text)) - char_coverage = 1 if lang in full_char_coverage_langs else 0.99 + if fixed_character_coverage: + char_coverage = 0.9995 + else: + char_coverage = 1 if lang in full_char_coverage_langs else 0.99 sp_params = [ f"--input={raw_text_path}",