wassname/multifit
1
0
Fork 0
mirror of https://github.com/wassname/multifit.git synced 2026-09-05 16:40:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

missing file

Browse Source
This commit is contained in:
Piotr Czapla
2019-06-10 20:14:57 +02:00
parent 7d59c4ecf2
commit a7ac4f5170
1 changed files with 5 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fastai_contrib/utils.py
+5 -2
View File
@@ -70,7 +70,7 @@ full_char_coverage_langs = ["bg", "cs", "da", "de", "el", "en", "es", "et", "fi"
"it","lt","lv","mt","nl","pl","pt","ro","sk","sl","sv"] # all European langus
def get_sentencepiece(cache_dir:PathOrStr, load_text, pre_rules: ListRules=None, post_rules:ListRules=None,
vocab_size:int=30000, model_type:str='unigram', input_sentence_size:int=1E7, lang='en'):
vocab_size:int=30000, model_type:str='unigram', input_sentence_size:int=1E7, lang='en', fixed_character_coverage=False):
try:
import sentencepiece as spm
except ImportError:
@@ -96,7 +96,10 @@ def get_sentencepiece(cache_dir:PathOrStr, load_text, pre_rules: ListRules=None,
raw_text_path = cache_dir / 'all_text.txt'
with open(raw_text_path, 'w') as f: f.write("\n".join(text))
char_coverage = 1 if lang in full_char_coverage_langs else 0.99
if fixed_character_coverage:
char_coverage = 0.9995
else:
char_coverage = 1 if lang in full_char_coverage_langs else 0.99
sp_params = [
f"--input={raw_text_path}",