wassname/multifit
1
0
Fork 0
mirror of https://github.com/wassname/multifit.git synced 2026-09-09 11:27:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

non-ascii char removed from code. Caused display bugs

Browse Source
This commit is contained in:
Tomasz Pietruszka
2019-01-13 17:29:55 +01:00
parent b00410e0cb
commit 5ba8ae4c59
1 changed files with 3 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ulmfit/postprocess_wikitext.py
+3 -2
View File
@@ -53,7 +53,7 @@ def limit_vocab(unk_path, vocab):
tokens = [''] + tokens
line = ' '.join(tokens)
f_out.write(line)
print(f'{unk_path.name}. # of tokens: {total_num_tokens}')
print(f'{unk_path.name}. # of tokens: {total_num_tokens}')
temp_file_path.replace(unk_path)
@@ -101,5 +101,6 @@ def postprocess_wikitext(path, lang):
unk_path = dest_path / f'{lang}.wiki.{split}.tokens'
limit_vocab(unk_path, vocab)
if __name__ == '__main__':
fire.Fire(postprocess_wikitext)
fire.Fire(postprocess_wikitext)