From 5ba8ae4c59a72813ce4338c983cd087cacb25b02 Mon Sep 17 00:00:00 2001 From: Tomasz Pietruszka Date: Sun, 13 Jan 2019 17:29:55 +0100 Subject: [PATCH] non-ascii char removed from code. Caused display bugs --- ulmfit/postprocess_wikitext.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ulmfit/postprocess_wikitext.py b/ulmfit/postprocess_wikitext.py index 7ae8774..481631c 100644 --- a/ulmfit/postprocess_wikitext.py +++ b/ulmfit/postprocess_wikitext.py @@ -53,7 +53,7 @@ def limit_vocab(unk_path, vocab): tokens = [''] + tokens line = ' '.join(tokens) f_out.write(line) - print(f'{unk_path.name}. # of tokens: {total_num_tokens}') + print(f'{unk_path.name}. # of tokens: {total_num_tokens}') temp_file_path.replace(unk_path) @@ -101,5 +101,6 @@ def postprocess_wikitext(path, lang): unk_path = dest_path / f'{lang}.wiki.{split}.tokens' limit_vocab(unk_path, vocab) + if __name__ == '__main__': - fire.Fire(postprocess_wikitext) \ No newline at end of file + fire.Fire(postprocess_wikitext)