wassname/Clover-Edition
1
0
Fork 0
mirror of https://github.com/wassname/Clover-Edition.git synced 2026-09-09 11:13:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

remove endoftext etc

Browse Source
This commit is contained in:
wassname
2019-12-25 06:39:20 +08:00
parent d1baf4cada
commit 69d1151e86
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gpt2generator.py
+1 -1
View File
@@ -202,7 +202,7 @@ class GPT2Generator:
out = out[:, len(context_tokens) :].tolist()
for o in out:
generated += 1
text = self.tokenizer.decode(o, clean_up_tokenization_spaces=True)
text = self.tokenizer.decode(o, clean_up_tokenization_spaces=True, skip_special_tokens=True)
if self.stop_token:
index = text.find(self.stop_token)
if index == -1: