wassname/multifit
1
0
Fork 0
mirror of https://github.com/wassname/multifit.git synced 2026-09-09 11:27:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix double EOS in read_imdb

Browse Source
This commit is contained in:
Aayush
2018-11-19 19:30:11 +05:30
committed by GitHub
parent 6e867811db
commit 73403bdcb3
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fastai_contrib/utils.py
+1 -1
View File
@@ -224,7 +224,7 @@ def read_imdb(dir_path, lang, split, spm_path=None) -> Tuple[List[List[str]], Li
for row in reader:
label, text = row
lbls.append(label)
raw_tokens = mt.tokenize(text, return_str=True).split(' ') + [EOS]
raw_tokens = mt.tokenize(text, return_str=True).split(' ')
tokens = []