wassname/multifit
1
0
Fork 0
mirror of https://github.com/wassname/multifit.git synced 2026-09-09 11:27:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix splitting by article for local languages and make it more memory efficient

Browse Source 
The issue was that the code assumed that empty lines have space followed by a new  line, which isn't the case for datasets generated by our scripts.
This commit is contained in:
Piotr Czapla
2019-01-03 12:16:41 +01:00
parent 8a2fed41c5
commit 0085c18ae0
1 changed files with 7 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ulmfit/pretrain_lm.py
+7 -7
View File
@@ -41,15 +41,15 @@ def read_wiki_articles(filename):
articles = []
with open(filename, encoding='utf8') as f:
lines = f.readlines()
current_article = ''
current_article = []
for i,line in enumerate(lines):
current_article += line
if i < len(lines)-2 and lines[i+1] == ' \n' and istitle(lines[i+2]):
articles.append(current_article)
current_article = ''
articles.append(current_article)
current_article.append(line)
if i < len(lines)-2 and lines[i+1].strip() == "" and istitle(lines[i+2]):
articles.append("".join(current_article))
current_article = []
articles.append("".join(current_article))
print(f"Wiki text was split to {len(articles)} articles")
return pd.DataFrame({'texts':np.array(articles)})
return pd.DataFrame({'texts': np.array(articles, dtype=np.object)})
@dataclass
class LMHyperParams: