wassname/multifit
1
0
Fork 0
mirror of https://github.com/wassname/multifit.git synced 2026-09-09 11:27:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/n-waves/ulmfit-multilingual into polyglot-lm

Browse Source
This commit is contained in:
NAUSICAA\Julian
2019-01-03 09:30:54 -03:00
parent 52702e32ae 0085c18ae0
commit 3f91460dc8
1 changed files with 7 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ulmfit/pretrain_lm.py
+7 -7
View File
@@ -42,15 +42,15 @@ def read_wiki_articles(filename):
articles = []
with open(filename, encoding='utf8') as f:
lines = f.readlines()
current_article = ''
current_article = []
for i,line in enumerate(lines):
current_article += line
if i < len(lines)-2 and lines[i+1] == ' \n' and istitle(lines[i+2]):
articles.append(current_article)
current_article = ''
articles.append(current_article)
current_article.append(line)
if i < len(lines)-2 and lines[i+1].strip() == "" and istitle(lines[i+2]):
articles.append("".join(current_article))
current_article = []
articles.append("".join(current_article))
print(f"Wiki text was split to {len(articles)} articles")
return pd.DataFrame({'texts':np.array(articles)})
return pd.DataFrame({'texts': np.array(articles, dtype=np.object)})
@dataclass
class LMHyperParams: