wassname/multifit
1
0
Fork 0
mirror of https://github.com/wassname/multifit.git synced 2026-09-09 11:27:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

Make the article detection code work with our wikitext

Browse Source
This commit is contained in:
Piotr Czapla
2019-01-01 15:13:14 +01:00
parent 0945c699c7
commit f784d7bcd2
1 changed files with 2 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ulmfit/pretrain_lm.py
+2 -1
View File
@@ -35,7 +35,7 @@ class Tokenizers(Enum):
FASTAI='f'
def istitle(line):
return len(re.findall(r'^ = [^=]* = $', line)) != 0
return len(re.findall(r'^ ?= [^=]* = ?$', line)) != 0
def read_wiki_articles(filename):
articles = []
@@ -48,6 +48,7 @@ def read_wiki_articles(filename):
articles.append(current_article)
current_article = ''
articles.append(current_article)
print(f"Wiki text was split to {len(articles)} articles")
return pd.DataFrame({'texts':np.array(articles)})
@dataclass