wassname/multifit
1
0
Fork 0
mirror of https://github.com/wassname/multifit.git synced 2026-09-09 11:27:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix bidir for fastai tokenizer

Browse Source
This commit is contained in:
Piotr Czapla
2018-12-01 23:58:14 +01:00
parent 0c4aed6d05
commit b9eb7388f6
2 changed files with 22 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_end_to_end.py
+19
View File
@@ -83,6 +83,25 @@ def test_ulmfit_fastai_end_to_end():
)
exp.train_lm(num_epochs=1)
def test_ulmfit_fastai_bidir_end_to_end():
""" Test ulmfit with sentencepiece tokenizer on small wikipedia dataset.
"""
imdb, wt2 = get_test_data()
lm_name = 'end-to-end-test-fastai'
cuda_id = 0
exp = ulmfit.pretrain_lm.LMHyperParams(
dataset_path=wt2,
lang='en',
cuda_id=cuda_id,
qrnn=True,
bidir=True,
tokenizer='f',
max_vocab=100,
bs=2,
name=lm_name,
)
exp.train_lm(num_epochs=1)
def test_ulmfit_sentencepiece_end_to_end():
""" Test ulmfit with sentencepiece tokenizer on small wikipedia dataset.
"""
ulmfit/pretrain_lm.py
+3 -5
View File
@@ -181,9 +181,7 @@ class LMHyperParams:
sp = get_sentencepiece(self.dataset_path, trn_path, self.name, vocab_size=self.max_vocab)
lm_type = contrib_data.LanguageModelType.BiLM if self.bidir else contrib_data.LanguageModelType.FwdLM
data_lm = TextLMDataBunch.from_csv(self.dataset_path, 'train.csv', **sp, bs=self.bs, bptt=self.bptt, lm_type=lm_type)
data_lm = TextLMDataBunch.from_csv(self.dataset_path, 'train.csv', **sp, bs=self.bs, bptt=self.bptt, lm_type=self.lm_type)
itos = data_lm.train_ds.vocab.itos
stoi = data_lm.train_ds.vocab.stoi
elif self.tokenizer is Tokenizers.MOSES:
@@ -219,12 +217,12 @@ class LMHyperParams:
lm_type=self.lm_type)
elif self.tokenizer is Tokenizers.FASTAI:
try:
data_lm = TextLMDataBunch.load(self.cache_dir, '.')
data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type)
print("Tokenized data loaded")
except FileNotFoundError:
print("Running tokenization")
data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=read_file(trn_path), valid_df=read_file(val_path),
test_df=read_file(tst_path), classes=None)
test_df=read_file(tst_path), classes=None, lm_type=self.lm_type)
data_lm.save('.')
else:
raise ValueError(f"self.tokenizer has wrong value {self.tokenizer}, Allowed values are taken from {Tokenizers}")