wassname/multifit
1
0
Fork 0
mirror of https://github.com/wassname/multifit.git synced 2026-09-09 11:27:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add moses with fastai preprocessing

Browse Source
This commit is contained in:
Piotr Czapla
2018-12-02 21:43:57 +01:00
parent b9eb7388f6
commit 83427aadc6
3 changed files with 45 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
results/sp30k_biqrnn_bs70
+13
View File
@@ -0,0 +1,13 @@
Starting from random weights
epoch train_loss valid_loss accuracy_fwd accuracy_bwd
1 3.669261 3.641705 0.399714 0.380689
2 3.574335 3.547273 0.404729 0.385104
3 3.573150 3.549644 0.403350 0.384167
4 3.518714 3.499090 0.408166 0.389015
5 3.477355 3.441828 0.413880 0.394777
6 3.408005 3.366269 0.422041 0.402934
7 3.314280 3.284519 0.431068 0.411727
8 3.244735 3.205757 0.440180 0.421078
9 3.170936 3.152495 0.446947 0.428045
10 3.131996 3.138446 0.448782 0.430013
Saving optimiser state at data/wiki/wikitext-103/models/sp30k/biqrnn_bs70.m
tests/test_end_to_end.py
+19
View File
@@ -102,6 +102,25 @@ def test_ulmfit_fastai_bidir_end_to_end():
)
exp.train_lm(num_epochs=1)
def test_ulmfit_moses_fa_bidir_end_to_end():
""" Test ulmfit with sentencepiece tokenizer on small wikipedia dataset.
"""
imdb, wt2 = get_test_data()
lm_name = 'end-to-end-test-fastai'
cuda_id = 0
exp = ulmfit.pretrain_lm.LMHyperParams(
dataset_path=wt2,
lang='en',
cuda_id=cuda_id,
qrnn=True,
bidir=True,
tokenizer='vf',
max_vocab=100,
bs=2,
name=lm_name,
)
exp.train_lm(num_epochs=1)
def test_ulmfit_sentencepiece_end_to_end():
""" Test ulmfit with sentencepiece tokenizer on small wikipedia dataset.
"""
ulmfit/pretrain_lm.py
+13 -4
View File
@@ -44,6 +44,7 @@ import fastai_contrib.data as contrib_data
class Tokenizers(Enum):
SUBWORD='sb'
MOSES='v'
MOSES_FA='vf'
FASTAI='f'
# tokenizers ={
@@ -182,8 +183,6 @@ class LMHyperParams:
sp = get_sentencepiece(self.dataset_path, trn_path, self.name, vocab_size=self.max_vocab)
data_lm = TextLMDataBunch.from_csv(self.dataset_path, 'train.csv', **sp, bs=self.bs, bptt=self.bptt, lm_type=self.lm_type)
itos = data_lm.train_ds.vocab.itos
stoi = data_lm.train_ds.vocab.stoi
elif self.tokenizer is Tokenizers.MOSES:
# read the already whitespace separated data without any preprocessing
trn_tok = read_whitespace_file(trn_path)
@@ -209,12 +208,22 @@ class LMHyperParams:
trn_ids = np.array([([stoi.get(w, stoi[UNK]) for w in s]) for s in trn_tok])
val_ids = np.array([([stoi.get(w, stoi[UNK]) for w in s]) for s in val_tok])
# data_lm = TextLMDataBunch.from_ids(dir_path, trn_ids, [], val_ids, [], len(itos))
data_lm = TextLMDataBunch.from_ids(path=self.dataset_path, vocab=vocab, train_ids=trn_ids,
valid_ids=val_ids, bs=self.bs, bptt=self.bptt,
lm_type=self.lm_type)
elif self.tokenizer is Tokenizers.MOSES_FA:
try:
data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type)
print("Tokenized data loaded")
except FileNotFoundError:
print("Running tokenization")
pretokenized = Tokenizer(tok_func=BaseTokenizer, lang='en', pre_rules=None, post_rules=None)
data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=read_file(trn_path),
valid_df=read_file(val_path), tokenizer=pretokenized,
test_df=read_file(tst_path), classes=None, lm_type=self.lm_type)
data_lm.save('.')
elif self.tokenizer is Tokenizers.FASTAI:
try:
data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type)