wassname/multifit
1
0
Fork 0
mirror of https://github.com/wassname/multifit.git synced 2026-09-09 11:27:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

add vocab_size to sentencepiece

Browse Source
This commit is contained in:
Aayush
2018-11-19 19:13:55 +05:30
committed by GitHub
parent 69ed7b4169
commit 5ffbe8ba5c
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ulmfit/pretrain_lm.py
+1 -1
View File
@@ -70,7 +70,7 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo
read_file(trn_path, 'train')
read_file(val_path, 'valid')
sp = get_sentencepiece(dir_path, trn_path, name)
sp = get_sentencepiece(dir_path, trn_path, name, vocab_size=max_vocab)
data_lm = TextLMDataBunch.from_csv(dir_path, **sp)
itos = data_lm.train_ds.vocab.itos