mirror of
https://github.com/wassname/multifit.git
synced 2026-10-27 13:00:53 +08:00
add vocab_size to sentencepiece
This commit is contained in:
1 parent
69ed7b4169
commit
5ffbe8ba5c
1 file changed
+1
-1
@@ -70,7 +70,7 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo
|
||||
read_file(trn_path, 'train')
|
||||
read_file(val_path, 'valid')
|
||||
|
||||
sp = get_sentencepiece(dir_path, trn_path, name)
|
||||
sp = get_sentencepiece(dir_path, trn_path, name, vocab_size=max_vocab)
|
||||
|
||||
data_lm = TextLMDataBunch.from_csv(dir_path, **sp)
|
||||
itos = data_lm.train_ds.vocab.itos
|
||||
|
||||
Reference in new issue
Block a user