From 7fe059c9a2d432da5475d1dc71b9b8f7cd9494e6 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sat, 17 Nov 2018 14:47:27 +0000 Subject: [PATCH] Fixed models path for vocabulary --- ulmfit/pretrain_lm.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 0847ca6..0a76e00 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -89,18 +89,17 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo itos = [o for o,c in cnt.most_common(n=max_vocab)] itos.insert(1, PAD) #  set pad id to 1 to conform to fast.ai standard assert UNK in itos, f'Unknown words are expected to have been replaced with {UNK} in the data.' - stoi = {w: i for i, w in enumerate(itos)} vocab = Vocab(itos) stoi = vocab.stoi # save vocabulary - print(f"Saving vocabulary as {dir_path / model_dir}") - results['itos_fname'] = dir_path / model_dir / f'itos_{name}.pkl' - with open(results['itos_fname'], 'wb') as f: + itos_fname = model_dir / f'itos_{name}.pkl' + print(f"Saving vocabulary as {itos_fname}") + results['itos_fname'] = itos_fname + with open(itos_fname, 'wb') as f: pickle.dump(itos, f) - trn_ids = np.array([([stoi.get(w, stoi[UNK]) for w in s]) for s in trn_tok]) val_ids = np.array([([stoi.get(w, stoi[UNK]) for w in s]) for s in val_tok]) @@ -108,7 +107,6 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo data_lm = TextLMDataBunch.from_ids(path=dir_path, vocab=vocab, train_ids=trn_ids, valid_ids=val_ids, bs=bs, bptt=bptt) - print('Size of vocabulary:', len(itos)) print('First 10 words in vocab:', ', '.join([itos[i] for i in range(10)])) @@ -135,8 +133,6 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo fit_one_cycle(learn, num_epochs, 5e-3, (0.8, 0.7), wd=1e-7) - - if not subword and max_vocab is None: # only if we use the unpreprocessed version and the full vocabulary # are the perplexity results comparable to previous work