diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py index 43c98aa..101bce3 100644 --- a/fastai_contrib/utils.py +++ b/fastai_contrib/utils.py @@ -11,7 +11,7 @@ import re import csv from functools import reduce -from fastai.text.transform import Tokenizer, BaseTokenizer, Vocab, default_rules +from fastai.text.transform import Tokenizer, BaseTokenizer, Vocab from fastai.torch_core import * import shutil @@ -47,11 +47,14 @@ class SentencepieceTokenizer(BaseTokenizer): raise Exception('sentencepiece module is missing: run `pip install sentencepiece`') self.tok = spm.SentencePieceProcessor() self.tok.Load(str(pathlib.Path(model_dir) / 'spm.model')) + def tokenizer(self, t:str) -> List[str]: return self.tok.EncodeAsPieces(t) + def add_special_cases(self, toks:Collection[str]): pass + def get_sentencepiece(path:PathOrStr, trn_path:Path, name:str, rules:ListRules=None, vocab_size:int=30000, model_type:str='unigram', input_sentence_size:int=1E7, pad_idx:int=PAD_TOKEN_ID): @@ -64,7 +67,7 @@ def get_sentencepiece(path:PathOrStr, trn_path:Path, name:str, rules:ListRules=N cache_name = 'tmp' os.makedirs(path / cache_name, exist_ok=True) os.makedirs(path / 'models', exist_ok=True) - rules = rules if rules else default_rules + rules = rules if rules else None # load the text frmo the train tokens file @@ -77,11 +80,11 @@ def get_sentencepiece(path:PathOrStr, trn_path:Path, name:str, rules:ListRules=N with open(raw_text_path, 'w') as f: f.write(raw_text) - sp_params = f'--input={raw_text_path} --pad_id={pad_idx} --unk_id=0' \ - f'--character_coverage=1.0 --bos_id=-1 --eos_id=-1 ' \ - f'--input_sentence_size={int(input_sentence_size)} ' \ + sp_params = f"--input={raw_text_path} --pad_id={pad_idx} --unk_id=0 " \ + f"--character_coverage=1.0 --bos_id=-1 --eos_id=-1 " \ + f"--input_sentence_size={int(input_sentence_size)} " \ f"--model_prefix={path / 'models' / 'spm'} " \ - f'--vocab_size={vocab_size} --model_type={model_type} ' + f"--vocab_size={vocab_size} --model_type={model_type} " spm.SentencePieceTrainer.Train(sp_params) with open(path / 'models' / 'spm.vocab', 'r') as f: @@ -200,40 +203,54 @@ def prepare_imdb(file_path: str, prepare_lm = False): df_val.to_csv(LM_PATH / 'test.csv', header=False, index=False) -def read_imdb(dir_path, lang, split) -> Tuple[List[List[str]], List[str]]: +def read_imdb(dir_path, lang, split, spm_path=None) -> Tuple[List[List[str]], List[str]]: """ Reads IMDb data. :param dir_path: the path to the imdb folder :param lang: the language (not used here as IMDb is only available in English) :param split: the split of the data that should be read (train, test, val) + :param spm_path: path to sentencepiece model :return: a tuple consisting of a list of lists of tokens and a list of labels """ file_path = dir_path / 'train.csv' if split == TRN else dir_path / 'test.csv' toks, lbls = [], [] + mt = MosesTokenizer('en') + if spm_path is not None: + sp = SentencepieceTokenizer(spm_path) + print(f'Reading {file_path}...') + with open(file_path, encoding='utf-8') as f: reader = csv.reader(f) for row in reader: label, text = row lbls.append(label) - raw_tokens = mt.tokenize(text, return_str=True).split(' ') + [EOS] + raw_tokens = mt.tokenize(text, return_str=True).split(' ') + tokens = [] + + # fix up occurences of numbers in text for token in raw_tokens: if number_match_re.match(token): tokens += number_split_re.sub(r' @\1@ ', token).split() else: tokens.append(token) - toks.append(tokens) + + if spm_path is not None: + tokens = sp.tokenizer(' '.join(tokens)) + + toks.append(tokens + [EOS]) return toks, lbls -def read_xnli(dir_path, lang, split) -> Tuple[List[List[str]], List[str]]: +def read_xnli(dir_path, lang, split, spm_path=None) -> Tuple[List[List[str]], List[str]]: """ Reads XNLI data. :param dir_path: the path to the xnli folder :param lang: the language :param split: the split of the data that should be read (train, test, val) + :param spm_path: path to sentencepiece model :return: a tuple consisting of a list of lists of tokens and a list of labels """ file_path = XNLI_PATHS[split] @@ -243,6 +260,10 @@ def read_xnli(dir_path, lang, split) -> Tuple[List[List[str]], List[str]]: file_name = 'xnli.dev.en.tsv' if split == VAL else 'xnli.test.en.tsv' file_path = f'XNLI-MT-1.0/xnli/{file_name}' file_path = dir_path / file_path + + if spm_path is not None: + sp = SentencepieceTokenizer(spm_path) + toks, lbls = [], [] print(f'Reading {file_path}...') with open(file_path, encoding='utf-8') as f: @@ -258,9 +279,15 @@ def read_xnli(dir_path, lang, split) -> Tuple[List[List[str]], List[str]]: if ex_lang != lang: continue premise, hypo, label = row[-3], row[-2], row[1] + # TODO add BOS - premise_toks = premise.split(' ') + [EOS] - hypo_toks = hypo.split(' ') + [EOS] + if spm_path is not None: + premise_toks = sp.tokenizer(premise) + [EOS] + hypo_toks = sp.tokenizer(hypo) + [EOS] + else: + premise_toks = premise.split(' ') + [EOS] + hypo_toks = hypo.split(' ') + [EOS] + toks.append(premise_toks + [SEP] + hypo_toks) lbls.append(label) return toks, lbls @@ -390,4 +417,4 @@ class TextReader(): if __name__ == "__main__": - fire.Fire() # allows using all functions via CLI e.g. python utils.py prepare_imdb aclImdb.tgz \ No newline at end of file + fire.Fire() # allows using all functions via CLI e.g. python utils.py prepare_imdb aclImdb.tgz diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 06853a0..fe2d4b7 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -89,18 +89,17 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo itos = [o for o,c in cnt.most_common(n=max_vocab)] itos.insert(1, PAD) #  set pad id to 1 to conform to fast.ai standard assert UNK in itos, f'Unknown words are expected to have been replaced with {UNK} in the data.' - stoi = {w: i for i, w in enumerate(itos)} vocab = Vocab(itos) stoi = vocab.stoi # save vocabulary - print(f"Saving vocabulary as {dir_path / model_dir}") - results['itos_fname'] = dir_path / model_dir / f'itos_{name}.pkl' - with open(results['itos_fname'], 'wb') as f: + itos_fname = model_dir / f'itos_{name}.pkl' + print(f"Saving vocabulary as {itos_fname}") + results['itos_fname'] = itos_fname + with open(itos_fname, 'wb') as f: pickle.dump(itos, f) - trn_ids = np.array([([stoi.get(w, stoi[UNK]) for w in s]) for s in trn_tok]) val_ids = np.array([([stoi.get(w, stoi[UNK]) for w in s]) for s in val_tok]) @@ -108,7 +107,6 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo data_lm = TextLMDataBunch.from_ids(path=dir_path, vocab=vocab, train_ids=trn_ids, valid_ids=val_ids, bs=bs, bptt=bptt) - print('Size of vocabulary:', len(itos)) print('First 10 words in vocab:', ', '.join([itos[i] for i in range(10)])) @@ -135,8 +133,6 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo fit_one_cycle(learn, num_epochs, 5e-3, (0.8, 0.7), wd=1e-7) - - if not subword and max_vocab is None: # only if we use the unpreprocessed version and the full vocabulary # are the perplexity results comparable to previous work