diff --git a/fastai_contrib/data.py b/fastai_contrib/data.py deleted file mode 100644 index 5f6e732..0000000 --- a/fastai_contrib/data.py +++ /dev/null @@ -1,71 +0,0 @@ -"NLP data loading pipeline. Supports csv, folders, and preprocessed data." -from fastai.text import * -from fastai.torch_core import * -from fastai.text.transform import * -from fastai.basic_data import * -from fastai.data_block import * - -#region Modified fastai classes - -LanguageModelType=Enum('LanguageModelType', 'FwdLM BwdLM BiLM') - -class LanguageModelLoader(): # copy of the original LanguageModelLoader - "Create a dataloader with bptt slightly changing." - def __init__(self, dataset:LabelList, bs:int=64, bptt:int=70, - lm_type:LanguageModelType=LanguageModelType.FwdLM, shuffle:bool=False, - max_len:int=25, p_bptt:int=0.95): - self.dataset,self.bs,self.bptt,self.lm_type,self.shuffle, self.p_bptt = dataset,bs,bptt,lm_type,shuffle,p_bptt - self.first,self.i,self.iter = True,0,0 - self.n = len(np.concatenate(dataset.x.items)) // self.bs if len(dataset.x.items) > 0 else 0 - self.max_len,self.num_workers = max_len,0 - self.init_kwargs = dict(bs=bs, bptt=bptt, lm_type=lm_type, shuffle=shuffle, max_len=max_len, p_bptt=p_bptt) - - def __iter__(self): - if getattr(self.dataset, 'item', None) is not None: - yield LongTensor(getattr(self.dataset, 'item'))[None],LongTensor([0]) - idx = np.random.permutation(len(self.dataset)) if self.shuffle else range(len(self.dataset)) - data = self.batchify(np.concatenate([self.dataset.x.items[i] for i in idx])) - - pos, itr = 0,0 - while pos < self.n-1 and itr int: return int(math.ceil((self.n-1) / self.bptt)) # so that it is always at least 1 - def __getattr__(self,k:str)->Any: return getattr(self.dataset, k) - - @property - def batch_size(self): return self.bs - @batch_size.setter - def batch_size(self, v): self.bs = v - - def batchify(self, data:np.ndarray) -> LongTensor: - "Split the corpus `data` in batches." - nb = data.shape[0] // self.bs - data = np.array(data[:nb*self.bs]).reshape(self.bs, -1) - if self.lm_type == LanguageModelType.BwdLM: data = data[:,::-1].copy() - elif self.lm_type == LanguageModelType.BiLM: data = np.stack([data, data[:,::-1].copy()], axis=2) - return LongTensor(data) - - def get_batch(self, data:LongTensor, i:int, seq_len:int) -> Tuple[LongTensor, LongTensor]: - "Create a batch at `i` of a given `seq_len`." - seq_len = min(seq_len, data.shape[1] - 1 - i) - x = data[:,i:i+seq_len] - y = data[:,i+1:i+1+seq_len] - y = y.contiguous().view(-1, 2) if self.lm_type == LanguageModelType.BiLM else y.contiguous().view(-1) - return x,y - -#endregion -#region Replaces fastai classes - -import fastai.text.data -fastai.text.data.LanguageModelLoader = LanguageModelLoader # Replace original LanguageModelLoader with new verion - -#endregion \ No newline at end of file diff --git a/fastai_contrib/learner.py b/fastai_contrib/learner.py deleted file mode 100644 index de5d26a..0000000 --- a/fastai_contrib/learner.py +++ /dev/null @@ -1,120 +0,0 @@ -from torch.nn import CrossEntropyLoss - -from fastai import * -from fastai.text import * - -#region New code -from fastai_contrib.models import * - - -def bilm_learner(data:DataBunch, bptt:int=70, emb_sz:int=400, nh:int=1150, nl:int=3, pad_token:int=1, - drop_mult:float=1., tie_weights:bool=True, bias:bool=True, qrnn:bool=False, pretrained_model=None, - pretrained_fnames:OptStrTuple=None, **kwargs) -> 'LanguageLearner': - "Create a `Learner` with a language model." - dps = default_dropout['language'] * drop_mult - vocab_size = len(data.vocab.itos) - model = get_bilm(vocab_size, emb_sz, nh, nl, pad_token, input_p=dps[0], output_p=dps[1], - weight_p=dps[2], embed_p=dps[3], hidden_p=dps[4], tie_weights=tie_weights, bias=bias, qrnn=qrnn) - learn = LanguageLearner(data, model, bptt, split_func=bilm_split, **kwargs) - if pretrained_model is not None: - model_path = untar_data(pretrained_model, data=False) - fnames = [list(model_path.glob(f'*.{ext}'))[0] for ext in ['pth', 'pkl']] - learn.load_pretrained(*fnames) - learn.freeze() - if pretrained_fnames is not None: - fnames = [learn.path/learn.model_dir/f'{fn}.{ext}' for fn,ext in zip(pretrained_fnames, ['pth', 'pkl'])] - learn.load_pretrained(*fnames) - learn.freeze() - learn.loss_func = CrossEntropyLoss() # I'm not sure why fast ai is using CrossEntropyFlat but it breaks bilm - return learn - -def bilm_text_classifier_learner(data: DataBunch, bptt: int = 70, max_len: int = 70 * 20, emb_sz: int = 400, - nh: int = 1150, nl: int = 3, - lin_ftrs: Collection[int] = None, ps: Collection[float] = None, pad_token: int = 1, - drop_mult: float = 1., qrnn: bool = False, bicls_head:str='BiPoolingLinearClassifier', **kwargs) -> 'TextClassifierLearner': - "Create a RNN classifier." - dps = default_dropout['classifier'] * drop_mult - if lin_ftrs is None: lin_ftrs = [50] - if ps is None: ps = [0.1] - ds = data.train_ds - vocab_size, n_class = len(data.vocab.itos), data.c - if bicls_head == 'BiPoolingLinearClassifier': - count = 3*2 - else: - count = 3 - layers = [emb_sz * count] + lin_ftrs + [n_class] - ps = [dps[4]] + ps - model = get_birnn_classifier(bptt, max_len, n_class, vocab_size, emb_sz, nh, nl, pad_token, - layers, ps, input_p=dps[0], weight_p=dps[1], embed_p=dps[2], hidden_p=dps[3], - qrnn=qrnn, bicls_head=bicls_head) - learn = RNNLearner(data, model, bptt, split_func=birnn_classifier_split, **kwargs) - return learn - -def bilm_split(model:nn.Module) -> List[nn.Module]: - "Split a RNN `model` in groups for differential learning rates." - - return [f+b for f,b in zip(lm_split(model.fwd_lm),lm_split(model.bwd_lm))] - -def birnn_classifier_split(model:nn.Module) -> List[nn.Module]: - "Split a RNN `model` in groups for differential learning rates." - f_rnn,b_rnn = model[0].fwd_lm,model[0].bwd_lm - groups = [[f_rnn.encoder, f_rnn.encoder_dp,b_rnn.encoder, b_rnn.encoder_dp]] - groups += [a for a in zip(f_rnn.rnns, f_rnn.hidden_dps, b_rnn.rnns, b_rnn.hidden_dps, )] - groups.append([model[1]]) - return groups - -def accuracy_fwd(input, targs): - return accuracy(input[...,0], targs[...,0]) - -def accuracy_bwd(input, targs): - return accuracy(input[...,1], targs[...,1]) - - -#endregion -#region Modified fastai code - -def convert_weights(wgts:Weights, stoi_wgts:Dict[str,int], itos_new:Collection[str]) -> Weights: - "Convert the model weights to go with a new vocabulary." - if 'fwd_lm.0.encoder.weight' in wgts: #todo share embedding matrix computation - wgts = convert_weights_with_prefix(wgts, stoi_wgts, itos_new, prefix='fwd_lm.') - return convert_weights_with_prefix(wgts, stoi_wgts, itos_new, prefix='bwd_lm.') - else: - return convert_weights_with_prefix(wgts, stoi_wgts, itos_new, prefix='') - -def convert_weights_with_prefix(wgts:Weights, stoi_wgts:Dict[str,int], itos_new:Collection[str], prefix='') -> Weights: - "Convert the model weights to go with a new vocabulary." - if 'model' in wgts: - wgts['model'] = convert_weights_with_prefix(wgts['model'], stoi_wgts, itos_new, prefix) - else: - #dec_bias, enc_wgts = wgts[prefix+'1.decoder.bias'], wgts[prefix+'0.encoder.weight'] - has_bias = prefix+'1.decoder.bias' in wgts - enc_wgts = wgts[prefix+'0.encoder.weight'] - if has_bias: - dec_bias = wgts[prefix+'1.decoder.bias'] - else: - dec_bias = enc_wgts.new_zeros((len(stoi_wgts),)) - bias_m, wgts_m = dec_bias.mean(0), enc_wgts.mean(0) - new_w = enc_wgts.new_zeros((len(itos_new),enc_wgts.size(1))).zero_() - new_b = dec_bias.new_zeros((len(itos_new),)).zero_() - unk_tokens=[] - for i,w in enumerate(itos_new): - r = stoi_wgts[w] if w in stoi_wgts else -1 - if r < 0: - unk_tokens.append(w) - new_w[i] = enc_wgts[r] if r>=0 else wgts_m - new_b[i] = dec_bias[r] if r>=0 else bias_m - print(f"Unknown tokens {len(unk_tokens)}, first 100: {unk_tokens[:100]}") - wgts[prefix+'0.encoder.weight'] = new_w - wgts[prefix+'0.encoder_dp.emb.weight'] = new_w.clone() - wgts[prefix+'1.decoder.weight'] = new_w.clone() - if has_bias: - wgts[prefix+'1.decoder.bias'] = new_b - return wgts - -#endregion -#region Replace code in fastai - -import fastai.text.learner -fastai.text.learner.convert_weights = convert_weights - -#endregion diff --git a/fastai_contrib/models.py b/fastai_contrib/models.py deleted file mode 100644 index 8b5afdb..0000000 --- a/fastai_contrib/models.py +++ /dev/null @@ -1,166 +0,0 @@ -from fastai.torch_core import * -from fastai.layers import * -from fastai.text.models import * -from fastai.text.learner import * -#region New code - -class BiLMModel(nn.Module): - - def __init__(self, fwd_lm:nn.Module, bwd_lm:nn.Module, squash_bs_sl=False): - super().__init__() - self.fwd_lm = fwd_lm - self.bwd_lm = bwd_lm - self.squash_bs_sl = squash_bs_sl - - def __getitem__(self, idx): - return BiLMModel(self.fwd_lm[idx], self.bwd_lm[idx]) - - def __len__(self): - return len(self.fwd_lm) - - def stack(self, fwd_o, bwd_o): - if is_listy(fwd_o): - return [self.stack(f, b) for f,b in zip(fwd_o,bwd_o)] - else: - return torch.stack([fwd_o, bwd_o], dim=len(fwd_o.shape)) - - def forward(self, input): - if len(input.shape) == 3: # sl, bs, tracks - f = input[..., 0] - b = input[..., 1] - elif len(input.shape) == 2: # sl, bs - support during classification mode - f = input - b = torch.flip(input, [1]) # todo test if we are duplicating the backward pass correctly - else: - raise AttributeError(f"Inorrect size of input, {input.shape}") - fwd_o = self.fwd_lm(f) - bwd_o = self.bwd_lm(b) - - outs = self.stack(fwd_o, bwd_o) - if self.squash_bs_sl: - o = outs[0] - o = o.view(o.shape[0]*o.shape[1],o.shape[2],o.shape[3]) - outs[0] = o - return outs - - def reset(self): - "Reset the hidden states of underlaying lms." - self.fwd_lm.reset() - self.bwd_lm.reset() - -class MultiBatchBiLMModel(BiLMModel): - "Create a RNNCore module that can process a full sentence." - - def __init__(self, bptt:int, max_seq:int, *args, **kwargs): - self.max_seq,self.bptt = max_seq,bptt - super().__init__(*args, **kwargs) - - def concat(self, arrs:Collection[Tensor])->Tensor: - "Concatenate the `arrs` along the batch dimension." - return [torch.cat([l[si] for l in arrs], dim=1) for si in range_of(arrs[0])] - - def forward(self, input:LongTensor)->Tuple[Tensor,Tensor]: - bs,sl = input.size() - self.reset() - raw_outputs, outputs = [],[] - for i in range(0, sl, self.bptt): - r, o = super().forward(input[:,i: min(i+self.bptt, sl)]) - if i>(sl-self.max_seq): - raw_outputs.append(r) - outputs.append(o) - return self.concat(raw_outputs), self.concat(outputs) - -class BiPoolingLinearClassifier(PoolingLinearClassifier): - "Create a linear classifier with pooling." - - def forward(self, input:Tuple[Tensor,Tensor])->Tuple[Tensor,Tensor,Tensor]: - raw_outputs, outputs = input - output = outputs[-1] - if len(output.size()) == 3: - return super().forward(input) - elif len(output.size()) == 4: - bs, sl, em_sz, passes = output.size() - - f_avgpool = self.pool(output[...,0], bs, False) - f_mxpool = self.pool(output[...,0], bs, True) - b_avgpool = self.pool(output[..., 1], bs, False) - b_mxpool = self.pool(output[..., 1], bs, True) - x = torch.cat([output[:,-1,..., 0], f_mxpool, f_avgpool, - output[:,-1,..., 1], b_mxpool, b_avgpool,], 1) - x = self.layers(x) - return x, raw_outputs, outputs - - -class AvgPoolingLinearClassifier(nn.Module): - "Create a linear classifier with pooling." - - def __init__(self, layers:Collection[int], drops:Collection[float]): - super().__init__() - mod_layers = [] - activs = [nn.ReLU(inplace=True)] * (len(layers) - 2) + [None] - for n_in,n_out,p,actn in zip(layers[:-1],layers[1:], drops, activs): - mod_layers += bn_drop_lin(n_in, n_out, p=p, actn=actn) - self.layers = nn.Sequential(*mod_layers) - - def pool(self, x:Tensor, bs:int, is_max:bool): - "Pool the tensor along the seq_len dimension." - f = F.adaptive_max_pool1d if is_max else F.adaptive_avg_pool1d - return f(x.permute(1,2,0), (1,)).view(bs,-1) - - def forward(self, input:Tuple[Tensor,Tensor])->Tuple[Tensor,Tensor,Tensor]: - raw_outputs, outputs = input - output = outputs[-1] - if len(output.size()) == 3: - sl,bs,_ = output.size() - avgpool = self.pool(output, bs, False) - mxpool = self.pool(output, bs, True) - x = torch.cat([output[-1], mxpool, avgpool], 1) - x = self.layers(x) - return x, raw_outputs, outputs - elif len(output.size()) == 4: - sl, bs, em_sz, passes = output.size() - - avgpool = (self.pool(output[...,0], bs, False) + self.pool(output[..., 1], bs, False))/2 - mxpool = (self.pool(output[...,0], bs, True) +self.pool(output[..., 1], bs, True))/2 - x = torch.cat([(output[-1][..., 0]+output[-1][..., 1])/2, mxpool, avgpool], 1) - x = self.layers(x) - return x, raw_outputs, outputs - - -def get_bilm(vocab_sz:int, emb_sz:int, n_hid:int, n_layers:int, pad_token:int, tie_weights:bool=True, - qrnn:bool=False, bias:bool=True, bidir:bool=False, output_p:float=0.4, hidden_p:float=0.2, input_p:float=0.6, - embed_p:float=0.1, weight_p:float=0.5)->nn.Module: - "Create a two AWD-LSTM one for each direction " - fwd_rnn_enc = RNNCore(vocab_sz, emb_sz, n_hid=n_hid, n_layers=n_layers, pad_token=pad_token, qrnn=qrnn, bidir=bidir, - hidden_p=hidden_p, input_p=input_p, embed_p=embed_p, weight_p=weight_p) - bwd_rnn_enc = RNNCore(vocab_sz, emb_sz, n_hid=n_hid, n_layers=n_layers, pad_token=pad_token, qrnn=qrnn, bidir=bidir, - hidden_p=hidden_p, input_p=input_p, embed_p=embed_p, weight_p=weight_p) - enc = None - if tie_weights: - enc = fwd_rnn_enc.encoder - fwd_rnn_enc.encoder.weight = enc.weight - bwd_rnn_enc.encoder.weight = enc.weight - - return BiLMModel( - fwd_lm=SequentialRNN(fwd_rnn_enc, LinearDecoder(vocab_sz, emb_sz, output_p, tie_encoder=enc, bias=bias)), - bwd_lm=SequentialRNN(bwd_rnn_enc, LinearDecoder(vocab_sz, emb_sz, output_p, tie_encoder=enc, bias=bias)), - squash_bs_sl=True) - -def get_birnn_classifier(bptt:int, max_seq:int, n_class:int, vocab_sz:int, emb_sz:int, n_hid:int, n_layers:int, - pad_token:int, layers:Collection[int], drops:Collection[float], bidir:bool=False, qrnn:bool=False, - hidden_p:float=0.2, input_p:float=0.6, embed_p:float=0.1, weight_p:float=0.5, bicls_head:str='BiPoolingLinearClassifier')->nn.Module: - "Create a RNN classifier model." - fwd_rnn_enc = MultiBatchRNNCore(bptt, max_seq, vocab_sz, emb_sz, n_hid, n_layers, pad_token=pad_token, bidir=bidir, - qrnn=qrnn, hidden_p=hidden_p, input_p=input_p, embed_p=embed_p, weight_p=weight_p) - bwd_rnn_enc = MultiBatchRNNCore(bptt, max_seq, vocab_sz, emb_sz, n_hid, n_layers, pad_token=pad_token, bidir=bidir, - qrnn=qrnn, hidden_p=hidden_p, input_p=input_p, embed_p=embed_p, weight_p=weight_p) - - head = BiPoolingLinearClassifier - if bicls_head == 'BiPoolingLinearClassifier': head = BiPoolingLinearClassifier - elif bicls_head == 'AvgPoolingLinearClassifier': head = AvgPoolingLinearClassifier - - model = SequentialRNN(BiLMModel(fwd_rnn_enc, bwd_rnn_enc), head(layers, drops)) - model.reset() - return model - -#endregion diff --git a/fastai_contrib/text_data.py b/fastai_contrib/text_data.py new file mode 100644 index 0000000..c1e17f6 --- /dev/null +++ b/fastai_contrib/text_data.py @@ -0,0 +1,169 @@ +import os +import pathlib +import pickle +from functools import reduce +from typing import Collection, List + +from pandas import DataFrame +from sacremoses import MosesTokenizer + +import fastai +from fastai.basic_data import DataBunch + +from fastai.core import ListRules, PathOrStr, defaults, IntsOrStrs, is_listy +from fastai.data_block import ItemLists +from fastai.text import Tokenizer, BaseTokenizer, Vocab, SPProcessor, TextList, TextLMDataBunch + + +class MosesPreprocessingFunc(): + def __init__(self, lang: str): + self.mt = MosesTokenizer(lang) + + def __call__(self, t: str) -> str: + return self.mt.tokenize(t, return_str=True, escape=True) + + +class SentencePieceTokenizer(Tokenizer): + "Put together rules and a tokenizer function to tokenize text with multiprocessing." + def __init__(self, spm_model, lang:str='en', pre_rules:ListRules=None, + post_rules:ListRules=None, special_cases:Collection[str]=None, n_cpus:int=None): + # moses is added to preprocessing functions + super().__init__(self.tok_fun_with_sp, lang, pre_rules, post_rules, special_cases, n_cpus) + self.spm_model = spm_model + + def tok_fun_with_sp(self, lang): + try: + import sentencepiece as spm + except ImportError: + raise Exception('sentencepiece module is missing: run `pip install sentencepiece`') + tok = BaseTokenizer(lang) + tok.sp = spm.SentencePieceProcessor() + tok.sp.Load(str(self.spm_model)) + return tok + + def process_text(self, t:str, tok:BaseTokenizer) -> List[str]: + "Process one text `t` with tokenizer `tok`." + toks = super().process_text(t, tok) + toks = tok.sp.EncodeAsPieces(" ".join(toks)) + return toks + + +full_char_coverage_langs = ["bg", "cs", "da", "de", "el", "en", "es", "et", "fi", "fr", "ga", "hr", "hu", + "it","lt","lv","mt","nl","pl","pt","ro","sk","sl","sv"] # all European langus + + +def get_sentencepiece(cache_dir:PathOrStr, load_text, pre_rules: ListRules=None, post_rules:ListRules=None, + vocab_size:int=30000, model_type:str='unigram', input_sentence_size:int=1E7, lang='en', fixed_character_coverage=False): + try: + import sentencepiece as spm + except ImportError: + raise Exception('sentencepiece module is missing: run `pip install sentencepiece`') + + cache_dir = pathlib.Path(cache_dir) + pre_rules = pre_rules if pre_rules is not None else defaults.text_pre_rules + post_rules = post_rules if post_rules is not None else defaults.text_post_rules + + special_cases = defaults.text_spec_tok # + ['xxlink', 'xxuser', 'xxnumber', 'xxemoji', 'yyemoji'] + if not os.path.isfile(cache_dir / 'spm.model') or not os.path.isfile(cache_dir / f'itos.pkl'): + # load the text from the train tokens file + text = load_text() + text = filter(lambda x: len(x.rstrip(" ")), text) + text = (reduce(lambda t, rule: rule(t), pre_rules, line) for line in text) + def cleanup_n_postprocess(t): + t = t.split() + for r in post_rules: + t = r(t) + return ' '.join(t) + text = map(cleanup_n_postprocess, text) + raw_text_path = cache_dir / 'all_text.txt' + with open(raw_text_path, 'w') as f: f.write("\n".join(text)) + + if fixed_character_coverage: + char_coverage = 0.9995 + else: + char_coverage = 1 if lang in full_char_coverage_langs else 0.99 + + sp_params = [ + f"--input={raw_text_path}", + f"--character_coverage={char_coverage}", + f"--unk_id={len(special_cases)}", + f"--pad_id=-1", + f"--bos_id=-1", + f"--eos_id=-1", + f"--max_sentence_length=20480", + f"--input_sentence_size={int(input_sentence_size)}", + f"--user_defined_symbols={','.join(special_cases)}", + f"--model_prefix={cache_dir/'spm'}", + f"--vocab_size={vocab_size} --model_type={model_type}"] + spm.SentencePieceTrainer.Train(" ".join(sp_params)) + + with open(cache_dir / 'spm.vocab', 'r') as f: + vocab = [line.split('\t')[0] for line in f.readlines()] + + pickle.dump(vocab, open(cache_dir/ f'itos.pkl', 'wb')) + # todo add post rules + vocab = Vocab(pickle.load(open(cache_dir / f'itos.pkl', 'rb'))) + # We cannot use lambdas or local methods here, since `tok_func` needs to be + # pickle-able in order to be called in subprocesses when multithread tokenizing + tokenizer = SentencePieceTokenizer(cache_dir/'spm.model', + lang=lang, + pre_rules=pre_rules, + post_rules=post_rules) + return {'tokenizer': tokenizer, 'vocab': vocab} + +class SPProcessor2(SPProcessor): + def process(self, ds): + super().process(ds) + ds.vocab.sp_model = self.sp_model + ds.vocab.sp_vocab = self.sp_vocab + +def get_sentencepiece_fastai(cache_dir: PathOrStr, pre_rules: ListRules = None, + post_rules: ListRules = None, + vocab_size: int = 30000, lang='en'): + cache_dir = pathlib.Path(cache_dir) + + sp_model = cache_dir / 'spm.model' + if not sp_model.is_file(): + sp_model = None + sp_vocab = cache_dir / 'spm.vocab' + if not sp_vocab.is_file(): + sp_vocab = None + processor = SPProcessor2( + pre_rules=pre_rules, + post_rules=post_rules, + mark_fields=True, + vocab_sz=vocab_size, + sp_model=sp_model, + sp_vocab=sp_vocab, + lang=lang, + tmp_dir=cache_dir.absolute() # absolute make sure that dataset path is not added as prefix + ) + return {'processor': processor} + +# temporary loading function as from_df does not support processors +def make_data_bunch_from_df(cls, path: PathOrStr, train_df: DataFrame, valid_df: DataFrame, + tokenizer: Tokenizer = None, vocab: Vocab = None, classes: Collection[str] = None, + text_cols: IntsOrStrs = 1, + label_cols: IntsOrStrs = 0, label_delim: str = None, chunksize: int = 10000, + max_vocab: int = 60000, + min_freq: int = 2, mark_fields: bool = False, include_bos: bool = True, + include_eos: bool = False, processor=None, **kwargs) -> DataBunch: + "Create a `TextDataBunch` from DataFrames. `kwargs` are passed to the dataloader creation." + assert processor is None or tokenizer is None, "Processor and tokenizer are mutually exclusive." + + if processor is None: + processor = fastai.text.data._get_processor(tokenizer=tokenizer, vocab=vocab, chunksize=chunksize, max_vocab=max_vocab, + min_freq=min_freq, mark_fields=mark_fields, + include_bos=include_bos, include_eos=include_eos) + + if classes is None and is_listy(label_cols) and len(label_cols) > 1: classes = label_cols + src = ItemLists(path, TextList.from_df(train_df, path, cols=text_cols, processor=processor), + TextList.from_df(valid_df, path, cols=text_cols, processor=processor)) + if cls == TextLMDataBunch: + src = src.label_for_lm() + else: + if label_delim is not None: + src = src.label_from_df(cols=label_cols, classes=classes, label_delim=label_delim) + else: + src = src.label_from_df(cols=label_cols, classes=classes) + return src.databunch(**kwargs) \ No newline at end of file diff --git a/ulmfit/README.md b/ulmfit/README.md deleted file mode 100644 index 17b2388..0000000 --- a/ulmfit/README.md +++ /dev/null @@ -1,41 +0,0 @@ -# Todo -- [ ] Update these docs - -Getting Started ---- - -## Download and Extract the Wikipedia corpus - -In Linux, you can do all the following steps automatically with [prepare_wiki.sh](./prepare_wiki.sh) - -**Manual Instructions** - -We use the [WikiExtractor.py](http://medialab.di.unipi.it/wiki/Wikipedia_Extractor). It is a Python script that extracts and cleans text from a [Wikipedia database dump](http://download.wikimedia.org/). - -At the end of this step, you should have the following directory structure inside ulmfit: -```bash - -|- data - |- wiki - |- wiki_dumps - |- wiki_extr -|- wikiextractor -``` -The extracted data should be in the folder `wiki_extr` -> language name e.g.`en` (english), `fr` (french) `hi` (hindi) and so on. - -## Create and Post Process WikiText - -### Create and Post-Process -If you used the automated shell script from previous step, this might look something like -```bash -python create_wikitext.py -i data/wiki_extr/hi -o data/wiki/hi -l hi -``` -for hindi (unicode: 'hi') - -This should create two splits of your Wikimedia Dumps: a small and large one. - -_**Then**_, use the [postprocess_wikitext.py](./postprocess_wikitext.py) script to finish post processing. This processes numbers, builds a vocab, and limits the vocabulary size. This might look following for Hindi (`hi`) -```bash -python postprocess_wikitext.py data/wiki/hi-2 hi -python postprocess_wikitext.py data/wiki/hi-100 hi -``` diff --git a/ulmfit/__main__.py b/ulmfit/__main__.py index 9ffa500..fc9fa05 100644 --- a/ulmfit/__main__.py +++ b/ulmfit/__main__.py @@ -12,7 +12,6 @@ from .pretrain_lm import LMHyperParams, folder_name_to_model_name, DataSetParams from .train_clas import CLSHyperParams from pathlib import Path from string import Template -from fastai.metrics import fbeta import torch class FireView: @@ -146,6 +145,27 @@ class ULMFiT: lr_sched=lr_sched, **kwargs) + + def multifit_seeds(self, base, name=None, seed_name='clsweightseed', model_num=10, **kwargs): + if name is None: + name = folder_name_to_model_name(Path(base).name) + for seed in range(0, model_num, 1): + kwargs[seed_name] = seed + print("Seed: ", seed_name, seed) + self.multifit_eval(glob=base, name=name, num_lm_epochs=0, **kwargs) + + + def multifit_eval(self, glob, name=None, num_lm_epochs=20, num_cls_epochs=8, bs=20, lr_sched="1cycle", label_smoothing_eps=0.1, **kwargs): + return self.eval( + glob=glob, + name=name, + num_lm_epochs=num_lm_epochs, + num_cls_epochs=num_cls_epochs, + bs=bs, + lr_sched=lr_sched, + label_smoothing_eps=label_smoothing_eps, + **kwargs) + def ls(self, glob, dataset_template='${ds_name}'): data_dir = Path("data").absolute() glob = str(glob) @@ -252,8 +272,8 @@ class ULMFiT: if name is None: _name = folder_name_to_model_name(base_model.name) params = CLSHyperParams.from_lm(dataset_path, base_model, lang=lang, name=_name, **model_args) - last_model_dir = params.model_dir.relative_to(data_dir.parent) - if (params.model_dir/"cls_best.pth").exists(): + last_model_dir = params.model_path.relative_to(data_dir.parent) + if (params.model_path / "cls_best.pth").exists(): print("Evaluating previously trained model") d_tst = params.evaluate_cls(save_name=save_name, label_smoothing_eps=label_smoothing_eps, use_cache=True, mode="test") d_val = params.evaluate_cls(save_name=save_name, label_smoothing_eps=label_smoothing_eps, use_cache=True, mode="valid") @@ -264,12 +284,12 @@ class ULMFiT: print("Training") d = params.train_cls(num_lm_epochs=num_lm_epochs, label_smoothing_eps=label_smoothing_eps, **trn_params) else: - print("Skipping", (params.model_dir/"cls_best.pth")) - d = None + print("Skipping", (params.model_path / "cls_best.pth")) + d = None if d is not None: - d['model_dir_parent'] = params.model_dir.relative_to(data_dir.parent).parent + d['model_dir_parent'] = params.model_path.relative_to(data_dir.parent).parent d['model_name'] = params.model_name - np.save(params.model_dir / "results.npy", d) + np.save(params.model_path / "results.npy", d) results.append(d) del params except Exception as e: diff --git a/ulmfit/datasets/__init__.py b/ulmfit/datasets/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/ulmfit/datasets/__init__.py @@ -0,0 +1 @@ + diff --git a/ulmfit/create_wikitext.py b/ulmfit/datasets/create_wikitext.py similarity index 100% rename from ulmfit/create_wikitext.py rename to ulmfit/datasets/create_wikitext.py diff --git a/ulmfit/postprocess_wikitext.py b/ulmfit/datasets/postprocess_wikitext.py similarity index 98% rename from ulmfit/postprocess_wikitext.py rename to ulmfit/datasets/postprocess_wikitext.py index 481631c..7ae2c67 100644 --- a/ulmfit/postprocess_wikitext.py +++ b/ulmfit/datasets/postprocess_wikitext.py @@ -10,7 +10,7 @@ from pathlib import Path import fire -from fastai_contrib.utils import replace_number, UNK +from multifit.datasets.utils import replace_number, UNK def build_vocab(file_path, cutoff=3): diff --git a/fastai_contrib/utils.py b/ulmfit/datasets/utils.py similarity index 64% rename from fastai_contrib/utils.py rename to ulmfit/datasets/utils.py index 6583e25..dc96e9f 100644 --- a/fastai_contrib/utils.py +++ b/ulmfit/datasets/utils.py @@ -2,25 +2,18 @@ Utility methods for data processing. """ import fire -from fastai import * from fastai.text import * import shutil import pathlib import tarfile -from sklearn import model_selection -from sacremoses import MosesTokenizer from typing import Dict, Tuple, List +from fastai_contrib.text_data import SentencePieceTokenizer + EOS = 'xxeos' # fastai does not use eos, but we do SEP = 'xxsep' # special separator token for NLI -def replace_std_toks(x:str) -> str: - "Replace standard token names with fastai supported tokens" - # We change tokens to f'xx{token_name}' as it is not split by Moses tokenizer, - # while f'<{token_name}>' is being split to: '<' f'{token_name}' '>' - return x.replace('', UNK).replace('', BOS).replace('', EOS) - PAD_TOKEN_ID = 1 IMDB, XNLI, TRN, VAL, TST, EN = 'imdb', 'xnli', 'train', 'val', 'test', 'en' DATASETS = ['imdb', 'xnli'] @@ -35,124 +28,6 @@ CLASSES = ['neg', 'pos', 'unsup'] number_match_re = re.compile(r'^([0-9]+[,.]?)+$') number_split_re = re.compile(r'([,.])') -class MosesPreprocessingFunc(): - - def __init__(self, lang: str): - self.mt = MosesTokenizer(lang) - - def __call__(self, t: str) -> str: - return self.mt.tokenize(t, return_str=True, escape=True) - -class SentencePieceTokenizer(Tokenizer): - "Put together rules and a tokenizer function to tokenize text with multiprocessing." - def __init__(self, spm_model, lang:str='en', pre_rules:ListRules=None, - post_rules:ListRules=None, special_cases:Collection[str]=None, n_cpus:int=None): - # moses is added to preprocessing functions - super().__init__(self.tok_fun_with_sp, lang, pre_rules, post_rules, special_cases, n_cpus) - self.spm_model = spm_model - - def tok_fun_with_sp(self, lang): - try: - import sentencepiece as spm - except ImportError: - raise Exception('sentencepiece module is missing: run `pip install sentencepiece`') - tok = BaseTokenizer(lang) - tok.sp = spm.SentencePieceProcessor() - tok.sp.Load(str(self.spm_model)) - return tok - - def process_text(self, t:str, tok:BaseTokenizer) -> List[str]: - "Process one text `t` with tokenizer `tok`." - toks = super().process_text(t, tok) - toks = tok.sp.EncodeAsPieces(" ".join(toks)) - return toks -full_char_coverage_langs = ["bg", "cs", "da", "de", "el", "en", "es", "et", "fi", "fr", "ga", "hr", "hu", - "it","lt","lv","mt","nl","pl","pt","ro","sk","sl","sv"] # all European langus - -def get_sentencepiece(cache_dir:PathOrStr, load_text, pre_rules: ListRules=None, post_rules:ListRules=None, - vocab_size:int=30000, model_type:str='unigram', input_sentence_size:int=1E7, lang='en', fixed_character_coverage=False): - try: - import sentencepiece as spm - except ImportError: - raise Exception('sentencepiece module is missing: run `pip install sentencepiece`') - - cache_dir = pathlib.Path(cache_dir) - pre_rules = pre_rules if pre_rules is not None else defaults.text_pre_rules - post_rules = post_rules if post_rules is not None else defaults.text_post_rules - - special_cases = defaults.text_spec_tok # + ['xxlink', 'xxuser', 'xxnumber', 'xxemoji', 'yyemoji'] - if not os.path.isfile(cache_dir / 'spm.model') or not os.path.isfile(cache_dir / f'itos.pkl'): - # load the text from the train tokens file - text = load_text() - text = filter(lambda x: len(x.rstrip(" ")), text) - text = (reduce(lambda t, rule: rule(t), pre_rules, line) for line in text) - def cleanup_n_postprocess(t): - t = t.split() - for r in post_rules: - t = r(t) - return ' '.join(t) - text = map(cleanup_n_postprocess, text) - raw_text_path = cache_dir / 'all_text.txt' - with open(raw_text_path, 'w') as f: f.write("\n".join(text)) - - if fixed_character_coverage: - char_coverage = 0.9995 - else: - char_coverage = 1 if lang in full_char_coverage_langs else 0.99 - - sp_params = [ - f"--input={raw_text_path}", - f"--character_coverage={char_coverage}", - f"--unk_id={len(special_cases)}", - f"--pad_id=-1", - f"--bos_id=-1", - f"--eos_id=-1", - f"--max_sentence_length=20480", - f"--input_sentence_size={int(input_sentence_size)}", - f"--user_defined_symbols={','.join(special_cases)}", - f"--model_prefix={cache_dir/'spm'}", - f"--vocab_size={vocab_size} --model_type={model_type}"] - spm.SentencePieceTrainer.Train(" ".join(sp_params)) - - with open(cache_dir / 'spm.vocab', 'r') as f: - vocab = [line.split('\t')[0] for line in f.readlines()] - - pickle.dump(vocab, open(cache_dir/ f'itos.pkl', 'wb')) - # todo add post rules - vocab = Vocab(pickle.load(open(cache_dir / f'itos.pkl', 'rb'))) - # We cannot use lambdas or local methods here, since `tok_func` needs to be - # pickle-able in order to be called in subprocesses when multithread tokenizing - tokenizer = SentencePieceTokenizer(cache_dir/'spm.model', - lang=lang, - pre_rules=pre_rules, - post_rules=post_rules) - return {'tokenizer': tokenizer, 'vocab': vocab} - -def get_sentencepiece_fastai(cache_dir: PathOrStr, pre_rules: ListRules = None, - post_rules: ListRules = None, - vocab_size: int = 30000, lang='en'): - cache_dir = pathlib.Path(cache_dir) - - sp_model = cache_dir / 'spm.model' - if not sp_model.is_file(): - sp_model = None - - sp_vocab = cache_dir / 'spm.vocab' - if not sp_vocab.is_file(): - sp_vocab = None - - processor = SPProcessor( - pre_rules=pre_rules, - post_rules=post_rules, - mark_fields=True, - vocab_sz=vocab_size, - sp_model=sp_model, - sp_vocab=sp_vocab, - lang=lang, - tmp_dir=cache_dir.absolute() # absolute make sure that dataset path is not added as prefix - ) - return {'processor': processor} - def clear_cache_directory(path:PathOrStr, cache_name:str='tmp'): path = pathlib.Path(path) shutil.rmtree(path / cache_name) @@ -165,7 +40,6 @@ def get_texts(path): labels.append(idx) return np.array(texts), np.array(labels) - def ensure_paths_exists(*paths, message="One or more required files cannot be found."): error = False for path in paths: @@ -175,12 +49,6 @@ def ensure_paths_exists(*paths, message="One or more required files cannot be fo if error: raise FileNotFoundError(message) -def get_data_folder() -> Path: - """ - return data folder to use for future processing - """ - return (pathlib.Path(__file__).parent.parent / "data") - def get_scripts_folder(): """ return data folder to use for future processing @@ -339,8 +207,6 @@ def read_file(file_path, outname=None): df.to_csv(file_path.parent / f'{outname}.csv', header=False, index=False) return df - - def read_whitespace_file(filepath): """Reads a file and prepares the tokens.""" tokens = [] @@ -356,7 +222,6 @@ class DataStump: self.ids = ids self.loss_func = F.cross_entropy - def validate(model, ids, bptt=2000): """ Return the validation loss and perplexity of a model @@ -369,7 +234,7 @@ def validate(model, ids, bptt=2000): model.eval() model.reset() total_loss, num_examples = 0., 0 - for inputs, targets in tqdm(data): + for inputs, targets in data: outputs, raws, outs = model(to_device(inputs, None)) p_vocab = F.softmax(outputs, 1) for i, pv in enumerate(p_vocab): @@ -379,7 +244,6 @@ def validate(model, ids, bptt=2000): mean = total_loss / num_examples # divide by total number of tokens return mean, np.exp(mean) - class TextReader(): """ Returns a language model iterator that iterates through batches that are of length N(bptt,5) The first batch returned is always bptt+25; the max possible width. This is done because of they way that pytorch diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 67f49c9..2eadc46 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -4,28 +4,25 @@ expected to have been tokenized with Moses and processed with `postprocess_wikit That is, the data is expected to be white-space separated and numbers are expected to be split. """ -from dataclasses import InitVar, asdict +import pathlib +from dataclasses import asdict from string import Template import fastai -import fire from fastai import * -from fastai.callbacks import CSVLogger, SaveModelCallback +from fastai.callbacks import CSVLogger import fastai.text from fastai.text import * import torch -from fastai_contrib.utils import read_file, read_whitespace_file, \ - validate, PAD, UNK, get_sentencepiece, read_clas_data, TRN, VAL, TST, PAD_TOKEN_ID, \ - replace_std_toks, MosesPreprocessingFunc, get_sentencepiece_fastai -from fastai_contrib.learner import bilm_learner, accuracy_fwd, accuracy_bwd, bilm_text_classifier_learner +from ulmfit.datasets.utils import read_whitespace_file, \ + validate, UNK +from fastai_contrib.text_data import MosesPreprocessingFunc, get_sentencepiece, get_sentencepiece_fastai, \ + make_data_bunch_from_df import pickle from pathlib import Path -from collections import Counter -import fastai_contrib.data as contrib_data - LM_BEST = "lm_best" ENC_BEST = "enc_best" @@ -37,11 +34,9 @@ class Tokenizers(Enum): MOSES_FA='vf' FASTAI='f' - def istitle(line): return len(re.findall(r'^ ?= [^=]* = ?$', line)) != 0 - def read_wiki_articles(filename): if "reddit" in str(filename): # Temporary hack to handle poleval reddit dataset return pd.read_csv(filename, header=None, names=["texts"]).fillna("") @@ -89,35 +84,6 @@ class DataSetParams: except KeyError as e: raise KeyError(f"{e} , options:{repr(list(params.keys()))}") -# temporary loading function as from_df does not support processors -def make_data_bunch_from_df(cls, path: PathOrStr, train_df: DataFrame, valid_df: DataFrame, - tokenizer: Tokenizer = None, vocab: Vocab = None, classes: Collection[str] = None, - text_cols: IntsOrStrs = 1, - label_cols: IntsOrStrs = 0, label_delim: str = None, chunksize: int = 10000, - max_vocab: int = 60000, - min_freq: int = 2, mark_fields: bool = False, include_bos: bool = True, - include_eos: bool = False, processor=None, **kwargs) -> DataBunch: - "Create a `TextDataBunch` from DataFrames. `kwargs` are passed to the dataloader creation." - assert processor is None or tokenizer is None, "Processor and tokenizer are mutually exclusive." - - if processor is None: - processor = fastai.text.data._get_processor(tokenizer=tokenizer, vocab=vocab, chunksize=chunksize, max_vocab=max_vocab, - min_freq=min_freq, mark_fields=mark_fields, - include_bos=include_bos, include_eos=include_eos) - - if classes is None and is_listy(label_cols) and len(label_cols) > 1: classes = label_cols - src = ItemLists(path, TextList.from_df(train_df, path, cols=text_cols, processor=processor), - TextList.from_df(valid_df, path, cols=text_cols, processor=processor)) - if cls == TextLMDataBunch: - src = src.label_for_lm() - else: - if label_delim is not None: - src = src.label_from_df(cols=label_cols, classes=classes, label_delim=label_delim) - else: - src = src.label_from_df(cols=label_cols, classes=classes) - return src.databunch(**kwargs) - - @dataclass class LMHyperParams(DataSetParams): base_lm_path: str = None @@ -135,10 +101,6 @@ class LMHyperParams(DataSetParams): lmseed: int = None - # these hyperparameters are for training on ~100M tokens (e.g. WikiText-103) - # for training on smaller datasets, more dropout is necessary - # buggy dps = dict(output_p=0.25, hidden_p=0.1, input_p=0.2, embed_p=0.02, weight_p=0.15) # consider removing dps & clip from the default hyperparams and put them to train - dps = dict(input_p=0.25, output_p=0.1, weight_p=0.2, embed_p=0.02, hidden_p=0.15) # consider removing dps & clip from the default hyperparams and put them to train clip: float = 0.12 bptt: int = 70 # alpha and beta - defaults like in fastai/text/learner.py:RNNLearner() @@ -162,7 +124,7 @@ class LMHyperParams(DataSetParams): assert self.dataset_path.exists(), f"The dataset_path {self.dataset_path} does not exists" self.cache_dir = self.dataset_path / 'models' / self.tokenizer_prefix - self.model_dir = self.cache_dir / self.model_name + self.model_path = self.cache_dir / self.model_name if self.nh is None: self.nh = 1552 if self.qrnn else 1152 if self.name is None: self.name = self.lang @@ -191,15 +153,6 @@ class LMHyperParams(DataSetParams): @property def pretrained_fnames(self): return [self.base_lm_path / LM_BEST, self.base_lm_path / '../itos'] if self.base_lm_path else None - @property - def lm_type(self): - if self.bidir: - return contrib_data.LanguageModelType.BiLM - if self.backwards: - return contrib_data.LanguageModelType.BwdLM - else: - return contrib_data.LanguageModelType.FwdLM - def tokenizer_to_fastai_args(self, sp_data_func, use_moses): moses_preproc = [MosesPreprocessingFunc(self.lang)] if use_moses else [] if self.tokenizer is Tokenizers.FASTAI_SUBWORD: @@ -227,7 +180,7 @@ class LMHyperParams(DataSetParams): elif self.tokenizer is Tokenizers.MOSES: args = dict(tokenizer=Tokenizer(tok_func=BaseTokenizer, lang=self.lang, - pre_rules=moses_preproc + [replace_std_toks], + pre_rules=moses_preproc, post_rules=[])) elif self.tokenizer is Tokenizers.MOSES_FA: args = dict(tokenizer=Tokenizer(tok_func=BaseTokenizer, @@ -255,20 +208,20 @@ class LMHyperParams(DataSetParams): vals.pop('name', None) vals.pop('lang', None) vals['tokenizer'] = self.tokenizer.value - with (self.model_dir / 'info.json').open("w") as fp: json.dump(vals, fp) - print("Saving info", self.model_dir / 'info.json') + with (self.model_path / 'info.json').open("w") as fp: json.dump(vals, fp) + print("Saving info", self.model_path / 'info.json') - def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, lr=5e-3, label_smoothing_eps=0.0): + def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, label_smoothing_eps=0.0): print("Training lm") print('Max vocab:', self.max_vocab) print('Cache dir:', self.cache_dir) - print('Model dir:', self.model_dir) + print('Model dir:', self.model_path) if self.pretrained_fnames or self.pretrained_model: self.set_seed(self.ftseed, "fine-tune") else: self.set_seed(self.lmseed, "LM") - self.model_dir.mkdir(exist_ok=True, parents=True) + self.model_path.mkdir(exist_ok=True, parents=True) data_lm = self.load_wiki_data(bs=bs) if data_lm is None else data_lm learn = self.create_lm_learner(data_lm, drop_mult=drop_mult, label_smoothing_eps=label_smoothing_eps) print("Bptt", data_lm.bptt) @@ -290,8 +243,8 @@ class LMHyperParams(DataSetParams): else: print("Training lm from random weights") learn.unfreeze() - if not learn.true_wd: learn.fit_one_cycle(num_epochs, lr, (0.8, 0.7), wd=1e-7) - else: learn.fit_one_cycle(num_epochs, lr, (0.8, 0.7)) # TODO find proper values + if not learn.true_wd: learn.fit_one_cycle(num_epochs, 5e-3, (0.8, 0.7), wd=1e-7) + else: learn.fit_one_cycle(num_epochs, 5e-3, (0.8, 0.7)) # TODO find proper values learn.save("lm_best_with_opt", with_opt=True) learn.save_encoder(ENC_BEST) learn.save(LM_BEST, with_opt=False) @@ -303,13 +256,16 @@ class LMHyperParams(DataSetParams): def create_lm_learner(self, data_lm, dps=None, label_smoothing_eps=0.0, **kwargs): assert self.bidir == False, "bidirectional model is not yet supported" - config = dict(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, pad_token=PAD_TOKEN_ID, qrnn=self.qrnn, - tie_weights=True, out_bias=self.out_bias) - config.update(dps or self.dps) + config = awd_lstm_lm_config.copy() + config.update(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, qrnn=self.qrnn, + tie_weights=True, out_bias=self.out_bias) + if dps is not None: + config.update(dps) + trn_args = dict(clip=self.clip, alpha=self.rnn_alpha, beta=self.rnn_beta) trn_args.update(kwargs) - print ("Training args: ", trn_args, "dps: ", dps or self.dps) - learn = language_model_learner(data_lm, AWD_LSTM, config=config, model_dir=self.model_dir.relative_to(data_lm.path), pretrained=False, **trn_args) + print ("Training args: ", trn_args, "config: ", config) + learn = language_model_learner(data_lm, AWD_LSTM, config=config, model_dir=self.model_path.relative_to(data_lm.path), pretrained=False, **trn_args) if self.pretrained_model is not None: print("Loading pretrained model") model_path = untar_data(self.pretrained_model, data=False) @@ -323,10 +279,7 @@ class LMHyperParams(DataSetParams): learn.freeze() # compared to standard Adam, we set beta_1 to 0.8 learn.opt_fn = partial(optim.Adam, betas=(0.8, 0.99)) - learn.metrics = [accuracy_fwd, accuracy_bwd] if self.bidir else [accuracy] - learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/lm-history"), - # partial(SaveModelCallback, every='improvement', name='lm') disabled due to Memory issues - ] + learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/lm-history")] if label_smoothing_eps > 0.0: learn.loss_func = FlattenedLoss(LabelSmoothingCrossEntropy, eps=label_smoothing_eps) return learn @@ -337,7 +290,7 @@ class LMHyperParams(DataSetParams): return [line.rstrip('\n') for line in f] def load_wiki_data(self, bs=70): - self.model_dir.mkdir(exist_ok=True, parents=True) + self.model_path.mkdir(exist_ok=True, parents=True) trn_path = self.dataset_path / f'{self.lang}.wiki.train.tokens' val_path = self.dataset_path / f'{self.lang}.wiki.valid.tokens' tst_path = self.dataset_path / f'{self.lang}.wiki.test.tokens' @@ -382,11 +335,11 @@ class LMHyperParams(DataSetParams): else: print(f"Running tokenization {name}...") data = make_data_bunch_from_df(cls=bunch_class, path=self.cache_dir, - train_df=train_df, - valid_df=valid_df, - max_vocab=self.max_vocab, - bs=bs, - **args) + train_df=train_df, + valid_df=valid_df, + max_vocab=self.max_vocab, + bs=bs, + **args) data.save(name) with open(self.cache_dir/"itos.pkl", 'wb') as f: @@ -430,8 +383,9 @@ class LMHyperParams(DataSetParams): d['lang'] = infer_lang_from_dataset(dataset_path.name) return cls(**d) + def resolve_template(self, template, **additional_options): - return super().resolve_template(template, model_dir=self.model_dir, **additional_options) + return super().resolve_template(template, model_dir=self.model_path, **additional_options) def infer_lang_from_dataset(name:str): return name.split("-")[0] @@ -452,5 +406,8 @@ def validate_lm(self): logloss, perplexity = validate(learn.model, tst_ids, self.exp.bptt) print('Test logloss:', logloss.item(), 'perplexity:', perplexity.item()) -if __name__ == '__main__': - fire.Fire(LMHyperParams) +def get_data_folder() -> Path: + """ + return data folder to use for future processing + """ + return (pathlib.Path(__file__).parent.parent / "data") \ No newline at end of file diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 36220fc..47af4c5 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -7,14 +7,12 @@ import re from fastai.callbacks import CSVLogger, SaveModelCallback from fastai.text import * -from fastai_contrib.utils import PAD_TOKEN_ID +from ulmfit.datasets.utils import PAD_TOKEN_ID import fire from ulmfit.pretrain_lm import LMHyperParams, ENC_BEST -from sklearn.metrics import f1_score as f1s, precision_score, recall_score - @dataclass class CLSHyperParams(LMHyperParams): # dir_path -> data/imdb/ @@ -43,7 +41,7 @@ class CLSHyperParams(LMHyperParams): return '' @property - def need_fine_tune_lm(self): return not (self.model_dir/f"enc_best.pth").exists() + def need_fine_tune_lm(self): return not (self.model_path / f"enc_best.pth").exists() def lr_schedule_layered(self, learn, num_cls_epochs): learn.freeze_to(-1) @@ -123,15 +121,15 @@ class CLSHyperParams(LMHyperParams): f"{mode} Accuracy": results[6].item()} return {k:float(str(v)) for k,v in d.items()} # float(str(x)) to avoid float32 -> float64 conversion isssues - def train_cls(self, num_lm_epochs, unfreeze=True, num_cls_frozen_epochs=1, bs=40, drop_mul_lm=0.3, drop_mul_cls=0.5, + def train_cls(self, num_lm_epochs, unfreeze=True, bs=40, drop_mul_lm=0.3, drop_mul_cls=0.5, use_test_for_validation=False, num_cls_epochs=2, limit=None, noise=0.0, cls_max_len=20*70, lr_sched='layered', - label_smoothing_eps=0.0, random_init=False, dump_preds=None, early_stopping=True, weighted_cross_entropy=True): + label_smoothing_eps=0.0, random_init=False, early_stopping=True, weighted_cross_entropy=True): print("Training CLS") print('Max vocab:', self.max_vocab) print('Cache dir:', self.cache_dir) - print('Model dir:', self.model_dir) + print('Model dir:', self.model_path) assert use_test_for_validation == False, "use_test_for_validation=True is not supported" - self.model_dir.mkdir(exist_ok=True, parents=True) + self.model_path.mkdir(exist_ok=True, parents=True) if not unfreeze: num_cls_epochs = 1 @@ -139,7 +137,7 @@ class CLSHyperParams(LMHyperParams): data_clas, data_lm, data_tst = self.load_cls_data(bs, limit=limit, noise=noise) if self.need_fine_tune_lm and not random_init: - if not (self.model_dir/(ENC_BEST+".pth")).exists(): + if not (self.model_path / (ENC_BEST + ".pth")).exists(): self.train_lm(num_lm_epochs, data_lm=data_lm, drop_mult=drop_mul_lm, label_smoothing_eps=label_smoothing_eps) else: print("Language model already exist, skipping finetuning") @@ -180,7 +178,7 @@ class CLSHyperParams(LMHyperParams): def evaluate_cls(self, save_name='cls_best', bs=40, data_tst=None, learn=None, dump_preds=None, mode="test", label_smoothing_eps=None, use_cache=False): - cache_file = (self.model_dir / f'results_{mode+("" if save_name == "cls_best" else str(save_name))}.json') + cache_file = (self.model_path / f'results_{mode + ("" if save_name == "cls_best" else str(save_name))}.json') if use_cache and cache_file.exists(): with cache_file.open("r") as fp: return json.load(fp) @@ -212,7 +210,7 @@ class CLSHyperParams(LMHyperParams): if dump_preds: with open(dump_preds, 'w') as f: f.write('\n'.join([str(x) for x in preds])) - np.save(self.model_dir / f"preds-on-{mode}.npy", probs.cpu().numpy()) + np.save(self.model_path / f"preds-on-{mode}.npy", probs.cpu().numpy()) results = learn.validate(ds) print(f"Model: {self.name}") print(f"Evaluation on: {mode}") @@ -224,13 +222,16 @@ class CLSHyperParams(LMHyperParams): return labeled_results def create_cls_learner(self, data_clas, dps=None, label_smoothing_eps=0.0, random_init=False, early_stopping=True, **kwargs): - assert self.bidir == False, "bidirectional model is not yet supported" - config = dict(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, pad_token=PAD_TOKEN_ID, qrnn=self.qrnn) - config.update(dps or self.dps) - trn_args=dict(bptt=self.bptt, clip=self.clip) + + config = awd_lstm_clas_config.copy() + config.update(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, qrnn=self.qrnn) + if dps is not None: + config.update(dps) + + trn_args = dict(bptt=self.bptt, clip=self.clip) trn_args.update(kwargs) learn = text_classifier_learner(data_clas, AWD_LSTM, config=config, - pretrained=False, path=self.model_dir.parent, model_dir=self.model_dir.name, **trn_args) + pretrained=False, path=self.model_path.parent, model_dir=self.model_path.name, **trn_args) if self.pretrained_model is not None and not random_init: print("Loading pretrained model", self.pretrained_model) @@ -252,7 +253,7 @@ class CLSHyperParams(LMHyperParams): return learn def load_cls_data(self, bs, **kwargs): - self.model_dir.mkdir(exist_ok=True, parents=True) + self.model_path.mkdir(exist_ok=True, parents=True) add_trn_to_lm = True lang = self.lang use_moses = True @@ -338,7 +339,4 @@ class CLSHyperParams(LMHyperParams): return data_cls, data_lm, data_tst def cls_databunch(self, name, *args, **kwargs): - return self.databunch(name, bunch_class=TextClasDataBunch, *args, **kwargs) - -if __name__ == '__main__': - fire.Fire(CLSHyperParams) \ No newline at end of file + return self.databunch(name, bunch_class=TextClasDataBunch, *args, **kwargs) \ No newline at end of file