From 9f494b3ec6be72df06395ef4ea10dc9e0f0540b2 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Fri, 19 Apr 2019 10:32:35 +0200 Subject: [PATCH] Remove old bilm code that wasn't working --- fastai_contrib/data.py | 71 ---------------- fastai_contrib/learner.py | 113 -------------------------- fastai_contrib/models.py | 166 -------------------------------------- fastai_contrib/utils.py | 13 ++- tests/test_end_to_end.py | 45 ----------- ulmfit/pretrain_lm.py | 31 ++----- ulmfit/train_clas.py | 10 +-- 7 files changed, 15 insertions(+), 434 deletions(-) delete mode 100644 fastai_contrib/data.py delete mode 100644 fastai_contrib/learner.py delete mode 100644 fastai_contrib/models.py diff --git a/fastai_contrib/data.py b/fastai_contrib/data.py deleted file mode 100644 index 5f6e732..0000000 --- a/fastai_contrib/data.py +++ /dev/null @@ -1,71 +0,0 @@ -"NLP data loading pipeline. Supports csv, folders, and preprocessed data." -from fastai.text import * -from fastai.torch_core import * -from fastai.text.transform import * -from fastai.basic_data import * -from fastai.data_block import * - -#region Modified fastai classes - -LanguageModelType=Enum('LanguageModelType', 'FwdLM BwdLM BiLM') - -class LanguageModelLoader(): # copy of the original LanguageModelLoader - "Create a dataloader with bptt slightly changing." - def __init__(self, dataset:LabelList, bs:int=64, bptt:int=70, - lm_type:LanguageModelType=LanguageModelType.FwdLM, shuffle:bool=False, - max_len:int=25, p_bptt:int=0.95): - self.dataset,self.bs,self.bptt,self.lm_type,self.shuffle, self.p_bptt = dataset,bs,bptt,lm_type,shuffle,p_bptt - self.first,self.i,self.iter = True,0,0 - self.n = len(np.concatenate(dataset.x.items)) // self.bs if len(dataset.x.items) > 0 else 0 - self.max_len,self.num_workers = max_len,0 - self.init_kwargs = dict(bs=bs, bptt=bptt, lm_type=lm_type, shuffle=shuffle, max_len=max_len, p_bptt=p_bptt) - - def __iter__(self): - if getattr(self.dataset, 'item', None) is not None: - yield LongTensor(getattr(self.dataset, 'item'))[None],LongTensor([0]) - idx = np.random.permutation(len(self.dataset)) if self.shuffle else range(len(self.dataset)) - data = self.batchify(np.concatenate([self.dataset.x.items[i] for i in idx])) - - pos, itr = 0,0 - while pos < self.n-1 and itr int: return int(math.ceil((self.n-1) / self.bptt)) # so that it is always at least 1 - def __getattr__(self,k:str)->Any: return getattr(self.dataset, k) - - @property - def batch_size(self): return self.bs - @batch_size.setter - def batch_size(self, v): self.bs = v - - def batchify(self, data:np.ndarray) -> LongTensor: - "Split the corpus `data` in batches." - nb = data.shape[0] // self.bs - data = np.array(data[:nb*self.bs]).reshape(self.bs, -1) - if self.lm_type == LanguageModelType.BwdLM: data = data[:,::-1].copy() - elif self.lm_type == LanguageModelType.BiLM: data = np.stack([data, data[:,::-1].copy()], axis=2) - return LongTensor(data) - - def get_batch(self, data:LongTensor, i:int, seq_len:int) -> Tuple[LongTensor, LongTensor]: - "Create a batch at `i` of a given `seq_len`." - seq_len = min(seq_len, data.shape[1] - 1 - i) - x = data[:,i:i+seq_len] - y = data[:,i+1:i+1+seq_len] - y = y.contiguous().view(-1, 2) if self.lm_type == LanguageModelType.BiLM else y.contiguous().view(-1) - return x,y - -#endregion -#region Replaces fastai classes - -import fastai.text.data -fastai.text.data.LanguageModelLoader = LanguageModelLoader # Replace original LanguageModelLoader with new verion - -#endregion \ No newline at end of file diff --git a/fastai_contrib/learner.py b/fastai_contrib/learner.py deleted file mode 100644 index 67e79cb..0000000 --- a/fastai_contrib/learner.py +++ /dev/null @@ -1,113 +0,0 @@ -from torch.nn import CrossEntropyLoss - -from fastai import * -from fastai.text import * - -#region New code -from fastai_contrib.models import * - - -def bilm_learner(data:DataBunch, bptt:int=70, emb_sz:int=400, nh:int=1150, nl:int=3, pad_token:int=1, - drop_mult:float=1., tie_weights:bool=True, bias:bool=True, qrnn:bool=False, pretrained_model=None, - pretrained_fnames:OptStrTuple=None, **kwargs) -> 'LanguageLearner': - "Create a `Learner` with a language model." - dps = default_dropout['language'] * drop_mult - vocab_size = len(data.vocab.itos) - model = get_bilm(vocab_size, emb_sz, nh, nl, pad_token, input_p=dps[0], output_p=dps[1], - weight_p=dps[2], embed_p=dps[3], hidden_p=dps[4], tie_weights=tie_weights, bias=bias, qrnn=qrnn) - learn = LanguageLearner(data, model, bptt, split_func=bilm_split, **kwargs) - if pretrained_model is not None: - model_path = untar_data(pretrained_model, data=False) - fnames = [list(model_path.glob(f'*.{ext}'))[0] for ext in ['pth', 'pkl']] - learn.load_pretrained(*fnames) - learn.freeze() - if pretrained_fnames is not None: - fnames = [learn.path/learn.model_dir/f'{fn}.{ext}' for fn,ext in zip(pretrained_fnames, ['pth', 'pkl'])] - learn.load_pretrained(*fnames) - learn.freeze() - learn.loss_func = CrossEntropyLoss() # I'm not sure why fast ai is using CrossEntropyFlat but it breaks bilm - return learn - -def bilm_text_classifier_learner(data: DataBunch, bptt: int = 70, max_len: int = 70 * 20, emb_sz: int = 400, - nh: int = 1150, nl: int = 3, - lin_ftrs: Collection[int] = None, ps: Collection[float] = None, pad_token: int = 1, - drop_mult: float = 1., qrnn: bool = False, bicls_head:str='BiPoolingLinearClassifier', **kwargs) -> 'TextClassifierLearner': - "Create a RNN classifier." - dps = default_dropout['classifier'] * drop_mult - if lin_ftrs is None: lin_ftrs = [50] - if ps is None: ps = [0.1] - ds = data.train_ds - vocab_size, n_class = len(data.vocab.itos), data.c - if bicls_head == 'BiPoolingLinearClassifier': - count = 3*2 - else: - count = 3 - layers = [emb_sz * count] + lin_ftrs + [n_class] - ps = [dps[4]] + ps - model = get_birnn_classifier(bptt, max_len, n_class, vocab_size, emb_sz, nh, nl, pad_token, - layers, ps, input_p=dps[0], weight_p=dps[1], embed_p=dps[2], hidden_p=dps[3], - qrnn=qrnn, bicls_head=bicls_head) - learn = RNNLearner(data, model, bptt, split_func=birnn_classifier_split, **kwargs) - return learn - -def bilm_split(model:nn.Module) -> List[nn.Module]: - "Split a RNN `model` in groups for differential learning rates." - - return [f+b for f,b in zip(lm_split(model.fwd_lm),lm_split(model.bwd_lm))] - -def birnn_classifier_split(model:nn.Module) -> List[nn.Module]: - "Split a RNN `model` in groups for differential learning rates." - f_rnn,b_rnn = model[0].fwd_lm,model[0].bwd_lm - groups = [[f_rnn.encoder, f_rnn.encoder_dp,b_rnn.encoder, b_rnn.encoder_dp]] - groups += [a for a in zip(f_rnn.rnns, f_rnn.hidden_dps, b_rnn.rnns, b_rnn.hidden_dps, )] - groups.append([model[1]]) - return groups - -def accuracy_fwd(input, targs): - return accuracy(input[...,0], targs[...,0]) - -def accuracy_bwd(input, targs): - return accuracy(input[...,1], targs[...,1]) - - -#endregion -#region Modified fastai code - -def convert_weights(wgts:Weights, stoi_wgts:Dict[str,int], itos_new:Collection[str]) -> Weights: - "Convert the model weights to go with a new vocabulary." - if 'fwd_lm.0.encoder.weight' in wgts: #todo share embedding matrix computation - wgts = convert_weights_with_prefix(wgts, stoi_wgts, itos_new, prefix='fwd_lm.') - return convert_weights_with_prefix(wgts, stoi_wgts, itos_new, prefix='bwd_lm.') - else: - return convert_weights_with_prefix(wgts, stoi_wgts, itos_new, prefix='') - -def convert_weights_with_prefix(wgts:Weights, stoi_wgts:Dict[str,int], itos_new:Collection[str], prefix='') -> Weights: - "Convert the model weights to go with a new vocabulary." - if 'model' in wgts: - wgts['model'] = convert_weights_with_prefix(wgts['model'], stoi_wgts, itos_new, prefix) - else: - dec_bias, enc_wgts = wgts[prefix+'1.decoder.bias'], wgts[prefix+'0.encoder.weight'] - bias_m, wgts_m = dec_bias.mean(0), enc_wgts.mean(0) - new_w = enc_wgts.new_zeros((len(itos_new),enc_wgts.size(1))).zero_() - new_b = dec_bias.new_zeros((len(itos_new),)).zero_() - unk_tokens=[] - for i,w in enumerate(itos_new): - r = stoi_wgts[w] if w in stoi_wgts else -1 - if r < 0: - unk_tokens.append(w) - new_w[i] = enc_wgts[r] if r>=0 else wgts_m - new_b[i] = dec_bias[r] if r>=0 else bias_m - print(f"Unknown tokens {len(unk_tokens)}, first 100: {unk_tokens[:100]}") - wgts[prefix+'0.encoder.weight'] = new_w - wgts[prefix+'0.encoder_dp.emb.weight'] = new_w.clone() - wgts[prefix+'1.decoder.weight'] = new_w.clone() - wgts[prefix+'1.decoder.bias'] = new_b - return wgts - -#endregion -#region Replace code in fastai - -import fastai.text.learner -fastai.text.learner.convert_weights = convert_weights - -#endregion \ No newline at end of file diff --git a/fastai_contrib/models.py b/fastai_contrib/models.py deleted file mode 100644 index fbed679..0000000 --- a/fastai_contrib/models.py +++ /dev/null @@ -1,166 +0,0 @@ -from fastai.torch_core import * -from fastai.layers import * -from fastai.text.models import * - -#region New code - -class BiLMModel(nn.Module): - - def __init__(self, fwd_lm:nn.Module, bwd_lm:nn.Module, squash_bs_sl=False): - super().__init__() - self.fwd_lm = fwd_lm - self.bwd_lm = bwd_lm - self.squash_bs_sl = squash_bs_sl - - def __getitem__(self, idx): - return BiLMModel(self.fwd_lm[idx], self.bwd_lm[idx]) - - def __len__(self): - return len(self.fwd_lm) - - def stack(self, fwd_o, bwd_o): - if is_listy(fwd_o): - return [self.stack(f, b) for f,b in zip(fwd_o,bwd_o)] - else: - return torch.stack([fwd_o, bwd_o], dim=len(fwd_o.shape)) - - def forward(self, input): - if len(input.shape) == 3: # sl, bs, tracks - f = input[..., 0] - b = input[..., 1] - elif len(input.shape) == 2: # sl, bs - support during classification mode - f = input - b = torch.flip(input, [1]) # todo test if we are duplicating the backward pass correctly - else: - raise AttributeError(f"Inorrect size of input, {input.shape}") - fwd_o = self.fwd_lm(f) - bwd_o = self.bwd_lm(b) - - outs = self.stack(fwd_o, bwd_o) - if self.squash_bs_sl: - o = outs[0] - o = o.view(o.shape[0]*o.shape[1],o.shape[2],o.shape[3]) - outs[0] = o - return outs - - def reset(self): - "Reset the hidden states of underlaying lms." - self.fwd_lm.reset() - self.bwd_lm.reset() - -class MultiBatchBiLMModel(BiLMModel): - "Create a RNNCore module that can process a full sentence." - - def __init__(self, bptt:int, max_seq:int, *args, **kwargs): - self.max_seq,self.bptt = max_seq,bptt - super().__init__(*args, **kwargs) - - def concat(self, arrs:Collection[Tensor])->Tensor: - "Concatenate the `arrs` along the batch dimension." - return [torch.cat([l[si] for l in arrs], dim=1) for si in range_of(arrs[0])] - - def forward(self, input:LongTensor)->Tuple[Tensor,Tensor]: - bs,sl = input.size() - self.reset() - raw_outputs, outputs = [],[] - for i in range(0, sl, self.bptt): - r, o = super().forward(input[:,i: min(i+self.bptt, sl)]) - if i>(sl-self.max_seq): - raw_outputs.append(r) - outputs.append(o) - return self.concat(raw_outputs), self.concat(outputs) -#PoolingLinearClassifier -class BiPoolingLinearClassifier(nn.Module): - "Create a linear classifier with pooling." - - def forward(self, input:Tuple[Tensor,Tensor])->Tuple[Tensor,Tensor,Tensor]: - raw_outputs, outputs = input - output = outputs[-1] - if len(output.size()) == 3: - return super().forward(input) - elif len(output.size()) == 4: - bs, sl, em_sz, passes = output.size() - - f_avgpool = self.pool(output[...,0], bs, False) - f_mxpool = self.pool(output[...,0], bs, True) - b_avgpool = self.pool(output[..., 1], bs, False) - b_mxpool = self.pool(output[..., 1], bs, True) - x = torch.cat([output[:,-1,..., 0], f_mxpool, f_avgpool, - output[:,-1,..., 1], b_mxpool, b_avgpool,], 1) - x = self.layers(x) - return x, raw_outputs, outputs - - -class AvgPoolingLinearClassifier(nn.Module): - "Create a linear classifier with pooling." - - def __init__(self, layers:Collection[int], drops:Collection[float]): - super().__init__() - mod_layers = [] - activs = [nn.ReLU(inplace=True)] * (len(layers) - 2) + [None] - for n_in,n_out,p,actn in zip(layers[:-1],layers[1:], drops, activs): - mod_layers += bn_drop_lin(n_in, n_out, p=p, actn=actn) - self.layers = nn.Sequential(*mod_layers) - - def pool(self, x:Tensor, bs:int, is_max:bool): - "Pool the tensor along the seq_len dimension." - f = F.adaptive_max_pool1d if is_max else F.adaptive_avg_pool1d - return f(x.permute(1,2,0), (1,)).view(bs,-1) - - def forward(self, input:Tuple[Tensor,Tensor])->Tuple[Tensor,Tensor,Tensor]: - raw_outputs, outputs = input - output = outputs[-1] - if len(output.size()) == 3: - sl,bs,_ = output.size() - avgpool = self.pool(output, bs, False) - mxpool = self.pool(output, bs, True) - x = torch.cat([output[-1], mxpool, avgpool], 1) - x = self.layers(x) - return x, raw_outputs, outputs - elif len(output.size()) == 4: - sl, bs, em_sz, passes = output.size() - - avgpool = (self.pool(output[...,0], bs, False) + self.pool(output[..., 1], bs, False))/2 - mxpool = (self.pool(output[...,0], bs, True) +self.pool(output[..., 1], bs, True))/2 - x = torch.cat([(output[-1][..., 0]+output[-1][..., 1])/2, mxpool, avgpool], 1) - x = self.layers(x) - return x, raw_outputs, outputs - - -def get_bilm(vocab_sz:int, emb_sz:int, n_hid:int, n_layers:int, pad_token:int, tie_weights:bool=True, - qrnn:bool=False, bias:bool=True, bidir:bool=False, output_p:float=0.4, hidden_p:float=0.2, input_p:float=0.6, - embed_p:float=0.1, weight_p:float=0.5)->nn.Module: - "Create a two AWD-LSTM one for each direction " - fwd_rnn_enc = RNNCore(vocab_sz, emb_sz, n_hid=n_hid, n_layers=n_layers, pad_token=pad_token, qrnn=qrnn, bidir=bidir, - hidden_p=hidden_p, input_p=input_p, embed_p=embed_p, weight_p=weight_p) - bwd_rnn_enc = RNNCore(vocab_sz, emb_sz, n_hid=n_hid, n_layers=n_layers, pad_token=pad_token, qrnn=qrnn, bidir=bidir, - hidden_p=hidden_p, input_p=input_p, embed_p=embed_p, weight_p=weight_p) - enc = None - if tie_weights: - enc = fwd_rnn_enc.encoder - fwd_rnn_enc.encoder.weight = enc.weight - bwd_rnn_enc.encoder.weight = enc.weight - - return BiLMModel( - fwd_lm=SequentialRNN(fwd_rnn_enc, LinearDecoder(vocab_sz, emb_sz, output_p, tie_encoder=enc, bias=bias)), - bwd_lm=SequentialRNN(bwd_rnn_enc, LinearDecoder(vocab_sz, emb_sz, output_p, tie_encoder=enc, bias=bias)), - squash_bs_sl=True) - -def get_birnn_classifier(bptt:int, max_seq:int, n_class:int, vocab_sz:int, emb_sz:int, n_hid:int, n_layers:int, - pad_token:int, layers:Collection[int], drops:Collection[float], bidir:bool=False, qrnn:bool=False, - hidden_p:float=0.2, input_p:float=0.6, embed_p:float=0.1, weight_p:float=0.5, bicls_head:str='BiPoolingLinearClassifier')->nn.Module: - "Create a RNN classifier model." - fwd_rnn_enc = MultiBatchRNNCore(bptt, max_seq, vocab_sz, emb_sz, n_hid, n_layers, pad_token=pad_token, bidir=bidir, - qrnn=qrnn, hidden_p=hidden_p, input_p=input_p, embed_p=embed_p, weight_p=weight_p) - bwd_rnn_enc = MultiBatchRNNCore(bptt, max_seq, vocab_sz, emb_sz, n_hid, n_layers, pad_token=pad_token, bidir=bidir, - qrnn=qrnn, hidden_p=hidden_p, input_p=input_p, embed_p=embed_p, weight_p=weight_p) - - head = BiPoolingLinearClassifier - if bicls_head == 'BiPoolingLinearClassifier': head = BiPoolingLinearClassifier - elif bicls_head == 'AvgPoolingLinearClassifier': head = AvgPoolingLinearClassifier - - model = SequentialRNN(BiLMModel(fwd_rnn_enc, bwd_rnn_enc), head(layers, drops)) - model.reset() - return model - -#endregion \ No newline at end of file diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py index d0c93ea..1441ab0 100644 --- a/fastai_contrib/utils.py +++ b/fastai_contrib/utils.py @@ -1,17 +1,16 @@ """ Utility methods for data processing. """ -import fire -from fastai import * -from fastai.text import * - -import shutil import pathlib +import shutil import tarfile -from sklearn import model_selection -from sacremoses import MosesTokenizer from typing import Dict, Tuple, List +import fire +from sacremoses import MosesTokenizer + +from fastai.text import * + EOS = 'xxeos' # fastai does not use eos, but we do SEP = 'xxsep' # special separator token for NLI diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index 127c8fa..5c25258 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -136,51 +136,6 @@ def test_ulmfit_fastai_end_to_end_label_smoothing(): exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, label_smoothing_eps=0.1 ) - -def test_ulmfit_fastai_bidir_end_to_end(): - """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset. - """ - test_data, wt2 = get_test_data() - lm_name = 'end-to-end-test-fastai' - - exp = ulmfit.pretrain_lm.LMHyperParams( - dataset_path=wt2, - lang='en', - cuda_id=cuda_id, - qrnn=False, - bidir=True, - tokenizer='f', - max_vocab=100, - name=lm_name, - ) - exp.train_lm(num_epochs=1, bs=2) - exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(str(test_data / 'imdb'), str(exp.model_dir)) - exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, ) - -def test_ulmfit_moses_fa_bidir_end_to_end(): - """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset. - """ - test_data, wt2 = get_test_data() - lm_name = 'end-to-end-test-fastai' - - exp = ulmfit.pretrain_lm.LMHyperParams( - dataset_path=wt2, - lang='en', - cuda_id=cuda_id, - qrnn=False, - bidir=True, - tokenizer='vf', - max_vocab=100, - name=lm_name, - ) - exp.train_lm(num_epochs=1, bs=2) - exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) - exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, ) - -# def test_classification_model_work_with_different_dropmul(): -# learn = self.create_cls_learner(data_clas, drop_mult=0.1) -# learn = self.create_cls_learner(data_clas, drop_mult=0.0) - def test_ulmfit_sentencepiece_end_to_end(): """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset. """ diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 94efd8a..6b0b760 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -4,25 +4,14 @@ expected to have been tokenized with Moses and processed with `postprocess_wikit That is, the data is expected to be white-space separated and numbers are expected to be split. """ -from dataclasses import InitVar -import fastai import fire -from fastai import * -from fastai.callbacks import CSVLogger, SaveModelCallback +from fastai.callbacks import CSVLogger from fastai.text import * -import torch -from fastai_contrib.utils import read_file, read_whitespace_file, \ - validate, PAD, UNK, get_sentencepiece, read_clas_data, TRN, VAL, TST, PAD_TOKEN_ID, \ +from fastai_contrib.utils import read_whitespace_file, \ + validate, UNK, get_sentencepiece, PAD_TOKEN_ID, \ replace_std_toks, MosesPreprocessingFunc -from fastai_contrib.learner import bilm_learner, accuracy_fwd, accuracy_bwd, bilm_text_classifier_learner -import pickle - -from pathlib import Path - -from collections import Counter -import fastai_contrib.data as contrib_data LM_BEST = "lm_best" ENC_BEST = "enc_best" @@ -62,9 +51,9 @@ def json_load(f): @dataclass class LMHyperParams: - dataset_path: str # data_dir + dataset_path: Union[str, Path] # data_dir - base_lm_path: str = None + base_lm_path: Union[str, Path] = None backwards: str = False bidir: bool =False qrnn: bool = True @@ -131,15 +120,6 @@ class LMHyperParams: @property def pretrained_fnames(self): return [self.base_lm_path / LM_BEST, self.base_lm_path / '../itos'] if self.base_lm_path else None - @property - def lm_type(self): - if self.bidir: - return contrib_data.LanguageModelType.BiLM - if self.backwards: - return contrib_data.LanguageModelType.BwdLM - else: - return contrib_data.LanguageModelType.FwdLM - def tokenizer_to_fastai_args(self, sp_data_func, use_moses): moses_preproc = [MosesPreprocessingFunc(self.lang)] if use_moses else [] if self.tokenizer is Tokenizers.SUBWORD or self.tokenizer is Tokenizers.BROKENSUBWORD: @@ -234,7 +214,6 @@ class LMHyperParams: learn.freeze() # compared to standard Adam, we set beta_1 to 0.8 learn.opt_fn = partial(optim.Adam, betas=(0.8, 0.99)) - learn.metrics = [accuracy_fwd, accuracy_bwd] if self.bidir else [accuracy] learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/lm-history"), # partial(SaveModelCallback, every='improvement', name='lm') disabled due to Memory issues ] diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index b230387..2fac573 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -3,14 +3,12 @@ Train a classifier on top of a language model trained with `pretrain_lm.py`. Optionally fine-tune LM before. """ -from fastai.callbacks import CSVLogger -from fastai.text import * - -from fastai_contrib.utils import PAD_TOKEN_ID - import fire -from ulmfit.pretrain_lm import LMHyperParams, ENC_BEST, json_save +from fastai.callbacks import CSVLogger +from fastai.text import * +from fastai_contrib.utils import PAD_TOKEN_ID +from ulmfit.pretrain_lm import LMHyperParams, ENC_BEST class CLSHyperParams(LMHyperParams):