Clean up the old multfit training code

fastai_contrib/data.py

@@ -1,71 +0,0 @@
-"NLP data loading pipeline. Supports csv, folders, and preprocessed data."
-from fastai.text import *
-from fastai.torch_core import *
-from fastai.text.transform import *
-from fastai.basic_data import *
-from fastai.data_block import *
-
-#region Modified fastai classes
-
-LanguageModelType=Enum('LanguageModelType', 'FwdLM BwdLM BiLM')
-
-class LanguageModelLoader(): # copy of the original LanguageModelLoader
-    "Create a dataloader with bptt slightly changing."
-    def __init__(self, dataset:LabelList, bs:int=64, bptt:int=70,
-                 lm_type:LanguageModelType=LanguageModelType.FwdLM, shuffle:bool=False,
-                 max_len:int=25, p_bptt:int=0.95):
-        self.dataset,self.bs,self.bptt,self.lm_type,self.shuffle, self.p_bptt = dataset,bs,bptt,lm_type,shuffle,p_bptt
-        self.first,self.i,self.iter = True,0,0
-        self.n = len(np.concatenate(dataset.x.items)) // self.bs if len(dataset.x.items) > 0 else 0
-        self.max_len,self.num_workers = max_len,0
-        self.init_kwargs = dict(bs=bs, bptt=bptt, lm_type=lm_type, shuffle=shuffle, max_len=max_len, p_bptt=p_bptt)
-
-    def __iter__(self):
-        if getattr(self.dataset, 'item', None) is not None:
-            yield LongTensor(getattr(self.dataset, 'item'))[None],LongTensor([0])
-        idx = np.random.permutation(len(self.dataset)) if self.shuffle else range(len(self.dataset))
-        data = self.batchify(np.concatenate([self.dataset.x.items[i] for i in idx]))
-
-        pos, itr = 0,0
-        while pos < self.n-1 and itr<len(self):
-            if self.first and pos == 0: self.first,seq_len = False,self.bptt + self.max_len
-            else:
-                bptt = self.bptt if np.random.random() < self.p_bptt else self.bptt / 2.
-                seq_len = max(5, int(np.random.normal(bptt, 5)))
-                seq_len = min(seq_len, self.bptt + self.max_len)
-            res = self.get_batch(data, pos, seq_len)
-            pos += seq_len
-            itr += 1
-            yield res
-
-    def __len__(self) -> int: return int(math.ceil((self.n-1) / self.bptt)) # so that it is always at least 1
-    def __getattr__(self,k:str)->Any: return getattr(self.dataset, k)
-
-    @property
-    def batch_size(self): return self.bs
-    @batch_size.setter
-    def batch_size(self, v): self.bs = v
-
-    def batchify(self, data:np.ndarray) -> LongTensor:
-        "Split the corpus `data` in batches."
-        nb = data.shape[0] // self.bs
-        data = np.array(data[:nb*self.bs]).reshape(self.bs, -1)
-        if self.lm_type == LanguageModelType.BwdLM: data = data[:,::-1].copy()
-        elif self.lm_type == LanguageModelType.BiLM: data = np.stack([data, data[:,::-1].copy()], axis=2)
-        return LongTensor(data)
-
-    def get_batch(self, data:LongTensor,  i:int, seq_len:int) -> Tuple[LongTensor, LongTensor]:
-        "Create a batch at `i` of a given `seq_len`."
-        seq_len = min(seq_len, data.shape[1] - 1 - i)
-        x = data[:,i:i+seq_len]
-        y = data[:,i+1:i+1+seq_len]
-        y = y.contiguous().view(-1, 2) if self.lm_type == LanguageModelType.BiLM else y.contiguous().view(-1)
-        return x,y
-
-#endregion
-#region Replaces fastai classes
-
-import fastai.text.data
-fastai.text.data.LanguageModelLoader = LanguageModelLoader # Replace original LanguageModelLoader with new verion
-
-#endregion

fastai_contrib/learner.py

@@ -1,120 +0,0 @@
-from torch.nn import CrossEntropyLoss
-
-from fastai import *
-from fastai.text import *
-
-#region New code
-from fastai_contrib.models import *
-
-
-def bilm_learner(data:DataBunch, bptt:int=70, emb_sz:int=400, nh:int=1150, nl:int=3, pad_token:int=1,
-                  drop_mult:float=1., tie_weights:bool=True, bias:bool=True, qrnn:bool=False, pretrained_model=None,
-                  pretrained_fnames:OptStrTuple=None, **kwargs) -> 'LanguageLearner':
-    "Create a `Learner` with a language model."
-    dps = default_dropout['language'] * drop_mult
-    vocab_size = len(data.vocab.itos)
-    model = get_bilm(vocab_size, emb_sz, nh, nl, pad_token, input_p=dps[0], output_p=dps[1],
-                weight_p=dps[2], embed_p=dps[3], hidden_p=dps[4], tie_weights=tie_weights, bias=bias, qrnn=qrnn)
-    learn = LanguageLearner(data, model, bptt, split_func=bilm_split, **kwargs)
-    if pretrained_model is not None:
-        model_path = untar_data(pretrained_model, data=False)
-        fnames = [list(model_path.glob(f'*.{ext}'))[0] for ext in ['pth', 'pkl']]
-        learn.load_pretrained(*fnames)
-        learn.freeze()
-    if pretrained_fnames is not None:
-        fnames = [learn.path/learn.model_dir/f'{fn}.{ext}' for fn,ext in zip(pretrained_fnames, ['pth', 'pkl'])]
-        learn.load_pretrained(*fnames)
-        learn.freeze()
-    learn.loss_func = CrossEntropyLoss()  # I'm not sure why fast ai is using CrossEntropyFlat but it breaks bilm
-    return learn
-
-def bilm_text_classifier_learner(data: DataBunch, bptt: int = 70, max_len: int = 70 * 20, emb_sz: int = 400,
-                            nh: int = 1150, nl: int = 3,
-                            lin_ftrs: Collection[int] = None, ps: Collection[float] = None, pad_token: int = 1,
-                            drop_mult: float = 1., qrnn: bool = False, bicls_head:str='BiPoolingLinearClassifier',  **kwargs) -> 'TextClassifierLearner':
-    "Create a RNN classifier."
-    dps = default_dropout['classifier'] * drop_mult
-    if lin_ftrs is None: lin_ftrs = [50]
-    if ps is None:  ps = [0.1]
-    ds = data.train_ds
-    vocab_size, n_class = len(data.vocab.itos), data.c
-    if bicls_head == 'BiPoolingLinearClassifier':
-        count = 3*2
-    else:
-        count = 3
-    layers = [emb_sz * count] + lin_ftrs + [n_class]
-    ps = [dps[4]] + ps
-    model = get_birnn_classifier(bptt, max_len, n_class, vocab_size, emb_sz, nh, nl, pad_token,
-                               layers, ps, input_p=dps[0], weight_p=dps[1], embed_p=dps[2], hidden_p=dps[3],
-                               qrnn=qrnn, bicls_head=bicls_head)
-    learn = RNNLearner(data, model, bptt, split_func=birnn_classifier_split, **kwargs)
-    return learn
-
-def bilm_split(model:nn.Module) -> List[nn.Module]:
-    "Split a RNN `model` in groups for differential learning rates."
-
-    return [f+b for f,b in zip(lm_split(model.fwd_lm),lm_split(model.bwd_lm))]
-
-def birnn_classifier_split(model:nn.Module) -> List[nn.Module]:
-    "Split a RNN `model` in groups for differential learning rates."
-    f_rnn,b_rnn = model[0].fwd_lm,model[0].bwd_lm
-    groups = [[f_rnn.encoder, f_rnn.encoder_dp,b_rnn.encoder, b_rnn.encoder_dp]]
-    groups += [a for a in zip(f_rnn.rnns, f_rnn.hidden_dps, b_rnn.rnns, b_rnn.hidden_dps, )]
-    groups.append([model[1]])
-    return groups
-
-def accuracy_fwd(input, targs):
-    return accuracy(input[...,0], targs[...,0])
-
-def accuracy_bwd(input, targs):
-    return accuracy(input[...,1], targs[...,1])
-
-
-#endregion
-#region Modified fastai code
-
-def convert_weights(wgts:Weights, stoi_wgts:Dict[str,int], itos_new:Collection[str]) -> Weights:
-    "Convert the model weights to go with a new vocabulary."
-    if 'fwd_lm.0.encoder.weight' in wgts: #todo share embedding matrix computation
-        wgts = convert_weights_with_prefix(wgts, stoi_wgts, itos_new, prefix='fwd_lm.')
-        return convert_weights_with_prefix(wgts, stoi_wgts, itos_new, prefix='bwd_lm.')
-    else:
-        return convert_weights_with_prefix(wgts, stoi_wgts, itos_new, prefix='')
-
-def convert_weights_with_prefix(wgts:Weights, stoi_wgts:Dict[str,int], itos_new:Collection[str], prefix='') -> Weights:
-    "Convert the model weights to go with a new vocabulary."
-    if 'model' in wgts:
-        wgts['model'] = convert_weights_with_prefix(wgts['model'], stoi_wgts, itos_new, prefix)
-    else:
-        #dec_bias, enc_wgts = wgts[prefix+'1.decoder.bias'], wgts[prefix+'0.encoder.weight']
-        has_bias = prefix+'1.decoder.bias' in wgts
-        enc_wgts = wgts[prefix+'0.encoder.weight']
-        if has_bias:
-          dec_bias = wgts[prefix+'1.decoder.bias']
-        else:
-          dec_bias = enc_wgts.new_zeros((len(stoi_wgts),))
-        bias_m, wgts_m = dec_bias.mean(0), enc_wgts.mean(0)
-        new_w = enc_wgts.new_zeros((len(itos_new),enc_wgts.size(1))).zero_()
-        new_b = dec_bias.new_zeros((len(itos_new),)).zero_()
-        unk_tokens=[]
-        for i,w in enumerate(itos_new):
-            r = stoi_wgts[w] if w in stoi_wgts else -1
-            if r < 0:
-                unk_tokens.append(w)
-            new_w[i] = enc_wgts[r] if r>=0 else wgts_m
-            new_b[i] = dec_bias[r] if r>=0 else bias_m
-        print(f"Unknown tokens {len(unk_tokens)}, first 100: {unk_tokens[:100]}")
-        wgts[prefix+'0.encoder.weight'] = new_w
-        wgts[prefix+'0.encoder_dp.emb.weight'] = new_w.clone()
-        wgts[prefix+'1.decoder.weight'] = new_w.clone()
-        if has_bias:
-          wgts[prefix+'1.decoder.bias'] = new_b
-    return wgts
-
-#endregion
-#region Replace code in fastai
-
-import fastai.text.learner
-fastai.text.learner.convert_weights = convert_weights
-
-#endregion

fastai_contrib/models.py

@@ -1,166 +0,0 @@
-from fastai.torch_core import *
-from fastai.layers import *
-from fastai.text.models import *
-from fastai.text.learner import *
-#region New code
-
-class BiLMModel(nn.Module):
-
-    def __init__(self, fwd_lm:nn.Module, bwd_lm:nn.Module, squash_bs_sl=False):
-        super().__init__()
-        self.fwd_lm = fwd_lm
-        self.bwd_lm = bwd_lm
-        self.squash_bs_sl = squash_bs_sl
-
-    def __getitem__(self, idx):
-        return BiLMModel(self.fwd_lm[idx], self.bwd_lm[idx])
-
-    def __len__(self):
-        return len(self.fwd_lm)
-
-    def stack(self, fwd_o, bwd_o):
-        if is_listy(fwd_o):
-            return [self.stack(f, b) for f,b in zip(fwd_o,bwd_o)]
-        else:
-            return torch.stack([fwd_o, bwd_o], dim=len(fwd_o.shape))
-
-    def forward(self, input):
-        if len(input.shape) == 3: # sl, bs, tracks
-            f = input[..., 0]
-            b = input[..., 1]
-        elif len(input.shape) == 2: # sl, bs - support during classification mode
-            f = input
-            b = torch.flip(input, [1]) # todo test if we are duplicating the backward pass correctly
-        else:
-            raise AttributeError(f"Inorrect size of input, {input.shape}")
-        fwd_o = self.fwd_lm(f)
-        bwd_o = self.bwd_lm(b)
-
-        outs = self.stack(fwd_o, bwd_o)
-        if self.squash_bs_sl:
-            o = outs[0]
-            o = o.view(o.shape[0]*o.shape[1],o.shape[2],o.shape[3])
-            outs[0] = o
-        return outs
-
-    def reset(self):
-        "Reset the hidden states of underlaying lms."
-        self.fwd_lm.reset()
-        self.bwd_lm.reset()
-
-class MultiBatchBiLMModel(BiLMModel):
-    "Create a RNNCore module that can process a full sentence."
-
-    def __init__(self, bptt:int, max_seq:int, *args, **kwargs):
-        self.max_seq,self.bptt = max_seq,bptt
-        super().__init__(*args, **kwargs)
-
-    def concat(self, arrs:Collection[Tensor])->Tensor:
-        "Concatenate the `arrs` along the batch dimension."
-        return [torch.cat([l[si] for l in arrs], dim=1) for si in range_of(arrs[0])]
-
-    def forward(self, input:LongTensor)->Tuple[Tensor,Tensor]:
-        bs,sl = input.size()
-        self.reset()
-        raw_outputs, outputs = [],[]
-        for i in range(0, sl, self.bptt):
-            r, o = super().forward(input[:,i: min(i+self.bptt, sl)])
-            if i>(sl-self.max_seq):
-                raw_outputs.append(r)
-                outputs.append(o)
-        return self.concat(raw_outputs), self.concat(outputs)
-
-class BiPoolingLinearClassifier(PoolingLinearClassifier):
-    "Create a linear classifier with pooling."
-
-    def forward(self, input:Tuple[Tensor,Tensor])->Tuple[Tensor,Tensor,Tensor]:
-        raw_outputs, outputs = input
-        output = outputs[-1]
-        if len(output.size()) == 3:
-            return super().forward(input)
-        elif len(output.size()) == 4:
-            bs, sl, em_sz, passes = output.size()
-
-            f_avgpool = self.pool(output[...,0], bs, False)
-            f_mxpool = self.pool(output[...,0], bs, True)
-            b_avgpool = self.pool(output[..., 1], bs, False)
-            b_mxpool = self.pool(output[..., 1], bs, True)
-            x = torch.cat([output[:,-1,..., 0], f_mxpool, f_avgpool,
-                           output[:,-1,..., 1], b_mxpool, b_avgpool,], 1)
-            x = self.layers(x)
-            return x, raw_outputs, outputs
-
-
-class AvgPoolingLinearClassifier(nn.Module):
-    "Create a linear classifier with pooling."
-
-    def __init__(self, layers:Collection[int], drops:Collection[float]):
-        super().__init__()
-        mod_layers = []
-        activs = [nn.ReLU(inplace=True)] * (len(layers) - 2) + [None]
-        for n_in,n_out,p,actn in zip(layers[:-1],layers[1:], drops, activs):
-            mod_layers += bn_drop_lin(n_in, n_out, p=p, actn=actn)
-        self.layers = nn.Sequential(*mod_layers)
-
-    def pool(self, x:Tensor, bs:int, is_max:bool):
-        "Pool the tensor along the seq_len dimension."
-        f = F.adaptive_max_pool1d if is_max else F.adaptive_avg_pool1d
-        return f(x.permute(1,2,0), (1,)).view(bs,-1)
-
-    def forward(self, input:Tuple[Tensor,Tensor])->Tuple[Tensor,Tensor,Tensor]:
-        raw_outputs, outputs = input
-        output = outputs[-1]
-        if len(output.size()) == 3:
-            sl,bs,_ = output.size()
-            avgpool = self.pool(output, bs, False)
-            mxpool = self.pool(output, bs, True)
-            x = torch.cat([output[-1], mxpool, avgpool], 1)
-            x = self.layers(x)
-            return x, raw_outputs, outputs
-        elif len(output.size()) == 4:
-            sl, bs, em_sz, passes = output.size()
-
-            avgpool = (self.pool(output[...,0], bs, False) + self.pool(output[..., 1], bs, False))/2
-            mxpool = (self.pool(output[...,0], bs, True) +self.pool(output[..., 1], bs, True))/2
-            x = torch.cat([(output[-1][..., 0]+output[-1][..., 1])/2, mxpool, avgpool], 1)
-            x = self.layers(x)
-            return x, raw_outputs, outputs
-
-
-def get_bilm(vocab_sz:int, emb_sz:int, n_hid:int, n_layers:int, pad_token:int, tie_weights:bool=True,
-                       qrnn:bool=False, bias:bool=True, bidir:bool=False, output_p:float=0.4, hidden_p:float=0.2, input_p:float=0.6,
-                       embed_p:float=0.1, weight_p:float=0.5)->nn.Module:
-    "Create a two AWD-LSTM one for each direction "
-    fwd_rnn_enc = RNNCore(vocab_sz, emb_sz, n_hid=n_hid, n_layers=n_layers, pad_token=pad_token, qrnn=qrnn, bidir=bidir,
-                          hidden_p=hidden_p, input_p=input_p, embed_p=embed_p, weight_p=weight_p)
-    bwd_rnn_enc = RNNCore(vocab_sz, emb_sz, n_hid=n_hid, n_layers=n_layers, pad_token=pad_token, qrnn=qrnn, bidir=bidir,
-                          hidden_p=hidden_p, input_p=input_p, embed_p=embed_p, weight_p=weight_p)
-    enc = None
-    if tie_weights:
-        enc = fwd_rnn_enc.encoder
-        fwd_rnn_enc.encoder.weight = enc.weight
-        bwd_rnn_enc.encoder.weight = enc.weight
-
-    return BiLMModel(
-        fwd_lm=SequentialRNN(fwd_rnn_enc, LinearDecoder(vocab_sz, emb_sz, output_p, tie_encoder=enc, bias=bias)),
-        bwd_lm=SequentialRNN(bwd_rnn_enc, LinearDecoder(vocab_sz, emb_sz, output_p, tie_encoder=enc, bias=bias)),
-        squash_bs_sl=True)
-
-def get_birnn_classifier(bptt:int, max_seq:int, n_class:int, vocab_sz:int, emb_sz:int, n_hid:int, n_layers:int,
-                       pad_token:int, layers:Collection[int], drops:Collection[float], bidir:bool=False, qrnn:bool=False,
-                       hidden_p:float=0.2, input_p:float=0.6, embed_p:float=0.1, weight_p:float=0.5, bicls_head:str='BiPoolingLinearClassifier')->nn.Module:
-    "Create a RNN classifier model."
-    fwd_rnn_enc = MultiBatchRNNCore(bptt, max_seq, vocab_sz, emb_sz, n_hid, n_layers, pad_token=pad_token, bidir=bidir,
-                      qrnn=qrnn, hidden_p=hidden_p, input_p=input_p, embed_p=embed_p, weight_p=weight_p)
-    bwd_rnn_enc = MultiBatchRNNCore(bptt, max_seq, vocab_sz, emb_sz, n_hid, n_layers, pad_token=pad_token, bidir=bidir,
-                                qrnn=qrnn, hidden_p=hidden_p, input_p=input_p, embed_p=embed_p, weight_p=weight_p)
-
-    head = BiPoolingLinearClassifier
-    if bicls_head == 'BiPoolingLinearClassifier': head = BiPoolingLinearClassifier
-    elif bicls_head == 'AvgPoolingLinearClassifier': head = AvgPoolingLinearClassifier
-
-    model = SequentialRNN(BiLMModel(fwd_rnn_enc, bwd_rnn_enc), head(layers, drops))
-    model.reset()
-    return model
-
-#endregion

fastai_contrib/text_data.py

@@ -0,0 +1,169 @@
+import os
+import pathlib
+import pickle
+from functools import reduce
+from typing import Collection, List
+
+from pandas import DataFrame
+from sacremoses import MosesTokenizer
+
+import fastai
+from fastai.basic_data import DataBunch
+
+from fastai.core import ListRules, PathOrStr, defaults, IntsOrStrs, is_listy
+from fastai.data_block import ItemLists
+from fastai.text import Tokenizer, BaseTokenizer, Vocab, SPProcessor, TextList, TextLMDataBunch
+
+
+class MosesPreprocessingFunc():
+    def __init__(self, lang: str):
+        self.mt = MosesTokenizer(lang)
+
+    def __call__(self, t: str) -> str:
+        return self.mt.tokenize(t, return_str=True, escape=True)
+
+
+class SentencePieceTokenizer(Tokenizer):
+    "Put together rules and a tokenizer function to tokenize text with multiprocessing."
+    def __init__(self, spm_model, lang:str='en', pre_rules:ListRules=None,
+                 post_rules:ListRules=None, special_cases:Collection[str]=None, n_cpus:int=None):
+        # moses is added to preprocessing functions
+        super().__init__(self.tok_fun_with_sp, lang, pre_rules, post_rules, special_cases, n_cpus)
+        self.spm_model = spm_model
+
+    def tok_fun_with_sp(self, lang):
+        try:
+            import sentencepiece as spm
+        except ImportError:
+            raise Exception('sentencepiece module is missing: run `pip install sentencepiece`')
+        tok = BaseTokenizer(lang)
+        tok.sp = spm.SentencePieceProcessor()
+        tok.sp.Load(str(self.spm_model))
+        return tok
+
+    def process_text(self, t:str, tok:BaseTokenizer) -> List[str]:
+        "Process one text `t` with tokenizer `tok`."
+        toks = super().process_text(t, tok)
+        toks = tok.sp.EncodeAsPieces(" ".join(toks))
+        return toks
+
+
+full_char_coverage_langs = ["bg", "cs", "da", "de", "el", "en", "es", "et", "fi", "fr", "ga", "hr", "hu",
+                       "it","lt","lv","mt","nl","pl","pt","ro","sk","sl","sv"] # all European langus
+
+
+def get_sentencepiece(cache_dir:PathOrStr, load_text, pre_rules: ListRules=None, post_rules:ListRules=None,
+                      vocab_size:int=30000, model_type:str='unigram', input_sentence_size:int=1E7, lang='en', fixed_character_coverage=False):
+    try:
+        import sentencepiece as spm
+    except ImportError:
+        raise Exception('sentencepiece module is missing: run `pip install sentencepiece`')
+
+    cache_dir = pathlib.Path(cache_dir)
+    pre_rules = pre_rules if pre_rules is not None else defaults.text_pre_rules
+    post_rules = post_rules if post_rules is not None else defaults.text_post_rules
+
+    special_cases = defaults.text_spec_tok # + ['xxlink', 'xxuser', 'xxnumber', 'xxemoji', 'yyemoji']
+    if not os.path.isfile(cache_dir / 'spm.model') or not os.path.isfile(cache_dir / f'itos.pkl'):
+        # load the text from the train tokens file
+        text = load_text()
+        text = filter(lambda x: len(x.rstrip(" ")), text)
+        text = (reduce(lambda t, rule: rule(t), pre_rules, line) for line in text)
+        def cleanup_n_postprocess(t):
+            t = t.split()
+            for r in post_rules:
+                t = r(t)
+            return ' '.join(t)
+        text = map(cleanup_n_postprocess, text)
+        raw_text_path = cache_dir / 'all_text.txt'
+        with open(raw_text_path, 'w') as f: f.write("\n".join(text))
+
+        if fixed_character_coverage:
+            char_coverage = 0.9995
+        else:
+            char_coverage = 1 if lang in full_char_coverage_langs else 0.99
+
+        sp_params = [
+            f"--input={raw_text_path}",
+            f"--character_coverage={char_coverage}",
+            f"--unk_id={len(special_cases)}",
+            f"--pad_id=-1",
+            f"--bos_id=-1",
+            f"--eos_id=-1",
+            f"--max_sentence_length=20480",
+            f"--input_sentence_size={int(input_sentence_size)}",
+            f"--user_defined_symbols={','.join(special_cases)}",
+            f"--model_prefix={cache_dir/'spm'}",
+            f"--vocab_size={vocab_size} --model_type={model_type}"]
+        spm.SentencePieceTrainer.Train(" ".join(sp_params))
+
+        with open(cache_dir / 'spm.vocab', 'r') as f:
+            vocab = [line.split('\t')[0] for line in f.readlines()]
+
+        pickle.dump(vocab, open(cache_dir/ f'itos.pkl', 'wb'))
+    # todo add post rules
+    vocab = Vocab(pickle.load(open(cache_dir / f'itos.pkl', 'rb')))
+    # We cannot use lambdas or local methods here, since `tok_func` needs to be
+    # pickle-able in order to be called in subprocesses when multithread tokenizing
+    tokenizer = SentencePieceTokenizer(cache_dir/'spm.model',
+                                lang=lang,
+                                pre_rules=pre_rules,
+                                post_rules=post_rules)
+    return {'tokenizer': tokenizer, 'vocab': vocab}
+
+class SPProcessor2(SPProcessor):
+    def process(self, ds):
+        super().process(ds)
+        ds.vocab.sp_model = self.sp_model
+        ds.vocab.sp_vocab = self.sp_vocab
+
+def get_sentencepiece_fastai(cache_dir: PathOrStr,  pre_rules: ListRules = None,
+                          post_rules: ListRules = None,
+                          vocab_size: int = 30000, lang='en'):
+    cache_dir = pathlib.Path(cache_dir)
+
+    sp_model = cache_dir / 'spm.model'
+    if not sp_model.is_file():
+        sp_model = None
+    sp_vocab = cache_dir / 'spm.vocab'
+    if not sp_vocab.is_file():
+        sp_vocab = None
+    processor = SPProcessor2(
+        pre_rules=pre_rules,
+        post_rules=post_rules,
+        mark_fields=True,
+        vocab_sz=vocab_size,
+        sp_model=sp_model,
+        sp_vocab=sp_vocab,
+        lang=lang,
+        tmp_dir=cache_dir.absolute()  # absolute make sure that dataset path is not added as prefix
+    )
+    return {'processor': processor}
+
+# temporary loading function as from_df does not support processors
+def make_data_bunch_from_df(cls, path: PathOrStr, train_df: DataFrame, valid_df: DataFrame,
+            tokenizer: Tokenizer = None, vocab: Vocab = None, classes: Collection[str] = None,
+            text_cols: IntsOrStrs = 1,
+            label_cols: IntsOrStrs = 0, label_delim: str = None, chunksize: int = 10000,
+            max_vocab: int = 60000,
+            min_freq: int = 2, mark_fields: bool = False, include_bos: bool = True,
+            include_eos: bool = False, processor=None, **kwargs) -> DataBunch:
+    "Create a `TextDataBunch` from DataFrames. `kwargs` are passed to the dataloader creation."
+    assert processor is None or tokenizer is None, "Processor and tokenizer are mutually exclusive."
+
+    if processor is None:
+        processor = fastai.text.data._get_processor(tokenizer=tokenizer, vocab=vocab, chunksize=chunksize, max_vocab=max_vocab,
+                                   min_freq=min_freq, mark_fields=mark_fields,
+                                   include_bos=include_bos, include_eos=include_eos)
+
+    if classes is None and is_listy(label_cols) and len(label_cols) > 1: classes = label_cols
+    src = ItemLists(path, TextList.from_df(train_df, path, cols=text_cols, processor=processor),
+                    TextList.from_df(valid_df, path, cols=text_cols, processor=processor))
+    if cls == TextLMDataBunch:
+        src = src.label_for_lm()
+    else:
+        if label_delim is not None:
+            src = src.label_from_df(cols=label_cols, classes=classes, label_delim=label_delim)
+        else:
+            src = src.label_from_df(cols=label_cols, classes=classes)
+    return src.databunch(**kwargs)

ulmfit/README.md

@@ -1,41 +0,0 @@
-# Todo 
-- [ ] Update these docs
-
-Getting Started
----
-
-## Download and Extract the Wikipedia corpus
-
-In Linux, you can do all the following steps automatically with [prepare_wiki.sh](./prepare_wiki.sh)
-
-**Manual Instructions**
-
-We use the [WikiExtractor.py](http://medialab.di.unipi.it/wiki/Wikipedia_Extractor). It is a Python script that extracts and cleans text from a [Wikipedia database dump](http://download.wikimedia.org/).
-
-At the end of this step, you should have the following directory structure inside ulmfit:
-```bash
-
-|- data
-  |- wiki
-  |- wiki_dumps
-  |- wiki_extr
-|- wikiextractor
-```
-The extracted data should be in the folder `wiki_extr` -> language name e.g.`en` (english), `fr` (french) `hi` (hindi) and so on. 
-
-## Create and Post Process WikiText
-
-### Create and Post-Process 
-If you used the automated shell script from previous step, this might look something like
-```bash
-python create_wikitext.py -i data/wiki_extr/hi -o data/wiki/hi -l hi
-``` 
-for hindi (unicode: 'hi')
-
-This should create two splits of your Wikimedia Dumps: a small and large one. 
-
-_**Then**_, use the [postprocess_wikitext.py](./postprocess_wikitext.py) script to finish post processing. This processes numbers, builds a vocab, and limits the vocabulary size. This might look following for Hindi (`hi`)
-```bash
-python postprocess_wikitext.py data/wiki/hi-2 hi
-python postprocess_wikitext.py data/wiki/hi-100 hi
-```

ulmfit/__main__.py

@@ -12,7 +12,6 @@ from .pretrain_lm import LMHyperParams, folder_name_to_model_name, DataSetParams
 from .train_clas import CLSHyperParams
 from pathlib import Path
 from string import Template
-from fastai.metrics import fbeta
 import torch
 
 class FireView:
@@ -146,6 +145,27 @@ class ULMFiT:
                   lr_sched=lr_sched,
                   **kwargs)
 
+
+    def multifit_seeds(self, base, name=None, seed_name='clsweightseed', model_num=10, **kwargs):
+        if name is None:
+            name = folder_name_to_model_name(Path(base).name)
+        for seed in range(0, model_num, 1):
+            kwargs[seed_name] = seed
+            print("Seed: ", seed_name, seed)
+            self.multifit_eval(glob=base, name=name, num_lm_epochs=0, **kwargs)
+
+
+    def multifit_eval(self, glob, name=None, num_lm_epochs=20, num_cls_epochs=8, bs=20, lr_sched="1cycle", label_smoothing_eps=0.1, **kwargs):
+        return self.eval(
+                  glob=glob,
+                  name=name,
+                  num_lm_epochs=num_lm_epochs,
+                  num_cls_epochs=num_cls_epochs,
+                  bs=bs,
+                  lr_sched=lr_sched,
+                  label_smoothing_eps=label_smoothing_eps,
+                  **kwargs)
+
     def ls(self, glob, dataset_template='${ds_name}'):
         data_dir = Path("data").absolute()
         glob = str(glob)
@@ -252,8 +272,8 @@ class ULMFiT:
                 if name is None:
                     _name = folder_name_to_model_name(base_model.name)
                 params = CLSHyperParams.from_lm(dataset_path, base_model, lang=lang, name=_name, **model_args)
-                last_model_dir = params.model_dir.relative_to(data_dir.parent)
-                if (params.model_dir/"cls_best.pth").exists():
+                last_model_dir = params.model_path.relative_to(data_dir.parent)
+                if (params.model_path / "cls_best.pth").exists():
                     print("Evaluating previously trained model")
                     d_tst = params.evaluate_cls(save_name=save_name, label_smoothing_eps=label_smoothing_eps, use_cache=True, mode="test")
                     d_val = params.evaluate_cls(save_name=save_name, label_smoothing_eps=label_smoothing_eps, use_cache=True, mode="valid")
@@ -264,12 +284,12 @@ class ULMFiT:
                     print("Training")
                     d = params.train_cls(num_lm_epochs=num_lm_epochs, label_smoothing_eps=label_smoothing_eps, **trn_params)
                 else:
-                    print("Skipping", (params.model_dir/"cls_best.pth"))
-                    d  = None
+                    print("Skipping", (params.model_path / "cls_best.pth"))
+                    d = None
                 if d is not None:
-                    d['model_dir_parent'] = params.model_dir.relative_to(data_dir.parent).parent
+                    d['model_dir_parent'] = params.model_path.relative_to(data_dir.parent).parent
                     d['model_name'] = params.model_name
-                    np.save(params.model_dir / "results.npy", d)
+                    np.save(params.model_path / "results.npy", d)
                     results.append(d)
                 del params
             except Exception as e:

ulmfit/datasets/__init__.py

@@ -0,0 +1 @@
+

ulmfit/datasets/postprocess_wikitext.py

@@ -10,7 +10,7 @@ from pathlib import Path
 
 import fire
 
-from fastai_contrib.utils import replace_number, UNK
+from multifit.datasets.utils import replace_number, UNK
 
 
 def build_vocab(file_path, cutoff=3):

ulmfit/datasets/utils.py

@@ -2,25 +2,18 @@
 Utility methods for data processing.
 """
 import fire
-from fastai import *
 from fastai.text import *
 
 import shutil
 import pathlib
 import tarfile
-from sklearn import model_selection
-from sacremoses import MosesTokenizer
 from typing import Dict, Tuple, List
 
+from fastai_contrib.text_data import SentencePieceTokenizer
+
 EOS = 'xxeos' # fastai does not use eos, but we do
 SEP = 'xxsep' # special separator token for NLI
 
-def replace_std_toks(x:str) -> str:
-    "Replace standard token names with fastai supported tokens"
-    # We change tokens to f'xx{token_name}' as it is not split by Moses tokenizer,
-    # while f'<{token_name}>' is being split to: '<' f'{token_name}' '>'
-    return x.replace('<unk>', UNK).replace('<bos>', BOS).replace('<eos>', EOS)
-
 PAD_TOKEN_ID = 1
 IMDB, XNLI, TRN, VAL, TST, EN = 'imdb', 'xnli', 'train', 'val', 'test', 'en'
 DATASETS = ['imdb', 'xnli']
@@ -35,124 +28,6 @@ CLASSES = ['neg', 'pos', 'unsup']
 number_match_re = re.compile(r'^([0-9]+[,.]?)+$')
 number_split_re = re.compile(r'([,.])')
 
-class MosesPreprocessingFunc():
-
-    def __init__(self, lang: str):
-        self.mt = MosesTokenizer(lang)
-
-    def __call__(self, t: str) -> str:
-        return self.mt.tokenize(t, return_str=True, escape=True)
-
-class SentencePieceTokenizer(Tokenizer):
-    "Put together rules and a tokenizer function to tokenize text with multiprocessing."
-    def __init__(self, spm_model, lang:str='en', pre_rules:ListRules=None,
-                 post_rules:ListRules=None, special_cases:Collection[str]=None, n_cpus:int=None):
-        # moses is added to preprocessing functions
-        super().__init__(self.tok_fun_with_sp, lang, pre_rules, post_rules, special_cases, n_cpus)
-        self.spm_model = spm_model
-
-    def tok_fun_with_sp(self, lang):
-        try:
-            import sentencepiece as spm
-        except ImportError:
-            raise Exception('sentencepiece module is missing: run `pip install sentencepiece`')
-        tok = BaseTokenizer(lang)
-        tok.sp = spm.SentencePieceProcessor()
-        tok.sp.Load(str(self.spm_model))
-        return tok
-
-    def process_text(self, t:str, tok:BaseTokenizer) -> List[str]:
-        "Process one text `t` with tokenizer `tok`."
-        toks = super().process_text(t, tok)
-        toks = tok.sp.EncodeAsPieces(" ".join(toks))
-        return toks
-full_char_coverage_langs = ["bg", "cs", "da", "de", "el", "en", "es", "et", "fi", "fr", "ga", "hr", "hu",
-                       "it","lt","lv","mt","nl","pl","pt","ro","sk","sl","sv"] # all European langus
-
-def get_sentencepiece(cache_dir:PathOrStr, load_text, pre_rules: ListRules=None, post_rules:ListRules=None,
-                      vocab_size:int=30000, model_type:str='unigram', input_sentence_size:int=1E7, lang='en', fixed_character_coverage=False):
-    try:
-        import sentencepiece as spm
-    except ImportError:
-        raise Exception('sentencepiece module is missing: run `pip install sentencepiece`')
-
-    cache_dir = pathlib.Path(cache_dir)
-    pre_rules = pre_rules if pre_rules is not None else defaults.text_pre_rules
-    post_rules = post_rules if post_rules is not None else defaults.text_post_rules
-
-    special_cases = defaults.text_spec_tok # + ['xxlink', 'xxuser', 'xxnumber', 'xxemoji', 'yyemoji']
-    if not os.path.isfile(cache_dir / 'spm.model') or not os.path.isfile(cache_dir / f'itos.pkl'):
-        # load the text from the train tokens file
-        text = load_text()
-        text = filter(lambda x: len(x.rstrip(" ")), text)
-        text = (reduce(lambda t, rule: rule(t), pre_rules, line) for line in text)
-        def cleanup_n_postprocess(t):
-            t = t.split()
-            for r in post_rules:
-                t = r(t)
-            return ' '.join(t)
-        text = map(cleanup_n_postprocess, text)
-        raw_text_path = cache_dir / 'all_text.txt'
-        with open(raw_text_path, 'w') as f: f.write("\n".join(text))
-
-        if fixed_character_coverage:
-            char_coverage = 0.9995
-        else:
-            char_coverage = 1 if lang in full_char_coverage_langs else 0.99
-
-        sp_params = [
-            f"--input={raw_text_path}",
-            f"--character_coverage={char_coverage}",
-            f"--unk_id={len(special_cases)}",
-            f"--pad_id=-1",
-            f"--bos_id=-1",
-            f"--eos_id=-1",
-            f"--max_sentence_length=20480",
-            f"--input_sentence_size={int(input_sentence_size)}",
-            f"--user_defined_symbols={','.join(special_cases)}",
-            f"--model_prefix={cache_dir/'spm'}",
-            f"--vocab_size={vocab_size} --model_type={model_type}"]
-        spm.SentencePieceTrainer.Train(" ".join(sp_params))
-
-        with open(cache_dir / 'spm.vocab', 'r') as f:
-            vocab = [line.split('\t')[0] for line in f.readlines()]
-
-        pickle.dump(vocab, open(cache_dir/ f'itos.pkl', 'wb'))
-    # todo add post rules
-    vocab = Vocab(pickle.load(open(cache_dir / f'itos.pkl', 'rb')))
-    # We cannot use lambdas or local methods here, since `tok_func` needs to be
-    # pickle-able in order to be called in subprocesses when multithread tokenizing
-    tokenizer = SentencePieceTokenizer(cache_dir/'spm.model',
-                                lang=lang,
-                                pre_rules=pre_rules,
-                                post_rules=post_rules)
-    return {'tokenizer': tokenizer, 'vocab': vocab}
-
-def get_sentencepiece_fastai(cache_dir: PathOrStr,  pre_rules: ListRules = None,
-                          post_rules: ListRules = None,
-                          vocab_size: int = 30000, lang='en'):
-    cache_dir = pathlib.Path(cache_dir)
-
-    sp_model = cache_dir / 'spm.model'
-    if not sp_model.is_file():
-        sp_model = None
-
-    sp_vocab = cache_dir / 'spm.vocab'
-    if not sp_vocab.is_file():
-        sp_vocab = None
-
-    processor = SPProcessor(
-        pre_rules=pre_rules,
-        post_rules=post_rules,
-        mark_fields=True,
-        vocab_sz=vocab_size,
-        sp_model=sp_model,
-        sp_vocab=sp_vocab,
-        lang=lang,
-        tmp_dir=cache_dir.absolute()  # absolute make sure that dataset path is not added as prefix
-    )
-    return {'processor': processor}
-
 def clear_cache_directory(path:PathOrStr, cache_name:str='tmp'):
     path = pathlib.Path(path)
     shutil.rmtree(path / cache_name)
@@ -165,7 +40,6 @@ def get_texts(path):
             labels.append(idx)
     return np.array(texts), np.array(labels)
 
-
 def ensure_paths_exists(*paths, message="One or more required files cannot be found."):
     error = False
     for path in paths:
@@ -175,12 +49,6 @@ def ensure_paths_exists(*paths, message="One or more required files cannot be fo
     if error:
         raise FileNotFoundError(message)
 
-def get_data_folder() -> Path:
-    """
-    return data folder to use for future processing
-    """
-    return (pathlib.Path(__file__).parent.parent / "data")
-
 def get_scripts_folder():
     """
     return data folder to use for future processing
@@ -339,8 +207,6 @@ def read_file(file_path, outname=None):
         df.to_csv(file_path.parent / f'{outname}.csv', header=False, index=False)
     return df
 
-
-
 def read_whitespace_file(filepath):
     """Reads a file and prepares the tokens."""
     tokens = []
@@ -356,7 +222,6 @@ class DataStump:
         self.ids = ids
         self.loss_func = F.cross_entropy
 
-
 def validate(model, ids, bptt=2000):
     """
     Return the validation loss and perplexity of a model
@@ -369,7 +234,7 @@ def validate(model, ids, bptt=2000):
     model.eval()
     model.reset()
     total_loss, num_examples = 0., 0
-    for inputs, targets in tqdm(data):
+    for inputs, targets in data:
         outputs, raws, outs = model(to_device(inputs, None))
         p_vocab = F.softmax(outputs, 1)
         for i, pv in enumerate(p_vocab):
@@ -379,7 +244,6 @@ def validate(model, ids, bptt=2000):
     mean = total_loss / num_examples  # divide by total number of tokens
     return mean, np.exp(mean)
 
-
 class TextReader():
     """ Returns a language model iterator that iterates through batches that are of length N(bptt,5)
     The first batch returned is always bptt+25; the max possible width.  This is done because of they way that pytorch

ulmfit/pretrain_lm.py

@@ -4,28 +4,25 @@ expected to have been tokenized with Moses and processed with `postprocess_wikit
 That is, the data is expected to be white-space separated and numbers are expected
 to be split.
 """
-from dataclasses import InitVar, asdict
+import pathlib
+from dataclasses import asdict
 from string import Template
 
 import fastai
-import fire
 
 from fastai import *
-from fastai.callbacks import CSVLogger, SaveModelCallback
+from fastai.callbacks import CSVLogger
 import fastai.text
 from fastai.text import *
 import torch
-from fastai_contrib.utils import read_file, read_whitespace_file, \
-    validate, PAD, UNK, get_sentencepiece, read_clas_data, TRN, VAL, TST, PAD_TOKEN_ID, \
-    replace_std_toks, MosesPreprocessingFunc, get_sentencepiece_fastai
-from fastai_contrib.learner import bilm_learner, accuracy_fwd, accuracy_bwd, bilm_text_classifier_learner
+from ulmfit.datasets.utils import read_whitespace_file, \
+    validate, UNK
+from fastai_contrib.text_data import MosesPreprocessingFunc, get_sentencepiece, get_sentencepiece_fastai, \
+    make_data_bunch_from_df
 import pickle
 
 from pathlib import Path
 
-from collections import Counter
-import fastai_contrib.data as contrib_data
-
 LM_BEST = "lm_best"
 ENC_BEST = "enc_best"
 
@@ -37,11 +34,9 @@ class Tokenizers(Enum):
     MOSES_FA='vf'
     FASTAI='f'
 
-
 def istitle(line):
     return len(re.findall(r'^ ?= [^=]* = ?$', line)) != 0
 
-
 def read_wiki_articles(filename):
     if "reddit" in str(filename): # Temporary hack to handle poleval reddit dataset
         return pd.read_csv(filename, header=None, names=["texts"]).fillna("")
@@ -89,35 +84,6 @@ class DataSetParams:
         except KeyError as e:
             raise KeyError(f"{e} , options:{repr(list(params.keys()))}")
 
-# temporary loading function as from_df does not support processors
-def make_data_bunch_from_df(cls, path: PathOrStr, train_df: DataFrame, valid_df: DataFrame,
-            tokenizer: Tokenizer = None, vocab: Vocab = None, classes: Collection[str] = None,
-            text_cols: IntsOrStrs = 1,
-            label_cols: IntsOrStrs = 0, label_delim: str = None, chunksize: int = 10000,
-            max_vocab: int = 60000,
-            min_freq: int = 2, mark_fields: bool = False, include_bos: bool = True,
-            include_eos: bool = False, processor=None, **kwargs) -> DataBunch:
-    "Create a `TextDataBunch` from DataFrames. `kwargs` are passed to the dataloader creation."
-    assert processor is None or tokenizer is None, "Processor and tokenizer are mutually exclusive."
-
-    if processor is None:
-        processor = fastai.text.data._get_processor(tokenizer=tokenizer, vocab=vocab, chunksize=chunksize, max_vocab=max_vocab,
-                                   min_freq=min_freq, mark_fields=mark_fields,
-                                   include_bos=include_bos, include_eos=include_eos)
-
-    if classes is None and is_listy(label_cols) and len(label_cols) > 1: classes = label_cols
-    src = ItemLists(path, TextList.from_df(train_df, path, cols=text_cols, processor=processor),
-                    TextList.from_df(valid_df, path, cols=text_cols, processor=processor))
-    if cls == TextLMDataBunch:
-        src = src.label_for_lm()
-    else:
-        if label_delim is not None:
-            src = src.label_from_df(cols=label_cols, classes=classes, label_delim=label_delim)
-        else:
-            src = src.label_from_df(cols=label_cols, classes=classes)
-    return src.databunch(**kwargs)
-
-
 @dataclass
 class LMHyperParams(DataSetParams):
     base_lm_path: str = None
@@ -135,10 +101,6 @@ class LMHyperParams(DataSetParams):
 
     lmseed: int = None
 
-    # these hyperparameters are for training on ~100M tokens (e.g. WikiText-103)
-    # for training on smaller datasets, more dropout is necessary
-    # buggy dps = dict(output_p=0.25, hidden_p=0.1, input_p=0.2, embed_p=0.02, weight_p=0.15) # consider removing dps & clip from the default hyperparams and put them to train
-    dps = dict(input_p=0.25, output_p=0.1, weight_p=0.2, embed_p=0.02, hidden_p=0.15) # consider removing dps & clip from the default hyperparams and put them to train
     clip: float = 0.12
     bptt: int = 70
     # alpha and beta - defaults like in fastai/text/learner.py:RNNLearner()
@@ -162,7 +124,7 @@ class LMHyperParams(DataSetParams):
 
         assert self.dataset_path.exists(), f"The dataset_path {self.dataset_path} does not exists"
         self.cache_dir = self.dataset_path / 'models' / self.tokenizer_prefix
-        self.model_dir = self.cache_dir / self.model_name
+        self.model_path = self.cache_dir / self.model_name
 
         if self.nh is None: self.nh = 1552 if self.qrnn else 1152
         if self.name is None: self.name = self.lang
@@ -191,15 +153,6 @@ class LMHyperParams(DataSetParams):
     @property
     def pretrained_fnames(self): return [self.base_lm_path / LM_BEST, self.base_lm_path / '../itos'] if self.base_lm_path else None
 
-    @property
-    def lm_type(self):
-        if self.bidir:
-            return contrib_data.LanguageModelType.BiLM
-        if self.backwards:
-            return contrib_data.LanguageModelType.BwdLM
-        else:
-            return contrib_data.LanguageModelType.FwdLM
-
     def tokenizer_to_fastai_args(self, sp_data_func, use_moses):
         moses_preproc = [MosesPreprocessingFunc(self.lang)] if use_moses else []
         if self.tokenizer is Tokenizers.FASTAI_SUBWORD:
@@ -227,7 +180,7 @@ class LMHyperParams(DataSetParams):
         elif self.tokenizer is Tokenizers.MOSES:
             args = dict(tokenizer=Tokenizer(tok_func=BaseTokenizer,
                                             lang=self.lang,
-                                            pre_rules=moses_preproc + [replace_std_toks],
+                                            pre_rules=moses_preproc,
                                             post_rules=[]))
         elif self.tokenizer is Tokenizers.MOSES_FA:
             args = dict(tokenizer=Tokenizer(tok_func=BaseTokenizer,
@@ -255,20 +208,20 @@ class LMHyperParams(DataSetParams):
         vals.pop('name', None)
         vals.pop('lang', None)
         vals['tokenizer'] = self.tokenizer.value
-        with (self.model_dir / 'info.json').open("w") as fp: json.dump(vals, fp)
-        print("Saving info", self.model_dir / 'info.json')
+        with (self.model_path / 'info.json').open("w") as fp: json.dump(vals, fp)
+        print("Saving info", self.model_path / 'info.json')
 
-    def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, lr=5e-3, label_smoothing_eps=0.0):
+    def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, label_smoothing_eps=0.0):
         print("Training lm")
         print('Max vocab:', self.max_vocab)
         print('Cache dir:', self.cache_dir)
-        print('Model dir:', self.model_dir)
+        print('Model dir:', self.model_path)
         if self.pretrained_fnames or self.pretrained_model:
             self.set_seed(self.ftseed, "fine-tune")
         else:
             self.set_seed(self.lmseed, "LM")
 
-        self.model_dir.mkdir(exist_ok=True, parents=True)
+        self.model_path.mkdir(exist_ok=True, parents=True)
         data_lm = self.load_wiki_data(bs=bs) if data_lm is None else data_lm
         learn = self.create_lm_learner(data_lm, drop_mult=drop_mult, label_smoothing_eps=label_smoothing_eps)
         print("Bptt", data_lm.bptt)
@@ -290,8 +243,8 @@ class LMHyperParams(DataSetParams):
             else:
                 print("Training lm from random weights")
                 learn.unfreeze()
-                if not learn.true_wd: learn.fit_one_cycle(num_epochs, lr, (0.8, 0.7), wd=1e-7)
-                else:                 learn.fit_one_cycle(num_epochs, lr, (0.8, 0.7)) # TODO find proper values
+                if not learn.true_wd: learn.fit_one_cycle(num_epochs, 5e-3, (0.8, 0.7), wd=1e-7)
+                else:                 learn.fit_one_cycle(num_epochs, 5e-3, (0.8, 0.7)) # TODO find proper values
         learn.save("lm_best_with_opt", with_opt=True)
         learn.save_encoder(ENC_BEST)
         learn.save(LM_BEST, with_opt=False)
@@ -303,13 +256,16 @@ class LMHyperParams(DataSetParams):
 
     def create_lm_learner(self, data_lm, dps=None, label_smoothing_eps=0.0, **kwargs):
         assert self.bidir == False, "bidirectional model is not yet supported"
-        config = dict(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, pad_token=PAD_TOKEN_ID, qrnn=self.qrnn,
-                          tie_weights=True, out_bias=self.out_bias)
-        config.update(dps or self.dps)
+        config = awd_lstm_lm_config.copy()
+        config.update(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, qrnn=self.qrnn,
+                      tie_weights=True, out_bias=self.out_bias)
+        if dps is not None:
+            config.update(dps)
+
         trn_args = dict(clip=self.clip, alpha=self.rnn_alpha, beta=self.rnn_beta)
         trn_args.update(kwargs)
-        print ("Training args: ", trn_args, "dps: ", dps or self.dps)
-        learn = language_model_learner(data_lm, AWD_LSTM, config=config, model_dir=self.model_dir.relative_to(data_lm.path), pretrained=False, **trn_args)
+        print ("Training args: ", trn_args, "config: ", config)
+        learn = language_model_learner(data_lm, AWD_LSTM, config=config, model_dir=self.model_path.relative_to(data_lm.path), pretrained=False, **trn_args)
         if self.pretrained_model is not None:
             print("Loading pretrained model")
             model_path = untar_data(self.pretrained_model, data=False)
@@ -323,10 +279,7 @@ class LMHyperParams(DataSetParams):
             learn.freeze()
         # compared to standard Adam, we set beta_1 to 0.8
         learn.opt_fn = partial(optim.Adam, betas=(0.8, 0.99))
-        learn.metrics = [accuracy_fwd, accuracy_bwd] if self.bidir else [accuracy]
-        learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/lm-history"),
-                               # partial(SaveModelCallback, every='improvement', name='lm') disabled due to Memory issues
-                               ]
+        learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/lm-history")]
         if label_smoothing_eps > 0.0:
             learn.loss_func = FlattenedLoss(LabelSmoothingCrossEntropy, eps=label_smoothing_eps)
         return learn
@@ -337,7 +290,7 @@ class LMHyperParams(DataSetParams):
             return [line.rstrip('\n') for line in f]
 
     def load_wiki_data(self, bs=70):
-        self.model_dir.mkdir(exist_ok=True, parents=True)
+        self.model_path.mkdir(exist_ok=True, parents=True)
         trn_path = self.dataset_path / f'{self.lang}.wiki.train.tokens'
         val_path = self.dataset_path / f'{self.lang}.wiki.valid.tokens'
         tst_path = self.dataset_path / f'{self.lang}.wiki.test.tokens'
@@ -382,11 +335,11 @@ class LMHyperParams(DataSetParams):
         else:
             print(f"Running tokenization {name}...")
             data = make_data_bunch_from_df(cls=bunch_class, path=self.cache_dir,
-                                       train_df=train_df,
-                                       valid_df=valid_df,
-                                       max_vocab=self.max_vocab,
-                                       bs=bs,
-                                       **args)
+                                           train_df=train_df,
+                                           valid_df=valid_df,
+                                           max_vocab=self.max_vocab,
+                                           bs=bs,
+                                           **args)
 
             data.save(name)
         with open(self.cache_dir/"itos.pkl", 'wb') as f:
@@ -430,8 +383,9 @@ class LMHyperParams(DataSetParams):
         d['lang'] = infer_lang_from_dataset(dataset_path.name)
         return cls(**d)
 
+
     def resolve_template(self, template, **additional_options):
-        return super().resolve_template(template, model_dir=self.model_dir, **additional_options)
+        return super().resolve_template(template, model_dir=self.model_path, **additional_options)
 
 def infer_lang_from_dataset(name:str):
     return name.split("-")[0]
@@ -452,5 +406,8 @@ def validate_lm(self):
         logloss, perplexity = validate(learn.model, tst_ids, self.exp.bptt)
         print('Test logloss:', logloss.item(), 'perplexity:', perplexity.item())
 
-if __name__ == '__main__':
-    fire.Fire(LMHyperParams)
+def get_data_folder() -> Path:
+    """
+    return data folder to use for future processing
+    """
+    return (pathlib.Path(__file__).parent.parent / "data")

ulmfit/train_clas.py

@@ -7,14 +7,12 @@ import re
 from fastai.callbacks import CSVLogger, SaveModelCallback
 from fastai.text import *
 
-from fastai_contrib.utils import PAD_TOKEN_ID
+from ulmfit.datasets.utils import PAD_TOKEN_ID
 
 import fire
 
 from ulmfit.pretrain_lm import LMHyperParams, ENC_BEST
 
-from sklearn.metrics import f1_score as f1s, precision_score, recall_score
-
 @dataclass
 class CLSHyperParams(LMHyperParams):
     # dir_path -> data/imdb/
@@ -43,7 +41,7 @@ class CLSHyperParams(LMHyperParams):
         return ''
 
     @property
-    def need_fine_tune_lm(self): return not (self.model_dir/f"enc_best.pth").exists()
+    def need_fine_tune_lm(self): return not (self.model_path / f"enc_best.pth").exists()
 
     def lr_schedule_layered(self, learn, num_cls_epochs):
         learn.freeze_to(-1)
@@ -123,15 +121,15 @@ class CLSHyperParams(LMHyperParams):
                 f"{mode} Accuracy": results[6].item()}
         return {k:float(str(v)) for k,v in d.items()} # float(str(x)) to avoid float32 -> float64 conversion isssues
 
-    def train_cls(self, num_lm_epochs, unfreeze=True, num_cls_frozen_epochs=1, bs=40, drop_mul_lm=0.3, drop_mul_cls=0.5,
+    def train_cls(self, num_lm_epochs, unfreeze=True, bs=40, drop_mul_lm=0.3, drop_mul_cls=0.5,
                   use_test_for_validation=False, num_cls_epochs=2, limit=None, noise=0.0, cls_max_len=20*70, lr_sched='layered',
-                  label_smoothing_eps=0.0, random_init=False, dump_preds=None, early_stopping=True, weighted_cross_entropy=True):
+                  label_smoothing_eps=0.0, random_init=False, early_stopping=True, weighted_cross_entropy=True):
         print("Training CLS")
         print('Max vocab:', self.max_vocab)
         print('Cache dir:', self.cache_dir)
-        print('Model dir:', self.model_dir)
+        print('Model dir:', self.model_path)
         assert use_test_for_validation == False, "use_test_for_validation=True is not supported"
-        self.model_dir.mkdir(exist_ok=True, parents=True)
+        self.model_path.mkdir(exist_ok=True, parents=True)
 
         if not unfreeze:
            num_cls_epochs = 1
@@ -139,7 +137,7 @@ class CLSHyperParams(LMHyperParams):
         data_clas, data_lm, data_tst = self.load_cls_data(bs, limit=limit, noise=noise)
 
         if self.need_fine_tune_lm and not random_init:
-            if not (self.model_dir/(ENC_BEST+".pth")).exists():
+            if not (self.model_path / (ENC_BEST + ".pth")).exists():
                 self.train_lm(num_lm_epochs, data_lm=data_lm, drop_mult=drop_mul_lm, label_smoothing_eps=label_smoothing_eps)
             else:
                 print("Language model already exist, skipping finetuning")
@@ -180,7 +178,7 @@ class CLSHyperParams(LMHyperParams):
 
     def evaluate_cls(self, save_name='cls_best', bs=40, data_tst=None, learn=None,
                      dump_preds=None, mode="test", label_smoothing_eps=None, use_cache=False):
-        cache_file = (self.model_dir / f'results_{mode+("" if save_name == "cls_best" else str(save_name))}.json')
+        cache_file = (self.model_path / f'results_{mode + ("" if save_name == "cls_best" else str(save_name))}.json')
         if use_cache and cache_file.exists():
             with cache_file.open("r") as fp:
                 return json.load(fp)
@@ -212,7 +210,7 @@ class CLSHyperParams(LMHyperParams):
             if dump_preds:
                 with open(dump_preds, 'w') as f:
                     f.write('\n'.join([str(x) for x in preds]))
-            np.save(self.model_dir / f"preds-on-{mode}.npy", probs.cpu().numpy())
+            np.save(self.model_path / f"preds-on-{mode}.npy", probs.cpu().numpy())
         results = learn.validate(ds)
         print(f"Model: {self.name}")
         print(f"Evaluation on: {mode}")
@@ -224,13 +222,16 @@ class CLSHyperParams(LMHyperParams):
         return labeled_results
 
     def create_cls_learner(self, data_clas, dps=None, label_smoothing_eps=0.0, random_init=False, early_stopping=True, **kwargs):
-        assert self.bidir == False, "bidirectional model is not yet supported"
-        config = dict(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, pad_token=PAD_TOKEN_ID, qrnn=self.qrnn)
-        config.update(dps or self.dps)
-        trn_args=dict(bptt=self.bptt, clip=self.clip)
+
+        config = awd_lstm_clas_config.copy()
+        config.update(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, qrnn=self.qrnn)
+        if dps is not None:
+            config.update(dps)
+
+        trn_args = dict(bptt=self.bptt, clip=self.clip)
         trn_args.update(kwargs)
         learn = text_classifier_learner(data_clas, AWD_LSTM, config=config,
-            pretrained=False, path=self.model_dir.parent, model_dir=self.model_dir.name, **trn_args)
+                                        pretrained=False, path=self.model_path.parent, model_dir=self.model_path.name, **trn_args)
 
         if self.pretrained_model is not None and not random_init:
             print("Loading pretrained model", self.pretrained_model)
@@ -252,7 +253,7 @@ class CLSHyperParams(LMHyperParams):
         return learn
 
     def load_cls_data(self, bs, **kwargs):
-        self.model_dir.mkdir(exist_ok=True, parents=True)
+        self.model_path.mkdir(exist_ok=True, parents=True)
         add_trn_to_lm = True
         lang = self.lang
         use_moses = True
@@ -338,7 +339,4 @@ class CLSHyperParams(LMHyperParams):
         return data_cls, data_lm, data_tst
 
     def cls_databunch(self, name, *args, **kwargs):
-        return self.databunch(name, bunch_class=TextClasDataBunch, *args, **kwargs)
-
-if __name__ == '__main__':
-    fire.Fire(CLSHyperParams)
+        return self.databunch(name, bunch_class=TextClasDataBunch, *args, **kwargs)