made a binary classification version making binary category label list

.gitignore

@@ -1,3 +1,5 @@
+/data
+
 *.bak
 *.log
 *~

fastai_contrib/data_block.py

@@ -0,0 +1,34 @@
+from typing import Iterator, Collection
+from fastai.data_block import CategoryListBase
+from fastai.text import *
+
+
+class BinaryProcessor(CategoryProcessor):
+    def create_classes(self, classes):
+        self.classes = classes
+        if classes is not None: self.c2i = {0:0, 1:1}
+    def generate_classes(self, items):
+        return [0]
+
+class BinaryCategoryList(CategoryListBase):
+    "Basic `ItemList` for single classification labels."
+    _processor=BinaryProcessor
+    def __init__(self, items:Iterator, classes:Collection=None, label_delim:str=None, **kwargs):
+        super().__init__(items, classes=classes, **kwargs)
+        mean = self.items.mean()
+        if mean and mean!=0:
+            weight = torch.tensor([1 / self.items.mean()]).cuda()
+        else:
+            weight = None
+            raise Exception('debug')
+        self.loss_func = BCEWithLogitsFlat(weight=weight)
+
+    def reconstruct(self, t):
+        return Category(t, self.c2i[t.item()])
+
+    def get(self, i):
+        o = self.items[i]
+        if o is None: return None
+        return Category(o, self.c2i[o])
+
+    def analyze_pred(self, pred, thresh:float=0.5): return pred.argmax()

fastai_contrib/text_data.py

@@ -130,7 +130,7 @@ def make_data_bunch_from_df(cls, path: PathOrStr, train_df: DataFrame, valid_df:
             tokenizer: Tokenizer = None, vocab: Vocab = None, classes: Collection[str] = None,
             text_cols: IntsOrStrs = 1,
             label_cols: IntsOrStrs = 0, label_delim: str = None, chunksize: int = 10000,
-            max_vocab: int = 60000,
+            max_vocab: int = 60000, label_cls: Callable = None,
             min_freq: int = 2, mark_fields: bool = False, include_bos: bool = True,
             include_eos: bool = False, processor=None, **kwargs) -> DataBunch:
     "Create a `TextDataBunch` from DataFrames. `kwargs` are passed to the dataloader creation."
@@ -142,8 +142,8 @@ def make_data_bunch_from_df(cls, path: PathOrStr, train_df: DataFrame, valid_df:
                                    include_bos=include_bos, include_eos=include_eos)
 
     if classes is None and is_listy(label_cols) and len(label_cols) > 1: classes = label_cols
-    src = ItemLists(path, TextList.from_df(train_df, path, cols=text_cols, processor=processor),
-                    TextList.from_df(valid_df, path, cols=text_cols, processor=processor))
+    src = ItemLists(path, TextList.from_df(train_df, path, cols=text_cols, processor=processor, label_cls=label_cls),
+                    TextList.from_df(valid_df, path, cols=text_cols, processor=processor, label_cls=label_cls))
     if cls == TextLMDataBunch:
         src = src.label_for_lm()
     else:
@@ -151,4 +151,4 @@ def make_data_bunch_from_df(cls, path: PathOrStr, train_df: DataFrame, valid_df:
             src = src.label_from_df(cols=label_cols, classes=classes, label_delim=label_delim)
         else:
             src = src.label_from_df(cols=label_cols, classes=classes)
-    return src.databunch(**kwargs)
+    return src.databunch(**kwargs)

multifit/configurations.py

@@ -95,6 +95,36 @@ def multifit_paper_version():
     self.classifier.replace_(num_epochs=8, drop_mult=0.5, bs=18, label_smoothing_eps=0.1, early_stopping=None)
     return self
 
+def multifit_paper_version_bce():
+    self = ULMFiTBinary()
+    dps = {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
+    self.replace_(
+        label_smoothing_eps=0.0,
+        label_smoothing_eps_norm_by_classes=True,
+        true_wd=True,
+        wd=0.01, ## important :)
+        seed=0,
+        fp16=False,
+        bs=64,
+        use_adam_08=False,
+        early_stopping=None,
+        clip=0.12,
+        dropout_values=dps,
+        name=_use_caller_name()
+    )
+    self.arch.replace_(
+        tokenizer_type='sp',
+        max_vocab=15000,
+        qrnn=True,
+        n_layers=4,
+        n_hid=1550 # vs 1552
+    )
+    self.pretrain_lm.replace_(num_epochs=10, drop_mult=0.0, lr=5e-3, use_adam_08=True, true_wd=False, wd=1e-7, bs=50,)
+    self.finetune_lm.replace_(num_epochs=20, drop_mult=0.3, lr=1e-3, true_wd=False, wd=1e-7, bs=20)
+    # TODO check I can't do label_smoothing_eps on binary classifier
+    self.classifier.replace_(num_epochs=8, drop_mult=0.5, bs=18, label_smoothing_eps=0.0, early_stopping=None)
+    return self
+
 def ulmfit_orig():
     self = multifit_paper_version()
     self.replace_(
@@ -112,4 +142,4 @@ def ulmfit_orig():
 
 
 def _use_caller_name():
-    return inspect.stack()[1].function
+    return inspect.stack()[1].function

multifit/datasets/dataset.py

@@ -202,8 +202,8 @@ class ULMFiTDataset(Dataset):
 
     def load_lm_databunch(self, bs, bptt):
         lm_suffix = str(bptt) if bptt != 70 else ""
-        lm_suffix += "" if self.use_tst_for_lm else "-notst"
-        data_lm = self.load_n_cache_databunch(f"lm{lm_suffix}",
+        lm_suffix += "" if self.use_tst_for_lm else "-not-test"
+        data_lm = self.load_n_cache_databunch(f"lm{lm_suffix}.cache.databunch",
                                               bunch_class=TextLMDataBunch,
                                               data_loader=self.load_unsupervised_data,
                                               bptt=bptt,
@@ -223,27 +223,28 @@ class ULMFiTDataset(Dataset):
             self._vocab = self.load_lm_databunch(bs=20, bptt=70).vocab
         return self._vocab
 
-    def load_clas_databunch(self, bs):
+    def load_clas_databunch(self, bs, label_cls=None, **args):
         vocab = self._load_vocab()
 
-        cls_name = "cls"
+        cls_name = "cls.cache.databunch"
         if self.limit is not None:
             cls_name = f'{cls_name}limit{self.limit}'
         if self.noise > 0.0:
             cls_name = f'{cls_name}noise{self.noise}'
 
-        args = dict(vocab=vocab, bunch_class=TextClasDataBunch, bs=bs)
-        data_cls = self.load_n_cache_databunch(cls_name, data_loader=lambda: self.load_supervised_data()[:2], **args)
+        args.update(dict(vocab=vocab, bunch_class=TextClasDataBunch, bs=bs))
+        data_cls = self.load_n_cache_databunch(cls_name, data_loader=lambda: self.load_supervised_data()[:2], label_cls=label_cls, **args)
         # Hack to load test dataset with labels
-        data_tst = self.load_n_cache_databunch('tst', data_loader=lambda: self.load_supervised_data()[1:], **args)
+        data_tst = self.load_n_cache_databunch('tst.cache.databunch', data_loader=lambda: self.load_supervised_data()[1:], label_cls=label_cls, **args)
         data_cls.test_dl = data_tst.valid_dl # data_tst.valid_dl holds test data
         data_cls.lang = self.lang
         return data_cls
 
-    def load_n_cache_databunch(self, name, bunch_class, data_loader, bs, **args):
+    def load_n_cache_databunch(self, name, bunch_class, data_loader, bs, label_cls=None, **args):
         bunch_path = self.cache_path / name
         databunch = None
         if bunch_path.exists():
+            print(f'loading data bunch from {name}')
             try:
                 databunch = load_data(self.cache_path, name, bs=bs)
             except (AttributeError, ImportError):
@@ -251,17 +252,18 @@ class ULMFiTDataset(Dataset):
         if databunch is None:
             print(f"Running tokenization: '{name}' ...")
             train_df, valid_df = data_loader()
-            databunch = self.databunch_from_df(bunch_class, train_df, valid_df, **args)
+            databunch = self.databunch_from_df(bunch_class, train_df, valid_df, label_cls=label_cls, **args)
             databunch.save(name)
         print(f"Data {name}, trn: {len(databunch.train_ds)}, val: {len(databunch.valid_ds)}")
         return databunch
 
-    def databunch_from_df(self, bunch_class, train_df, valid_df, **args):
+    def databunch_from_df(self, bunch_class, train_df, valid_df, label_cls=None, **args):
         args.update(**self.tokenizer.get_fastai_config(dataset_uses_moses=self.uses_moses))  # TODO depends on the previous model
         databunch = make_data_bunch_from_df(cls=bunch_class,
                                             path=self.cache_path,
                                             train_df=train_df,
                                             valid_df=valid_df,
+                                            label_cls=label_cls,
                                             mark_fields=True,
                                             text_cols=list(train_df.columns.values)[1:],
                                             **args)

multifit/metrics.py

@@ -0,0 +1,13 @@
+import torch
+from torch import Tensor, LongTensor
+from fastai.metrics import auc_roc_score, fbeta
+
+
+def auc_roc_score_multi(input, targ):
+    """area under curve for multi category list (multiple bce losses)."""
+    n = input.shape[1]
+    scores = [auc_roc_score(input[:, i], targ[:, i]) for i in range(n)]
+    return torch.tensor(scores).mean()
+
+def fbeta_binary(y_pred, y_true, **args):
+    return fbeta(y_pred[:, None], y_true[:, None], **args)

multifit/training.py

@@ -5,7 +5,9 @@ import dataclasses
 from fastai.callbacks import CSVLogger, SaveModelCallback
 from fastai.text import *
 
-from multifit.datasets import ULMFiTDataset,ULMFiTTokenizer
+from multifit.metrics import auc_roc_score_multi, fbeta_binary, auc_roc_score
+from multifit.datasets import ULMFiTDataset, ULMFiTTokenizer
+from fastai_contrib.data_block import BinaryCategoryList
 
 CLS_BEST = 'cls_best'
 LM_BEST = "lm_best"
@@ -225,6 +227,7 @@ class ULMFiTPretraining(ULMFiTTrainingCommand):
                                        config=config,
                                        model_dir=self.model_name,
                                        **trn_args)
+        learn.metrics = [accuracy]
         learn = patch_learner(learn)
         # compared to standard Adam, we set beta_1 to 0.8
         if self.use_adam_08:
@@ -378,6 +381,7 @@ class ULMFiTClassifier(ULMFiTTrainingCommand):
                                         config=config,
                                         model_dir=self.model_name,
                                         **trn_args)
+        learn.metrics =[accuracy, dice]
         learn = patch_learner(learn)
         if self.base.encoder_fname and not self.random_init:
             print("Loading pretrained model", self.base.encoder_fname)
@@ -442,8 +446,184 @@ class ULMFiTClassifier(ULMFiTTrainingCommand):
 
         learn = self.get_learner(data_cls, eval_only=True)
         # avg = 'binary' if learn.data.c == 2 else 'macro'
-        # FBeta(beta=1.0, average=avg), Precision(average=avg), Recall(average=avg),
-        learn.metrics = [accuracy]
+        # learn.metrics = [accuracy, FBeta(beta=1.0, average=avg), Precision(average=avg), Recall(average=avg)]
+        # learn.metrics = [accuracy, fbeta, auc_roc_score,]
+        learn.metrics = [accuracy, dice]
+        print(f"Loading model {save_name}")
+        learn.load(save_name)
+        if save_preds:
+            probs, targets = learn.get_preds(ordered=True, ds_type=DatasetType.Test, activ=partial(F.softmax, dim=-1))
+            np.save(str(self.experiment_path / f"preds-on-test.npy"), probs.cpu().numpy())
+
+        results_dict = {}
+        for split in splits:
+            results_dict.update(self._validate(learn, split))
+        print(results_dict)
+        with cache_file.open("w") as fp:
+            json.dump(results_dict, fp)
+        return results_dict
+
+    def _fit_schedule(self, learn):
+        getattr(self, '_fit_schedule_' + self.fit_schedule)(learn)
+
+    def _fit_schedule_1cycle(self, learn):
+        learn.unfreeze()
+        learn.fit_one_cycle(self.num_epochs, slice(1e-2 / (2.6 ** 4), 2e-2), moms=(0.8, 0.7))
+
+    def _fit_schedule_layered(self, learn):
+        learn.freeze_to(-1)
+        learn.fit_one_cycle(1, 2e-2, moms=(0.8, 0.7))
+        if self.num_epochs > 1:
+            learn.freeze_to(-2)
+            learn.fit_one_cycle(1, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7))
+            learn.freeze_to(-3)
+            learn.fit_one_cycle(1, slice(5e-3 / (2.6 ** 4), 5e-3), moms=(0.8, 0.7))
+            learn.unfreeze()
+        if self.num_epochs > 5:
+            learn.fit_one_cycle(self.num_epochs - 4, slice(1e-3 / (2.6 ** 4), 1e-3), moms=(0.8, 0.7))
+
+    def _fit_schedule_2cycle(self, learn):
+        learn.freeze_to(-1)
+        learn.fit_one_cycle(1, 2e-2, moms=(0.8, 0.7))
+        learn.unfreeze()
+        if self.num_epochs > 1:
+            learn.fit_one_cycle(self.num_epochs - 1, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7))
+
+    def _fit_schedule_reverse_2cycle(self, learn):
+        learn.unfreeze()
+        for g in learn.layer_groups[-1:]:
+            for l in g:
+                if not learn.train_bn or not isinstance(l, bn_types): requires_grad(l, False)
+        learn.create_opt(defaults.lr)
+        learn.fit_one_cycle(self.num_epochs, slice(1e-2 / (2.6 ** 4), 2e-2), moms=(0.8, 0.7))
+        learn.unfreeze()
+        learn.fit_one_cycle(self.num_epochs, slice(1e-3 / (2.6 ** 4), 2e-3), moms=(0.8, 0.7))
+
+    def _fit_schedule_false_wd(self, learn):
+        learn.true_wd = False
+        learn.fit_one_cycle(1, 5e-2, moms=(0.8, 0.7), wd=1e-7)
+        if self.num_epochs > 1:
+            learn.freeze_to(-2)
+            learn.fit_one_cycle(1, slice(5e-2 / (2.6 ** 4), 5e-2), moms=(0.8, 0.7), wd=1e-7)
+            learn.freeze_to(-3)
+            learn.fit_one_cycle(1, slice(5e-4 / (2.6 ** 4), 5e-4), moms=(0.8, 0.7), wd=1e-7)
+            learn.unfreeze()
+            if self.num_epochs > 5:
+                learn.fit_one_cycle(self.num_epochs - 4, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7), wd=1e-7)
+
+
+@dataclass
+class ULMFiTBinaryClassifier(ULMFiTTrainingCommand):
+    bs: int = 20
+    num_epochs: int = 10
+    drop_mult: float = 0.5
+    dropout_values: dict = field(default_factory=dict)
+    wd: float = 0.01
+    clip: float = None
+    label_smoothing_eps: float = 0.0
+    label_smoothing_eps_norm_by_classes: bool = False
+    weighted_cross_entropy: tuple = None
+    early_stopping: str = 'accuracy'
+    fit_schedule: str = '1cycle'
+    base: ULMFiTFinetuning = field(repr=False, default=None)
+    random_init: bool = False
+    seed: int = 0
+    bptt: int = 70
+    fp16: bool = False
+    arch: ULMFiTArchitecture = None
+
+    def get_learner(self, data_clas, eval_only=False, **additional_trn_args):
+        assert self.weighted_cross_entropy is None or self.label_smoothing_eps == 0, "Label smoothing not implemented with weighted_cross_entropy"
+        if self.weighted_cross_entropy is not None:
+            loss_func = BCEWithLogitsFlat(weight=torch.tensor(self.weighted_cross_entropy, dtype=torch.float32).cuda())
+        elif self.label_smoothing_eps > 0.0:
+            raise Exception("label_smoothing is not implemented in the binary classifier")
+        else:
+            loss_func = None
+
+        set_seed(self.seed, "Classifier weights seed")
+        config = awd_lstm_clas_config.copy()
+        config.update(emb_sz=self.arch.emb_sz, n_hid=self.arch.n_hid, n_layers=self.arch.n_layers, qrnn=self.arch.qrnn,
+                      **self.dropout_values)
+
+        trn_args = dict(drop_mult=self.drop_mult, wd=self.wd, pretrained=False, bptt=self.bptt,
+                        loss_func=loss_func, clip=self.clip)
+        if hasattr(Learner, 'silent'):
+            trn_args.update(silent=eval_only)
+
+        trn_args.update(**additional_trn_args)
+        print("Training args: ", trn_args, "config: ", config)
+        learn = text_classifier_learner(data_clas,
+                                        AWD_LSTM,
+                                        config=config,
+                                        model_dir=self.model_name,
+                                        **trn_args)
+        # learn.metrics =[accuracy, dice]
+        learn = patch_learner(learn)
+        if self.base.encoder_fname and not self.random_init:
+            print("Loading pretrained model", self.base.encoder_fname)
+            learn.load_encoder(self.base.encoder_fname)
+            learn.freeze()
+        else:
+            warn("No pretrained encoder")
+
+        set_seed(self.seed, "Classifier training seed")
+        if not eval_only:
+            learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/cls-history")]
+            if self.early_stopping:
+                learn.callback_fns += [partial(SaveModelCallback, every='improvement',
+                                               name='cls_best_tmp',
+                                               monitor=self.early_stopping)]
+        if self.fp16:
+            learn.to_fp16()
+        return learn
+
+    def train_(self, dataset_or_path=None, label_cls=BinaryCategoryList, metrics=[accuracy, dice, partial(fbeta_binary, beta=1), auc_roc_score_multi], label_cols=[0,0], **train_config):
+        self.replace_(**train_config, _strict=True)
+
+        base_tokenizer = self.base.tokenizer
+        dataset = self._set_dataset_(dataset_or_path, base_tokenizer)
+        data_clas = dataset.load_clas_databunch(bs=self.bs, label_cls=label_cls, label_cols=label_cols)
+        learn = self.get_learner(data_clas=data_clas)
+        learn.metrics = metrics
+        print(f"Training: {learn.path / learn.model_dir}")
+        learn.unfreeze()
+        self._fit_schedule(learn)
+
+        self.experiment_path = learn.path / learn.model_dir
+        base_tokenizer.save(self.experiment_path, learn=learn)
+        learn.to_fp32()
+        learn.save(CLS_BEST, with_opt=False)
+        print("Classifier model saved to", self.experiment_path)
+        self.save_paramters()
+        learn.destroy()
+        return
+
+    def _validate(self, learn, ds_type):
+        ds_name = ds_type.name.lower()
+        print(f"Model: {self.name}, ds_name: {ds_name}")
+        results_dict = dict(zip(
+            [f'{ds_name} loss'] + [f"{ds_name} {getattr(m, '__name__', m.__class__.__name__)}" for m in learn.metrics],
+            map(float, learn.validate(learn.data.dl(ds_type)))))
+        results_dict['name'] = self.name
+        return results_dict
+
+    def validate(self, *splits, data_cls=None, save_name=CLS_BEST, use_cache=True, save_preds=False):
+        """Validates
+            splits - Dataset Types to validate on default DatasetType.Test, DatasetType.Valid, DatasetType.Train
+        """
+        if len(splits) == 0:
+            splits = [DatasetType.Test, DatasetType.Valid, DatasetType.Train]
+        cache_file = (self.experiment_path / f'results{"" if save_name == CLS_BEST else "-" + save_name}.json')
+        if use_cache and cache_file.exists():
+            with cache_file.open("r") as fp:
+                return json.load(fp)
+
+        if data_cls is None:
+            data_cls = self.dataset.load_clas_databunch(bs=self.bs)
+
+        learn = self.get_learner(data_cls, eval_only=True)
+        learn.metrics = [accuracy, dice, fbeta, auc_roc_score_multi]
         print(f"Loading model {save_name}")
         learn.load(save_name)
         if save_preds:
@@ -579,6 +759,72 @@ class ULMFiT:
         return self.load_(path)
 
 
+@dataclass
+class ULMFiTBinary:
+    arch: ULMFiTArchitecture = None
+    pretrain_lm: ULMFiTPretraining = None
+    finetune_lm: ULMFiTFinetuning = None
+    classifier: ULMFiTBinaryClassifier = None
+
+    def __post_init__(self):
+        self.arch = ULMFiTArchitecture()
+        self.pretrain_lm = ULMFiTPretraining(arch=self.arch)
+        self.finetune_lm = ULMFiTFinetuning(arch=self.arch, base=self.pretrain_lm)
+        self.classifier = ULMFiTBinaryClassifier(arch=self.arch, base=self.finetune_lm)
+
+    def load_(self, experiment_path:Path, silent=False):
+        success = (self.classifier.load_(experiment_path, silent=silent) or
+                   self.finetune_lm.load_(experiment_path, silent=silent) or
+                   self.pretrain_lm.load_(experiment_path, silent=silent) or
+                   self.load_legacy_(experiment_path, silent=silent))
+        if not success:
+            warn(f'Unable to load experiment {experiment_path}')
+        return self
+
+    def load_legacy_(self, experiment_path, silent=True):
+        if not (experiment_path / "info.json").exists():
+            return False
+        with (experiment_path / "info.json").open('r') as f:
+            d = json.load(f)
+        dataset_path = d.pop('dataset_path', "")
+        d['n_hid'] = d['nh']
+        d['n_layers'] = d['nl']
+        d['lang'] = detect_lang_from_dataset_path(Path(dataset_path))
+        if "wiki" in str(dataset_path):
+            self.arch.replace_(**d)
+            self.pretrain_lm.replace_(**d)
+            self.pretrain_lm.experiment_path = path_if_model_exists(experiment_path, LM_BEST)
+            self.pretrain_lm.dataset_path = dataset_path if dataset_path in str(experiment_path) else None
+        else:
+            self.replace_(**d)
+            self.finetune_lm.experiment_path = path_if_model_exists(experiment_path, ENC_BEST)
+            self.finetune_lm.dataset_path = dataset_path if dataset_path in str(experiment_path) else None
+            self.classifier.experiment_path = path_if_model_exists(experiment_path, CLS_BEST)
+            self.classifier.dataset_path = dataset_path if dataset_path in str(experiment_path) else None
+        return True
+
+    def replace_(self, **kwargs):
+        self.arch.replace_(**kwargs)
+        self.pretrain_lm.replace_(**kwargs)
+        self.finetune_lm.replace_(**kwargs)
+        self.classifier.replace_(**kwargs)
+        return self
+
+    def pprint(self):
+        print(f"""ULMFiT(
+    {self.arch},
+    {self.pretrain_lm},
+    {self.finetune_lm},
+    {self.classifier},
+)""")
+
+    def from_pretrained_(self, name, repo="n-waves/multifit-models"):
+        name = name.rstrip(".tgz")  # incase someone put's tgz name the name
+        url = f"https://github.com/{repo}/releases/download/{name}/{name}.tgz"
+        path = untar_data(url.rstrip(".tgz"), data=False)  # untar_data adds .tgz
+        return self.load_(path)
+
+
 def from_pretrained(name):
     #TODO: Detect name and load configuration
     from . import configurations