multiclass (but single classes don't work)

fastai_contrib/data_block.py

@@ -16,11 +16,12 @@ class BinaryCategoryList(CategoryListBase):
     def __init__(self, items:Iterator, classes:Collection=None, label_delim:str=None, **kwargs):
         super().__init__(items, classes=classes, **kwargs)
         mean = self.items.mean()
-        if mean and mean!=0:
+        # if mean and mean != 0:
-            weight = torch.tensor([1 / self.items.mean()]).cuda()
+        #     weight = torch.tensor([1 / mean]).cuda()
-        else:
+        #     print(f'Weighting BCEWithLogitsFlat by {weight.item()}')
-            weight = None
+        # else:
-            raise Exception('debug')
+        weight = None
+        # raise Exception('debug')
         self.loss_func = BCEWithLogitsFlat(weight=weight)
 
     def reconstruct(self, t):

multifit/datasets/dataset.py

@@ -224,6 +224,7 @@ class ULMFiTDataset(Dataset):
         return self._vocab
 
     def load_clas_databunch(self, bs, label_cls=None, **args):
+        print('DEBUG', bs, label_cls, args)
         vocab = self._load_vocab()
 
         cls_name = "cls.cache.databunch"

multifit/metrics.py

@@ -6,8 +6,61 @@ from fastai.metrics import auc_roc_score, fbeta
 def auc_roc_score_multi(input, targ):
     """area under curve for multi category list (multiple bce losses)."""
     n = input.shape[1]
+    targ = targ * 1
+    if targ.shape != input.shape:
+        targ = targ.expand(input.T.shape).T
     scores = [auc_roc_score(input[:, i], targ[:, i]) for i in range(n)]
     return torch.tensor(scores).mean()
 
+
 def fbeta_binary(y_pred, y_true, **args):
     return fbeta(y_pred[:, None], y_true[:, None], **args)
+
+
+def auc_roc_score(input: Tensor, targ: Tensor):
+    "Computes the area under the receiver operator characteristic (ROC) curve using the trapezoid method. Restricted binary classification tasks."
+    fpr, tpr = roc_curve(input.squeeze(), targ.squeeze())
+    d = fpr[1:] - fpr[:-1]
+    sl1, sl2 = [slice(None)], [slice(None)]
+    sl1[-1], sl2[-1] = slice(1, None), slice(None, -1)
+    return (d * (tpr[tuple(sl1)] + tpr[tuple(sl2)]) / 2.0).sum(-1)
+
+
+def roc_curve(input: Tensor, targ: Tensor):
+    "Computes the receiver operator characteristic (ROC) curve by determining the true positive ratio (TPR) and false positive ratio (FPR) for various classification thresholds. Restricted binary classification tasks."
+    # wassname: fix this by making LongTensor([0]=>device)
+    targ = targ == 1
+    desc_score_indices = torch.flip(input.argsort(-1), [-1])
+    input = input[desc_score_indices]
+    targ = targ[desc_score_indices]
+    d = input[1:] - input[:-1]
+    distinct_value_indices = torch.nonzero(d).transpose(0, 1)[0]
+    threshold_idxs = torch.cat(
+        (distinct_value_indices, LongTensor([len(targ) - 1]).to(targ.device))
+    )
+    tps = torch.cumsum(targ * 1, dim=-1)[threshold_idxs]
+    fps = 1 + threshold_idxs - tps
+    if tps[0] != 0 or fps[0] != 0:
+        zer = torch.zeros(1, dtype=fps.dtype, device=fps.device)
+        fps = torch.cat((zer, fps))
+        tps = torch.cat((zer, tps))
+    fpr, tpr = fps.float() / fps[-1], tps.float() / tps[-1]
+    return fpr, tpr
+
+
+def accuracy_binary(input, targs):
+    input = torch.sigmoid(input) > 0.5
+    return (input == targs).float().mean()
+
+
+def dice_binary(input, targs, iou=False, eps=1e-8):
+    "Dice coefficient metric for binary target. If iou=True, returns iou metric, classic for segmentation problems."
+    input = torch.sigmoid(input) > 0.5
+    intersect = (input * targs).sum(dim=1).float()
+    union = (input + targs).sum(dim=1).float()
+    if not iou:
+        l = 2.0 * intersect / union
+    else:
+        l = intersect / (union - intersect + eps)
+    l[union == 0.0] = 1.0
+    return l.mean()

multifit/training.py

@@ -5,7 +5,7 @@ import dataclasses
 from fastai.callbacks import CSVLogger, SaveModelCallback
 from fastai.text import *
 
-from multifit.metrics import auc_roc_score_multi, fbeta_binary, auc_roc_score
+from multifit.metrics import auc_roc_score_multi, fbeta_binary, auc_roc_score, accuracy_binary, dice_binary
 from multifit.datasets import ULMFiTDataset, ULMFiTTokenizer
 from fastai_contrib.data_block import BinaryCategoryList
 
@@ -351,18 +351,17 @@ class ULMFiTClassifier(ULMFiTTrainingCommand):
     fp16: bool = False
     arch: ULMFiTArchitecture = None
 
-    def get_learner(self, data_clas, eval_only=False, **additional_trn_args):
+    def get_learner(self, data_clas, eval_only=False, loss_func=None, **additional_trn_args):
         assert self.weighted_cross_entropy is None or self.label_smoothing_eps == 0, "Label smoohting not implemented with weighted_cross_entropy"
-        if self.weighted_cross_entropy is not None:
+        if loss_func is None:
-            loss_func = CrossEntropyFlat(weight=torch.tensor(self.weighted_cross_entropy, dtype=torch.float32).cuda())
+            if self.weighted_cross_entropy is not None:
-        elif self.label_smoothing_eps > 0.0:
+                loss_func = CrossEntropyFlat(weight=torch.tensor(self.weighted_cross_entropy, dtype=torch.float32).cuda())
-            eps = self.label_smoothing_eps
+            elif self.label_smoothing_eps > 0.0:
-            if self.label_smoothing_eps_norm_by_classes:
+                eps = self.label_smoothing_eps
-                eps = eps / data_clas.c
+                if self.label_smoothing_eps_norm_by_classes:
-            print("Using Label smoothing with eps = ", eps)
+                    eps = eps / data_clas.c
-            loss_func = FlattenedLoss(LabelSmoothingCrossEntropy, eps=eps)
+                print("Using Label smoothing with eps = ", eps)
-        else:
+                loss_func = FlattenedLoss(LabelSmoothingCrossEntropy, eps=eps)
-            loss_func = None
 
         set_seed(self.seed, "Classifier weights seed")
         config = awd_lstm_clas_config.copy()
@@ -381,7 +380,6 @@ class ULMFiTClassifier(ULMFiTTrainingCommand):
                                         config=config,
                                         model_dir=self.model_name,
                                         **trn_args)
-        learn.metrics =[accuracy, dice]
         learn = patch_learner(learn)
         if self.base.encoder_fname and not self.random_init:
             print("Loading pretrained model", self.base.encoder_fname)
@@ -401,13 +399,15 @@ class ULMFiTClassifier(ULMFiTTrainingCommand):
             learn.to_fp16()
         return learn
 
-    def train_(self, dataset_or_path=None, **train_config):
+
+    def train_(self, dataset_or_path=None, label_cls=None, loss_func=None, label_cols=None, metrics=[accuracy], **train_config):
         self.replace_(**train_config, _strict=True)
 
         base_tokenizer = self.base.tokenizer
         dataset = self._set_dataset_(dataset_or_path, base_tokenizer)
-        data_clas = dataset.load_clas_databunch(bs=self.bs)
+        data_clas = dataset.load_clas_databunch(bs=self.bs, label_cls=label_cls, label_cols=label_cols)
-        learn = self.get_learner(data_clas=data_clas)
+        learn = self.get_learner(data_clas=data_clas, loss_func=loss_func)
+        learn.metrics = metrics
         print(f"Training: {learn.path / learn.model_dir}")
         learn.unfreeze()
         self._fit_schedule(learn)
@@ -578,7 +578,7 @@ class ULMFiTBinaryClassifier(ULMFiTTrainingCommand):
             learn.to_fp16()
         return learn
 
-    def train_(self, dataset_or_path=None, label_cls=BinaryCategoryList, metrics=[accuracy, dice, partial(fbeta_binary, beta=1), auc_roc_score_multi], label_cols=[0,0], **train_config):
+    def train_(self, dataset_or_path=None, label_cls=BinaryCategoryList, metrics=[accuracy_binary, dice_binary, partial(fbeta_binary, beta=1), auc_roc_score_multi], label_cols=[0,0], **train_config):
         self.replace_(**train_config, _strict=True)
 
         base_tokenizer = self.base.tokenizer