diff --git a/fastai_contrib/data_block.py b/fastai_contrib/data_block.py index 95b7010..46ce2aa 100644 --- a/fastai_contrib/data_block.py +++ b/fastai_contrib/data_block.py @@ -16,11 +16,12 @@ class BinaryCategoryList(CategoryListBase): def __init__(self, items:Iterator, classes:Collection=None, label_delim:str=None, **kwargs): super().__init__(items, classes=classes, **kwargs) mean = self.items.mean() - if mean and mean!=0: - weight = torch.tensor([1 / self.items.mean()]).cuda() - else: - weight = None - raise Exception('debug') + # if mean and mean != 0: + # weight = torch.tensor([1 / mean]).cuda() + # print(f'Weighting BCEWithLogitsFlat by {weight.item()}') + # else: + weight = None + # raise Exception('debug') self.loss_func = BCEWithLogitsFlat(weight=weight) def reconstruct(self, t): diff --git a/multifit/datasets/dataset.py b/multifit/datasets/dataset.py index 715295c..ce7857a 100644 --- a/multifit/datasets/dataset.py +++ b/multifit/datasets/dataset.py @@ -224,6 +224,7 @@ class ULMFiTDataset(Dataset): return self._vocab def load_clas_databunch(self, bs, label_cls=None, **args): + print('DEBUG', bs, label_cls, args) vocab = self._load_vocab() cls_name = "cls.cache.databunch" diff --git a/multifit/metrics.py b/multifit/metrics.py index 35aad2c..8cdbdaf 100644 --- a/multifit/metrics.py +++ b/multifit/metrics.py @@ -6,8 +6,61 @@ from fastai.metrics import auc_roc_score, fbeta def auc_roc_score_multi(input, targ): """area under curve for multi category list (multiple bce losses).""" n = input.shape[1] + targ = targ * 1 + if targ.shape != input.shape: + targ = targ.expand(input.T.shape).T scores = [auc_roc_score(input[:, i], targ[:, i]) for i in range(n)] return torch.tensor(scores).mean() + def fbeta_binary(y_pred, y_true, **args): return fbeta(y_pred[:, None], y_true[:, None], **args) + + +def auc_roc_score(input: Tensor, targ: Tensor): + "Computes the area under the receiver operator characteristic (ROC) curve using the trapezoid method. Restricted binary classification tasks." + fpr, tpr = roc_curve(input.squeeze(), targ.squeeze()) + d = fpr[1:] - fpr[:-1] + sl1, sl2 = [slice(None)], [slice(None)] + sl1[-1], sl2[-1] = slice(1, None), slice(None, -1) + return (d * (tpr[tuple(sl1)] + tpr[tuple(sl2)]) / 2.0).sum(-1) + + +def roc_curve(input: Tensor, targ: Tensor): + "Computes the receiver operator characteristic (ROC) curve by determining the true positive ratio (TPR) and false positive ratio (FPR) for various classification thresholds. Restricted binary classification tasks." + # wassname: fix this by making LongTensor([0]=>device) + targ = targ == 1 + desc_score_indices = torch.flip(input.argsort(-1), [-1]) + input = input[desc_score_indices] + targ = targ[desc_score_indices] + d = input[1:] - input[:-1] + distinct_value_indices = torch.nonzero(d).transpose(0, 1)[0] + threshold_idxs = torch.cat( + (distinct_value_indices, LongTensor([len(targ) - 1]).to(targ.device)) + ) + tps = torch.cumsum(targ * 1, dim=-1)[threshold_idxs] + fps = 1 + threshold_idxs - tps + if tps[0] != 0 or fps[0] != 0: + zer = torch.zeros(1, dtype=fps.dtype, device=fps.device) + fps = torch.cat((zer, fps)) + tps = torch.cat((zer, tps)) + fpr, tpr = fps.float() / fps[-1], tps.float() / tps[-1] + return fpr, tpr + + +def accuracy_binary(input, targs): + input = torch.sigmoid(input) > 0.5 + return (input == targs).float().mean() + + +def dice_binary(input, targs, iou=False, eps=1e-8): + "Dice coefficient metric for binary target. If iou=True, returns iou metric, classic for segmentation problems." + input = torch.sigmoid(input) > 0.5 + intersect = (input * targs).sum(dim=1).float() + union = (input + targs).sum(dim=1).float() + if not iou: + l = 2.0 * intersect / union + else: + l = intersect / (union - intersect + eps) + l[union == 0.0] = 1.0 + return l.mean() diff --git a/multifit/training.py b/multifit/training.py index c5e7457..a1302dd 100644 --- a/multifit/training.py +++ b/multifit/training.py @@ -5,7 +5,7 @@ import dataclasses from fastai.callbacks import CSVLogger, SaveModelCallback from fastai.text import * -from multifit.metrics import auc_roc_score_multi, fbeta_binary, auc_roc_score +from multifit.metrics import auc_roc_score_multi, fbeta_binary, auc_roc_score, accuracy_binary, dice_binary from multifit.datasets import ULMFiTDataset, ULMFiTTokenizer from fastai_contrib.data_block import BinaryCategoryList @@ -351,18 +351,17 @@ class ULMFiTClassifier(ULMFiTTrainingCommand): fp16: bool = False arch: ULMFiTArchitecture = None - def get_learner(self, data_clas, eval_only=False, **additional_trn_args): + def get_learner(self, data_clas, eval_only=False, loss_func=None, **additional_trn_args): assert self.weighted_cross_entropy is None or self.label_smoothing_eps == 0, "Label smoohting not implemented with weighted_cross_entropy" - if self.weighted_cross_entropy is not None: - loss_func = CrossEntropyFlat(weight=torch.tensor(self.weighted_cross_entropy, dtype=torch.float32).cuda()) - elif self.label_smoothing_eps > 0.0: - eps = self.label_smoothing_eps - if self.label_smoothing_eps_norm_by_classes: - eps = eps / data_clas.c - print("Using Label smoothing with eps = ", eps) - loss_func = FlattenedLoss(LabelSmoothingCrossEntropy, eps=eps) - else: - loss_func = None + if loss_func is None: + if self.weighted_cross_entropy is not None: + loss_func = CrossEntropyFlat(weight=torch.tensor(self.weighted_cross_entropy, dtype=torch.float32).cuda()) + elif self.label_smoothing_eps > 0.0: + eps = self.label_smoothing_eps + if self.label_smoothing_eps_norm_by_classes: + eps = eps / data_clas.c + print("Using Label smoothing with eps = ", eps) + loss_func = FlattenedLoss(LabelSmoothingCrossEntropy, eps=eps) set_seed(self.seed, "Classifier weights seed") config = awd_lstm_clas_config.copy() @@ -381,7 +380,6 @@ class ULMFiTClassifier(ULMFiTTrainingCommand): config=config, model_dir=self.model_name, **trn_args) - learn.metrics =[accuracy, dice] learn = patch_learner(learn) if self.base.encoder_fname and not self.random_init: print("Loading pretrained model", self.base.encoder_fname) @@ -401,13 +399,15 @@ class ULMFiTClassifier(ULMFiTTrainingCommand): learn.to_fp16() return learn - def train_(self, dataset_or_path=None, **train_config): + + def train_(self, dataset_or_path=None, label_cls=None, loss_func=None, label_cols=None, metrics=[accuracy], **train_config): self.replace_(**train_config, _strict=True) base_tokenizer = self.base.tokenizer dataset = self._set_dataset_(dataset_or_path, base_tokenizer) - data_clas = dataset.load_clas_databunch(bs=self.bs) - learn = self.get_learner(data_clas=data_clas) + data_clas = dataset.load_clas_databunch(bs=self.bs, label_cls=label_cls, label_cols=label_cols) + learn = self.get_learner(data_clas=data_clas, loss_func=loss_func) + learn.metrics = metrics print(f"Training: {learn.path / learn.model_dir}") learn.unfreeze() self._fit_schedule(learn) @@ -578,7 +578,7 @@ class ULMFiTBinaryClassifier(ULMFiTTrainingCommand): learn.to_fp16() return learn - def train_(self, dataset_or_path=None, label_cls=BinaryCategoryList, metrics=[accuracy, dice, partial(fbeta_binary, beta=1), auc_roc_score_multi], label_cols=[0,0], **train_config): + def train_(self, dataset_or_path=None, label_cls=BinaryCategoryList, metrics=[accuracy_binary, dice_binary, partial(fbeta_binary, beta=1), auc_roc_score_multi], label_cols=[0,0], **train_config): self.replace_(**train_config, _strict=True) base_tokenizer = self.base.tokenizer