From a3216db80398d0bb7617ae6708c8ad279ec842be Mon Sep 17 00:00:00 2001 From: wassname Date: Wed, 27 Nov 2019 11:08:32 +0800 Subject: [PATCH] fix sklearn warning on f1 score --- multifit/metrics.py | 13 ++++++++++++- multifit/training.py | 5 ++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/multifit/metrics.py b/multifit/metrics.py index 5c9b72f..38d5d23 100644 --- a/multifit/metrics.py +++ b/multifit/metrics.py @@ -1,7 +1,12 @@ import torch from torch import Tensor, LongTensor from fastai.metrics import auc_roc_score, fbeta +import sklearn.metrics.classification +# ignore: sklearn UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no predicted samples. +import warnings +from sklearn.exceptions import UndefinedMetricWarning +warnings.filterwarnings(action='ignore', category=UndefinedMetricWarning) def auc_roc_score_multi(input, targ): """area under curve for multi category list (multiple bce losses).""" @@ -27,6 +32,12 @@ def auc_roc_score_cls_n(y_pred, y_true, class_n=1, **args): def fbeta_binary(y_pred, y_true, **args): return fbeta(y_pred[:, None], y_true[:, None], **args) +# def f1_binary(y_pred, y_true, **args): +# return fbeta(y_pred[:, None], y_true[:, None], thresh=0.5, beta=1, **args) + +def f1_binary(y_pred, y_true, **args): + return torch.tensor(sklearn.metrics.classification.f1_score(y_true[:, 1].cpu().numpy(), y_pred[:, 1].cpu().numpy()>0.5)) + def auc_roc_score(input: Tensor, targ: Tensor): "Computes the area under the receiver operator characteristic (ROC) curve using the trapezoid method. Restricted binary classification tasks." @@ -61,7 +72,7 @@ def roc_curve(input: Tensor, targ: Tensor): def accuracy_binary(input, targs): input = torch.sigmoid(input) > 0.5 - return (input == targs).float().mean() + return (input.squeeze() == targs.squeeze()).float().mean() def dice_binary(input, targs, iou=False, eps=1e-8): diff --git a/multifit/training.py b/multifit/training.py index bd18c6e..2548759 100644 --- a/multifit/training.py +++ b/multifit/training.py @@ -5,7 +5,7 @@ import dataclasses from fastai.callbacks import CSVLogger, SaveModelCallback from fastai.text import * -from multifit.metrics import auc_roc_score_multi, fbeta_binary, auc_roc_score, accuracy_binary, dice_binary +from multifit.metrics import auc_roc_score_multi, fbeta_binary, auc_roc_score, accuracy_binary, dice_binary, f1_binary from multifit.datasets import ULMFiTDataset, ULMFiTTokenizer from fastai_contrib.data_block import BinaryCategoryList @@ -562,7 +562,6 @@ class ULMFiTBinaryClassifier(ULMFiTTrainingCommand): config=config, model_dir=self.model_name, **trn_args) - # learn.metrics =[accuracy, dice] learn = patch_learner(learn) if self.base.encoder_fname and not self.random_init: print("Loading pretrained model", self.base.encoder_fname) @@ -582,7 +581,7 @@ class ULMFiTBinaryClassifier(ULMFiTTrainingCommand): learn.to_fp16() return learn - def train_(self, dataset_or_path=None, label_cls=BinaryCategoryList, metrics=[accuracy_binary, dice_binary, partial(fbeta_binary, beta=1), auc_roc_score_multi], label_cols=[0,0], **train_config): + def train_(self, dataset_or_path=None, label_cls=BinaryCategoryList, metrics=[accuracy_binary, dice_binary, f1_binary, auc_roc_score_multi], label_cols=[0,0], **train_config): self.replace_(**train_config, _strict=True) base_tokenizer = self.base.tokenizer