fix sklearn warning on f1 score

multifit/metrics.py

@@ -1,7 +1,12 @@
 import torch
 from torch import Tensor, LongTensor
 from fastai.metrics import auc_roc_score, fbeta
+import sklearn.metrics.classification
 
+# ignore: sklearn UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no predicted samples.
+import warnings
+from sklearn.exceptions import UndefinedMetricWarning
+warnings.filterwarnings(action='ignore', category=UndefinedMetricWarning)
 
 def auc_roc_score_multi(input, targ):
     """area under curve for multi category list (multiple bce losses)."""
@@ -27,6 +32,12 @@ def auc_roc_score_cls_n(y_pred, y_true, class_n=1, **args):
 def fbeta_binary(y_pred, y_true, **args):
     return fbeta(y_pred[:, None], y_true[:, None], **args)
 
+# def f1_binary(y_pred, y_true, **args):
+#     return fbeta(y_pred[:, None], y_true[:, None], thresh=0.5, beta=1, **args)
+
+def f1_binary(y_pred, y_true, **args):
+    return torch.tensor(sklearn.metrics.classification.f1_score(y_true[:, 1].cpu().numpy(), y_pred[:, 1].cpu().numpy()>0.5))
+
 
 def auc_roc_score(input: Tensor, targ: Tensor):
     "Computes the area under the receiver operator characteristic (ROC) curve using the trapezoid method. Restricted binary classification tasks."
@@ -61,7 +72,7 @@ def roc_curve(input: Tensor, targ: Tensor):
 
 def accuracy_binary(input, targs):
     input = torch.sigmoid(input) > 0.5
-    return (input == targs).float().mean()
+    return (input.squeeze() == targs.squeeze()).float().mean()
 
 
 def dice_binary(input, targs, iou=False, eps=1e-8):

multifit/training.py

@@ -5,7 +5,7 @@ import dataclasses
 from fastai.callbacks import CSVLogger, SaveModelCallback
 from fastai.text import *
 
-from multifit.metrics import auc_roc_score_multi, fbeta_binary, auc_roc_score, accuracy_binary, dice_binary
+from multifit.metrics import auc_roc_score_multi, fbeta_binary, auc_roc_score, accuracy_binary, dice_binary, f1_binary
 from multifit.datasets import ULMFiTDataset, ULMFiTTokenizer
 from fastai_contrib.data_block import BinaryCategoryList
 
@@ -562,7 +562,6 @@ class ULMFiTBinaryClassifier(ULMFiTTrainingCommand):
                                         config=config,
                                         model_dir=self.model_name,
                                         **trn_args)
-        # learn.metrics =[accuracy, dice]
         learn = patch_learner(learn)
         if self.base.encoder_fname and not self.random_init:
             print("Loading pretrained model", self.base.encoder_fname)
@@ -582,7 +581,7 @@ class ULMFiTBinaryClassifier(ULMFiTTrainingCommand):
             learn.to_fp16()
         return learn
 
-    def train_(self, dataset_or_path=None, label_cls=BinaryCategoryList, metrics=[accuracy_binary, dice_binary, partial(fbeta_binary, beta=1), auc_roc_score_multi], label_cols=[0,0], **train_config):
+    def train_(self, dataset_or_path=None, label_cls=BinaryCategoryList, metrics=[accuracy_binary, dice_binary, f1_binary, auc_roc_score_multi], label_cols=[0,0], **train_config):
         self.replace_(**train_config, _strict=True)
 
         base_tokenizer = self.base.tokenizer