tidy and metrics

fastai_contrib/metrics.py

@@ -14,7 +14,7 @@ def auc_roc_score_cls_n(y_pred, y_true, class_n=1, **args):
 
 def auc_roc_score(input: Tensor, targ: Tensor):
     "Computes the area under the receiver operator characteristic (ROC) curve using the trapezoid method. Restricted binary classification tasks."
-    fpr, tpr = roc_curve(input.squeeze(), targ.squeeze())
+    fpr, tpr = roc_curve(input, targ)
     d = fpr[1:] - fpr[:-1]
     sl1, sl2 = [slice(None)], [slice(None)]
     sl1[-1], sl2[-1] = slice(1, None), slice(None, -1)
@@ -43,6 +43,12 @@ def roc_curve(input: Tensor, targ: Tensor):
     return fpr, tpr
 
 
+def f1_score(*args, **kwargs):
+    return fbeta_cls_n(beta=1, thresh=0.5, *args, **kwargs)
+
 def accuracy_binary(input, targs):
     input = torch.sigmoid(input) > 0.5
     return (input == targs).float().mean()
+
+def mean_output(input, targs):
+    return input.mean()

multifit/configurations.py

@@ -95,35 +95,6 @@ def multifit_paper_version():
     self.classifier.replace_(num_epochs=8, drop_mult=0.5, bs=18, label_smoothing_eps=0.1, early_stopping=None)
     return self
 
-def multifit_paper_version_bce():
-    self = ULMFiTBinary()
-    dps = {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}
-    self.replace_(
-        label_smoothing_eps=0.0,
-        label_smoothing_eps_norm_by_classes=True,
-        true_wd=True,
-        wd=0.01, ## important :)
-        seed=0,
-        fp16=False,
-        bs=64,
-        use_adam_08=False,
-        early_stopping=None,
-        clip=0.12,
-        dropout_values=dps,
-        name=_use_caller_name()
-    )
-    self.arch.replace_(
-        tokenizer_type='sp',
-        max_vocab=15000,
-        qrnn=True,
-        n_layers=4,
-        n_hid=1550 # vs 1552
-    )
-    self.pretrain_lm.replace_(num_epochs=10, drop_mult=0.0, lr=5e-3, use_adam_08=True, true_wd=False, wd=1e-7, bs=50,)
-    self.finetune_lm.replace_(num_epochs=20, drop_mult=0.3, lr=1e-3, true_wd=False, wd=1e-7, bs=20)
-    # TODO check I can't do label_smoothing_eps on binary classifier
-    self.classifier.replace_(num_epochs=8, drop_mult=0.5, bs=18, label_smoothing_eps=0.0, early_stopping=None)
-    return self
 
 def ulmfit_orig():
     self = multifit_paper_version()

multifit/training.py

@@ -5,9 +5,7 @@ import dataclasses
 from fastai.callbacks import CSVLogger, SaveModelCallback
 from fastai.text import *
 
-from multifit.metrics import auc_roc_score_multi, fbeta_binary, auc_roc_score, accuracy_binary, dice_binary
 from multifit.datasets import ULMFiTDataset, ULMFiTTokenizer
-from fastai_contrib.data_block import BinaryCategoryList
 
 CLS_BEST = 'cls_best'
 LM_BEST = "lm_best"
@@ -449,9 +447,6 @@ class ULMFiTClassifier(ULMFiTTrainingCommand):
             data_cls = self.dataset.load_clas_databunch(bs=self.bs)
 
         learn = self.get_learner(data_cls, eval_only=True)
-        # avg = 'binary' if learn.data.c == 2 else 'macro'
-        # learn.metrics = [accuracy, FBeta(beta=1.0, average=avg), Precision(average=avg), Recall(average=avg)]
-        # learn.metrics = [accuracy, fbeta, auc_roc_score,]
         learn.metrics = [accuracy, dice]
         print(f"Loading model {save_name}")
         learn.load(save_name)