From 67da707648234275ecb1819b6caf99d13b891ce2 Mon Sep 17 00:00:00 2001 From: wassname Date: Tue, 26 Nov 2019 21:03:17 +0800 Subject: [PATCH] tidy and metrics --- {multifit => fastai_contrib}/metrics.py | 8 ++++++- multifit/configurations.py | 29 ------------------------- multifit/training.py | 5 ----- 3 files changed, 7 insertions(+), 35 deletions(-) rename {multifit => fastai_contrib}/metrics.py (91%) diff --git a/multifit/metrics.py b/fastai_contrib/metrics.py similarity index 91% rename from multifit/metrics.py rename to fastai_contrib/metrics.py index 8b9e1df..e0f6fa5 100644 --- a/multifit/metrics.py +++ b/fastai_contrib/metrics.py @@ -14,7 +14,7 @@ def auc_roc_score_cls_n(y_pred, y_true, class_n=1, **args): def auc_roc_score(input: Tensor, targ: Tensor): "Computes the area under the receiver operator characteristic (ROC) curve using the trapezoid method. Restricted binary classification tasks." - fpr, tpr = roc_curve(input.squeeze(), targ.squeeze()) + fpr, tpr = roc_curve(input, targ) d = fpr[1:] - fpr[:-1] sl1, sl2 = [slice(None)], [slice(None)] sl1[-1], sl2[-1] = slice(1, None), slice(None, -1) @@ -43,6 +43,12 @@ def roc_curve(input: Tensor, targ: Tensor): return fpr, tpr +def f1_score(*args, **kwargs): + return fbeta_cls_n(beta=1, thresh=0.5, *args, **kwargs) + def accuracy_binary(input, targs): input = torch.sigmoid(input) > 0.5 return (input == targs).float().mean() + +def mean_output(input, targs): + return input.mean() diff --git a/multifit/configurations.py b/multifit/configurations.py index 2149395..b0ef7a3 100644 --- a/multifit/configurations.py +++ b/multifit/configurations.py @@ -95,35 +95,6 @@ def multifit_paper_version(): self.classifier.replace_(num_epochs=8, drop_mult=0.5, bs=18, label_smoothing_eps=0.1, early_stopping=None) return self -def multifit_paper_version_bce(): - self = ULMFiTBinary() - dps = {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} - self.replace_( - label_smoothing_eps=0.0, - label_smoothing_eps_norm_by_classes=True, - true_wd=True, - wd=0.01, ## important :) - seed=0, - fp16=False, - bs=64, - use_adam_08=False, - early_stopping=None, - clip=0.12, - dropout_values=dps, - name=_use_caller_name() - ) - self.arch.replace_( - tokenizer_type='sp', - max_vocab=15000, - qrnn=True, - n_layers=4, - n_hid=1550 # vs 1552 - ) - self.pretrain_lm.replace_(num_epochs=10, drop_mult=0.0, lr=5e-3, use_adam_08=True, true_wd=False, wd=1e-7, bs=50,) - self.finetune_lm.replace_(num_epochs=20, drop_mult=0.3, lr=1e-3, true_wd=False, wd=1e-7, bs=20) - # TODO check I can't do label_smoothing_eps on binary classifier - self.classifier.replace_(num_epochs=8, drop_mult=0.5, bs=18, label_smoothing_eps=0.0, early_stopping=None) - return self def ulmfit_orig(): self = multifit_paper_version() diff --git a/multifit/training.py b/multifit/training.py index d0e4944..990d9fa 100644 --- a/multifit/training.py +++ b/multifit/training.py @@ -5,9 +5,7 @@ import dataclasses from fastai.callbacks import CSVLogger, SaveModelCallback from fastai.text import * -from multifit.metrics import auc_roc_score_multi, fbeta_binary, auc_roc_score, accuracy_binary, dice_binary from multifit.datasets import ULMFiTDataset, ULMFiTTokenizer -from fastai_contrib.data_block import BinaryCategoryList CLS_BEST = 'cls_best' LM_BEST = "lm_best" @@ -449,9 +447,6 @@ class ULMFiTClassifier(ULMFiTTrainingCommand): data_cls = self.dataset.load_clas_databunch(bs=self.bs) learn = self.get_learner(data_cls, eval_only=True) - # avg = 'binary' if learn.data.c == 2 else 'macro' - # learn.metrics = [accuracy, FBeta(beta=1.0, average=avg), Precision(average=avg), Recall(average=avg)] - # learn.metrics = [accuracy, fbeta, auc_roc_score,] learn.metrics = [accuracy, dice] print(f"Loading model {save_name}") learn.load(save_name)