diff --git a/fastai_contrib/data_block.py b/fastai_contrib/data_block.py deleted file mode 100644 index 46ce2aa..0000000 --- a/fastai_contrib/data_block.py +++ /dev/null @@ -1,35 +0,0 @@ -from typing import Iterator, Collection -from fastai.data_block import CategoryListBase -from fastai.text import * - - -class BinaryProcessor(CategoryProcessor): - def create_classes(self, classes): - self.classes = classes - if classes is not None: self.c2i = {0:0, 1:1} - def generate_classes(self, items): - return [0] - -class BinaryCategoryList(CategoryListBase): - "Basic `ItemList` for single classification labels." - _processor=BinaryProcessor - def __init__(self, items:Iterator, classes:Collection=None, label_delim:str=None, **kwargs): - super().__init__(items, classes=classes, **kwargs) - mean = self.items.mean() - # if mean and mean != 0: - # weight = torch.tensor([1 / mean]).cuda() - # print(f'Weighting BCEWithLogitsFlat by {weight.item()}') - # else: - weight = None - # raise Exception('debug') - self.loss_func = BCEWithLogitsFlat(weight=weight) - - def reconstruct(self, t): - return Category(t, self.c2i[t.item()]) - - def get(self, i): - o = self.items[i] - if o is None: return None - return Category(o, self.c2i[o]) - - def analyze_pred(self, pred, thresh:float=0.5): return pred.argmax() diff --git a/multifit/metrics.py b/multifit/metrics.py index 5c9b72f..8b9e1df 100644 --- a/multifit/metrics.py +++ b/multifit/metrics.py @@ -2,32 +2,16 @@ import torch from torch import Tensor, LongTensor from fastai.metrics import auc_roc_score, fbeta - -def auc_roc_score_multi(input, targ): - """area under curve for multi category list (multiple bce losses).""" - n = input.shape[1] - targ = targ * 1 - if targ.shape != input.shape: - targ = targ.expand(input.T.shape).T - scores = [auc_roc_score(input[:, i], targ[:, i]) for i in range(n)] - return torch.tensor(scores).mean() - - def fbeta_cls_n(y_pred, y_true, class_n=1, **args): """F1 score of class 1, to be used with 2 classes.""" y_pred = torch.nn.functional.softmax(y_pred, dim=-1) return fbeta(y_pred, y_true[:, None], sigmoid=False, **args) def auc_roc_score_cls_n(y_pred, y_true, class_n=1, **args): - """F1 score of class 1, to be used with 2 classes.""" + """auc_roc_score score of class 1, to be used with 2 classes.""" y_pred = torch.nn.functional.softmax(y_pred, dim=-1) return auc_roc_score(y_pred[:, class_n], y_true==class_n, **args) - -def fbeta_binary(y_pred, y_true, **args): - return fbeta(y_pred[:, None], y_true[:, None], **args) - - def auc_roc_score(input: Tensor, targ: Tensor): "Computes the area under the receiver operator characteristic (ROC) curve using the trapezoid method. Restricted binary classification tasks." fpr, tpr = roc_curve(input.squeeze(), targ.squeeze()) @@ -62,16 +46,3 @@ def roc_curve(input: Tensor, targ: Tensor): def accuracy_binary(input, targs): input = torch.sigmoid(input) > 0.5 return (input == targs).float().mean() - - -def dice_binary(input, targs, iou=False, eps=1e-8): - "Dice coefficient metric for binary target. If iou=True, returns iou metric, classic for segmentation problems." - input = torch.sigmoid(input) > 0.5 - intersect = (input * targs).sum(dim=1).float() - union = (input + targs).sum(dim=1).float() - if not iou: - l = 2.0 * intersect / union - else: - l = intersect / (union - intersect + eps) - l[union == 0.0] = 1.0 - return l.mean() diff --git a/multifit/training.py b/multifit/training.py index bd18c6e..d0e4944 100644 --- a/multifit/training.py +++ b/multifit/training.py @@ -516,187 +516,6 @@ class ULMFiTClassifier(ULMFiTTrainingCommand): learn.fit_one_cycle(self.num_epochs - 4, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7), wd=1e-7) -@dataclass -class ULMFiTBinaryClassifier(ULMFiTTrainingCommand): - bs: int = 20 - num_epochs: int = 10 - drop_mult: float = 0.5 - dropout_values: dict = field(default_factory=dict) - wd: float = 0.01 - clip: float = None - label_smoothing_eps: float = 0.0 - label_smoothing_eps_norm_by_classes: bool = False - weighted_cross_entropy: tuple = None - early_stopping: str = 'accuracy' - fit_schedule: str = '1cycle' - base: ULMFiTFinetuning = field(repr=False, default=None) - random_init: bool = False - seed: int = 0 - bptt: int = 70 - fp16: bool = False - arch: ULMFiTArchitecture = None - - def get_learner(self, data_clas, eval_only=False, **additional_trn_args): - assert self.weighted_cross_entropy is None or self.label_smoothing_eps == 0, "Label smoothing not implemented with weighted_cross_entropy" - if self.weighted_cross_entropy is not None: - loss_func = BCEWithLogitsFlat(weight=torch.tensor(self.weighted_cross_entropy, dtype=torch.float32).cuda()) - elif self.label_smoothing_eps > 0.0: - raise Exception("label_smoothing is not implemented in the binary classifier") - else: - loss_func = None - - set_seed(self.seed, "Classifier weights seed") - config = awd_lstm_clas_config.copy() - config.update(emb_sz=self.arch.emb_sz, n_hid=self.arch.n_hid, n_layers=self.arch.n_layers, qrnn=self.arch.qrnn, - **self.dropout_values) - - trn_args = dict(drop_mult=self.drop_mult, wd=self.wd, pretrained=False, bptt=self.bptt, - loss_func=loss_func, clip=self.clip) - if hasattr(Learner, 'silent'): - trn_args.update(silent=eval_only) - - trn_args.update(**additional_trn_args) - print("Training args: ", trn_args, "config: ", config) - learn = text_classifier_learner(data_clas, - AWD_LSTM, - config=config, - model_dir=self.model_name, - **trn_args) - # learn.metrics =[accuracy, dice] - learn = patch_learner(learn) - if self.base.encoder_fname and not self.random_init: - print("Loading pretrained model", self.base.encoder_fname) - learn.load_encoder(self.base.encoder_fname) - learn.freeze() - else: - warn("No pretrained encoder") - - set_seed(self.seed, "Classifier training seed") - if not eval_only: - learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/cls-history")] - if self.early_stopping: - learn.callback_fns += [partial(SaveModelCallback, every='improvement', - name='cls_best_tmp', - monitor=self.early_stopping)] - if self.fp16: - learn.to_fp16() - return learn - - def train_(self, dataset_or_path=None, label_cls=BinaryCategoryList, metrics=[accuracy_binary, dice_binary, partial(fbeta_binary, beta=1), auc_roc_score_multi], label_cols=[0,0], **train_config): - self.replace_(**train_config, _strict=True) - - base_tokenizer = self.base.tokenizer - dataset = self._set_dataset_(dataset_or_path, base_tokenizer) - data_clas = dataset.load_clas_databunch(bs=self.bs, label_cls=label_cls, label_cols=label_cols) - learn = self.get_learner(data_clas=data_clas) - learn.metrics = metrics - print(f"Training: {learn.path / learn.model_dir}") - learn.unfreeze() - self._fit_schedule(learn) - - self.experiment_path = learn.path / learn.model_dir - base_tokenizer.save(self.experiment_path, learn=learn) - learn.to_fp32() - learn.save(CLS_BEST, with_opt=False) - print("Classifier model saved to", self.experiment_path) - self.save_paramters() - learn.destroy() - return - - def _validate(self, learn, ds_type): - ds_name = ds_type.name.lower() - print(f"Model: {self.name}, ds_name: {ds_name}") - results_dict = dict(zip( - [f'{ds_name} loss'] + [f"{ds_name} {getattr(m, '__name__', m.__class__.__name__)}" for m in learn.metrics], - map(float, learn.validate(learn.data.dl(ds_type))))) - results_dict['name'] = self.name - return results_dict - - def validate(self, *splits, data_cls=None, save_name=CLS_BEST, use_cache=True, save_preds=False): - """Validates - splits - Dataset Types to validate on default DatasetType.Test, DatasetType.Valid, DatasetType.Train - """ - if len(splits) == 0: - splits = [DatasetType.Test, DatasetType.Valid, DatasetType.Train] - cache_file = (self.experiment_path / f'results{"" if save_name == CLS_BEST else "-" + save_name}.json') - if use_cache and cache_file.exists(): - with cache_file.open("r") as fp: - return json.load(fp) - - if data_cls is None: - data_cls = self.dataset.load_clas_databunch(bs=self.bs) - - learn = self.get_learner(data_cls, eval_only=True) - learn.metrics = [accuracy, dice, fbeta, auc_roc_score_multi] - print(f"Loading model {save_name}") - learn.load(save_name) - if save_preds: - probs, targets = learn.get_preds(ordered=True, ds_type=DatasetType.Test, activ=partial(F.softmax, dim=-1)) - np.save(str(self.experiment_path / f"preds-on-test.npy"), probs.cpu().numpy()) - - results_dict = {} - for split in splits: - results_dict.update(self._validate(learn, split)) - print(results_dict) - with cache_file.open("w") as fp: - json.dump(results_dict, fp) - return results_dict - - def _fit_schedule(self, learn): - getattr(self, '_fit_schedule_' + self.fit_schedule)(learn) - - def _fit_schedule_1cycle(self, learn): - learn.unfreeze() - learn.fit_one_cycle(self.num_epochs, slice(1e-2 / (2.6 ** 4), 2e-2), moms=(0.8, 0.7)) - - def _fit_schedule_layered(self, learn): - learn.freeze_to(-1) - learn.fit_one_cycle(1, 2e-2, moms=(0.8, 0.7)) - if self.num_epochs > 1: - learn.freeze_to(-2) - learn.fit_one_cycle(1, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7)) - learn.freeze_to(-3) - learn.fit_one_cycle(1, slice(5e-3 / (2.6 ** 4), 5e-3), moms=(0.8, 0.7)) - learn.unfreeze() - if self.num_epochs > 5: - learn.fit_one_cycle(self.num_epochs - 4, slice(1e-3 / (2.6 ** 4), 1e-3), moms=(0.8, 0.7)) - - def _fit_schedule_2cycle(self, learn): - learn.freeze_to(-1) - learn.fit_one_cycle(1, 2e-2, moms=(0.8, 0.7)) - learn.unfreeze() - if self.num_epochs > 1: - learn.fit_one_cycle(self.num_epochs - 1, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7)) - - def _fit_schedule_reverse_2cycle(self, learn): - learn.unfreeze() - for g in learn.layer_groups[-1:]: - for l in g: - if not learn.train_bn or not isinstance(l, bn_types): requires_grad(l, False) - learn.create_opt(defaults.lr) - learn.fit_one_cycle(self.num_epochs, slice(1e-2 / (2.6 ** 4), 2e-2), moms=(0.8, 0.7)) - learn.unfreeze() - learn.fit_one_cycle(self.num_epochs, slice(1e-3 / (2.6 ** 4), 2e-3), moms=(0.8, 0.7)) - - def _fit_schedule_false_wd(self, learn): - learn.true_wd = False - learn.fit_one_cycle(1, 5e-2, moms=(0.8, 0.7), wd=1e-7) - if self.num_epochs > 1: - learn.freeze_to(-2) - learn.fit_one_cycle(1, slice(5e-2 / (2.6 ** 4), 5e-2), moms=(0.8, 0.7), wd=1e-7) - learn.freeze_to(-3) - learn.fit_one_cycle(1, slice(5e-4 / (2.6 ** 4), 5e-4), moms=(0.8, 0.7), wd=1e-7) - learn.unfreeze() - if self.num_epochs > 5: - learn.fit_one_cycle(self.num_epochs - 4, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7), wd=1e-7) - - -def path_if_model_exists(path, weights_name): - """Return path to model if it exists""" - model_path = path / (weights_name + ".pth") - return path if model_path.exists() else None - - @dataclass class ULMFiT: arch: ULMFiTArchitecture = None @@ -763,72 +582,6 @@ class ULMFiT: return self.load_(path) -@dataclass -class ULMFiTBinary: - arch: ULMFiTArchitecture = None - pretrain_lm: ULMFiTPretraining = None - finetune_lm: ULMFiTFinetuning = None - classifier: ULMFiTBinaryClassifier = None - - def __post_init__(self): - self.arch = ULMFiTArchitecture() - self.pretrain_lm = ULMFiTPretraining(arch=self.arch) - self.finetune_lm = ULMFiTFinetuning(arch=self.arch, base=self.pretrain_lm) - self.classifier = ULMFiTBinaryClassifier(arch=self.arch, base=self.finetune_lm) - - def load_(self, experiment_path:Path, silent=False): - success = (self.classifier.load_(experiment_path, silent=silent) or - self.finetune_lm.load_(experiment_path, silent=silent) or - self.pretrain_lm.load_(experiment_path, silent=silent) or - self.load_legacy_(experiment_path, silent=silent)) - if not success: - warn(f'Unable to load experiment {experiment_path}') - return self - - def load_legacy_(self, experiment_path, silent=True): - if not (experiment_path / "info.json").exists(): - return False - with (experiment_path / "info.json").open('r') as f: - d = json.load(f) - dataset_path = d.pop('dataset_path', "") - d['n_hid'] = d['nh'] - d['n_layers'] = d['nl'] - d['lang'] = detect_lang_from_dataset_path(Path(dataset_path)) - if "wiki" in str(dataset_path): - self.arch.replace_(**d) - self.pretrain_lm.replace_(**d) - self.pretrain_lm.experiment_path = path_if_model_exists(experiment_path, LM_BEST) - self.pretrain_lm.dataset_path = dataset_path if dataset_path in str(experiment_path) else None - else: - self.replace_(**d) - self.finetune_lm.experiment_path = path_if_model_exists(experiment_path, ENC_BEST) - self.finetune_lm.dataset_path = dataset_path if dataset_path in str(experiment_path) else None - self.classifier.experiment_path = path_if_model_exists(experiment_path, CLS_BEST) - self.classifier.dataset_path = dataset_path if dataset_path in str(experiment_path) else None - return True - - def replace_(self, **kwargs): - self.arch.replace_(**kwargs) - self.pretrain_lm.replace_(**kwargs) - self.finetune_lm.replace_(**kwargs) - self.classifier.replace_(**kwargs) - return self - - def pprint(self): - print(f"""ULMFiT( - {self.arch}, - {self.pretrain_lm}, - {self.finetune_lm}, - {self.classifier}, -)""") - - def from_pretrained_(self, name, repo="n-waves/multifit-models"): - name = name.rstrip(".tgz") # incase someone put's tgz name the name - url = f"https://github.com/{repo}/releases/download/{name}/{name}.tgz" - path = untar_data(url.rstrip(".tgz"), data=False) # untar_data adds .tgz - return self.load_(path) - - def from_pretrained(name): #TODO: Detect name and load configuration from . import configurations