From 7557805043a6c2d89270f16fe0fe9d8afe1b6a22 Mon Sep 17 00:00:00 2001 From: wassname Date: Tue, 26 Nov 2019 10:16:23 +0800 Subject: [PATCH] made a binary classification version making binary category label list --- .gitignore | 2 + fastai_contrib/data_block.py | 34 +++++ fastai_contrib/text_data.py | 8 +- multifit/configurations.py | 32 ++++- multifit/datasets/dataset.py | 22 +-- multifit/metrics.py | 13 ++ multifit/training.py | 252 ++++++++++++++++++++++++++++++++++- 7 files changed, 345 insertions(+), 18 deletions(-) create mode 100644 fastai_contrib/data_block.py create mode 100644 multifit/metrics.py diff --git a/.gitignore b/.gitignore index d148f25..b13bbb3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +/data + *.bak *.log *~ diff --git a/fastai_contrib/data_block.py b/fastai_contrib/data_block.py new file mode 100644 index 0000000..95b7010 --- /dev/null +++ b/fastai_contrib/data_block.py @@ -0,0 +1,34 @@ +from typing import Iterator, Collection +from fastai.data_block import CategoryListBase +from fastai.text import * + + +class BinaryProcessor(CategoryProcessor): + def create_classes(self, classes): + self.classes = classes + if classes is not None: self.c2i = {0:0, 1:1} + def generate_classes(self, items): + return [0] + +class BinaryCategoryList(CategoryListBase): + "Basic `ItemList` for single classification labels." + _processor=BinaryProcessor + def __init__(self, items:Iterator, classes:Collection=None, label_delim:str=None, **kwargs): + super().__init__(items, classes=classes, **kwargs) + mean = self.items.mean() + if mean and mean!=0: + weight = torch.tensor([1 / self.items.mean()]).cuda() + else: + weight = None + raise Exception('debug') + self.loss_func = BCEWithLogitsFlat(weight=weight) + + def reconstruct(self, t): + return Category(t, self.c2i[t.item()]) + + def get(self, i): + o = self.items[i] + if o is None: return None + return Category(o, self.c2i[o]) + + def analyze_pred(self, pred, thresh:float=0.5): return pred.argmax() diff --git a/fastai_contrib/text_data.py b/fastai_contrib/text_data.py index 30a4313..c10b0d5 100644 --- a/fastai_contrib/text_data.py +++ b/fastai_contrib/text_data.py @@ -130,7 +130,7 @@ def make_data_bunch_from_df(cls, path: PathOrStr, train_df: DataFrame, valid_df: tokenizer: Tokenizer = None, vocab: Vocab = None, classes: Collection[str] = None, text_cols: IntsOrStrs = 1, label_cols: IntsOrStrs = 0, label_delim: str = None, chunksize: int = 10000, - max_vocab: int = 60000, + max_vocab: int = 60000, label_cls: Callable = None, min_freq: int = 2, mark_fields: bool = False, include_bos: bool = True, include_eos: bool = False, processor=None, **kwargs) -> DataBunch: "Create a `TextDataBunch` from DataFrames. `kwargs` are passed to the dataloader creation." @@ -142,8 +142,8 @@ def make_data_bunch_from_df(cls, path: PathOrStr, train_df: DataFrame, valid_df: include_bos=include_bos, include_eos=include_eos) if classes is None and is_listy(label_cols) and len(label_cols) > 1: classes = label_cols - src = ItemLists(path, TextList.from_df(train_df, path, cols=text_cols, processor=processor), - TextList.from_df(valid_df, path, cols=text_cols, processor=processor)) + src = ItemLists(path, TextList.from_df(train_df, path, cols=text_cols, processor=processor, label_cls=label_cls), + TextList.from_df(valid_df, path, cols=text_cols, processor=processor, label_cls=label_cls)) if cls == TextLMDataBunch: src = src.label_for_lm() else: @@ -151,4 +151,4 @@ def make_data_bunch_from_df(cls, path: PathOrStr, train_df: DataFrame, valid_df: src = src.label_from_df(cols=label_cols, classes=classes, label_delim=label_delim) else: src = src.label_from_df(cols=label_cols, classes=classes) - return src.databunch(**kwargs) \ No newline at end of file + return src.databunch(**kwargs) diff --git a/multifit/configurations.py b/multifit/configurations.py index 0870bf2..2149395 100644 --- a/multifit/configurations.py +++ b/multifit/configurations.py @@ -95,6 +95,36 @@ def multifit_paper_version(): self.classifier.replace_(num_epochs=8, drop_mult=0.5, bs=18, label_smoothing_eps=0.1, early_stopping=None) return self +def multifit_paper_version_bce(): + self = ULMFiTBinary() + dps = {'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15} + self.replace_( + label_smoothing_eps=0.0, + label_smoothing_eps_norm_by_classes=True, + true_wd=True, + wd=0.01, ## important :) + seed=0, + fp16=False, + bs=64, + use_adam_08=False, + early_stopping=None, + clip=0.12, + dropout_values=dps, + name=_use_caller_name() + ) + self.arch.replace_( + tokenizer_type='sp', + max_vocab=15000, + qrnn=True, + n_layers=4, + n_hid=1550 # vs 1552 + ) + self.pretrain_lm.replace_(num_epochs=10, drop_mult=0.0, lr=5e-3, use_adam_08=True, true_wd=False, wd=1e-7, bs=50,) + self.finetune_lm.replace_(num_epochs=20, drop_mult=0.3, lr=1e-3, true_wd=False, wd=1e-7, bs=20) + # TODO check I can't do label_smoothing_eps on binary classifier + self.classifier.replace_(num_epochs=8, drop_mult=0.5, bs=18, label_smoothing_eps=0.0, early_stopping=None) + return self + def ulmfit_orig(): self = multifit_paper_version() self.replace_( @@ -112,4 +142,4 @@ def ulmfit_orig(): def _use_caller_name(): - return inspect.stack()[1].function \ No newline at end of file + return inspect.stack()[1].function diff --git a/multifit/datasets/dataset.py b/multifit/datasets/dataset.py index 34204e0..715295c 100644 --- a/multifit/datasets/dataset.py +++ b/multifit/datasets/dataset.py @@ -202,8 +202,8 @@ class ULMFiTDataset(Dataset): def load_lm_databunch(self, bs, bptt): lm_suffix = str(bptt) if bptt != 70 else "" - lm_suffix += "" if self.use_tst_for_lm else "-notst" - data_lm = self.load_n_cache_databunch(f"lm{lm_suffix}", + lm_suffix += "" if self.use_tst_for_lm else "-not-test" + data_lm = self.load_n_cache_databunch(f"lm{lm_suffix}.cache.databunch", bunch_class=TextLMDataBunch, data_loader=self.load_unsupervised_data, bptt=bptt, @@ -223,27 +223,28 @@ class ULMFiTDataset(Dataset): self._vocab = self.load_lm_databunch(bs=20, bptt=70).vocab return self._vocab - def load_clas_databunch(self, bs): + def load_clas_databunch(self, bs, label_cls=None, **args): vocab = self._load_vocab() - cls_name = "cls" + cls_name = "cls.cache.databunch" if self.limit is not None: cls_name = f'{cls_name}limit{self.limit}' if self.noise > 0.0: cls_name = f'{cls_name}noise{self.noise}' - args = dict(vocab=vocab, bunch_class=TextClasDataBunch, bs=bs) - data_cls = self.load_n_cache_databunch(cls_name, data_loader=lambda: self.load_supervised_data()[:2], **args) + args.update(dict(vocab=vocab, bunch_class=TextClasDataBunch, bs=bs)) + data_cls = self.load_n_cache_databunch(cls_name, data_loader=lambda: self.load_supervised_data()[:2], label_cls=label_cls, **args) # Hack to load test dataset with labels - data_tst = self.load_n_cache_databunch('tst', data_loader=lambda: self.load_supervised_data()[1:], **args) + data_tst = self.load_n_cache_databunch('tst.cache.databunch', data_loader=lambda: self.load_supervised_data()[1:], label_cls=label_cls, **args) data_cls.test_dl = data_tst.valid_dl # data_tst.valid_dl holds test data data_cls.lang = self.lang return data_cls - def load_n_cache_databunch(self, name, bunch_class, data_loader, bs, **args): + def load_n_cache_databunch(self, name, bunch_class, data_loader, bs, label_cls=None, **args): bunch_path = self.cache_path / name databunch = None if bunch_path.exists(): + print(f'loading data bunch from {name}') try: databunch = load_data(self.cache_path, name, bs=bs) except (AttributeError, ImportError): @@ -251,17 +252,18 @@ class ULMFiTDataset(Dataset): if databunch is None: print(f"Running tokenization: '{name}' ...") train_df, valid_df = data_loader() - databunch = self.databunch_from_df(bunch_class, train_df, valid_df, **args) + databunch = self.databunch_from_df(bunch_class, train_df, valid_df, label_cls=label_cls, **args) databunch.save(name) print(f"Data {name}, trn: {len(databunch.train_ds)}, val: {len(databunch.valid_ds)}") return databunch - def databunch_from_df(self, bunch_class, train_df, valid_df, **args): + def databunch_from_df(self, bunch_class, train_df, valid_df, label_cls=None, **args): args.update(**self.tokenizer.get_fastai_config(dataset_uses_moses=self.uses_moses)) # TODO depends on the previous model databunch = make_data_bunch_from_df(cls=bunch_class, path=self.cache_path, train_df=train_df, valid_df=valid_df, + label_cls=label_cls, mark_fields=True, text_cols=list(train_df.columns.values)[1:], **args) diff --git a/multifit/metrics.py b/multifit/metrics.py new file mode 100644 index 0000000..35aad2c --- /dev/null +++ b/multifit/metrics.py @@ -0,0 +1,13 @@ +import torch +from torch import Tensor, LongTensor +from fastai.metrics import auc_roc_score, fbeta + + +def auc_roc_score_multi(input, targ): + """area under curve for multi category list (multiple bce losses).""" + n = input.shape[1] + scores = [auc_roc_score(input[:, i], targ[:, i]) for i in range(n)] + return torch.tensor(scores).mean() + +def fbeta_binary(y_pred, y_true, **args): + return fbeta(y_pred[:, None], y_true[:, None], **args) diff --git a/multifit/training.py b/multifit/training.py index 34349b2..c5e7457 100644 --- a/multifit/training.py +++ b/multifit/training.py @@ -5,7 +5,9 @@ import dataclasses from fastai.callbacks import CSVLogger, SaveModelCallback from fastai.text import * -from multifit.datasets import ULMFiTDataset,ULMFiTTokenizer +from multifit.metrics import auc_roc_score_multi, fbeta_binary, auc_roc_score +from multifit.datasets import ULMFiTDataset, ULMFiTTokenizer +from fastai_contrib.data_block import BinaryCategoryList CLS_BEST = 'cls_best' LM_BEST = "lm_best" @@ -225,6 +227,7 @@ class ULMFiTPretraining(ULMFiTTrainingCommand): config=config, model_dir=self.model_name, **trn_args) + learn.metrics = [accuracy] learn = patch_learner(learn) # compared to standard Adam, we set beta_1 to 0.8 if self.use_adam_08: @@ -378,6 +381,7 @@ class ULMFiTClassifier(ULMFiTTrainingCommand): config=config, model_dir=self.model_name, **trn_args) + learn.metrics =[accuracy, dice] learn = patch_learner(learn) if self.base.encoder_fname and not self.random_init: print("Loading pretrained model", self.base.encoder_fname) @@ -442,8 +446,184 @@ class ULMFiTClassifier(ULMFiTTrainingCommand): learn = self.get_learner(data_cls, eval_only=True) # avg = 'binary' if learn.data.c == 2 else 'macro' - # FBeta(beta=1.0, average=avg), Precision(average=avg), Recall(average=avg), - learn.metrics = [accuracy] + # learn.metrics = [accuracy, FBeta(beta=1.0, average=avg), Precision(average=avg), Recall(average=avg)] + # learn.metrics = [accuracy, fbeta, auc_roc_score,] + learn.metrics = [accuracy, dice] + print(f"Loading model {save_name}") + learn.load(save_name) + if save_preds: + probs, targets = learn.get_preds(ordered=True, ds_type=DatasetType.Test, activ=partial(F.softmax, dim=-1)) + np.save(str(self.experiment_path / f"preds-on-test.npy"), probs.cpu().numpy()) + + results_dict = {} + for split in splits: + results_dict.update(self._validate(learn, split)) + print(results_dict) + with cache_file.open("w") as fp: + json.dump(results_dict, fp) + return results_dict + + def _fit_schedule(self, learn): + getattr(self, '_fit_schedule_' + self.fit_schedule)(learn) + + def _fit_schedule_1cycle(self, learn): + learn.unfreeze() + learn.fit_one_cycle(self.num_epochs, slice(1e-2 / (2.6 ** 4), 2e-2), moms=(0.8, 0.7)) + + def _fit_schedule_layered(self, learn): + learn.freeze_to(-1) + learn.fit_one_cycle(1, 2e-2, moms=(0.8, 0.7)) + if self.num_epochs > 1: + learn.freeze_to(-2) + learn.fit_one_cycle(1, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7)) + learn.freeze_to(-3) + learn.fit_one_cycle(1, slice(5e-3 / (2.6 ** 4), 5e-3), moms=(0.8, 0.7)) + learn.unfreeze() + if self.num_epochs > 5: + learn.fit_one_cycle(self.num_epochs - 4, slice(1e-3 / (2.6 ** 4), 1e-3), moms=(0.8, 0.7)) + + def _fit_schedule_2cycle(self, learn): + learn.freeze_to(-1) + learn.fit_one_cycle(1, 2e-2, moms=(0.8, 0.7)) + learn.unfreeze() + if self.num_epochs > 1: + learn.fit_one_cycle(self.num_epochs - 1, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7)) + + def _fit_schedule_reverse_2cycle(self, learn): + learn.unfreeze() + for g in learn.layer_groups[-1:]: + for l in g: + if not learn.train_bn or not isinstance(l, bn_types): requires_grad(l, False) + learn.create_opt(defaults.lr) + learn.fit_one_cycle(self.num_epochs, slice(1e-2 / (2.6 ** 4), 2e-2), moms=(0.8, 0.7)) + learn.unfreeze() + learn.fit_one_cycle(self.num_epochs, slice(1e-3 / (2.6 ** 4), 2e-3), moms=(0.8, 0.7)) + + def _fit_schedule_false_wd(self, learn): + learn.true_wd = False + learn.fit_one_cycle(1, 5e-2, moms=(0.8, 0.7), wd=1e-7) + if self.num_epochs > 1: + learn.freeze_to(-2) + learn.fit_one_cycle(1, slice(5e-2 / (2.6 ** 4), 5e-2), moms=(0.8, 0.7), wd=1e-7) + learn.freeze_to(-3) + learn.fit_one_cycle(1, slice(5e-4 / (2.6 ** 4), 5e-4), moms=(0.8, 0.7), wd=1e-7) + learn.unfreeze() + if self.num_epochs > 5: + learn.fit_one_cycle(self.num_epochs - 4, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7), wd=1e-7) + + +@dataclass +class ULMFiTBinaryClassifier(ULMFiTTrainingCommand): + bs: int = 20 + num_epochs: int = 10 + drop_mult: float = 0.5 + dropout_values: dict = field(default_factory=dict) + wd: float = 0.01 + clip: float = None + label_smoothing_eps: float = 0.0 + label_smoothing_eps_norm_by_classes: bool = False + weighted_cross_entropy: tuple = None + early_stopping: str = 'accuracy' + fit_schedule: str = '1cycle' + base: ULMFiTFinetuning = field(repr=False, default=None) + random_init: bool = False + seed: int = 0 + bptt: int = 70 + fp16: bool = False + arch: ULMFiTArchitecture = None + + def get_learner(self, data_clas, eval_only=False, **additional_trn_args): + assert self.weighted_cross_entropy is None or self.label_smoothing_eps == 0, "Label smoothing not implemented with weighted_cross_entropy" + if self.weighted_cross_entropy is not None: + loss_func = BCEWithLogitsFlat(weight=torch.tensor(self.weighted_cross_entropy, dtype=torch.float32).cuda()) + elif self.label_smoothing_eps > 0.0: + raise Exception("label_smoothing is not implemented in the binary classifier") + else: + loss_func = None + + set_seed(self.seed, "Classifier weights seed") + config = awd_lstm_clas_config.copy() + config.update(emb_sz=self.arch.emb_sz, n_hid=self.arch.n_hid, n_layers=self.arch.n_layers, qrnn=self.arch.qrnn, + **self.dropout_values) + + trn_args = dict(drop_mult=self.drop_mult, wd=self.wd, pretrained=False, bptt=self.bptt, + loss_func=loss_func, clip=self.clip) + if hasattr(Learner, 'silent'): + trn_args.update(silent=eval_only) + + trn_args.update(**additional_trn_args) + print("Training args: ", trn_args, "config: ", config) + learn = text_classifier_learner(data_clas, + AWD_LSTM, + config=config, + model_dir=self.model_name, + **trn_args) + # learn.metrics =[accuracy, dice] + learn = patch_learner(learn) + if self.base.encoder_fname and not self.random_init: + print("Loading pretrained model", self.base.encoder_fname) + learn.load_encoder(self.base.encoder_fname) + learn.freeze() + else: + warn("No pretrained encoder") + + set_seed(self.seed, "Classifier training seed") + if not eval_only: + learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/cls-history")] + if self.early_stopping: + learn.callback_fns += [partial(SaveModelCallback, every='improvement', + name='cls_best_tmp', + monitor=self.early_stopping)] + if self.fp16: + learn.to_fp16() + return learn + + def train_(self, dataset_or_path=None, label_cls=BinaryCategoryList, metrics=[accuracy, dice, partial(fbeta_binary, beta=1), auc_roc_score_multi], label_cols=[0,0], **train_config): + self.replace_(**train_config, _strict=True) + + base_tokenizer = self.base.tokenizer + dataset = self._set_dataset_(dataset_or_path, base_tokenizer) + data_clas = dataset.load_clas_databunch(bs=self.bs, label_cls=label_cls, label_cols=label_cols) + learn = self.get_learner(data_clas=data_clas) + learn.metrics = metrics + print(f"Training: {learn.path / learn.model_dir}") + learn.unfreeze() + self._fit_schedule(learn) + + self.experiment_path = learn.path / learn.model_dir + base_tokenizer.save(self.experiment_path, learn=learn) + learn.to_fp32() + learn.save(CLS_BEST, with_opt=False) + print("Classifier model saved to", self.experiment_path) + self.save_paramters() + learn.destroy() + return + + def _validate(self, learn, ds_type): + ds_name = ds_type.name.lower() + print(f"Model: {self.name}, ds_name: {ds_name}") + results_dict = dict(zip( + [f'{ds_name} loss'] + [f"{ds_name} {getattr(m, '__name__', m.__class__.__name__)}" for m in learn.metrics], + map(float, learn.validate(learn.data.dl(ds_type))))) + results_dict['name'] = self.name + return results_dict + + def validate(self, *splits, data_cls=None, save_name=CLS_BEST, use_cache=True, save_preds=False): + """Validates + splits - Dataset Types to validate on default DatasetType.Test, DatasetType.Valid, DatasetType.Train + """ + if len(splits) == 0: + splits = [DatasetType.Test, DatasetType.Valid, DatasetType.Train] + cache_file = (self.experiment_path / f'results{"" if save_name == CLS_BEST else "-" + save_name}.json') + if use_cache and cache_file.exists(): + with cache_file.open("r") as fp: + return json.load(fp) + + if data_cls is None: + data_cls = self.dataset.load_clas_databunch(bs=self.bs) + + learn = self.get_learner(data_cls, eval_only=True) + learn.metrics = [accuracy, dice, fbeta, auc_roc_score_multi] print(f"Loading model {save_name}") learn.load(save_name) if save_preds: @@ -579,6 +759,72 @@ class ULMFiT: return self.load_(path) +@dataclass +class ULMFiTBinary: + arch: ULMFiTArchitecture = None + pretrain_lm: ULMFiTPretraining = None + finetune_lm: ULMFiTFinetuning = None + classifier: ULMFiTBinaryClassifier = None + + def __post_init__(self): + self.arch = ULMFiTArchitecture() + self.pretrain_lm = ULMFiTPretraining(arch=self.arch) + self.finetune_lm = ULMFiTFinetuning(arch=self.arch, base=self.pretrain_lm) + self.classifier = ULMFiTBinaryClassifier(arch=self.arch, base=self.finetune_lm) + + def load_(self, experiment_path:Path, silent=False): + success = (self.classifier.load_(experiment_path, silent=silent) or + self.finetune_lm.load_(experiment_path, silent=silent) or + self.pretrain_lm.load_(experiment_path, silent=silent) or + self.load_legacy_(experiment_path, silent=silent)) + if not success: + warn(f'Unable to load experiment {experiment_path}') + return self + + def load_legacy_(self, experiment_path, silent=True): + if not (experiment_path / "info.json").exists(): + return False + with (experiment_path / "info.json").open('r') as f: + d = json.load(f) + dataset_path = d.pop('dataset_path', "") + d['n_hid'] = d['nh'] + d['n_layers'] = d['nl'] + d['lang'] = detect_lang_from_dataset_path(Path(dataset_path)) + if "wiki" in str(dataset_path): + self.arch.replace_(**d) + self.pretrain_lm.replace_(**d) + self.pretrain_lm.experiment_path = path_if_model_exists(experiment_path, LM_BEST) + self.pretrain_lm.dataset_path = dataset_path if dataset_path in str(experiment_path) else None + else: + self.replace_(**d) + self.finetune_lm.experiment_path = path_if_model_exists(experiment_path, ENC_BEST) + self.finetune_lm.dataset_path = dataset_path if dataset_path in str(experiment_path) else None + self.classifier.experiment_path = path_if_model_exists(experiment_path, CLS_BEST) + self.classifier.dataset_path = dataset_path if dataset_path in str(experiment_path) else None + return True + + def replace_(self, **kwargs): + self.arch.replace_(**kwargs) + self.pretrain_lm.replace_(**kwargs) + self.finetune_lm.replace_(**kwargs) + self.classifier.replace_(**kwargs) + return self + + def pprint(self): + print(f"""ULMFiT( + {self.arch}, + {self.pretrain_lm}, + {self.finetune_lm}, + {self.classifier}, +)""") + + def from_pretrained_(self, name, repo="n-waves/multifit-models"): + name = name.rstrip(".tgz") # incase someone put's tgz name the name + url = f"https://github.com/{repo}/releases/download/{name}/{name}.tgz" + path = untar_data(url.rstrip(".tgz"), data=False) # untar_data adds .tgz + return self.load_(path) + + def from_pretrained(name): #TODO: Detect name and load configuration from . import configurations