mirror of
https://github.com/wassname/multifit.git
synced 2026-09-09 11:27:26 +08:00
remove binary databunch
This commit is contained in:
wassname
@@ -1,35 +0,0 @@
|
||||
from typing import Iterator, Collection
|
||||
from fastai.data_block import CategoryListBase
|
||||
from fastai.text import *
|
||||
|
||||
|
||||
class BinaryProcessor(CategoryProcessor):
|
||||
def create_classes(self, classes):
|
||||
self.classes = classes
|
||||
if classes is not None: self.c2i = {0:0, 1:1}
|
||||
def generate_classes(self, items):
|
||||
return [0]
|
||||
|
||||
class BinaryCategoryList(CategoryListBase):
|
||||
"Basic `ItemList` for single classification labels."
|
||||
_processor=BinaryProcessor
|
||||
def __init__(self, items:Iterator, classes:Collection=None, label_delim:str=None, **kwargs):
|
||||
super().__init__(items, classes=classes, **kwargs)
|
||||
mean = self.items.mean()
|
||||
# if mean and mean != 0:
|
||||
# weight = torch.tensor([1 / mean]).cuda()
|
||||
# print(f'Weighting BCEWithLogitsFlat by {weight.item()}')
|
||||
# else:
|
||||
weight = None
|
||||
# raise Exception('debug')
|
||||
self.loss_func = BCEWithLogitsFlat(weight=weight)
|
||||
|
||||
def reconstruct(self, t):
|
||||
return Category(t, self.c2i[t.item()])
|
||||
|
||||
def get(self, i):
|
||||
o = self.items[i]
|
||||
if o is None: return None
|
||||
return Category(o, self.c2i[o])
|
||||
|
||||
def analyze_pred(self, pred, thresh:float=0.5): return pred.argmax()
|
||||
+1
-30
@@ -2,32 +2,16 @@ import torch
|
||||
from torch import Tensor, LongTensor
|
||||
from fastai.metrics import auc_roc_score, fbeta
|
||||
|
||||
|
||||
def auc_roc_score_multi(input, targ):
|
||||
"""area under curve for multi category list (multiple bce losses)."""
|
||||
n = input.shape[1]
|
||||
targ = targ * 1
|
||||
if targ.shape != input.shape:
|
||||
targ = targ.expand(input.T.shape).T
|
||||
scores = [auc_roc_score(input[:, i], targ[:, i]) for i in range(n)]
|
||||
return torch.tensor(scores).mean()
|
||||
|
||||
|
||||
def fbeta_cls_n(y_pred, y_true, class_n=1, **args):
|
||||
"""F1 score of class 1, to be used with 2 classes."""
|
||||
y_pred = torch.nn.functional.softmax(y_pred, dim=-1)
|
||||
return fbeta(y_pred, y_true[:, None], sigmoid=False, **args)
|
||||
|
||||
def auc_roc_score_cls_n(y_pred, y_true, class_n=1, **args):
|
||||
"""F1 score of class 1, to be used with 2 classes."""
|
||||
"""auc_roc_score score of class 1, to be used with 2 classes."""
|
||||
y_pred = torch.nn.functional.softmax(y_pred, dim=-1)
|
||||
return auc_roc_score(y_pred[:, class_n], y_true==class_n, **args)
|
||||
|
||||
|
||||
def fbeta_binary(y_pred, y_true, **args):
|
||||
return fbeta(y_pred[:, None], y_true[:, None], **args)
|
||||
|
||||
|
||||
def auc_roc_score(input: Tensor, targ: Tensor):
|
||||
"Computes the area under the receiver operator characteristic (ROC) curve using the trapezoid method. Restricted binary classification tasks."
|
||||
fpr, tpr = roc_curve(input.squeeze(), targ.squeeze())
|
||||
@@ -62,16 +46,3 @@ def roc_curve(input: Tensor, targ: Tensor):
|
||||
def accuracy_binary(input, targs):
|
||||
input = torch.sigmoid(input) > 0.5
|
||||
return (input == targs).float().mean()
|
||||
|
||||
|
||||
def dice_binary(input, targs, iou=False, eps=1e-8):
|
||||
"Dice coefficient metric for binary target. If iou=True, returns iou metric, classic for segmentation problems."
|
||||
input = torch.sigmoid(input) > 0.5
|
||||
intersect = (input * targs).sum(dim=1).float()
|
||||
union = (input + targs).sum(dim=1).float()
|
||||
if not iou:
|
||||
l = 2.0 * intersect / union
|
||||
else:
|
||||
l = intersect / (union - intersect + eps)
|
||||
l[union == 0.0] = 1.0
|
||||
return l.mean()
|
||||
|
||||
@@ -516,187 +516,6 @@ class ULMFiTClassifier(ULMFiTTrainingCommand):
|
||||
learn.fit_one_cycle(self.num_epochs - 4, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7), wd=1e-7)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ULMFiTBinaryClassifier(ULMFiTTrainingCommand):
|
||||
bs: int = 20
|
||||
num_epochs: int = 10
|
||||
drop_mult: float = 0.5
|
||||
dropout_values: dict = field(default_factory=dict)
|
||||
wd: float = 0.01
|
||||
clip: float = None
|
||||
label_smoothing_eps: float = 0.0
|
||||
label_smoothing_eps_norm_by_classes: bool = False
|
||||
weighted_cross_entropy: tuple = None
|
||||
early_stopping: str = 'accuracy'
|
||||
fit_schedule: str = '1cycle'
|
||||
base: ULMFiTFinetuning = field(repr=False, default=None)
|
||||
random_init: bool = False
|
||||
seed: int = 0
|
||||
bptt: int = 70
|
||||
fp16: bool = False
|
||||
arch: ULMFiTArchitecture = None
|
||||
|
||||
def get_learner(self, data_clas, eval_only=False, **additional_trn_args):
|
||||
assert self.weighted_cross_entropy is None or self.label_smoothing_eps == 0, "Label smoothing not implemented with weighted_cross_entropy"
|
||||
if self.weighted_cross_entropy is not None:
|
||||
loss_func = BCEWithLogitsFlat(weight=torch.tensor(self.weighted_cross_entropy, dtype=torch.float32).cuda())
|
||||
elif self.label_smoothing_eps > 0.0:
|
||||
raise Exception("label_smoothing is not implemented in the binary classifier")
|
||||
else:
|
||||
loss_func = None
|
||||
|
||||
set_seed(self.seed, "Classifier weights seed")
|
||||
config = awd_lstm_clas_config.copy()
|
||||
config.update(emb_sz=self.arch.emb_sz, n_hid=self.arch.n_hid, n_layers=self.arch.n_layers, qrnn=self.arch.qrnn,
|
||||
**self.dropout_values)
|
||||
|
||||
trn_args = dict(drop_mult=self.drop_mult, wd=self.wd, pretrained=False, bptt=self.bptt,
|
||||
loss_func=loss_func, clip=self.clip)
|
||||
if hasattr(Learner, 'silent'):
|
||||
trn_args.update(silent=eval_only)
|
||||
|
||||
trn_args.update(**additional_trn_args)
|
||||
print("Training args: ", trn_args, "config: ", config)
|
||||
learn = text_classifier_learner(data_clas,
|
||||
AWD_LSTM,
|
||||
config=config,
|
||||
model_dir=self.model_name,
|
||||
**trn_args)
|
||||
# learn.metrics =[accuracy, dice]
|
||||
learn = patch_learner(learn)
|
||||
if self.base.encoder_fname and not self.random_init:
|
||||
print("Loading pretrained model", self.base.encoder_fname)
|
||||
learn.load_encoder(self.base.encoder_fname)
|
||||
learn.freeze()
|
||||
else:
|
||||
warn("No pretrained encoder")
|
||||
|
||||
set_seed(self.seed, "Classifier training seed")
|
||||
if not eval_only:
|
||||
learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/cls-history")]
|
||||
if self.early_stopping:
|
||||
learn.callback_fns += [partial(SaveModelCallback, every='improvement',
|
||||
name='cls_best_tmp',
|
||||
monitor=self.early_stopping)]
|
||||
if self.fp16:
|
||||
learn.to_fp16()
|
||||
return learn
|
||||
|
||||
def train_(self, dataset_or_path=None, label_cls=BinaryCategoryList, metrics=[accuracy_binary, dice_binary, partial(fbeta_binary, beta=1), auc_roc_score_multi], label_cols=[0,0], **train_config):
|
||||
self.replace_(**train_config, _strict=True)
|
||||
|
||||
base_tokenizer = self.base.tokenizer
|
||||
dataset = self._set_dataset_(dataset_or_path, base_tokenizer)
|
||||
data_clas = dataset.load_clas_databunch(bs=self.bs, label_cls=label_cls, label_cols=label_cols)
|
||||
learn = self.get_learner(data_clas=data_clas)
|
||||
learn.metrics = metrics
|
||||
print(f"Training: {learn.path / learn.model_dir}")
|
||||
learn.unfreeze()
|
||||
self._fit_schedule(learn)
|
||||
|
||||
self.experiment_path = learn.path / learn.model_dir
|
||||
base_tokenizer.save(self.experiment_path, learn=learn)
|
||||
learn.to_fp32()
|
||||
learn.save(CLS_BEST, with_opt=False)
|
||||
print("Classifier model saved to", self.experiment_path)
|
||||
self.save_paramters()
|
||||
learn.destroy()
|
||||
return
|
||||
|
||||
def _validate(self, learn, ds_type):
|
||||
ds_name = ds_type.name.lower()
|
||||
print(f"Model: {self.name}, ds_name: {ds_name}")
|
||||
results_dict = dict(zip(
|
||||
[f'{ds_name} loss'] + [f"{ds_name} {getattr(m, '__name__', m.__class__.__name__)}" for m in learn.metrics],
|
||||
map(float, learn.validate(learn.data.dl(ds_type)))))
|
||||
results_dict['name'] = self.name
|
||||
return results_dict
|
||||
|
||||
def validate(self, *splits, data_cls=None, save_name=CLS_BEST, use_cache=True, save_preds=False):
|
||||
"""Validates
|
||||
splits - Dataset Types to validate on default DatasetType.Test, DatasetType.Valid, DatasetType.Train
|
||||
"""
|
||||
if len(splits) == 0:
|
||||
splits = [DatasetType.Test, DatasetType.Valid, DatasetType.Train]
|
||||
cache_file = (self.experiment_path / f'results{"" if save_name == CLS_BEST else "-" + save_name}.json')
|
||||
if use_cache and cache_file.exists():
|
||||
with cache_file.open("r") as fp:
|
||||
return json.load(fp)
|
||||
|
||||
if data_cls is None:
|
||||
data_cls = self.dataset.load_clas_databunch(bs=self.bs)
|
||||
|
||||
learn = self.get_learner(data_cls, eval_only=True)
|
||||
learn.metrics = [accuracy, dice, fbeta, auc_roc_score_multi]
|
||||
print(f"Loading model {save_name}")
|
||||
learn.load(save_name)
|
||||
if save_preds:
|
||||
probs, targets = learn.get_preds(ordered=True, ds_type=DatasetType.Test, activ=partial(F.softmax, dim=-1))
|
||||
np.save(str(self.experiment_path / f"preds-on-test.npy"), probs.cpu().numpy())
|
||||
|
||||
results_dict = {}
|
||||
for split in splits:
|
||||
results_dict.update(self._validate(learn, split))
|
||||
print(results_dict)
|
||||
with cache_file.open("w") as fp:
|
||||
json.dump(results_dict, fp)
|
||||
return results_dict
|
||||
|
||||
def _fit_schedule(self, learn):
|
||||
getattr(self, '_fit_schedule_' + self.fit_schedule)(learn)
|
||||
|
||||
def _fit_schedule_1cycle(self, learn):
|
||||
learn.unfreeze()
|
||||
learn.fit_one_cycle(self.num_epochs, slice(1e-2 / (2.6 ** 4), 2e-2), moms=(0.8, 0.7))
|
||||
|
||||
def _fit_schedule_layered(self, learn):
|
||||
learn.freeze_to(-1)
|
||||
learn.fit_one_cycle(1, 2e-2, moms=(0.8, 0.7))
|
||||
if self.num_epochs > 1:
|
||||
learn.freeze_to(-2)
|
||||
learn.fit_one_cycle(1, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7))
|
||||
learn.freeze_to(-3)
|
||||
learn.fit_one_cycle(1, slice(5e-3 / (2.6 ** 4), 5e-3), moms=(0.8, 0.7))
|
||||
learn.unfreeze()
|
||||
if self.num_epochs > 5:
|
||||
learn.fit_one_cycle(self.num_epochs - 4, slice(1e-3 / (2.6 ** 4), 1e-3), moms=(0.8, 0.7))
|
||||
|
||||
def _fit_schedule_2cycle(self, learn):
|
||||
learn.freeze_to(-1)
|
||||
learn.fit_one_cycle(1, 2e-2, moms=(0.8, 0.7))
|
||||
learn.unfreeze()
|
||||
if self.num_epochs > 1:
|
||||
learn.fit_one_cycle(self.num_epochs - 1, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7))
|
||||
|
||||
def _fit_schedule_reverse_2cycle(self, learn):
|
||||
learn.unfreeze()
|
||||
for g in learn.layer_groups[-1:]:
|
||||
for l in g:
|
||||
if not learn.train_bn or not isinstance(l, bn_types): requires_grad(l, False)
|
||||
learn.create_opt(defaults.lr)
|
||||
learn.fit_one_cycle(self.num_epochs, slice(1e-2 / (2.6 ** 4), 2e-2), moms=(0.8, 0.7))
|
||||
learn.unfreeze()
|
||||
learn.fit_one_cycle(self.num_epochs, slice(1e-3 / (2.6 ** 4), 2e-3), moms=(0.8, 0.7))
|
||||
|
||||
def _fit_schedule_false_wd(self, learn):
|
||||
learn.true_wd = False
|
||||
learn.fit_one_cycle(1, 5e-2, moms=(0.8, 0.7), wd=1e-7)
|
||||
if self.num_epochs > 1:
|
||||
learn.freeze_to(-2)
|
||||
learn.fit_one_cycle(1, slice(5e-2 / (2.6 ** 4), 5e-2), moms=(0.8, 0.7), wd=1e-7)
|
||||
learn.freeze_to(-3)
|
||||
learn.fit_one_cycle(1, slice(5e-4 / (2.6 ** 4), 5e-4), moms=(0.8, 0.7), wd=1e-7)
|
||||
learn.unfreeze()
|
||||
if self.num_epochs > 5:
|
||||
learn.fit_one_cycle(self.num_epochs - 4, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7), wd=1e-7)
|
||||
|
||||
|
||||
def path_if_model_exists(path, weights_name):
|
||||
"""Return path to model if it exists"""
|
||||
model_path = path / (weights_name + ".pth")
|
||||
return path if model_path.exists() else None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ULMFiT:
|
||||
arch: ULMFiTArchitecture = None
|
||||
@@ -763,72 +582,6 @@ class ULMFiT:
|
||||
return self.load_(path)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ULMFiTBinary:
|
||||
arch: ULMFiTArchitecture = None
|
||||
pretrain_lm: ULMFiTPretraining = None
|
||||
finetune_lm: ULMFiTFinetuning = None
|
||||
classifier: ULMFiTBinaryClassifier = None
|
||||
|
||||
def __post_init__(self):
|
||||
self.arch = ULMFiTArchitecture()
|
||||
self.pretrain_lm = ULMFiTPretraining(arch=self.arch)
|
||||
self.finetune_lm = ULMFiTFinetuning(arch=self.arch, base=self.pretrain_lm)
|
||||
self.classifier = ULMFiTBinaryClassifier(arch=self.arch, base=self.finetune_lm)
|
||||
|
||||
def load_(self, experiment_path:Path, silent=False):
|
||||
success = (self.classifier.load_(experiment_path, silent=silent) or
|
||||
self.finetune_lm.load_(experiment_path, silent=silent) or
|
||||
self.pretrain_lm.load_(experiment_path, silent=silent) or
|
||||
self.load_legacy_(experiment_path, silent=silent))
|
||||
if not success:
|
||||
warn(f'Unable to load experiment {experiment_path}')
|
||||
return self
|
||||
|
||||
def load_legacy_(self, experiment_path, silent=True):
|
||||
if not (experiment_path / "info.json").exists():
|
||||
return False
|
||||
with (experiment_path / "info.json").open('r') as f:
|
||||
d = json.load(f)
|
||||
dataset_path = d.pop('dataset_path', "")
|
||||
d['n_hid'] = d['nh']
|
||||
d['n_layers'] = d['nl']
|
||||
d['lang'] = detect_lang_from_dataset_path(Path(dataset_path))
|
||||
if "wiki" in str(dataset_path):
|
||||
self.arch.replace_(**d)
|
||||
self.pretrain_lm.replace_(**d)
|
||||
self.pretrain_lm.experiment_path = path_if_model_exists(experiment_path, LM_BEST)
|
||||
self.pretrain_lm.dataset_path = dataset_path if dataset_path in str(experiment_path) else None
|
||||
else:
|
||||
self.replace_(**d)
|
||||
self.finetune_lm.experiment_path = path_if_model_exists(experiment_path, ENC_BEST)
|
||||
self.finetune_lm.dataset_path = dataset_path if dataset_path in str(experiment_path) else None
|
||||
self.classifier.experiment_path = path_if_model_exists(experiment_path, CLS_BEST)
|
||||
self.classifier.dataset_path = dataset_path if dataset_path in str(experiment_path) else None
|
||||
return True
|
||||
|
||||
def replace_(self, **kwargs):
|
||||
self.arch.replace_(**kwargs)
|
||||
self.pretrain_lm.replace_(**kwargs)
|
||||
self.finetune_lm.replace_(**kwargs)
|
||||
self.classifier.replace_(**kwargs)
|
||||
return self
|
||||
|
||||
def pprint(self):
|
||||
print(f"""ULMFiT(
|
||||
{self.arch},
|
||||
{self.pretrain_lm},
|
||||
{self.finetune_lm},
|
||||
{self.classifier},
|
||||
)""")
|
||||
|
||||
def from_pretrained_(self, name, repo="n-waves/multifit-models"):
|
||||
name = name.rstrip(".tgz") # incase someone put's tgz name the name
|
||||
url = f"https://github.com/{repo}/releases/download/{name}/{name}.tgz"
|
||||
path = untar_data(url.rstrip(".tgz"), data=False) # untar_data adds .tgz
|
||||
return self.load_(path)
|
||||
|
||||
|
||||
def from_pretrained(name):
|
||||
#TODO: Detect name and load configuration
|
||||
from . import configurations
|
||||
|
||||
Reference in New Issue
Block a user