mirror of
https://github.com/wassname/multifit.git
synced 2026-10-09 12:20:23 +08:00
323 lines
14 KiB
Python
323 lines
14 KiB
Python
import gc
|
|
import os
|
|
import tarfile
|
|
from collections import defaultdict
|
|
import re
|
|
from functools import wraps
|
|
import numpy as np
|
|
import pandas as pd
|
|
import fire
|
|
|
|
from .pretrain_lm import LMHyperParams, folder_name_to_model_name, DataSetParams
|
|
from .train_clas import CLSHyperParams
|
|
from pathlib import Path
|
|
from string import Template
|
|
import torch
|
|
|
|
class FireView:
|
|
def __init__(self, **kwargs):
|
|
for k,v in kwargs.items():
|
|
setattr(self, k, v)
|
|
|
|
def get_lang_from_dataset_path(ds):
|
|
lang,*_ = ds.name.split("-")
|
|
if len(lang) == 2:
|
|
return lang
|
|
return "en"
|
|
|
|
def get_dataset_path(p, dataset_template):
|
|
ds = [x for x in p.parents if x.name == "models"][0].parent
|
|
lang = get_lang_from_dataset_path(ds)
|
|
pattern = Template(dataset_template).substitute(lang=lang, ds_name=ds.name)
|
|
print("Selected cls dataset", (ds.parent.relative_to(Path.cwd())/pattern).resolve())
|
|
for ds_path in ds.parent.glob(pattern):
|
|
yield lang, ds_path
|
|
|
|
class ULMFiT:
|
|
@wraps(LMHyperParams)
|
|
def lm(self, dataset_path, **changes):
|
|
changes['dataset_path'] = dataset_path
|
|
params = LMHyperParams(**changes)
|
|
return FireView(train=params.train_lm)
|
|
|
|
lm2 = LMHyperParams
|
|
@wraps(CLSHyperParams)
|
|
def cls(self, dataset_path, base_lm_path=None, **changes):
|
|
print(dataset_path, repr(base_lm_path))
|
|
if base_lm_path is not None:
|
|
params = CLSHyperParams.from_lm(dataset_path, base_lm_path, **changes)
|
|
else:
|
|
params = CLSHyperParams(dataset_path=dataset_path, **changes)
|
|
return FireView(train=params.train_cls, evaluate_cls=params.evaluate_cls)
|
|
|
|
|
|
@wraps(CLSHyperParams)
|
|
def load_cls(self, model_path, **changes):
|
|
params = CLSHyperParams.from_json(model_path, **changes)
|
|
return FireView(train=params.train_cls, evaluate_cls=params.evaluate_cls)
|
|
|
|
|
|
def eval_noise_resistance(self, lang="de", size=1, prefix_name="", model="sp15k/qrnn_nl4.m",
|
|
num_cls_epochs=8, bs=18, lr_sched="1cycle", label_smoothing_eps=0.0, **kwargs):
|
|
results= []
|
|
for noise in range(0, 80, 5):
|
|
print("Noise: ", noise)
|
|
d = self.eval(glob=f"mldoc/{lang}-1/models/{model}",
|
|
name=f"nl4_{prefix_name}{noise}",
|
|
noise=noise/100,
|
|
dataset_template='${lang}-'+str(size),
|
|
num_cls_epochs=num_cls_epochs,
|
|
bs=bs,
|
|
lr_sched=lr_sched,
|
|
label_smoothing_eps=label_smoothing_eps,
|
|
**kwargs)
|
|
|
|
def tar(self, model_path):
|
|
data_dir = (Path.cwd()/"data").resolve()
|
|
params = CLSHyperParams.from_json(model_path)
|
|
name = str(params.dataset_dir.resolve().relative_to(data_dir)).replace("/", "-")
|
|
|
|
tar_name = f"models/{name}-{params.tokenizer_prefix}-{params.model_name}.tar"
|
|
print("Storing model in", tar_name)
|
|
with tarfile.open(tar_name, mode="w") as tar:
|
|
for g in map(params.model_dir.glob, ['*_best.pth', 'info.json', '../spm.*', '../itos.*',]):
|
|
for f in g:
|
|
dest = f.resolve().relative_to(data_dir.parent)
|
|
print("Adding", f, dest)
|
|
tar.add(f, dest)
|
|
|
|
|
|
def poleval19_full(self, base, num_lm_epochs=6, lmtype=None, skip_train_seed=False, **kwargs):
|
|
clsbase = self.poleval19_init(base, num_lm_epochs=num_lm_epochs, lmtype=lmtype, **kwargs)
|
|
self.poleval19_seeds(clsbase, seed_name='clsweightseed', **kwargs)
|
|
if skip_train_seed:
|
|
self.poleval19_seeds(clsbase, seed_name='clstrainseed', **kwargs)
|
|
|
|
def poleval19_init(self, base, name=None, lmseed=None, lmtype=None, **kwargs):
|
|
clstrainseed = kwargs.pop('clstrainseed', 0)
|
|
clsweightseed = kwargs.pop('clsweightseed', 0)
|
|
ftseed = kwargs.pop('ftseed', 0)
|
|
|
|
if lmtype is None:
|
|
if "wiki" in base:
|
|
lmtype = "wiki"
|
|
elif "reddit" in base:
|
|
lmtype = "reddit"
|
|
else:
|
|
raise AttributeError("unkown lm ty")
|
|
|
|
if "seed0" in base:
|
|
lmseed = 0
|
|
print("Setting lmseed ", lmseed)
|
|
elif "seed1" in base:
|
|
lmseed = 1
|
|
print("Setting lmseed ", lmseed)
|
|
|
|
dataset_template=f"../hate/pl-10-{lmtype}"
|
|
if name is None:
|
|
name = f"ft{kwargs.get('num_lm_epochs',6)}_cl{kwargs.get('num_cls_epochs',6)}"
|
|
print("Setting name to ", name)
|
|
|
|
return self.poleval19_eval(glob=base,
|
|
name=name,
|
|
dataset_template=dataset_template,
|
|
lmseed=lmseed,
|
|
ftseed=ftseed,
|
|
clstrainseed=clstrainseed,
|
|
clsweightseed=clsweightseed,
|
|
**kwargs)
|
|
|
|
|
|
def poleval19_seeds(self, base, name=None, seed_name='clsweightseed', model_num=10, **kwargs):
|
|
if name is None:
|
|
name = folder_name_to_model_name(Path(base).name)
|
|
for seed in range(0, model_num, 1):
|
|
kwargs[seed_name] = seed
|
|
print("Seed: ", seed_name, seed)
|
|
self.poleval19_eval(glob=base, name=name, num_lm_epochs=0, **kwargs)
|
|
|
|
def poleval19_eval(self, glob, name=None, num_lm_epochs=6, num_cls_epochs=8, bs=160, lr_sched="1cycle", **kwargs):
|
|
return self.eval(glob=glob,
|
|
name=name,
|
|
num_lm_epochs=num_lm_epochs,
|
|
num_cls_epochs=num_cls_epochs,
|
|
bs=bs,
|
|
lr_sched=lr_sched,
|
|
**kwargs)
|
|
|
|
|
|
def multifit_seeds(self, base, name=None, seed_name='clsweightseed', model_num=10, **kwargs):
|
|
if name is None:
|
|
name = folder_name_to_model_name(Path(base).name)
|
|
for seed in range(0, model_num, 1):
|
|
kwargs[seed_name] = seed
|
|
print("Seed: ", seed_name, seed)
|
|
self.multifit_eval(glob=base, name=name, num_lm_epochs=0, **kwargs)
|
|
|
|
|
|
def multifit_eval(self, glob, name=None, num_lm_epochs=20, num_cls_epochs=8, bs=20, lr_sched="1cycle", label_smoothing_eps=0.1, **kwargs):
|
|
return self.eval(
|
|
glob=glob,
|
|
name=name,
|
|
num_lm_epochs=num_lm_epochs,
|
|
num_cls_epochs=num_cls_epochs,
|
|
bs=bs,
|
|
lr_sched=lr_sched,
|
|
label_smoothing_eps=label_smoothing_eps,
|
|
**kwargs)
|
|
|
|
def ls(self, glob, dataset_template='${ds_name}'):
|
|
data_dir = Path("data").absolute()
|
|
glob = str(glob)
|
|
if "data" not in glob and not glob.startswith("/"):
|
|
glob = "data/" + glob
|
|
results = []
|
|
for base_model in sorted(data_dir.parent.glob(glob)):
|
|
datasets = list(sorted(get_dataset_path(base_model, dataset_template)))
|
|
print(f"Base model: {base_model} length: {len(datasets)}")
|
|
if len(datasets) == 0:
|
|
print(f"Debug: {dataset_template}")
|
|
for lang, dataset_path in datasets:
|
|
results.append((base_model, lang, dataset_path))
|
|
return results
|
|
|
|
|
|
# file_glob = "${ds_name}/${lang}.train.csv"
|
|
def ensemble(self, glob="data/mldoc*/*-1/models/sp15k/qrnn_*.m",
|
|
file_template="${model_dir}/preds-on-test.npy",
|
|
gold_labels_template="${dataset_path}/${lang}.test.csv",
|
|
out_template="${key}.ensemble.csv",
|
|
key_template='${lang}', verbose=False, exclude_re=None):
|
|
|
|
def load_labels(file, verbose=False):
|
|
if file.suffix == ".npy":
|
|
labels = np.load(str(file))
|
|
|
|
elif file.suffix == ".csv":
|
|
df = pd.read_csv(file, header=None)
|
|
labels = np.array([df[c] for c in df.columns if np.issubdtype(df[c].dtype, np.number)]).T.squeeze()
|
|
else:
|
|
raise AttributeError("Unknown result file type", file.extension)
|
|
if verbose: print(file, labels.shape)
|
|
return labels
|
|
|
|
files_for_ensemble = defaultdict(list)
|
|
gold_labels = {}
|
|
for folder in Path.cwd().glob(glob):
|
|
if exclude_re is not None and re.match(exclude_re, str(folder)):
|
|
print("Skipping", folder)
|
|
continue
|
|
if folder.suffix == ".m":
|
|
params = CLSHyperParams.from_json(folder)
|
|
else:
|
|
params = DataSetParams(folder)
|
|
|
|
key = params.resolve_template(key_template)
|
|
file_glob = params.resolve_template(file_template)
|
|
files_for_ensemble[key].append(Path(file_glob))
|
|
gold_label_glob = params.resolve_template(gold_labels_template)
|
|
gold_file = Path(gold_label_glob)
|
|
gold_labels[key] = gold_file
|
|
|
|
def fbeta(y_pred, y_true, thresh: float = 0.2, beta: float = 2, eps: float = 1e-9,
|
|
sigmoid: bool = True, dim=1):
|
|
"Computes the f_beta between `preds` and `targets`"
|
|
beta2 = beta ** 2
|
|
if sigmoid: y_pred = y_pred.sigmoid()
|
|
y_pred = (y_pred > thresh).float()
|
|
y_true = y_true.float()
|
|
TP = (y_pred * y_true).sum(dim=dim)
|
|
prec = TP / (y_pred.sum(dim=dim) + eps)
|
|
rec = TP / (y_true.sum(dim=dim) + eps)
|
|
res = (prec * rec) / (prec * beta2 + rec + eps) * (1 + beta2)
|
|
return res.mean()
|
|
|
|
for key, files in files_for_ensemble.items():
|
|
ensemble = np.array([load_labels(file, verbose) for file in files if file.exists()]).mean(axis=0)
|
|
if len(ensemble.shape) != 1:
|
|
ensemble = np.argmax(ensemble, axis=1)
|
|
test = pd.read_csv(gold_labels[key], header=None)
|
|
|
|
f1beta = fbeta(torch.tensor(ensemble), torch.tensor(test[0]), sigmoid=False, beta=1, dim=0)
|
|
print({"Key": key, "Test Accuracy": (test[0] == ensemble).mean(), "Test F1":f1beta, "on": gold_labels[key], 'files_count':len(files)})
|
|
test[0] = ensemble
|
|
if out_template:
|
|
out_file = Path(Template(out_template).substitute(key=key))
|
|
out_file.parent.mkdir(exist_ok=True, parents=True)
|
|
test.to_csv(out_file, header=None, index=False)
|
|
print({"File saved to": out_file})
|
|
|
|
def eval(self, glob="data/mldoc/*-1/models/sp30k/lstm_nl4.m", dataset_template='${ds_name}', name=None,
|
|
num_lm_epochs=0, train=True, to_csv=None, return_df=False, label_smoothing_eps=0.0,
|
|
lmseed=None, ftseed=None, clsweightseed=None, clstrainseed=None, save_name="cls_best",
|
|
skip_on_error=True, **trn_params):
|
|
results = []
|
|
model_args = {}
|
|
last_model_dir = None
|
|
if clsweightseed is not None:
|
|
model_args["clsweightseed"] = clsweightseed
|
|
if clstrainseed is not None:
|
|
model_args['clstrainseed'] = clstrainseed
|
|
if ftseed is not None:
|
|
model_args['ftseed'] = ftseed
|
|
if lmseed is not None:
|
|
model_args['lmseed'] = lmseed
|
|
data_dir = Path("data").absolute()
|
|
glob=str(glob)
|
|
if "data" not in glob and not glob.startswith("/"):
|
|
glob = "data/"+glob
|
|
for base_model, lang, dataset_path in self.ls(glob, dataset_template):
|
|
try:
|
|
_name = name
|
|
if name is None:
|
|
_name = folder_name_to_model_name(base_model.name)
|
|
params = CLSHyperParams.from_lm(dataset_path, base_model, lang=lang, name=_name, **model_args)
|
|
last_model_dir = params.model_path.relative_to(data_dir.parent)
|
|
if (params.model_path / "cls_best.pth").exists():
|
|
print("Evaluating previously trained model")
|
|
d_tst = params.evaluate_cls(save_name=save_name, label_smoothing_eps=label_smoothing_eps, use_cache=True, mode="test")
|
|
d_val = params.evaluate_cls(save_name=save_name, label_smoothing_eps=label_smoothing_eps, use_cache=True, mode="valid")
|
|
d={}
|
|
d.update(d_val)
|
|
d.update(d_tst)
|
|
elif train:
|
|
print("Training")
|
|
d = params.train_cls(num_lm_epochs=num_lm_epochs, label_smoothing_eps=label_smoothing_eps, **trn_params)
|
|
else:
|
|
print("Skipping", (params.model_path / "cls_best.pth"))
|
|
d = None
|
|
if d is not None:
|
|
d['model_dir_parent'] = params.model_path.relative_to(data_dir.parent).parent
|
|
d['model_name'] = params.model_name
|
|
np.save(params.model_path / "results.npy", d)
|
|
results.append(d)
|
|
del params
|
|
except Exception as e:
|
|
print("Error", e)
|
|
if not skip_on_error:
|
|
raise e
|
|
gc.collect()
|
|
df = pd.DataFrame.from_records(results)
|
|
print(df)
|
|
if to_csv is not None:
|
|
print(f"Saving result to: {to_csv}")
|
|
df.to_csv(to_csv)
|
|
if return_df:
|
|
return last_model_dir, df
|
|
return str(last_model_dir)
|
|
|
|
def remove_lm_saves(self):
|
|
for lm_save in Path("data").glob("**/lm_*.pth"):
|
|
num = lm_save.stem.split("_")[-1]
|
|
if not num.isdigit():
|
|
continue
|
|
if int(num) not in [5, 10, 15]:
|
|
print("rm ", lm_save)
|
|
os.remove(lm_save)
|
|
|
|
|
|
# python -m ulmfit cls --dataset-path data/mldoc/de-1-laser --base-lm-path data/mldoc/de-1/models/sp30k/lstm_nl4.m --lang=de --name 'nl4' --cuda-id=1 - train 0 --bs 40 --num-cls-epochs=2
|
|
|
|
if __name__ == '__main__':
|
|
fire.Fire(ULMFiT())
|