wassname/multifit
1
0
Fork 0
mirror of https://github.com/wassname/multifit.git synced 2026-09-09 11:27:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

Make ulmfit eval more secure and give more flexibility in dataset_template

Browse Source 
The dataset_template can use lang as additional token to construct globs patterns.
This commit is contained in:
Piotr Czapla
2019-02-14 22:28:25 +01:00
parent 1340f4235c
commit c28c0fde16
1 changed files with 17 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ulmfit/__main__.py
+17 -4
View File
@@ -1,4 +1,5 @@
import gc
import os
import pprint
import shutil
from collections import OrderedDict
@@ -8,6 +9,7 @@ import fire
from .pretrain_lm import LMHyperParams
from .train_clas import CLSHyperParams
from pathlib import Path
from string import Template
class FireView:
def __init__(self, **kwargs):
@@ -22,7 +24,9 @@ def get_lang_from_dataset_path(ds):
def get_dataset_path(p, dataset_template):
ds = [x for x in p.parents if x.name == "models"][0].parent
return ds.parent.glob(dataset_template.format(ds.name))
lang = get_lang_from_dataset_path(ds)
for ds_path in ds.parent.glob(Template(dataset_template).substitute(lang=lang, ds_name=ds.name)):
yield lang, ds_path
class ULMFiT:
@wraps(LMHyperParams)
@@ -37,11 +41,10 @@ class ULMFiT:
params = CLSHyperParams.from_lm(dataset_path, base_lm_path, **changes)
return FireView(train=params.train_cls, validate_cls=params.validate_cls)
def eval(self, glob="mldoc/*-1/models/sp30k/lstm_nl4.m", dataset_template='{}', name="tmp-100", cuda_id=0, **trn_params):
def eval(self, glob="mldoc/*-1/models/sp30k/lstm_nl4.m", dataset_template='${lang}-1', name="tmp-100", cuda_id=0, **trn_params):
results = OrderedDict()
for base_model in sorted(Path("data").glob(glob)):
for dataset_path in sorted(get_dataset_path(base_model, dataset_template)):
lang = get_lang_from_dataset_path(dataset_path)
for lang, dataset_path in sorted(get_dataset_path(base_model, dataset_template)):
params = CLSHyperParams.from_lm(dataset_path, base_model, lang=lang, name=name, cuda_id=cuda_id)
key = str(params.model_dir.relative_to(Path.cwd()))
if params.model_dir.exists():
@@ -54,6 +57,16 @@ class ULMFiT:
gc.collect()
pprint.pprint(results)
def remove_lm_saves(self):
for lm_save in Path("data").glob("**/lm_*.pth"):
num = lm_save.stem.split("_")[-1]
if not num.isdigit():
continue
if int(num) not in [5, 10, 15]:
print("rm ", lm_save)
os.remove(lm_save)
# python -m ulmfit cls --dataset-path data/mldoc/de-1-laser --base-lm-path data/mldoc/de-1/models/sp30k/lstm_nl4.m --lang=de --name 'nl4' --cuda-id=1 - train 0 --bs 40 --num-cls-epochs=2
if __name__ == '__main__':