wassname/multifit
1
0
Fork 0
mirror of https://github.com/wassname/multifit.git synced 2026-09-09 11:27:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

finall clean ups

Browse Source
This commit is contained in:
Piotr Czapla
2019-10-07 14:50:03 +02:00
parent 3fe6c19af2
commit 1fe2bd46a6
2 changed files with 19 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ulmfit/datasets/__init__.py
+5 -2
View File
@@ -203,8 +203,9 @@ class ULMFiTDataset(Dataset):
and we expect finetuning to handle the conversion
"""
# reuse base model sentencepiece vocabulary
self.cache_path.mkdir(exist_ok=True)
if (base_lm_path / '..' / 'spm.vocab').exists() and (base_lm_path.parent.resolve() != self.cache_path.resolve()):
self.cache_path.mkdir(exist_ok=True, parents=True)
if base_lm_path and (base_lm_path / '..' / 'spm.vocab').exists() and \
(base_lm_path.parent.resolve() != self.cache_path.resolve()):
shutil.copy(str(base_lm_path / '..' / 'itos.pkl'), str(self.cache_path))
shutil.copy(str(base_lm_path / '..' / 'spm.model'), str(self.cache_path))
shutil.copy(str(base_lm_path / '..' / 'spm.vocab'), str(self.cache_path))
@@ -309,4 +310,6 @@ class ULMFiTDataset(Dataset):
post_rules=defaults.text_post_rules))
def _get_processor_pure_fastai(self, ds_need_moses):
if ds_need_moses:
warn("Fast ai dont use moses, make sure you trained from wikpiedia that wasm't tokenized with moses.")
return dict()
ulmfit/training.py
+14 -13
View File
@@ -113,7 +113,7 @@ class ULMFiTTrainingCommand(Params):
f.write(json_str)
return json_str
def load_(self, experiment_path, tantetive=True):
def load_(self, experiment_path, tantetive=True, update_arch=True):
fn = experiment_path / self.info_json
if not fn.exists():
if not tantetive:
@@ -124,14 +124,13 @@ class ULMFiTTrainingCommand(Params):
d = json.load(f)
base = d.pop('base', None)
arch = d.pop('arch')
self.arch.replace_(**arch)
if hasattr(self, 'base'):
self.base.load_(Path(base), tantetive=True, update_arch=False)
if update_arch:
self.arch.replace_(**arch)
self.replace_(**d)
if base is not None:
other_arch = getattr(self, 'base').load_(Path(base), tantetive=True)
if other_arch and other_arch != arch:
warn(f"Architecuture does not match {arch}, {other_arch}")
self.name = experiment_path.name
dataset_path = experiment_path.parent.parent.parent # data/mldoc/de-1/models/fsp15k/multfit_fp16 -> data/mldoc/de-1
dataset_path = experiment_path.parent.parent.parent # ./de-1/models/fsp15k/multfit_fp16 -> ./de-1
self.dataset_path = Path(dataset_path)
self.experiment_path = Path(experiment_path)
return arch
@@ -208,7 +207,9 @@ class ULMFiTPretraining(ULMFiTTrainingCommand):
@property
def model_fnames(self):
if self.experiment_path:
return [self.experiment_path.absolute() / LM_BEST, self.experiment_path.parent.absolute() / 'itos']
model_path = self.experiment_path.absolute()
cache_path = (model_path if (model_path / "itos.pkl").exists() else model_path.parent)
return [model_path / LM_BEST, cache_path /'itos']
return None
@property
@@ -434,18 +435,18 @@ class ULMFiT:
return False
with (experiment_path / "info.json").open('r') as f:
d = json.load(f)
d.pop('dataset_path', None)
dataset_path = d.pop('dataset_path', "")
d['n_hid'] = d['nh']
d['n_layers'] = d['nl']
self.replace_(**d)
if "wiki" in str(experiment_path):
if "wiki" in str(dataset_path):
self.pretrain_lm.experiment_path = path_if_model_exists(experiment_path, LM_BEST)
self.pretrain_lm.dataset_path = experiment_path.parent.parent.parent
self.pretrain_lm.dataset_path = dataset_path if dataset_path in str(experiment_path) else None
else:
self.finetune_lm.experiment_path = path_if_model_exists(experiment_path, ENC_BEST)
self.finetune_lm.dataset_path = experiment_path.parent.parent.parent
self.finetune_lm.dataset_path = dataset_path if dataset_path in str(experiment_path) else None
self.classifier.experiment_path = path_if_model_exists(experiment_path, CLS_BEST)
self.classifier.dataset_path = experiment_path.parent.parent.parent
self.classifier.dataset_path = dataset_path if dataset_path in str(experiment_path) else None
return True
def replace_(self, **kwargs):