From c28c0fde16ef4113b844c239359e7a6cddc32e76 Mon Sep 17 00:00:00 2001
From: Piotr Czapla <github@piotrczapla.com>
Date: Thu, 14 Feb 2019 22:28:25 +0100
Subject: [PATCH] Make ulmfit eval more secure and give more flexibility in
 dataset_template

The dataset_template can use lang as additional token to construct globs patterns.
---
 ulmfit/__main__.py | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/ulmfit/__main__.py b/ulmfit/__main__.py
index bbf69cb..a36fd84 100644
--- a/ulmfit/__main__.py
+++ b/ulmfit/__main__.py
@@ -1,4 +1,5 @@
 import gc
+import os
 import pprint
 import shutil
 from collections import OrderedDict
@@ -8,6 +9,7 @@ import fire
 from .pretrain_lm import LMHyperParams
 from .train_clas import CLSHyperParams
 from pathlib import Path
+from string import Template
 
 class FireView:
     def __init__(self, **kwargs):
@@ -22,7 +24,9 @@ def get_lang_from_dataset_path(ds):
 
 def get_dataset_path(p, dataset_template):
     ds = [x for x in p.parents if x.name == "models"][0].parent
-    return ds.parent.glob(dataset_template.format(ds.name))
+    lang = get_lang_from_dataset_path(ds)
+    for ds_path in ds.parent.glob(Template(dataset_template).substitute(lang=lang, ds_name=ds.name)):
+        yield lang, ds_path
 
 class ULMFiT:
     @wraps(LMHyperParams)
@@ -37,11 +41,10 @@ class ULMFiT:
         params = CLSHyperParams.from_lm(dataset_path, base_lm_path, **changes)
         return FireView(train=params.train_cls, validate_cls=params.validate_cls)
 
-    def eval(self, glob="mldoc/*-1/models/sp30k/lstm_nl4.m", dataset_template='{}', name="tmp-100", cuda_id=0, **trn_params):
+    def eval(self, glob="mldoc/*-1/models/sp30k/lstm_nl4.m", dataset_template='${lang}-1', name="tmp-100", cuda_id=0, **trn_params):
         results = OrderedDict()
         for base_model in sorted(Path("data").glob(glob)):
-            for dataset_path in sorted(get_dataset_path(base_model, dataset_template)):
-                lang = get_lang_from_dataset_path(dataset_path)
+            for lang, dataset_path in sorted(get_dataset_path(base_model, dataset_template)):
                 params = CLSHyperParams.from_lm(dataset_path, base_model, lang=lang, name=name, cuda_id=cuda_id)
                 key = str(params.model_dir.relative_to(Path.cwd()))
                 if params.model_dir.exists():
@@ -54,6 +57,16 @@ class ULMFiT:
                 gc.collect()
 
         pprint.pprint(results)
+
+    def remove_lm_saves(self):
+        for lm_save in Path("data").glob("**/lm_*.pth"):
+            num = lm_save.stem.split("_")[-1]
+            if not num.isdigit():
+                continue
+            if int(num) not in [5, 10, 15]:
+                print("rm ", lm_save)
+                os.remove(lm_save)
+
 # python -m ulmfit cls --dataset-path data/mldoc/de-1-laser  --base-lm-path data/mldoc/de-1/models/sp30k/lstm_nl4.m  --lang=de --name 'nl4' --cuda-id=1 - train 0 --bs 40 --num-cls-epochs=2
 
 if __name__ == '__main__':