Merge branch 'reproduce-poleval' of https://github.com/n-waves/ulmfit-multilingual into reproduce-poleval

split-cls.py

@@ -3,14 +3,48 @@ import fire
 from pathlib import Path
 from sys import stderr
 from sklearn.model_selection import train_test_split
+import re
 
 def to_csv(df, path):
     df.to_csv(path, header=None, index=None)
 
-def split(data_dir):
+def remove_rt(df):
+    return df.assign(text=df.text.str.replace('^RT @anonymized_account ',''))
+
+def remove_duplicates(df):
+    exact = df[~df.duplicated('text')]
+    prefixes = exact.text.map(lambda t: t.endswith('…') and exact.text.str.startswith(t[:-1]).sum()>1)
+    return exact[~prefixes]
+
+def cross_remove_duplicates(from_df, other_df):
+    exact = from_df[~from_df.text.isin(other_df.text)]
+    other_prefixes = other_df.text[other_df.text.str.endswith('…')].str[:-1]
+    if len(other_prefixes):
+        other_prefixes_re = re.compile('^'+'|'.join([f'({re.escape(t)})' for t in other_prefixes]))
+    else:
+        other_prefixes_re = re.compile('^$')
+    prefixes = exact.text.map(lambda t:
+            (t.endswith('…') and other_df.text.str.startswith(t[:-1]).any()) or
+            other_prefixes_re.match(t) is not None
+        )
+    return exact[~prefixes]
+
+def split(data_dir, dedup=False):
     data_dir = Path(data_dir)
-    train = pd.read_csv(data_dir / "pl.unsup.csv", header=None)
+    train = pd.read_csv(data_dir / "pl.unsup.csv", header=None, names=["label", "text"])
-    trn, val = train_test_split(train, test_size=0.1, random_state=12345, stratify=train[0])
+    val_ratio = 0.1
+    train = remove_rt(train)
+    trn, val = train_test_split(train, test_size=val_ratio, random_state=12345, stratify=train.label)
+
+    if dedup:
+        trn = remove_duplicates(trn)
+        val = remove_duplicates(val)
+        val = cross_remove_duplicates(val, trn)
+        l1, l2, l3 = len(remove_duplicates(train)), len(trn), len(val)
+        if l1 != l2 + l3:
+            print("Warning: some condition believed by me to be invariant is not hold")
+            print(f"{l1} should be equal to {l2} + {l3} = {l2+l3}")
+
 
     to_csv(trn, data_dir / "pl.train.csv")
     to_csv(val, data_dir / "pl.dev.csv")

ulmfit/__main__.py

@@ -33,9 +33,9 @@ def get_dataset_path(p, dataset_template):
     for ds_path in ds.parent.glob(pattern):
         yield lang, ds_path
 
-name_re = re.compile("(lstm|qrnn)_(.*)_(lmseed-)?.*\.m")
+name_re = re.compile("(bwd)?(lstm|qrnn)_(.*)_(lmseed-)?.*\.m")
 def folder_name_to_model_name(folder_name):
-    return name_re.match(folder_name).group(2)
+    return name_re.match(folder_name).group(3)
 
 class ULMFiT:
     @wraps(LMHyperParams)
@@ -86,19 +86,20 @@ class ULMFiT:
                     tar.add(f, dest)
 
 
-    def poleval19_full(self, base, num_lm_epochs=6, **kwargs):
+    def poleval19_full(self, base, num_lm_epochs=6, lmtype=None, **kwargs):
-        clsbase = self.poleval19_init(base, num_lm_epochs=num_lm_epochs, **kwargs)
+        clsbase = self.poleval19_init(base, num_lm_epochs=num_lm_epochs, lmtype=lmtype, **kwargs)
         self.poleval19_seeds(clsbase, seed_name='clsweightseed', **kwargs)
         self.poleval19_seeds(clsbase, seed_name='clstrainseed', **kwargs)
 
-    def poleval19_init(self, base, name=None, lmseed=None, **kwargs):
+    def poleval19_init(self, base, name=None, lmseed=None, lmtype=None, **kwargs):
-        clstrainseed = clsweightseed = ftseed = lmseed = 0
+        clstrainseed = clsweightseed = ftseed = 0
-        if "wiki" in base:
+        if lmtype is None:
-            lmtype = "wiki"
+            if "wiki" in base:
-        elif "reddit" in base:
+                lmtype = "wiki"
-            lmtype = "reddit"
+            elif "reddit" in base:
-        else:
+                lmtype = "reddit"
-            raise AttributeError("unkown lm ty")
+            else:
+                raise AttributeError("unkown lm ty")
 
         if "seed0" in base:
             lmseed = 0