From 84405b6e6c7e5e5f641fd68356ee4d99675ab933 Mon Sep 17 00:00:00 2001
From: Marcin <marcin.kardas@gmail.com>
Date: Mon, 13 May 2019 17:51:04 +0200
Subject: [PATCH 1/3] Allow bwd in model name

---
 ulmfit/__main__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ulmfit/__main__.py b/ulmfit/__main__.py
index 93e8b8a..be5ba92 100644
--- a/ulmfit/__main__.py
+++ b/ulmfit/__main__.py
@@ -34,9 +34,9 @@ def get_dataset_path(p, dataset_template):
     for ds_path in ds.parent.glob(pattern):
         yield lang, ds_path
 
-name_re = re.compile("(lstm|qrnn)_(.*)_(lmseed-)?.*\.m")
+name_re = re.compile("(bwd)?(lstm|qrnn)_(.*)_(lmseed-)?.*\.m")
 def folder_name_to_model_name(folder_name):
-    return name_re.match(folder_name).group(2)
+    return name_re.match(folder_name).group(3)
 
 class ULMFiT:
     @wraps(LMHyperParams)

From fe31778d7009b33b25851540f937b5254757aa38 Mon Sep 17 00:00:00 2001
From: Marcin <marcin.kardas@gmail.com>
Date: Tue, 14 May 2019 02:30:25 +0200
Subject: [PATCH 2/3] Add option to remove duplicates

---
 split-cls.py | 40 +++++++++++++++++++++++++++++++++++++---
 1 file changed, 37 insertions(+), 3 deletions(-)

diff --git a/split-cls.py b/split-cls.py
index c89c428..b3b4415 100644
--- a/split-cls.py
+++ b/split-cls.py
@@ -3,14 +3,48 @@ import fire
 from pathlib import Path
 from sys import stderr
 from sklearn.model_selection import train_test_split
+import re
 
 def to_csv(df, path):
     df.to_csv(path, header=None, index=None)
 
-def split(data_dir):
+def remove_rt(df):
+    return df.assign(text=df.text.str.replace('^RT @anonymized_account ',''))
+
+def remove_duplicates(df):
+    exact = df[~df.duplicated('text')]
+    prefixes = exact.text.map(lambda t: t.endswith('…') and exact.text.str.startswith(t[:-1]).sum()>1)
+    return exact[~prefixes]
+
+def cross_remove_duplicates(from_df, other_df):
+    exact = from_df[~from_df.text.isin(other_df.text)]
+    other_prefixes = other_df.text[other_df.text.str.endswith('…')].str[:-1]
+    if len(other_prefixes):
+        other_prefixes_re = re.compile('^'+'|'.join([f'({re.escape(t)})' for t in other_prefixes]))
+    else:
+        other_prefixes_re = re.compile('^$')
+    prefixes = exact.text.map(lambda t:
+            (t.endswith('…') and other_df.text.str.startswith(t[:-1]).any()) or
+            other_prefixes_re.match(t) is not None
+        )
+    return exact[~prefixes]
+
+def split(data_dir, dedup=False):
     data_dir = Path(data_dir)
-    train = pd.read_csv(data_dir / "pl.unsup.csv", header=None)
-    trn, val = train_test_split(train, test_size=0.1, random_state=12345, stratify=train[0])
+    train = pd.read_csv(data_dir / "pl.unsup.csv", header=None, names=["label", "text"])
+    val_ratio = 0.1
+    train = remove_rt(train)
+    trn, val = train_test_split(train, test_size=val_ratio, random_state=12345, stratify=train.label)
+
+    if dedup:
+        trn = remove_duplicates(trn)
+        val = remove_duplicates(val)
+        val = cross_remove_duplicates(val, trn)
+        l1, l2, l3 = len(remove_duplicates(train)), len(trn), len(val)
+        if l1 != l2 + l3:
+            print("Warning: some condition believed by me to be invariant is not hold")
+            print(f"{l1} should be equal to {l2} + {l3} = {l2+l3}")
+
 
     to_csv(trn, data_dir / "pl.train.csv")
     to_csv(val, data_dir / "pl.dev.csv")

From 746c6ca18580d70603c41248f3c179790de9e3d1 Mon Sep 17 00:00:00 2001
From: Marcin <marcin.kardas@gmail.com>
Date: Tue, 14 May 2019 06:57:02 +0200
Subject: [PATCH 3/3] Don't reset lmseed

---
 ulmfit/__main__.py | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/ulmfit/__main__.py b/ulmfit/__main__.py
index be5ba92..83bfe1f 100644
--- a/ulmfit/__main__.py
+++ b/ulmfit/__main__.py
@@ -87,19 +87,20 @@ class ULMFiT:
                     tar.add(f, dest)
 
 
-    def poleval19_full(self, base, num_lm_epochs=6, **kwargs):
-        clsbase = self.poleval19_init(base, num_lm_epochs=num_lm_epochs, **kwargs)
+    def poleval19_full(self, base, num_lm_epochs=6, lmtype=None, **kwargs):
+        clsbase = self.poleval19_init(base, num_lm_epochs=num_lm_epochs, lmtype=lmtype, **kwargs)
         self.poleval19_seeds(clsbase, seed_name='clsweightseed', **kwargs)
         self.poleval19_seeds(clsbase, seed_name='clstrainseed', **kwargs)
 
-    def poleval19_init(self, base, name=None, lmseed=None, **kwargs):
-        clstrainseed = clsweightseed = ftseed = lmseed = 0
-        if "wiki" in base:
-            lmtype = "wiki"
-        elif "reddit" in base:
-            lmtype = "reddit"
-        else:
-            raise AttributeError("unkown lm ty")
+    def poleval19_init(self, base, name=None, lmseed=None, lmtype=None, **kwargs):
+        clstrainseed = clsweightseed = ftseed = 0
+        if lmtype is None:
+            if "wiki" in base:
+                lmtype = "wiki"
+            elif "reddit" in base:
+                lmtype = "reddit"
+            else:
+                raise AttributeError("unkown lm ty")
 
         if "seed0" in base:
             lmseed = 0