From 7fe059c9a2d432da5475d1dc71b9b8f7cd9494e6 Mon Sep 17 00:00:00 2001
From: Sebastian <ruder.sebastian@gmail.com>
Date: Sat, 17 Nov 2018 14:47:27 +0000
Subject: [PATCH 01/11] Fixed models path for vocabulary

---
 ulmfit/pretrain_lm.py | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py
index 0847ca6..0a76e00 100644
--- a/ulmfit/pretrain_lm.py
+++ b/ulmfit/pretrain_lm.py
@@ -89,18 +89,17 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo
         itos = [o for o,c in cnt.most_common(n=max_vocab)]
         itos.insert(1, PAD)  #  set pad id to 1 to conform to fast.ai standard
         assert UNK in itos, f'Unknown words are expected to have been replaced with {UNK} in the data.'
-        stoi = {w: i for i, w in enumerate(itos)}
 
         vocab = Vocab(itos)
         stoi = vocab.stoi
 
         # save vocabulary
-        print(f"Saving vocabulary as {dir_path / model_dir}")
-        results['itos_fname'] = dir_path / model_dir / f'itos_{name}.pkl'
-        with open(results['itos_fname'], 'wb') as f:
+        itos_fname = model_dir / f'itos_{name}.pkl'
+        print(f"Saving vocabulary as {itos_fname}")
+        results['itos_fname'] = itos_fname
+        with open(itos_fname, 'wb') as f:
             pickle.dump(itos, f)
 
-
         trn_ids = np.array([([stoi.get(w, stoi[UNK]) for w in s]) for s in trn_tok])
         val_ids = np.array([([stoi.get(w, stoi[UNK]) for w in s]) for s in val_tok])
 
@@ -108,7 +107,6 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo
         data_lm = TextLMDataBunch.from_ids(path=dir_path, vocab=vocab, train_ids=trn_ids,
                                            valid_ids=val_ids, bs=bs, bptt=bptt)
 
-
     print('Size of vocabulary:', len(itos))
     print('First 10 words in vocab:', ', '.join([itos[i] for i in range(10)]))
 
@@ -135,8 +133,6 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo
 
     fit_one_cycle(learn, num_epochs, 5e-3, (0.8, 0.7), wd=1e-7)
 
-
-
     if not subword and max_vocab is None:
         # only if we use the unpreprocessed version and the full vocabulary
         # are the perplexity results comparable to previous work

From d79661045257bb75f810b631d0c065fa56e32294 Mon Sep 17 00:00:00 2001
From: Sebastian <ruder.sebastian@gmail.com>
Date: Sat, 17 Nov 2018 14:48:06 +0000
Subject: [PATCH 02/11] Removed deprecated default_rules

---
 fastai_contrib/utils.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py
index 9004132..688e28d 100644
--- a/fastai_contrib/utils.py
+++ b/fastai_contrib/utils.py
@@ -11,7 +11,7 @@ import re
 import csv
 
 from functools import reduce
-from fastai.text.transform import Tokenizer, BaseTokenizer, Vocab, default_rules
+from fastai.text.transform import Tokenizer, BaseTokenizer, Vocab
 from fastai.torch_core import *
 
 import shutil
@@ -52,6 +52,7 @@ class SentencepieceTokenizer(BaseTokenizer):
     def add_special_cases(self, toks:Collection[str]):
         pass
 
+
 def get_sentencepiece(path:PathOrStr, trn_path:Path, name:str, rules:ListRules=None,
                       vocab_size:int=30000, model_type:str='unigram', input_sentence_size:int=1E7, 
                       pad_idx:int=PAD_TOKEN_ID):
@@ -62,7 +63,7 @@ def get_sentencepiece(path:PathOrStr, trn_path:Path, name:str, rules:ListRules=N
     
     path = pathlib.Path(path)
     os.makedirs(path / 'models', exist_ok=True)
-    rules = rules if rules else default_rules
+    rules = rules if rules else None
 
     cache_name = 'tmp'
     

From 6292412eff6fa98152c01936c876e3c06359ac23 Mon Sep 17 00:00:00 2001
From: "NAUSICAA\\Julian" <eisenjulian@gmail.com>
Date: Sun, 18 Nov 2018 22:29:07 -0300
Subject: [PATCH 03/11] Adding scripts to download and format RCV and Webis CLS
 X-Language datasets

---
 prepare_cls.sh            | 15 ++++++++++++
 prepare_rcv.sh            | 50 +++++++++++++++++++++++++++++++++++++++
 ulmfit/postprocess_cls.py | 23 ++++++++++++++++++
 ulmfit/postprocess_rcv.py | 12 ++++++++++
 4 files changed, 100 insertions(+)
 create mode 100644 prepare_cls.sh
 create mode 100644 prepare_rcv.sh
 create mode 100644 ulmfit/postprocess_cls.py
 create mode 100644 ulmfit/postprocess_rcv.py

diff --git a/prepare_cls.sh b/prepare_cls.sh
new file mode 100644
index 0000000..286e4e4
--- /dev/null
+++ b/prepare_cls.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+ROOT=data
+DATA_DIR=$ROOT/cls
+mkdir -p $DATA_DIR/tmp
+echo "Saving data in $DATA_DIR"
+
+if [ ! -d $DATA_DIR/tmp/cls-acl10-unprocessed ]; then
+  wget -c http://www.uni-weimar.de/medien/webis/corpora/corpus-webis-cls-10/cls-acl10-unprocessed.tar.gz  -P $DATA_DIR/tmp
+  tar -xzvf $DATA_DIR/tmp/cls-acl10-unprocessed.tar.gz -C $DATA_DIR/tmp/
+else
+  echo "CLS already exists. Skipping download."
+fi
+
+python ulmfit/postprocess_cls.py --input_dir $DATA_DIR/tmp/cls-acl10-unprocessed --output_dir $DATA_DIR
diff --git a/prepare_rcv.sh b/prepare_rcv.sh
new file mode 100644
index 0000000..acea5d4
--- /dev/null
+++ b/prepare_rcv.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+if [[ $# -ne 3 ]] ; then
+  echo 'Usage: ./prepare_rcv.sh <RCV_URL> <RCV_USER> <RCV_PASSWORD>'
+  echo 'This dataset has restricted access: apply at https://trec.nist.gov/data/reuters/reuters.html'
+  exit 1
+fi
+
+ROOT="data"
+DATA_DIR="${ROOT}/rcv"
+mkdir -p "${DATA_DIR}/tmp"
+echo "Saving data in $DATA_DIR"
+
+RCV_URL=$1
+RCV_USER=$2
+RCV_PASSWORD=$3
+
+MLDOC=https://github.com/facebookresearch/MLDoc/raw/master
+wget -c $MLDOC/generate_documents.py -P "${DATA_DIR}/tmp"
+
+if [ ! -d "${DATA_DIR}/tmp/RCV2_Multilingual_Corpus" ]; then
+  wget -c --user $RCV_USER --password $RCV_PASSWORD $RCV_URL/rcv2.tar.xz -P "${DATA_DIR}/tmp"  
+  tar xvf "${DATA_DIR}/tmp/rcv2.tar.xz" -C "${DATA_DIR}/tmp/"
+else
+  echo "RCV2 already exists. Skipping download."
+fi
+
+if [ ! -d "${DATA_DIR}/tmp/RCV2_Multilingual_Corpus/english" ]; then
+  wget -c --user $RCV_USER --password $RCV_PASSWORD $RCV_URL/rcv1.tar.xz -P "${DATA_DIR}/tmp"
+  tar xvf "${DATA_DIR}/tmp/rcv1.tar.xz" -C "${DATA_DIR}/tmp/RCV2_Multilingual_Corpus/"
+  mv "${DATA_DIR}/tmp/RCV2_Multilingual_Corpus/rcv1" "${DATA_DIR}/tmp/RCV2_Multilingual_Corpus/english"
+else
+  echo "RCV1 already exists. Skipping download."
+fi
+
+for LANGUAGE in spanish chinese french japanese german italian russian english
+do
+  mkdir -p "${DATA_DIR}/${LANGUAGE}"
+  for FILE_EXT in train.1000 train.2000 train.5000 train.10000 dev test
+  do
+    wget -c $MLDOC/mldoc-indices/$LANGUAGE.$FILE_EXT -P "${DATA_DIR}/tmp"
+
+    python $DATA_DIR/tmp/generate_documents.py \
+        --indices-file $DATA_DIR/tmp/$LANGUAGE.$FILE_EXT \
+        --output-filename $DATA_DIR/tmp/$LANGUAGE.$FILE_EXT.raw \
+        --rcv-dir $DATA_DIR/tmp/RCV2_Multilingual_Corpus/$LANGUAGE
+    python ulmfit/postprocess_rcv.py --input_file $DATA_DIR/tmp/$LANGUAGE.$FILE_EXT.raw \
+        --output_file $DATA_DIR/$LANGUAGE/$FILE_EXT.csv
+  done
+done
diff --git a/ulmfit/postprocess_cls.py b/ulmfit/postprocess_cls.py
new file mode 100644
index 0000000..ad55a22
--- /dev/null
+++ b/ulmfit/postprocess_cls.py
@@ -0,0 +1,23 @@
+import fire
+import pandas as pd
+import os
+from bs4 import BeautifulSoup
+
+def main(input_dir, output_dir):
+  for lang in ['en', 'de', 'fr', 'jp']:
+    for cat in ['dvd', 'music', 'books']:
+      for mode in ['train', 'test']: # , 'unlabeled']:
+        os.makedirs(os.path.join(input_dir, lang), exist_ok=True)
+        with open(os.path.join(input_dir, lang, cat, mode + '.review'), 'r') as f:
+          items = BeautifulSoup(f.read(), features="html.parser").find_all('item')
+          text = [item.find('text').text.strip() for item in items]
+          summary = [item.find('summary').text.strip() for item in items]
+          if mode == 'unlabeled':
+            out = pd.DataFrame({'summary': summary, 'text': text})
+          else:
+            labels = [1 if item.rating.text in ('4.0', '5.0') else 0 for item in items]
+            out = pd.DataFrame({'labels': labels, 'summary': summary, 'text': text})
+          file_name = os.path.join(output_dir, f'{lang}/{cat}.{mode}.csv')
+          out.to_csv(file_name, header=None, index=False)
+
+if __name__ == '__main__': fire.Fire(main)
diff --git a/ulmfit/postprocess_rcv.py b/ulmfit/postprocess_rcv.py
new file mode 100644
index 0000000..27753a2
--- /dev/null
+++ b/ulmfit/postprocess_rcv.py
@@ -0,0 +1,12 @@
+import fire
+import pandas as pd
+
+def main(input_file, output_file):
+  df = pd.read_csv(input_file, sep='\t', header=None)
+  unique_labels = sorted(list(df[0].unique()))
+  labels = [unique_labels.index(label) for label in df[0]]
+  texts = [eval(text).decode('utf-8').strip() for text in df[1]]
+  out = pd.DataFrame({'labels': labels, 'texts': texts})
+  out.to_csv(output_file, header=None, index=False)
+
+if __name__ == '__main__': fire.Fire(main)

From 6e867811db82a62707199ffdeb34907065f992ad Mon Sep 17 00:00:00 2001
From: Aayush <aayushyadav96@gmail.com>
Date: Mon, 19 Nov 2018 19:28:19 +0530
Subject: [PATCH 04/11] Support for running sentencepiece with train_clas

Committing into this branch. Only changes are in `read_xnli` and `read_imdb`.
---
 fastai_contrib/utils.py | 46 ++++++++++++++++++++++++++++++++---------
 1 file changed, 36 insertions(+), 10 deletions(-)

diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py
index 688e28d..1eab00b 100644
--- a/fastai_contrib/utils.py
+++ b/fastai_contrib/utils.py
@@ -47,8 +47,10 @@ class SentencepieceTokenizer(BaseTokenizer):
             raise Exception('sentencepiece module is missing: run `pip install sentencepiece`')
         self.tok = spm.SentencePieceProcessor()
         self.tok.Load(str(pathlib.Path(model_dir) / 'spm.model'))
+    
     def tokenizer(self, t:str) -> List[str]:
         return self.tok.EncodeAsPieces(t)
+    
     def add_special_cases(self, toks:Collection[str]):
         pass
 
@@ -77,11 +79,11 @@ def get_sentencepiece(path:PathOrStr, trn_path:Path, name:str, rules:ListRules=N
         with open(raw_text_path, 'w') as f:
             f.write(raw_text)
       
-        sp_params = f'--input={raw_text_path} --pad_id={pad_idx} --unk_id=0' \
-                    f'--character_coverage=1.0 --bos_id=-1 --eos_id=-1 ' \
-                    f'--input_sentence_size={int(input_sentence_size)} ' \
+        sp_params = f"--input={raw_text_path} --pad_id={pad_idx} --unk_id=0 " \
+                    f"--character_coverage=1.0 --bos_id=-1 --eos_id=-1 " \
+                    f"--input_sentence_size={int(input_sentence_size)} " \
                     f"--model_prefix={path / 'models' / 'spm'} " \
-                    f'--vocab_size={vocab_size} --model_type={model_type} '
+                    f"--vocab_size={vocab_size} --model_type={model_type} "
         spm.SentencePieceTrainer.Train(sp_params)
   
         with open(path / 'models' / 'spm.vocab', 'r') as f:
@@ -199,40 +201,54 @@ def prepare_imdb(file_path: str, prepare_lm = False):
         df_val.to_csv(LM_PATH / 'test.csv', header=False, index=False)
 
 
-def read_imdb(dir_path, lang, split) -> Tuple[List[List[str]], List[str]]:
+def read_imdb(dir_path, lang, split, spm_path=None) -> Tuple[List[List[str]], List[str]]:
     """
     Reads IMDb data.
     :param dir_path: the path to the imdb folder
     :param lang: the language (not used here as IMDb is only available in English)
     :param split: the split of the data that should be read (train, test, val)
+    :param spm_path: path to sentencepiece model
     :return: a tuple consisting of a list of lists of tokens and a list of labels
     """
     file_path = dir_path / 'train.csv' if split == TRN else dir_path / 'test.csv'
     toks, lbls = [], []
+    
     mt = MosesTokenizer('en')
+    if spm_path is not None:
+        sp = SentencepieceTokenizer(spm_path)
+
     print(f'Reading {file_path}...')
+    
     with open(file_path, encoding='utf-8') as f:
         reader = csv.reader(f)
         for row in reader:
             label, text = row
             lbls.append(label)
             raw_tokens = mt.tokenize(text, return_str=True).split(' ') + [EOS]
+            
             tokens = []
+            
+            # fix up occurences of numbers in text
             for token in raw_tokens:
                 if number_match_re.match(token):
                     tokens += number_split_re.sub(r' @\1@ ', token).split()
                 else:
                     tokens.append(token)
-            toks.append(tokens)
+            
+            if spm_path is not None:
+                tokens = sp.tokenizer(' '.join(tokens))
+
+            toks.append(tokens + [EOS])
     return toks, lbls
 
 
-def read_xnli(dir_path, lang, split) -> Tuple[List[List[str]], List[str]]:
+def read_xnli(dir_path, lang, split, spm_path=None) -> Tuple[List[List[str]], List[str]]:
     """
     Reads XNLI data.
     :param dir_path: the path to the xnli folder
     :param lang: the language
     :param split: the split of the data that should be read (train, test, val)
+    :param spm_path: path to sentencepiece model
     :return: a tuple consisting of a list of lists of tokens and a list of labels
     """
     file_path = XNLI_PATHS[split]
@@ -242,6 +258,10 @@ def read_xnli(dir_path, lang, split) -> Tuple[List[List[str]], List[str]]:
         file_name = 'xnli.dev.en.tsv' if split == VAL else 'xnli.test.en.tsv'
         file_path = f'XNLI-MT-1.0/xnli/{file_name}'
     file_path = dir_path / file_path
+    
+    if spm_path is not None:
+        sp = SentencepieceTokenizer(spm_path)
+
     toks, lbls = [], []
     print(f'Reading {file_path}...')
     with open(file_path, encoding='utf-8') as f:
@@ -257,9 +277,15 @@ def read_xnli(dir_path, lang, split) -> Tuple[List[List[str]], List[str]]:
                 if ex_lang != lang:
                     continue
                 premise, hypo, label = row[-3], row[-2], row[1]
+            
             # TODO add BOS
-            premise_toks = premise.split(' ') + [EOS]
-            hypo_toks = hypo.split(' ') + [EOS]
+            if spm_path is not None:
+                premise_toks = sp.tokenizer(premise) + [EOS]
+                hypo_toks = sp.tokenizer(hypo) + [EOS]
+            else:
+                premise_toks = premise.split(' ') + [EOS]
+                hypo_toks = hypo.split(' ') + [EOS]
+            
             toks.append(premise_toks + [SEP] + hypo_toks)
             lbls.append(label)
     return toks, lbls
@@ -389,4 +415,4 @@ class TextReader():
 
 
 if __name__ == "__main__":
-    fire.Fire()  # allows using all functions via CLI e.g. python utils.py prepare_imdb aclImdb.tgz
\ No newline at end of file
+    fire.Fire()  # allows using all functions via CLI e.g. python utils.py prepare_imdb aclImdb.tgz

From 73403bdcb310a12f8cab3868d980ef8860d17b14 Mon Sep 17 00:00:00 2001
From: Aayush <aayushyadav96@gmail.com>
Date: Mon, 19 Nov 2018 19:30:11 +0530
Subject: [PATCH 05/11] fix double EOS in read_imdb

---
 fastai_contrib/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py
index 1eab00b..9e4c9fa 100644
--- a/fastai_contrib/utils.py
+++ b/fastai_contrib/utils.py
@@ -224,7 +224,7 @@ def read_imdb(dir_path, lang, split, spm_path=None) -> Tuple[List[List[str]], Li
         for row in reader:
             label, text = row
             lbls.append(label)
-            raw_tokens = mt.tokenize(text, return_str=True).split(' ') + [EOS]
+            raw_tokens = mt.tokenize(text, return_str=True).split(' ')
             
             tokens = []
             

From 470f984ca47f3c71cf16391d00e52a0e6fbdf8b1 Mon Sep 17 00:00:00 2001
From: "NAUSICAA\\Julian" <eisenjulian@gmail.com>
Date: Mon, 19 Nov 2018 16:31:38 -0300
Subject: [PATCH 06/11] Script to merge two languages

---
 ulmfit/merge_langs.py | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)
 create mode 100644 ulmfit/merge_langs.py

diff --git a/ulmfit/merge_langs.py b/ulmfit/merge_langs.py
new file mode 100644
index 0000000..7fee7ed
--- /dev/null
+++ b/ulmfit/merge_langs.py
@@ -0,0 +1,35 @@
+"""
+Script to merge WikiText files created with `create_wikitext.py`.
+"""
+import fire
+from pathlib import Path
+from contextlib import ExitStack
+
+def merge_wikitext(paths, langs, dest_path, num_sentences):
+    wiki_paths = [Path(path) for path in paths]
+    for wiki_path in wiki_paths:
+        assert wiki_path.exists(), f'Error: {wiki_path} does not exist.'
+    dest_path = Path(dest_path)
+    dest_path.mkdir(exist_ok=True)
+    splits = ['train', 'valid', 'test']
+    concat_langs = '-'.join(langs)
+    for split in splits:
+        with ExitStack() as stack:
+            files = [stack.enter_context(open(
+                wiki_path / f'{lang}.wiki.{split}.tokens', 'r', encoding='utf-8')) 
+                for lang, wiki_path in zip(langs, wiki_paths)]
+            
+            output = stack.enter_context(open(dest_path / f'{concat_langs}.wiki.{split}.tokens', 'w', encoding='utf-8'))
+            done = False
+            while not done:
+                for file in files:
+                    lines = [file.readline() for x in range(num_sentences)]
+                    size = len(lines)
+                    lines = [line for line in lines if line]
+                    if len(lines) < size:
+                        done = True
+                    for line in lines:
+                        output.write(line)
+
+if __name__ == '__main__':
+    fire.Fire(merge_wikitext)
\ No newline at end of file

From 0ecc6342a6071b40ba5d4b0c0de52a83e4b321a0 Mon Sep 17 00:00:00 2001
From: "NAUSICAA\\Julian" <eisenjulian@gmail.com>
Date: Tue, 20 Nov 2018 00:38:44 -0300
Subject: [PATCH 07/11] Fixes when running sentence piece end to end

---
 fastai_contrib/utils.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py
index 9004132..43c98aa 100644
--- a/fastai_contrib/utils.py
+++ b/fastai_contrib/utils.py
@@ -61,16 +61,17 @@ def get_sentencepiece(path:PathOrStr, trn_path:Path, name:str, rules:ListRules=N
         raise Exception('sentencepiece module is missing: run `pip install sentencepiece`')
     
     path = pathlib.Path(path)
+    cache_name = 'tmp'
+    os.makedirs(path / cache_name, exist_ok=True)
     os.makedirs(path / 'models', exist_ok=True)
     rules = rules if rules else default_rules
 
-    cache_name = 'tmp'
     
     # load the text frmo the train tokens file
     text = [line.rstrip('\n') for line in open(trn_path)]
     text = list(filter(None, text))
 
-    if not os.path.isfile(path / 'models' / 'spm.model') or not os.path.isfile(path / f'itos_{name}.pkl'):
+    if not os.path.isfile(path / 'models' / 'spm.model') or not os.path.isfile(path / 'models' / f'itos_{name}.pkl'):
         raw_text = reduce(lambda t, rule: rule(t), rules, '\n'.join(text))
         raw_text_path = path / cache_name / 'all_text.txt'
         with open(raw_text_path, 'w') as f:
@@ -88,11 +89,12 @@ def get_sentencepiece(path:PathOrStr, trn_path:Path, name:str, rules:ListRules=N
             vocab[0] = UNK
             vocab[pad_idx] = PAD
   
-        pickle.dump(vocab, open(path / 'models'/ f'itos_{name}.pkl', 'wb'))
+        pickle.dump(vocab, open(path / 'models' / f'itos_{name}.pkl', 'wb'))
     
-    vocab = Vocab(pickle.load(open(path / 'models'/ f'itos_{name}.pkl', 'rb')))
-    spt = SentencepieceTokenizer(path)
-    tokenizer = Tokenizer(tok_func=lambda lang: spt, rules=rules)
+    vocab = Vocab(pickle.load(open(path / 'models' / f'itos_{name}.pkl', 'rb')))
+    # We cannot use lambdas or local methods here, since `tok_func` needs to be
+    # pickle-able in order to be called in subprocesses when multithread tokenizing
+    tokenizer = Tokenizer(tok_func=SentencepieceTokenizer, lang: str(path / 'models'), rules=rules)
     
     clear_cache_directory(path, cache_name)
 

From 3b5629c0c5cfdc5e43bf8bd12e4f72b9a500021d Mon Sep 17 00:00:00 2001
From: "NAUSICAA\\Julian" <eisenjulian@gmail.com>
Date: Tue, 20 Nov 2018 00:46:04 -0300
Subject: [PATCH 08/11] Removing files

---
 prepare_cls.sh            | 15 ------------
 prepare_rcv.sh            | 50 ---------------------------------------
 ulmfit/merge_langs.py     | 35 ---------------------------
 ulmfit/postprocess_cls.py | 23 ------------------
 ulmfit/postprocess_rcv.py | 12 ----------
 5 files changed, 135 deletions(-)
 delete mode 100644 prepare_cls.sh
 delete mode 100644 prepare_rcv.sh
 delete mode 100644 ulmfit/merge_langs.py
 delete mode 100644 ulmfit/postprocess_cls.py
 delete mode 100644 ulmfit/postprocess_rcv.py

diff --git a/prepare_cls.sh b/prepare_cls.sh
deleted file mode 100644
index 286e4e4..0000000
--- a/prepare_cls.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/usr/bin/env bash
-
-ROOT=data
-DATA_DIR=$ROOT/cls
-mkdir -p $DATA_DIR/tmp
-echo "Saving data in $DATA_DIR"
-
-if [ ! -d $DATA_DIR/tmp/cls-acl10-unprocessed ]; then
-  wget -c http://www.uni-weimar.de/medien/webis/corpora/corpus-webis-cls-10/cls-acl10-unprocessed.tar.gz  -P $DATA_DIR/tmp
-  tar -xzvf $DATA_DIR/tmp/cls-acl10-unprocessed.tar.gz -C $DATA_DIR/tmp/
-else
-  echo "CLS already exists. Skipping download."
-fi
-
-python ulmfit/postprocess_cls.py --input_dir $DATA_DIR/tmp/cls-acl10-unprocessed --output_dir $DATA_DIR
diff --git a/prepare_rcv.sh b/prepare_rcv.sh
deleted file mode 100644
index acea5d4..0000000
--- a/prepare_rcv.sh
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/usr/bin/env bash
-
-if [[ $# -ne 3 ]] ; then
-  echo 'Usage: ./prepare_rcv.sh <RCV_URL> <RCV_USER> <RCV_PASSWORD>'
-  echo 'This dataset has restricted access: apply at https://trec.nist.gov/data/reuters/reuters.html'
-  exit 1
-fi
-
-ROOT="data"
-DATA_DIR="${ROOT}/rcv"
-mkdir -p "${DATA_DIR}/tmp"
-echo "Saving data in $DATA_DIR"
-
-RCV_URL=$1
-RCV_USER=$2
-RCV_PASSWORD=$3
-
-MLDOC=https://github.com/facebookresearch/MLDoc/raw/master
-wget -c $MLDOC/generate_documents.py -P "${DATA_DIR}/tmp"
-
-if [ ! -d "${DATA_DIR}/tmp/RCV2_Multilingual_Corpus" ]; then
-  wget -c --user $RCV_USER --password $RCV_PASSWORD $RCV_URL/rcv2.tar.xz -P "${DATA_DIR}/tmp"  
-  tar xvf "${DATA_DIR}/tmp/rcv2.tar.xz" -C "${DATA_DIR}/tmp/"
-else
-  echo "RCV2 already exists. Skipping download."
-fi
-
-if [ ! -d "${DATA_DIR}/tmp/RCV2_Multilingual_Corpus/english" ]; then
-  wget -c --user $RCV_USER --password $RCV_PASSWORD $RCV_URL/rcv1.tar.xz -P "${DATA_DIR}/tmp"
-  tar xvf "${DATA_DIR}/tmp/rcv1.tar.xz" -C "${DATA_DIR}/tmp/RCV2_Multilingual_Corpus/"
-  mv "${DATA_DIR}/tmp/RCV2_Multilingual_Corpus/rcv1" "${DATA_DIR}/tmp/RCV2_Multilingual_Corpus/english"
-else
-  echo "RCV1 already exists. Skipping download."
-fi
-
-for LANGUAGE in spanish chinese french japanese german italian russian english
-do
-  mkdir -p "${DATA_DIR}/${LANGUAGE}"
-  for FILE_EXT in train.1000 train.2000 train.5000 train.10000 dev test
-  do
-    wget -c $MLDOC/mldoc-indices/$LANGUAGE.$FILE_EXT -P "${DATA_DIR}/tmp"
-
-    python $DATA_DIR/tmp/generate_documents.py \
-        --indices-file $DATA_DIR/tmp/$LANGUAGE.$FILE_EXT \
-        --output-filename $DATA_DIR/tmp/$LANGUAGE.$FILE_EXT.raw \
-        --rcv-dir $DATA_DIR/tmp/RCV2_Multilingual_Corpus/$LANGUAGE
-    python ulmfit/postprocess_rcv.py --input_file $DATA_DIR/tmp/$LANGUAGE.$FILE_EXT.raw \
-        --output_file $DATA_DIR/$LANGUAGE/$FILE_EXT.csv
-  done
-done
diff --git a/ulmfit/merge_langs.py b/ulmfit/merge_langs.py
deleted file mode 100644
index 7fee7ed..0000000
--- a/ulmfit/merge_langs.py
+++ /dev/null
@@ -1,35 +0,0 @@
-"""
-Script to merge WikiText files created with `create_wikitext.py`.
-"""
-import fire
-from pathlib import Path
-from contextlib import ExitStack
-
-def merge_wikitext(paths, langs, dest_path, num_sentences):
-    wiki_paths = [Path(path) for path in paths]
-    for wiki_path in wiki_paths:
-        assert wiki_path.exists(), f'Error: {wiki_path} does not exist.'
-    dest_path = Path(dest_path)
-    dest_path.mkdir(exist_ok=True)
-    splits = ['train', 'valid', 'test']
-    concat_langs = '-'.join(langs)
-    for split in splits:
-        with ExitStack() as stack:
-            files = [stack.enter_context(open(
-                wiki_path / f'{lang}.wiki.{split}.tokens', 'r', encoding='utf-8')) 
-                for lang, wiki_path in zip(langs, wiki_paths)]
-            
-            output = stack.enter_context(open(dest_path / f'{concat_langs}.wiki.{split}.tokens', 'w', encoding='utf-8'))
-            done = False
-            while not done:
-                for file in files:
-                    lines = [file.readline() for x in range(num_sentences)]
-                    size = len(lines)
-                    lines = [line for line in lines if line]
-                    if len(lines) < size:
-                        done = True
-                    for line in lines:
-                        output.write(line)
-
-if __name__ == '__main__':
-    fire.Fire(merge_wikitext)
\ No newline at end of file
diff --git a/ulmfit/postprocess_cls.py b/ulmfit/postprocess_cls.py
deleted file mode 100644
index ad55a22..0000000
--- a/ulmfit/postprocess_cls.py
+++ /dev/null
@@ -1,23 +0,0 @@
-import fire
-import pandas as pd
-import os
-from bs4 import BeautifulSoup
-
-def main(input_dir, output_dir):
-  for lang in ['en', 'de', 'fr', 'jp']:
-    for cat in ['dvd', 'music', 'books']:
-      for mode in ['train', 'test']: # , 'unlabeled']:
-        os.makedirs(os.path.join(input_dir, lang), exist_ok=True)
-        with open(os.path.join(input_dir, lang, cat, mode + '.review'), 'r') as f:
-          items = BeautifulSoup(f.read(), features="html.parser").find_all('item')
-          text = [item.find('text').text.strip() for item in items]
-          summary = [item.find('summary').text.strip() for item in items]
-          if mode == 'unlabeled':
-            out = pd.DataFrame({'summary': summary, 'text': text})
-          else:
-            labels = [1 if item.rating.text in ('4.0', '5.0') else 0 for item in items]
-            out = pd.DataFrame({'labels': labels, 'summary': summary, 'text': text})
-          file_name = os.path.join(output_dir, f'{lang}/{cat}.{mode}.csv')
-          out.to_csv(file_name, header=None, index=False)
-
-if __name__ == '__main__': fire.Fire(main)
diff --git a/ulmfit/postprocess_rcv.py b/ulmfit/postprocess_rcv.py
deleted file mode 100644
index 27753a2..0000000
--- a/ulmfit/postprocess_rcv.py
+++ /dev/null
@@ -1,12 +0,0 @@
-import fire
-import pandas as pd
-
-def main(input_file, output_file):
-  df = pd.read_csv(input_file, sep='\t', header=None)
-  unique_labels = sorted(list(df[0].unique()))
-  labels = [unique_labels.index(label) for label in df[0]]
-  texts = [eval(text).decode('utf-8').strip() for text in df[1]]
-  out = pd.DataFrame({'labels': labels, 'texts': texts})
-  out.to_csv(output_file, header=None, index=False)
-
-if __name__ == '__main__': fire.Fire(main)

From 40a29903228b53db28d41deee0a87c117b3e8f95 Mon Sep 17 00:00:00 2001
From: "NAUSICAA\\Julian" <eisenjulian@gmail.com>
Date: Tue, 20 Nov 2018 09:06:03 -0300
Subject: [PATCH 09/11] Add suggested changes to rules system

---
 fastai_contrib/utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py
index 101bce3..4507c7c 100644
--- a/fastai_contrib/utils.py
+++ b/fastai_contrib/utils.py
@@ -67,7 +67,7 @@ def get_sentencepiece(path:PathOrStr, trn_path:Path, name:str, rules:ListRules=N
     cache_name = 'tmp'
     os.makedirs(path / cache_name, exist_ok=True)
     os.makedirs(path / 'models', exist_ok=True)
-    rules = rules if rules else None
+    rules = rules if rules is not None else []
 
     
     # load the text frmo the train tokens file
@@ -97,7 +97,7 @@ def get_sentencepiece(path:PathOrStr, trn_path:Path, name:str, rules:ListRules=N
     vocab = Vocab(pickle.load(open(path / 'models' / f'itos_{name}.pkl', 'rb')))
     # We cannot use lambdas or local methods here, since `tok_func` needs to be
     # pickle-able in order to be called in subprocesses when multithread tokenizing
-    tokenizer = Tokenizer(tok_func=SentencepieceTokenizer, lang: str(path / 'models'), rules=rules)
+    tokenizer = Tokenizer(tok_func=SentencepieceTokenizer, lang: str(path / 'models'), pre_rules=rules, post_rules=[])
     
     clear_cache_directory(path, cache_name)
 

From 79691791b3e7a86516bf20c282576c89a621d9ab Mon Sep 17 00:00:00 2001
From: "NAUSICAA\\Julian" <eisenjulian@gmail.com>
Date: Tue, 20 Nov 2018 18:53:23 -0300
Subject: [PATCH 10/11] Typo fix

---
 fastai_contrib/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py
index 4507c7c..0cd855e 100644
--- a/fastai_contrib/utils.py
+++ b/fastai_contrib/utils.py
@@ -97,7 +97,7 @@ def get_sentencepiece(path:PathOrStr, trn_path:Path, name:str, rules:ListRules=N
     vocab = Vocab(pickle.load(open(path / 'models' / f'itos_{name}.pkl', 'rb')))
     # We cannot use lambdas or local methods here, since `tok_func` needs to be
     # pickle-able in order to be called in subprocesses when multithread tokenizing
-    tokenizer = Tokenizer(tok_func=SentencepieceTokenizer, lang: str(path / 'models'), pre_rules=rules, post_rules=[])
+    tokenizer = Tokenizer(tok_func=SentencepieceTokenizer, lang=str(path / 'models'), pre_rules=rules, post_rules=[])
     
     clear_cache_directory(path, cache_name)
 

From 8cb867b066d7994c5c533fcc9540f966664318c4 Mon Sep 17 00:00:00 2001
From: "NAUSICAA\\Julian" <eisenjulian@gmail.com>
Date: Tue, 20 Nov 2018 19:21:02 -0300
Subject: [PATCH 11/11] Compatibility with new fastai version

---
 fastai_contrib/utils.py | 2 +-
 ulmfit/pretrain_lm.py   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py
index 0cd855e..3017ffc 100644
--- a/fastai_contrib/utils.py
+++ b/fastai_contrib/utils.py
@@ -97,7 +97,7 @@ def get_sentencepiece(path:PathOrStr, trn_path:Path, name:str, rules:ListRules=N
     vocab = Vocab(pickle.load(open(path / 'models' / f'itos_{name}.pkl', 'rb')))
     # We cannot use lambdas or local methods here, since `tok_func` needs to be
     # pickle-able in order to be called in subprocesses when multithread tokenizing
-    tokenizer = Tokenizer(tok_func=SentencepieceTokenizer, lang=str(path / 'models'), pre_rules=rules, post_rules=[])
+    tokenizer = Tokenizer(tok_func=SentencepieceTokenizer, lang=str(path / 'models'), rules=rules)
     
     clear_cache_directory(path, cache_name)
 
diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py
index fe2d4b7..64b2da2 100644
--- a/ulmfit/pretrain_lm.py
+++ b/ulmfit/pretrain_lm.py
@@ -72,7 +72,7 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo
 
         sp = get_sentencepiece(dir_path, trn_path, name, vocab_size=max_vocab)
 
-        data_lm = TextLMDataBunch.from_csv(dir_path, **sp)
+        data_lm = TextLMDataBunch.from_csv(dir_path, 'train.csv', **sp)
         itos = data_lm.train_ds.vocab.itos
         stoi = data_lm.train_ds.vocab.stoi
     else: