From 6292412eff6fa98152c01936c876e3c06359ac23 Mon Sep 17 00:00:00 2001 From: "NAUSICAA\\Julian" Date: Sun, 18 Nov 2018 22:29:07 -0300 Subject: [PATCH 1/5] Adding scripts to download and format RCV and Webis CLS X-Language datasets --- prepare_cls.sh | 15 ++++++++++++ prepare_rcv.sh | 50 +++++++++++++++++++++++++++++++++++++++ ulmfit/postprocess_cls.py | 23 ++++++++++++++++++ ulmfit/postprocess_rcv.py | 12 ++++++++++ 4 files changed, 100 insertions(+) create mode 100644 prepare_cls.sh create mode 100644 prepare_rcv.sh create mode 100644 ulmfit/postprocess_cls.py create mode 100644 ulmfit/postprocess_rcv.py diff --git a/prepare_cls.sh b/prepare_cls.sh new file mode 100644 index 0000000..286e4e4 --- /dev/null +++ b/prepare_cls.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +ROOT=data +DATA_DIR=$ROOT/cls +mkdir -p $DATA_DIR/tmp +echo "Saving data in $DATA_DIR" + +if [ ! -d $DATA_DIR/tmp/cls-acl10-unprocessed ]; then + wget -c http://www.uni-weimar.de/medien/webis/corpora/corpus-webis-cls-10/cls-acl10-unprocessed.tar.gz -P $DATA_DIR/tmp + tar -xzvf $DATA_DIR/tmp/cls-acl10-unprocessed.tar.gz -C $DATA_DIR/tmp/ +else + echo "CLS already exists. Skipping download." +fi + +python ulmfit/postprocess_cls.py --input_dir $DATA_DIR/tmp/cls-acl10-unprocessed --output_dir $DATA_DIR diff --git a/prepare_rcv.sh b/prepare_rcv.sh new file mode 100644 index 0000000..acea5d4 --- /dev/null +++ b/prepare_rcv.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +if [[ $# -ne 3 ]] ; then + echo 'Usage: ./prepare_rcv.sh ' + echo 'This dataset has restricted access: apply at https://trec.nist.gov/data/reuters/reuters.html' + exit 1 +fi + +ROOT="data" +DATA_DIR="${ROOT}/rcv" +mkdir -p "${DATA_DIR}/tmp" +echo "Saving data in $DATA_DIR" + +RCV_URL=$1 +RCV_USER=$2 +RCV_PASSWORD=$3 + +MLDOC=https://github.com/facebookresearch/MLDoc/raw/master +wget -c $MLDOC/generate_documents.py -P "${DATA_DIR}/tmp" + +if [ ! -d "${DATA_DIR}/tmp/RCV2_Multilingual_Corpus" ]; then + wget -c --user $RCV_USER --password $RCV_PASSWORD $RCV_URL/rcv2.tar.xz -P "${DATA_DIR}/tmp" + tar xvf "${DATA_DIR}/tmp/rcv2.tar.xz" -C "${DATA_DIR}/tmp/" +else + echo "RCV2 already exists. Skipping download." +fi + +if [ ! -d "${DATA_DIR}/tmp/RCV2_Multilingual_Corpus/english" ]; then + wget -c --user $RCV_USER --password $RCV_PASSWORD $RCV_URL/rcv1.tar.xz -P "${DATA_DIR}/tmp" + tar xvf "${DATA_DIR}/tmp/rcv1.tar.xz" -C "${DATA_DIR}/tmp/RCV2_Multilingual_Corpus/" + mv "${DATA_DIR}/tmp/RCV2_Multilingual_Corpus/rcv1" "${DATA_DIR}/tmp/RCV2_Multilingual_Corpus/english" +else + echo "RCV1 already exists. Skipping download." +fi + +for LANGUAGE in spanish chinese french japanese german italian russian english +do + mkdir -p "${DATA_DIR}/${LANGUAGE}" + for FILE_EXT in train.1000 train.2000 train.5000 train.10000 dev test + do + wget -c $MLDOC/mldoc-indices/$LANGUAGE.$FILE_EXT -P "${DATA_DIR}/tmp" + + python $DATA_DIR/tmp/generate_documents.py \ + --indices-file $DATA_DIR/tmp/$LANGUAGE.$FILE_EXT \ + --output-filename $DATA_DIR/tmp/$LANGUAGE.$FILE_EXT.raw \ + --rcv-dir $DATA_DIR/tmp/RCV2_Multilingual_Corpus/$LANGUAGE + python ulmfit/postprocess_rcv.py --input_file $DATA_DIR/tmp/$LANGUAGE.$FILE_EXT.raw \ + --output_file $DATA_DIR/$LANGUAGE/$FILE_EXT.csv + done +done diff --git a/ulmfit/postprocess_cls.py b/ulmfit/postprocess_cls.py new file mode 100644 index 0000000..ad55a22 --- /dev/null +++ b/ulmfit/postprocess_cls.py @@ -0,0 +1,23 @@ +import fire +import pandas as pd +import os +from bs4 import BeautifulSoup + +def main(input_dir, output_dir): + for lang in ['en', 'de', 'fr', 'jp']: + for cat in ['dvd', 'music', 'books']: + for mode in ['train', 'test']: # , 'unlabeled']: + os.makedirs(os.path.join(input_dir, lang), exist_ok=True) + with open(os.path.join(input_dir, lang, cat, mode + '.review'), 'r') as f: + items = BeautifulSoup(f.read(), features="html.parser").find_all('item') + text = [item.find('text').text.strip() for item in items] + summary = [item.find('summary').text.strip() for item in items] + if mode == 'unlabeled': + out = pd.DataFrame({'summary': summary, 'text': text}) + else: + labels = [1 if item.rating.text in ('4.0', '5.0') else 0 for item in items] + out = pd.DataFrame({'labels': labels, 'summary': summary, 'text': text}) + file_name = os.path.join(output_dir, f'{lang}/{cat}.{mode}.csv') + out.to_csv(file_name, header=None, index=False) + +if __name__ == '__main__': fire.Fire(main) diff --git a/ulmfit/postprocess_rcv.py b/ulmfit/postprocess_rcv.py new file mode 100644 index 0000000..27753a2 --- /dev/null +++ b/ulmfit/postprocess_rcv.py @@ -0,0 +1,12 @@ +import fire +import pandas as pd + +def main(input_file, output_file): + df = pd.read_csv(input_file, sep='\t', header=None) + unique_labels = sorted(list(df[0].unique())) + labels = [unique_labels.index(label) for label in df[0]] + texts = [eval(text).decode('utf-8').strip() for text in df[1]] + out = pd.DataFrame({'labels': labels, 'texts': texts}) + out.to_csv(output_file, header=None, index=False) + +if __name__ == '__main__': fire.Fire(main) From 470f984ca47f3c71cf16391d00e52a0e6fbdf8b1 Mon Sep 17 00:00:00 2001 From: "NAUSICAA\\Julian" Date: Mon, 19 Nov 2018 16:31:38 -0300 Subject: [PATCH 2/5] Script to merge two languages --- ulmfit/merge_langs.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 ulmfit/merge_langs.py diff --git a/ulmfit/merge_langs.py b/ulmfit/merge_langs.py new file mode 100644 index 0000000..7fee7ed --- /dev/null +++ b/ulmfit/merge_langs.py @@ -0,0 +1,35 @@ +""" +Script to merge WikiText files created with `create_wikitext.py`. +""" +import fire +from pathlib import Path +from contextlib import ExitStack + +def merge_wikitext(paths, langs, dest_path, num_sentences): + wiki_paths = [Path(path) for path in paths] + for wiki_path in wiki_paths: + assert wiki_path.exists(), f'Error: {wiki_path} does not exist.' + dest_path = Path(dest_path) + dest_path.mkdir(exist_ok=True) + splits = ['train', 'valid', 'test'] + concat_langs = '-'.join(langs) + for split in splits: + with ExitStack() as stack: + files = [stack.enter_context(open( + wiki_path / f'{lang}.wiki.{split}.tokens', 'r', encoding='utf-8')) + for lang, wiki_path in zip(langs, wiki_paths)] + + output = stack.enter_context(open(dest_path / f'{concat_langs}.wiki.{split}.tokens', 'w', encoding='utf-8')) + done = False + while not done: + for file in files: + lines = [file.readline() for x in range(num_sentences)] + size = len(lines) + lines = [line for line in lines if line] + if len(lines) < size: + done = True + for line in lines: + output.write(line) + +if __name__ == '__main__': + fire.Fire(merge_wikitext) \ No newline at end of file From 0ecc6342a6071b40ba5d4b0c0de52a83e4b321a0 Mon Sep 17 00:00:00 2001 From: "NAUSICAA\\Julian" Date: Tue, 20 Nov 2018 00:38:44 -0300 Subject: [PATCH 3/5] Fixes when running sentence piece end to end --- fastai_contrib/utils.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py index 9004132..43c98aa 100644 --- a/fastai_contrib/utils.py +++ b/fastai_contrib/utils.py @@ -61,16 +61,17 @@ def get_sentencepiece(path:PathOrStr, trn_path:Path, name:str, rules:ListRules=N raise Exception('sentencepiece module is missing: run `pip install sentencepiece`') path = pathlib.Path(path) + cache_name = 'tmp' + os.makedirs(path / cache_name, exist_ok=True) os.makedirs(path / 'models', exist_ok=True) rules = rules if rules else default_rules - cache_name = 'tmp' # load the text frmo the train tokens file text = [line.rstrip('\n') for line in open(trn_path)] text = list(filter(None, text)) - if not os.path.isfile(path / 'models' / 'spm.model') or not os.path.isfile(path / f'itos_{name}.pkl'): + if not os.path.isfile(path / 'models' / 'spm.model') or not os.path.isfile(path / 'models' / f'itos_{name}.pkl'): raw_text = reduce(lambda t, rule: rule(t), rules, '\n'.join(text)) raw_text_path = path / cache_name / 'all_text.txt' with open(raw_text_path, 'w') as f: @@ -88,11 +89,12 @@ def get_sentencepiece(path:PathOrStr, trn_path:Path, name:str, rules:ListRules=N vocab[0] = UNK vocab[pad_idx] = PAD - pickle.dump(vocab, open(path / 'models'/ f'itos_{name}.pkl', 'wb')) + pickle.dump(vocab, open(path / 'models' / f'itos_{name}.pkl', 'wb')) - vocab = Vocab(pickle.load(open(path / 'models'/ f'itos_{name}.pkl', 'rb'))) - spt = SentencepieceTokenizer(path) - tokenizer = Tokenizer(tok_func=lambda lang: spt, rules=rules) + vocab = Vocab(pickle.load(open(path / 'models' / f'itos_{name}.pkl', 'rb'))) + # We cannot use lambdas or local methods here, since `tok_func` needs to be + # pickle-able in order to be called in subprocesses when multithread tokenizing + tokenizer = Tokenizer(tok_func=SentencepieceTokenizer, lang: str(path / 'models'), rules=rules) clear_cache_directory(path, cache_name) From 3b5629c0c5cfdc5e43bf8bd12e4f72b9a500021d Mon Sep 17 00:00:00 2001 From: "NAUSICAA\\Julian" Date: Tue, 20 Nov 2018 00:46:04 -0300 Subject: [PATCH 4/5] Removing files --- prepare_cls.sh | 15 ------------ prepare_rcv.sh | 50 --------------------------------------- ulmfit/merge_langs.py | 35 --------------------------- ulmfit/postprocess_cls.py | 23 ------------------ ulmfit/postprocess_rcv.py | 12 ---------- 5 files changed, 135 deletions(-) delete mode 100644 prepare_cls.sh delete mode 100644 prepare_rcv.sh delete mode 100644 ulmfit/merge_langs.py delete mode 100644 ulmfit/postprocess_cls.py delete mode 100644 ulmfit/postprocess_rcv.py diff --git a/prepare_cls.sh b/prepare_cls.sh deleted file mode 100644 index 286e4e4..0000000 --- a/prepare_cls.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -ROOT=data -DATA_DIR=$ROOT/cls -mkdir -p $DATA_DIR/tmp -echo "Saving data in $DATA_DIR" - -if [ ! -d $DATA_DIR/tmp/cls-acl10-unprocessed ]; then - wget -c http://www.uni-weimar.de/medien/webis/corpora/corpus-webis-cls-10/cls-acl10-unprocessed.tar.gz -P $DATA_DIR/tmp - tar -xzvf $DATA_DIR/tmp/cls-acl10-unprocessed.tar.gz -C $DATA_DIR/tmp/ -else - echo "CLS already exists. Skipping download." -fi - -python ulmfit/postprocess_cls.py --input_dir $DATA_DIR/tmp/cls-acl10-unprocessed --output_dir $DATA_DIR diff --git a/prepare_rcv.sh b/prepare_rcv.sh deleted file mode 100644 index acea5d4..0000000 --- a/prepare_rcv.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env bash - -if [[ $# -ne 3 ]] ; then - echo 'Usage: ./prepare_rcv.sh ' - echo 'This dataset has restricted access: apply at https://trec.nist.gov/data/reuters/reuters.html' - exit 1 -fi - -ROOT="data" -DATA_DIR="${ROOT}/rcv" -mkdir -p "${DATA_DIR}/tmp" -echo "Saving data in $DATA_DIR" - -RCV_URL=$1 -RCV_USER=$2 -RCV_PASSWORD=$3 - -MLDOC=https://github.com/facebookresearch/MLDoc/raw/master -wget -c $MLDOC/generate_documents.py -P "${DATA_DIR}/tmp" - -if [ ! -d "${DATA_DIR}/tmp/RCV2_Multilingual_Corpus" ]; then - wget -c --user $RCV_USER --password $RCV_PASSWORD $RCV_URL/rcv2.tar.xz -P "${DATA_DIR}/tmp" - tar xvf "${DATA_DIR}/tmp/rcv2.tar.xz" -C "${DATA_DIR}/tmp/" -else - echo "RCV2 already exists. Skipping download." -fi - -if [ ! -d "${DATA_DIR}/tmp/RCV2_Multilingual_Corpus/english" ]; then - wget -c --user $RCV_USER --password $RCV_PASSWORD $RCV_URL/rcv1.tar.xz -P "${DATA_DIR}/tmp" - tar xvf "${DATA_DIR}/tmp/rcv1.tar.xz" -C "${DATA_DIR}/tmp/RCV2_Multilingual_Corpus/" - mv "${DATA_DIR}/tmp/RCV2_Multilingual_Corpus/rcv1" "${DATA_DIR}/tmp/RCV2_Multilingual_Corpus/english" -else - echo "RCV1 already exists. Skipping download." -fi - -for LANGUAGE in spanish chinese french japanese german italian russian english -do - mkdir -p "${DATA_DIR}/${LANGUAGE}" - for FILE_EXT in train.1000 train.2000 train.5000 train.10000 dev test - do - wget -c $MLDOC/mldoc-indices/$LANGUAGE.$FILE_EXT -P "${DATA_DIR}/tmp" - - python $DATA_DIR/tmp/generate_documents.py \ - --indices-file $DATA_DIR/tmp/$LANGUAGE.$FILE_EXT \ - --output-filename $DATA_DIR/tmp/$LANGUAGE.$FILE_EXT.raw \ - --rcv-dir $DATA_DIR/tmp/RCV2_Multilingual_Corpus/$LANGUAGE - python ulmfit/postprocess_rcv.py --input_file $DATA_DIR/tmp/$LANGUAGE.$FILE_EXT.raw \ - --output_file $DATA_DIR/$LANGUAGE/$FILE_EXT.csv - done -done diff --git a/ulmfit/merge_langs.py b/ulmfit/merge_langs.py deleted file mode 100644 index 7fee7ed..0000000 --- a/ulmfit/merge_langs.py +++ /dev/null @@ -1,35 +0,0 @@ -""" -Script to merge WikiText files created with `create_wikitext.py`. -""" -import fire -from pathlib import Path -from contextlib import ExitStack - -def merge_wikitext(paths, langs, dest_path, num_sentences): - wiki_paths = [Path(path) for path in paths] - for wiki_path in wiki_paths: - assert wiki_path.exists(), f'Error: {wiki_path} does not exist.' - dest_path = Path(dest_path) - dest_path.mkdir(exist_ok=True) - splits = ['train', 'valid', 'test'] - concat_langs = '-'.join(langs) - for split in splits: - with ExitStack() as stack: - files = [stack.enter_context(open( - wiki_path / f'{lang}.wiki.{split}.tokens', 'r', encoding='utf-8')) - for lang, wiki_path in zip(langs, wiki_paths)] - - output = stack.enter_context(open(dest_path / f'{concat_langs}.wiki.{split}.tokens', 'w', encoding='utf-8')) - done = False - while not done: - for file in files: - lines = [file.readline() for x in range(num_sentences)] - size = len(lines) - lines = [line for line in lines if line] - if len(lines) < size: - done = True - for line in lines: - output.write(line) - -if __name__ == '__main__': - fire.Fire(merge_wikitext) \ No newline at end of file diff --git a/ulmfit/postprocess_cls.py b/ulmfit/postprocess_cls.py deleted file mode 100644 index ad55a22..0000000 --- a/ulmfit/postprocess_cls.py +++ /dev/null @@ -1,23 +0,0 @@ -import fire -import pandas as pd -import os -from bs4 import BeautifulSoup - -def main(input_dir, output_dir): - for lang in ['en', 'de', 'fr', 'jp']: - for cat in ['dvd', 'music', 'books']: - for mode in ['train', 'test']: # , 'unlabeled']: - os.makedirs(os.path.join(input_dir, lang), exist_ok=True) - with open(os.path.join(input_dir, lang, cat, mode + '.review'), 'r') as f: - items = BeautifulSoup(f.read(), features="html.parser").find_all('item') - text = [item.find('text').text.strip() for item in items] - summary = [item.find('summary').text.strip() for item in items] - if mode == 'unlabeled': - out = pd.DataFrame({'summary': summary, 'text': text}) - else: - labels = [1 if item.rating.text in ('4.0', '5.0') else 0 for item in items] - out = pd.DataFrame({'labels': labels, 'summary': summary, 'text': text}) - file_name = os.path.join(output_dir, f'{lang}/{cat}.{mode}.csv') - out.to_csv(file_name, header=None, index=False) - -if __name__ == '__main__': fire.Fire(main) diff --git a/ulmfit/postprocess_rcv.py b/ulmfit/postprocess_rcv.py deleted file mode 100644 index 27753a2..0000000 --- a/ulmfit/postprocess_rcv.py +++ /dev/null @@ -1,12 +0,0 @@ -import fire -import pandas as pd - -def main(input_file, output_file): - df = pd.read_csv(input_file, sep='\t', header=None) - unique_labels = sorted(list(df[0].unique())) - labels = [unique_labels.index(label) for label in df[0]] - texts = [eval(text).decode('utf-8').strip() for text in df[1]] - out = pd.DataFrame({'labels': labels, 'texts': texts}) - out.to_csv(output_file, header=None, index=False) - -if __name__ == '__main__': fire.Fire(main) From 40a29903228b53db28d41deee0a87c117b3e8f95 Mon Sep 17 00:00:00 2001 From: "NAUSICAA\\Julian" Date: Tue, 20 Nov 2018 09:06:03 -0300 Subject: [PATCH 5/5] Add suggested changes to rules system --- fastai_contrib/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py index 101bce3..4507c7c 100644 --- a/fastai_contrib/utils.py +++ b/fastai_contrib/utils.py @@ -67,7 +67,7 @@ def get_sentencepiece(path:PathOrStr, trn_path:Path, name:str, rules:ListRules=N cache_name = 'tmp' os.makedirs(path / cache_name, exist_ok=True) os.makedirs(path / 'models', exist_ok=True) - rules = rules if rules else None + rules = rules if rules is not None else [] # load the text frmo the train tokens file @@ -97,7 +97,7 @@ def get_sentencepiece(path:PathOrStr, trn_path:Path, name:str, rules:ListRules=N vocab = Vocab(pickle.load(open(path / 'models' / f'itos_{name}.pkl', 'rb'))) # We cannot use lambdas or local methods here, since `tok_func` needs to be # pickle-able in order to be called in subprocesses when multithread tokenizing - tokenizer = Tokenizer(tok_func=SentencepieceTokenizer, lang: str(path / 'models'), rules=rules) + tokenizer = Tokenizer(tok_func=SentencepieceTokenizer, lang: str(path / 'models'), pre_rules=rules, post_rules=[]) clear_cache_directory(path, cache_name)