diff --git a/prepare_cls.sh b/prepare_cls.sh new file mode 100644 index 0000000..286e4e4 --- /dev/null +++ b/prepare_cls.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +ROOT=data +DATA_DIR=$ROOT/cls +mkdir -p $DATA_DIR/tmp +echo "Saving data in $DATA_DIR" + +if [ ! -d $DATA_DIR/tmp/cls-acl10-unprocessed ]; then + wget -c http://www.uni-weimar.de/medien/webis/corpora/corpus-webis-cls-10/cls-acl10-unprocessed.tar.gz -P $DATA_DIR/tmp + tar -xzvf $DATA_DIR/tmp/cls-acl10-unprocessed.tar.gz -C $DATA_DIR/tmp/ +else + echo "CLS already exists. Skipping download." +fi + +python ulmfit/postprocess_cls.py --input_dir $DATA_DIR/tmp/cls-acl10-unprocessed --output_dir $DATA_DIR diff --git a/prepare_rcv.sh b/prepare_rcv.sh new file mode 100644 index 0000000..acea5d4 --- /dev/null +++ b/prepare_rcv.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +if [[ $# -ne 3 ]] ; then + echo 'Usage: ./prepare_rcv.sh ' + echo 'This dataset has restricted access: apply at https://trec.nist.gov/data/reuters/reuters.html' + exit 1 +fi + +ROOT="data" +DATA_DIR="${ROOT}/rcv" +mkdir -p "${DATA_DIR}/tmp" +echo "Saving data in $DATA_DIR" + +RCV_URL=$1 +RCV_USER=$2 +RCV_PASSWORD=$3 + +MLDOC=https://github.com/facebookresearch/MLDoc/raw/master +wget -c $MLDOC/generate_documents.py -P "${DATA_DIR}/tmp" + +if [ ! -d "${DATA_DIR}/tmp/RCV2_Multilingual_Corpus" ]; then + wget -c --user $RCV_USER --password $RCV_PASSWORD $RCV_URL/rcv2.tar.xz -P "${DATA_DIR}/tmp" + tar xvf "${DATA_DIR}/tmp/rcv2.tar.xz" -C "${DATA_DIR}/tmp/" +else + echo "RCV2 already exists. Skipping download." +fi + +if [ ! -d "${DATA_DIR}/tmp/RCV2_Multilingual_Corpus/english" ]; then + wget -c --user $RCV_USER --password $RCV_PASSWORD $RCV_URL/rcv1.tar.xz -P "${DATA_DIR}/tmp" + tar xvf "${DATA_DIR}/tmp/rcv1.tar.xz" -C "${DATA_DIR}/tmp/RCV2_Multilingual_Corpus/" + mv "${DATA_DIR}/tmp/RCV2_Multilingual_Corpus/rcv1" "${DATA_DIR}/tmp/RCV2_Multilingual_Corpus/english" +else + echo "RCV1 already exists. Skipping download." +fi + +for LANGUAGE in spanish chinese french japanese german italian russian english +do + mkdir -p "${DATA_DIR}/${LANGUAGE}" + for FILE_EXT in train.1000 train.2000 train.5000 train.10000 dev test + do + wget -c $MLDOC/mldoc-indices/$LANGUAGE.$FILE_EXT -P "${DATA_DIR}/tmp" + + python $DATA_DIR/tmp/generate_documents.py \ + --indices-file $DATA_DIR/tmp/$LANGUAGE.$FILE_EXT \ + --output-filename $DATA_DIR/tmp/$LANGUAGE.$FILE_EXT.raw \ + --rcv-dir $DATA_DIR/tmp/RCV2_Multilingual_Corpus/$LANGUAGE + python ulmfit/postprocess_rcv.py --input_file $DATA_DIR/tmp/$LANGUAGE.$FILE_EXT.raw \ + --output_file $DATA_DIR/$LANGUAGE/$FILE_EXT.csv + done +done diff --git a/ulmfit/postprocess_cls.py b/ulmfit/postprocess_cls.py new file mode 100644 index 0000000..ad55a22 --- /dev/null +++ b/ulmfit/postprocess_cls.py @@ -0,0 +1,23 @@ +import fire +import pandas as pd +import os +from bs4 import BeautifulSoup + +def main(input_dir, output_dir): + for lang in ['en', 'de', 'fr', 'jp']: + for cat in ['dvd', 'music', 'books']: + for mode in ['train', 'test']: # , 'unlabeled']: + os.makedirs(os.path.join(input_dir, lang), exist_ok=True) + with open(os.path.join(input_dir, lang, cat, mode + '.review'), 'r') as f: + items = BeautifulSoup(f.read(), features="html.parser").find_all('item') + text = [item.find('text').text.strip() for item in items] + summary = [item.find('summary').text.strip() for item in items] + if mode == 'unlabeled': + out = pd.DataFrame({'summary': summary, 'text': text}) + else: + labels = [1 if item.rating.text in ('4.0', '5.0') else 0 for item in items] + out = pd.DataFrame({'labels': labels, 'summary': summary, 'text': text}) + file_name = os.path.join(output_dir, f'{lang}/{cat}.{mode}.csv') + out.to_csv(file_name, header=None, index=False) + +if __name__ == '__main__': fire.Fire(main) diff --git a/ulmfit/postprocess_rcv.py b/ulmfit/postprocess_rcv.py new file mode 100644 index 0000000..27753a2 --- /dev/null +++ b/ulmfit/postprocess_rcv.py @@ -0,0 +1,12 @@ +import fire +import pandas as pd + +def main(input_file, output_file): + df = pd.read_csv(input_file, sep='\t', header=None) + unique_labels = sorted(list(df[0].unique())) + labels = [unique_labels.index(label) for label in df[0]] + texts = [eval(text).decode('utf-8').strip() for text in df[1]] + out = pd.DataFrame({'labels': labels, 'texts': texts}) + out.to_csv(output_file, header=None, index=False) + +if __name__ == '__main__': fire.Fire(main)