Adding scripts to download and format RCV and Webis CLS X-Language datasets

prepare_cls.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+ROOT=data
+DATA_DIR=$ROOT/cls
+mkdir -p $DATA_DIR/tmp
+echo "Saving data in $DATA_DIR"
+
+if [ ! -d $DATA_DIR/tmp/cls-acl10-unprocessed ]; then
+  wget -c http://www.uni-weimar.de/medien/webis/corpora/corpus-webis-cls-10/cls-acl10-unprocessed.tar.gz  -P $DATA_DIR/tmp
+  tar -xzvf $DATA_DIR/tmp/cls-acl10-unprocessed.tar.gz -C $DATA_DIR/tmp/
+else
+  echo "CLS already exists. Skipping download."
+fi
+
+python ulmfit/postprocess_cls.py --input_dir $DATA_DIR/tmp/cls-acl10-unprocessed --output_dir $DATA_DIR

prepare_rcv.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+if [[ $# -ne 3 ]] ; then
+  echo 'Usage: ./prepare_rcv.sh <RCV_URL> <RCV_USER> <RCV_PASSWORD>'
+  echo 'This dataset has restricted access: apply at https://trec.nist.gov/data/reuters/reuters.html'
+  exit 1
+fi
+
+ROOT="data"
+DATA_DIR="${ROOT}/rcv"
+mkdir -p "${DATA_DIR}/tmp"
+echo "Saving data in $DATA_DIR"
+
+RCV_URL=$1
+RCV_USER=$2
+RCV_PASSWORD=$3
+
+MLDOC=https://github.com/facebookresearch/MLDoc/raw/master
+wget -c $MLDOC/generate_documents.py -P "${DATA_DIR}/tmp"
+
+if [ ! -d "${DATA_DIR}/tmp/RCV2_Multilingual_Corpus" ]; then
+  wget -c --user $RCV_USER --password $RCV_PASSWORD $RCV_URL/rcv2.tar.xz -P "${DATA_DIR}/tmp"  
+  tar xvf "${DATA_DIR}/tmp/rcv2.tar.xz" -C "${DATA_DIR}/tmp/"
+else
+  echo "RCV2 already exists. Skipping download."
+fi
+
+if [ ! -d "${DATA_DIR}/tmp/RCV2_Multilingual_Corpus/english" ]; then
+  wget -c --user $RCV_USER --password $RCV_PASSWORD $RCV_URL/rcv1.tar.xz -P "${DATA_DIR}/tmp"
+  tar xvf "${DATA_DIR}/tmp/rcv1.tar.xz" -C "${DATA_DIR}/tmp/RCV2_Multilingual_Corpus/"
+  mv "${DATA_DIR}/tmp/RCV2_Multilingual_Corpus/rcv1" "${DATA_DIR}/tmp/RCV2_Multilingual_Corpus/english"
+else
+  echo "RCV1 already exists. Skipping download."
+fi
+
+for LANGUAGE in spanish chinese french japanese german italian russian english
+do
+  mkdir -p "${DATA_DIR}/${LANGUAGE}"
+  for FILE_EXT in train.1000 train.2000 train.5000 train.10000 dev test
+  do
+    wget -c $MLDOC/mldoc-indices/$LANGUAGE.$FILE_EXT -P "${DATA_DIR}/tmp"
+
+    python $DATA_DIR/tmp/generate_documents.py \
+        --indices-file $DATA_DIR/tmp/$LANGUAGE.$FILE_EXT \
+        --output-filename $DATA_DIR/tmp/$LANGUAGE.$FILE_EXT.raw \
+        --rcv-dir $DATA_DIR/tmp/RCV2_Multilingual_Corpus/$LANGUAGE
+    python ulmfit/postprocess_rcv.py --input_file $DATA_DIR/tmp/$LANGUAGE.$FILE_EXT.raw \
+        --output_file $DATA_DIR/$LANGUAGE/$FILE_EXT.csv
+  done
+done

ulmfit/postprocess_cls.py

@@ -0,0 +1,23 @@
+import fire
+import pandas as pd
+import os
+from bs4 import BeautifulSoup
+
+def main(input_dir, output_dir):
+  for lang in ['en', 'de', 'fr', 'jp']:
+    for cat in ['dvd', 'music', 'books']:
+      for mode in ['train', 'test']: # , 'unlabeled']:
+        os.makedirs(os.path.join(input_dir, lang), exist_ok=True)
+        with open(os.path.join(input_dir, lang, cat, mode + '.review'), 'r') as f:
+          items = BeautifulSoup(f.read(), features="html.parser").find_all('item')
+          text = [item.find('text').text.strip() for item in items]
+          summary = [item.find('summary').text.strip() for item in items]
+          if mode == 'unlabeled':
+            out = pd.DataFrame({'summary': summary, 'text': text})
+          else:
+            labels = [1 if item.rating.text in ('4.0', '5.0') else 0 for item in items]
+            out = pd.DataFrame({'labels': labels, 'summary': summary, 'text': text})
+          file_name = os.path.join(output_dir, f'{lang}/{cat}.{mode}.csv')
+          out.to_csv(file_name, header=None, index=False)
+
+if __name__ == '__main__': fire.Fire(main)

ulmfit/postprocess_rcv.py

@@ -0,0 +1,12 @@
+import fire
+import pandas as pd
+
+def main(input_file, output_file):
+  df = pd.read_csv(input_file, sep='\t', header=None)
+  unique_labels = sorted(list(df[0].unique()))
+  labels = [unique_labels.index(label) for label in df[0]]
+  texts = [eval(text).decode('utf-8').strip() for text in df[1]]
+  out = pd.DataFrame({'labels': labels, 'texts': texts})
+  out.to_csv(output_file, header=None, index=False)
+
+if __name__ == '__main__': fire.Fire(main)