Removing files

prepare_cls.sh

@@ -1,15 +0,0 @@
-#!/usr/bin/env bash
-
-ROOT=data
-DATA_DIR=$ROOT/cls
-mkdir -p $DATA_DIR/tmp
-echo "Saving data in $DATA_DIR"
-
-if [ ! -d $DATA_DIR/tmp/cls-acl10-unprocessed ]; then
-  wget -c http://www.uni-weimar.de/medien/webis/corpora/corpus-webis-cls-10/cls-acl10-unprocessed.tar.gz  -P $DATA_DIR/tmp
-  tar -xzvf $DATA_DIR/tmp/cls-acl10-unprocessed.tar.gz -C $DATA_DIR/tmp/
-else
-  echo "CLS already exists. Skipping download."
-fi
-
-python ulmfit/postprocess_cls.py --input_dir $DATA_DIR/tmp/cls-acl10-unprocessed --output_dir $DATA_DIR

prepare_rcv.sh

@@ -1,50 +0,0 @@
-#!/usr/bin/env bash
-
-if [[ $# -ne 3 ]] ; then
-  echo 'Usage: ./prepare_rcv.sh <RCV_URL> <RCV_USER> <RCV_PASSWORD>'
-  echo 'This dataset has restricted access: apply at https://trec.nist.gov/data/reuters/reuters.html'
-  exit 1
-fi
-
-ROOT="data"
-DATA_DIR="${ROOT}/rcv"
-mkdir -p "${DATA_DIR}/tmp"
-echo "Saving data in $DATA_DIR"
-
-RCV_URL=$1
-RCV_USER=$2
-RCV_PASSWORD=$3
-
-MLDOC=https://github.com/facebookresearch/MLDoc/raw/master
-wget -c $MLDOC/generate_documents.py -P "${DATA_DIR}/tmp"
-
-if [ ! -d "${DATA_DIR}/tmp/RCV2_Multilingual_Corpus" ]; then
-  wget -c --user $RCV_USER --password $RCV_PASSWORD $RCV_URL/rcv2.tar.xz -P "${DATA_DIR}/tmp"  
-  tar xvf "${DATA_DIR}/tmp/rcv2.tar.xz" -C "${DATA_DIR}/tmp/"
-else
-  echo "RCV2 already exists. Skipping download."
-fi
-
-if [ ! -d "${DATA_DIR}/tmp/RCV2_Multilingual_Corpus/english" ]; then
-  wget -c --user $RCV_USER --password $RCV_PASSWORD $RCV_URL/rcv1.tar.xz -P "${DATA_DIR}/tmp"
-  tar xvf "${DATA_DIR}/tmp/rcv1.tar.xz" -C "${DATA_DIR}/tmp/RCV2_Multilingual_Corpus/"
-  mv "${DATA_DIR}/tmp/RCV2_Multilingual_Corpus/rcv1" "${DATA_DIR}/tmp/RCV2_Multilingual_Corpus/english"
-else
-  echo "RCV1 already exists. Skipping download."
-fi
-
-for LANGUAGE in spanish chinese french japanese german italian russian english
-do
-  mkdir -p "${DATA_DIR}/${LANGUAGE}"
-  for FILE_EXT in train.1000 train.2000 train.5000 train.10000 dev test
-  do
-    wget -c $MLDOC/mldoc-indices/$LANGUAGE.$FILE_EXT -P "${DATA_DIR}/tmp"
-
-    python $DATA_DIR/tmp/generate_documents.py \
-        --indices-file $DATA_DIR/tmp/$LANGUAGE.$FILE_EXT \
-        --output-filename $DATA_DIR/tmp/$LANGUAGE.$FILE_EXT.raw \
-        --rcv-dir $DATA_DIR/tmp/RCV2_Multilingual_Corpus/$LANGUAGE
-    python ulmfit/postprocess_rcv.py --input_file $DATA_DIR/tmp/$LANGUAGE.$FILE_EXT.raw \
-        --output_file $DATA_DIR/$LANGUAGE/$FILE_EXT.csv
-  done
-done

ulmfit/merge_langs.py

@@ -1,35 +0,0 @@
-"""
-Script to merge WikiText files created with `create_wikitext.py`.
-"""
-import fire
-from pathlib import Path
-from contextlib import ExitStack
-
-def merge_wikitext(paths, langs, dest_path, num_sentences):
-    wiki_paths = [Path(path) for path in paths]
-    for wiki_path in wiki_paths:
-        assert wiki_path.exists(), f'Error: {wiki_path} does not exist.'
-    dest_path = Path(dest_path)
-    dest_path.mkdir(exist_ok=True)
-    splits = ['train', 'valid', 'test']
-    concat_langs = '-'.join(langs)
-    for split in splits:
-        with ExitStack() as stack:
-            files = [stack.enter_context(open(
-                wiki_path / f'{lang}.wiki.{split}.tokens', 'r', encoding='utf-8')) 
-                for lang, wiki_path in zip(langs, wiki_paths)]
-            
-            output = stack.enter_context(open(dest_path / f'{concat_langs}.wiki.{split}.tokens', 'w', encoding='utf-8'))
-            done = False
-            while not done:
-                for file in files:
-                    lines = [file.readline() for x in range(num_sentences)]
-                    size = len(lines)
-                    lines = [line for line in lines if line]
-                    if len(lines) < size:
-                        done = True
-                    for line in lines:
-                        output.write(line)
-
-if __name__ == '__main__':
-    fire.Fire(merge_wikitext)

ulmfit/postprocess_cls.py

@@ -1,23 +0,0 @@
-import fire
-import pandas as pd
-import os
-from bs4 import BeautifulSoup
-
-def main(input_dir, output_dir):
-  for lang in ['en', 'de', 'fr', 'jp']:
-    for cat in ['dvd', 'music', 'books']:
-      for mode in ['train', 'test']: # , 'unlabeled']:
-        os.makedirs(os.path.join(input_dir, lang), exist_ok=True)
-        with open(os.path.join(input_dir, lang, cat, mode + '.review'), 'r') as f:
-          items = BeautifulSoup(f.read(), features="html.parser").find_all('item')
-          text = [item.find('text').text.strip() for item in items]
-          summary = [item.find('summary').text.strip() for item in items]
-          if mode == 'unlabeled':
-            out = pd.DataFrame({'summary': summary, 'text': text})
-          else:
-            labels = [1 if item.rating.text in ('4.0', '5.0') else 0 for item in items]
-            out = pd.DataFrame({'labels': labels, 'summary': summary, 'text': text})
-          file_name = os.path.join(output_dir, f'{lang}/{cat}.{mode}.csv')
-          out.to_csv(file_name, header=None, index=False)
-
-if __name__ == '__main__': fire.Fire(main)

ulmfit/postprocess_rcv.py

@@ -1,12 +0,0 @@
-import fire
-import pandas as pd
-
-def main(input_file, output_file):
-  df = pd.read_csv(input_file, sep='\t', header=None)
-  unique_labels = sorted(list(df[0].unique()))
-  labels = [unique_labels.index(label) for label in df[0]]
-  texts = [eval(text).decode('utf-8').strip() for text in df[1]]
-  out = pd.DataFrame({'labels': labels, 'texts': texts})
-  out.to_csv(output_file, header=None, index=False)
-
-if __name__ == '__main__': fire.Fire(main)