From e657423e7f1a0470c004f0df42eee7ff96251ba8 Mon Sep 17 00:00:00 2001 From: "NAUSICAA\\Julian" Date: Wed, 13 Feb 2019 10:41:24 -0300 Subject: [PATCH] CLS Download Script --- prepare_cls.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 prepare_cls.py diff --git a/prepare_cls.py b/prepare_cls.py new file mode 100644 index 0000000..12b6905 --- /dev/null +++ b/prepare_cls.py @@ -0,0 +1,24 @@ +import fire +import urllib.request +from pathlib import Path + +lang_codes = ['fr', 'jp', 'en', 'de'] + +def fetch_cls(url_prefix, cls_path="data/cls"): + """ Fetch CLS from server using basic auth + url_prefix should point to CLS stored as follow + "https://user:passwd@server/path/[en|fr|de|jp]/[dvd|music|books].[test|train|unlabeled].csv" + """ + def fetch(url, CLS): + CLS.parent.mkdir(parents=True, exist_ok=True) + print("fetching", url, CLS) + urllib.request.urlretrieve(url, CLS) + for code in lang_codes: + for category in ['books', 'music', 'dvd']: + dir = Path(cls_path)/f'{code}-{category}' + fetch(f"{url_prefix}/{code}/{category}/train.csv", dir / f"{code}.train.csv") + fetch(f"{url_prefix}/{code}/{category}/test.csv", dir / f"{code}.test.csv") + fetch(f"{url_prefix}/{code}/{category}/unlabeled.csv", dir / f"{code}.unsup.csv") + +if __name__ == "__main__": + fire.Fire(fetch_cls)