From fc74fdb0a78e4875ccbe8f4ea6e878001a72232f Mon Sep 17 00:00:00 2001 From: Scott Sanderson Date: Tue, 10 May 2016 20:48:33 -0400 Subject: [PATCH 1/6] BUG: Don't print a tuple on bundles in py2. --- zipline/__main__.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/zipline/__main__.py b/zipline/__main__.py index b1f8d394..d8171130 100644 --- a/zipline/__main__.py +++ b/zipline/__main__.py @@ -360,14 +360,11 @@ def bundles(): raise ingestions = [] - print( - '\n'.join( - '%s %s' % (bundle, line) - for line in ( - ingestions if ingestions else ('',) - ) - ), - ) + # If we got no ingestions, either because the directory didn't exist or + # becuase there were no entries, print a single message indicating that + # no ingestions have yet been made. + for timestamp in ingestions or [""]: + print("%s %s" % (bundle, timestamp)) if __name__ == '__main__': From a642ce3ae45110307a3080f10458947625907d43 Mon Sep 17 00:00:00 2001 From: Scott Sanderson Date: Tue, 10 May 2016 20:50:54 -0400 Subject: [PATCH 2/6] ENH: Add progressbar for quantopian-quandl download. --- zipline/data/bundles/quandl.py | 68 ++++++++++++++++++++++++++++++++-- 1 file changed, 65 insertions(+), 3 deletions(-) diff --git a/zipline/data/bundles/quandl.py b/zipline/data/bundles/quandl.py index 8636bc54..3ebb5c07 100644 --- a/zipline/data/bundles/quandl.py +++ b/zipline/data/bundles/quandl.py @@ -7,8 +7,10 @@ from itertools import count import tarfile from time import time, sleep +from click import progressbar from logbook import Logger import pandas as pd +import requests from six.moves.urllib.parse import urlencode from six.moves.urllib.request import urlopen @@ -307,6 +309,58 @@ def quandl_bundle(environ, QUANTOPIAN_QUANDL_URL = ( 'https://s3.amazonaws.com/quantopian-public-zipline-data/quandl' ) +ONE_MEGABYTE = 1024 * 1024 + + +def download_with_progress(url, chunk_size, **progress_kwargs): + """ + Download streaming data from a URL, printing progress information to the + terminal. + + Parameters + ---------- + url : str + A URL that can be understood by ``requests.get``. + chunk_size : int + Number of bytes to read at a time from requests. + **progress_kwargs + Forwarded to click.progressbar. + + Returns + ------- + data : BytesIO + A BytesIO containing the downloaded data. + """ + resp = requests.get(url, stream=True) + resp.raise_for_status() + + total_size = int(resp.headers['content-length']) + data = BytesIO() + with progressbar(length=total_size, **progress_kwargs) as pbar: + for chunk in resp.iter_content(chunk_size=chunk_size): + data.write(chunk) + pbar.update(len(chunk)) + + return data + + +def download_without_progress(url): + """ + Download data from a URL, returning a BytesIO containing the loaded data. + + Parameters + ---------- + url : str + A URL that can be understood by ``requests.get``. + + Returns + ------- + data : BytesIO + A BytesIO containing the downloaded data. + """ + resp = requests.get(QUANTOPIAN_QUANDL_URL) + resp.raise_for_status() + return BytesIO(resp.content) @bundles.register('quantopian-quandl', create_writers=False) @@ -320,8 +374,16 @@ def quantopian_quandl_bundle(environ, show_progress, output_dir): if show_progress: - print('Downloading quandl data. This can take about 1 minute.') + data = download_with_progress( + QUANTOPIAN_QUANDL_URL, + chunk_size=ONE_MEGABYTE, + label="Downloading Bundle: quantopian-quandl", + ) + else: + data = download_without_progress(QUANTOPIAN_QUANDL_URL) + # use closing for py2 compat - with closing(urlopen(QUANTOPIAN_QUANDL_URL)) as f, \ - tarfile.open('r', fileobj=BytesIO(f.read())) as tar: + with tarfile.open('r', fileobj=data) as tar: + if show_progress: + print("Writing data to %s." % output_dir) tar.extractall(output_dir) From 4ab8ea7c29fe320a01e86e0c411c9ed7d44f70ea Mon Sep 17 00:00:00 2001 From: Scott Sanderson Date: Wed, 11 May 2016 14:53:24 -0400 Subject: [PATCH 3/6] STY: flake8. --- zipline/data/bundles/quandl.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/zipline/data/bundles/quandl.py b/zipline/data/bundles/quandl.py index 3ebb5c07..bfb486ed 100644 --- a/zipline/data/bundles/quandl.py +++ b/zipline/data/bundles/quandl.py @@ -1,7 +1,6 @@ """ Module for building a complete daily dataset from Quandl's WIKI dataset. """ -from contextlib import closing from io import BytesIO from itertools import count import tarfile @@ -12,7 +11,6 @@ from logbook import Logger import pandas as pd import requests from six.moves.urllib.parse import urlencode -from six.moves.urllib.request import urlopen from . import core as bundles from zipline.utils.cli import maybe_show_progress From 3a2c0e4ec8719c68789342e81ec3a00187294295 Mon Sep 17 00:00:00 2001 From: Scott Sanderson Date: Wed, 11 May 2016 14:54:36 -0400 Subject: [PATCH 4/6] DOC: Remove old comment. --- zipline/data/bundles/quandl.py | 1 - 1 file changed, 1 deletion(-) diff --git a/zipline/data/bundles/quandl.py b/zipline/data/bundles/quandl.py index bfb486ed..e670e1c1 100644 --- a/zipline/data/bundles/quandl.py +++ b/zipline/data/bundles/quandl.py @@ -380,7 +380,6 @@ def quantopian_quandl_bundle(environ, else: data = download_without_progress(QUANTOPIAN_QUANDL_URL) - # use closing for py2 compat with tarfile.open('r', fileobj=data) as tar: if show_progress: print("Writing data to %s." % output_dir) From 7dbabc013e2910eb194360f186475a6b6d4e22a6 Mon Sep 17 00:00:00 2001 From: Scott Sanderson Date: Wed, 11 May 2016 14:54:48 -0400 Subject: [PATCH 5/6] BUG: Use the URL actually passed. And move constants closer to where they're used. --- zipline/data/bundles/quandl.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/zipline/data/bundles/quandl.py b/zipline/data/bundles/quandl.py index e670e1c1..edf6d049 100644 --- a/zipline/data/bundles/quandl.py +++ b/zipline/data/bundles/quandl.py @@ -304,12 +304,6 @@ def quandl_bundle(environ, ) -QUANTOPIAN_QUANDL_URL = ( - 'https://s3.amazonaws.com/quantopian-public-zipline-data/quandl' -) -ONE_MEGABYTE = 1024 * 1024 - - def download_with_progress(url, chunk_size, **progress_kwargs): """ Download streaming data from a URL, printing progress information to the @@ -356,11 +350,17 @@ def download_without_progress(url): data : BytesIO A BytesIO containing the downloaded data. """ - resp = requests.get(QUANTOPIAN_QUANDL_URL) + resp = requests.get(url) resp.raise_for_status() return BytesIO(resp.content) +QUANTOPIAN_QUANDL_URL = ( + 'https://s3.amazonaws.com/quantopian-public-zipline-data/quandl' +) +ONE_MEGABYTE = 1024 * 1024 + + @bundles.register('quantopian-quandl', create_writers=False) def quantopian_quandl_bundle(environ, asset_db_writer, From a46787711c7dd262dfef350025f7c5f526b43dd9 Mon Sep 17 00:00:00 2001 From: Scott Sanderson Date: Wed, 11 May 2016 14:57:06 -0400 Subject: [PATCH 6/6] DOC: Spell 'because' correctly. --- zipline/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zipline/__main__.py b/zipline/__main__.py index d8171130..2f303f26 100644 --- a/zipline/__main__.py +++ b/zipline/__main__.py @@ -361,7 +361,7 @@ def bundles(): ingestions = [] # If we got no ingestions, either because the directory didn't exist or - # becuase there were no entries, print a single message indicating that + # because there were no entries, print a single message indicating that # no ingestions have yet been made. for timestamp in ingestions or [""]: print("%s %s" % (bundle, timestamp))