diff --git a/tests/data/bundles/test_core.py b/tests/data/bundles/test_core.py index d85c437e..285916cc 100644 --- a/tests/data/bundles/test_core.py +++ b/tests/data/bundles/test_core.py @@ -6,7 +6,7 @@ from toolz import valmap import toolz.curried.operator as op from zipline.assets.synthetic import make_simple_equity_info -from zipline.data.bundles import UnknownBundle +from zipline.data.bundles import UnknownBundle, from_bundle_ingest_dirname from zipline.data.bundles.core import _make_bundle_core from zipline.lib.adjustment import Float64Multiply from zipline.pipeline.loaders.synthetic import ( @@ -362,7 +362,7 @@ class BundleCoreTestCase(WithInstanceTmpDir, ZiplineTestCase): @staticmethod def _ts_of_run(run): - return pd.Timestamp(int(run.rsplit(os.path.sep, 1)[-1])) + return from_bundle_ingest_dirname(run.rsplit(os.path.sep, 1)[-1]) def test_clean_before_after(self): first = self._empty_ingest() diff --git a/tests/resources/example_data.tar.gz b/tests/resources/example_data.tar.gz index 7957345e..8e4cd103 100644 Binary files a/tests/resources/example_data.tar.gz and b/tests/resources/example_data.tar.gz differ diff --git a/zipline/__main__.py b/zipline/__main__.py index db496e86..b1f8d394 100644 --- a/zipline/__main__.py +++ b/zipline/__main__.py @@ -1,3 +1,4 @@ +import errno import os from functools import wraps @@ -347,12 +348,18 @@ def bundles(): """List all of the available data bundles. """ for bundle in sorted(bundles_module.bundles.keys()): - ingestions = sorted( - (str(pd.Timestamp(int(ing))) - for ing in os.listdir(pth.data_path([bundle])) - if not pth.hidden(ing)), - reverse=True, - ) + try: + ingestions = sorted( + (str(bundles_module.from_bundle_ingest_dirname(ing)) + for ing in os.listdir(pth.data_path([bundle])) + if not pth.hidden(ing)), + reverse=True, + ) + except IOError as e: + if e.errno != errno.ENOENT: + raise + ingestions = [] + print( '\n'.join( '%s %s' % (bundle, line) diff --git a/zipline/data/bundles/__init__.py b/zipline/data/bundles/__init__.py index 68bbfcd4..5d433f52 100644 --- a/zipline/data/bundles/__init__.py +++ b/zipline/data/bundles/__init__.py @@ -3,9 +3,11 @@ from .core import ( UnknownBundle, bundles, clean, + from_bundle_ingest_dirname, ingest, load, register, + to_bundle_ingest_dirname, unregister, ) from .yahoo import yahoo_equities @@ -15,9 +17,11 @@ __all__ = [ 'UnknownBundle', 'bundles', 'clean', + 'from_bundle_ingest_dirname', 'ingest', 'load', 'register', + 'to_bundle_ingest_dirname', 'unregister', 'yahoo_equities', ] diff --git a/zipline/data/bundles/core.py b/zipline/data/bundles/core.py index 9643c522..bcb5f1c6 100644 --- a/zipline/data/bundles/core.py +++ b/zipline/data/bundles/core.py @@ -67,6 +67,39 @@ def cache_path(bundle_name, environ=None): ) +def to_bundle_ingest_dirname(ts): + """Convert a pandas Timestamp into the name of the directory for the + ingestion. + + Parameters + ---------- + ts : pandas.Timestamp + The time of the ingestions + + Returns + ------- + name : str + The name of the directory for this ingestion. + """ + return ts.isoformat().replace(':', ';') + + +def from_bundle_ingest_dirname(cs): + """Read a bundle ingestion directory name into a pandas Timestamp. + + Parameters + ---------- + cs : str + The name of the directory. + + Returns + ------- + ts : pandas.Timestamp + The time when this ingestion happened. + """ + return pd.Timestamp(cs.replace(';', ':')) + + _BundlePayload = namedtuple( '_BundlePayload', 'calendar opens closes minutes_per_day ingest create_writers', @@ -279,7 +312,7 @@ def _make_bundle_core(): if timestamp is None: timestamp = pd.Timestamp.utcnow() timestamp = timestamp.tz_convert('utc').tz_localize(None) - timestr = str(timestamp.value) + timestr = to_bundle_ingest_dirname(timestamp) cachepath = cache_path(name, environ=environ) pth.ensure_directory(pth.data_path([name, timestr], environ=environ)) pth.ensure_directory(cachepath) @@ -366,7 +399,7 @@ def _make_bundle_core(): [bundle_name, max( filter(complement(pth.hidden), candidates), - key=compose(pd.Timestamp, int), + key=from_bundle_ingest_dirname, )], environ=environ, ) @@ -463,7 +496,7 @@ def _make_bundle_core(): complement(pth.hidden), os.listdir(pth.data_path([name], environ=environ)), ), - key=compose(pd.Timestamp, int), + key=from_bundle_ingest_dirname, ) except OSError as e: if e.errno != errno.ENOENT: @@ -475,7 +508,7 @@ def _make_bundle_core(): if keep_last is None: def should_clean(name): - dt = pd.Timestamp(int(name)) + dt = from_bundle_ingest_dirname(name) return ( (before is not None and dt < before) or (after is not None and dt > after)