ENH: use more human readable format for bundle ingest directories

We are now using isoformats with ':' replaced with ';'. We cannot use a
normal isoformat because windows does not allow files or directories
with ':' in the name.

tests/data/bundles/test_core.py

@@ -6,7 +6,7 @@ from toolz import valmap
 import toolz.curried.operator as op
 
 from zipline.assets.synthetic import make_simple_equity_info
-from zipline.data.bundles import UnknownBundle
+from zipline.data.bundles import UnknownBundle, from_bundle_ingest_dirname
 from zipline.data.bundles.core import _make_bundle_core
 from zipline.lib.adjustment import Float64Multiply
 from zipline.pipeline.loaders.synthetic import (
@@ -362,7 +362,7 @@ class BundleCoreTestCase(WithInstanceTmpDir, ZiplineTestCase):
 
     @staticmethod
     def _ts_of_run(run):
-        return pd.Timestamp(int(run.rsplit(os.path.sep, 1)[-1]))
+        return from_bundle_ingest_dirname(run.rsplit(os.path.sep, 1)[-1])
 
     def test_clean_before_after(self):
         first = self._empty_ingest()

zipline/__main__.py

@@ -1,3 +1,4 @@
+import errno
 import os
 from functools import wraps
 
@@ -347,12 +348,18 @@ def bundles():
     """List all of the available data bundles.
     """
     for bundle in sorted(bundles_module.bundles.keys()):
-        ingestions = sorted(
-            (str(pd.Timestamp(int(ing)))
-             for ing in os.listdir(pth.data_path([bundle]))
-             if not pth.hidden(ing)),
-            reverse=True,
-        )
+        try:
+            ingestions = sorted(
+                (str(bundles_module.from_bundle_ingest_dirname(ing))
+                 for ing in os.listdir(pth.data_path([bundle]))
+                 if not pth.hidden(ing)),
+                reverse=True,
+            )
+        except IOError as e:
+            if e.errno != errno.ENOENT:
+                raise
+            ingestions = []
+
         print(
             '\n'.join(
                 '%s %s' % (bundle, line)

zipline/data/bundles/__init__.py

@@ -3,9 +3,11 @@ from .core import (
     UnknownBundle,
     bundles,
     clean,
+    from_bundle_ingest_dirname,
     ingest,
     load,
     register,
+    to_bundle_ingest_dirname,
     unregister,
 )
 from .yahoo import yahoo_equities
@@ -15,9 +17,11 @@ __all__ = [
     'UnknownBundle',
     'bundles',
     'clean',
+    'from_bundle_ingest_dirname',
     'ingest',
     'load',
     'register',
+    'to_bundle_ingest_dirname',
     'unregister',
     'yahoo_equities',
 ]

zipline/data/bundles/core.py

@@ -67,6 +67,39 @@ def cache_path(bundle_name, environ=None):
     )
 
 
+def to_bundle_ingest_dirname(ts):
+    """Convert a pandas Timestamp into the name of the directory for the
+    ingestion.
+
+    Parameters
+    ----------
+    ts : pandas.Timestamp
+        The time of the ingestions
+
+    Returns
+    -------
+    name : str
+        The name of the directory for this ingestion.
+    """
+    return ts.isoformat().replace(':', ';')
+
+
+def from_bundle_ingest_dirname(cs):
+    """Read a bundle ingestion directory name into a pandas Timestamp.
+
+    Parameters
+    ----------
+    cs : str
+        The name of the directory.
+
+    Returns
+    -------
+    ts : pandas.Timestamp
+        The time when this ingestion happened.
+    """
+    return pd.Timestamp(cs.replace(';', ':'))
+
+
 _BundlePayload = namedtuple(
     '_BundlePayload',
     'calendar opens closes minutes_per_day ingest create_writers',
@@ -279,7 +312,7 @@ def _make_bundle_core():
         if timestamp is None:
             timestamp = pd.Timestamp.utcnow()
         timestamp = timestamp.tz_convert('utc').tz_localize(None)
-        timestr = str(timestamp.value)
+        timestr = to_bundle_ingest_dirname(timestamp)
         cachepath = cache_path(name, environ=environ)
         pth.ensure_directory(pth.data_path([name, timestr], environ=environ))
         pth.ensure_directory(cachepath)
@@ -366,7 +399,7 @@ def _make_bundle_core():
                 [bundle_name,
                  max(
                      filter(complement(pth.hidden), candidates),
-                     key=compose(pd.Timestamp, int),
+                     key=from_bundle_ingest_dirname,
                  )],
                 environ=environ,
             )
@@ -463,7 +496,7 @@ def _make_bundle_core():
                     complement(pth.hidden),
                     os.listdir(pth.data_path([name], environ=environ)),
                 ),
-                key=compose(pd.Timestamp, int),
+                key=from_bundle_ingest_dirname,
             )
         except OSError as e:
             if e.errno != errno.ENOENT:
@@ -475,7 +508,7 @@ def _make_bundle_core():
 
         if keep_last is None:
             def should_clean(name):
-                dt = pd.Timestamp(int(name))
+                dt = from_bundle_ingest_dirname(name)
                 return (
                     (before is not None and dt < before) or
                     (after is not None and dt > after)