BLD: working on ingesting alt data sources

catalyst/marketplace/marketplace.py

@@ -6,13 +6,14 @@ from web3 import Web3, HTTPProvider
 
 from catalyst.exchange.utils.stats_utils import set_print_settings
 from catalyst.constants import ROOT_DIR
+from catalyst.marketplace.utils.paths import get_temp_bundles_folder
 
 REMOTE_NODE = 'http://localhost:7545'
 CONTRACT_PATH = os.path.join(
     ROOT_DIR, '..', 'marketplace', 'build', 'contracts', 'Marketplace.json'
 )
 CONTRACT_ADDRESS = Web3.toChecksumAddress(
-    '0xaa862ddac09f6736a61e1124040fd883a6533c19'
+    '0xe2b6cf3863240892d59664d209a28289a73ef644'
 )
 
 
@@ -112,7 +113,9 @@ class Marketplace:
 
         pass
 
-    def ingest(self, data_source_name, data_frequency, start, end):
+    def ingest(self, data_source_name, data_frequency=None, start=None,
+               end=None):
+        temp_folder = get_temp_bundles_folder(data_source_name)
         pass
 
     def clean(self, data_source_name):

catalyst/marketplace/utils/paths.py

@@ -0,0 +1,87 @@
+import os
+import tarfile
+
+from catalyst.data.bundles.core import download_without_progress
+from catalyst.utils.paths import data_root, ensure_directory
+
+
+def get_data_source_folder(data_source_name, environ=None):
+    """
+    The root path of an data_source folder.
+
+    Parameters
+    ----------
+    data_source_name: str
+    environ:
+
+    Returns
+    -------
+    str
+
+    """
+    if not environ:
+        environ = os.environ
+
+    root = data_root(environ)
+    data_source_folder = os.path.join(root, 'marketplace', data_source_name)
+    ensure_directory(data_source_folder)
+
+    return data_source_folder
+
+
+def get_temp_bundles_folder(data_source_name, environ=None):
+    """
+    The temp folder for bundle downloads by algo name.
+
+    Parameters
+    ----------
+    data_source_name: str
+    environ:
+
+    Returns
+    -------
+    str
+
+    """
+    data_source_folder = get_data_source_folder(data_source_name, environ)
+
+    temp_bundles = os.path.join(data_source_folder, 'temp_bundles')
+    ensure_directory(temp_bundles)
+
+    return temp_bundles
+
+
+def get_data_source(data_source_name, period):
+    """
+    Download and extract a bcolz bundle.
+
+    Parameters
+    ----------
+    exchange_name: str
+    symbol: str
+    data_frequency: str
+    period: str
+
+    Returns
+    -------
+    str
+        Filename: bitfinex-daily-neo_eth-2017-10.tar.gz
+
+    """
+    root = get_temp_bundles_folder(data_source_name)
+    name = '{data_source}-{period}'.format(
+        data_source=data_source_name,
+        period=period,
+    )
+    path = os.path.join(root, name)
+
+    if not os.path.isdir(path):
+        url = 'http://127.0.0.1:8080/{data_source}/{name}.tar.gz'.format(
+            data_source=data_source_name,
+            name=name,
+        )
+        bytes = download_without_progress(url)
+        with tarfile.open('r', fileobj=bytes) as tar:
+            tar.extractall(path)
+
+    return path

tests/marketplace/test_marketplace.py

@@ -1,5 +1,6 @@
 from catalyst.marketplace.marketplace import Marketplace
 from catalyst.testing.fixtures import WithLogger, ZiplineTestCase
+import pandas as pd
 
 
 class TestMarketplace(WithLogger, ZiplineTestCase):
@@ -12,3 +13,12 @@ class TestMarketplace(WithLogger, ZiplineTestCase):
         marketplace = Marketplace()
         marketplace.register('GitHub')
         pass
+
+    def test_ingest(self):
+        marketplace = Marketplace()
+        marketplace.ingest(
+            data_source_name='GitHub',
+            data_frequency='finest',
+            start=pd.Timestamp.utcnow(),
+        )
+        pass