From 354e422be84d50bef799453fed9020c9dd0a8416 Mon Sep 17 00:00:00 2001
From: Frederic Fortier <fredfortier@gmail.com>
Date: Wed, 10 Jan 2018 21:54:09 -0500
Subject: [PATCH] BLD: working on ingesting alt data sources

---
 catalyst/marketplace/marketplace.py    |  7 ++-
 catalyst/marketplace/utils/__init__.py |  0
 catalyst/marketplace/utils/paths.py    | 87 ++++++++++++++++++++++++++
 tests/marketplace/test_marketplace.py  | 10 +++
 4 files changed, 102 insertions(+), 2 deletions(-)
 create mode 100644 catalyst/marketplace/utils/__init__.py
 create mode 100644 catalyst/marketplace/utils/paths.py

diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py
index e8288037..e22d3f49 100644
--- a/catalyst/marketplace/marketplace.py
+++ b/catalyst/marketplace/marketplace.py
@@ -6,13 +6,14 @@ from web3 import Web3, HTTPProvider
 
 from catalyst.exchange.utils.stats_utils import set_print_settings
 from catalyst.constants import ROOT_DIR
+from catalyst.marketplace.utils.paths import get_temp_bundles_folder
 
 REMOTE_NODE = 'http://localhost:7545'
 CONTRACT_PATH = os.path.join(
     ROOT_DIR, '..', 'marketplace', 'build', 'contracts', 'Marketplace.json'
 )
 CONTRACT_ADDRESS = Web3.toChecksumAddress(
-    '0xaa862ddac09f6736a61e1124040fd883a6533c19'
+    '0xe2b6cf3863240892d59664d209a28289a73ef644'
 )
 
 
@@ -112,7 +113,9 @@ class Marketplace:
 
         pass
 
-    def ingest(self, data_source_name, data_frequency, start, end):
+    def ingest(self, data_source_name, data_frequency=None, start=None,
+               end=None):
+        temp_folder = get_temp_bundles_folder(data_source_name)
         pass
 
     def clean(self, data_source_name):
diff --git a/catalyst/marketplace/utils/__init__.py b/catalyst/marketplace/utils/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/catalyst/marketplace/utils/paths.py b/catalyst/marketplace/utils/paths.py
new file mode 100644
index 00000000..008ed2b7
--- /dev/null
+++ b/catalyst/marketplace/utils/paths.py
@@ -0,0 +1,87 @@
+import os
+import tarfile
+
+from catalyst.data.bundles.core import download_without_progress
+from catalyst.utils.paths import data_root, ensure_directory
+
+
+def get_data_source_folder(data_source_name, environ=None):
+    """
+    The root path of an data_source folder.
+
+    Parameters
+    ----------
+    data_source_name: str
+    environ:
+
+    Returns
+    -------
+    str
+
+    """
+    if not environ:
+        environ = os.environ
+
+    root = data_root(environ)
+    data_source_folder = os.path.join(root, 'marketplace', data_source_name)
+    ensure_directory(data_source_folder)
+
+    return data_source_folder
+
+
+def get_temp_bundles_folder(data_source_name, environ=None):
+    """
+    The temp folder for bundle downloads by algo name.
+
+    Parameters
+    ----------
+    data_source_name: str
+    environ:
+
+    Returns
+    -------
+    str
+
+    """
+    data_source_folder = get_data_source_folder(data_source_name, environ)
+
+    temp_bundles = os.path.join(data_source_folder, 'temp_bundles')
+    ensure_directory(temp_bundles)
+
+    return temp_bundles
+
+
+def get_data_source(data_source_name, period):
+    """
+    Download and extract a bcolz bundle.
+
+    Parameters
+    ----------
+    exchange_name: str
+    symbol: str
+    data_frequency: str
+    period: str
+
+    Returns
+    -------
+    str
+        Filename: bitfinex-daily-neo_eth-2017-10.tar.gz
+
+    """
+    root = get_temp_bundles_folder(data_source_name)
+    name = '{data_source}-{period}'.format(
+        data_source=data_source_name,
+        period=period,
+    )
+    path = os.path.join(root, name)
+
+    if not os.path.isdir(path):
+        url = 'http://127.0.0.1:8080/{data_source}/{name}.tar.gz'.format(
+            data_source=data_source_name,
+            name=name,
+        )
+        bytes = download_without_progress(url)
+        with tarfile.open('r', fileobj=bytes) as tar:
+            tar.extractall(path)
+
+    return path
diff --git a/tests/marketplace/test_marketplace.py b/tests/marketplace/test_marketplace.py
index 1faf0648..3feeb9a6 100644
--- a/tests/marketplace/test_marketplace.py
+++ b/tests/marketplace/test_marketplace.py
@@ -1,5 +1,6 @@
 from catalyst.marketplace.marketplace import Marketplace
 from catalyst.testing.fixtures import WithLogger, ZiplineTestCase
+import pandas as pd
 
 
 class TestMarketplace(WithLogger, ZiplineTestCase):
@@ -12,3 +13,12 @@ class TestMarketplace(WithLogger, ZiplineTestCase):
         marketplace = Marketplace()
         marketplace.register('GitHub')
         pass
+
+    def test_ingest(self):
+        marketplace = Marketplace()
+        marketplace.ingest(
+            data_source_name='GitHub',
+            data_frequency='finest',
+            start=pd.Timestamp.utcnow(),
+        )
+        pass