Unit tested exchange loader extension and backtest data portal refactoring

tests/exchange/test_bundle.py

@@ -1,14 +1,13 @@
-from datetime import timedelta
-
 import os
+from datetime import timedelta
+from logging import Logger, DEBUG
+
 import pandas as pd
-from logging import Logger
 
 from catalyst import get_calendar
-
 from catalyst.data.minute_bars import BcolzMinuteBarWriter
 from catalyst.exchange.exchange_bundle import exchange_bundle
-from catalyst.exchange.exchange_utils import get_exchange_minute_writer_root
+from catalyst.utils.paths import ensure_directory, data_root
 
 log = Logger('test_exchange_bundle')
 
@@ -17,44 +16,52 @@ class ExchangeBundleTestCase:
     def test_ingest(self):
         exchange_name = 'bitfinex'
 
-        start = pd.Timestamp.utcnow() - timedelta(days=2)
+        start = pd.Timestamp.utcnow() - timedelta(days=365)
         end = pd.Timestamp.utcnow()
         open_calendar = get_calendar('OPEN')
 
-        root = get_exchange_minute_writer_root(exchange_name)
-        filename = os.path.join(root, 'metadata.json')
+        root = data_root(os.environ)
+        output_dir = '{root}/exchange_{exchange}/test'.format(
+            root=root,
+            exchange=exchange_name
+        )
+        ensure_directory(output_dir)
 
+        filename = os.path.join(output_dir, 'metadata.json')
+
+        start_session = start.floor('1d')
         if os.path.isfile(filename):
-            minute_bar_writer = BcolzMinuteBarWriter.open(root, end)
+            minute_bar_writer = BcolzMinuteBarWriter.open(output_dir, end)
         else:
             # TODO: need to be able to write more precise numbers
             minute_bar_writer = BcolzMinuteBarWriter(
-                rootdir=root,
+                rootdir=output_dir,
                 calendar=open_calendar,
                 minutes_per_day=1440,
-                start_session=start.floor('1d'),
+                start_session=start_session,
                 end_session=end,
                 write_metadata=True
             )
 
         ingest = exchange_bundle(
             exchange_name=exchange_name,
-            symbols=['btc_usd']
+            symbols=['eth_btc'],
+            log_level=DEBUG
         )
 
-        ingest(
-            environ=os.environ,
-            asset_db_writer=None,  # TODO: nice to have
-            minute_bar_writer=minute_bar_writer,
-            daily_bar_writer=None,  # TODO: add later
-            adjustment_writer=None,  # Not applicable to crypto
-            calendar=open_calendar,
-            start_session=start,
-            end_session=end,
-            cache=dict(),
-            show_progress=True,
-            output_dir=exchange_name,  # TODO: not sure
-            start=start,
-            end=end
-        )
+        ingest(environ=os.environ,
+               asset_db_writer=None,
+               minute_bar_writer=minute_bar_writer,
+               five_minute_bar_writer=None,
+               daily_bar_writer=None,
+               adjustment_writer=None,
+               calendar=open_calendar,
+               start_session=start_session,
+               end_session=end,
+               cache=dict(),
+               show_progress=True,
+               is_compile=False,
+               output_dir=output_dir,
+               start=start,
+               end=end)
         pass

tests/exchange/test_data_portal.py

@@ -1,5 +1,6 @@
 from datetime import timedelta
 
+import os
 import pandas as pd
 from catalyst import get_calendar
 from logbook import Logger
@@ -10,7 +11,9 @@ from catalyst.exchange.bitfinex.bitfinex import Bitfinex
 from catalyst.exchange.bittrex.bittrex import Bittrex
 from catalyst.exchange.data_portal_exchange import DataPortalExchangeBacktest, \
     DataPortalExchangeLive
+from catalyst.exchange.exchange_bundle import exchange_bundle
 from catalyst.exchange.exchange_utils import get_exchange_auth
+from catalyst.utils.run_algo import load_extensions
 
 log = Logger('test_bitfinex')
 
@@ -44,13 +47,14 @@ class ExchangeDataPortalTestCase:
             first_trading_day=pd.to_datetime('today', utc=True)
         )
         self.data_portal_backtest = DataPortalExchangeBacktest(
-            exchanges=dict(bitfinex=self.bitfinex, bittrex=self.bittrex),
+            exchanges=dict(bitfinex=self.bitfinex),
             asset_finder=asset_finder,
             trading_calendar=open_calendar,
             first_trading_day=pd.to_datetime('today', utc=True)
         )
 
     def test_get_history_window_live(self):
+
         asset_finder = self.data_portal_live.asset_finder
 
         assets = [
@@ -79,13 +83,14 @@ class ExchangeDataPortalTestCase:
         pass
 
     def test_get_spot_value_backtest(self):
+
         asset_finder = self.data_portal_backtest.asset_finder
 
         assets = [
-            asset_finder.lookup_symbol('btc_usd', self.bitfinex),
+            asset_finder.lookup_symbol('neo_btc', self.bitfinex),
         ]
 
-        date = pd.Timestamp.utcnow() - timedelta(hours=2)
+        date = pd.Timestamp.utcnow() - timedelta(hours=8)
         value = self.data_portal_backtest.get_spot_value(
             assets, 'close', date, 'minute')
         pass