Filtered out starting dates before the calendar

catalyst/exchange/bundle_utils.py

@@ -103,42 +103,6 @@ def get_start_dt(end_dt, bar_count, data_frequency):
     return start_dt
 
 
-def get_adj_dates(start, end, assets, data_frequency):
-    """
-    Contains a date range to the trading availability of the specified pairs.
-
-    :param start:
-    :param end:
-    :param assets:
-    :param data_frequency:
-    :return:
-    """
-    earliest_trade = None
-    last_entry = None
-    for asset in assets:
-        if earliest_trade is None or earliest_trade > asset.start_date:
-            earliest_trade = asset.start_date
-
-        end_asset = asset.end_minute if data_frequency == 'minute' else \
-            asset.end_daily
-        if end_asset is not None and \
-                (last_entry is None or end_asset > last_entry):
-            last_entry = end_asset
-
-    if start is None or earliest_trade > start:
-        start = earliest_trade
-
-    if end is None or (last_entry is not None and end > last_entry):
-        end = last_entry
-
-    if end is None or start >= end:
-        raise NoDataAvailableOnExchange(
-            exchange=asset.exchange.title(),
-            symbol=[asset.symbol],
-            data_frequency=data_frequency,
-        )
-
-    return start, end
 
 
 def get_month_start_end(dt):

catalyst/exchange/exchange_bundle.py

@@ -10,7 +10,7 @@ from catalyst.constants import LOG_LEVEL
 from catalyst.data.minute_bars import BcolzMinuteOverlappingData, \
     BcolzMinuteBarMetadata
 from catalyst.exchange.bundle_utils import range_in_bundle, \
-    get_bcolz_chunk, get_delta, get_adj_dates, get_month_start_end, \
+    get_bcolz_chunk, get_delta, get_month_start_end, \
     get_year_start_end, get_periods_range, get_df_from_arrays, get_start_dt
 from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \
     BcolzExchangeBarWriter
@@ -18,8 +18,7 @@ from catalyst.exchange.exchange_errors import EmptyValuesInBundleError, \
     InvalidHistoryFrequencyError, TempBundleNotFoundError, \
     NoDataAvailableOnExchange, \
     PricingDataNotLoadedError
-from catalyst.exchange.exchange_utils import get_exchange_folder, \
-    get_exchange_bundles_folder
+from catalyst.exchange.exchange_utils import get_exchange_folder
 from catalyst.utils.cli import maybe_show_progress
 from catalyst.utils.paths import ensure_directory
 
@@ -305,6 +304,45 @@ class ExchangeBundle:
 
         return path
 
+    def get_adj_dates(self, start, end, assets, data_frequency):
+        """
+        Contains a date range to the trading availability of the specified pairs.
+
+        :param start:
+        :param end:
+        :param assets:
+        :param data_frequency:
+        :return:
+        """
+        earliest_trade = None
+        last_entry = None
+        for asset in assets:
+            if (earliest_trade is None or earliest_trade > asset.start_date) \
+                    and asset.start_date >= self.calendar.first_session:
+                earliest_trade = asset.start_date
+
+            end_asset = asset.end_minute if data_frequency == 'minute' else \
+                asset.end_daily
+            if end_asset is not None and \
+                    (last_entry is None or end_asset > last_entry):
+                last_entry = end_asset
+
+        if start is None or \
+                (earliest_trade is not None and earliest_trade > start):
+            start = earliest_trade
+
+        if end is None or (last_entry is not None and end > last_entry):
+            end = last_entry
+
+        if end is None or start is None or start >= end:
+            raise NoDataAvailableOnExchange(
+                exchange=asset.exchange.title(),
+                symbol=[asset.symbol],
+                data_frequency=data_frequency,
+            )
+
+        return start, end
+
     def prepare_chunks(self, assets, data_frequency, start_dt, end_dt):
         """
         Split a price data request into chunks corresponding to individual
@@ -321,8 +359,9 @@ class ExchangeBundle:
         chunks = []
         for asset in assets:
             try:
-                asset_start, asset_end = \
-                    get_adj_dates(start_dt, end_dt, [asset], data_frequency)
+                asset_start, asset_end = self.get_adj_dates(
+                    start_dt, end_dt, [asset], data_frequency
+                )
 
             except NoDataAvailableOnExchange:
                 continue
@@ -464,7 +503,9 @@ class ExchangeBundle:
         :return:
         """
         assets = self.get_assets(include_symbols, exclude_symbols)
-        start_dt, end_dt = get_adj_dates(start, end, assets, data_frequency)
+        start_dt, end_dt = self.get_adj_dates(
+            start, end, assets, data_frequency
+        )
 
         for frequency in data_frequency.split(','):
             self.ingest_assets(assets, start_dt, end_dt, frequency,
@@ -551,8 +592,9 @@ class ExchangeBundle:
                                   data_frequency,
                                   reset_reader=False):
         start_dt = get_start_dt(end_dt, bar_count, data_frequency)
-        start_dt, end_dt = \
-            get_adj_dates(start_dt, end_dt, assets, data_frequency)
+        start_dt, end_dt = self.get_adj_dates(
+            start_dt, end_dt, assets, data_frequency
+        )
 
         reader = self.get_reader(data_frequency)
         if reset_reader:
@@ -571,8 +613,9 @@ class ExchangeBundle:
             )
 
         for asset in assets:
-            asset_start_dt, asset_end_dt = \
-                get_adj_dates(start_dt, end_dt, assets, data_frequency)
+            asset_start_dt, asset_end_dt = self.get_adj_dates(
+                start_dt, end_dt, assets, data_frequency
+            )
 
             in_bundle = range_in_bundle(
                 asset, asset_start_dt, asset_end_dt, reader
@@ -628,7 +671,7 @@ class ExchangeBundle:
         symbols = os.path.join(root, 'symbols.json')
         if os.path.isfile(symbols):
             os.remove(symbols)
-            
+
         temp_bundles = os.path.join(root, 'temp_bundles')
 
         if os.path.isdir(temp_bundles):

tests/exchange/test_bundle.py

@@ -17,7 +17,7 @@ from catalyst.utils.paths import ensure_directory
 log = Logger('test_exchange_bundle')
 
 
-class TestExchangeBundleTestCase:
+class TestExchangeBundle:
     def test_spot_value(self):
         data_frequency = 'daily'
         exchange_name = 'poloniex'
@@ -99,12 +99,8 @@ class TestExchangeBundleTestCase:
         # data_frequency = 'daily'
         # include_symbols = 'neo_btc,bch_btc,eth_btc'
 
-        exchange_name = 'bittrex'
-        data_frequency = 'daily'
-
-        start = pd.to_datetime('2015-1-1', utc=True)
-        end = pd.to_datetime('2017-10-16', utc=True)
-        periods = get_periods_range(start, end, data_frequency)
+        exchange_name = 'bitfinex'
+        data_frequency = 'minute'
 
         exchange = get_exchange(exchange_name)
         exchange_bundle = ExchangeBundle(exchange)
@@ -114,8 +110,8 @@ class TestExchangeBundleTestCase:
             data_frequency=data_frequency,
             include_symbols=None,
             exclude_symbols=None,
-            start=start,
-            end=end,
+            start=None,
+            end=None,
             show_progress=True
         )