Unit testing an issue with the daily loader

catalyst/exchange/exchange.py

@@ -486,17 +486,17 @@ class Exchange:
 
         # We check again for data which may be too recent for the consolidated
         # exchanges service
-        missing_assets = self.bundle.filter_existing_assets(
+        trailing_assets = self.bundle.filter_existing_assets(
             assets=assets,
             start_dt=start_dt,
             end_dt=end_dt,
             data_frequency=data_frequency
         )
-        if missing_assets:
+        if trailing_assets:
             # Adding bars too recent to be contained in the consolidated
             # exchanges bundles. We go directly against the exchange
             # to retrieve the candles.
-            for asset in missing_assets:
+            for asset in trailing_assets:
                 trailing_candles_dt = get_trailing_candles_dt(
                     asset=asset,
                     start_dt=start_dt,

tests/exchange/test_bundle.py

@@ -71,32 +71,77 @@ class ExchangeBundleTestCase:
         pass
 
     def test_merge_ctables(self):
-        exchange_name = 'bitfinex'
+        exchange_name = 'poloniex'
+
+        # Switch between daily and minute for testing
         data_frequency = 'daily'
+        # data_frequency = 'minute'
 
         exchange = get_exchange(exchange_name)
-        asset = exchange.get_asset('neo_btc')
+        assets = [
+            exchange.get_asset('eth_btc'),
+            exchange.get_asset('etc_btc'),
+        ]
 
         start = pd.to_datetime('2017-9-1', utc=True)
         end = pd.to_datetime('2017-9-30', utc=True)
 
-        # asset = exchange.get_asset('neo_btc')
-        #
-        # start = pd.to_datetime('2017-9-1', utc=True)
-        # end = pd.to_datetime('2017-9-30', utc=True)
-
         exchange_bundle = ExchangeBundle(exchange)
 
         writer = exchange_bundle.get_writer(start, end, data_frequency)
+
+        # In the interest of avoiding abstractions, this is writing a chunk
+        # to the ctable. It does not include the logic which creates chunks.
         exchange_bundle.ingest_ctable(
-            asset=asset,
+            asset=assets[0],
             data_frequency=data_frequency,
+            # period='2017-9',
+            period='2017',
+            # Dont't forget to update if you change your dates
+            start_dt=start,
+            end_dt=end,
+            writer=writer,
+            empty_rows_behavior='strip'
+        )
+        exchange_bundle.ingest_ctable(
+            asset=assets[1],
+            data_frequency=data_frequency,
+            # period='2017-9',
             period='2017',
             start_dt=start,
             end_dt=end,
             writer=writer,
             empty_rows_behavior='strip'
         )
+
+        # Since this pair was loaded last. It should be there in daily mode.
+        last_asset_array = exchange_bundle.get_raw_arrays(
+            assets=[assets[1]],
+            start_dt=start,
+            end_dt=end,
+            fields=['close'],
+            data_frequency=data_frequency
+        )
+        print('found {} rows for last ingestion'.format(
+            len(last_asset_array[0]))
+        )
+
+        # In daily mode, this returns an error. It appears that writing
+        # a second asset in the same date range removed the first asset.
+
+        # In minute mode, the data is there too. This signals that the minute
+        # writer / reader is more powerful. This explains why I did not
+        # encounter these problems as I have been focusing on minute data.
+        first_asset_array = exchange_bundle.get_raw_arrays(
+            assets=[assets[0]],
+            start_dt=start,
+            end_dt=end,
+            fields=['close'],
+            data_frequency=data_frequency
+        )
+        print('found {} rows for first ingestion'.format(
+            len(first_asset_array[0]))
+        )
         pass
 
     def test_minute_bundle(self):
@@ -112,5 +157,5 @@ class ExchangeBundleTestCase:
             data_frequency=data_frequency,
             period='2017-5',
         )
-        reader = BcolzMinuteBarReader(path)
+
         pass