From 105fee0fb95ffa173ee22b644b59d217fb8cf298 Mon Sep 17 00:00:00 2001
From: fredfortier <fredfortier@gmail.com>
Date: Tue, 17 Oct 2017 15:35:49 -0400
Subject: [PATCH] Fixes to the daily data

---
 catalyst/exchange/exchange_bundle.py | 22 ++++++++++------------
 tests/exchange/test_bundle.py        | 19 ++++++++-----------
 2 files changed, 18 insertions(+), 23 deletions(-)

diff --git a/catalyst/exchange/exchange_bundle.py b/catalyst/exchange/exchange_bundle.py
index d85f417e..6da300f7 100644
--- a/catalyst/exchange/exchange_bundle.py
+++ b/catalyst/exchange/exchange_bundle.py
@@ -175,17 +175,13 @@ class ExchangeBundle:
                 )
 
         elif data_frequency == 'daily':
-            if len(os.listdir(output_dir)) > 0:
-                self._writers[key] = \
-                    BcolzDailyBarWriter.open(output_dir, end_dt)
-            else:
-                end_session = end_dt.floor('1d')
-                self._writers[key] = BcolzDailyBarWriter(
-                    filename=output_dir,
-                    calendar=self.calendar,
-                    start_session=start_dt,
-                    end_session=end_session
-                )
+            end_session = end_dt.floor('1d')
+            self._writers[key] = BcolzDailyBarWriter(
+                filename=output_dir,
+                calendar=self.calendar,
+                start_session=start_dt,
+                end_session=end_session
+            )
         else:
             raise InvalidHistoryFrequencyError(
                 frequency=data_frequency
@@ -327,7 +323,7 @@ class ExchangeBundle:
         """
 
     def ingest_ctable(self, asset, data_frequency, period, writer,
-                      empty_rows_behavior='warn', cleanup=False):
+                      empty_rows_behavior='strip', cleanup=False):
         """
         Merge a ctable bundle chunk into the main bundle for the exchange.
 
@@ -440,6 +436,8 @@ class ExchangeBundle:
                         end_minute=asset.end_minute,
                         dates=dates
                     )
+                else:
+                    df.dropna(inplace=True)
 
         data = []
         if not df.empty:
diff --git a/tests/exchange/test_bundle.py b/tests/exchange/test_bundle.py
index a0738dab..ea3da2f8 100644
--- a/tests/exchange/test_bundle.py
+++ b/tests/exchange/test_bundle.py
@@ -71,14 +71,14 @@ class ExchangeBundleTestCase:
         pass
 
     def test_merge_ctables(self):
-        exchange_name = 'poloniex'
-        data_frequency = 'minute'
+        exchange_name = 'bitfinex'
+        data_frequency = 'daily'
 
         exchange = get_exchange(exchange_name)
-        asset = exchange.get_asset('gno_btc')
+        asset = exchange.get_asset('neo_btc')
 
-        start = pd.to_datetime('2017-5-1', utc=True)
-        end = pd.to_datetime('2017-5-31', utc=True)
+        start = pd.to_datetime('2017-9-1', utc=True)
+        end = pd.to_datetime('2017-9-30', utc=True)
 
         # asset = exchange.get_asset('neo_btc')
         #
@@ -91,9 +91,9 @@ class ExchangeBundleTestCase:
         exchange_bundle.ingest_ctable(
             asset=asset,
             data_frequency=data_frequency,
-            period='2017-5',
+            period='2017',
             writer=writer,
-            empty_rows_behavior='raise'
+            empty_rows_behavior='strip'
         )
         pass
 
@@ -102,10 +102,7 @@ class ExchangeBundleTestCase:
         data_frequency = 'minute'
 
         exchange = get_exchange(exchange_name)
-        asset = exchange.get_asset('gno_btc')
-
-        start = pd.to_datetime('2017-5-1', utc=True)
-        end = pd.to_datetime('2017-5-31', utc=True)
+        asset = exchange.get_asset('neo_btc')
 
         path = get_bcolz_chunk(
             exchange_name=exchange_name,