unit test for Bcolz writer expanded

catalyst/curate/poloniex.py

@@ -212,32 +212,32 @@ class PoloniexCurator(object):
     def write_ohlcv_file(self, currencyPair):        
         csv_trades = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv'
         csv_1min   = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv'
-        if( os.path.isfile(csv_1min) ):
-            log.debug(currencyPair+': 1min data already present. Delete the file if you want to rebuild it.')
-        else:
-            df = pd.read_csv(csv_trades, names=['tradeID','date','type','rate','amount','total','globalTradeID'], 
-                    dtype = {'tradeID': int, 'date': str, 'type': str, 'rate': float, 'amount': float, 'total': float, 'globalTradeID': int } )
-            df.drop(['tradeID','type','amount','globalTradeID'], axis=1, inplace=True)
-            df['date'] = pd.to_datetime(df['date'], infer_datetime_format=True) 
-            ohlcv = self.generate_ohlcv(df)
-            try: 
-                with open(csv_1min, 'ab') as csvfile:
-                    csvwriter = csv.writer(csvfile)
-                    for item in ohlcv.itertuples():
-                        if item.Index == 0:
-                            continue
-                        csvwriter.writerow([
-                            item.Index.value // 10 ** 9,
-                            item.open,
-                            item.high,
-                            item.low,
-                            item.close,
-                            item.volume,
-                        ])
-            except Exception as e:
-                log.error('Error opening %s' % csv_fn)
-                log.exception(e)
-            log.debug(currencyPair+': Generated 1min OHLCV data.')
+        #if( os.path.isfile(csv_1min) ):
+        #    log.debug(currencyPair+': 1min data already present. Delete the file if you want to rebuild it.')
+        #else:
+        df = pd.read_csv(csv_trades, names=['tradeID','date','type','rate','amount','total','globalTradeID'], 
+                dtype = {'tradeID': int, 'date': str, 'type': str, 'rate': float, 'amount': float, 'total': float, 'globalTradeID': int } )
+        df.drop(['tradeID','type','amount','globalTradeID'], axis=1, inplace=True)
+        df['date'] = pd.to_datetime(df['date'], infer_datetime_format=True) 
+        ohlcv = self.generate_ohlcv(df)
+        try: 
+            with open(csv_1min, 'ab') as csvfile:
+                csvwriter = csv.writer(csvfile)
+                for item in ohlcv.itertuples():
+                    if item.Index == 0:
+                        continue
+                    csvwriter.writerow([
+                        item.Index.value // 10 ** 9,
+                        item.open,
+                        item.high,
+                        item.low,
+                        item.close,
+                        item.volume,
+                    ])
+        except Exception as e:
+            log.error('Error opening %s' % csv_fn)
+            log.exception(e)
+        log.debug(currencyPair+': Generated 1min OHLCV data.')
 
 
     '''

tests/exchange/test_bcolz.py

@@ -5,27 +5,24 @@ import pandas as pd
 from catalyst import get_calendar
 from catalyst.exchange.exchange_bcolz import BcolzExchangeBarWriter
 
-
 class TestBcolzWriter(object):
+
     @classmethod
     def setup_class(cls):
-        cls.columns = ['open', 'high', 'low', 'close', 'volume']
+    	cls.columns = ['open', 'high', 'low', 'close', 'volume']
 
     def setUp(self):
-        # Create a temporary directory
-        self.root_dir = tempfile.mkdtemp()
+    	self.root_dir = tempfile.mkdtemp()			# Create a temporary directory
 
     def tearDown(self):
-        # Remove the directory after the test
-        shutil.rmtree(self.root_dir)
+    	shutil.rmtree(self.root_dir)				# Remove the directory after the test
 
-    def test_bcolz_write_daily(self):
+    def test_bcolz_write_daily_past(self):
         start = pd.to_datetime('2015-01-01')
         end = pd.to_datetime('2015-12-31')
         freq = 'daily'
 
         calendar = get_calendar('OPEN')
-        # index = pd.date_range(start=start, end=end, freq='D', name='date')
         index = calendar.sessions_in_range(start, end)
         df = pd.DataFrame(index=index, columns=self.columns)
         df.fillna(1, inplace=True)
@@ -39,11 +36,72 @@ class TestBcolzWriter(object):
 
         data = []
         data.append((1, df))
-
         writer.write(data)
         pass
 
-    def test_bcolz_write_minute(self):
-        index = pd.date_range(start=pd.to_datetime('2015-01-01'),
-                              end=pd.to_datetime('2015-01-31'), freq='T',
-                              name='date')
+    def test_bcolz_write_daily_present(self):
+        start = pd.to_datetime('2017-01-01')
+        end   = pd.to_datetime('today')
+        freq  = 'daily'
+
+        calendar = get_calendar('OPEN')
+        index = calendar.sessions_in_range(start, end)
+        df = pd.DataFrame(index=index, columns=self.columns)
+        df.fillna(1, inplace=True)
+
+        writer = BcolzExchangeBarWriter(
+        				rootdir  = self.root_dir,
+        				start_session   = start,
+                        end_session     = end,
+                        data_frequency  = freq,
+                        write_metadata  = True )
+
+        data = []
+        data.append((1,df))
+        writer.write(data)
+        pass
+
+    def test_bcolz_write_minute_past(self):
+        start = pd.to_datetime('2015-03-01')
+        end   = pd.to_datetime('2015-03-31')
+        freq  = 'minute'
+
+        calendar = get_calendar('OPEN')
+        index = calendar.sessions_in_range(start, end)
+        df = pd.DataFrame(index=index, columns=self.columns)
+        df.fillna(1, inplace=True)
+
+        writer = BcolzExchangeBarWriter(
+        				rootdir  = self.root_dir,
+        				start_session   = start,
+                        end_session     = end,
+                        data_frequency  = freq,
+                        write_metadata  = True )
+
+        data = []
+        data.append((1,df))
+        writer.write(data)
+        pass
+
+    def test_bcolz_write_minute_present(self):
+        start = pd.to_datetime('2017-10-01')
+        end   = pd.to_datetime('today')
+        freq  = 'minute'
+
+        calendar = get_calendar('OPEN')
+        index = calendar.sessions_in_range(start, end)
+        df = pd.DataFrame(index=index, columns=self.columns)
+        df.fillna(1, inplace=True)
+
+        writer = BcolzExchangeBarWriter(
+        				rootdir  = self.root_dir,
+        				start_session   = start,
+                        end_session     = end,
+                        data_frequency  = freq,
+                        write_metadata  = True )
+
+        data = []
+        data.append((1,df))
+        writer.write(data)
+        pass
+