From 4e833981e4c05be66f00ea83b437317aa1ab3de7 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Tue, 24 Oct 2017 09:55:37 -0600 Subject: [PATCH] unit test for Bcolz writer expanded --- catalyst/curate/poloniex.py | 52 +++++++++++----------- tests/exchange/test_bcolz.py | 84 ++++++++++++++++++++++++++++++------ 2 files changed, 97 insertions(+), 39 deletions(-) diff --git a/catalyst/curate/poloniex.py b/catalyst/curate/poloniex.py index e2a88476..113af497 100644 --- a/catalyst/curate/poloniex.py +++ b/catalyst/curate/poloniex.py @@ -212,32 +212,32 @@ class PoloniexCurator(object): def write_ohlcv_file(self, currencyPair): csv_trades = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv' csv_1min = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv' - if( os.path.isfile(csv_1min) ): - log.debug(currencyPair+': 1min data already present. Delete the file if you want to rebuild it.') - else: - df = pd.read_csv(csv_trades, names=['tradeID','date','type','rate','amount','total','globalTradeID'], - dtype = {'tradeID': int, 'date': str, 'type': str, 'rate': float, 'amount': float, 'total': float, 'globalTradeID': int } ) - df.drop(['tradeID','type','amount','globalTradeID'], axis=1, inplace=True) - df['date'] = pd.to_datetime(df['date'], infer_datetime_format=True) - ohlcv = self.generate_ohlcv(df) - try: - with open(csv_1min, 'ab') as csvfile: - csvwriter = csv.writer(csvfile) - for item in ohlcv.itertuples(): - if item.Index == 0: - continue - csvwriter.writerow([ - item.Index.value // 10 ** 9, - item.open, - item.high, - item.low, - item.close, - item.volume, - ]) - except Exception as e: - log.error('Error opening %s' % csv_fn) - log.exception(e) - log.debug(currencyPair+': Generated 1min OHLCV data.') + #if( os.path.isfile(csv_1min) ): + # log.debug(currencyPair+': 1min data already present. Delete the file if you want to rebuild it.') + #else: + df = pd.read_csv(csv_trades, names=['tradeID','date','type','rate','amount','total','globalTradeID'], + dtype = {'tradeID': int, 'date': str, 'type': str, 'rate': float, 'amount': float, 'total': float, 'globalTradeID': int } ) + df.drop(['tradeID','type','amount','globalTradeID'], axis=1, inplace=True) + df['date'] = pd.to_datetime(df['date'], infer_datetime_format=True) + ohlcv = self.generate_ohlcv(df) + try: + with open(csv_1min, 'ab') as csvfile: + csvwriter = csv.writer(csvfile) + for item in ohlcv.itertuples(): + if item.Index == 0: + continue + csvwriter.writerow([ + item.Index.value // 10 ** 9, + item.open, + item.high, + item.low, + item.close, + item.volume, + ]) + except Exception as e: + log.error('Error opening %s' % csv_fn) + log.exception(e) + log.debug(currencyPair+': Generated 1min OHLCV data.') ''' diff --git a/tests/exchange/test_bcolz.py b/tests/exchange/test_bcolz.py index 5b2486f1..92d5e684 100644 --- a/tests/exchange/test_bcolz.py +++ b/tests/exchange/test_bcolz.py @@ -5,27 +5,24 @@ import pandas as pd from catalyst import get_calendar from catalyst.exchange.exchange_bcolz import BcolzExchangeBarWriter - class TestBcolzWriter(object): + @classmethod def setup_class(cls): - cls.columns = ['open', 'high', 'low', 'close', 'volume'] + cls.columns = ['open', 'high', 'low', 'close', 'volume'] def setUp(self): - # Create a temporary directory - self.root_dir = tempfile.mkdtemp() + self.root_dir = tempfile.mkdtemp() # Create a temporary directory def tearDown(self): - # Remove the directory after the test - shutil.rmtree(self.root_dir) + shutil.rmtree(self.root_dir) # Remove the directory after the test - def test_bcolz_write_daily(self): + def test_bcolz_write_daily_past(self): start = pd.to_datetime('2015-01-01') end = pd.to_datetime('2015-12-31') freq = 'daily' calendar = get_calendar('OPEN') - # index = pd.date_range(start=start, end=end, freq='D', name='date') index = calendar.sessions_in_range(start, end) df = pd.DataFrame(index=index, columns=self.columns) df.fillna(1, inplace=True) @@ -39,11 +36,72 @@ class TestBcolzWriter(object): data = [] data.append((1, df)) - writer.write(data) pass - def test_bcolz_write_minute(self): - index = pd.date_range(start=pd.to_datetime('2015-01-01'), - end=pd.to_datetime('2015-01-31'), freq='T', - name='date') + def test_bcolz_write_daily_present(self): + start = pd.to_datetime('2017-01-01') + end = pd.to_datetime('today') + freq = 'daily' + + calendar = get_calendar('OPEN') + index = calendar.sessions_in_range(start, end) + df = pd.DataFrame(index=index, columns=self.columns) + df.fillna(1, inplace=True) + + writer = BcolzExchangeBarWriter( + rootdir = self.root_dir, + start_session = start, + end_session = end, + data_frequency = freq, + write_metadata = True ) + + data = [] + data.append((1,df)) + writer.write(data) + pass + + def test_bcolz_write_minute_past(self): + start = pd.to_datetime('2015-03-01') + end = pd.to_datetime('2015-03-31') + freq = 'minute' + + calendar = get_calendar('OPEN') + index = calendar.sessions_in_range(start, end) + df = pd.DataFrame(index=index, columns=self.columns) + df.fillna(1, inplace=True) + + writer = BcolzExchangeBarWriter( + rootdir = self.root_dir, + start_session = start, + end_session = end, + data_frequency = freq, + write_metadata = True ) + + data = [] + data.append((1,df)) + writer.write(data) + pass + + def test_bcolz_write_minute_present(self): + start = pd.to_datetime('2017-10-01') + end = pd.to_datetime('today') + freq = 'minute' + + calendar = get_calendar('OPEN') + index = calendar.sessions_in_range(start, end) + df = pd.DataFrame(index=index, columns=self.columns) + df.fillna(1, inplace=True) + + writer = BcolzExchangeBarWriter( + rootdir = self.root_dir, + start_session = start, + end_session = end, + data_frequency = freq, + write_metadata = True ) + + data = [] + data.append((1,df)) + writer.write(data) + pass +