From b230b7382924cbba35d1711d5a4a9d6392cddd47 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Tue, 24 Oct 2017 11:28:31 -0600 Subject: [PATCH 1/2] unit test bcolz writer --- tests/exchange/test_bcolz.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/exchange/test_bcolz.py b/tests/exchange/test_bcolz.py index 2ea82d35..75c43900 100644 --- a/tests/exchange/test_bcolz.py +++ b/tests/exchange/test_bcolz.py @@ -65,8 +65,8 @@ class TestBcolzWriter(object): pass def test_bcolz_write_minute_past(self): - start = pd.to_datetime('2015-04-01') - end = pd.to_datetime('2015-04-30') + start = pd.to_datetime('2015-04-01 00:00') + end = pd.to_datetime('2015-04-30 23:59') freq = 'minute' bundle = ExchangeBundle('bitfinex') @@ -89,7 +89,7 @@ class TestBcolzWriter(object): pass def test_bcolz_write_minute_present(self): - start = pd.to_datetime('2017-10-01') + start = pd.to_datetime('2017-10-01 00:00') end = pd.to_datetime('today') freq = 'minute' From fa018e2e0c13b357c3a0894f60f100bc9590ba17 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Tue, 24 Oct 2017 13:42:53 -0600 Subject: [PATCH 2/2] more bcolz unit tests --- tests/exchange/test_bcolz.py | 68 +++++++++++++++++++++--------- tests/exchange/test_bitfinex.py | 2 +- tests/exchange/test_bittrex.py | 2 +- tests/exchange/test_bundle.py | 2 +- tests/exchange/test_clock.py | 2 +- tests/exchange/test_data_portal.py | 2 +- tests/exchange/test_poloniex.py | 2 +- 7 files changed, 55 insertions(+), 25 deletions(-) diff --git a/tests/exchange/test_bcolz.py b/tests/exchange/test_bcolz.py index 75c43900..88f47def 100644 --- a/tests/exchange/test_bcolz.py +++ b/tests/exchange/test_bcolz.py @@ -3,7 +3,7 @@ import tempfile import pandas as pd from catalyst.exchange.exchange_bundle import ExchangeBundle -from catalyst.exchange.exchange_bcolz import BcolzExchangeBarWriter +from catalyst.exchange.exchange_bcolz import BcolzExchangeBarWriter, BcolzExchangeBarReader from nose.tools import assert_equals @@ -19,16 +19,20 @@ class TestBcolzWriter(object): def tearDown(self): shutil.rmtree(self.root_dir) # Remove the directory after the test + def generate_df(self, exchange_name, freq, start, end): + bundle = ExchangeBundle(exchange_name) + index = bundle.get_calendar_periods_range(start, end,freq) + df = pd.DataFrame(index=index, columns=self.columns) + df.fillna(1, inplace=True) + return df + + def test_bcolz_write_daily_past(self): start = pd.to_datetime('2016-01-01') end = pd.to_datetime('2016-12-31') freq = 'daily' - bundle = ExchangeBundle('bitfinex') - index = bundle.get_calendar_periods_range(start, end,freq) - df = pd.DataFrame(index=index, columns=self.columns) - assert_equals(len(df.index), 366) - df.fillna(1, inplace=True) + df = self.generate_df('bitfinex', freq, start, end) writer = BcolzExchangeBarWriter( rootdir=self.root_dir, @@ -47,10 +51,7 @@ class TestBcolzWriter(object): end = pd.to_datetime('today') freq = 'daily' - bundle = ExchangeBundle('bitfinex') - index = bundle.get_calendar_periods_range(start, end,freq) - df = pd.DataFrame(index=index, columns=self.columns) - df.fillna(1, inplace=True) + df = self.generate_df('bitfinex', freq, start, end) writer = BcolzExchangeBarWriter( rootdir = self.root_dir, @@ -69,11 +70,7 @@ class TestBcolzWriter(object): end = pd.to_datetime('2015-04-30 23:59') freq = 'minute' - bundle = ExchangeBundle('bitfinex') - index = bundle.get_calendar_periods_range(start, end,freq) - df = pd.DataFrame(index=index, columns=self.columns) - assert_equals(len(df.index), 30*24*60) - df.fillna(1, inplace=True) + df = self.generate_df('bitfinex', freq, start, end) writer = BcolzExchangeBarWriter( rootdir = self.root_dir, @@ -93,10 +90,7 @@ class TestBcolzWriter(object): end = pd.to_datetime('today') freq = 'minute' - bundle = ExchangeBundle('bitfinex') - index = bundle.get_calendar_periods_range(start, end,freq) - df = pd.DataFrame(index=index, columns=self.columns) - df.fillna(1, inplace=True) + df = self.generate_df('bitfinex', freq, start, end) writer = BcolzExchangeBarWriter( rootdir = self.root_dir, @@ -110,3 +104,39 @@ class TestBcolzWriter(object): writer.write(data) pass + def bcolz_exchange_daily_write_read(self, exchange_name): + start = pd.to_datetime('2017-10-01 00:00') + end = pd.to_datetime('today') + freq = 'minute' + + df = self.generate_df(exchange_name, freq, start, end) + + writer = BcolzExchangeBarWriter( + rootdir = self.root_dir, + start_session = start, + end_session = end, + data_frequency = freq, + write_metadata = True ) + + data = [] + data.append((1,df)) + writer.write(data) + + reader = BcolzExchangeBarReader(rootdir = self.root_dir) + + dx = reader.load_raw_arrays(self.columns, start, end, [1,]) + + assert_equals(dx,df) + + def test_bcolz_bitfinex_daily_write_read(self): + self.bcolz_exchange_daily_write_read('bitfinex') + + def test_bcolz_poloniex_daily_write_read(self): + self.bcolz_exchange_daily_write_read('poloniex') + + + + + + + diff --git a/tests/exchange/test_bitfinex.py b/tests/exchange/test_bitfinex.py index ded6c8ca..94968e9e 100644 --- a/tests/exchange/test_bitfinex.py +++ b/tests/exchange/test_bitfinex.py @@ -8,7 +8,7 @@ from catalyst.finance.execution import (LimitOrder) log = Logger('test_bitfinex') -class BitfinexTestCase(BaseExchangeTestCase): +class TestBitfinexTestCase(BaseExchangeTestCase): @classmethod def setup(self): log.info('creating bitfinex object') diff --git a/tests/exchange/test_bittrex.py b/tests/exchange/test_bittrex.py index f1becbcc..bbf52fa1 100644 --- a/tests/exchange/test_bittrex.py +++ b/tests/exchange/test_bittrex.py @@ -7,7 +7,7 @@ from catalyst.exchange.exchange_utils import get_exchange_auth log = Logger('test_bittrex') -class BittrexTestCase(BaseExchangeTestCase): +class TestBittrexTestCase(BaseExchangeTestCase): @classmethod def setup(self): print ('creating bittrex object') diff --git a/tests/exchange/test_bundle.py b/tests/exchange/test_bundle.py index c4575fe8..05bbb4b1 100644 --- a/tests/exchange/test_bundle.py +++ b/tests/exchange/test_bundle.py @@ -16,7 +16,7 @@ from catalyst.utils.paths import ensure_directory log = Logger('test_exchange_bundle') -class ExchangeBundleTestCase: +class TestExchangeBundleTestCase: def test_spot_value(self): data_frequency = 'daily' exchange_name = 'poloniex' diff --git a/tests/exchange/test_clock.py b/tests/exchange/test_clock.py index ff74986b..5857269b 100644 --- a/tests/exchange/test_clock.py +++ b/tests/exchange/test_clock.py @@ -11,7 +11,7 @@ import pandas as pd log = Logger('ExchangeClockTestCase') -class ExchangeClockTestCase(TestCase): +class TestExchangeClockTestCase(TestCase): @classmethod def setUpClass(cls): cls.open_calendar = get_calendar("OPEN") diff --git a/tests/exchange/test_data_portal.py b/tests/exchange/test_data_portal.py index 80ed5531..8a605a0b 100644 --- a/tests/exchange/test_data_portal.py +++ b/tests/exchange/test_data_portal.py @@ -12,7 +12,7 @@ from catalyst.exchange.exchange_utils import get_exchange_auth log = Logger('test_bitfinex') -class ExchangeDataPortalTestCase: +class TestExchangeDataPortalTestCase: @classmethod def setup(self): log.info('creating bitfinex exchange') diff --git a/tests/exchange/test_poloniex.py b/tests/exchange/test_poloniex.py index 4f2f12a7..16ab45b8 100644 --- a/tests/exchange/test_poloniex.py +++ b/tests/exchange/test_poloniex.py @@ -8,7 +8,7 @@ from catalyst.exchange.exchange_utils import get_exchange_auth log = Logger('test_poloniex') -class PoloniexTestCase(BaseExchangeTestCase): +class TestPoloniexTestCase(BaseExchangeTestCase): @classmethod def setup(self): print ('creating poloniex object')