From 96cc6b6588534e3c40922bdfe664bc8816787a8a Mon Sep 17 00:00:00 2001 From: Andrew Daniels Date: Mon, 19 Sep 2016 17:07:33 -0400 Subject: [PATCH] MAINT: Adds option for minute bar writer to not write metadata With the addition of the truncate function, there are cases where we'll want to construct a BcolzMinuteBarWriter to call truncate, without gathering all the metadata. This commit adds a write_metadata arg to its init, which is True by default. If False is specified, no metadata is written. Requires adding logic to truncate to update end_session in metadata to the truncate date. --- tests/data/test_minute_bars.py | 80 ++++++---------------------------- zipline/data/minute_bars.py | 31 ++++++++----- 2 files changed, 35 insertions(+), 76 deletions(-) diff --git a/tests/data/test_minute_bars.py b/tests/data/test_minute_bars.py index a61308a9..514933bc 100644 --- a/tests/data/test_minute_bars.py +++ b/tests/data/test_minute_bars.py @@ -33,7 +33,6 @@ from pandas import ( Timedelta, NaT, date_range, - isnull, ) from zipline.data.bar_reader import NoDataOnDate @@ -971,8 +970,15 @@ class BcolzMinuteBarTestCase(WithTradingCalendars, # Truncate to first day with data. self.writer.truncate(days[0]) + # Refresh the reader since truncate update the metadata. + self.reader = BcolzMinuteBarReader(self.dest) + self.assertEqual(self.writer.last_date_in_output_for_sid(sid), days[0]) + cal = self.trading_calendar + _, last_close = cal.open_and_close_for_session(days[0]) + self.assertEqual(self.reader.last_available_dt, last_close) + minute = minutes[0] open_price = self.reader.get_value(sid, minute, 'open') @@ -995,28 +1001,6 @@ class BcolzMinuteBarTestCase(WithTradingCalendars, self.assertEquals(50.0, volume_price) - minute = minutes[1] - - open_price = self.reader.get_value(sid, minute, 'open') - - self.assertTrue(isnull(open_price)) - - high_price = self.reader.get_value(sid, minute, 'high') - - self.assertTrue(isnull(high_price)) - - low_price = self.reader.get_value(sid, minute, 'low') - - self.assertTrue(isnull(low_price)) - - close_price = self.reader.get_value(sid, minute, 'close') - - self.assertTrue(isnull(close_price)) - - volume_price = self.reader.get_value(sid, minute, 'volume') - - self.assertEqual(0.0, volume_price) - def test_truncate_all_data_points(self): tds = self.market_opens.index @@ -1044,51 +1028,15 @@ class BcolzMinuteBarTestCase(WithTradingCalendars, # day with minute data. self.writer.truncate(self.test_calendar_start) + # Refresh the reader since truncate update the metadata. + self.reader = BcolzMinuteBarReader(self.dest) + self.assertEqual( self.writer.last_date_in_output_for_sid(sid), self.test_calendar_start, ) - minute = minutes[0] - - open_price = self.reader.get_value(sid, minute, 'open') - - self.assertTrue(isnull(open_price)) - - high_price = self.reader.get_value(sid, minute, 'high') - - self.assertTrue(isnull(high_price)) - - low_price = self.reader.get_value(sid, minute, 'low') - - self.assertTrue(isnull(low_price)) - - close_price = self.reader.get_value(sid, minute, 'close') - - self.assertTrue(isnull(close_price)) - - volume_price = self.reader.get_value(sid, minute, 'volume') - - self.assertEquals(0.0, volume_price) - - minute = minutes[1] - - open_price = self.reader.get_value(sid, minute, 'open') - - self.assertTrue(isnull(open_price)) - - high_price = self.reader.get_value(sid, minute, 'high') - - self.assertTrue(isnull(high_price)) - - low_price = self.reader.get_value(sid, minute, 'low') - - self.assertTrue(isnull(low_price)) - - close_price = self.reader.get_value(sid, minute, 'close') - - self.assertTrue(isnull(close_price)) - - volume_price = self.reader.get_value(sid, minute, 'volume') - - self.assertEqual(0.0, volume_price) + cal = self.trading_calendar + _, last_close = cal.open_and_close_for_session( + self.test_calendar_start) + self.assertEqual(self.reader.last_available_dt, last_close) diff --git a/zipline/data/minute_bars.py b/zipline/data/minute_bars.py index 2c7e307f..9a2d4864 100644 --- a/zipline/data/minute_bars.py +++ b/zipline/data/minute_bars.py @@ -317,6 +317,10 @@ class BcolzMinuteBarWriter(object): Defaults to supporting 15 years of NYSE equity market data. see: http://bcolz.blosc.org/opt-tips.html#informing-about-the-length-of-your-carrays # noqa + write_metadata : bool, optional + If True, writes the minute bar metadata (on init of the writer). + If False, no metadata is written (existing metadata is + retained). Default is True. Notes ----- @@ -373,7 +377,8 @@ class BcolzMinuteBarWriter(object): minutes_per_day, default_ohlc_ratio=OHLC_RATIO, ohlc_ratios_per_sid=None, - expectedlen=DEFAULT_EXPECTEDLEN): + expectedlen=DEFAULT_EXPECTEDLEN, + write_metadata=True): self._rootdir = rootdir self._start_session = start_session @@ -391,15 +396,16 @@ class BcolzMinuteBarWriter(object): self._minute_index = _calc_minute_index( self._schedule.market_open, self._minutes_per_day) - metadata = BcolzMinuteBarMetadata( - self._default_ohlc_ratio, - self._ohlc_ratios_per_sid, - self._calendar, - self._start_session, - self._end_session, - self._minutes_per_day, - ) - metadata.write(self._rootdir) + if write_metadata: + metadata = BcolzMinuteBarMetadata( + self._default_ohlc_ratio, + self._ohlc_ratios_per_sid, + self._calendar, + self._start_session, + self._end_session, + self._minutes_per_day, + ) + metadata.write(self._rootdir) @property def first_trading_day(self): @@ -795,6 +801,11 @@ class BcolzMinuteBarWriter(object): ) shutil.move(tmp_path, sid_path) + # Update end session in metadata. + metadata = BcolzMinuteBarMetadata.read(self._rootdir) + metadata.end_session = date + metadata.write(self._rootdir) + class BcolzMinuteBarReader(MinuteBarReader): """