From 151c3e45a78c6a3dd77f7020fb2087e87b9a0fff Mon Sep 17 00:00:00 2001 From: Eddie Hebert Date: Wed, 31 Aug 2016 13:01:18 -0400 Subject: [PATCH] TST: Fix get_last_traded_dt on bcolz daily reader. Remove special handling for the last session of an asset, which was moving the last traded back a session. If the asset has data on a session, `get_last_traded_dt` should always return that session if it is the parameter to the method. --- tests/data/test_resample.py | 6 --- tests/data/test_us_equity_pricing.py | 7 ++-- tests/test_bar_data.py | 55 +++++++++++++++++----------- zipline/data/us_equity_pricing.py | 27 +++++++++----- 4 files changed, 56 insertions(+), 39 deletions(-) diff --git a/tests/data/test_resample.py b/tests/data/test_resample.py index 5cbba336..9f8a839c 100644 --- a/tests/data/test_resample.py +++ b/tests/data/test_resample.py @@ -11,8 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from unittest import skip - from collections import OrderedDict from numbers import Real @@ -666,10 +664,6 @@ class TestReindexSessionBars(WithBcolzEquityDailyBarReader, def test_last_availabe_dt(self): self.assertEqual(self.reader.last_available_dt, self.END_DATE) - @skip("This test revealed a bug in BcolzDailyBarReader.get_last_traded_dt." - " When requesting data on the last session of an asset, the date is " - "overriden by the previous day. When that errant handling is this " - "test should be enabled.") def test_get_last_traded_dt(self): asset = self.asset_finder.retrieve_asset(1) self.assertEqual(self.reader.get_last_traded_dt(asset, diff --git a/tests/data/test_us_equity_pricing.py b/tests/data/test_us_equity_pricing.py index f6eb1cac..506d517f 100644 --- a/tests/data/test_us_equity_pricing.py +++ b/tests/data/test_us_equity_pricing.py @@ -30,7 +30,8 @@ from pandas.util.testing import assert_index_equal from zipline.data.us_equity_pricing import ( BcolzDailyBarReader, - NoDataOnDate, + NoDataBeforeDate, + NoDataAfterDate, ) from zipline.pipeline.loaders.synthetic import ( OHLCV, @@ -316,11 +317,11 @@ class BcolzDailyBarTestCase(WithBcolzEquityDailyBarReader, ZiplineTestCase): table = self.bcolz_daily_bar_ctable reader = BcolzDailyBarReader(table) # before - with self.assertRaises(NoDataOnDate): + with self.assertRaises(NoDataBeforeDate): reader.get_value(2, Timestamp('2015-06-08', tz='UTC'), 'close') # after - with self.assertRaises(NoDataOnDate): + with self.assertRaises(NoDataAfterDate): reader.get_value(4, Timestamp('2015-06-16', tz='UTC'), 'close') def test_unadjusted_get_value_empty_value(self): diff --git a/tests/test_bar_data.py b/tests/test_bar_data.py index 46da4704..795a45b5 100644 --- a/tests/test_bar_data.py +++ b/tests/test_bar_data.py @@ -15,6 +15,8 @@ from datetime import timedelta from nose_parameterized import parameterized import numpy as np +from numpy import nan +from numpy.testing import assert_almost_equal import pandas as pd from zipline._protocol import handle_non_market_minutes @@ -600,7 +602,7 @@ class TestDailyBarData(WithBarDataChecks, ZiplineTestCase): START_DATE = pd.Timestamp('2016-01-05', tz='UTC') END_DATE = ASSET_FINDER_EQUITY_END_DATE = pd.Timestamp( - '2016-01-08', + '2016-01-11', tz='UTC', ) @@ -613,6 +615,12 @@ class TestDailyBarData(WithBarDataChecks, DIVIDEND_ASSET_SID = 7 ILLIQUID_DIVIDEND_ASSET_SID = 8 + @classmethod + def make_equity_info(cls): + frame = super(TestDailyBarData, cls).make_equity_info() + frame.loc[[1, 2], 'end_date'] = pd.Timestamp('2016-01-08', tz='UTC') + return frame + @classmethod def make_splits_data(cls): return pd.DataFrame.from_records([ @@ -688,10 +696,11 @@ class TestDailyBarData(WithBarDataChecks, @classmethod def make_equity_daily_bar_data(cls): for sid in cls.sids: + asset = cls.asset_finder.retrieve_asset(sid) yield sid, create_daily_df_for_asset( cls.trading_calendar, - cls.equity_daily_bar_days[0], - cls.equity_daily_bar_days[-1], + asset.start_date, + asset.end_date, interval=2 - sid % 2 ) @@ -829,25 +838,31 @@ class TestDailyBarData(WithBarDataChecks, self.check_internal_consistency(bar_data) for asset in self.ASSETS: - self.assertTrue(bar_data.can_trade(asset)) + if asset in (1, 2): + self.assertFalse(bar_data.can_trade(asset)) + else: + self.assertTrue(bar_data.can_trade(asset)) self.assertFalse(bar_data.is_stale(asset)) - self.assertEqual(6, bar_data.current(asset, "open")) - self.assertEqual(7, bar_data.current(asset, "high")) - self.assertEqual(4, bar_data.current(asset, "low")) - self.assertEqual(5, bar_data.current(asset, "close")) - self.assertEqual(500, bar_data.current(asset, "volume")) - self.assertEqual(5, bar_data.current(asset, "price")) + if asset in (1, 2): + assert_almost_equal(nan, bar_data.current(asset, "open")) + assert_almost_equal(nan, bar_data.current(asset, "high")) + assert_almost_equal(nan, bar_data.current(asset, "low")) + assert_almost_equal(nan, bar_data.current(asset, "close")) + assert_almost_equal(0, bar_data.current(asset, "volume")) + assert_almost_equal(nan, bar_data.current(asset, "price")) + else: + self.assertEqual(6, bar_data.current(asset, "open")) + self.assertEqual(7, bar_data.current(asset, "high")) + self.assertEqual(4, bar_data.current(asset, "low")) + self.assertEqual(5, bar_data.current(asset, "close")) + self.assertEqual(500, bar_data.current(asset, "volume")) + self.assertEqual(5, bar_data.current(asset, "price")) def test_after_assets_dead(self): - # both assets end on self.day[-1], so let's try the next day - minute = self.get_last_minute_of_session( - self.trading_calendar.next_session_label( - self.equity_daily_bar_days[-1] - ) - ) + session = self.END_DATE - bar_data = BarData(self.data_portal, lambda: minute, "daily") + bar_data = BarData(self.data_portal, lambda: session, "daily") self.check_internal_consistency(bar_data) for asset in self.ASSETS: @@ -861,11 +876,9 @@ class TestDailyBarData(WithBarDataChecks, last_traded_dt = bar_data.current(asset, "last_traded") - if asset == self.ASSET1: - self.assertEqual(self.equity_daily_bar_days[-2], + if asset in (self.ASSET1, self.ASSET2): + self.assertEqual(self.equity_daily_bar_days[3], last_traded_dt) - else: - self.assertEqual(self.equity_daily_bar_days[1], last_traded_dt) @parameterized.expand([ ("split", 2, 3, 3, 1.5), diff --git a/zipline/data/us_equity_pricing.py b/zipline/data/us_equity_pricing.py index 29acdf0d..9b8dcfe4 100644 --- a/zipline/data/us_equity_pricing.py +++ b/zipline/data/us_equity_pricing.py @@ -106,6 +106,14 @@ class NoDataOnDate(Exception): pass +class NoDataBeforeDate(Exception): + pass + + +class NoDataAfterDate(Exception): + pass + + def check_uint32_safe(value, colname): if value >= UINT32_MAX: raise ValueError( @@ -631,19 +639,20 @@ class BcolzDailyBarReader(SessionBarReader): def get_last_traded_dt(self, asset, day): volumes = self._spot_col('volume') - if day >= asset.end_date: - # go back to one day before the asset ended - search_day = self.sessions[ - self.sessions.searchsorted(asset.end_date) - 1 - ] - else: - search_day = day + search_day = day while True: try: ix = self.sid_day_index(asset, search_day) except NoDataOnDate: return None + except NoDataBeforeDate: + return None + except NoDataAfterDate: + prev_day_ix = self.sessions.get_loc(search_day) - 1 + if prev_day_ix > -1: + search_day = self.sessions[prev_day_ix] + continue if volumes[ix] != 0: return search_day prev_day_ix = self.sessions.get_loc(search_day) - 1 @@ -675,12 +684,12 @@ class BcolzDailyBarReader(SessionBarReader): day, self.sessions)) offset = day_loc - self._calendar_offsets[sid] if offset < 0: - raise NoDataOnDate( + raise NoDataBeforeDate( "No data on or before day={0} for sid={1}".format( day, sid)) ix = self._first_rows[sid] + offset if ix > self._last_rows[sid]: - raise NoDataOnDate( + raise NoDataAfterDate( "No data on or after day={0} for sid={1}".format( day, sid)) return ix