From 3d521561f572913b70d005fee440337659a9689c Mon Sep 17 00:00:00 2001 From: Maya Tydykov Date: Tue, 12 Apr 2016 15:38:26 -0400 Subject: [PATCH] TST: update tests to handle new buyback auth design MAINT: add back cash amount constant BUG: fix field names BUG: pass remaining args WIP: make buyback units parameterized so that user can choose BUG: fix filtering based on units parameter WIP: test for undesired units Revert "WIP: make buyback units parameterized so that user can choose" This reverts commit df3b838d525bff5026eba1d81865c6645d534c88. --- tests/pipeline/test_buyback_auth.py | 196 ++++++------------ zipline/pipeline/common.py | 1 + zipline/pipeline/data/__init__.py | 2 +- zipline/pipeline/data/buyback_auth.py | 7 +- zipline/pipeline/factors/__init__.py | 6 +- .../pipeline/loaders/blaze/buyback_auth.py | 9 +- zipline/pipeline/loaders/buyback_auth.py | 22 +- zipline/pipeline/loaders/utils.py | 4 + 8 files changed, 95 insertions(+), 152 deletions(-) diff --git a/tests/pipeline/test_buyback_auth.py b/tests/pipeline/test_buyback_auth.py index e88e664c..35165097 100644 --- a/tests/pipeline/test_buyback_auth.py +++ b/tests/pipeline/test_buyback_auth.py @@ -8,35 +8,26 @@ from six import iteritems from zipline.pipeline.common import( BUYBACK_ANNOUNCEMENT_FIELD_NAME, - CASH_FIELD_NAME, + BUYBACK_TYPE_FIELD_NAME, DAYS_SINCE_PREV, PREVIOUS_BUYBACK_ANNOUNCEMENT, - PREVIOUS_BUYBACK_CASH, - PREVIOUS_BUYBACK_SHARE_COUNT, - SHARE_COUNT_FIELD_NAME, SID_FIELD_NAME, - TS_FIELD_NAME -) -from zipline.pipeline.data import ( - CashBuybackAuthorizations, - ShareBuybackAuthorizations -) -from zipline.pipeline.factors.events import ( - BusinessDaysSinceCashBuybackAuth, - BusinessDaysSinceShareBuybackAuth -) -from zipline.pipeline.loaders.buyback_auth import ( - CashBuybackAuthorizationsLoader, - ShareBuybackAuthorizationsLoader -) -from zipline.pipeline.loaders.blaze import ( - BlazeCashBuybackAuthorizationsLoader, - BlazeShareBuybackAuthorizationsLoader, + TS_FIELD_NAME, + VALUE_FIELD_NAME, + VALUE_TYPE_FIELD_NAME, + PREVIOUS_VALUE, + PREVIOUS_VALUE_TYPE, + PREVIOUS_BUYBACK_TYPE, ) +from zipline.pipeline.data import BuybackAuthorizations +from zipline.pipeline.factors.events import BusinessDaysSinceBuybackAuth +from zipline.pipeline.loaders.buyback_auth import BuybackAuthorizationsLoader +from zipline.pipeline.loaders.blaze import BlazeBuybackAuthorizationsLoader from zipline.pipeline.loaders.utils import ( zip_with_floats, - zip_with_dates -) + zip_with_dates, + get_values_for_date_ranges, + zip_with_strs) from zipline.testing.fixtures import ( WithPipelineEventDataLoader, ZiplineTestCase ) @@ -48,15 +39,17 @@ date_intervals = [ buyback_authorizations_cases = [ pd.DataFrame({ - SHARE_COUNT_FIELD_NAME: [1, 15], - CASH_FIELD_NAME: [10, 20], + VALUE_FIELD_NAME: [1, 15], + VALUE_TYPE_FIELD_NAME: ["$M", "Mshares"], + BUYBACK_TYPE_FIELD_NAME: ["New", "Additional"], TS_FIELD_NAME: pd.to_datetime(['2014-01-05', '2014-01-10']), BUYBACK_ANNOUNCEMENT_FIELD_NAME: pd.to_datetime(['2014-01-04', '2014-01-09']) }), pd.DataFrame( - columns=[SHARE_COUNT_FIELD_NAME, - CASH_FIELD_NAME, + columns=[VALUE_FIELD_NAME, + VALUE_TYPE_FIELD_NAME, + BUYBACK_TYPE_FIELD_NAME, BUYBACK_ANNOUNCEMENT_FIELD_NAME, TS_FIELD_NAME], dtype='datetime64[ns]' @@ -64,18 +57,33 @@ buyback_authorizations_cases = [ ] -class CashBuybackAuthLoaderTestCase(WithPipelineEventDataLoader, - ZiplineTestCase): +def get_expected_previous_values(zip_date_index_with_vals, + dates, + vals_for_date_intervals): + return pd.DataFrame({ + 0: get_values_for_date_ranges(zip_date_index_with_vals, + vals_for_date_intervals, + date_intervals, + dates), + 1: zip_date_index_with_vals(dates, ['NaN'] * len(dates)), + }, index=dates) + + +class BuybackAuthLoaderTestCase(WithPipelineEventDataLoader, ZiplineTestCase): """ Test for cash buyback authorizations dataset. """ pipeline_columns = { - PREVIOUS_BUYBACK_CASH: - CashBuybackAuthorizations.cash_amount.latest, + PREVIOUS_VALUE: + BuybackAuthorizations.previous_value.latest, PREVIOUS_BUYBACK_ANNOUNCEMENT: - CashBuybackAuthorizations.announcement_date.latest, + BuybackAuthorizations.previous_date.latest, + PREVIOUS_VALUE_TYPE: + BuybackAuthorizations.previous_value_type.latest, + PREVIOUS_BUYBACK_TYPE: + BuybackAuthorizations.previous_buyback_type.latest, DAYS_SINCE_PREV: - BusinessDaysSinceCashBuybackAuth(), + BusinessDaysSinceBuybackAuth(), } @classmethod @@ -84,24 +92,30 @@ class CashBuybackAuthLoaderTestCase(WithPipelineEventDataLoader, @classmethod def get_dataset(cls): - return {sid: - frame.drop(SHARE_COUNT_FIELD_NAME, axis=1) + return {sid: frame for sid, frame in enumerate(buyback_authorizations_cases)} - loader_type = CashBuybackAuthorizationsLoader + loader_type = BuybackAuthorizationsLoader def setup(self, dates): cols = { - PREVIOUS_BUYBACK_CASH: self.get_sids_to_frames(zip_with_floats, - [['NaN', 10, 20]], - date_intervals, - dates), + PREVIOUS_VALUE: self.get_sids_to_frames(zip_with_floats, + [['NaN', 1, 15]], + date_intervals, + dates), PREVIOUS_BUYBACK_ANNOUNCEMENT: self.get_sids_to_frames( zip_with_dates, [['NaT', '2014-01-04', '2014-01-09']], date_intervals, dates), + PREVIOUS_VALUE_TYPE: self.get_sids_to_frames( + zip_with_strs, [["", "$M", "Mshares"]], date_intervals, dates + ), + PREVIOUS_BUYBACK_TYPE: self.get_sids_to_frames( + zip_with_strs, [["", "New", "Additional"]], date_intervals, + dates + ) } cols[DAYS_SINCE_PREV] = self._compute_busday_offsets( cols[PREVIOUS_BUYBACK_ANNOUNCEMENT] @@ -109,68 +123,26 @@ class CashBuybackAuthLoaderTestCase(WithPipelineEventDataLoader, return cols -class ShareBuybackAuthLoaderTestCase(WithPipelineEventDataLoader, - ZiplineTestCase): - """ - Test for share buyback authorizations dataset. - """ - pipeline_columns = { - PREVIOUS_BUYBACK_SHARE_COUNT: - ShareBuybackAuthorizations.share_count.latest, - PREVIOUS_BUYBACK_ANNOUNCEMENT: - ShareBuybackAuthorizations.announcement_date.latest, - DAYS_SINCE_PREV: - BusinessDaysSinceShareBuybackAuth(), - } - - @classmethod - def get_sids(cls): - return range(2) - - @classmethod - def get_dataset(cls): - return {sid: - frame.drop(CASH_FIELD_NAME, axis=1) - for sid, frame - in enumerate(buyback_authorizations_cases)} - - loader_type = ShareBuybackAuthorizationsLoader - - def setup(self, dates): - cols = {PREVIOUS_BUYBACK_SHARE_COUNT: - self.get_sids_to_frames(zip_with_floats, - [['NaN', 1, 15]], - date_intervals, - dates,), - PREVIOUS_BUYBACK_ANNOUNCEMENT: - self.get_sids_to_frames(zip_with_dates, - [['NaT', '2014-01-04', '2014-01-09']], - date_intervals, - dates,), - } - - cols[DAYS_SINCE_PREV] = self._compute_busday_offsets( - cols[PREVIOUS_BUYBACK_ANNOUNCEMENT] - ) - return cols - - -class BlazeCashBuybackAuthLoaderTestCase(CashBuybackAuthLoaderTestCase): +class BlazeBuybackAuthLoaderTestCase(BuybackAuthLoaderTestCase): """ Test case for loading via blaze. """ - loader_type = BlazeCashBuybackAuthorizationsLoader + loader_type = BlazeBuybackAuthorizationsLoader def pipeline_event_loader_args(self, dates): _, mapping = super( - BlazeCashBuybackAuthLoaderTestCase, + BlazeBuybackAuthLoaderTestCase, self, ).pipeline_event_loader_args(dates) return (bz.data(pd.concat( pd.DataFrame({ BUYBACK_ANNOUNCEMENT_FIELD_NAME: frame[BUYBACK_ANNOUNCEMENT_FIELD_NAME], - CASH_FIELD_NAME: - frame[CASH_FIELD_NAME], + VALUE_FIELD_NAME: + frame[VALUE_FIELD_NAME], + VALUE_TYPE_FIELD_NAME: + frame[VALUE_TYPE_FIELD_NAME], + BUYBACK_TYPE_FIELD_NAME: + frame[BUYBACK_TYPE_FIELD_NAME], TS_FIELD_NAME: frame[TS_FIELD_NAME], SID_FIELD_NAME: sid, @@ -179,49 +151,13 @@ class BlazeCashBuybackAuthLoaderTestCase(CashBuybackAuthLoaderTestCase): ).reset_index(drop=True)),) -class BlazeShareBuybackAuthLoaderTestCase(ShareBuybackAuthLoaderTestCase): - """ Test case for loading via blaze. - """ - loader_type = BlazeShareBuybackAuthorizationsLoader - - def pipeline_event_loader_args(self, dates): - _, mapping = super( - BlazeShareBuybackAuthLoaderTestCase, - self, - ).pipeline_event_loader_args(dates) - return (bz.data(pd.concat( - pd.DataFrame({ - BUYBACK_ANNOUNCEMENT_FIELD_NAME: - frame[BUYBACK_ANNOUNCEMENT_FIELD_NAME], - SHARE_COUNT_FIELD_NAME: - frame[SHARE_COUNT_FIELD_NAME], - TS_FIELD_NAME: - frame[TS_FIELD_NAME], - SID_FIELD_NAME: sid, - }) - for sid, frame in iteritems(mapping) - ).reset_index(drop=True)),) - - -class BlazeShareBuybackAuthLoaderNotInteractiveTestCase( - BlazeShareBuybackAuthLoaderTestCase): +class BlazeBuybackAuthLoaderNotInteractiveTestCase( + BlazeBuybackAuthLoaderTestCase): """Test case for passing a non-interactive symbol and a dict of resources. """ def pipeline_event_loader_args(self, dates): (bound_expr,) = super( - BlazeShareBuybackAuthLoaderNotInteractiveTestCase, - self, - ).pipeline_event_loader_args(dates) - return swap_resources_into_scope(bound_expr, {}) - - -class BlazeCashBuybackAuthLoaderNotInteractiveTestCase( - BlazeCashBuybackAuthLoaderTestCase): - """Test case for passing a non-interactive symbol and a dict of resources. - """ - def pipeline_event_loader_args(self, dates): - (bound_expr,) = super( - BlazeCashBuybackAuthLoaderNotInteractiveTestCase, + BlazeBuybackAuthLoaderNotInteractiveTestCase, self, ).pipeline_event_loader_args(dates) return swap_resources_into_scope(bound_expr, {}) diff --git a/zipline/pipeline/common.py b/zipline/pipeline/common.py index f820c7cc..a264c7f0 100644 --- a/zipline/pipeline/common.py +++ b/zipline/pipeline/common.py @@ -9,6 +9,7 @@ CASH_AMOUNT_FIELD_NAME = 'cash_amount' COUNT_FIELD_NAME = 'count' BUYBACK_ANNOUNCEMENT_FIELD_NAME = 'buyback_date' BUYBACK_TYPE_FIELD_NAME = 'buyback_type' +CASH_AMOUNT_FIELD_NAME = 'cash_amount' DAYS_SINCE_PREV = 'days_since_prev' DAYS_SINCE_PREV_DISCLOSURE = 'days_since_prev_disclosure' DAYS_SINCE_PREV_DIVIDEND_ANNOUNCEMENT = 'days_since_prev_dividend_announcement' diff --git a/zipline/pipeline/data/__init__.py b/zipline/pipeline/data/__init__.py index 626e11c8..063316ad 100644 --- a/zipline/pipeline/data/__init__.py +++ b/zipline/pipeline/data/__init__.py @@ -20,6 +20,6 @@ __all__ = [ 'DividendsByExDate', 'DividendsByPayDate', 'EarningsCalendar', - 'ConsensusEstimates', + 'ConsensusEstimates', 'USEquityPricing', ] diff --git a/zipline/pipeline/data/buyback_auth.py b/zipline/pipeline/data/buyback_auth.py index dadce8cf..781e01aa 100644 --- a/zipline/pipeline/data/buyback_auth.py +++ b/zipline/pipeline/data/buyback_auth.py @@ -1,7 +1,8 @@ """ Datasets representing dates of recently announced buyback authorizations. """ -from zipline.utils.numpy_utils import datetime64ns_dtype, float64_dtype +from zipline.utils.numpy_utils import datetime64ns_dtype, float64_dtype, \ + categorical_dtype from .dataset import Column, DataSet @@ -13,5 +14,5 @@ class BuybackAuthorizations(DataSet): """ previous_value = Column(float64_dtype) previous_date = Column(datetime64ns_dtype) - previous_value_type = Column(float64_dtype) # TODO: should be string - previous_buyback_type = Column(float64_dtype) # TODO: should be string + previous_value_type = Column(categorical_dtype, missing_value="<>") + previous_buyback_type = Column(categorical_dtype, missing_value="<>") diff --git a/zipline/pipeline/factors/__init__.py b/zipline/pipeline/factors/__init__.py index d6d32b98..b5c7e2e8 100644 --- a/zipline/pipeline/factors/__init__.py +++ b/zipline/pipeline/factors/__init__.py @@ -6,13 +6,12 @@ from .factor import ( ) from .events import ( BusinessDaysSince13DFilingsDate, - BusinessDaysSinceCashBuybackAuth, + BusinessDaysSinceBuybackAuth, BusinessDaysSinceDividendAnnouncement, BusinessDaysUntilNextExDate, BusinessDaysSincePreviousExDate, BusinessDaysUntilNextEarnings, BusinessDaysSincePreviousEarnings, - BusinessDaysSinceShareBuybackAuth, ) from .technical import ( AverageDollarVolume, @@ -30,13 +29,12 @@ from .technical import ( __all__ = [ 'BusinessDaysSince13DFilingsDate', - 'BusinessDaysSinceCashBuybackAuth', + 'BusinessDaysSinceBuybackAuth', 'BusinessDaysSinceDividendAnnouncement', 'BusinessDaysUntilNextExDate', 'BusinessDaysSincePreviousExDate', 'BusinessDaysUntilNextEarnings', 'BusinessDaysSincePreviousEarnings', - 'BusinessDaysSinceShareBuybackAuth', 'CustomFactor', 'AverageDollarVolume', 'EWMA', diff --git a/zipline/pipeline/loaders/blaze/buyback_auth.py b/zipline/pipeline/loaders/blaze/buyback_auth.py index deab8422..7fe15900 100644 --- a/zipline/pipeline/loaders/blaze/buyback_auth.py +++ b/zipline/pipeline/loaders/blaze/buyback_auth.py @@ -4,7 +4,10 @@ from .core import ( ) from zipline.pipeline.common import ( BUYBACK_ANNOUNCEMENT_FIELD_NAME, - VALUE_FIELD_NAME, VALUE_TYPE_FIELD_NAME, BUYBACK_TYPE_FIELD_NAME) + BUYBACK_TYPE_FIELD_NAME, + VALUE_FIELD_NAME, + VALUE_TYPE_FIELD_NAME, +) from zipline.pipeline.data import BuybackAuthorizations from zipline.pipeline.loaders import BuybackAuthorizationsLoader from .events import BlazeEventsLoader @@ -38,8 +41,8 @@ class BlazeBuybackAuthorizationsLoader(BlazeEventsLoader): {TS_FIELD_NAME}: datetime, {BUYBACK_ANNOUNCEMENT_FIELD_NAME}: ?datetime, {VALUE_FIELD_NAME}: ?float64, - {VALUE_TYPE_FIELD_NAME}: ?float64, - {BUYBACK_TYPE_FIELD_NAME}: ?float64, + {VALUE_TYPE_FIELD_NAME}: ?str, + {BUYBACK_TYPE_FIELD_NAME}: ?str, }} Where each row of the table is a record including the sid to identify the diff --git a/zipline/pipeline/loaders/buyback_auth.py b/zipline/pipeline/loaders/buyback_auth.py index bb6b6d86..327d5743 100644 --- a/zipline/pipeline/loaders/buyback_auth.py +++ b/zipline/pipeline/loaders/buyback_auth.py @@ -44,26 +44,26 @@ class BuybackAuthorizationsLoader(EventsLoader): @lazyval def previous_value_loader(self): return self._previous_event_value_loader( - self.dataset.cash_amount, + self.dataset.previous_value, VALUE_FIELD_NAME ) @lazyval def previous_date_loader(self): return self._previous_event_date_loader( - self.dataset.announcement_date, - ) - - @lazyval - def previous_buyback_type_loader(self): - return self._previous_event_date_loader( - self.dataset.announcement_date, - BUYBACK_TYPE_FIELD_NAME, + self.dataset.previous_date, ) @lazyval def previous_value_type_loader(self): - return self._previous_event_date_loader( - self.dataset.announcement_date, + return self._previous_event_value_loader( + self.dataset.previous_value_type, VALUE_TYPE_FIELD_NAME, ) + + @lazyval + def previous_buyback_type_loader(self): + return self._previous_event_value_loader( + self.dataset.previous_buyback_type, + BUYBACK_TYPE_FIELD_NAME, + ) diff --git a/zipline/pipeline/loaders/utils.py b/zipline/pipeline/loaders/utils.py index 80642a25..f4b267db 100644 --- a/zipline/pipeline/loaders/utils.py +++ b/zipline/pipeline/loaders/utils.py @@ -280,6 +280,10 @@ def zip_with_floats(dates, flts): return pd.Series(flts, index=dates, dtype='float') +def zip_with_strs(dates, flts): + return pd.Series(flts, index=dates, dtype='string') + + def zip_with_dates(index_dates, dts): return pd.Series(pd.to_datetime(dts), index=index_dates)