diff --git a/tests/pipeline/test_dividends.py b/tests/pipeline/test_dividends.py index c4e226ab..1b0c9791 100644 --- a/tests/pipeline/test_dividends.py +++ b/tests/pipeline/test_dividends.py @@ -8,23 +8,28 @@ from six import iteritems from zipline.pipeline.common import ( ANNOUNCEMENT_FIELD_NAME, + CASH_AMOUNT_FIELD_NAME, + CURRENCY_FIELD_NAME, DAYS_SINCE_PREV_DIVIDEND_ANNOUNCEMENT, DAYS_SINCE_PREV_EX_DATE, DAYS_TO_NEXT_EX_DATE, + DIVIDEND_TYPE_FIELD_NAME, + EX_DATE_FIELD_NAME, NEXT_AMOUNT, + NEXT_CURRENCY_TYPE, + NEXT_DIVIDEND_TYPE, NEXT_EX_DATE, NEXT_PAY_DATE, + PAY_DATE_FIELD_NAME, + PREVIOUS_AMOUNT, PREVIOUS_ANNOUNCEMENT, PREVIOUS_CURRENCY_TYPE, + PREVIOUS_DIVIDEND_TYPE, PREVIOUS_EX_DATE, PREVIOUS_PAY_DATE, - PREVIOUS_AMOUNT, SID_FIELD_NAME, TS_FIELD_NAME, - CASH_AMOUNT_FIELD_NAME, - EX_DATE_FIELD_NAME, - PAY_DATE_FIELD_NAME, - CURRENCY_FIELD_NAME, NEXT_CURRENCY_TYPE) +) from zipline.pipeline.data.dividends import ( DividendsByAnnouncementDate, DividendsByExDate, @@ -63,7 +68,8 @@ dividends_cases = [ PAY_DATE_FIELD_NAME: pd.to_datetime(['2014-01-15', '2014-01-20']), TS_FIELD_NAME: pd.to_datetime(['2014-01-05', '2014-01-10']), ANNOUNCEMENT_FIELD_NAME: pd.to_datetime(['2014-01-04', '2014-01-09']), - CURRENCY_FIELD_NAME: ["$", "EUR"] + CURRENCY_FIELD_NAME: ["$", "EUR"], + DIVIDEND_TYPE_FIELD_NAME: ["Stock", "Mixed"] }), # K1--K2--A2--A1. pd.DataFrame({ @@ -72,7 +78,8 @@ dividends_cases = [ PAY_DATE_FIELD_NAME: pd.to_datetime(['2014-01-20', '2014-01-15']), TS_FIELD_NAME: pd.to_datetime(['2014-01-05', '2014-01-10']), ANNOUNCEMENT_FIELD_NAME: pd.to_datetime(['2014-01-04', '2014-01-09']), - CURRENCY_FIELD_NAME: ["EUR", "$"] + CURRENCY_FIELD_NAME: ["EUR", "$"], + DIVIDEND_TYPE_FIELD_NAME: ["Mixed", "Stock"] }), # K1--A1--K2--A2. pd.DataFrame({ @@ -81,7 +88,8 @@ dividends_cases = [ PAY_DATE_FIELD_NAME: pd.to_datetime(['2014-01-10', '2014-01-20']), TS_FIELD_NAME: pd.to_datetime(['2014-01-05', '2014-01-15']), ANNOUNCEMENT_FIELD_NAME: pd.to_datetime(['2014-01-04', '2014-01-14']), - CURRENCY_FIELD_NAME: ["$", "EUR"] + CURRENCY_FIELD_NAME: ["$", "EUR"], + DIVIDEND_TYPE_FIELD_NAME: ["Stock", "Mixed"] }), # K1 == K2. pd.DataFrame({ @@ -90,7 +98,8 @@ dividends_cases = [ PAY_DATE_FIELD_NAME: pd.to_datetime(['2014-01-10', '2014-01-15']), TS_FIELD_NAME: pd.to_datetime(['2014-01-05'] * 2), ANNOUNCEMENT_FIELD_NAME: pd.to_datetime(['2014-01-04', '2014-01-04']), - CURRENCY_FIELD_NAME: ["$", "EUR"] + CURRENCY_FIELD_NAME: ["$", "EUR"], + DIVIDEND_TYPE_FIELD_NAME: ["Stock", "Mixed"] }), pd.DataFrame( columns=[CASH_AMOUNT_FIELD_NAME, @@ -98,7 +107,8 @@ dividends_cases = [ PAY_DATE_FIELD_NAME, TS_FIELD_NAME, ANNOUNCEMENT_FIELD_NAME, - CURRENCY_FIELD_NAME], + CURRENCY_FIELD_NAME, + DIVIDEND_TYPE_FIELD_NAME], dtype='datetime64[ns]' ), ] @@ -174,6 +184,16 @@ next_currency_types = [[None, "$", "EUR", None], [None, "$", None, "EUR", None], [None, "$", "EUR", None]] +prev_dividend_types = [[None, "Stock", "Mixed"], + [None, "Stock", "Mixed"], + [None, "Stock", "Mixed"], + [None, "Stock", "Mixed"]] + +next_dividend_types = [[None, "Stock", "Mixed", None], + [None, "Mixed", "Stock", "Mixed", None], + [None, "Stock", None, "Mixed", None], + [None, "Stock", "Mixed", None]] + class DividendsByAnnouncementDateTestCase(WithPipelineEventDataLoader, ZiplineTestCase): @@ -188,6 +208,8 @@ class DividendsByAnnouncementDateTestCase(WithPipelineEventDataLoader, BusinessDaysSinceDividendAnnouncement(), PREVIOUS_CURRENCY_TYPE: DividendsByAnnouncementDate.previous_currency.latest, + PREVIOUS_DIVIDEND_TYPE: + DividendsByAnnouncementDate.previous_type.latest, } @classmethod @@ -223,8 +245,10 @@ class DividendsByAnnouncementDateTestCase(WithPipelineEventDataLoader, ['NaT', '2014-01-04', '2014-01-14'], ['NaT', '2014-01-04']] amounts = [['NaN', 1, 15], ['NaN', 7, 13], ['NaN', 3, 1], ['NaN', 23]] - currency_types = [[None, "$", "EUR"], [None, "$", "EUR"], - [None, "$", "EUR"]] + currency_types = [[None, "$", "EUR"], [None, "EUR", "$"], + [None, "$", "EUR"], [None, "EUR"]] + dividend_types = [[None, "Stock", "Mixed"], [None, "Mixed", "Stock"], + [None, "Stock", "Mixed"], [None, "Mixed"]] cols = { PREVIOUS_ANNOUNCEMENT: self.get_sids_to_frames( zip_with_dates, announcement_dates, date_intervals, dates, @@ -237,6 +261,10 @@ class DividendsByAnnouncementDateTestCase(WithPipelineEventDataLoader, zip_with_strs, currency_types, date_intervals, dates, 'category', None ), + PREVIOUS_DIVIDEND_TYPE: self.get_sids_to_frames( + zip_with_strs, dividend_types, date_intervals, dates, + 'category', None + ), } cols[ @@ -260,7 +288,9 @@ class BlazeDividendsByAnnouncementDateTestCase( ANNOUNCEMENT_FIELD_NAME: df[ANNOUNCEMENT_FIELD_NAME], TS_FIELD_NAME: df[TS_FIELD_NAME], SID_FIELD_NAME: sid, - CASH_AMOUNT_FIELD_NAME: df[CASH_AMOUNT_FIELD_NAME] + CASH_AMOUNT_FIELD_NAME: df[CASH_AMOUNT_FIELD_NAME], + CURRENCY_FIELD_NAME: df[CURRENCY_FIELD_NAME], + DIVIDEND_TYPE_FIELD_NAME: df[DIVIDEND_TYPE_FIELD_NAME], }) for sid, df in iteritems(mapping) ).reset_index(drop=True)),) @@ -290,6 +320,8 @@ class DividendsByExDateTestCase(WithPipelineEventDataLoader, ZiplineTestCase): PREVIOUS_AMOUNT: DividendsByExDate.previous_amount.latest, PREVIOUS_CURRENCY_TYPE: DividendsByExDate.previous_currency.latest, NEXT_CURRENCY_TYPE: DividendsByExDate.next_currency.latest, + PREVIOUS_DIVIDEND_TYPE: DividendsByExDate.previous_type.latest, + NEXT_DIVIDEND_TYPE: DividendsByExDate.next_type.latest, DAYS_TO_NEXT_EX_DATE: BusinessDaysUntilNextExDate(), DAYS_SINCE_PREV_EX_DATE: BusinessDaysSincePreviousExDate() } @@ -331,7 +363,15 @@ class DividendsByExDateTestCase(WithPipelineEventDataLoader, ZiplineTestCase): NEXT_CURRENCY_TYPE: self.get_sids_to_frames( zip_with_strs, next_currency_types, next_date_intervals, dates, 'category', None - ) + ), + PREVIOUS_DIVIDEND_TYPE: self.get_sids_to_frames( + zip_with_strs, prev_dividend_types, prev_date_intervals, dates, + 'category', None + ), + NEXT_DIVIDEND_TYPE: self.get_sids_to_frames( + zip_with_strs, next_dividend_types, next_date_intervals, dates, + 'category', None + ), } cols[DAYS_TO_NEXT_EX_DATE] = self._compute_busday_offsets( @@ -356,7 +396,9 @@ class BlazeDividendsByExDateLoaderTestCase(DividendsByExDateTestCase): EX_DATE_FIELD_NAME: df[EX_DATE_FIELD_NAME], TS_FIELD_NAME: df[TS_FIELD_NAME], SID_FIELD_NAME: sid, - CASH_AMOUNT_FIELD_NAME: df[CASH_AMOUNT_FIELD_NAME] + CASH_AMOUNT_FIELD_NAME: df[CASH_AMOUNT_FIELD_NAME], + CURRENCY_FIELD_NAME: df[CURRENCY_FIELD_NAME], + DIVIDEND_TYPE_FIELD_NAME: df[DIVIDEND_TYPE_FIELD_NAME], }) for sid, df in iteritems(mapping) ).reset_index(drop=True)),) @@ -386,6 +428,8 @@ class DividendsByPayDateTestCase(WithPipelineEventDataLoader, ZiplineTestCase): PREVIOUS_AMOUNT: DividendsByPayDate.previous_amount.latest, PREVIOUS_CURRENCY_TYPE: DividendsByPayDate.previous_currency.latest, NEXT_CURRENCY_TYPE: DividendsByPayDate.next_currency.latest, + PREVIOUS_DIVIDEND_TYPE: DividendsByPayDate.previous_type.latest, + NEXT_DIVIDEND_TYPE: DividendsByPayDate.next_type.latest, } @classmethod @@ -425,7 +469,15 @@ class DividendsByPayDateTestCase(WithPipelineEventDataLoader, ZiplineTestCase): NEXT_CURRENCY_TYPE: self.get_sids_to_frames( zip_with_strs, next_currency_types, next_date_intervals, dates, 'category', None - ) + ), + PREVIOUS_DIVIDEND_TYPE: self.get_sids_to_frames( + zip_with_strs, prev_dividend_types, prev_date_intervals, dates, + 'category', None + ), + NEXT_DIVIDEND_TYPE: self.get_sids_to_frames( + zip_with_strs, next_dividend_types, next_date_intervals, dates, + 'category', None + ), } @@ -442,7 +494,9 @@ class BlazeDividendsByPayDateLoaderTestCase(DividendsByPayDateTestCase): PAY_DATE_FIELD_NAME: df[PAY_DATE_FIELD_NAME], TS_FIELD_NAME: df[TS_FIELD_NAME], SID_FIELD_NAME: sid, - CASH_AMOUNT_FIELD_NAME: df[CASH_AMOUNT_FIELD_NAME] + CASH_AMOUNT_FIELD_NAME: df[CASH_AMOUNT_FIELD_NAME], + CURRENCY_FIELD_NAME: df[CURRENCY_FIELD_NAME], + DIVIDEND_TYPE_FIELD_NAME: df[DIVIDEND_TYPE_FIELD_NAME], }) for sid, df in iteritems(mapping) ).reset_index(drop=True)),) diff --git a/tests/pipeline/test_earnings.py b/tests/pipeline/test_earnings.py index f645edfa..74f58597 100644 --- a/tests/pipeline/test_earnings.py +++ b/tests/pipeline/test_earnings.py @@ -55,7 +55,8 @@ class EarningsCalendarLoaderTestCase(WithNextAndPreviousEventDataLoader, 'datetime64[ns]', 'NaN' ), NEXT_ANNOUNCEMENT: self.get_expected_next_event_dates( - dates, 'datetime64[ns]', 'NaN'), + dates, 'datetime64[ns]', 'NaN' + ), } cols[DAYS_TO_NEXT] = self._compute_busday_offsets( cols[NEXT_ANNOUNCEMENT] diff --git a/zipline/pipeline/common.py b/zipline/pipeline/common.py index d75ee558..d45a85f4 100644 --- a/zipline/pipeline/common.py +++ b/zipline/pipeline/common.py @@ -22,6 +22,7 @@ DAYS_SINCE_PREV_EX_DATE = 'days_since_prev_ex_date' DAYS_TO_NEXT = 'days_to_next' DAYS_TO_NEXT_EX_DATE = 'days_to_next_ex_date' DISCLOSURE_DATE = 'disclosure_date' +DIVIDEND_TYPE_FIELD_NAME = 'dividend_type' EX_DATE_FIELD_NAME = 'ex_date' FISCAL_QUARTER_FIELD_NAME = 'fiscal_quarter' FISCAL_YEAR_FIELD_NAME = 'fiscal_year' @@ -32,6 +33,7 @@ NEXT_AMOUNT = 'next_amount' NEXT_ANNOUNCEMENT = 'next_announcement' NEXT_COUNT = 'next_count' NEXT_CURRENCY_TYPE = 'next_currency_type' +NEXT_DIVIDEND_TYPE = 'next_dividend_type' NEXT_EX_DATE = 'next_ex_date' NEXT_FISCAL_QUARTER = 'next_fiscal_quarter' NEXT_FISCAL_YEAR = 'next_fiscal_year' @@ -51,12 +53,12 @@ PREVIOUS_BUYBACK_AMOUNT = 'previous_value' PREVIOUS_BUYBACK_ANNOUNCEMENT = 'previous_buyback_announcement' PREVIOUS_BUYBACK_CASH = 'previous_buyback_cash' PREVIOUS_BUYBACK_SHARE_COUNT = 'previous_buyback_share_count' -PREVIOUS_DISCLOSURE_DATE = 'previous_disclosure_date' PREVIOUS_COUNT = 'previous_count' PREVIOUS_BUYBACK_TYPE = 'previous_buyback_type' -PREVIOUS_VALUE = 'previous_value' -PREVIOUS_VALUE_TYPE = 'previous_value_type' +PREVIOUS_BUYBACK_UNIT = 'previous_buyback_unit' PREVIOUS_CURRENCY_TYPE = 'previous_currency_type' +PREVIOUS_DISCLOSURE_DATE = 'previous_disclosure_date' +PREVIOUS_DIVIDEND_TYPE = 'previous_dividend_type' PREVIOUS_EX_DATE = 'previous_ex_date' PREVIOUS_FISCAL_QUARTER = 'previous_fiscal_quarter' PREVIOUS_FISCAL_YEAR = 'previous_fiscal_year' diff --git a/zipline/pipeline/data/dividends.py b/zipline/pipeline/data/dividends.py index 1178ed9a..042e74e8 100644 --- a/zipline/pipeline/data/dividends.py +++ b/zipline/pipeline/data/dividends.py @@ -17,6 +17,8 @@ class DividendsByExDate(DataSet): previous_amount = Column(float64_dtype) next_currency = Column(categorical_dtype) previous_currency = Column(categorical_dtype) + previous_type = Column(categorical_dtype) + next_type = Column(categorical_dtype) class DividendsByPayDate(DataSet): @@ -26,9 +28,12 @@ class DividendsByPayDate(DataSet): previous_amount = Column(float64_dtype) next_currency = Column(categorical_dtype) previous_currency = Column(categorical_dtype) + previous_type = Column(categorical_dtype) + next_type = Column(categorical_dtype) class DividendsByAnnouncementDate(DataSet): previous_announcement_date = Column(datetime64ns_dtype) previous_amount = Column(float64_dtype) previous_currency = Column(categorical_dtype) + previous_type = Column(categorical_dtype) diff --git a/zipline/pipeline/loaders/blaze/dividends.py b/zipline/pipeline/loaders/blaze/dividends.py index e7b84058..081e84ed 100644 --- a/zipline/pipeline/loaders/blaze/dividends.py +++ b/zipline/pipeline/loaders/blaze/dividends.py @@ -2,6 +2,7 @@ from zipline.pipeline.common import ( ANNOUNCEMENT_FIELD_NAME, CASH_AMOUNT_FIELD_NAME, CURRENCY_FIELD_NAME, + DIVIDEND_TYPE_FIELD_NAME, EX_DATE_FIELD_NAME, PAY_DATE_FIELD_NAME, SID_FIELD_NAME, @@ -49,6 +50,7 @@ class BlazeDividendsByAnnouncementDateLoader(BlazeEventsLoader): {CASH_AMOUNT_FIELD_NAME}: ?float64, {ANNOUNCEMENT_FIELD_NAME}: ?datetime, {CURRENCY_FIELD_NAME}: ?string, + {DIVIDEND_TYPE_FIELD_NAME}: ?string, }} Where each row of the table is a record including the sid to identify the @@ -65,6 +67,7 @@ class BlazeDividendsByAnnouncementDateLoader(BlazeEventsLoader): CASH_AMOUNT_FIELD_NAME=CASH_AMOUNT_FIELD_NAME, ANNOUNCEMENT_FIELD_NAME=ANNOUNCEMENT_FIELD_NAME, CURRENCY_FIELD_NAME=CURRENCY_FIELD_NAME, + DIVIDEND_TYPE_FIELD_NAME=DIVIDEND_TYPE_FIELD_NAME, ) _expected_fields = frozenset({ @@ -72,7 +75,8 @@ class BlazeDividendsByAnnouncementDateLoader(BlazeEventsLoader): SID_FIELD_NAME, CASH_AMOUNT_FIELD_NAME, CURRENCY_FIELD_NAME, - ANNOUNCEMENT_FIELD_NAME + ANNOUNCEMENT_FIELD_NAME, + DIVIDEND_TYPE_FIELD_NAME }) concrete_loader = DividendsByAnnouncementDateLoader @@ -108,6 +112,7 @@ class BlazeDividendsByExDateLoader(BlazeEventsLoader): {EX_DATE_FIELD_NAME}: ?datetime, {CASH_AMOUNT_FIELD_NAME}: ?datetime, {CURRENCY_FIELD_NAME}: ?string, + {DIVIDEND_TYPE_FIELD_NAME}: ?string, }} Where each row of the table is a record including the sid to identify the @@ -124,6 +129,7 @@ class BlazeDividendsByExDateLoader(BlazeEventsLoader): EX_DATE_FIELD_NAME=EX_DATE_FIELD_NAME, CASH_AMOUNT_FIELD_NAME=CASH_AMOUNT_FIELD_NAME, CURRENCY_FIELD_NAME=CURRENCY_FIELD_NAME, + DIVIDEND_TYPE_FIELD_NAME=DIVIDEND_TYPE_FIELD_NAME, ) _expected_fields = frozenset({ @@ -132,6 +138,7 @@ class BlazeDividendsByExDateLoader(BlazeEventsLoader): EX_DATE_FIELD_NAME, CASH_AMOUNT_FIELD_NAME, CURRENCY_FIELD_NAME, + DIVIDEND_TYPE_FIELD_NAME, }) concrete_loader = DividendsByExDateLoader @@ -167,6 +174,7 @@ class BlazeDividendsByPayDateLoader(BlazeEventsLoader): {PAY_DATE_FIELD_NAME}: ?datetime, {CASH_AMOUNT_FIELD_NAME}: ?datetime, {CURRENCY_FIELD_NAME}: ?string, + {DIVIDEND_TYPE_FIELD_NAME}: ?string, }} Where each row of the table is a record including the sid to identify the @@ -182,7 +190,8 @@ class BlazeDividendsByPayDateLoader(BlazeEventsLoader): SID_FIELD_NAME=SID_FIELD_NAME, PAY_DATE_FIELD_NAME=PAY_DATE_FIELD_NAME, CASH_AMOUNT_FIELD_NAME=CASH_AMOUNT_FIELD_NAME, - CURRENCY_FIELD_NAME=CURRENCY_FIELD_NAME + CURRENCY_FIELD_NAME=CURRENCY_FIELD_NAME, + DIVIDEND_TYPE_FIELD_NAME=DIVIDEND_TYPE_FIELD_NAME ) _expected_fields = frozenset({ @@ -191,6 +200,7 @@ class BlazeDividendsByPayDateLoader(BlazeEventsLoader): PAY_DATE_FIELD_NAME, CASH_AMOUNT_FIELD_NAME, CURRENCY_FIELD_NAME, + DIVIDEND_TYPE_FIELD_NAME, }) concrete_loader = DividendsByPayDateLoader diff --git a/zipline/pipeline/loaders/dividends.py b/zipline/pipeline/loaders/dividends.py index 51723216..5732c4cc 100644 --- a/zipline/pipeline/loaders/dividends.py +++ b/zipline/pipeline/loaders/dividends.py @@ -2,6 +2,7 @@ from zipline.pipeline.common import ( ANNOUNCEMENT_FIELD_NAME, CASH_AMOUNT_FIELD_NAME, CURRENCY_FIELD_NAME, + DIVIDEND_TYPE_FIELD_NAME, EX_DATE_FIELD_NAME, PAY_DATE_FIELD_NAME, ) @@ -17,7 +18,8 @@ from zipline.utils.memoize import lazyval class DividendsByAnnouncementDateLoader(EventsLoader): expected_cols = frozenset([ANNOUNCEMENT_FIELD_NAME, CASH_AMOUNT_FIELD_NAME, - CURRENCY_FIELD_NAME]) + CURRENCY_FIELD_NAME, + DIVIDEND_TYPE_FIELD_NAME]) event_date_col = ANNOUNCEMENT_FIELD_NAME @@ -48,11 +50,19 @@ class DividendsByAnnouncementDateLoader(EventsLoader): CURRENCY_FIELD_NAME ) + @lazyval + def previous_type_loader(self): + return self._previous_event_value_loader( + self.dataset.previous_type, + DIVIDEND_TYPE_FIELD_NAME + ) + class DividendsByPayDateLoader(EventsLoader): expected_cols = frozenset([PAY_DATE_FIELD_NAME, CASH_AMOUNT_FIELD_NAME, - CURRENCY_FIELD_NAME]) + CURRENCY_FIELD_NAME, + DIVIDEND_TYPE_FIELD_NAME]) event_date_col = PAY_DATE_FIELD_NAME @@ -99,11 +109,26 @@ class DividendsByPayDateLoader(EventsLoader): CURRENCY_FIELD_NAME ) + @lazyval + def previous_type_loader(self): + return self._previous_event_value_loader( + self.dataset.previous_type, + DIVIDEND_TYPE_FIELD_NAME + ) + + @lazyval + def next_type_loader(self): + return self._next_event_value_loader( + self.dataset.next_type, + DIVIDEND_TYPE_FIELD_NAME + ) + class DividendsByExDateLoader(EventsLoader): expected_cols = frozenset([EX_DATE_FIELD_NAME, CASH_AMOUNT_FIELD_NAME, - CURRENCY_FIELD_NAME]) + CURRENCY_FIELD_NAME, + DIVIDEND_TYPE_FIELD_NAME]) event_date_col = EX_DATE_FIELD_NAME @@ -149,3 +174,17 @@ class DividendsByExDateLoader(EventsLoader): self.dataset.next_currency, CURRENCY_FIELD_NAME ) + + @lazyval + def previous_type_loader(self): + return self._previous_event_value_loader( + self.dataset.previous_type, + DIVIDEND_TYPE_FIELD_NAME + ) + + @lazyval + def next_type_loader(self): + return self._next_event_value_loader( + self.dataset.next_type, + DIVIDEND_TYPE_FIELD_NAME + )