From 8a3b82c5369c74a12dbad0f07943f0faa58130b3 Mon Sep 17 00:00:00 2001 From: Maya Tydykov Date: Thu, 5 May 2016 11:20:53 -0400 Subject: [PATCH] ENH: add column for currency type --- tests/pipeline/test_dividends.py | 61 ++++++++++++++++++--- zipline/pipeline/common.py | 13 ++++- zipline/pipeline/data/dividends.py | 11 +++- zipline/pipeline/loaders/blaze/dividends.py | 12 +++- zipline/pipeline/loaders/dividends.py | 49 +++++++++++++++-- 5 files changed, 128 insertions(+), 18 deletions(-) diff --git a/tests/pipeline/test_dividends.py b/tests/pipeline/test_dividends.py index 7534ef73..c4e226ab 100644 --- a/tests/pipeline/test_dividends.py +++ b/tests/pipeline/test_dividends.py @@ -15,6 +15,7 @@ from zipline.pipeline.common import ( NEXT_EX_DATE, NEXT_PAY_DATE, PREVIOUS_ANNOUNCEMENT, + PREVIOUS_CURRENCY_TYPE, PREVIOUS_EX_DATE, PREVIOUS_PAY_DATE, PREVIOUS_AMOUNT, @@ -22,8 +23,8 @@ from zipline.pipeline.common import ( TS_FIELD_NAME, CASH_AMOUNT_FIELD_NAME, EX_DATE_FIELD_NAME, - PAY_DATE_FIELD_NAME -) + PAY_DATE_FIELD_NAME, + CURRENCY_FIELD_NAME, NEXT_CURRENCY_TYPE) from zipline.pipeline.data.dividends import ( DividendsByAnnouncementDate, DividendsByExDate, @@ -46,7 +47,8 @@ from zipline.pipeline.loaders.dividends import ( ) from zipline.pipeline.loaders.utils import ( zip_with_dates, - zip_with_floats + zip_with_floats, + zip_with_strs, ) from zipline.testing.fixtures import ( WithPipelineEventDataLoader, @@ -60,7 +62,8 @@ dividends_cases = [ EX_DATE_FIELD_NAME: pd.to_datetime(['2014-01-15', '2014-01-20']), PAY_DATE_FIELD_NAME: pd.to_datetime(['2014-01-15', '2014-01-20']), TS_FIELD_NAME: pd.to_datetime(['2014-01-05', '2014-01-10']), - ANNOUNCEMENT_FIELD_NAME: pd.to_datetime(['2014-01-04', '2014-01-09']) + ANNOUNCEMENT_FIELD_NAME: pd.to_datetime(['2014-01-04', '2014-01-09']), + CURRENCY_FIELD_NAME: ["$", "EUR"] }), # K1--K2--A2--A1. pd.DataFrame({ @@ -68,7 +71,8 @@ dividends_cases = [ EX_DATE_FIELD_NAME: pd.to_datetime(['2014-01-20', '2014-01-15']), PAY_DATE_FIELD_NAME: pd.to_datetime(['2014-01-20', '2014-01-15']), TS_FIELD_NAME: pd.to_datetime(['2014-01-05', '2014-01-10']), - ANNOUNCEMENT_FIELD_NAME: pd.to_datetime(['2014-01-04', '2014-01-09']) + ANNOUNCEMENT_FIELD_NAME: pd.to_datetime(['2014-01-04', '2014-01-09']), + CURRENCY_FIELD_NAME: ["EUR", "$"] }), # K1--A1--K2--A2. pd.DataFrame({ @@ -76,7 +80,8 @@ dividends_cases = [ EX_DATE_FIELD_NAME: pd.to_datetime(['2014-01-10', '2014-01-20']), PAY_DATE_FIELD_NAME: pd.to_datetime(['2014-01-10', '2014-01-20']), TS_FIELD_NAME: pd.to_datetime(['2014-01-05', '2014-01-15']), - ANNOUNCEMENT_FIELD_NAME: pd.to_datetime(['2014-01-04', '2014-01-14']) + ANNOUNCEMENT_FIELD_NAME: pd.to_datetime(['2014-01-04', '2014-01-14']), + CURRENCY_FIELD_NAME: ["$", "EUR"] }), # K1 == K2. pd.DataFrame({ @@ -84,14 +89,16 @@ dividends_cases = [ EX_DATE_FIELD_NAME: pd.to_datetime(['2014-01-10', '2014-01-15']), PAY_DATE_FIELD_NAME: pd.to_datetime(['2014-01-10', '2014-01-15']), TS_FIELD_NAME: pd.to_datetime(['2014-01-05'] * 2), - ANNOUNCEMENT_FIELD_NAME: pd.to_datetime(['2014-01-04', '2014-01-04']) + ANNOUNCEMENT_FIELD_NAME: pd.to_datetime(['2014-01-04', '2014-01-04']), + CURRENCY_FIELD_NAME: ["$", "EUR"] }), pd.DataFrame( columns=[CASH_AMOUNT_FIELD_NAME, EX_DATE_FIELD_NAME, PAY_DATE_FIELD_NAME, TS_FIELD_NAME, - ANNOUNCEMENT_FIELD_NAME], + ANNOUNCEMENT_FIELD_NAME, + CURRENCY_FIELD_NAME], dtype='datetime64[ns]' ), ] @@ -157,6 +164,16 @@ next_amounts = [['NaN', 1, 15, 'NaN'], ['NaN', 3, 'NaN', 1, 'NaN'], ['NaN', 6, 23, 'NaN']] +prev_currency_types = [[None, "$", "EUR"], + [None, "$", "EUR"], + [None, "$", "EUR"], + [None, "$", "EUR"]] + +next_currency_types = [[None, "$", "EUR", None], + [None, "EUR", "$", "EUR", None], + [None, "$", None, "EUR", None], + [None, "$", "EUR", None]] + class DividendsByAnnouncementDateTestCase(WithPipelineEventDataLoader, ZiplineTestCase): @@ -169,6 +186,8 @@ class DividendsByAnnouncementDateTestCase(WithPipelineEventDataLoader, PREVIOUS_AMOUNT: DividendsByAnnouncementDate.previous_amount.latest, DAYS_SINCE_PREV_DIVIDEND_ANNOUNCEMENT: BusinessDaysSinceDividendAnnouncement(), + PREVIOUS_CURRENCY_TYPE: + DividendsByAnnouncementDate.previous_currency.latest, } @classmethod @@ -204,6 +223,8 @@ class DividendsByAnnouncementDateTestCase(WithPipelineEventDataLoader, ['NaT', '2014-01-04', '2014-01-14'], ['NaT', '2014-01-04']] amounts = [['NaN', 1, 15], ['NaN', 7, 13], ['NaN', 3, 1], ['NaN', 23]] + currency_types = [[None, "$", "EUR"], [None, "$", "EUR"], + [None, "$", "EUR"]] cols = { PREVIOUS_ANNOUNCEMENT: self.get_sids_to_frames( zip_with_dates, announcement_dates, date_intervals, dates, @@ -212,6 +233,10 @@ class DividendsByAnnouncementDateTestCase(WithPipelineEventDataLoader, PREVIOUS_AMOUNT: self.get_sids_to_frames( zip_with_floats, amounts, date_intervals, dates, 'float', 'NaN' ), + PREVIOUS_CURRENCY_TYPE: self.get_sids_to_frames( + zip_with_strs, currency_types, date_intervals, dates, + 'category', None + ), } cols[ @@ -263,6 +288,8 @@ class DividendsByExDateTestCase(WithPipelineEventDataLoader, ZiplineTestCase): PREVIOUS_EX_DATE: DividendsByExDate.previous_date.latest, NEXT_AMOUNT: DividendsByExDate.next_amount.latest, PREVIOUS_AMOUNT: DividendsByExDate.previous_amount.latest, + PREVIOUS_CURRENCY_TYPE: DividendsByExDate.previous_currency.latest, + NEXT_CURRENCY_TYPE: DividendsByExDate.next_currency.latest, DAYS_TO_NEXT_EX_DATE: BusinessDaysUntilNextExDate(), DAYS_SINCE_PREV_EX_DATE: BusinessDaysSincePreviousExDate() } @@ -296,6 +323,14 @@ class DividendsByExDateTestCase(WithPipelineEventDataLoader, ZiplineTestCase): PREVIOUS_AMOUNT: self.get_sids_to_frames( zip_with_floats, prev_amounts, prev_date_intervals, dates, 'float', 'NaN' + ), + PREVIOUS_CURRENCY_TYPE: self.get_sids_to_frames( + zip_with_strs, prev_currency_types, prev_date_intervals, dates, + 'category', None + ), + NEXT_CURRENCY_TYPE: self.get_sids_to_frames( + zip_with_strs, next_currency_types, next_date_intervals, dates, + 'category', None ) } @@ -349,6 +384,8 @@ class DividendsByPayDateTestCase(WithPipelineEventDataLoader, ZiplineTestCase): PREVIOUS_PAY_DATE: DividendsByPayDate.previous_date.latest, NEXT_AMOUNT: DividendsByPayDate.next_amount.latest, PREVIOUS_AMOUNT: DividendsByPayDate.previous_amount.latest, + PREVIOUS_CURRENCY_TYPE: DividendsByPayDate.previous_currency.latest, + NEXT_CURRENCY_TYPE: DividendsByPayDate.next_currency.latest, } @classmethod @@ -380,6 +417,14 @@ class DividendsByPayDateTestCase(WithPipelineEventDataLoader, ZiplineTestCase): PREVIOUS_AMOUNT: self.get_sids_to_frames( zip_with_floats, prev_amounts, prev_date_intervals, dates, 'float', 'NaN' + ), + PREVIOUS_CURRENCY_TYPE: self.get_sids_to_frames( + zip_with_strs, prev_currency_types, prev_date_intervals, dates, + 'category', None + ), + NEXT_CURRENCY_TYPE: self.get_sids_to_frames( + zip_with_strs, next_currency_types, next_date_intervals, dates, + 'category', None ) } diff --git a/zipline/pipeline/common.py b/zipline/pipeline/common.py index 9dbab69d..d75ee558 100644 --- a/zipline/pipeline/common.py +++ b/zipline/pipeline/common.py @@ -5,6 +5,10 @@ ACTUAL_VALUE_FIELD_NAME = 'actual_value' AD_FIELD_NAME = 'asof_date' ANNOUNCEMENT_FIELD_NAME = 'announcement_date' BUYBACK_AMOUNT_FIELD_NAME = 'buyback_amount' +CASH_FIELD_NAME = 'cash' +CASH_AMOUNT_FIELD_NAME = 'cash_amount' +COUNT_FIELD_NAME = 'count' +CURRENCY_FIELD_NAME = 'currency_type' BUYBACK_ANNOUNCEMENT_FIELD_NAME = 'buyback_date' BUYBACK_TYPE_FIELD_NAME = 'buyback_type' BUYBACK_UNIT_FIELD_NAME = 'buyback_unit' @@ -27,6 +31,7 @@ MEAN_FIELD_NAME = 'mean' NEXT_AMOUNT = 'next_amount' NEXT_ANNOUNCEMENT = 'next_announcement' NEXT_COUNT = 'next_count' +NEXT_CURRENCY_TYPE = 'next_currency_type' NEXT_EX_DATE = 'next_ex_date' NEXT_FISCAL_QUARTER = 'next_fiscal_quarter' NEXT_FISCAL_YEAR = 'next_fiscal_year' @@ -46,10 +51,12 @@ PREVIOUS_BUYBACK_AMOUNT = 'previous_value' PREVIOUS_BUYBACK_ANNOUNCEMENT = 'previous_buyback_announcement' PREVIOUS_BUYBACK_CASH = 'previous_buyback_cash' PREVIOUS_BUYBACK_SHARE_COUNT = 'previous_buyback_share_count' -PREVIOUS_BUYBACK_TYPE = 'previous_buyback_type' -PREVIOUS_BUYBACK_UNIT = 'previous_value_type' -PREVIOUS_COUNT = 'previous_count' PREVIOUS_DISCLOSURE_DATE = 'previous_disclosure_date' +PREVIOUS_COUNT = 'previous_count' +PREVIOUS_BUYBACK_TYPE = 'previous_buyback_type' +PREVIOUS_VALUE = 'previous_value' +PREVIOUS_VALUE_TYPE = 'previous_value_type' +PREVIOUS_CURRENCY_TYPE = 'previous_currency_type' PREVIOUS_EX_DATE = 'previous_ex_date' PREVIOUS_FISCAL_QUARTER = 'previous_fiscal_quarter' PREVIOUS_FISCAL_YEAR = 'previous_fiscal_year' diff --git a/zipline/pipeline/data/dividends.py b/zipline/pipeline/data/dividends.py index 176aac43..1178ed9a 100644 --- a/zipline/pipeline/data/dividends.py +++ b/zipline/pipeline/data/dividends.py @@ -1,7 +1,11 @@ """ Dataset representing dates of upcoming dividends. """ -from zipline.utils.numpy_utils import datetime64ns_dtype, float64_dtype +from zipline.utils.numpy_utils import ( + datetime64ns_dtype, + float64_dtype, + categorical_dtype +) from .dataset import Column, DataSet @@ -11,6 +15,8 @@ class DividendsByExDate(DataSet): previous_date = Column(datetime64ns_dtype) next_amount = Column(float64_dtype) previous_amount = Column(float64_dtype) + next_currency = Column(categorical_dtype) + previous_currency = Column(categorical_dtype) class DividendsByPayDate(DataSet): @@ -18,8 +24,11 @@ class DividendsByPayDate(DataSet): previous_date = Column(datetime64ns_dtype) next_amount = Column(float64_dtype) previous_amount = Column(float64_dtype) + next_currency = Column(categorical_dtype) + previous_currency = Column(categorical_dtype) class DividendsByAnnouncementDate(DataSet): previous_announcement_date = Column(datetime64ns_dtype) previous_amount = Column(float64_dtype) + previous_currency = Column(categorical_dtype) diff --git a/zipline/pipeline/loaders/blaze/dividends.py b/zipline/pipeline/loaders/blaze/dividends.py index 6fd86be1..e7b84058 100644 --- a/zipline/pipeline/loaders/blaze/dividends.py +++ b/zipline/pipeline/loaders/blaze/dividends.py @@ -1,6 +1,7 @@ from zipline.pipeline.common import ( ANNOUNCEMENT_FIELD_NAME, CASH_AMOUNT_FIELD_NAME, + CURRENCY_FIELD_NAME, EX_DATE_FIELD_NAME, PAY_DATE_FIELD_NAME, SID_FIELD_NAME, @@ -47,6 +48,7 @@ class BlazeDividendsByAnnouncementDateLoader(BlazeEventsLoader): {TS_FIELD_NAME}: datetime, {CASH_AMOUNT_FIELD_NAME}: ?float64, {ANNOUNCEMENT_FIELD_NAME}: ?datetime, + {CURRENCY_FIELD_NAME}: ?string, }} Where each row of the table is a record including the sid to identify the @@ -61,13 +63,15 @@ class BlazeDividendsByAnnouncementDateLoader(BlazeEventsLoader): TS_FIELD_NAME=TS_FIELD_NAME, SID_FIELD_NAME=SID_FIELD_NAME, CASH_AMOUNT_FIELD_NAME=CASH_AMOUNT_FIELD_NAME, - ANNOUNCEMENT_FIELD_NAME=ANNOUNCEMENT_FIELD_NAME + ANNOUNCEMENT_FIELD_NAME=ANNOUNCEMENT_FIELD_NAME, + CURRENCY_FIELD_NAME=CURRENCY_FIELD_NAME, ) _expected_fields = frozenset({ TS_FIELD_NAME, SID_FIELD_NAME, CASH_AMOUNT_FIELD_NAME, + CURRENCY_FIELD_NAME, ANNOUNCEMENT_FIELD_NAME }) @@ -103,6 +107,7 @@ class BlazeDividendsByExDateLoader(BlazeEventsLoader): {TS_FIELD_NAME}: datetime, {EX_DATE_FIELD_NAME}: ?datetime, {CASH_AMOUNT_FIELD_NAME}: ?datetime, + {CURRENCY_FIELD_NAME}: ?string, }} Where each row of the table is a record including the sid to identify the @@ -118,6 +123,7 @@ class BlazeDividendsByExDateLoader(BlazeEventsLoader): SID_FIELD_NAME=SID_FIELD_NAME, EX_DATE_FIELD_NAME=EX_DATE_FIELD_NAME, CASH_AMOUNT_FIELD_NAME=CASH_AMOUNT_FIELD_NAME, + CURRENCY_FIELD_NAME=CURRENCY_FIELD_NAME, ) _expected_fields = frozenset({ @@ -125,6 +131,7 @@ class BlazeDividendsByExDateLoader(BlazeEventsLoader): SID_FIELD_NAME, EX_DATE_FIELD_NAME, CASH_AMOUNT_FIELD_NAME, + CURRENCY_FIELD_NAME, }) concrete_loader = DividendsByExDateLoader @@ -159,6 +166,7 @@ class BlazeDividendsByPayDateLoader(BlazeEventsLoader): {TS_FIELD_NAME}: datetime, {PAY_DATE_FIELD_NAME}: ?datetime, {CASH_AMOUNT_FIELD_NAME}: ?datetime, + {CURRENCY_FIELD_NAME}: ?string, }} Where each row of the table is a record including the sid to identify the @@ -174,6 +182,7 @@ class BlazeDividendsByPayDateLoader(BlazeEventsLoader): SID_FIELD_NAME=SID_FIELD_NAME, PAY_DATE_FIELD_NAME=PAY_DATE_FIELD_NAME, CASH_AMOUNT_FIELD_NAME=CASH_AMOUNT_FIELD_NAME, + CURRENCY_FIELD_NAME=CURRENCY_FIELD_NAME ) _expected_fields = frozenset({ @@ -181,6 +190,7 @@ class BlazeDividendsByPayDateLoader(BlazeEventsLoader): SID_FIELD_NAME, PAY_DATE_FIELD_NAME, CASH_AMOUNT_FIELD_NAME, + CURRENCY_FIELD_NAME, }) concrete_loader = DividendsByPayDateLoader diff --git a/zipline/pipeline/loaders/dividends.py b/zipline/pipeline/loaders/dividends.py index 339435c1..51723216 100644 --- a/zipline/pipeline/loaders/dividends.py +++ b/zipline/pipeline/loaders/dividends.py @@ -1,8 +1,9 @@ from zipline.pipeline.common import ( + ANNOUNCEMENT_FIELD_NAME, + CASH_AMOUNT_FIELD_NAME, + CURRENCY_FIELD_NAME, EX_DATE_FIELD_NAME, PAY_DATE_FIELD_NAME, - CASH_AMOUNT_FIELD_NAME, - ANNOUNCEMENT_FIELD_NAME ) from zipline.pipeline.loaders.events import EventsLoader from ..data import ( @@ -15,7 +16,8 @@ from zipline.utils.memoize import lazyval class DividendsByAnnouncementDateLoader(EventsLoader): expected_cols = frozenset([ANNOUNCEMENT_FIELD_NAME, - CASH_AMOUNT_FIELD_NAME]) + CASH_AMOUNT_FIELD_NAME, + CURRENCY_FIELD_NAME]) event_date_col = ANNOUNCEMENT_FIELD_NAME @@ -39,10 +41,18 @@ class DividendsByAnnouncementDateLoader(EventsLoader): CASH_AMOUNT_FIELD_NAME ) + @lazyval + def previous_currency_loader(self): + return self._previous_event_value_loader( + self.dataset.previous_currency, + CURRENCY_FIELD_NAME + ) + class DividendsByPayDateLoader(EventsLoader): expected_cols = frozenset([PAY_DATE_FIELD_NAME, - CASH_AMOUNT_FIELD_NAME]) + CASH_AMOUNT_FIELD_NAME, + CURRENCY_FIELD_NAME]) event_date_col = PAY_DATE_FIELD_NAME @@ -75,10 +85,25 @@ class DividendsByPayDateLoader(EventsLoader): CASH_AMOUNT_FIELD_NAME ) + @lazyval + def previous_currency_loader(self): + return self._previous_event_value_loader( + self.dataset.previous_currency, + CURRENCY_FIELD_NAME + ) + + @lazyval + def next_currency_loader(self): + return self._next_event_value_loader( + self.dataset.next_currency, + CURRENCY_FIELD_NAME + ) + class DividendsByExDateLoader(EventsLoader): expected_cols = frozenset([EX_DATE_FIELD_NAME, - CASH_AMOUNT_FIELD_NAME]) + CASH_AMOUNT_FIELD_NAME, + CURRENCY_FIELD_NAME]) event_date_col = EX_DATE_FIELD_NAME @@ -110,3 +135,17 @@ class DividendsByExDateLoader(EventsLoader): self.dataset.previous_amount, CASH_AMOUNT_FIELD_NAME ) + + @lazyval + def previous_currency_loader(self): + return self._previous_event_value_loader( + self.dataset.previous_currency, + CURRENCY_FIELD_NAME + ) + + @lazyval + def next_currency_loader(self): + return self._next_event_value_loader( + self.dataset.next_currency, + CURRENCY_FIELD_NAME + )