From c9e165aa2de0ed3ec9c79ab2302849c9ea4ebede Mon Sep 17 00:00:00 2001 From: Scott Sanderson Date: Thu, 22 Oct 2015 07:22:35 -0400 Subject: [PATCH] ENH: Rewrite Canadian treasury loader. --- zipline/data/treasuries_can.py | 213 ++++++++++++++++++--------------- 1 file changed, 119 insertions(+), 94 deletions(-) diff --git a/zipline/data/treasuries_can.py b/zipline/data/treasuries_can.py index 6d579aa2..bb457fd7 100644 --- a/zipline/data/treasuries_can.py +++ b/zipline/data/treasuries_can.py @@ -13,113 +13,138 @@ # See the License for the specific language governing permissions and # limitations under the License. -import datetime -import requests +import pandas as pd +import six +from toolz import curry +from toolz.curried.operator import add as prepend -from . loader_utils import ( - source_to_records -) - -from zipline.data.treasuries import ( - treasury_mappings, get_treasury_date, get_treasury_rate -) - - -_CURVE_MAPPINGS = { - 'date': (get_treasury_date, "Date"), - '1month': (get_treasury_rate, "V39063"), - '3month': (get_treasury_rate, "V39065"), - '6month': (get_treasury_rate, "V39066"), - '1year': (get_treasury_rate, "V39067"), - '2year': (get_treasury_rate, "V39051"), - '3year': (get_treasury_rate, "V39052"), - '5year': (get_treasury_rate, "V39053"), - '7year': (get_treasury_rate, "V39054"), - '10year': (get_treasury_rate, "V39055"), +COLUMN_NAMES = { + "V39063": '1month', + "V39065": '3month', + "V39066": '6month', + "V39067": '1year', + "V39051": '2year', + "V39052": '3year', + "V39053": '5year', + "V39054": '7year', + "V39055": '10year', # Bank of Canada refers to this as 'Long' Rate, approximately 30 years. - '30year': (get_treasury_rate, "V39056"), + "V39056": '30year', } +BILL_IDS = ['V39063', 'V39065', 'V39066', 'V39067'] +BOND_IDS = ['V39051', 'V39052', 'V39053', 'V39054', 'V39055', 'V39056'] -BILLS = ['V39063', 'V39065', 'V39066', 'V39067'] -BONDS = ['V39051', 'V39052', 'V39053', 'V39054', 'V39055', 'V39056'] + +@curry +def _format_url(instrument_type, + instrument_ids, + start_date, + end_date, + earliest_allowed_date): + """ + Format a URL for loading data from Bank of Canada. + """ + return ( + "http://www.bankofcanada.ca/stats/results/csv" + "?lP=lookup_{instrument_type}_yields.php" + "&sR={restrict}" + "&se={instrument_ids}" + "&dF={start}" + "&dT={end}".format( + instrument_type=instrument_type, + instrument_ids='-'.join(map(prepend("L_"), instrument_ids)), + restrict=earliest_allowed_date.strftime("%Y-%m-%d"), + start=start_date.strftime("%Y-%m-%d"), + end=end_date.strftime("%Y-%m-%d"), + ) + ) + + +format_bill_url = _format_url('tbill', BILL_IDS) +format_bond_url = _format_url('bond', BOND_IDS) + + +def load_frame(url, skiprows): + """ + Load a DataFrame of data from a Bank of Canada site. + """ + return pd.read_csv( + url, + skiprows=skiprows, + skipinitialspace=True, + na_values=["Bank holiday", "Not available"], + parse_dates=["Date"], + index_col="Date", + ).dropna(how='all') \ + .tz_localize('UTC') \ + .rename(columns=COLUMN_NAMES) + + +def check_known_inconsistencies(bill_data, bond_data): + """ + There are a couple quirks in the data provided by Bank of Canada. + Check that no new quirks have been introduced in the latest download. + """ + inconsistent_dates = bill_data.index.sym_diff(bond_data.index) + known_inconsistencies = [ + # bill_data has an entry for 2010-02-15, which bond_data doesn't. + # bond_data has an entry for 2006-09-04, which bill_data doesn't. + # Both of these dates are bank holidays (Flag Day and Labor Day, + # respectively). + pd.Timestamp('2006-09-04', tz='UTC'), + pd.Timestamp('2010-02-15', tz='UTC'), + # 2013-07-25 comes back as "Not available" from the bills endpoint. + # This date doesn't seem to be a bank holiday, but the previous + # calendar implementation dropped this entry, so we drop it as well. + # If someone cares deeply about the integrity of the Canadian trading + # calendar, they may want to consider forward-filling here rather than + # dropping the row. + pd.Timestamp('2013-07-25', tz='UTC'), + ] + unexpected_inconsistences = inconsistent_dates.drop(known_inconsistencies) + if len(unexpected_inconsistences): + in_bills = bill_data.index.difference(bond_data.index).difference( + known_inconsistencies + ) + in_bonds = bond_data.index.difference(bill_data.index).difference( + known_inconsistencies + ) + raise ValueError( + "Inconsistent dates for Canadian treasury bills vs bonds. \n" + "Dates with bills but not bonds: {in_bills}.\n" + "Dates with bonds but not bills: {in_bonds}.".format( + in_bills=in_bills, + in_bonds=in_bonds, + ) + ) def get_treasury_source(start_date=None, end_date=None): - today = datetime.date.today() - # Bank of Canada only has 10 years of data and has this in the URL. - restriction = datetime.date(today.year - 10, today.month, today.day) - + today = pd.Timestamp('now').normalize() + # Bank of Canada only has the last 10 years of data at any given time. + earliest_date = today.date().replace(year=today.year - 10) if not end_date: end_date = today - if not start_date: - start_date = restriction + start_date = earliest_date - bill_url = ( - "http://www.bankofcanada.ca/stats/results/csv?" - "lP=lookup_tbill_yields.php&sR={restrict}&se=" - "L_V39063-L_V39065-L_V39066-L_V39067&dF={start}&dT={end}" - .format(restrict=restriction.strftime("%Y-%m-%d"), - start=start_date.strftime("%Y-%m-%d"), - end=end_date.strftime("%Y-%m-%d"), - ) + bill_data = load_frame( + format_bill_url(start_date, end_date, earliest_date), + # We skip fewer rows here because we query for fewer bill fields, + # which makes the header smaller. + skiprows=18, ) - - bond_url = ( - "http://www.bankofcanada.ca/stats/results/csv?" - "lP=lookup_bond_yields.php&sR={restrict}&se=" - "L_V39051-L_V39052-L_V39053-L_V39054-L_V39055-L_V39056" - "&dF={start}&dT={end}" - .format(restrict=restriction.strftime("%Y-%m-%d"), - start=start_date.strftime("%Y-%m-%d"), - end=end_date.strftime("%Y-%m-%d") - ) + bond_data = load_frame( + format_bond_url(start_date, end_date, earliest_date), + skiprows=22, ) + check_known_inconsistencies(bill_data, bond_data) - res_bill = requests.get(bill_url, stream=True) - res_bond = requests.get(bond_url, stream=True) - bill_iter = res_bill.iter_lines() - bond_iter = res_bond.iter_lines() + # dropna('any') removes the rows for which we only had data for one of + # bills/bonds. + out = pd.concat([bond_data, bill_data], axis=1).dropna(how='any') + assert set(out.columns) == set(six.itervalues(COLUMN_NAMES)) - bill_row = "" - while ",".join(BILLS) not in bill_row: - bill_row = bill_iter.next() - if 'Daily series:' in bill_row: - bill_end_date = datetime.datetime.strptime( - bill_row.split(' - ')[1].strip(), - "%Y-%m-%d").date() - bill_header = bill_row.split(",") - - bond_row = "" - while ",".join(BONDS) not in bond_row: - bond_row = bond_iter.next() - if 'Daily series:' in bond_row: - bond_end_date = datetime.datetime.strptime( - bond_row.split(' - ')[1].strip(), - "%Y-%m-%d").date() - bond_header = bond_row.split(",") - - # Line up the two dates - if bill_end_date > bond_end_date: - bill_iter.next() - elif bond_end_date > bill_end_date: - bond_iter.next() - - for bill_row in bill_iter: - bond_row = bond_iter.next() - bill_dict = dict(zip(bill_header, bill_row.split(","))) - bond_dict = dict(zip(bond_header, bond_row.split(","))) - if ' Bank holiday' in bond_row.split(",") + bill_row.split(","): - continue - if ' Not available' in bond_row.split(",") + bill_row.split(","): - continue - - bill_dict.update(bond_dict) - yield bill_dict - - -def get_treasury_data(): - mappings = treasury_mappings(_CURVE_MAPPINGS) - source = get_treasury_source() - return source_to_records(mappings, source) + # Multiply by 0.01 to convert from percentages to expected output format. + return out * 0.01