mirror of
https://github.com/wassname/catalyst.git
synced 2026-07-15 11:22:18 +08:00
ENH: Rewrite treasury loader using pandas.
Replaces our custom XML parsing with a single call to `pd.read_csv`
against the federal reserve's API. This produces nearly identical
results as compared to the old loader, but it's dramatically simpler and
roughly 10x faster on my machine.
The average difference in magnitude between new and old is approximately
10e-7, and only one entry is different to a degree greater than the
number of significant figures provided by treasury.gov.
Additionally, the new loader correctly ignores Columbus Day of 2010, for
which the old loader erroneously produced an all-NaN row.
This also changes the interface that treasury modules modules are
required to implement. Modules must now supply a `get_treasury_data`
function that returns a `DataFrame` with a daily `DatetimeIndex` and a
column for each supported treasury duration.
Detailed comparison between results from new and old loader::
from zipline.data.treasuries import get_treasury_data
new = get_treasury_data() # New implementation
old = pd.read_csv( # Previously cached data
'/home/ssanderson/.zipline/data/treasury_curves.csv'
parse_dates=[0],
index_col=0,
)
# These columns were unused.
del old['tid']; del old['date']
old = old.tz_localize('UTC')
old.dropna(how='all')
# old data erroneously contained an all-NaN entry for Columbus Day
# in 2010. Remove before comparing.
old = old.dropna(how='all')
In [25]: len(new) == len(old)
Out[25]: True
In [26]: abs(old - new).max()
Out[26]:
10year 2.000000e-04
1month 6.938894e-18
1year 1.000000e-04
20year 1.000000e-04
2year 2.000000e-04
30year 1.000000e-04
3month 1.000000e-03
3year 1.000000e-04
5year 1.387779e-17
6month 1.000000e-04
7year 1.000000e-04
dtype: float64
In [27]: abs(old - new).mean()
Out[27]:
10year 3.097414e-08
1month 4.396534e-19
1year 1.548707e-08
20year 3.624502e-08
2year 4.646120e-08
30year 1.830496e-08
3month 1.549427e-07
3year 1.548707e-08
5year 1.702619e-18
6month 1.548707e-08
7year 1.548707e-08
dtype: float64
Since www.treasury.gov only reports values up to three significant
digits, we should only care about differences of greater than 1e-3.
There is exactly one such difference: the entry for the three month bond
on 1999-10-01::
In [60]: new[(abs(new - old) >= 1e-3).any(axis=1)].T
Out[60]:
Time Period 1999-10-01 00:00:00+00:00
1month NaN
3month 0.0498
6month 0.0501
1year 0.0530
2year 0.0573
3year 0.0583
5year 0.0590
7year 0.0622
10year 0.0600
20year 0.0657
30year 0.0615
In [61]: old[(abs(new - old) >= 1e-3).any(axis=1)].T
Out[61]:
1999-10-01 00:00:00+00:00
10year 0.0600
1month NaN
1year 0.0530
20year 0.0657
2year 0.0573
30year 0.0615
3month 0.0488
3year 0.0583
5year 0.0590
6month 0.0501
7year 0.0622
The US Treasury website (our old source) provides a value of 0.488 here,
whereas the Federal Reserve site (our new source) provides a value of
0.498.
This commit is contained in:
Scott Sanderson
@@ -84,17 +84,10 @@ def dump_treasury_curves(module_name, filename):
|
||||
raise NotImplementedError(
|
||||
'Treasury curve {0} module not implemented'.format(module_name))
|
||||
|
||||
tr_data = {}
|
||||
|
||||
for curve in m.get_treasury_data():
|
||||
# Not ideal but massaging data into expected format
|
||||
tr_data[curve['date']] = curve
|
||||
|
||||
curves = pd.DataFrame(tr_data).T
|
||||
curves = m.get_treasury_data()
|
||||
|
||||
data_filepath = get_data_filepath(filename)
|
||||
curves.to_csv(data_filepath)
|
||||
|
||||
return curves
|
||||
|
||||
|
||||
|
||||
+42
-110
@@ -16,127 +16,59 @@ import re
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import requests
|
||||
|
||||
from collections import OrderedDict
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
from six import iteritems
|
||||
|
||||
from . loader_utils import (
|
||||
guarded_conversion,
|
||||
safe_int,
|
||||
Mapping,
|
||||
date_conversion,
|
||||
source_to_records
|
||||
)
|
||||
|
||||
|
||||
def get_treasury_date(dstring):
|
||||
return date_conversion(dstring.split("T")[0], date_pattern='%Y-%m-%d',
|
||||
to_utc=False)
|
||||
def getkeys(d, keys):
|
||||
return (d[key] for key in keys)
|
||||
|
||||
|
||||
def get_treasury_rate(string_val):
|
||||
val = guarded_conversion(float, string_val)
|
||||
if val is not None:
|
||||
val = round(val / 100.0, 4)
|
||||
return val
|
||||
|
||||
_CURVE_MAPPINGS = {
|
||||
'tid': (safe_int, "Id"),
|
||||
'date': (get_treasury_date, "NEW_DATE"),
|
||||
'1month': (get_treasury_rate, "BC_1MONTH"),
|
||||
'3month': (get_treasury_rate, "BC_3MONTH"),
|
||||
'6month': (get_treasury_rate, "BC_6MONTH"),
|
||||
'1year': (get_treasury_rate, "BC_1YEAR"),
|
||||
'2year': (get_treasury_rate, "BC_2YEAR"),
|
||||
'3year': (get_treasury_rate, "BC_3YEAR"),
|
||||
'5year': (get_treasury_rate, "BC_5YEAR"),
|
||||
'7year': (get_treasury_rate, "BC_7YEAR"),
|
||||
'10year': (get_treasury_rate, "BC_10YEAR"),
|
||||
'20year': (get_treasury_rate, "BC_20YEAR"),
|
||||
'30year': (get_treasury_rate, "BC_30YEAR"),
|
||||
}
|
||||
|
||||
|
||||
def treasury_mappings(mappings):
|
||||
return {key: Mapping(*value)
|
||||
for key, value
|
||||
in iteritems(mappings)}
|
||||
|
||||
|
||||
class iter_to_stream(object):
|
||||
def parse_treasury_csv_column(column):
|
||||
"""
|
||||
Exposes an iterable as an i/o stream
|
||||
Parse a treasury CSV column into a more human-readable format.
|
||||
|
||||
Columns are start with 'RIFLGFC', followed by Y or M (year or month),
|
||||
followed by a two-digit number, followed by _N.B. We only care about the
|
||||
middle two entries which we turn into a string like 3month or 30year.
|
||||
"""
|
||||
def __init__(self, iterable):
|
||||
self.buffered = ""
|
||||
self.iter = iter(iterable)
|
||||
column_re = re.compile(
|
||||
r"^(?P<prefix>RIFLGFC)"
|
||||
"(?P<unit>[YM])"
|
||||
"(?P<periods>[0-9]{2})"
|
||||
"(?P<suffix>_N.B)$"
|
||||
)
|
||||
|
||||
def read(self, size):
|
||||
result = ""
|
||||
while size > 0:
|
||||
data = self.buffered or next(self.iter, None)
|
||||
self.buffered = ""
|
||||
if data is None:
|
||||
break
|
||||
size -= len(data)
|
||||
if size < 0:
|
||||
data, self.buffered = data[:size], data[size:]
|
||||
result += data
|
||||
return result
|
||||
match = column_re.match(column)
|
||||
if match is None:
|
||||
raise ValueError("Couldn't parse CSV column %r." % column)
|
||||
unit, periods = getkeys(match.groupdict(), ['unit', 'periods'])
|
||||
|
||||
|
||||
def get_localname(element):
|
||||
qtag = ET.QName(element.tag).text
|
||||
return re.match("(\{.*\})(.*)", qtag).group(2)
|
||||
|
||||
|
||||
def get_treasury_source():
|
||||
url = """\
|
||||
http://data.treasury.gov/feed.svc/DailyTreasuryYieldCurveRateData\
|
||||
"""
|
||||
res = requests.get(url, stream=True)
|
||||
stream = iter_to_stream(res.text.splitlines())
|
||||
|
||||
elements = ET.iterparse(stream, ('end', 'start-ns', 'end-ns'))
|
||||
|
||||
namespaces = OrderedDict()
|
||||
properties_xpath = ['']
|
||||
|
||||
def updated_namespaces():
|
||||
if '' in namespaces and 'm' in namespaces:
|
||||
properties_xpath[0] = "{%s}content/{%s}properties" % (
|
||||
namespaces[''], namespaces['m']
|
||||
)
|
||||
else:
|
||||
properties_xpath[0] = ''
|
||||
|
||||
for event, element in elements:
|
||||
if event == 'end':
|
||||
tag = get_localname(element)
|
||||
if tag == "entry":
|
||||
properties = element.find(properties_xpath[0])
|
||||
datum = {get_localname(node): node.text
|
||||
for node in properties if ET.iselement(node)}
|
||||
# clear the element after we've dealt with it:
|
||||
element.clear()
|
||||
yield datum
|
||||
|
||||
elif event == 'start-ns':
|
||||
namespaces[element[0]] = element[1]
|
||||
updated_namespaces()
|
||||
|
||||
elif event == 'end-ns':
|
||||
namespaces.popitem()
|
||||
updated_namespaces()
|
||||
# Roundtrip through int to coerce '06' into '6'.
|
||||
return str(int(periods)) + ('year' if unit == 'Y' else 'month')
|
||||
|
||||
|
||||
def get_treasury_data():
|
||||
mappings = treasury_mappings(_CURVE_MAPPINGS)
|
||||
source = get_treasury_source()
|
||||
return source_to_records(mappings, source)
|
||||
return pd.read_csv(
|
||||
"http://www.federalreserve.gov/datadownload/Output.aspx"
|
||||
"?rel=H15"
|
||||
"&series=bf17364827e38702b42a58cf8eaa3f78"
|
||||
"&lastObs="
|
||||
"&from=" # An unbounded query is ~2x faster than specifying dates.
|
||||
"&to="
|
||||
"&filetype=csv"
|
||||
"&label=omit"
|
||||
"&layout=seriescolumn"
|
||||
"&type=package",
|
||||
skiprows=1, # First row is a useless header.
|
||||
parse_dates=['Time Period'],
|
||||
na_values=['ND'], # Presumably this stands for "No Data".
|
||||
index_col=0,
|
||||
).loc[
|
||||
'1990': # Truncate down to 1990.
|
||||
].dropna(
|
||||
how='all'
|
||||
).rename(
|
||||
columns=parse_treasury_csv_column
|
||||
).tz_localize('UTC') * 0.01
|
||||
|
||||
|
||||
def dataconverter(s):
|
||||
|
||||
Reference in New Issue
Block a user