mirror of
https://github.com/wassname/catalyst.git
synced 2026-08-19 12:00:15 +08:00
Scott Sanderson
8c38278783
Replaces our custom XML parsing with a single call to `pd.read_csv`
against the federal reserve's API. This produces nearly identical
results as compared to the old loader, but it's dramatically simpler and
roughly 10x faster on my machine.
The average difference in magnitude between new and old is approximately
10e-7, and only one entry is different to a degree greater than the
number of significant figures provided by treasury.gov.
Additionally, the new loader correctly ignores Columbus Day of 2010, for
which the old loader erroneously produced an all-NaN row.
This also changes the interface that treasury modules modules are
required to implement. Modules must now supply a `get_treasury_data`
function that returns a `DataFrame` with a daily `DatetimeIndex` and a
column for each supported treasury duration.
Detailed comparison between results from new and old loader::
from zipline.data.treasuries import get_treasury_data
new = get_treasury_data() # New implementation
old = pd.read_csv( # Previously cached data
'/home/ssanderson/.zipline/data/treasury_curves.csv'
parse_dates=[0],
index_col=0,
)
# These columns were unused.
del old['tid']; del old['date']
old = old.tz_localize('UTC')
old.dropna(how='all')
# old data erroneously contained an all-NaN entry for Columbus Day
# in 2010. Remove before comparing.
old = old.dropna(how='all')
In [25]: len(new) == len(old)
Out[25]: True
In [26]: abs(old - new).max()
Out[26]:
10year 2.000000e-04
1month 6.938894e-18
1year 1.000000e-04
20year 1.000000e-04
2year 2.000000e-04
30year 1.000000e-04
3month 1.000000e-03
3year 1.000000e-04
5year 1.387779e-17
6month 1.000000e-04
7year 1.000000e-04
dtype: float64
In [27]: abs(old - new).mean()
Out[27]:
10year 3.097414e-08
1month 4.396534e-19
1year 1.548707e-08
20year 3.624502e-08
2year 4.646120e-08
30year 1.830496e-08
3month 1.549427e-07
3year 1.548707e-08
5year 1.702619e-18
6month 1.548707e-08
7year 1.548707e-08
dtype: float64
Since www.treasury.gov only reports values up to three significant
digits, we should only care about differences of greater than 1e-3.
There is exactly one such difference: the entry for the three month bond
on 1999-10-01::
In [60]: new[(abs(new - old) >= 1e-3).any(axis=1)].T
Out[60]:
Time Period 1999-10-01 00:00:00+00:00
1month NaN
3month 0.0498
6month 0.0501
1year 0.0530
2year 0.0573
3year 0.0583
5year 0.0590
7year 0.0622
10year 0.0600
20year 0.0657
30year 0.0615
In [61]: old[(abs(new - old) >= 1e-3).any(axis=1)].T
Out[61]:
1999-10-01 00:00:00+00:00
10year 0.0600
1month NaN
1year 0.0530
20year 0.0657
2year 0.0573
30year 0.0615
3month 0.0488
3year 0.0583
5year 0.0590
6month 0.0501
7year 0.0622
The US Treasury website (our old source) provides a value of 0.488 here,
whereas the Federal Reserve site (our new source) provides a value of
0.498.
90 lines
2.8 KiB
Python
90 lines
2.8 KiB
Python
#
|
|
# Copyright 2013 Quantopian, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
import re
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
|
|
|
|
def getkeys(d, keys):
|
|
return (d[key] for key in keys)
|
|
|
|
|
|
def parse_treasury_csv_column(column):
|
|
"""
|
|
Parse a treasury CSV column into a more human-readable format.
|
|
|
|
Columns are start with 'RIFLGFC', followed by Y or M (year or month),
|
|
followed by a two-digit number, followed by _N.B. We only care about the
|
|
middle two entries which we turn into a string like 3month or 30year.
|
|
"""
|
|
column_re = re.compile(
|
|
r"^(?P<prefix>RIFLGFC)"
|
|
"(?P<unit>[YM])"
|
|
"(?P<periods>[0-9]{2})"
|
|
"(?P<suffix>_N.B)$"
|
|
)
|
|
|
|
match = column_re.match(column)
|
|
if match is None:
|
|
raise ValueError("Couldn't parse CSV column %r." % column)
|
|
unit, periods = getkeys(match.groupdict(), ['unit', 'periods'])
|
|
|
|
# Roundtrip through int to coerce '06' into '6'.
|
|
return str(int(periods)) + ('year' if unit == 'Y' else 'month')
|
|
|
|
|
|
def get_treasury_data():
|
|
return pd.read_csv(
|
|
"http://www.federalreserve.gov/datadownload/Output.aspx"
|
|
"?rel=H15"
|
|
"&series=bf17364827e38702b42a58cf8eaa3f78"
|
|
"&lastObs="
|
|
"&from=" # An unbounded query is ~2x faster than specifying dates.
|
|
"&to="
|
|
"&filetype=csv"
|
|
"&label=omit"
|
|
"&layout=seriescolumn"
|
|
"&type=package",
|
|
skiprows=1, # First row is a useless header.
|
|
parse_dates=['Time Period'],
|
|
na_values=['ND'], # Presumably this stands for "No Data".
|
|
index_col=0,
|
|
).loc[
|
|
'1990': # Truncate down to 1990.
|
|
].dropna(
|
|
how='all'
|
|
).rename(
|
|
columns=parse_treasury_csv_column
|
|
).tz_localize('UTC') * 0.01
|
|
|
|
|
|
def dataconverter(s):
|
|
try:
|
|
return float(s) / 100
|
|
except:
|
|
return np.nan
|
|
|
|
|
|
def get_daily_10yr_treasury_data():
|
|
"""Download daily 10 year treasury rates from the Federal Reserve and
|
|
return a pandas.Series."""
|
|
url = "http://www.federalreserve.gov/datadownload/Output.aspx?rel=H15" \
|
|
"&series=bcb44e57fb57efbe90002369321bfb3f&lastObs=&from=&to=" \
|
|
"&filetype=csv&label=include&layout=seriescolumn"
|
|
return pd.read_csv(url, header=5, index_col=0, names=['DATE', 'BC_10YEAR'],
|
|
parse_dates=True, converters={1: dataconverter},
|
|
squeeze=True)
|