From 0ab136f1c81df983c3076c794bd630a566d1c85c Mon Sep 17 00:00:00 2001 From: Eddie Hebert Date: Thu, 18 Oct 2012 15:11:04 -0400 Subject: [PATCH] Adds a loader for market data when it doesn't exist locally. Hopefully, this helps ease ramp up time for developing against market data, without us distributing the data. We do a check for the data when attempting to read the msgpack files, if they don't exist the loader makes a web request and retrieves and serializes the data for the user. Provides a loader for: - curves from data.treasury.gov - benchmarks from Yahoo! Finance Adds dependency of requests library in dev requirements. --- .gitignore | 3 + etc/requirements_dev.txt | 3 + zipline/data/__init__.py | 3 + zipline/data/benchmarks.py | 108 ++++++++++++++++++++++++ zipline/data/loader.py | 71 ++++++++++++++++ zipline/data/loader_utils.py | 155 +++++++++++++++++++++++++++++++++++ zipline/data/treasuries.py | 74 +++++++++++++++++ zipline/utils/factory.py | 27 +++++- 8 files changed, 442 insertions(+), 2 deletions(-) create mode 100644 zipline/data/benchmarks.py create mode 100644 zipline/data/loader.py create mode 100644 zipline/data/loader_utils.py create mode 100644 zipline/data/treasuries.py diff --git a/.gitignore b/.gitignore index 0d216d0e..e6161659 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,6 @@ docs/_build/* # database of vbench benchmarks.db + +# downloaded data +zipline/data/*.msgpack diff --git a/etc/requirements_dev.txt b/etc/requirements_dev.txt index 34460d2e..dedc64d1 100644 --- a/etc/requirements_dev.txt +++ b/etc/requirements_dev.txt @@ -4,6 +4,9 @@ ipython==0.12 unittest2 nose==1.1.2 +# Fetching sample data +requests==0.14.1 + # Linting flake8==1.4 diff --git a/zipline/data/__init__.py b/zipline/data/__init__.py index e69de29b..86eafd7d 100644 --- a/zipline/data/__init__.py +++ b/zipline/data/__init__.py @@ -0,0 +1,3 @@ +import loader + +__all__ = ['loader'] diff --git a/zipline/data/benchmarks.py b/zipline/data/benchmarks.py new file mode 100644 index 00000000..a024dbf0 --- /dev/null +++ b/zipline/data/benchmarks.py @@ -0,0 +1,108 @@ +# +# Copyright 2012 Quantopian, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from datetime import datetime + +import csv + +from StringIO import StringIO +from functools import partial + +import requests + +from loader_utils import ( + date_conversion, + source_to_records +) + +from loader_utils import Mapping + +from zipline.finance.risk import DailyReturn + +_BENCHMARK_MAPPING = { + # Need to add 'symbol' and GSPC as a constant + 'volume': (int, 'Volume'), + 'open': (float, 'Open'), + 'close': (float, 'Close'), + 'high': (float, 'High'), + 'low': (float, 'Low'), + 'adj_close': (float, 'Adj Close'), + 'date': (partial(date_conversion, date_pattern='%Y-%m-%d'), 'Date') +} + + +def benchmark_mappings(): + return {key: Mapping(*value) + for key, value + in _BENCHMARK_MAPPING.iteritems()} + + +def get_raw_benchmark_data(start_date, end_date): + + # create benchmark files + # ^GSPC 19500103 + params = { + # the s&p 500 + 's': '^GSPC', + # end_date month, zero indexed + 'd': end_date.month - 1, + # end_date day str(int(todate[6:8])) #day + 'e': end_date.day, + # end_date year str(int(todate[0:4])) + 'f': end_date.year, + # daily frequency + 'g': 'd', + # start_date month, zero indexed + 'a': start_date.month - 1, + # start_date day + 'b': start_date.day, + # start_date year + 'c': start_date.year + } + + res = requests.get('http://ichart.yahoo.com/table.csv', + params=params) + + return csv.DictReader(StringIO(res.content)) + + +def get_benchmark_data(): + """ + Benchmarks from Yahoo's GSPC source. + """ + start_date = datetime(year=1950, month=1, day=3) + end_date = datetime.utcnow() + + raw_benchmark_data = get_raw_benchmark_data(start_date, end_date) + # Reverse data so we can load it in reverse chron order. + benchmarks_source = reversed(list(raw_benchmark_data)) + + mappings = benchmark_mappings() + + return source_to_records(mappings, benchmarks_source) + + +def get_benchmark_returns(): + + benchmark_returns = [] + + for data_point in get_benchmark_data(): + returns = (data_point['close'] - data_point['open']) / \ + data_point['open'] + daily_return = DailyReturn(date=data_point['date'], returns=returns) + benchmark_returns.append(daily_return) + + return benchmark_returns diff --git a/zipline/data/loader.py b/zipline/data/loader.py new file mode 100644 index 00000000..8a5e9825 --- /dev/null +++ b/zipline/data/loader.py @@ -0,0 +1,71 @@ +# +# Copyright 2012 Quantopian, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + +if __name__ == "__main__": + import sys + sys.path.append(os.path.abspath('.')) + print sys.path + +import msgpack + +from treasuries import get_treasury_data +from benchmarks import get_benchmark_returns + + +def dump_treasury_curves(): + """ + Dumps data to be used with zipline. + + Puts source treasury and data into zipline. + """ + tr_data = [] + + for curve in get_treasury_data(): + print curve + date_as_tuple = curve['date'].timetuple()[0:6] + \ + (curve['date'].microsecond,) + # Not ideal but massaging data into expected format + del curve['date'] + tr = (date_as_tuple, curve) + tr_data.append(tr) + + tr_path = os.path.join(os.path.dirname(__file__), + "treasury_curves.msgpack") + tr_fp = open(tr_path, "wb") + tr_fp.write(msgpack.dumps(tr_data)) + + +def dump_benchmarks(): + """ + Dumps data to be used with zipline. + + Puts source treasury and data into zipline. + """ + benchmark_path = os.path.join(os.path.dirname(__file__), + "benchmark.msgpack") + benchmark_fp = open(benchmark_path, "wb") + benchmark_data = [] + for daily_return in get_benchmark_returns(): + print daily_return + date_as_tuple = daily_return.date.timetuple()[0:6] + \ + (daily_return.date.microsecond,) + # Not ideal but massaging data into expected format + benchmark = (date_as_tuple, daily_return.returns) + benchmark_data.append(benchmark) + + benchmark_fp.write(msgpack.dumps(benchmark_data)) diff --git a/zipline/data/loader_utils.py b/zipline/data/loader_utils.py new file mode 100644 index 00000000..700db778 --- /dev/null +++ b/zipline/data/loader_utils.py @@ -0,0 +1,155 @@ +# +# Copyright 2012 Quantopian, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +Various utilites used by different date loaders. + +Could stand to be broken up more into components. +e.g. the mapping utilities. + +""" + +import datetime + +import pytz + +from collections import namedtuple + +from functools import partial + + +def get_utc_from_exchange_time(naive): + local = pytz.timezone('US/Eastern') + local_dt = naive.replace(tzinfo=local) + utc_dt = local_dt.astimezone(pytz.utc) + return utc_dt + + +def get_exchange_time_from_utc(utc_dt): + """ + Takes in result from exchange time. + """ + dt = utc_dt.replace(tzinfo=pytz.utc) + local = pytz.timezone('US/Eastern') + dt = dt.astimezone(local) + + return dt + + +def guarded_conversion(conversion, str_val): + """ + Returns the result of applying the @conversion to @str_val + """ + if str_val in (None, ""): + return None + return conversion(str_val) + + +def safe_int(str_val): + """ + casts the @str_val to a float to handle the occassional + decimal point in int fields from data providers. + """ + f = float(str_val) + i = int(f) + return i + + +def date_conversion(date_str, date_pattern='%m/%d/%Y', to_utc=True): + """ + Convert date strings from TickData (or other source) into epoch values. + + Specify to_utc=False if the input date is already UTC (or is naive). + """ + dt = datetime.datetime.strptime(date_str, date_pattern) + if to_utc: + dt = get_utc_from_exchange_time(dt) + else: + dt = dt.replace(tzinfo=pytz.utc) + return dt + + +# Mapping is a structure for how want to convert the source data into +# the form we insert into the database. +# - conversion, a function used to convert source input to our target value +# - source, the key(s) in the original source to pass to the conversion +# method +# If a single string, then it's a direct lookup into the +# source row by that key +# If an iterator, pass the source to as a list of keys, +# in order, to the conversion function. +# If empty, then the conversion method provides a 'default' value. +Mapping = namedtuple('Mapping', ['conversion', 'source']) + + +def apply_mapping(mapping, row): + """ + Returns the value of a @mapping for a given @row. + + i.e. the @mapping.source values are extracted from @row and fed + into the @mapping.conversion method. + """ + if isinstance(mapping.source, str): + # Do a 'direct' conversion of one key from the source row. + return guarded_conversion(mapping.conversion, row[mapping.source]) + if mapping.source is None: + # For hardcoded values. + # conversion method will return a constant value + return mapping.conversion() + else: + # Assume we are using multiple source values. + # Feed the source values in order prescribed by mapping.source + # to mapping.conversion. + return mapping.conversion(*[row[source] for source in mapping.source]) + + +def _row_cb(mapping, row): + """ + Returns the dict created from our @mapping of the source @row. + + Not intended to be used directly, but rather to be the base of another + function that supplies the mapping value. + """ + return { + target: apply_mapping(mapping, row) + for target, mapping + in mapping.iteritems() + } + + +def make_row_cb(mapping): + """ + Returns a func that can be applied to a dict that returns the + application of the @mapping, which results in a dict. + """ + return partial(_row_cb, mapping) + + +def source_to_records(mappings, + source, + source_wrapper=None, + records_wrapper=None): + if source_wrapper: + source = source_wrapper(source) + + callback = make_row_cb(mappings) + + records = (callback(row) for row in source) + + if records_wrapper: + records = records_wrapper(records) + + return records diff --git a/zipline/data/treasuries.py b/zipline/data/treasuries.py new file mode 100644 index 00000000..bae14060 --- /dev/null +++ b/zipline/data/treasuries.py @@ -0,0 +1,74 @@ +import requests + +from StringIO import StringIO +from xml.dom.minidom import parse + +from loader_utils import ( + guarded_conversion, + safe_int, + Mapping, + date_conversion, + source_to_records +) + + +def get_treasury_date(dstring): + return date_conversion(dstring.split("T")[0], date_pattern='%Y-%m-%d', + to_utc=False) + + +def get_treasury_rate(string_val): + val = guarded_conversion(float, string_val) + if val is not None: + val = round(val / 100.0, 4) + return val + +_CURVE_MAPPINGS = { + 'tid': (safe_int, "Id"), + 'date': (get_treasury_date, "NEW_DATE"), + '1month': (get_treasury_rate, "BC_1MONTH"), + '3month': (get_treasury_rate, "BC_3MONTH"), + '6month': (get_treasury_rate, "BC_6MONTH"), + '1year': (get_treasury_rate, "BC_1YEAR"), + '2year': (get_treasury_rate, "BC_2YEAR"), + '3year': (get_treasury_rate, "BC_3YEAR"), + '5year': (get_treasury_rate, "BC_5YEAR"), + '7year': (get_treasury_rate, "BC_7YEAR"), + '10year': (get_treasury_rate, "BC_10YEAR"), + '20year': (get_treasury_rate, "BC_20YEAR"), + '30year': (get_treasury_rate, "BC_30YEAR"), +} + + +def treasury_mappings(): + return {key: Mapping(*value) + for key, value + in _CURVE_MAPPINGS.iteritems()} + + +def get_treasury_source(): + url = """\ +http://data.treasury.gov/feed.svc/DailyTreasuryYieldCurveRateData\ +""" + res = requests.get(url) + + content = StringIO(res.content) + dom = parse(content) + + entries = dom.getElementsByTagName("entry") + + for entry in entries: + properties = entry.getElementsByTagName("m:properties") + datum = {node.nodeName.replace('d:', ''): + node.childNodes[0].nodeValue + if len(node.childNodes) + else None + for node in properties[0].childNodes + if node.nodeType == dom.ELEMENT_NODE} + yield datum + + +def get_treasury_data(): + mappings = treasury_mappings() + source = get_treasury_source() + return source_to_records(mappings, source) diff --git a/zipline/utils/factory.py b/zipline/utils/factory.py index acf4a216..4a17db7f 100644 --- a/zipline/utils/factory.py +++ b/zipline/utils/factory.py @@ -34,16 +34,27 @@ from zipline.gens.tradegens import SpecificEquityTrades, DataFrameSource from zipline.gens.utils import create_trade from zipline.finance.trading import TradingEnvironment +from zipline import data + # TODO def data_path(): - from zipline import data data_path = dirname(abspath(data.__file__)) return data_path def load_market_data(): - fp_bm = open(join(data_path(), "benchmark.msgpack"), "rb") + benchmark_data_path = join(data_path(), "benchmark.msgpack") + try: + fp_bm = open(benchmark_data_path, "rb") + except IOError: + print """ +data msgpacks aren't distribute with source. +Fetching data from Yahoo Finance. +""".strip() + data.loader.dump_benchmarks() + fp_bm = open(benchmark_data_path, "rb") + bm_list = msgpack.loads(fp_bm.read()) bm_returns = [] for packed_date, returns in bm_list: @@ -59,6 +70,18 @@ def load_market_data(): bm_returns.append(daily_return) bm_returns = sorted(bm_returns, key=attrgetter('date')) + + treasury_data_path = join(data_path(), "treasury_curves.msgpack") + try: + fp_bm = open(treasury_data_path, "rb") + except IOError: + print """ +data msgpacks aren't distribute with source. +Fetching data from data.treasury.gov +""".strip() + data.loader.dump_treasury_curves() + fp_bm = open(treasury_data_path, "rb") + fp_tr = open(join(data_path(), "treasury_curves.msgpack"), "rb") tr_list = msgpack.loads(fp_tr.read()) tr_curves = {}