Adds a loader for market data when it doesn't exist locally.

Hopefully, this helps ease ramp up time for developing against
market data, without us distributing the data.

We do a check for the data when attempting to read the msgpack
files, if they don't exist the loader makes a web request and
retrieves and serializes the data for the user.

Provides a loader for:

- curves from data.treasury.gov
- benchmarks from Yahoo! Finance

Adds dependency of requests library in dev requirements.

.gitignore

@@ -53,3 +53,6 @@ docs/_build/*
 
 # database of vbench
 benchmarks.db
+
+# downloaded data
+zipline/data/*.msgpack

etc/requirements_dev.txt

@@ -4,6 +4,9 @@ ipython==0.12
 unittest2
 nose==1.1.2
 
+# Fetching sample data
+requests==0.14.1
+
 # Linting
 
 flake8==1.4

zipline/data/__init__.py

@@ -0,0 +1,3 @@
+import loader
+
+__all__ = ['loader']

zipline/data/benchmarks.py

@@ -0,0 +1,108 @@
+#
+# Copyright 2012 Quantopian, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from datetime import datetime
+
+import csv
+
+from StringIO import StringIO
+from functools import partial
+
+import requests
+
+from loader_utils import (
+    date_conversion,
+    source_to_records
+)
+
+from loader_utils import Mapping
+
+from zipline.finance.risk import DailyReturn
+
+_BENCHMARK_MAPPING = {
+    # Need to add 'symbol' and GSPC as a constant
+    'volume': (int, 'Volume'),
+    'open': (float, 'Open'),
+    'close': (float, 'Close'),
+    'high': (float, 'High'),
+    'low': (float, 'Low'),
+    'adj_close': (float, 'Adj Close'),
+    'date': (partial(date_conversion, date_pattern='%Y-%m-%d'), 'Date')
+}
+
+
+def benchmark_mappings():
+    return {key: Mapping(*value)
+            for key, value
+            in _BENCHMARK_MAPPING.iteritems()}
+
+
+def get_raw_benchmark_data(start_date, end_date):
+
+    # create benchmark files
+    # ^GSPC 19500103
+    params = {
+        # the s&p 500
+        's': '^GSPC',
+        # end_date month, zero indexed
+        'd': end_date.month - 1,
+        # end_date day str(int(todate[6:8])) #day
+        'e': end_date.day,
+        # end_date year str(int(todate[0:4]))
+        'f': end_date.year,
+        # daily frequency
+        'g': 'd',
+        # start_date month, zero indexed
+        'a': start_date.month - 1,
+        # start_date day
+        'b': start_date.day,
+        # start_date year
+        'c': start_date.year
+    }
+
+    res = requests.get('http://ichart.yahoo.com/table.csv',
+                       params=params)
+
+    return csv.DictReader(StringIO(res.content))
+
+
+def get_benchmark_data():
+    """
+    Benchmarks from Yahoo's GSPC source.
+    """
+    start_date = datetime(year=1950, month=1, day=3)
+    end_date = datetime.utcnow()
+
+    raw_benchmark_data = get_raw_benchmark_data(start_date, end_date)
+    # Reverse data so we can load it in reverse chron order.
+    benchmarks_source = reversed(list(raw_benchmark_data))
+
+    mappings = benchmark_mappings()
+
+    return source_to_records(mappings, benchmarks_source)
+
+
+def get_benchmark_returns():
+
+    benchmark_returns = []
+
+    for data_point in get_benchmark_data():
+        returns = (data_point['close'] - data_point['open']) / \
+                  data_point['open']
+        daily_return = DailyReturn(date=data_point['date'], returns=returns)
+        benchmark_returns.append(daily_return)
+
+    return benchmark_returns

zipline/data/loader.py

@@ -0,0 +1,71 @@
+#
+# Copyright 2012 Quantopian, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+
+if __name__ == "__main__":
+    import sys
+    sys.path.append(os.path.abspath('.'))
+    print sys.path
+
+import msgpack
+
+from treasuries import get_treasury_data
+from benchmarks import get_benchmark_returns
+
+
+def dump_treasury_curves():
+    """
+    Dumps data to be used with zipline.
+
+    Puts source treasury and data into zipline.
+    """
+    tr_data = []
+
+    for curve in get_treasury_data():
+        print curve
+        date_as_tuple = curve['date'].timetuple()[0:6] + \
+                        (curve['date'].microsecond,)
+        # Not ideal but massaging data into expected format
+        del curve['date']
+        tr = (date_as_tuple, curve)
+        tr_data.append(tr)
+
+    tr_path = os.path.join(os.path.dirname(__file__),
+                           "treasury_curves.msgpack")
+    tr_fp = open(tr_path, "wb")
+    tr_fp.write(msgpack.dumps(tr_data))
+
+
+def dump_benchmarks():
+    """
+    Dumps data to be used with zipline.
+
+    Puts source treasury and data into zipline.
+    """
+    benchmark_path = os.path.join(os.path.dirname(__file__),
+                           "benchmark.msgpack")
+    benchmark_fp = open(benchmark_path, "wb")
+    benchmark_data = []
+    for daily_return in get_benchmark_returns():
+        print daily_return
+        date_as_tuple = daily_return.date.timetuple()[0:6] + \
+                        (daily_return.date.microsecond,)
+        # Not ideal but massaging data into expected format
+        benchmark = (date_as_tuple, daily_return.returns)
+        benchmark_data.append(benchmark)
+
+    benchmark_fp.write(msgpack.dumps(benchmark_data))

zipline/data/loader_utils.py

@@ -0,0 +1,155 @@
+#
+# Copyright 2012 Quantopian, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""
+Various utilites used by different date loaders.
+
+Could stand to be broken up more into components.
+e.g. the mapping utilities.
+
+"""
+
+import datetime
+
+import pytz
+
+from collections import namedtuple
+
+from functools import partial
+
+
+def get_utc_from_exchange_time(naive):
+    local = pytz.timezone('US/Eastern')
+    local_dt = naive.replace(tzinfo=local)
+    utc_dt = local_dt.astimezone(pytz.utc)
+    return utc_dt
+
+
+def get_exchange_time_from_utc(utc_dt):
+    """
+    Takes in result from exchange time.
+    """
+    dt = utc_dt.replace(tzinfo=pytz.utc)
+    local = pytz.timezone('US/Eastern')
+    dt = dt.astimezone(local)
+
+    return dt
+
+
+def guarded_conversion(conversion, str_val):
+    """
+    Returns the result of applying the @conversion to @str_val
+    """
+    if str_val in (None, ""):
+        return None
+    return conversion(str_val)
+
+
+def safe_int(str_val):
+    """
+    casts the @str_val to a float to handle the occassional
+    decimal point in int fields from data providers.
+    """
+    f = float(str_val)
+    i = int(f)
+    return i
+
+
+def date_conversion(date_str, date_pattern='%m/%d/%Y', to_utc=True):
+    """
+    Convert date strings from TickData (or other source) into epoch values.
+
+    Specify to_utc=False if the input date is already UTC (or is naive).
+    """
+    dt = datetime.datetime.strptime(date_str, date_pattern)
+    if to_utc:
+        dt = get_utc_from_exchange_time(dt)
+    else:
+        dt = dt.replace(tzinfo=pytz.utc)
+    return dt
+
+
+# Mapping is a structure for how want to convert the source data into
+# the form we insert into the database.
+# - conversion, a function used to convert source input to our target value
+# - source, the key(s) in the original source to pass to the conversion
+#           method
+#           If a single string, then it's a direct lookup into the
+#           source row by that key
+#           If an iterator, pass the source to as a list of keys,
+#           in order, to the conversion function.
+#           If empty, then the conversion method provides a 'default' value.
+Mapping = namedtuple('Mapping', ['conversion', 'source'])
+
+
+def apply_mapping(mapping, row):
+    """
+    Returns the value of a @mapping for a given @row.
+
+    i.e. the @mapping.source values are extracted from @row and fed
+    into the @mapping.conversion method.
+    """
+    if isinstance(mapping.source, str):
+        # Do a 'direct' conversion of one key from the source row.
+        return guarded_conversion(mapping.conversion, row[mapping.source])
+    if mapping.source is None:
+        # For hardcoded values.
+        # conversion method will return a constant value
+        return mapping.conversion()
+    else:
+        # Assume we are using multiple source values.
+        # Feed the source values in order prescribed by mapping.source
+        # to mapping.conversion.
+        return mapping.conversion(*[row[source] for source in mapping.source])
+
+
+def _row_cb(mapping, row):
+    """
+    Returns the dict created from our @mapping of the source @row.
+
+    Not intended to be used directly, but rather to be the base of another
+    function that supplies the mapping value.
+    """
+    return {
+        target: apply_mapping(mapping, row)
+        for target, mapping
+        in mapping.iteritems()
+        }
+
+
+def make_row_cb(mapping):
+    """
+    Returns a func that can be applied to a dict that returns the
+    application of the @mapping, which results in a dict.
+    """
+    return partial(_row_cb, mapping)
+
+
+def source_to_records(mappings,
+                      source,
+                      source_wrapper=None,
+                      records_wrapper=None):
+    if source_wrapper:
+        source = source_wrapper(source)
+
+    callback = make_row_cb(mappings)
+
+    records = (callback(row) for row in source)
+
+    if records_wrapper:
+        records = records_wrapper(records)
+
+    return records

zipline/data/treasuries.py

@@ -0,0 +1,74 @@
+import requests
+
+from StringIO import StringIO
+from xml.dom.minidom import parse
+
+from loader_utils import (
+    guarded_conversion,
+    safe_int,
+    Mapping,
+    date_conversion,
+    source_to_records
+)
+
+
+def get_treasury_date(dstring):
+    return date_conversion(dstring.split("T")[0], date_pattern='%Y-%m-%d',
+                           to_utc=False)
+
+
+def get_treasury_rate(string_val):
+    val = guarded_conversion(float, string_val)
+    if val is not None:
+        val = round(val / 100.0, 4)
+    return val
+
+_CURVE_MAPPINGS = {
+    'tid': (safe_int, "Id"),
+    'date': (get_treasury_date, "NEW_DATE"),
+    '1month': (get_treasury_rate, "BC_1MONTH"),
+    '3month': (get_treasury_rate, "BC_3MONTH"),
+    '6month': (get_treasury_rate, "BC_6MONTH"),
+    '1year': (get_treasury_rate, "BC_1YEAR"),
+    '2year': (get_treasury_rate, "BC_2YEAR"),
+    '3year': (get_treasury_rate, "BC_3YEAR"),
+    '5year': (get_treasury_rate, "BC_5YEAR"),
+    '7year': (get_treasury_rate, "BC_7YEAR"),
+    '10year': (get_treasury_rate, "BC_10YEAR"),
+    '20year': (get_treasury_rate, "BC_20YEAR"),
+    '30year': (get_treasury_rate, "BC_30YEAR"),
+}
+
+
+def treasury_mappings():
+    return {key: Mapping(*value)
+            for key, value
+            in _CURVE_MAPPINGS.iteritems()}
+
+
+def get_treasury_source():
+    url = """\
+http://data.treasury.gov/feed.svc/DailyTreasuryYieldCurveRateData\
+"""
+    res = requests.get(url)
+
+    content = StringIO(res.content)
+    dom = parse(content)
+
+    entries = dom.getElementsByTagName("entry")
+
+    for entry in entries:
+        properties = entry.getElementsByTagName("m:properties")
+        datum = {node.nodeName.replace('d:', ''):
+                 node.childNodes[0].nodeValue
+                 if len(node.childNodes)
+                 else None
+                 for node in properties[0].childNodes
+                 if node.nodeType == dom.ELEMENT_NODE}
+        yield datum
+
+
+def get_treasury_data():
+    mappings = treasury_mappings()
+    source = get_treasury_source()
+    return source_to_records(mappings, source)

zipline/utils/factory.py

@@ -34,16 +34,27 @@ from zipline.gens.tradegens import SpecificEquityTrades, DataFrameSource
 from zipline.gens.utils import create_trade
 from zipline.finance.trading import TradingEnvironment
 
+from zipline import data
+
 
 # TODO
 def data_path():
-    from zipline import data
     data_path = dirname(abspath(data.__file__))
     return data_path
 
 
 def load_market_data():
-    fp_bm = open(join(data_path(), "benchmark.msgpack"), "rb")
+    benchmark_data_path = join(data_path(), "benchmark.msgpack")
+    try:
+        fp_bm = open(benchmark_data_path, "rb")
+    except IOError:
+        print """
+data msgpacks aren't distribute with source.
+Fetching data from Yahoo Finance.
+""".strip()
+        data.loader.dump_benchmarks()
+        fp_bm = open(benchmark_data_path, "rb")
+
     bm_list = msgpack.loads(fp_bm.read())
     bm_returns = []
     for packed_date, returns in bm_list:
@@ -59,6 +70,18 @@ def load_market_data():
         bm_returns.append(daily_return)
 
     bm_returns = sorted(bm_returns, key=attrgetter('date'))
+
+    treasury_data_path = join(data_path(), "treasury_curves.msgpack")
+    try:
+        fp_bm = open(treasury_data_path, "rb")
+    except IOError:
+        print """
+data msgpacks aren't distribute with source.
+Fetching data from data.treasury.gov
+""".strip()
+        data.loader.dump_treasury_curves()
+        fp_bm = open(treasury_data_path, "rb")
+
     fp_tr = open(join(data_path(), "treasury_curves.msgpack"), "rb")
     tr_list = msgpack.loads(fp_tr.read())
     tr_curves = {}