From a66f45b59821196b976d06e538b47572f105f854 Mon Sep 17 00:00:00 2001 From: Thomas Wiecki Date: Tue, 1 Oct 2013 10:10:36 -0400 Subject: [PATCH] MAINT: Moving yahoo loader from factory to utils. --- zipline/data/__init__.py | 3 +- zipline/data/loader.py | 129 +++++++++++++++++++++++++++++++++++++ zipline/utils/factory.py | 134 ++------------------------------------- 3 files changed, 137 insertions(+), 129 deletions(-) diff --git a/zipline/data/__init__.py b/zipline/data/__init__.py index c7dbfbe1..57f8e9f0 100644 --- a/zipline/data/__init__.py +++ b/zipline/data/__init__.py @@ -1,3 +1,4 @@ from . import loader +from .loader import load_from_yahoo, load_bars_from_yahoo -__all__ = ['loader'] +__all__ = ['loader', 'load_from_yahoo', 'load_bars_from_yahoo'] diff --git a/zipline/data/loader.py b/zipline/data/loader.py index 8115a617..323a0d51 100644 --- a/zipline/data/loader.py +++ b/zipline/data/loader.py @@ -22,6 +22,8 @@ from datetime import timedelta import logbook import pandas as pd +from pandas.io.data import DataReader +import pytz from . treasuries import get_treasury_data from . import benchmarks @@ -219,3 +221,130 @@ Fetching data from data.treasury.gov key=lambda t: t[0])) return bm_returns, tr_curves + + +def _load_raw_yahoo_data(indexes=None, stocks=None, start=None, end=None): + """Load closing prices from yahoo finance. + + :Optional: + indexes : dict (Default: {'SPX': '^GSPC'}) + Financial indexes to load. + stocks : list (Default: ['AAPL', 'GE', 'IBM', 'MSFT', + 'XOM', 'AA', 'JNJ', 'PEP', 'KO']) + Stock closing prices to load. + start : datetime (Default: datetime(1993, 1, 1, 0, 0, 0, 0, pytz.utc)) + Retrieve prices from start date on. + end : datetime (Default: datetime(2002, 1, 1, 0, 0, 0, 0, pytz.utc)) + Retrieve prices until end date. + + :Note: + This is based on code presented in a talk by Wes McKinney: + http://wesmckinney.com/files/20111017/notebook_output.pdf + """ + + assert indexes is not None or stocks is not None, """ +must specify stocks or indexes""" + + if start is None: + start = pd.datetime(1990, 1, 1, 0, 0, 0, 0, pytz.utc) + + if not start is None and not end is None: + assert start < end, "start date is later than end date." + + data = OrderedDict() + + if stocks is not None: + for stock in stocks: + print stock + stkd = DataReader(stock, 'yahoo', start, end).sort_index() + data[stock] = stkd + + if indexes is not None: + for name, ticker in indexes.iteritems(): + print name + stkd = DataReader(ticker, 'yahoo', start, end).sort_index() + data[name] = stkd + + return data + + +def load_from_yahoo(indexes=None, + stocks=None, + start=None, + end=None, + adjusted=True): + """ + Loads price data from Yahoo into a dataframe for each of the indicated + securities. By default, 'price' is taken from Yahoo's 'Adjusted Close', + which removes the impact of splits and dividends. If the argument + 'adjusted' is False, then the non-adjusted 'close' field is used instead. + + :param indexes: Financial indexes to load. + :type indexes: dict + :param stocks: Stock closing prices to load. + :type stocks: list + :param start: Retrieve prices from start date on. + :type start: datetime + :param end: Retrieve prices until end date. + :type end: datetime + :param adjusted: Adjust the price for splits and dividends. + :type adjusted: bool + + """ + data = _load_raw_yahoo_data(indexes, stocks, start, end) + if adjusted: + close_key = 'Adj Close' + else: + close_key = 'Close' + df = pd.DataFrame({key: d[close_key] for key, d in data.iteritems()}) + df.index = df.index.tz_localize(pytz.utc) + return df + + +def load_bars_from_yahoo(indexes=None, + stocks=None, + start=None, + end=None, + adjusted=True): + """ + Loads data from Yahoo into a panel with the following + column names for each indicated security: + + - open + - high + - low + - close + - volume + - price + + Note that 'price' is Yahoo's 'Adjusted Close', which removes the + impact of splits and dividends. If the argument 'adjusted' is True, then + the open, high, low, and close values are adjusted as well. + + :param indexes: Financial indexes to load. + :type indexes: dict + :param stocks: Stock closing prices to load. + :type stocks: list + :param start: Retrieve prices from start date on. + :type start: datetime + :param end: Retrieve prices until end date. + :type end: datetime + :param adjusted: Adjust open/high/low/close for splits and dividends. + The 'price' field is always adjusted. + :type adjusted: bool + + """ + data = _load_raw_yahoo_data(indexes, stocks, start, end) + panel = pd.Panel(data) + # Rename columns + panel.minor_axis = ['open', 'high', 'low', 'close', 'volume', 'price'] + panel.major_axis = panel.major_axis.tz_localize(pytz.utc) + # Adjust data + if adjusted: + adj_cols = ['open', 'high', 'low', 'close'] + for ticker in panel.items: + ratio = (panel[ticker]['price'] / panel[ticker]['close']) + ratio_filtered = ratio.fillna(0).values + for col in adj_cols: + panel[ticker][col] *= ratio_filtered + return panel diff --git a/zipline/utils/factory.py b/zipline/utils/factory.py index 1545e21e..2c0b392e 100644 --- a/zipline/utils/factory.py +++ b/zipline/utils/factory.py @@ -23,7 +23,6 @@ from collections import OrderedDict from delorean import Delorean import pandas as pd -from pandas.io.data import DataReader import numpy as np from datetime import datetime, timedelta @@ -38,6 +37,12 @@ from zipline.sources.test_source import ( create_trade ) +# For backwards compatibility +from zipline.data.loader import (load_from_yahoo, + load_bars_from_yahoo) + +__all__ = ['load_from_yahoo', 'load_bars_from_yahoo'] + def create_simulation_parameters(year=2006, start=None, end=None, capital_base=float("1.0e5"), @@ -389,130 +394,3 @@ def create_test_panel_ohlc_source(sim_params=None): panel = pd.Panel.from_dict({0: df}) return DataPanelSource(panel), panel - - -def _load_raw_yahoo_data(indexes=None, stocks=None, start=None, end=None): - """Load closing prices from yahoo finance. - - :Optional: - indexes : dict (Default: {'SPX': '^GSPC'}) - Financial indexes to load. - stocks : list (Default: ['AAPL', 'GE', 'IBM', 'MSFT', - 'XOM', 'AA', 'JNJ', 'PEP', 'KO']) - Stock closing prices to load. - start : datetime (Default: datetime(1993, 1, 1, 0, 0, 0, 0, pytz.utc)) - Retrieve prices from start date on. - end : datetime (Default: datetime(2002, 1, 1, 0, 0, 0, 0, pytz.utc)) - Retrieve prices until end date. - - :Note: - This is based on code presented in a talk by Wes McKinney: - http://wesmckinney.com/files/20111017/notebook_output.pdf - """ - - assert indexes is not None or stocks is not None, """ -must specify stocks or indexes""" - - if start is None: - start = pd.datetime(1990, 1, 1, 0, 0, 0, 0, pytz.utc) - - if not start is None and not end is None: - assert start < end, "start date is later than end date." - - data = OrderedDict() - - if stocks is not None: - for stock in stocks: - print stock - stkd = DataReader(stock, 'yahoo', start, end).sort_index() - data[stock] = stkd - - if indexes is not None: - for name, ticker in indexes.iteritems(): - print name - stkd = DataReader(ticker, 'yahoo', start, end).sort_index() - data[name] = stkd - - return data - - -def load_from_yahoo(indexes=None, - stocks=None, - start=None, - end=None, - adjusted=True): - """ - Loads price data from Yahoo into a dataframe for each of the indicated - securities. By default, 'price' is taken from Yahoo's 'Adjusted Close', - which removes the impact of splits and dividends. If the argument - 'adjusted' is False, then the non-adjusted 'close' field is used instead. - - :param indexes: Financial indexes to load. - :type indexes: dict - :param stocks: Stock closing prices to load. - :type stocks: list - :param start: Retrieve prices from start date on. - :type start: datetime - :param end: Retrieve prices until end date. - :type end: datetime - :param adjusted: Adjust the price for splits and dividends. - :type adjusted: bool - - """ - data = _load_raw_yahoo_data(indexes, stocks, start, end) - if adjusted: - close_key = 'Adj Close' - else: - close_key = 'Close' - df = pd.DataFrame({key: d[close_key] for key, d in data.iteritems()}) - df.index = df.index.tz_localize(pytz.utc) - return df - - -def load_bars_from_yahoo(indexes=None, - stocks=None, - start=None, - end=None, - adjusted=True): - """ - Loads data from Yahoo into a panel with the following - column names for each indicated security: - - - open - - high - - low - - close - - volume - - price - - Note that 'price' is Yahoo's 'Adjusted Close', which removes the - impact of splits and dividends. If the argument 'adjusted' is True, then - the open, high, low, and close values are adjusted as well. - - :param indexes: Financial indexes to load. - :type indexes: dict - :param stocks: Stock closing prices to load. - :type stocks: list - :param start: Retrieve prices from start date on. - :type start: datetime - :param end: Retrieve prices until end date. - :type end: datetime - :param adjusted: Adjust open/high/low/close for splits and dividends. - The 'price' field is always adjusted. - :type adjusted: bool - - """ - data = _load_raw_yahoo_data(indexes, stocks, start, end) - panel = pd.Panel(data) - # Rename columns - panel.minor_axis = ['open', 'high', 'low', 'close', 'volume', 'price'] - panel.major_axis = panel.major_axis.tz_localize(pytz.utc) - # Adjust data - if adjusted: - adj_cols = ['open', 'high', 'low', 'close'] - for ticker in panel.items: - ratio = (panel[ticker]['price'] / panel[ticker]['close']) - ratio_filtered = ratio.fillna(0).values - for col in adj_cols: - panel[ticker][col] *= ratio_filtered - return panel