From 763f2ab8b4996d1f89b40c0ff4d3cbd55655ec12 Mon Sep 17 00:00:00 2001 From: Nathan Wolfe Date: Wed, 13 Jul 2016 17:45:07 -0400 Subject: [PATCH] MAINT: Combine daily and minute into `PanelBarReader`. Also simplify `load_raw_arrays` and `get_last_traded_dt`. --- tests/test_panel_daily_bar_reader.py | 6 +- zipline/algorithm.py | 37 ++++----- zipline/data/data_portal.py | 2 - zipline/data/minute_bars.py | 111 --------------------------- zipline/data/us_equity_pricing.py | 53 +++++++------ 5 files changed, 42 insertions(+), 167 deletions(-) diff --git a/tests/test_panel_daily_bar_reader.py b/tests/test_panel_daily_bar_reader.py index ba30cca9..2e29bd69 100644 --- a/tests/test_panel_daily_bar_reader.py +++ b/tests/test_panel_daily_bar_reader.py @@ -18,7 +18,7 @@ from itertools import permutations, product import numpy as np import pandas as pd -from zipline.data.us_equity_pricing import PanelDailyBarReader +from zipline.data.us_equity_pricing import PanelBarReader from zipline.testing import ExplodingObject from zipline.testing.fixtures import ( WithAssetFinder, @@ -55,7 +55,7 @@ class TestPanelDailyBarReader(WithAssetFinder, minor_axis=minor_axis, ) - cls.reader = PanelDailyBarReader(days, cls.panel) + cls.reader = PanelBarReader(days, cls.panel) def test_spot_price(self): panel = self.panel @@ -83,7 +83,7 @@ class TestPanelDailyBarReader(WithAssetFinder, for axis_order in permutations((0, 1, 2)): transposed = panel.transpose(*axis_order) with self.assertRaises(ValueError) as e: - PanelDailyBarReader(unused, transposed) + PanelBarReader(unused, transposed) expected = ( "Duplicate entries in Panel.{name}: ['a', 'b'].".format( diff --git a/zipline/algorithm.py b/zipline/algorithm.py index 1d9e4b75..a84cacb9 100644 --- a/zipline/algorithm.py +++ b/zipline/algorithm.py @@ -37,8 +37,7 @@ from six import ( from zipline._protocol import handle_non_market_minutes from zipline.assets.synthetic import make_simple_equity_info from zipline.data.data_portal import DataPortal -from zipline.data.us_equity_pricing import PanelDailyBarReader -from zipline.data.minute_bars import PanelMinuteBarReader +from zipline.data.us_equity_pricing import PanelBarReader from zipline.errors import ( AttachPipelineAfterInitialize, HistoryInInitialize, @@ -649,29 +648,19 @@ class TradingAlgorithm(object): ) if self.sim_params.data_frequency == 'daily': - equity_daily_reader = PanelDailyBarReader( - self.trading_calendar.all_sessions, - copy_panel, - ) - self.data_portal = DataPortal( - self.asset_finder, - self.trading_calendar, - first_trading_day=equity_daily_reader - .first_trading_day, - equity_daily_reader=equity_daily_reader, - ) + equity_reader_arg = 'equity_daily_reader' + calendar = self.trading_calendar.all_sessions elif self.sim_params.data_frequency == 'minute': - equity_minute_reader = PanelMinuteBarReader( - self.trading_calendar.all_minutes, - copy_panel, - ) - self.data_portal = DataPortal( - self.asset_finder, - self.trading_calendar, - first_trading_day=equity_minute_reader - .first_trading_day, - equity_minute_reader=equity_minute_reader, - ) + equity_reader_arg = 'equity_minute_reader' + calendar = self.trading_calendar.all_minutes + equity_reader = PanelBarReader(calendar, copy_panel) + + self.data_portal = DataPortal( + self.asset_finder, + self.trading_calendar, + first_trading_day=equity_reader.first_trading_day, + **{equity_reader_arg: equity_reader} + ) # Force a reset of the performance tracker, in case # this is a repeat run of the algorithm. diff --git a/zipline/data/data_portal.py b/zipline/data/data_portal.py index dcb3e0b1..294ba86d 100644 --- a/zipline/data/data_portal.py +++ b/zipline/data/data_portal.py @@ -554,8 +554,6 @@ class DataPortal(object): self._equity_minute_reader, self._adjustment_reader ) - self.MINUTE_PRICE_ADJUSTMENT_FACTOR = \ - self._equity_minute_reader._ohlc_inverse self._first_trading_day = first_trading_day diff --git a/zipline/data/minute_bars.py b/zipline/data/minute_bars.py index 0fcb4917..df1b4328 100644 --- a/zipline/data/minute_bars.py +++ b/zipline/data/minute_bars.py @@ -21,9 +21,7 @@ import bcolz from bcolz import ctable from intervaltree import IntervalTree import numpy as np -from numpy import zeros import pandas as pd -from pandas import NaT from zipline.data._minute_bar_internal import ( minute_value, @@ -32,11 +30,6 @@ from zipline.data._minute_bar_internal import ( ) from zipline.gens.sim_engine import NANOS_IN_MINUTE -from zipline.utils.preprocess import call -from zipline.utils.input_validation import ( - preprocess, - verify_indices_all_unique, -) from zipline.utils.cli import maybe_show_progress from zipline.utils.memoize import lazyval @@ -986,107 +979,3 @@ class BcolzMinuteBarReader(object): out *= self._ohlc_inverse results.append(out) return results - - -class PanelMinuteBarReader(object): - """ - Reader for data passed as Panel. - - DataPanel Structure - ------- - items : Int64Index - Asset identifiers. Must be unique. - major_axis : DatetimeIndex - Datetimes for data provided by the Panel. Must be unique. - minor_axis : ['open', 'high', 'low', 'close', 'volume'] - Price attributes. Must be unique. - - Attributes - ---------- - The table with which this loader interacts contains the following - attributes: - - panel : pd.Panel - The panel from which to read OHLCV data. - first_trading_day : pd.Timestamp - The first trading day in the dataset. - """ - @preprocess(panel=call(verify_indices_all_unique)) - def __init__(self, calendar, panel): - - panel = panel.copy() - if 'volume' not in panel.minor_axis: - # Fake volume if it does not exist. - panel.loc[:, :, 'volume'] = int(1e9) - - self.first_trading_day = pd.datetools.normalize_date( - panel.major_axis[0] - ) - self._calendar = calendar - - self.panel = panel - - self._ohlc_inverse = 1. / OHLC_RATIO - - @property - def last_available_dt(self): - return self.panel.major_axis[-1] - - def load_raw_arrays(self, columns, start_dt, end_dt, assets): - columns = list(columns) - dts = self.panel.major_axis - index = dts[dts.slice_indexer(start_dt, end_dt)] - shape = (len(index), len(assets)) - results = [] - for col in columns: - outbuf = zeros(shape=shape) - for i, asset in enumerate(assets): - data = self.panel.loc[asset, start_dt:end_dt, col] - data = data.reindex_axis(index).values - outbuf[:, i] = data - results.append(outbuf) - return results - - def spot_price(self, sid, dt, colname): - """ - Parameters - ---------- - sid : int - The asset identifier. - dt : datetime64-like - Midnight of the day for which data is requested. - colname : string - The price field. e.g. ('open', 'high', 'low', 'close', 'volume') - - Returns - ------- - float - The spot price for colname of the given sid on the given day. - Raises a NoDataOnDate exception if the given day and sid is before - or after the date range of the equity. - Returns -1 if the day is within the date range, but the price is - 0. - """ - return self.panel.loc[sid, dt, colname] - - get_value = spot_price - - def get_last_traded_dt(self, sid, dt): - """ - Parameters - ---------- - sid : int - The asset identifier. - dt : datetime64-like - Midnight of the day for which data is requested. - - Returns - ------- - pd.Timestamp : The last known dt for the asset and dt; - NaT if no trade is found before the given dt. - """ - for ts in self.panel.major_axis[self.panel.major_axis - .slice_indexer(end=dt)][::-1]: - if not pd.isnull(self.panel.loc[sid, ts, 'close']): - return ts - return NaT diff --git a/zipline/data/us_equity_pricing.py b/zipline/data/us_equity_pricing.py index a6d92504..993a20ff 100644 --- a/zipline/data/us_equity_pricing.py +++ b/zipline/data/us_equity_pricing.py @@ -35,15 +35,15 @@ from numpy import ( issubdtype, nan, uint32, - zeros, ) from pandas import ( DataFrame, read_csv, Timestamp, NaT, - isnull, - DatetimeIndex) + DatetimeIndex +) +from pandas.core.datetools import normalize_date from pandas.tslib import iNaT from six import ( iteritems, @@ -746,7 +746,7 @@ class BcolzDailyBarReader(DailyBarReader): return price -class PanelDailyBarReader(DailyBarReader): +class PanelBarReader(DailyBarReader): """ Reader for data passed as Panel. @@ -777,7 +777,7 @@ class PanelDailyBarReader(DailyBarReader): # Fake volume if it does not exist. panel.loc[:, :, 'volume'] = int(1e9) - self.first_trading_day = panel.major_axis[0] + self.first_trading_day = normalize_date(panel.major_axis[0]) self._calendar = calendar self.panel = panel @@ -788,28 +788,28 @@ class PanelDailyBarReader(DailyBarReader): @property def last_available_dt(self): - return self._calendar[-1] + # Returns the last Panel index that is on the calendar. + # The slice end is converted from dt to date string so that + # dts on the last day of the calendar get included. + return self.panel.major_axis[ + self.panel.major_axis.slice_indexer( + end=self._calendar[-1].strftime('%Y-%m-%d') + ) + ][-1] @property def trading_calendar(self): return None - def load_raw_arrays(self, columns, start_date, end_date, assets): - columns = list(columns) + def load_raw_arrays(self, columns, start_dt, end_dt, assets): cal = self._calendar - index = cal[cal.slice_indexer(start_date, end_date)] - shape = (len(index), len(assets)) - results = [] - for col in columns: - outbuf = zeros(shape=shape) - for i, asset in enumerate(assets): - data = self.panel.loc[asset, start_date:end_date, col] - data = data.reindex_axis(index).values - outbuf[:, i] = data - results.append(outbuf) - return results + return self.panel.loc[ + list(assets), + start_dt:end_dt, + list(columns) + ].reindex(major_axis=cal[cal.slice_indexer(start_dt, end_dt)]).values.T - def spot_price(self, sid, day, colname): + def spot_price(self, sid, dt, colname): """ Parameters ---------- @@ -829,7 +829,9 @@ class PanelDailyBarReader(DailyBarReader): Returns -1 if the day is within the date range, but the price is 0. """ - return self.panel.loc[sid, day, colname] + return self.panel.loc[sid, dt, colname] + + get_value = spot_price def get_last_traded_dt(self, sid, dt): """ @@ -845,12 +847,9 @@ class PanelDailyBarReader(DailyBarReader): pd.Timestamp : The last know dt for the asset and dt; NaT if no trade is found before the given dt. """ - while dt in self.panel.major_axis: - freq = self.panel.major_axis.freq - if not isnull(self.panel.loc[sid, dt, 'close']): - return dt - dt -= freq - else: + try: + return self.panel.loc[sid, :dt, 'close'].last_valid_index() + except IndexError: return NaT