wassname/catalyst
1
0
Fork 0
mirror of https://github.com/wassname/catalyst.git synced 2026-07-10 16:11:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

MAINT: Combine daily and minute into PanelBarReader.

Browse Source 
Also simplify `load_raw_arrays` and `get_last_traded_dt`.
This commit is contained in:
Nathan Wolfe
2016-07-13 17:45:07 -04:00
parent 69506570dd
commit 763f2ab8b4
5 changed files with 42 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_panel_daily_bar_reader.py
+3 -3
View File
@@ -18,7 +18,7 @@ from itertools import permutations, product
import numpy as np
import pandas as pd
from zipline.data.us_equity_pricing import PanelDailyBarReader
from zipline.data.us_equity_pricing import PanelBarReader
from zipline.testing import ExplodingObject
from zipline.testing.fixtures import (
WithAssetFinder,
@@ -55,7 +55,7 @@ class TestPanelDailyBarReader(WithAssetFinder,
minor_axis=minor_axis,
)
cls.reader = PanelDailyBarReader(days, cls.panel)
cls.reader = PanelBarReader(days, cls.panel)
def test_spot_price(self):
panel = self.panel
@@ -83,7 +83,7 @@ class TestPanelDailyBarReader(WithAssetFinder,
for axis_order in permutations((0, 1, 2)):
transposed = panel.transpose(*axis_order)
with self.assertRaises(ValueError) as e:
PanelDailyBarReader(unused, transposed)
PanelBarReader(unused, transposed)
expected = (
"Duplicate entries in Panel.{name}: ['a', 'b'].".format(
zipline/algorithm.py
+13 -24
View File
@@ -37,8 +37,7 @@ from six import (
from zipline._protocol import handle_non_market_minutes
from zipline.assets.synthetic import make_simple_equity_info
from zipline.data.data_portal import DataPortal
from zipline.data.us_equity_pricing import PanelDailyBarReader
from zipline.data.minute_bars import PanelMinuteBarReader
from zipline.data.us_equity_pricing import PanelBarReader
from zipline.errors import (
AttachPipelineAfterInitialize,
HistoryInInitialize,
@@ -649,29 +648,19 @@ class TradingAlgorithm(object):
)
if self.sim_params.data_frequency == 'daily':
equity_daily_reader = PanelDailyBarReader(
self.trading_calendar.all_sessions,
copy_panel,
)
self.data_portal = DataPortal(
self.asset_finder,
self.trading_calendar,
first_trading_day=equity_daily_reader
.first_trading_day,
equity_daily_reader=equity_daily_reader,
)
equity_reader_arg = 'equity_daily_reader'
calendar = self.trading_calendar.all_sessions
elif self.sim_params.data_frequency == 'minute':
equity_minute_reader = PanelMinuteBarReader(
self.trading_calendar.all_minutes,
copy_panel,
)
self.data_portal = DataPortal(
self.asset_finder,
self.trading_calendar,
first_trading_day=equity_minute_reader
.first_trading_day,
equity_minute_reader=equity_minute_reader,
)
equity_reader_arg = 'equity_minute_reader'
calendar = self.trading_calendar.all_minutes
equity_reader = PanelBarReader(calendar, copy_panel)
self.data_portal = DataPortal(
self.asset_finder,
self.trading_calendar,
first_trading_day=equity_reader.first_trading_day,
**{equity_reader_arg: equity_reader}
)
# Force a reset of the performance tracker, in case
# this is a repeat run of the algorithm.
zipline/data/data_portal.py
-2
View File
@@ -554,8 +554,6 @@ class DataPortal(object):
self._equity_minute_reader,
self._adjustment_reader
)
self.MINUTE_PRICE_ADJUSTMENT_FACTOR = \
self._equity_minute_reader._ohlc_inverse
self._first_trading_day = first_trading_day
zipline/data/minute_bars.py
-111
View File
@@ -21,9 +21,7 @@ import bcolz
from bcolz import ctable
from intervaltree import IntervalTree
import numpy as np
from numpy import zeros
import pandas as pd
from pandas import NaT
from zipline.data._minute_bar_internal import (
minute_value,
@@ -32,11 +30,6 @@ from zipline.data._minute_bar_internal import (
)
from zipline.gens.sim_engine import NANOS_IN_MINUTE
from zipline.utils.preprocess import call
from zipline.utils.input_validation import (
preprocess,
verify_indices_all_unique,
)
from zipline.utils.cli import maybe_show_progress
from zipline.utils.memoize import lazyval
@@ -986,107 +979,3 @@ class BcolzMinuteBarReader(object):
out *= self._ohlc_inverse
results.append(out)
return results
class PanelMinuteBarReader(object):
"""
Reader for data passed as Panel.
DataPanel Structure
-------
items : Int64Index
Asset identifiers. Must be unique.
major_axis : DatetimeIndex
Datetimes for data provided by the Panel. Must be unique.
minor_axis : ['open', 'high', 'low', 'close', 'volume']
Price attributes. Must be unique.
Attributes
----------
The table with which this loader interacts contains the following
attributes:
panel : pd.Panel
The panel from which to read OHLCV data.
first_trading_day : pd.Timestamp
The first trading day in the dataset.
"""
@preprocess(panel=call(verify_indices_all_unique))
def __init__(self, calendar, panel):
panel = panel.copy()
if 'volume' not in panel.minor_axis:
# Fake volume if it does not exist.
panel.loc[:, :, 'volume'] = int(1e9)
self.first_trading_day = pd.datetools.normalize_date(
panel.major_axis[0]
)
self._calendar = calendar
self.panel = panel
self._ohlc_inverse = 1. / OHLC_RATIO
@property
def last_available_dt(self):
return self.panel.major_axis[-1]
def load_raw_arrays(self, columns, start_dt, end_dt, assets):
columns = list(columns)
dts = self.panel.major_axis
index = dts[dts.slice_indexer(start_dt, end_dt)]
shape = (len(index), len(assets))
results = []
for col in columns:
outbuf = zeros(shape=shape)
for i, asset in enumerate(assets):
data = self.panel.loc[asset, start_dt:end_dt, col]
data = data.reindex_axis(index).values
outbuf[:, i] = data
results.append(outbuf)
return results
def spot_price(self, sid, dt, colname):
"""
Parameters
----------
sid : int
The asset identifier.
dt : datetime64-like
Midnight of the day for which data is requested.
colname : string
The price field. e.g. ('open', 'high', 'low', 'close', 'volume')
Returns
-------
float
The spot price for colname of the given sid on the given day.
Raises a NoDataOnDate exception if the given day and sid is before
or after the date range of the equity.
Returns -1 if the day is within the date range, but the price is
0.
"""
return self.panel.loc[sid, dt, colname]
get_value = spot_price
def get_last_traded_dt(self, sid, dt):
"""
Parameters
----------
sid : int
The asset identifier.
dt : datetime64-like
Midnight of the day for which data is requested.
Returns
-------
pd.Timestamp : The last known dt for the asset and dt;
NaT if no trade is found before the given dt.
"""
for ts in self.panel.major_axis[self.panel.major_axis
.slice_indexer(end=dt)][::-1]:
if not pd.isnull(self.panel.loc[sid, ts, 'close']):
return ts
return NaT
zipline/data/us_equity_pricing.py
+26 -27
View File
@@ -35,15 +35,15 @@ from numpy import (
issubdtype,
nan,
uint32,
zeros,
)
from pandas import (
DataFrame,
read_csv,
Timestamp,
NaT,
isnull,
DatetimeIndex)
DatetimeIndex
)
from pandas.core.datetools import normalize_date
from pandas.tslib import iNaT
from six import (
iteritems,
@@ -746,7 +746,7 @@ class BcolzDailyBarReader(DailyBarReader):
return price
class PanelDailyBarReader(DailyBarReader):
class PanelBarReader(DailyBarReader):
"""
Reader for data passed as Panel.
@@ -777,7 +777,7 @@ class PanelDailyBarReader(DailyBarReader):
# Fake volume if it does not exist.
panel.loc[:, :, 'volume'] = int(1e9)
self.first_trading_day = panel.major_axis[0]
self.first_trading_day = normalize_date(panel.major_axis[0])
self._calendar = calendar
self.panel = panel
@@ -788,28 +788,28 @@ class PanelDailyBarReader(DailyBarReader):
@property
def last_available_dt(self):
return self._calendar[-1]
# Returns the last Panel index that is on the calendar.
# The slice end is converted from dt to date string so that
# dts on the last day of the calendar get included.
return self.panel.major_axis[
self.panel.major_axis.slice_indexer(
end=self._calendar[-1].strftime('%Y-%m-%d')
)
][-1]
@property
def trading_calendar(self):
return None
def load_raw_arrays(self, columns, start_date, end_date, assets):
columns = list(columns)
def load_raw_arrays(self, columns, start_dt, end_dt, assets):
cal = self._calendar
index = cal[cal.slice_indexer(start_date, end_date)]
shape = (len(index), len(assets))
results = []
for col in columns:
outbuf = zeros(shape=shape)
for i, asset in enumerate(assets):
data = self.panel.loc[asset, start_date:end_date, col]
data = data.reindex_axis(index).values
outbuf[:, i] = data
results.append(outbuf)
return results
return self.panel.loc[
list(assets),
start_dt:end_dt,
list(columns)
].reindex(major_axis=cal[cal.slice_indexer(start_dt, end_dt)]).values.T
def spot_price(self, sid, day, colname):
def spot_price(self, sid, dt, colname):
"""
Parameters
----------
@@ -829,7 +829,9 @@ class PanelDailyBarReader(DailyBarReader):
Returns -1 if the day is within the date range, but the price is
0.
"""
return self.panel.loc[sid, day, colname]
return self.panel.loc[sid, dt, colname]
get_value = spot_price
def get_last_traded_dt(self, sid, dt):
"""
@@ -845,12 +847,9 @@ class PanelDailyBarReader(DailyBarReader):
pd.Timestamp : The last know dt for the asset and dt;
NaT if no trade is found before the given dt.
"""
while dt in self.panel.major_axis:
freq = self.panel.major_axis.freq
if not isnull(self.panel.loc[sid, dt, 'close']):
return dt
dt -= freq
else:
try:
return self.panel.loc[sid, :dt, 'close'].last_valid_index()
except IndexError:
return NaT