mirror of
https://github.com/wassname/catalyst.git
synced 2026-07-13 17:42:42 +08:00
Merge pull request #1413 from quantopian/normalize-equity-future-in-data-portal
MAINT: Remove future/equity distinction.
This commit is contained in:
Eddie Hebert
@@ -1615,376 +1615,6 @@ cost of sole txn in test"
|
||||
net_leverage=-0.8181,
|
||||
net_liquidation=1100.0)
|
||||
|
||||
def test_long_future_position(self):
|
||||
"""
|
||||
verify that the performance period calculates properly for a
|
||||
single buy transaction
|
||||
"""
|
||||
self.create_environment_stuff()
|
||||
sim_params = copy.copy(self.sim_params)
|
||||
sim_params.data_frequency = 'minute'
|
||||
|
||||
# post some trades in the market
|
||||
trades = factory.create_trade_history(
|
||||
self.asset3,
|
||||
[10, 10, 10, 11],
|
||||
[100, 100, 100, 100],
|
||||
oneday,
|
||||
sim_params,
|
||||
trading_calendar=self.trading_calendar,
|
||||
)
|
||||
|
||||
data_portal = create_data_portal_from_trade_history(
|
||||
self.env.asset_finder,
|
||||
self.trading_calendar,
|
||||
self.instance_tmpdir,
|
||||
self.sim_params,
|
||||
{3: trades}
|
||||
)
|
||||
|
||||
txn = create_txn(self.asset3, trades[1].dt, 10.0, 1)
|
||||
pt = perf.PositionTracker(self.env.asset_finder,
|
||||
self.sim_params.data_frequency)
|
||||
pp = perf.PerformancePeriod(1000.0, self.env.asset_finder,
|
||||
self.sim_params.data_frequency)
|
||||
pp.position_tracker = pt
|
||||
|
||||
pt.execute_transaction(txn)
|
||||
pp.handle_execution(txn)
|
||||
|
||||
# This verifies that the last sale price is being correctly
|
||||
# set in the positions. If this is not the case then returns can
|
||||
# incorrectly show as sharply dipping if a transaction arrives
|
||||
# before a trade. This is caused by returns being based on holding
|
||||
# stocks with a last sale price of 0.
|
||||
self.assertEqual(pp.positions[3].last_sale_price, 10.0)
|
||||
|
||||
pt.sync_last_sale_prices(trades[-1].dt, False, data_portal)
|
||||
pp.calculate_performance()
|
||||
|
||||
self.assertEqual(
|
||||
pp.cash_flow,
|
||||
0,
|
||||
"there should be no cash flow on a futures txn"
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
len(pp.positions),
|
||||
1,
|
||||
"should be just one position")
|
||||
|
||||
self.assertEqual(
|
||||
pp.positions[3].sid,
|
||||
txn.sid,
|
||||
"position should be in security with id 1")
|
||||
|
||||
self.assertEqual(
|
||||
pp.positions[3].amount,
|
||||
txn.amount,
|
||||
"should have a position of {sharecount} shares".format(
|
||||
sharecount=txn.amount
|
||||
)
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
pp.positions[3].cost_basis,
|
||||
txn.price,
|
||||
"should have a cost basis of 10"
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
pp.positions[3].last_sale_price,
|
||||
trades[-1]['price'],
|
||||
"last sale should be same as last trade. \
|
||||
expected {exp} actual {act}".format(
|
||||
exp=trades[-1]['price'],
|
||||
act=pp.positions[3].last_sale_price)
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
pp.ending_value,
|
||||
0,
|
||||
"ending value should be 0 because only futures are held"
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
pp.ending_exposure,
|
||||
1100,
|
||||
"ending exposure should be price of last trade times number of \
|
||||
contracts in position")
|
||||
|
||||
self.assertEqual(pp.pnl, 100, "gain of 1 on 1 100x contract should be "
|
||||
"100")
|
||||
|
||||
check_perf_period(
|
||||
pp,
|
||||
gross_leverage=1.0,
|
||||
net_leverage=1.0,
|
||||
long_exposure=1100.0,
|
||||
longs_count=1,
|
||||
short_exposure=0.0,
|
||||
shorts_count=0)
|
||||
|
||||
# Validate that the account attributes were updated.
|
||||
account = pp.as_account()
|
||||
check_account(account,
|
||||
settled_cash=1100.0,
|
||||
equity_with_loan=1100.0,
|
||||
total_positions_value=0.0,
|
||||
total_positions_exposure=1100.0,
|
||||
regt_equity=1100.0,
|
||||
available_funds=1100.0,
|
||||
excess_liquidity=1100.0,
|
||||
cushion=1.0,
|
||||
leverage=1.0,
|
||||
net_leverage=1.0,
|
||||
net_liquidation=1100.0)
|
||||
|
||||
def test_short_future_position(self):
|
||||
"""verify that the performance period calculates properly for a \
|
||||
single short-sale transaction"""
|
||||
self.create_environment_stuff(num_days=6)
|
||||
|
||||
trades = factory.create_trade_history(
|
||||
self.asset3,
|
||||
[10, 10, 10, 11, 10, 9],
|
||||
[100, 100, 100, 100, 100, 100],
|
||||
oneday,
|
||||
self.sim_params,
|
||||
trading_calendar=self.trading_calendar,
|
||||
)
|
||||
|
||||
data_portal = create_data_portal_from_trade_history(
|
||||
self.env.asset_finder,
|
||||
self.trading_calendar,
|
||||
self.instance_tmpdir,
|
||||
self.sim_params,
|
||||
{3: trades}
|
||||
)
|
||||
trades_1 = trades[:-2]
|
||||
|
||||
txn = create_txn(self.asset3, trades[0].dt, 10.0, -1)
|
||||
pt = perf.PositionTracker(self.env.asset_finder,
|
||||
self.sim_params.data_frequency)
|
||||
pp = perf.PerformancePeriod(1000.0, self.env.asset_finder,
|
||||
self.sim_params.data_frequency)
|
||||
pp.position_tracker = pt
|
||||
|
||||
pt.execute_transaction(txn)
|
||||
pp.handle_execution(txn)
|
||||
|
||||
pt.sync_last_sale_prices(trades[-3].dt, False, data_portal)
|
||||
pp.calculate_performance()
|
||||
|
||||
self.assertEqual(
|
||||
pp.cash_flow,
|
||||
0,
|
||||
"there should be no cash flow on a futures txn"
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
len(pp.positions),
|
||||
1,
|
||||
"should be just one position")
|
||||
|
||||
self.assertEqual(
|
||||
pp.positions[3].sid,
|
||||
txn.sid,
|
||||
"position should be in future from the transaction"
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
pp.positions[3].amount,
|
||||
-1,
|
||||
"should have a position of -1 contract"
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
pp.positions[3].cost_basis,
|
||||
txn.price,
|
||||
"should have a cost basis of 10"
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
pp.positions[3].last_sale_price,
|
||||
trades_1[-1]['price'],
|
||||
"last sale should be price of last trade"
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
pp.ending_value,
|
||||
0,
|
||||
"ending value should be 0 because only futures are held"
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
pp.ending_exposure,
|
||||
-1100,
|
||||
"ending exposure should be price of last trade times number of \
|
||||
contracts in position")
|
||||
|
||||
self.assertEqual(pp.pnl, -100, "gain of 1 on 1 100x contract should be"
|
||||
" 100")
|
||||
|
||||
# simulate additional trades, and ensure that the position value
|
||||
# reflects the new price
|
||||
trades_2 = trades[-2:]
|
||||
|
||||
# simulate a rollover to a new period
|
||||
pp.rollover()
|
||||
|
||||
pt.sync_last_sale_prices(trades_2[-1].dt, False, data_portal)
|
||||
pp.calculate_performance()
|
||||
|
||||
self.assertEqual(
|
||||
pp.cash_flow,
|
||||
0,
|
||||
"capital used should be zero, there were no transactions in \
|
||||
performance period"
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
len(pp.positions),
|
||||
1,
|
||||
"should be just one position"
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
pp.positions[3].sid,
|
||||
txn.sid,
|
||||
"position should be in future from the transaction"
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
pp.positions[3].amount,
|
||||
-1,
|
||||
"should have a position of -1 contract"
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
pp.positions[3].cost_basis,
|
||||
txn.price,
|
||||
"should have a cost basis of 10"
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
pp.positions[3].last_sale_price,
|
||||
trades_2[-1].price,
|
||||
"last sale should be price of last trade"
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
pp.ending_value,
|
||||
0,
|
||||
"ending value should be 0 because only futures are held")
|
||||
|
||||
self.assertEqual(
|
||||
pp.ending_exposure,
|
||||
-900,
|
||||
"ending exposure should be price of last trade times number of \
|
||||
shares in position")
|
||||
|
||||
self.assertEqual(
|
||||
pp.pnl,
|
||||
200,
|
||||
"drop of 2 on -1 100x contract should be 200"
|
||||
)
|
||||
|
||||
# now run a performance period encompassing the entire trade sample.
|
||||
ptTotal = perf.PositionTracker(self.env.asset_finder,
|
||||
self.sim_params.data_frequency)
|
||||
ppTotal = perf.PerformancePeriod(1000.0, self.env.asset_finder,
|
||||
self.sim_params.data_frequency)
|
||||
ppTotal.position_tracker = ptTotal
|
||||
|
||||
for trade in trades_1:
|
||||
ptTotal.sync_last_sale_prices(trade.dt, False, data_portal)
|
||||
|
||||
ptTotal.execute_transaction(txn)
|
||||
ppTotal.handle_execution(txn)
|
||||
|
||||
for trade in trades_2:
|
||||
ptTotal.sync_last_sale_prices(trade.dt, False, data_portal)
|
||||
|
||||
ppTotal.calculate_performance()
|
||||
|
||||
self.assertEqual(
|
||||
ppTotal.cash_flow,
|
||||
0,
|
||||
"capital used should be equal to the opposite of the transaction \
|
||||
cost of sole txn in test"
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
len(ppTotal.positions),
|
||||
1,
|
||||
"should be just one position"
|
||||
)
|
||||
self.assertEqual(
|
||||
ppTotal.positions[3].sid,
|
||||
txn.sid,
|
||||
"position should be in security from the transaction"
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
ppTotal.positions[3].amount,
|
||||
-1,
|
||||
"should have a position of -1 contract"
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
ppTotal.positions[3].cost_basis,
|
||||
txn.price,
|
||||
"should have a cost basis of 10"
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
ppTotal.positions[3].last_sale_price,
|
||||
trades_2[-1].price,
|
||||
"last sale should be price of last trade"
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
pp.ending_value,
|
||||
0,
|
||||
"ending value should be 0 because only futures are held")
|
||||
|
||||
self.assertEqual(
|
||||
pp.ending_exposure,
|
||||
-900,
|
||||
"ending exposure should be price of last trade times number of \
|
||||
shares in position")
|
||||
|
||||
self.assertEqual(
|
||||
ppTotal.pnl,
|
||||
100,
|
||||
"drop of 1 on -1 100x contract should be 100"
|
||||
)
|
||||
|
||||
check_perf_period(
|
||||
pp,
|
||||
gross_leverage=0.8181,
|
||||
net_leverage=-0.8181,
|
||||
long_exposure=0.0,
|
||||
longs_count=0,
|
||||
short_exposure=-900.0,
|
||||
shorts_count=1)
|
||||
|
||||
# Validate that the account attributes.
|
||||
account = ppTotal.as_account()
|
||||
check_account(account,
|
||||
settled_cash=1100.0,
|
||||
equity_with_loan=1100.0,
|
||||
total_positions_value=0.0,
|
||||
total_positions_exposure=-900.0,
|
||||
regt_equity=1100.0,
|
||||
available_funds=1100.0,
|
||||
excess_liquidity=1100.0,
|
||||
cushion=1.0,
|
||||
leverage=0.8181,
|
||||
net_leverage=-0.8181,
|
||||
net_liquidation=1100.0)
|
||||
|
||||
def test_covering_short(self):
|
||||
"""verify performance where short is bought and covered, and shares \
|
||||
trade after cover"""
|
||||
|
||||
+33
-132
@@ -126,7 +126,7 @@ class DataPortal(object):
|
||||
|
||||
self._equity_daily_reader = equity_daily_reader
|
||||
if self._equity_daily_reader is not None:
|
||||
self._equity_history_loader = DailyHistoryLoader(
|
||||
self._history_loader = DailyHistoryLoader(
|
||||
self.trading_calendar,
|
||||
self._equity_daily_reader,
|
||||
self._adjustment_reader
|
||||
@@ -146,17 +146,16 @@ class DataPortal(object):
|
||||
}
|
||||
}
|
||||
|
||||
if self._equity_minute_reader is not None:
|
||||
self._equity_daily_aggregator = DailyHistoryAggregator(
|
||||
self.trading_calendar.schedule.market_open,
|
||||
self._equity_minute_reader,
|
||||
self.trading_calendar
|
||||
)
|
||||
self._equity_minute_history_loader = MinuteHistoryLoader(
|
||||
self.trading_calendar,
|
||||
self._equity_minute_reader,
|
||||
self._adjustment_reader
|
||||
)
|
||||
self._daily_aggregator = DailyHistoryAggregator(
|
||||
self.trading_calendar.schedule.market_open,
|
||||
self._equity_minute_reader,
|
||||
self.trading_calendar
|
||||
)
|
||||
self._minute_history_loader = MinuteHistoryLoader(
|
||||
self.trading_calendar,
|
||||
self._equity_minute_reader,
|
||||
self._adjustment_reader
|
||||
)
|
||||
|
||||
self._first_trading_day = first_trading_day
|
||||
|
||||
@@ -511,9 +510,9 @@ class DataPortal(object):
|
||||
)
|
||||
|
||||
def _get_daily_data(self, asset, column, dt):
|
||||
reader = self._pricing_readers[type(asset)]['daily']
|
||||
if column == "last_traded":
|
||||
last_traded_dt = \
|
||||
self._equity_daily_reader.get_last_traded_dt(asset, dt)
|
||||
last_traded_dt = reader.get_last_traded_dt(asset, dt)
|
||||
|
||||
if pd.isnull(last_traded_dt):
|
||||
return pd.NaT
|
||||
@@ -522,7 +521,7 @@ class DataPortal(object):
|
||||
elif column in OHLCV_FIELDS:
|
||||
# don't forward fill
|
||||
try:
|
||||
val = self._equity_daily_reader.spot_price(asset, dt, column)
|
||||
val = reader.spot_price(asset, dt, column)
|
||||
if val == -1:
|
||||
if column == "volume":
|
||||
return 0
|
||||
@@ -536,7 +535,7 @@ class DataPortal(object):
|
||||
found_dt = dt
|
||||
while True:
|
||||
try:
|
||||
value = self._equity_daily_reader.spot_price(
|
||||
value = reader.spot_price(
|
||||
asset, found_dt, "close"
|
||||
)
|
||||
if value != -1:
|
||||
@@ -581,88 +580,16 @@ class DataPortal(object):
|
||||
index=days_for_window,
|
||||
columns=None)
|
||||
|
||||
future_data = []
|
||||
eq_assets = []
|
||||
|
||||
for asset in assets:
|
||||
if isinstance(asset, Future):
|
||||
future_data.append(self._get_history_daily_window_future(
|
||||
asset, days_for_window, end_dt, field_to_use
|
||||
))
|
||||
else:
|
||||
eq_assets.append(asset)
|
||||
eq_data = self._get_history_daily_window_equities(
|
||||
eq_assets, days_for_window, end_dt, field_to_use
|
||||
data = self._get_history_daily_window_data(
|
||||
assets, days_for_window, end_dt, field_to_use
|
||||
)
|
||||
if future_data:
|
||||
# TODO: This case appears to be uncovered by testing.
|
||||
data = np.concatenate(eq_data, np.array(future_data).T)
|
||||
else:
|
||||
data = eq_data
|
||||
return pd.DataFrame(
|
||||
data,
|
||||
index=days_for_window,
|
||||
columns=assets
|
||||
)
|
||||
|
||||
def _get_history_daily_window_future(self, asset, days_for_window,
|
||||
end_dt, column):
|
||||
# Since we don't have daily bcolz files for futures (yet), use minute
|
||||
# bars to calculate the daily values.
|
||||
data = []
|
||||
data_groups = []
|
||||
|
||||
# get all the minutes for the days NOT including today
|
||||
for day in days_for_window[:-1]:
|
||||
minutes = self.sessions_in_range.minutes_for_session(day)
|
||||
|
||||
values_for_day = np.zeros(len(minutes), dtype=np.float64)
|
||||
|
||||
for idx, minute in enumerate(minutes):
|
||||
minute_val = self._get_minute_spot_value_future(
|
||||
asset, column, minute
|
||||
)
|
||||
|
||||
values_for_day[idx] = minute_val
|
||||
|
||||
data_groups.append(values_for_day)
|
||||
|
||||
# get the minutes for today
|
||||
last_day_minutes = pd.date_range(
|
||||
start=self.trading_calendar.open_and_close_for_session(end_dt)[0],
|
||||
end=end_dt,
|
||||
freq="T"
|
||||
)
|
||||
|
||||
values_for_last_day = np.zeros(len(last_day_minutes), dtype=np.float64)
|
||||
|
||||
for idx, minute in enumerate(last_day_minutes):
|
||||
minute_val = self._get_minute_spot_value_future(
|
||||
asset, column, minute
|
||||
)
|
||||
|
||||
values_for_last_day[idx] = minute_val
|
||||
|
||||
data_groups.append(values_for_last_day)
|
||||
|
||||
for group in data_groups:
|
||||
if len(group) == 0:
|
||||
continue
|
||||
|
||||
if column == 'volume':
|
||||
data.append(np.sum(group))
|
||||
elif column == 'open':
|
||||
data.append(group[0])
|
||||
elif column == 'close':
|
||||
data.append(group[-1])
|
||||
elif column == 'high':
|
||||
data.append(np.amax(group))
|
||||
elif column == 'low':
|
||||
data.append(np.amin(group))
|
||||
|
||||
return data
|
||||
|
||||
def _get_history_daily_window_equities(
|
||||
def _get_history_daily_window_data(
|
||||
self, assets, days_for_window, end_dt, field_to_use):
|
||||
ends_at_midnight = end_dt.hour == 0 and end_dt.minute == 0
|
||||
|
||||
@@ -686,19 +613,19 @@ class DataPortal(object):
|
||||
)
|
||||
|
||||
if field_to_use == 'open':
|
||||
minute_value = self._equity_daily_aggregator.opens(
|
||||
minute_value = self._daily_aggregator.opens(
|
||||
assets, end_dt)
|
||||
elif field_to_use == 'high':
|
||||
minute_value = self._equity_daily_aggregator.highs(
|
||||
minute_value = self._daily_aggregator.highs(
|
||||
assets, end_dt)
|
||||
elif field_to_use == 'low':
|
||||
minute_value = self._equity_daily_aggregator.lows(
|
||||
minute_value = self._daily_aggregator.lows(
|
||||
assets, end_dt)
|
||||
elif field_to_use == 'close':
|
||||
minute_value = self._equity_daily_aggregator.closes(
|
||||
minute_value = self._daily_aggregator.closes(
|
||||
assets, end_dt)
|
||||
elif field_to_use == 'volume':
|
||||
minute_value = self._equity_daily_aggregator.volumes(
|
||||
minute_value = self._daily_aggregator.volumes(
|
||||
assets, end_dt)
|
||||
|
||||
# append the partial day.
|
||||
@@ -860,40 +787,14 @@ class DataPortal(object):
|
||||
-------
|
||||
A numpy array with requested values.
|
||||
"""
|
||||
if isinstance(assets, Future):
|
||||
return self._get_minute_window_for_future([assets], field,
|
||||
minutes_for_window)
|
||||
else:
|
||||
# TODO: Make caller accept assets.
|
||||
window = self._get_minute_window_for_equities(assets, field,
|
||||
minutes_for_window)
|
||||
return window
|
||||
return self._get_minute_window_data(assets, field, minutes_for_window)
|
||||
|
||||
def _get_minute_window_for_future(self, asset, field, minutes_for_window):
|
||||
# THIS IS TEMPORARY. For now, we are only exposing futures within
|
||||
# equity trading hours (9:30 am to 4pm, Eastern). The easiest way to
|
||||
# do this is to simply do a spot lookup for each desired minute.
|
||||
return_data = np.zeros(len(minutes_for_window), dtype=np.float64)
|
||||
for idx, minute in enumerate(minutes_for_window):
|
||||
return_data[idx] = \
|
||||
self._get_minute_spot_value_future(asset, field, minute)
|
||||
|
||||
# Note: an improvement could be to find the consecutive runs within
|
||||
# minutes_for_window, and use them to read the underlying ctable
|
||||
# more efficiently.
|
||||
|
||||
# Once futures are on 24-hour clock, then we can just grab all the
|
||||
# requested minutes in one shot from the ctable.
|
||||
|
||||
# no adjustments for futures, yay.
|
||||
return return_data
|
||||
|
||||
def _get_minute_window_for_equities(
|
||||
def _get_minute_window_data(
|
||||
self, assets, field, minutes_for_window):
|
||||
return self._equity_minute_history_loader.history(assets,
|
||||
minutes_for_window,
|
||||
field,
|
||||
False)
|
||||
return self._minute_history_loader.history(assets,
|
||||
minutes_for_window,
|
||||
field,
|
||||
False)
|
||||
|
||||
def _apply_all_adjustments(self, data, asset, dts, field,
|
||||
price_adj_factor=1.0):
|
||||
@@ -1007,10 +908,10 @@ class DataPortal(object):
|
||||
return_array[:] = np.NAN
|
||||
|
||||
if bar_count != 0:
|
||||
data = self._equity_history_loader.history(assets,
|
||||
days_in_window,
|
||||
field,
|
||||
extra_slot)
|
||||
data = self._history_loader.history(assets,
|
||||
days_in_window,
|
||||
field,
|
||||
extra_slot)
|
||||
if extra_slot:
|
||||
return_array[:len(return_array) - 1, :] = data
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user