diff --git a/tests/test_perf_tracking.py b/tests/test_perf_tracking.py index 7e053720..077705dd 100644 --- a/tests/test_perf_tracking.py +++ b/tests/test_perf_tracking.py @@ -1615,376 +1615,6 @@ cost of sole txn in test" net_leverage=-0.8181, net_liquidation=1100.0) - def test_long_future_position(self): - """ - verify that the performance period calculates properly for a - single buy transaction - """ - self.create_environment_stuff() - sim_params = copy.copy(self.sim_params) - sim_params.data_frequency = 'minute' - - # post some trades in the market - trades = factory.create_trade_history( - self.asset3, - [10, 10, 10, 11], - [100, 100, 100, 100], - oneday, - sim_params, - trading_calendar=self.trading_calendar, - ) - - data_portal = create_data_portal_from_trade_history( - self.env.asset_finder, - self.trading_calendar, - self.instance_tmpdir, - self.sim_params, - {3: trades} - ) - - txn = create_txn(self.asset3, trades[1].dt, 10.0, 1) - pt = perf.PositionTracker(self.env.asset_finder, - self.sim_params.data_frequency) - pp = perf.PerformancePeriod(1000.0, self.env.asset_finder, - self.sim_params.data_frequency) - pp.position_tracker = pt - - pt.execute_transaction(txn) - pp.handle_execution(txn) - - # This verifies that the last sale price is being correctly - # set in the positions. If this is not the case then returns can - # incorrectly show as sharply dipping if a transaction arrives - # before a trade. This is caused by returns being based on holding - # stocks with a last sale price of 0. - self.assertEqual(pp.positions[3].last_sale_price, 10.0) - - pt.sync_last_sale_prices(trades[-1].dt, False, data_portal) - pp.calculate_performance() - - self.assertEqual( - pp.cash_flow, - 0, - "there should be no cash flow on a futures txn" - ) - - self.assertEqual( - len(pp.positions), - 1, - "should be just one position") - - self.assertEqual( - pp.positions[3].sid, - txn.sid, - "position should be in security with id 1") - - self.assertEqual( - pp.positions[3].amount, - txn.amount, - "should have a position of {sharecount} shares".format( - sharecount=txn.amount - ) - ) - - self.assertEqual( - pp.positions[3].cost_basis, - txn.price, - "should have a cost basis of 10" - ) - - self.assertEqual( - pp.positions[3].last_sale_price, - trades[-1]['price'], - "last sale should be same as last trade. \ - expected {exp} actual {act}".format( - exp=trades[-1]['price'], - act=pp.positions[3].last_sale_price) - ) - - self.assertEqual( - pp.ending_value, - 0, - "ending value should be 0 because only futures are held" - ) - - self.assertEqual( - pp.ending_exposure, - 1100, - "ending exposure should be price of last trade times number of \ - contracts in position") - - self.assertEqual(pp.pnl, 100, "gain of 1 on 1 100x contract should be " - "100") - - check_perf_period( - pp, - gross_leverage=1.0, - net_leverage=1.0, - long_exposure=1100.0, - longs_count=1, - short_exposure=0.0, - shorts_count=0) - - # Validate that the account attributes were updated. - account = pp.as_account() - check_account(account, - settled_cash=1100.0, - equity_with_loan=1100.0, - total_positions_value=0.0, - total_positions_exposure=1100.0, - regt_equity=1100.0, - available_funds=1100.0, - excess_liquidity=1100.0, - cushion=1.0, - leverage=1.0, - net_leverage=1.0, - net_liquidation=1100.0) - - def test_short_future_position(self): - """verify that the performance period calculates properly for a \ -single short-sale transaction""" - self.create_environment_stuff(num_days=6) - - trades = factory.create_trade_history( - self.asset3, - [10, 10, 10, 11, 10, 9], - [100, 100, 100, 100, 100, 100], - oneday, - self.sim_params, - trading_calendar=self.trading_calendar, - ) - - data_portal = create_data_portal_from_trade_history( - self.env.asset_finder, - self.trading_calendar, - self.instance_tmpdir, - self.sim_params, - {3: trades} - ) - trades_1 = trades[:-2] - - txn = create_txn(self.asset3, trades[0].dt, 10.0, -1) - pt = perf.PositionTracker(self.env.asset_finder, - self.sim_params.data_frequency) - pp = perf.PerformancePeriod(1000.0, self.env.asset_finder, - self.sim_params.data_frequency) - pp.position_tracker = pt - - pt.execute_transaction(txn) - pp.handle_execution(txn) - - pt.sync_last_sale_prices(trades[-3].dt, False, data_portal) - pp.calculate_performance() - - self.assertEqual( - pp.cash_flow, - 0, - "there should be no cash flow on a futures txn" - ) - - self.assertEqual( - len(pp.positions), - 1, - "should be just one position") - - self.assertEqual( - pp.positions[3].sid, - txn.sid, - "position should be in future from the transaction" - ) - - self.assertEqual( - pp.positions[3].amount, - -1, - "should have a position of -1 contract" - ) - - self.assertEqual( - pp.positions[3].cost_basis, - txn.price, - "should have a cost basis of 10" - ) - - self.assertEqual( - pp.positions[3].last_sale_price, - trades_1[-1]['price'], - "last sale should be price of last trade" - ) - - self.assertEqual( - pp.ending_value, - 0, - "ending value should be 0 because only futures are held" - ) - - self.assertEqual( - pp.ending_exposure, - -1100, - "ending exposure should be price of last trade times number of \ - contracts in position") - - self.assertEqual(pp.pnl, -100, "gain of 1 on 1 100x contract should be" - " 100") - - # simulate additional trades, and ensure that the position value - # reflects the new price - trades_2 = trades[-2:] - - # simulate a rollover to a new period - pp.rollover() - - pt.sync_last_sale_prices(trades_2[-1].dt, False, data_portal) - pp.calculate_performance() - - self.assertEqual( - pp.cash_flow, - 0, - "capital used should be zero, there were no transactions in \ - performance period" - ) - - self.assertEqual( - len(pp.positions), - 1, - "should be just one position" - ) - - self.assertEqual( - pp.positions[3].sid, - txn.sid, - "position should be in future from the transaction" - ) - - self.assertEqual( - pp.positions[3].amount, - -1, - "should have a position of -1 contract" - ) - - self.assertEqual( - pp.positions[3].cost_basis, - txn.price, - "should have a cost basis of 10" - ) - - self.assertEqual( - pp.positions[3].last_sale_price, - trades_2[-1].price, - "last sale should be price of last trade" - ) - - self.assertEqual( - pp.ending_value, - 0, - "ending value should be 0 because only futures are held") - - self.assertEqual( - pp.ending_exposure, - -900, - "ending exposure should be price of last trade times number of \ - shares in position") - - self.assertEqual( - pp.pnl, - 200, - "drop of 2 on -1 100x contract should be 200" - ) - - # now run a performance period encompassing the entire trade sample. - ptTotal = perf.PositionTracker(self.env.asset_finder, - self.sim_params.data_frequency) - ppTotal = perf.PerformancePeriod(1000.0, self.env.asset_finder, - self.sim_params.data_frequency) - ppTotal.position_tracker = ptTotal - - for trade in trades_1: - ptTotal.sync_last_sale_prices(trade.dt, False, data_portal) - - ptTotal.execute_transaction(txn) - ppTotal.handle_execution(txn) - - for trade in trades_2: - ptTotal.sync_last_sale_prices(trade.dt, False, data_portal) - - ppTotal.calculate_performance() - - self.assertEqual( - ppTotal.cash_flow, - 0, - "capital used should be equal to the opposite of the transaction \ -cost of sole txn in test" - ) - - self.assertEqual( - len(ppTotal.positions), - 1, - "should be just one position" - ) - self.assertEqual( - ppTotal.positions[3].sid, - txn.sid, - "position should be in security from the transaction" - ) - - self.assertEqual( - ppTotal.positions[3].amount, - -1, - "should have a position of -1 contract" - ) - - self.assertEqual( - ppTotal.positions[3].cost_basis, - txn.price, - "should have a cost basis of 10" - ) - - self.assertEqual( - ppTotal.positions[3].last_sale_price, - trades_2[-1].price, - "last sale should be price of last trade" - ) - - self.assertEqual( - pp.ending_value, - 0, - "ending value should be 0 because only futures are held") - - self.assertEqual( - pp.ending_exposure, - -900, - "ending exposure should be price of last trade times number of \ - shares in position") - - self.assertEqual( - ppTotal.pnl, - 100, - "drop of 1 on -1 100x contract should be 100" - ) - - check_perf_period( - pp, - gross_leverage=0.8181, - net_leverage=-0.8181, - long_exposure=0.0, - longs_count=0, - short_exposure=-900.0, - shorts_count=1) - - # Validate that the account attributes. - account = ppTotal.as_account() - check_account(account, - settled_cash=1100.0, - equity_with_loan=1100.0, - total_positions_value=0.0, - total_positions_exposure=-900.0, - regt_equity=1100.0, - available_funds=1100.0, - excess_liquidity=1100.0, - cushion=1.0, - leverage=0.8181, - net_leverage=-0.8181, - net_liquidation=1100.0) - def test_covering_short(self): """verify performance where short is bought and covered, and shares \ trade after cover""" diff --git a/zipline/data/data_portal.py b/zipline/data/data_portal.py index 739b7616..7839cefd 100644 --- a/zipline/data/data_portal.py +++ b/zipline/data/data_portal.py @@ -137,7 +137,7 @@ class DataPortal(object): self._equity_daily_reader = equity_daily_reader if self._equity_daily_reader is not None: - self._equity_history_loader = DailyHistoryLoader( + self._history_loader = DailyHistoryLoader( self.trading_calendar, self._equity_daily_reader, self._adjustment_reader @@ -157,17 +157,16 @@ class DataPortal(object): } } - if self._equity_minute_reader is not None: - self._equity_daily_aggregator = DailyHistoryAggregator( - self.trading_calendar.schedule.market_open, - self._equity_minute_reader, - self.trading_calendar - ) - self._equity_minute_history_loader = MinuteHistoryLoader( - self.trading_calendar, - self._equity_minute_reader, - self._adjustment_reader - ) + self._daily_aggregator = DailyHistoryAggregator( + self.trading_calendar.schedule.market_open, + self._equity_minute_reader, + self.trading_calendar + ) + self._minute_history_loader = MinuteHistoryLoader( + self.trading_calendar, + self._equity_minute_reader, + self._adjustment_reader + ) self._first_trading_day = first_trading_day @@ -522,9 +521,9 @@ class DataPortal(object): ) def _get_daily_data(self, asset, column, dt): + reader = self._pricing_readers[type(asset)]['daily'] if column == "last_traded": - last_traded_dt = \ - self._equity_daily_reader.get_last_traded_dt(asset, dt) + last_traded_dt = reader.get_last_traded_dt(asset, dt) if pd.isnull(last_traded_dt): return pd.NaT @@ -533,7 +532,7 @@ class DataPortal(object): elif column in OHLCV_FIELDS: # don't forward fill try: - val = self._equity_daily_reader.spot_price(asset, dt, column) + val = reader.spot_price(asset, dt, column) if val == -1: if column == "volume": return 0 @@ -547,7 +546,7 @@ class DataPortal(object): found_dt = dt while True: try: - value = self._equity_daily_reader.spot_price( + value = reader.spot_price( asset, found_dt, "close" ) if value != -1: @@ -592,88 +591,16 @@ class DataPortal(object): index=days_for_window, columns=None) - future_data = [] - eq_assets = [] - - for asset in assets: - if isinstance(asset, Future): - future_data.append(self._get_history_daily_window_future( - asset, days_for_window, end_dt, field_to_use - )) - else: - eq_assets.append(asset) - eq_data = self._get_history_daily_window_equities( - eq_assets, days_for_window, end_dt, field_to_use + data = self._get_history_daily_window_data( + assets, days_for_window, end_dt, field_to_use ) - if future_data: - # TODO: This case appears to be uncovered by testing. - data = np.concatenate(eq_data, np.array(future_data).T) - else: - data = eq_data return pd.DataFrame( data, index=days_for_window, columns=assets ) - def _get_history_daily_window_future(self, asset, days_for_window, - end_dt, column): - # Since we don't have daily bcolz files for futures (yet), use minute - # bars to calculate the daily values. - data = [] - data_groups = [] - - # get all the minutes for the days NOT including today - for day in days_for_window[:-1]: - minutes = self.sessions_in_range.minutes_for_session(day) - - values_for_day = np.zeros(len(minutes), dtype=np.float64) - - for idx, minute in enumerate(minutes): - minute_val = self._get_minute_spot_value_future( - asset, column, minute - ) - - values_for_day[idx] = minute_val - - data_groups.append(values_for_day) - - # get the minutes for today - last_day_minutes = pd.date_range( - start=self.trading_calendar.open_and_close_for_session(end_dt)[0], - end=end_dt, - freq="T" - ) - - values_for_last_day = np.zeros(len(last_day_minutes), dtype=np.float64) - - for idx, minute in enumerate(last_day_minutes): - minute_val = self._get_minute_spot_value_future( - asset, column, minute - ) - - values_for_last_day[idx] = minute_val - - data_groups.append(values_for_last_day) - - for group in data_groups: - if len(group) == 0: - continue - - if column == 'volume': - data.append(np.sum(group)) - elif column == 'open': - data.append(group[0]) - elif column == 'close': - data.append(group[-1]) - elif column == 'high': - data.append(np.amax(group)) - elif column == 'low': - data.append(np.amin(group)) - - return data - - def _get_history_daily_window_equities( + def _get_history_daily_window_data( self, assets, days_for_window, end_dt, field_to_use): ends_at_midnight = end_dt.hour == 0 and end_dt.minute == 0 @@ -697,19 +624,19 @@ class DataPortal(object): ) if field_to_use == 'open': - minute_value = self._equity_daily_aggregator.opens( + minute_value = self._daily_aggregator.opens( assets, end_dt) elif field_to_use == 'high': - minute_value = self._equity_daily_aggregator.highs( + minute_value = self._daily_aggregator.highs( assets, end_dt) elif field_to_use == 'low': - minute_value = self._equity_daily_aggregator.lows( + minute_value = self._daily_aggregator.lows( assets, end_dt) elif field_to_use == 'close': - minute_value = self._equity_daily_aggregator.closes( + minute_value = self._daily_aggregator.closes( assets, end_dt) elif field_to_use == 'volume': - minute_value = self._equity_daily_aggregator.volumes( + minute_value = self._daily_aggregator.volumes( assets, end_dt) # append the partial day. @@ -871,40 +798,14 @@ class DataPortal(object): ------- A numpy array with requested values. """ - if isinstance(assets, Future): - return self._get_minute_window_for_future([assets], field, - minutes_for_window) - else: - # TODO: Make caller accept assets. - window = self._get_minute_window_for_equities(assets, field, - minutes_for_window) - return window + return self._get_minute_window_data(assets, field, minutes_for_window) - def _get_minute_window_for_future(self, asset, field, minutes_for_window): - # THIS IS TEMPORARY. For now, we are only exposing futures within - # equity trading hours (9:30 am to 4pm, Eastern). The easiest way to - # do this is to simply do a spot lookup for each desired minute. - return_data = np.zeros(len(minutes_for_window), dtype=np.float64) - for idx, minute in enumerate(minutes_for_window): - return_data[idx] = \ - self._get_minute_spot_value_future(asset, field, minute) - - # Note: an improvement could be to find the consecutive runs within - # minutes_for_window, and use them to read the underlying ctable - # more efficiently. - - # Once futures are on 24-hour clock, then we can just grab all the - # requested minutes in one shot from the ctable. - - # no adjustments for futures, yay. - return return_data - - def _get_minute_window_for_equities( + def _get_minute_window_data( self, assets, field, minutes_for_window): - return self._equity_minute_history_loader.history(assets, - minutes_for_window, - field, - False) + return self._minute_history_loader.history(assets, + minutes_for_window, + field, + False) def _apply_all_adjustments(self, data, asset, dts, field, price_adj_factor=1.0): @@ -1018,10 +919,10 @@ class DataPortal(object): return_array[:] = np.NAN if bar_count != 0: - data = self._equity_history_loader.history(assets, - days_in_window, - field, - extra_slot) + data = self._history_loader.history(assets, + days_in_window, + field, + extra_slot) if extra_slot: return_array[:len(return_array) - 1, :] = data else: