From 40f42b43f5f36ccb39c6be9d511131fdb8d69cd7 Mon Sep 17 00:00:00 2001 From: Andrew Liang Date: Wed, 11 May 2016 11:28:07 -0400 Subject: [PATCH] DEV: Adjust performance calculations for capital changes Refactor PerformancePeriod so that it creates a sub-period every time a capital change happens within the period --- tests/test_algorithm.py | 543 +++++++++++++++++++++++++ tests/test_perf_tracking.py | 119 +++++- zipline/algorithm.py | 3 + zipline/finance/performance/period.py | 137 +++++-- zipline/finance/performance/tracker.py | 11 + zipline/gens/tradesimulation.py | 39 +- 6 files changed, 818 insertions(+), 34 deletions(-) diff --git a/tests/test_algorithm.py b/tests/test_algorithm.py index 07b58831..51847b2a 100644 --- a/tests/test_algorithm.py +++ b/tests/test_algorithm.py @@ -17,6 +17,7 @@ import datetime from datetime import timedelta from textwrap import dedent from unittest import TestCase, skip +from copy import deepcopy import logbook import toolz @@ -1854,6 +1855,548 @@ def handle_data(context, data): algo.run(self.data_portal) +class TestCapitalChanges(WithLogger, + WithDataPortal, + WithSimParams, + ZiplineTestCase): + + sids = 0, 1 + + @classmethod + def make_equity_info(cls): + data = make_simple_equity_info( + cls.sids, + pd.Timestamp('2006-01-03', tz='UTC'), + pd.Timestamp('2006-01-09', tz='UTC'), + ) + return data + + @classmethod + def make_minute_bar_data(cls): + minutes = cls.env.minutes_for_days_in_range( + pd.Timestamp('2006-01-03', tz='UTC'), + pd.Timestamp('2006-01-09', tz='UTC') + ) + return trades_by_sid_to_dfs( + { + 1: factory.create_trade_history( + 1, + np.arange(100.0, 100.0 + len(minutes), 1), + [10000] * len(minutes), + timedelta(minutes=1), + cls.sim_params, + cls.env), + }, + index=pd.DatetimeIndex(minutes), + ) + + @classmethod + def make_daily_bar_data(cls): + days = cls.env.days_in_range( + pd.Timestamp('2006-01-03', tz='UTC'), + pd.Timestamp('2006-01-09', tz='UTC') + ) + return trades_by_sid_to_dfs( + { + 0: factory.create_trade_history( + 0, + np.arange(10.0, 10.0 + len(days), 1.0), + [10000] * len(days), + timedelta(days=1), + cls.sim_params, + cls.env), + }, + index=pd.DatetimeIndex(days), + ) + + def test_capital_changes_daily_mode(self): + sim_params = factory.create_simulation_parameters( + start=pd.Timestamp('2006-01-03', tz='UTC'), + end=pd.Timestamp('2006-01-09', tz='UTC') + ) + + capital_changes = { + pd.Timestamp('2006-01-06', tz='UTC'): 50000 + } + + algocode = """ +from zipline.api import set_slippage, set_commission, slippage, commission, \ + schedule_function, time_rules, order, sid + +def initialize(context): + set_slippage(slippage.FixedSlippage(spread=0)) + set_commission(commission.PerShare(0, 0)) + schedule_function(order_stuff, time_rule=time_rules.market_open()) + +def order_stuff(context, data): + order(sid(0), 1000) +""" + + algo = TradingAlgorithm( + script=algocode, + sim_params=sim_params, + env=self.env, + data_portal=self.data_portal, + capital_changes=capital_changes + ) + + gen = algo.get_generator() + results = list(gen) + + cumulative_perf = \ + [r['cumulative_perf'] for r in results if 'cumulative_perf' in r] + daily_perf = [r['daily_perf'] for r in results if 'daily_perf' in r] + + # 1/03: price = 10, place orders + # 1/04: orders execute at price = 11, place orders + # 1/05: orders execute at price = 12, place orders + # 1/06: +50000 capital change, + # orders execute at price = 13, place orders + # 1/09: orders execute at price = 14, place orders + + expected_daily = {} + + expected_capital_changes = np.array([ + 0.0, 0.0, 0.0, 50000.0, 0.0 + ]) + + # Day 1, no transaction. Day 2, we transact, but the price of our stock + # does not change. Day 3, we start getting returns + expected_daily['returns'] = np.array([ + 0.0, + 0.0, + # 1000 shares * gain of 1 + (100000.0 + 1000.0)/100000.0 - 1.0, + # 2000 shares * gain of 1, capital change of +5000 + (151000.0 + 2000.0)/151000.0 - 1.0, + # 3000 shares * gain of 1 + (153000.0 + 3000.0)/153000.0 - 1.0, + ]) + + expected_daily['pnl'] = np.array([ + 0.0, + 0.0, + 1000.00, # 1000 shares * gain of 1 + 2000.00, # 2000 shares * gain of 1 + 3000.00, # 3000 shares * gain of 1 + ]) + + expected_daily['capital_used'] = np.array([ + 0.0, + -11000.0, # 1000 shares at price = 11 + -12000.0, # 1000 shares at price = 12 + -13000.0, # 1000 shares at price = 13 + -14000.0, # 1000 shares at price = 14 + ]) + + expected_daily['ending_cash'] = \ + np.array([100000.0] * 5) + \ + np.cumsum(expected_capital_changes) + \ + np.cumsum(expected_daily['capital_used']) + + expected_daily['starting_cash'] = \ + expected_daily['ending_cash'] - \ + expected_daily['capital_used'] + + expected_daily['starting_value'] = [ + 0.0, + 0.0, + 11000.0, # 1000 shares at price = 11 + 24000.0, # 2000 shares at price = 12 + 39000.0, # 3000 shares at price = 13 + ] + + expected_daily['ending_value'] = \ + expected_daily['starting_value'] + \ + expected_daily['pnl'] - \ + expected_daily['capital_used'] + + expected_daily['portfolio_value'] = \ + expected_daily['ending_value'] + \ + expected_daily['ending_cash'] + + stats = [ + 'returns', 'pnl', 'capital_used', 'starting_cash', 'ending_cash', + 'starting_value', 'ending_value', 'portfolio_value' + ] + + expected_cumulative = { + 'returns': np.cumprod(expected_daily['returns'] + 1) - 1, + 'pnl': np.cumsum(expected_daily['pnl']), + 'capital_used': np.cumsum(expected_daily['capital_used']), + 'starting_cash': + np.repeat(expected_daily['starting_cash'][0:1], 5), + 'ending_cash': expected_daily['ending_cash'], + 'starting_value': + np.repeat(expected_daily['starting_value'][0:1], 5), + 'ending_value': expected_daily['ending_value'], + 'portfolio_value': expected_daily['portfolio_value'], + } + + for stat in stats: + np.testing.assert_array_almost_equal( + np.array([perf[stat] for perf in daily_perf]), + expected_daily[stat] + ) + np.testing.assert_array_almost_equal( + np.array([perf[stat] for perf in cumulative_perf]), + expected_cumulative[stat] + ) + + @parameterized.expand([('interday',), ('intraday',)]) + def test_capital_changes_minute_mode_daily_emission(self, change): + sim_params = factory.create_simulation_parameters( + start=pd.Timestamp('2006-01-03', tz='UTC'), + end=pd.Timestamp('2006-01-05', tz='UTC'), + data_frequency='minute', + capital_base=1000.0 + ) + + if change == 'intraday': + capital_changes = { + pd.Timestamp('2006-01-04 17:00', tz='UTC'): 500.0, + pd.Timestamp('2006-01-04 18:00', tz='UTC'): 500.0, + } + else: + capital_changes = {pd.Timestamp('2006-01-04', tz='UTC'): 1000.0} + + algocode = """ +from zipline.api import set_slippage, set_commission, slippage, commission, \ + schedule_function, time_rules, order, sid + +def initialize(context): + set_slippage(slippage.FixedSlippage(spread=0)) + set_commission(commission.PerShare(0, 0)) + schedule_function(order_stuff, time_rule=time_rules.market_open()) + +def order_stuff(context, data): + order(sid(1), 1) +""" + + algo = TradingAlgorithm( + script=algocode, + sim_params=sim_params, + env=self.env, + data_portal=self.data_portal, + capital_changes=capital_changes + ) + + gen = algo.get_generator() + results = list(gen) + + cumulative_perf = \ + [r['cumulative_perf'] for r in results if 'cumulative_perf' in r] + daily_perf = [r['daily_perf'] for r in results if 'daily_perf' in r] + + # 1/03: place orders at price = 100, execute at 101 + # 1/04: place orders at price = 490, execute at 491, + # +500 capital change at 17:00 and 18:00 (intraday) + # or +1000 at 00:00 (interday), + # 1/05: place orders at price = 880, execute at 881 + + expected_daily = {} + + expected_capital_changes = np.array([ + 0.0, 1000.0, 0.0 + ]) + + if change == 'intraday': + # Fills at 491, +500 capital change comes at 638 (17:00) and + # 698 (18:00), ends day at 879 + day2_return = (1388.0 + 149.0 + 147.0)/1388.0 * \ + (2184.0 + 60.0 + 60.0)/2184.0 * \ + (2804.0 + 181.0 + 181.0)/2804.0 - 1.0 + else: + # Fills at 491, ends day at 879, capital change +1000 + day2_return = (2388.0 + 390.0 + 388.0)/2388.0 - 1 + + expected_daily['returns'] = np.array([ + # Fills at 101, ends day at 489 + (1000.0 + 388.0)/1000.0 - 1.0, + day2_return, + # Fills at 881, ends day at 1269 + (3166.0 + 390.0 + 390.0 + 388.0)/3166.0 - 1.0, + ]) + + expected_daily['pnl'] = np.array([ + 388.0, + 390.0 + 388.0, + 390.0 + 390.0 + 388.0, + ]) + + expected_daily['capital_used'] = np.array([ + -101.0, -491.0, -881.0 + ]) + + expected_daily['ending_cash'] = \ + np.array([1000.0] * 3) + \ + np.cumsum(expected_capital_changes) + \ + np.cumsum(expected_daily['capital_used']) + + expected_daily['starting_cash'] = \ + expected_daily['ending_cash'] - \ + expected_daily['capital_used'] + + if change == 'intraday': + # Capital changes come after day start + expected_daily['starting_cash'] -= expected_capital_changes + + expected_daily['starting_value'] = np.array([ + 0.0, 489.0, 879.0 * 2 + ]) + + expected_daily['ending_value'] = \ + expected_daily['starting_value'] + \ + expected_daily['pnl'] - \ + expected_daily['capital_used'] + + expected_daily['portfolio_value'] = \ + expected_daily['ending_value'] + \ + expected_daily['ending_cash'] + + stats = [ + 'returns', 'pnl', 'capital_used', 'starting_cash', 'ending_cash', + 'starting_value', 'ending_value', 'portfolio_value' + ] + + expected_cumulative = { + 'returns': np.cumprod(expected_daily['returns'] + 1) - 1, + 'pnl': np.cumsum(expected_daily['pnl']), + 'capital_used': np.cumsum(expected_daily['capital_used']), + 'starting_cash': + np.repeat(expected_daily['starting_cash'][0:1], 3), + 'ending_cash': expected_daily['ending_cash'], + 'starting_value': + np.repeat(expected_daily['starting_value'][0:1], 3), + 'ending_value': expected_daily['ending_value'], + 'portfolio_value': expected_daily['portfolio_value'], + } + + for stat in stats: + np.testing.assert_array_almost_equal( + np.array([perf[stat] for perf in daily_perf]), + expected_daily[stat] + ) + np.testing.assert_array_almost_equal( + np.array([perf[stat] for perf in cumulative_perf]), + expected_cumulative[stat] + ) + + @parameterized.expand([('interday',), ('intraday',)]) + def test_capital_changes_minute_mode_minute_emission(self, change): + sim_params = factory.create_simulation_parameters( + start=pd.Timestamp('2006-01-03', tz='UTC'), + end=pd.Timestamp('2006-01-05', tz='UTC'), + data_frequency='minute', + emission_rate='minute', + capital_base=1000.0 + ) + + if change == 'intraday': + capital_changes = { + pd.Timestamp('2006-01-04 17:00', tz='UTC'): 500.0, + pd.Timestamp('2006-01-04 18:00', tz='UTC'): 500.0, + } + else: + capital_changes = {pd.Timestamp('2006-01-04', tz='UTC'): 1000.0} + + algocode = """ +from zipline.api import set_slippage, set_commission, slippage, commission, \ + schedule_function, time_rules, order, sid + +def initialize(context): + set_slippage(slippage.FixedSlippage(spread=0)) + set_commission(commission.PerShare(0, 0)) + schedule_function(order_stuff, time_rule=time_rules.market_open()) + +def order_stuff(context, data): + order(sid(1), 1) +""" + + algo = TradingAlgorithm( + script=algocode, + sim_params=sim_params, + env=self.env, + data_portal=self.data_portal, + capital_changes=capital_changes + ) + + gen = algo.get_generator() + results = list(gen) + + cumulative_perf = \ + [r['cumulative_perf'] for r in results if 'cumulative_perf' in r] + minute_perf = [r['minute_perf'] for r in results if 'minute_perf' in r] + daily_perf = [r['daily_perf'] for r in results if 'daily_perf' in r] + + # 1/03: place orders at price = 100, execute at 101 + # 1/04: place orders at price = 490, execute at 491, + # +500 capital change at 17:00 and 18:00 (intraday) + # or +1000 at 00:00 (interday), + # 1/05: place orders at price = 880, execute at 881 + + # Minute perfs are cumulative for the day + expected_minute = {} + + capital_changes_after_start = np.array([0.0] * 1170) + if change == 'intraday': + capital_changes_after_start[539:599] = 500.0 + capital_changes_after_start[599:780] = 1000.0 + + expected_minute['pnl'] = np.array([0.0] * 1170) + expected_minute['pnl'][:2] = 0.0 + expected_minute['pnl'][2:392] = 1.0 + expected_minute['pnl'][392:782] = 2.0 + expected_minute['pnl'][782:] = 3.0 + for start, end in ((0, 390), (390, 780), (780, 1170)): + expected_minute['pnl'][start:end] = \ + np.cumsum(expected_minute['pnl'][start:end]) + + expected_minute['capital_used'] = np.concatenate(( + [0.0] * 1, [-101.0] * 389, + [0.0] * 1, [-491.0] * 389, + [0.0] * 1, [-881.0] * 389, + )) + + # +1000 capital changes comes before the day start if interday + day2adj = 0.0 if change == 'intraday' else 1000.0 + + expected_minute['starting_cash'] = np.concatenate(( + [1000.0] * 390, + # 101 spent on 1/03 + [1000.0 - 101.0 + day2adj] * 390, + # 101 spent on 1/03, 491 on 1/04, +1000 capital change on 1/04 + [1000.0 - 101.0 - 491.0 + 1000] * 390 + )) + + expected_minute['ending_cash'] = \ + expected_minute['starting_cash'] + \ + expected_minute['capital_used'] + \ + capital_changes_after_start + + expected_minute['starting_value'] = np.concatenate(( + [0.0] * 390, + [489.0] * 390, + [879.0 * 2] * 390 + )) + + expected_minute['ending_value'] = \ + expected_minute['starting_value'] + \ + expected_minute['pnl'] - \ + expected_minute['capital_used'] + + expected_minute['portfolio_value'] = \ + expected_minute['ending_value'] + \ + expected_minute['ending_cash'] + + expected_minute['returns'] = \ + expected_minute['pnl'] / \ + (expected_minute['starting_value'] + + expected_minute['starting_cash']) + + # If the change is interday, we can just calculate the returns from + # the pnl, starting_value and starting_cash. If the change is intraday, + # the returns after the change have to be calculated from two + # subperiods + if change == 'intraday': + # The last packet (at 1/04 16:59) before the first capital change + prev_subperiod_return = expected_minute['returns'][538] + + # From 1/04 17:00 to 17:59 + cur_subperiod_pnl = \ + expected_minute['pnl'][539:599] - expected_minute['pnl'][538] + cur_subperiod_starting_value = \ + np.array([expected_minute['ending_value'][538]] * 60) + cur_subperiod_starting_cash = \ + np.array([expected_minute['ending_cash'][538] + 500] * 60) + + cur_subperiod_returns = cur_subperiod_pnl / \ + (cur_subperiod_starting_value + cur_subperiod_starting_cash) + expected_minute['returns'][539:599] = \ + (cur_subperiod_returns + 1.0) * \ + (prev_subperiod_return + 1.0) - \ + 1.0 + + # The last packet (at 1/04 17:59) before the second capital change + prev_subperiod_return = expected_minute['returns'][598] + + # From 1/04 18:00 to 21:00 + cur_subperiod_pnl = \ + expected_minute['pnl'][599:780] - expected_minute['pnl'][598] + cur_subperiod_starting_value = \ + np.array([expected_minute['ending_value'][598]] * 181) + cur_subperiod_starting_cash = \ + np.array([expected_minute['ending_cash'][598] + 500] * 181) + + cur_subperiod_returns = cur_subperiod_pnl / \ + (cur_subperiod_starting_value + cur_subperiod_starting_cash) + expected_minute['returns'][599:780] = \ + (cur_subperiod_returns + 1.0) * \ + (prev_subperiod_return + 1.0) - \ + 1.0 + + # The last minute packet of each day + expected_daily = { + k: np.array([v[389], v[779], v[1169]]) + for k, v in iteritems(expected_minute) + } + + stats = [ + 'pnl', 'capital_used', 'starting_cash', 'ending_cash', + 'starting_value', 'ending_value', 'portfolio_value', 'returns' + ] + + expected_cumulative = deepcopy(expected_minute) + + # "Add" daily return from 1/03 to minute returns on 1/04 and 1/05 + # "Add" daily return from 1/04 to minute returns on 1/05 + expected_cumulative['returns'][390:] = \ + (expected_cumulative['returns'][390:] + 1) * \ + (expected_daily['returns'][0] + 1) - 1 + expected_cumulative['returns'][780:] = \ + (expected_cumulative['returns'][780:] + 1) * \ + (expected_daily['returns'][1] + 1) - 1 + + # Add daily pnl/capital_used from 1/03 to 1/04 and 1/05 + # Add daily pnl/capital_used from 1/04 to 1/05 + expected_cumulative['pnl'][390:] += expected_daily['pnl'][0] + expected_cumulative['pnl'][780:] += expected_daily['pnl'][1] + expected_cumulative['capital_used'][390:] += \ + expected_daily['capital_used'][0] + expected_cumulative['capital_used'][780:] += \ + expected_daily['capital_used'][1] + + # starting_cash, starting_value are same as those of the first daily + # packet + expected_cumulative['starting_cash'] = \ + np.repeat(expected_daily['starting_cash'][0:1], 1170) + expected_cumulative['starting_value'] = \ + np.repeat(expected_daily['starting_value'][0:1], 1170) + + # extra cumulative packet per day from the daily packet + for stat in stats: + for i in (390, 781, 1172): + expected_cumulative[stat] = np.insert( + expected_cumulative[stat], + i, + expected_cumulative[stat][i-1] + ) + + for stat in stats: + np.testing.assert_array_almost_equal( + np.array([perf[stat] for perf in minute_perf]), + expected_minute[stat] + ) + np.testing.assert_array_almost_equal( + np.array([perf[stat] for perf in daily_perf]), + expected_daily[stat] + ) + np.testing.assert_array_almost_equal( + np.array([perf[stat] for perf in cumulative_perf]), + expected_cumulative[stat] + ) + + class TestGetDatetime(WithLogger, WithSimParams, WithDataPortal, diff --git a/tests/test_perf_tracking.py b/tests/test_perf_tracking.py index 35e0770a..c8550c89 100644 --- a/tests/test_perf_tracking.py +++ b/tests/test_perf_tracking.py @@ -1277,7 +1277,7 @@ class TestPositionPerformance(WithInstanceTmpDir, ZiplineTestCase): pp.calculate_performance() self.assertEqual( - pp.period_cash_flow, + pp.cash_flow, -1 * txn.price * txn.amount, "capital used should be equal to the opposite of the transaction \ cost of sole txn in test" @@ -1387,7 +1387,7 @@ single short-sale transaction""" pp.calculate_performance() self.assertEqual( - pp.period_cash_flow, + pp.cash_flow, -1 * txn.price * txn.amount, "capital used should be equal to the opposite of the transaction\ cost of sole txn in test" @@ -1443,7 +1443,7 @@ single short-sale transaction""" pp.calculate_performance() self.assertEqual( - pp.period_cash_flow, + pp.cash_flow, 0, "capital used should be zero, there were no transactions in \ performance period" @@ -1506,7 +1506,7 @@ single short-sale transaction""" ppTotal.calculate_performance() self.assertEqual( - ppTotal.period_cash_flow, + ppTotal.cash_flow, -1 * txn.price * txn.amount, "capital used should be equal to the opposite of the transaction \ cost of sole txn in test" @@ -1624,7 +1624,7 @@ cost of sole txn in test" pp.calculate_performance() self.assertEqual( - pp.period_cash_flow, + pp.cash_flow, 0, "there should be no cash flow on a futures txn" ) @@ -1737,7 +1737,7 @@ single short-sale transaction""" pp.calculate_performance() self.assertEqual( - pp.period_cash_flow, + pp.cash_flow, 0, "there should be no cash flow on a futures txn" ) @@ -1797,7 +1797,7 @@ single short-sale transaction""" pp.calculate_performance() self.assertEqual( - pp.period_cash_flow, + pp.cash_flow, 0, "capital used should be zero, there were no transactions in \ performance period" @@ -1869,7 +1869,7 @@ single short-sale transaction""" ppTotal.calculate_performance() self.assertEqual( - ppTotal.period_cash_flow, + ppTotal.cash_flow, 0, "capital used should be equal to the opposite of the transaction \ cost of sole txn in test" @@ -1986,7 +1986,7 @@ trade after cover""" cover_txn_cost = cover_txn.price * cover_txn.amount self.assertEqual( - pp.period_cash_flow, + pp.cash_flow, -1 * short_txn_cost - cover_txn_cost, "capital used should be equal to the net transaction costs" ) @@ -2193,6 +2193,107 @@ shares in position" self.assertEqual(pp.positions[1].cost_basis, cost_bases[-1]) + def test_capital_change_intra_period(self): + self.create_environment_stuff() + + # post some trades in the market + trades = factory.create_trade_history( + self.asset1, + [10.0, 11.0, 12.0, 13.0], + [100, 100, 100, 100], + oneday, + self.sim_params, + env=self.env + ) + + data_portal = create_data_portal_from_trade_history( + self.env, + self.instance_tmpdir, + self.sim_params, + {1: trades}) + txn = create_txn(self.asset1, trades[0].dt, 10.0, 100) + pt = perf.PositionTracker(self.env.asset_finder, + self.sim_params.data_frequency) + pp = perf.PerformancePeriod(1000.0, self.env.asset_finder, + self.sim_params.data_frequency, + period_open=self.sim_params.period_start, + period_close=self.sim_params.period_end) + pp.position_tracker = pt + + pt.execute_transaction(txn) + pp.handle_execution(txn) + + # sync prices and calculate performance before we introduce a capital + # change + pt.sync_last_sale_prices(trades[2].dt, False, data_portal) + pp.calculate_performance() + + pp.subdivide_period(1000.0) + + pt.sync_last_sale_prices(trades[-1].dt, False, data_portal) + pp.calculate_performance() + + self.assertAlmostEqual(pp.returns, 1200/1000 * 2300/2200 - 1) + self.assertAlmostEqual(pp.pnl, 300) + self.assertAlmostEqual(pp.cash_flow, -1000) + + def test_capital_change_inter_period(self): + self.create_environment_stuff() + + # post some trades in the market + trades = factory.create_trade_history( + self.asset1, + [10.0, 11.0, 12.0, 13.0], + [100, 100, 100, 100], + oneday, + self.sim_params, + env=self.env + ) + + data_portal = create_data_portal_from_trade_history( + self.env, + self.instance_tmpdir, + self.sim_params, + {1: trades}) + txn = create_txn(self.asset1, trades[0].dt, 10.0, 100) + pt = perf.PositionTracker(self.env.asset_finder, + self.sim_params.data_frequency) + pp = perf.PerformancePeriod(1000.0, self.env.asset_finder, + self.sim_params.data_frequency, + period_open=self.sim_params.period_start, + period_close=self.sim_params.period_end) + pp.position_tracker = pt + + pt.execute_transaction(txn) + pp.handle_execution(txn) + pt.sync_last_sale_prices(trades[0].dt, False, data_portal) + pp.calculate_performance() + self.assertAlmostEqual(pp.returns, 0) + self.assertAlmostEqual(pp.pnl, 0) + self.assertAlmostEqual(pp.cash_flow, -1000) + pp.rollover() + + pt.sync_last_sale_prices(trades[1].dt, False, data_portal) + pp.calculate_performance() + self.assertAlmostEqual(pp.returns, 1100.0/1000.0 - 1) + self.assertAlmostEqual(pp.pnl, 100) + self.assertAlmostEqual(pp.cash_flow, 0) + pp.rollover() + + pp.adjust_period_starting_capital(1000) + pt.sync_last_sale_prices(trades[2].dt, False, data_portal) + pp.calculate_performance() + self.assertAlmostEqual(pp.returns, 2200.0/2100.0 - 1) + self.assertAlmostEqual(pp.pnl, 100) + self.assertAlmostEqual(pp.cash_flow, 0) + pp.rollover() + + pt.sync_last_sale_prices(trades[3].dt, False, data_portal) + pp.calculate_performance() + self.assertAlmostEqual(pp.returns, 2300.0/2200.0 - 1) + self.assertAlmostEqual(pp.pnl, 100) + self.assertAlmostEqual(pp.cash_flow, 0) + class TestPositionTracker(WithTradingEnvironment, WithInstanceTmpDir, diff --git a/zipline/algorithm.py b/zipline/algorithm.py index 2e6fc3c2..c2040abc 100644 --- a/zipline/algorithm.py +++ b/zipline/algorithm.py @@ -383,6 +383,9 @@ class TradingAlgorithm(object): self.benchmark_sid = kwargs.pop('benchmark_sid', None) + # A dictionary of capital change values keyed by timestamp + self.capital_changes = kwargs.pop('capital_changes', {}) + def init_engine(self, get_loader): """ Construct and store a PipelineEngine from loader. diff --git a/zipline/finance/performance/period.py b/zipline/finance/performance/period.py index 23a1864c..fce85263 100644 --- a/zipline/finance/performance/period.py +++ b/zipline/finance/performance/period.py @@ -97,6 +97,14 @@ PeriodStats = namedtuple('PeriodStats', 'gross_leverage', 'net_leverage']) +PrevSubPeriodStats = namedtuple( + 'PrevSubPeriodStats', ['returns', 'pnl', 'cash_flow'] +) + +CurrSubPeriodStats = namedtuple( + 'CurrSubPeriodStats', ['starting_value', 'starting_cash'] +) + def calc_net_liquidation(ending_cash, long_value, short_value): return ending_cash + long_value + short_value @@ -143,16 +151,20 @@ class PerformancePeriod(object): self.asset_finder = asset_finder self.data_frequency = data_frequency + # Start and end of the entire period self.period_open = period_open self.period_close = period_close + self.initialize(starting_cash=starting_cash, + starting_value=0.0, + starting_exposure=0.0) + self.ending_value = 0.0 self.ending_exposure = 0.0 - self.period_cash_flow = 0.0 - self.pnl = 0.0 - self.ending_cash = starting_cash + self.subperiod_divider = None + # Keyed by asset, the previous last sale price of positions with # payouts on price differences, e.g. Futures. # @@ -161,8 +173,6 @@ class PerformancePeriod(object): # start, or when the price at execution. self._payout_last_sale_prices = {} - # rollover initializes a number of self's attributes: - self.rollover() self.keep_transactions = keep_transactions self.keep_orders = keep_orders @@ -181,6 +191,23 @@ class PerformancePeriod(object): _position_tracker = None + def initialize(self, starting_cash, starting_value, starting_exposure): + + # Performance stats for the entire period, returned externally + self.pnl = 0.0 + self.returns = 0.0 + self.cash_flow = 0.0 + self.starting_value = starting_value + self.starting_exposure = starting_exposure + self.starting_cash = starting_cash + + # The cumulative capital change occurred within the period + self._total_intraperiod_capital_change = 0.0 + + self.processed_transactions = {} + self.orders_by_modified = {} + self.orders_by_id = OrderedDict() + @property def position_tracker(self): return self._position_tracker @@ -193,15 +220,17 @@ class PerformancePeriod(object): # we only calculate perf once we inject PositionTracker self.calculate_performance() + def adjust_period_starting_capital(self, capital_change): + self.ending_cash += capital_change + self.starting_cash += capital_change + def rollover(self): - self.starting_value = self.ending_value - self.starting_exposure = self.ending_exposure - self.starting_cash = self.ending_cash - self.period_cash_flow = 0.0 - self.pnl = 0.0 - self.processed_transactions = {} - self.orders_by_modified = {} - self.orders_by_id = OrderedDict() + # We are starting a new period + self.initialize(starting_cash=self.ending_cash, + starting_value=self.ending_value, + starting_exposure=self.ending_exposure) + + self.subperiod_divider = None payout_assets = self._payout_last_sale_prices.keys() @@ -212,6 +241,24 @@ class PerformancePeriod(object): else: del self._payout_last_sale_prices[asset] + def subdivide_period(self, capital_change): + self.calculate_performance() + + # Apply the capital change to the ending cash + self.ending_cash += capital_change + + # Increment the total capital change occurred within the period + self._total_intraperiod_capital_change += capital_change + + # Divide the period into subperiods + self.subperiod_divider = SubPeriodDivider( + prev_returns=self.returns, + prev_pnl=self.pnl, + prev_cash_flow=self.cash_flow, + curr_starting_value=self.ending_value, + curr_starting_cash=self.ending_cash + ) + def handle_dividends_paid(self, net_cash_payment): if net_cash_payment: self.handle_cash_payment(net_cash_payment) @@ -225,7 +272,7 @@ class PerformancePeriod(object): self.adjust_cash(-cost) def adjust_cash(self, amount): - self.period_cash_flow += amount + self.cash_flow += amount def adjust_field(self, field, value): setattr(self, field, value) @@ -252,15 +299,39 @@ class PerformancePeriod(object): payout = self._get_payout_total(pt.positions) - total_at_start = self.starting_cash + self.starting_value - self.ending_cash = self.starting_cash + self.period_cash_flow + payout + self.ending_cash = self.starting_cash + self.cash_flow + \ + self._total_intraperiod_capital_change + payout + total_at_end = self.ending_cash + self.ending_value - self.pnl = total_at_end - total_at_start - if total_at_start != 0: - self.returns = self.pnl / total_at_start + # If there is a previous subperiod, the performance is calculated + # from the previous and current subperiods. Otherwise, the performance + # is calculated based on the start and end values of the whole period + if self.subperiod_divider: + starting_cash = self.subperiod_divider.curr_subperiod.starting_cash + total_at_start = starting_cash + \ + self.subperiod_divider.curr_subperiod.starting_value + + # Performance for this subperiod + pnl = total_at_end - total_at_start + if total_at_start != 0: + returns = pnl / total_at_start + else: + returns = 0.0 + + # Performance for this whole period + self.pnl = self.subperiod_divider.prev_subperiod.pnl + pnl + self.returns = \ + (1 + self.subperiod_divider.prev_subperiod.returns) * \ + (1 + returns) - 1 else: - self.returns = 0.0 + total_at_start = self.starting_cash + self.starting_value + self.pnl = total_at_end - total_at_start + + if total_at_start != 0: + self.returns = self.pnl / total_at_start + else: + self.returns = 0.0 def record_order(self, order): if self.keep_orders: @@ -279,7 +350,7 @@ class PerformancePeriod(object): self.orders_by_id[order.id] = order def handle_execution(self, txn): - self.period_cash_flow += self._calculate_execution_cash_flow(txn) + self.cash_flow += self._calculate_execution_cash_flow(txn) asset = self.asset_finder.retrieve_asset(txn.sid) if isinstance(asset, Future): @@ -342,7 +413,7 @@ class PerformancePeriod(object): 'ending_exposure': self.ending_exposure, # this field is renamed to capital_used for backward # compatibility. - 'capital_used': self.period_cash_flow, + 'capital_used': self.cash_flow, 'starting_value': self.starting_value, 'starting_exposure': self.starting_exposure, 'starting_cash': self.starting_cash, @@ -423,7 +494,7 @@ class PerformancePeriod(object): portfolio = self._portfolio_store # maintaining the old name for the portfolio field for # backward compatibility - portfolio.capital_used = self.period_cash_flow + portfolio.capital_used = self.cash_flow portfolio.starting_cash = self.starting_cash portfolio.portfolio_value = self.ending_cash + self.ending_value portfolio.pnl = self.pnl @@ -484,3 +555,23 @@ class PerformancePeriod(object): account.net_liquidation = getattr(self, 'net_liquidation', period_stats.net_liquidation) return account + + +class SubPeriodDivider(object): + """ + A marker for subdividing the period at the latest intraperiod capital + change. prev_subperiod and curr_subperiod hold information respective to + the previous and current subperiods. + """ + + def __init__(self, prev_returns, prev_pnl, prev_cash_flow, + curr_starting_value, curr_starting_cash): + + self.prev_subperiod = PrevSubPeriodStats( + returns=prev_returns, + pnl=prev_pnl, + cash_flow=prev_cash_flow) + + self.curr_subperiod = CurrSubPeriodStats( + starting_value=curr_starting_value, + starting_cash=curr_starting_cash) diff --git a/zipline/finance/performance/tracker.py b/zipline/finance/performance/tracker.py index 3c3be3dc..1ca6148c 100644 --- a/zipline/finance/performance/tracker.py +++ b/zipline/finance/performance/tracker.py @@ -231,6 +231,17 @@ class PerformanceTracker(object): return _dict + def process_capital_changes(self, capital_change, is_interday): + self.cumulative_performance.subdivide_period(capital_change) + + if is_interday: + # Change comes between days + self.todays_performance.adjust_period_starting_capital( + capital_change) + else: + # Change comes in the middle of day + self.todays_performance.subdivide_period(capital_change) + def process_transaction(self, transaction): self.txn_count += 1 self.cumulative_performance.handle_execution(transaction) diff --git a/zipline/gens/tradesimulation.py b/zipline/gens/tradesimulation.py index 44e7e0f1..138a049f 100644 --- a/zipline/gens/tradesimulation.py +++ b/zipline/gens/tradesimulation.py @@ -102,6 +102,9 @@ class AlgorithmSimulator(object): handle_data=algo.event_manager.handle_data): # called every tick (minute or day). + if dt_to_use in algo.capital_changes: + process_minute_capital_changes(dt_to_use) + self.simulation_dt = dt_to_use algo.on_dt_changed(dt_to_use) @@ -145,6 +148,16 @@ class AlgorithmSimulator(object): def once_a_day(midnight_dt, current_data=self.current_data, data_portal=self.data_portal): + + perf_tracker = algo.perf_tracker + + if midnight_dt in algo.capital_changes: + # process any capital changes that came overnight + perf_tracker.process_capital_changes( + algo.capital_changes[midnight_dt], + is_interday=True + ) + # Get the positions before updating the date so that prices are # fetched for trading close instead of midnight positions = algo.perf_tracker.position_tracker.positions @@ -158,8 +171,6 @@ class AlgorithmSimulator(object): # before cleaning up expired assets. self._cleanup_expired_assets(midnight_dt, position_assets) - perf_tracker = algo.perf_tracker - # handle any splits that impact any positions or any open orders. assets_we_care_about = \ viewkeys(perf_tracker.position_tracker.positions) | \ @@ -190,10 +201,34 @@ class AlgorithmSimulator(object): if algo.data_frequency == 'minute': def execute_order_cancellation_policy(): algo.blotter.execute_cancel_policy(DAY_END) + + def process_minute_capital_changes(dt): + # If we are running daily emission, prices won't + # necessarily be synced at the end of every minute, and we + # need the up-to-date prices for capital change + # calculations. We want to sync the prices as of the + # last market minute, and this is okay from a data portal + # perspective as we have technically not "advanced" to the + # current dt yet. + algo.perf_tracker.position_tracker.sync_last_sale_prices( + self.env.previous_market_minute(dt), + False, + self.data_portal + ) + + # process any capital changes that came between the last + # and current minutes + algo.perf_tracker.process_capital_changes( + algo.capital_changes[dt], + is_interday=False + ) else: def execute_order_cancellation_policy(): pass + def process_minute_capital_changes(dt): + pass + for dt, action in self.clock: if action == BAR: every_bar(dt)