From 52cb0fc70a2836315a1d1de2c545368248cf887b Mon Sep 17 00:00:00 2001 From: Andrew Liang Date: Mon, 11 Jul 2016 11:19:32 -0400 Subject: [PATCH 1/5] DEV: Allow net_leverage value to be forced by broker --- zipline/finance/performance/period.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zipline/finance/performance/period.py b/zipline/finance/performance/period.py index fce85263..ab63d4ae 100644 --- a/zipline/finance/performance/period.py +++ b/zipline/finance/performance/period.py @@ -550,8 +550,8 @@ class PerformancePeriod(object): getattr(self, 'day_trades_remaining', float('inf')) account.leverage = getattr(self, 'leverage', period_stats.gross_leverage) - account.net_leverage = period_stats.net_leverage - + account.net_leverage = getattr(self, 'net_leverage', + period_stats.net_leverage) account.net_liquidation = getattr(self, 'net_liquidation', period_stats.net_liquidation) return account From f146d6d8c1751ec8e7c314707837ad4c74789e2f Mon Sep 17 00:00:00 2001 From: Andrew Liang Date: Mon, 11 Jul 2016 12:09:17 -0400 Subject: [PATCH 2/5] MAINT: For capital changes, support input of delta or target value For target changes, calculate the delta using the portfolio value of the current minute --- zipline/algorithm.py | 2 +- zipline/finance/performance/tracker.py | 23 +++++++++++++++++++---- zipline/gens/tradesimulation.py | 8 +++----- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/zipline/algorithm.py b/zipline/algorithm.py index bf8c5935..663103a7 100644 --- a/zipline/algorithm.py +++ b/zipline/algorithm.py @@ -408,7 +408,7 @@ class TradingAlgorithm(object): self.benchmark_sid = kwargs.pop('benchmark_sid', None) - # A dictionary of capital change values keyed by timestamp + # A dictionary of capital changes keyed by timestamp self.capital_changes = kwargs.pop('capital_changes', {}) def init_engine(self, get_loader): diff --git a/zipline/finance/performance/tracker.py b/zipline/finance/performance/tracker.py index f4a3cb82..7e39a1da 100644 --- a/zipline/finance/performance/tracker.py +++ b/zipline/finance/performance/tracker.py @@ -238,16 +238,31 @@ class PerformanceTracker(object): return _dict - def process_capital_changes(self, capital_change, is_interday): - self.cumulative_performance.subdivide_period(capital_change) + def process_capital_changes(self, capital_change, dt, is_interday): + if capital_change['type'] == 'target': + capital_change_amount = capital_change['value'] - \ + self.cumulative_performance.as_portfolio().portfolio_value + log.info('Processing capital change to target %s at %s. Capital ' + 'change delta is %s' % (capital_change['value'], dt, + capital_change_amount)) + elif capital_change['type'] == 'delta': + capital_change_amount = capital_change['delta'] + log.info('Processing capital change of delta %s at %s' + % (capital_change_amount, dt)) + else: + log.error("Capital change %s does not indicate a valid type " + "('target' or 'delta')" % capital_change) + return + + self.cumulative_performance.subdivide_period(capital_change_amount) if is_interday: # Change comes between days self.todays_performance.adjust_period_starting_capital( - capital_change) + capital_change_amount) else: # Change comes in the middle of day - self.todays_performance.subdivide_period(capital_change) + self.todays_performance.subdivide_period(capital_change_amount) def process_transaction(self, transaction): self.txn_count += 1 diff --git a/zipline/gens/tradesimulation.py b/zipline/gens/tradesimulation.py index 101fdf34..8c0736b0 100644 --- a/zipline/gens/tradesimulation.py +++ b/zipline/gens/tradesimulation.py @@ -152,9 +152,8 @@ class AlgorithmSimulator(object): if midnight_dt in algo.capital_changes: # process any capital changes that came overnight change = algo.capital_changes[midnight_dt] - log.info('Processing capital change of %s at %s' % - (change, midnight_dt)) - perf_tracker.process_capital_changes(change, is_interday=True) + perf_tracker.process_capital_changes(change, dt, + is_interday=True) # Get the positions before updating the date so that prices are # fetched for trading close instead of midnight @@ -221,10 +220,9 @@ class AlgorithmSimulator(object): # process any capital changes that came between the last # and current minutes change = algo.capital_changes[dt] - log.info('Processing capital change of %s at %s' % - (change, dt)) algo.perf_tracker.process_capital_changes( change, + dt, is_interday=False ) else: From a9d698018ac40e74b1445e10e432c3420acde83b Mon Sep 17 00:00:00 2001 From: Andrew Liang Date: Tue, 12 Jul 2016 16:10:31 -0400 Subject: [PATCH 3/5] MAINT: Refactor checking, calculation and processing of capital changes AlgorithmSimulator will no longer check for capital changes. Instead, TradingAlgorithm find and calculate the changes, and PerformanceTracker will apply the changes --- zipline/algorithm.py | 70 +++++++++++++++++++++++++- zipline/finance/performance/period.py | 2 - zipline/finance/performance/tracker.py | 17 +------ zipline/gens/tradesimulation.py | 39 ++++---------- 4 files changed, 80 insertions(+), 48 deletions(-) diff --git a/zipline/algorithm.py b/zipline/algorithm.py index 663103a7..4c759678 100644 --- a/zipline/algorithm.py +++ b/zipline/algorithm.py @@ -408,9 +408,13 @@ class TradingAlgorithm(object): self.benchmark_sid = kwargs.pop('benchmark_sid', None) - # A dictionary of capital changes keyed by timestamp + # A dictionary of capital changes, keyed by timestamp, indicating the + # target/delta of the capital changes, along with values self.capital_changes = kwargs.pop('capital_changes', {}) + # A dictionary of the actual capital change deltas, keyed by timestamp + self.capital_change_deltas = {} + def init_engine(self, get_loader): """ Construct and store a PipelineEngine from loader. @@ -785,6 +789,70 @@ class TradingAlgorithm(object): return daily_stats + def calculate_capital_changes(self, dt, emission_rate, is_interday): + """ + If there is a capital change for a given dt, this means the the change + occurs before `handle_data` on the given dt. In the case of the + change being a target value, the change will be computed on the + portfolio value according to prices at the given dt + """ + try: + capital_change = self.capital_changes[dt] + except KeyError: + return + + if emission_rate == 'daily': + # If we are running daily emission, prices won't + # necessarily be synced at the end of every minute, and we + # need the up-to-date prices for capital change + # calculations. We want to sync the prices as of the + # last market minute, and this is okay from a data portal + # perspective as we have technically not "advanced" to the + # current dt yet. + self.perf_tracker.position_tracker.sync_last_sale_prices( + self.trading_calendar.previous_minute( + dt + ), + False, + self.data_portal + ) + + # Calculate performance before we sync prices price for the current dt + self.perf_tracker.cumulative_performance.calculate_performance() + self.perf_tracker.todays_performance.calculate_performance() + + if capital_change['type'] == 'target': + # Get an updated portfolio value as of this dt, but do it in a way + # so that the performance is not recalculated. This is done so + # that `process_capital_change` can find the performance values + # for the end of the subperiod, which is the previous dt + self.perf_tracker.position_tracker.sync_last_sale_prices( + dt, + self._in_before_trading_start, + self.data_portal + ) + portfolio_value = \ + self.perf_tracker.position_tracker.stats().net_value + \ + self.perf_tracker.cumulative_performance.ending_cash + + capital_change_amount = capital_change['value'] - portfolio_value + + log.info('Processing capital change to target %s at %s. Capital ' + 'change delta is %s' % (capital_change['value'], dt, + capital_change_amount)) + elif capital_change['type'] == 'delta': + capital_change_amount = capital_change['value'] + log.info('Processing capital change of delta %s at %s' + % (capital_change_amount, dt)) + else: + log.error("Capital change %s does not indicate a valid type " + "('target' or 'delta')" % capital_change) + return + + self.capital_change_deltas.update({dt: capital_change_amount}) + self.perf_tracker.process_capital_change(capital_change_amount, + is_interday) + @api_method def get_environment(self, field='platform'): """Query the execution environment. diff --git a/zipline/finance/performance/period.py b/zipline/finance/performance/period.py index ab63d4ae..2aeab406 100644 --- a/zipline/finance/performance/period.py +++ b/zipline/finance/performance/period.py @@ -242,8 +242,6 @@ class PerformancePeriod(object): del self._payout_last_sale_prices[asset] def subdivide_period(self, capital_change): - self.calculate_performance() - # Apply the capital change to the ending cash self.ending_cash += capital_change diff --git a/zipline/finance/performance/tracker.py b/zipline/finance/performance/tracker.py index 7e39a1da..017bfea5 100644 --- a/zipline/finance/performance/tracker.py +++ b/zipline/finance/performance/tracker.py @@ -238,22 +238,7 @@ class PerformanceTracker(object): return _dict - def process_capital_changes(self, capital_change, dt, is_interday): - if capital_change['type'] == 'target': - capital_change_amount = capital_change['value'] - \ - self.cumulative_performance.as_portfolio().portfolio_value - log.info('Processing capital change to target %s at %s. Capital ' - 'change delta is %s' % (capital_change['value'], dt, - capital_change_amount)) - elif capital_change['type'] == 'delta': - capital_change_amount = capital_change['delta'] - log.info('Processing capital change of delta %s at %s' - % (capital_change_amount, dt)) - else: - log.error("Capital change %s does not indicate a valid type " - "('target' or 'delta')" % capital_change) - return - + def process_capital_change(self, capital_change_amount, is_interday): self.cumulative_performance.subdivide_period(capital_change_amount) if is_interday: diff --git a/zipline/gens/tradesimulation.py b/zipline/gens/tradesimulation.py index 8c0736b0..e44e1196 100644 --- a/zipline/gens/tradesimulation.py +++ b/zipline/gens/tradesimulation.py @@ -95,13 +95,13 @@ class AlgorithmSimulator(object): Main generator work loop. """ algo = self.algo + emission_rate = algo.perf_tracker.emission_rate def every_bar(dt_to_use, current_data=self.current_data, handle_data=algo.event_manager.handle_data): # called every tick (minute or day). - if dt_to_use in algo.capital_changes: - process_minute_capital_changes(dt_to_use) + calculate_minute_capital_changes(dt_to_use) self.simulation_dt = dt_to_use algo.on_dt_changed(dt_to_use) @@ -149,11 +149,9 @@ class AlgorithmSimulator(object): perf_tracker = algo.perf_tracker - if midnight_dt in algo.capital_changes: - # process any capital changes that came overnight - change = algo.capital_changes[midnight_dt] - perf_tracker.process_capital_changes(change, dt, - is_interday=True) + # process any capital changes that came overnight + algo.calculate_capital_changes( + midnight_dt, emission_rate=emission_rate, is_interday=True) # Get the positions before updating the date so that prices are # fetched for trading close instead of midnight @@ -203,33 +201,16 @@ class AlgorithmSimulator(object): def execute_order_cancellation_policy(): algo.blotter.execute_cancel_policy(DAY_END) - def process_minute_capital_changes(dt): - # If we are running daily emission, prices won't - # necessarily be synced at the end of every minute, and we - # need the up-to-date prices for capital change - # calculations. We want to sync the prices as of the - # last market minute, and this is okay from a data portal - # perspective as we have technically not "advanced" to the - # current dt yet. - algo.perf_tracker.position_tracker.sync_last_sale_prices( - self.algo.trading_calendar.previous_minute(dt), - False, - self.data_portal - ) - + def calculate_minute_capital_changes(dt): # process any capital changes that came between the last # and current minutes - change = algo.capital_changes[dt] - algo.perf_tracker.process_capital_changes( - change, - dt, - is_interday=False - ) + algo.calculate_capital_changes( + dt, emission_rate=emission_rate, is_interday=False) else: def execute_order_cancellation_policy(): pass - def process_minute_capital_changes(dt): + def calculate_minute_capital_changes(dt): pass for dt, action in self.clock: @@ -239,7 +220,7 @@ class AlgorithmSimulator(object): once_a_day(dt) elif action == DAY_END: # End of the day. - if algo.perf_tracker.emission_rate == 'daily': + if emission_rate == 'daily': handle_benchmark(normalize_date(dt)) execute_order_cancellation_policy() From fdf8cdcd68e0767794f75064f0e6676308d1d6a8 Mon Sep 17 00:00:00 2001 From: Andrew Liang Date: Fri, 15 Jul 2016 18:03:19 -0400 Subject: [PATCH 4/5] BUG: Account object missing initial `total_positions_exposure` attr --- zipline/protocol.py | 1 + 1 file changed, 1 insertion(+) diff --git a/zipline/protocol.py b/zipline/protocol.py index 54fe118f..eb704b58 100644 --- a/zipline/protocol.py +++ b/zipline/protocol.py @@ -126,6 +126,7 @@ class Account(object): self.buying_power = float('inf') self.equity_with_loan = 0.0 self.total_positions_value = 0.0 + self.total_positions_exposure = 0.0 self.regt_equity = 0.0 self.regt_margin = float('inf') self.initial_margin_requirement = 0.0 From 0955515c46058c1a9f4d7680a4dcc70d2c38241c Mon Sep 17 00:00:00 2001 From: Andrew Liang Date: Wed, 13 Jul 2016 14:01:50 -0400 Subject: [PATCH 5/5] TEST: Test capital changes using target values --- tests/test_algorithm.py | 91 ++++++++++++++++++++++++++++++----------- 1 file changed, 66 insertions(+), 25 deletions(-) diff --git a/tests/test_algorithm.py b/tests/test_algorithm.py index 2ce0cb2d..1161e4ff 100644 --- a/tests/test_algorithm.py +++ b/tests/test_algorithm.py @@ -2023,14 +2023,18 @@ class TestCapitalChanges(WithLogger, index=pd.DatetimeIndex(days), ) - def test_capital_changes_daily_mode(self): + @parameterized.expand([ + ('target', 153000.0), ('delta', 50000.0) + ]) + def test_capital_changes_daily_mode(self, change_type, value): sim_params = factory.create_simulation_parameters( start=pd.Timestamp('2006-01-03', tz='UTC'), end=pd.Timestamp('2006-01-09', tz='UTC') ) capital_changes = { - pd.Timestamp('2006-01-06', tz='UTC'): 50000 + pd.Timestamp('2006-01-06', tz='UTC'): + {'type': change_type, 'value': value} } algocode = """ @@ -2157,8 +2161,22 @@ def order_stuff(context, data): expected_cumulative[stat] ) - @parameterized.expand([('interday',), ('intraday',)]) - def test_capital_changes_minute_mode_daily_emission(self, change): + self.assertEqual( + algo.capital_change_deltas, + {pd.Timestamp('2006-01-06', tz='UTC'): 50000.0} + ) + + @parameterized.expand([ + ('interday_target', [('2006-01-04', 2388.0)]), + ('interday_delta', [('2006-01-04', 1000.0)]), + ('intraday_target', [('2006-01-04 17:00', 2186.0), + ('2006-01-04 18:00', 2806.0)]), + ('intraday_delta', [('2006-01-04 17:00', 500.0), + ('2006-01-04 18:00', 500.0)]), + ]) + def test_capital_changes_minute_mode_daily_emission(self, change, values): + change_loc, change_type = change.split('_') + sim_params = factory.create_simulation_parameters( start=pd.Timestamp('2006-01-03', tz='UTC'), end=pd.Timestamp('2006-01-05', tz='UTC'), @@ -2166,13 +2184,8 @@ def order_stuff(context, data): capital_base=1000.0 ) - if change == 'intraday': - capital_changes = { - pd.Timestamp('2006-01-04 17:00', tz='UTC'): 500.0, - pd.Timestamp('2006-01-04 18:00', tz='UTC'): 500.0, - } - else: - capital_changes = {pd.Timestamp('2006-01-04', tz='UTC'): 1000.0} + capital_changes = {pd.Timestamp(val[0], tz='UTC'): { + 'type': change_type, 'value': val[1]} for val in values} algocode = """ from zipline.api import set_slippage, set_commission, slippage, commission, \ @@ -2214,7 +2227,7 @@ def order_stuff(context, data): 0.0, 1000.0, 0.0 ]) - if change == 'intraday': + if change_loc == 'intraday': # Fills at 491, +500 capital change comes at 638 (17:00) and # 698 (18:00), ends day at 879 day2_return = (1388.0 + 149.0 + 147.0)/1388.0 * \ @@ -2251,7 +2264,7 @@ def order_stuff(context, data): expected_daily['ending_cash'] - \ expected_daily['capital_used'] - if change == 'intraday': + if change_loc == 'intraday': # Capital changes come after day start expected_daily['starting_cash'] -= expected_capital_changes @@ -2296,8 +2309,29 @@ def order_stuff(context, data): expected_cumulative[stat] ) - @parameterized.expand([('interday',), ('intraday',)]) - def test_capital_changes_minute_mode_minute_emission(self, change): + if change_loc == 'interday': + self.assertEqual( + algo.capital_change_deltas, + {pd.Timestamp('2006-01-04', tz='UTC'): 1000.0} + ) + else: + self.assertEqual( + algo.capital_change_deltas, + {pd.Timestamp('2006-01-04 17:00', tz='UTC'): 500.0, + pd.Timestamp('2006-01-04 18:00', tz='UTC'): 500.0} + ) + + @parameterized.expand([ + ('interday_target', [('2006-01-04', 2388.0)]), + ('interday_delta', [('2006-01-04', 1000.0)]), + ('intraday_target', [('2006-01-04 17:00', 2186.0), + ('2006-01-04 18:00', 2806.0)]), + ('intraday_delta', [('2006-01-04 17:00', 500.0), + ('2006-01-04 18:00', 500.0)]), + ]) + def test_capital_changes_minute_mode_minute_emission(self, change, values): + change_loc, change_type = change.split('_') + sim_params = factory.create_simulation_parameters( start=pd.Timestamp('2006-01-03', tz='UTC'), end=pd.Timestamp('2006-01-05', tz='UTC'), @@ -2306,13 +2340,8 @@ def order_stuff(context, data): capital_base=1000.0 ) - if change == 'intraday': - capital_changes = { - pd.Timestamp('2006-01-04 17:00', tz='UTC'): 500.0, - pd.Timestamp('2006-01-04 18:00', tz='UTC'): 500.0, - } - else: - capital_changes = {pd.Timestamp('2006-01-04', tz='UTC'): 1000.0} + capital_changes = {pd.Timestamp(val[0], tz='UTC'): { + 'type': change_type, 'value': val[1]} for val in values} algocode = """ from zipline.api import set_slippage, set_commission, slippage, commission, \ @@ -2353,7 +2382,7 @@ def order_stuff(context, data): expected_minute = {} capital_changes_after_start = np.array([0.0] * 1170) - if change == 'intraday': + if change_loc == 'intraday': capital_changes_after_start[539:599] = 500.0 capital_changes_after_start[599:780] = 1000.0 @@ -2373,7 +2402,7 @@ def order_stuff(context, data): )) # +1000 capital changes comes before the day start if interday - day2adj = 0.0 if change == 'intraday' else 1000.0 + day2adj = 0.0 if change_loc == 'intraday' else 1000.0 expected_minute['starting_cash'] = np.concatenate(( [1000.0] * 390, @@ -2412,7 +2441,7 @@ def order_stuff(context, data): # the pnl, starting_value and starting_cash. If the change is intraday, # the returns after the change have to be calculated from two # subperiods - if change == 'intraday': + if change_loc == 'intraday': # The last packet (at 1/04 16:59) before the first capital change prev_subperiod_return = expected_minute['returns'][538] @@ -2510,6 +2539,18 @@ def order_stuff(context, data): expected_cumulative[stat] ) + if change_loc == 'interday': + self.assertEqual( + algo.capital_change_deltas, + {pd.Timestamp('2006-01-04', tz='UTC'): 1000.0} + ) + else: + self.assertEqual( + algo.capital_change_deltas, + {pd.Timestamp('2006-01-04 17:00', tz='UTC'): 500.0, + pd.Timestamp('2006-01-04 18:00', tz='UTC'): 500.0} + ) + class TestGetDatetime(WithLogger, WithSimParams,