From ec63397d280598c7cd2b0f8dc9d642dd08bb78e6 Mon Sep 17 00:00:00 2001 From: Eddie Hebert Date: Mon, 20 Apr 2015 22:04:49 -0400 Subject: [PATCH 1/2] BUG: Stop addition of new list in orders_by_modified on every dt. Remove use of defaultdict for orders_by_modified, which was causing an empty list to be added every time to_dict was called with a specified dt. Nnoticed in the minute emission case when hunting another memory leak, every simulation minute a new Timestamp and list was created and never let go. --- zipline/finance/performance/period.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/zipline/finance/performance/period.py b/zipline/finance/performance/period.py index cfd336e4..ca244a62 100644 --- a/zipline/finance/performance/period.py +++ b/zipline/finance/performance/period.py @@ -167,7 +167,7 @@ class PerformancePeriod(object): self.period_cash_flow = 0.0 self.pnl = 0.0 self.processed_transactions = defaultdict(list) - self.orders_by_modified = defaultdict(OrderedDict) + self.orders_by_modified = {} self.orders_by_id = OrderedDict() def handle_dividends_paid(self, net_cash_payment): @@ -203,9 +203,12 @@ class PerformancePeriod(object): def record_order(self, order): if self.keep_orders: - dt_orders = self.orders_by_modified[order.dt] - if order.id in dt_orders: - del dt_orders[order.id] + try: + dt_orders = self.orders_by_modified[order.dt] + if order.id in dt_orders: + del dt_orders[order.id] + except KeyError: + self.orders_by_modified[order.dt] = dt_orders = OrderedDict() dt_orders[order.id] = order # to preserve the order of the orders by modified date # we delete and add back. (ordered dictionary is sorted by @@ -339,8 +342,11 @@ class PerformancePeriod(object): if self.keep_orders: if dt: # only include orders modified as of the given dt. - orders = [x.to_dict() - for x in itervalues(self.orders_by_modified[dt])] + try: + orders = [x.to_dict() + for x in itervalues(self.orders_by_modified[dt])] + except KeyError: + orders = [] else: orders = [x.to_dict() for x in itervalues(self.orders_by_id)] rval['orders'] = orders @@ -456,7 +462,7 @@ class PerformancePeriod(object): orders_by_id = OrderedDict() orders_by_id.update(state.pop('orders_by_id')) - orders_by_modified = defaultdict(OrderedDict) + orders_by_modified = {} orders_by_modified.update(state.pop('orders_by_modified')) self.processed_transactions = processed_transactions self.orders_by_id = orders_by_id From d343e06593833bad76f78d6fb6a8656b9c391b2e Mon Sep 17 00:00:00 2001 From: Eddie Hebert Date: Mon, 20 Apr 2015 23:09:29 -0400 Subject: [PATCH 2/2] BUG: Remove use of defaultdict leading to memory growth. defaultdict behavior was adding and never releasing a list and Timestamp every time to_dict was called. --- zipline/finance/performance/period.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/zipline/finance/performance/period.py b/zipline/finance/performance/period.py index ca244a62..ee405c90 100644 --- a/zipline/finance/performance/period.py +++ b/zipline/finance/performance/period.py @@ -74,9 +74,6 @@ from __future__ import division import logbook import numpy as np -from collections import ( - defaultdict, -) try: # optional cython based OrderedDict @@ -166,7 +163,7 @@ class PerformancePeriod(object): self.starting_cash = self.ending_cash self.period_cash_flow = 0.0 self.pnl = 0.0 - self.processed_transactions = defaultdict(list) + self.processed_transactions = {} self.orders_by_modified = {} self.orders_by_id = OrderedDict() @@ -221,7 +218,10 @@ class PerformancePeriod(object): self.period_cash_flow -= txn.price * txn.amount if self.keep_transactions: - self.processed_transactions[txn.dt].append(txn) + try: + self.processed_transactions[txn.dt].append(txn) + except KeyError: + self.processed_transactions[txn.dt] = [txn] # backwards compat. TODO: remove? @property @@ -330,8 +330,11 @@ class PerformancePeriod(object): if self.keep_transactions: if dt: # Only include transactions for given dt - transactions = [x.to_dict() - for x in self.processed_transactions[dt]] + try: + transactions = [x.to_dict() + for x in self.processed_transactions[dt]] + except KeyError: + transactions = [] else: transactions = \ [y.to_dict() @@ -456,7 +459,7 @@ class PerformancePeriod(object): if version < OLDEST_SUPPORTED_STATE: raise BaseException("PerformancePeriod saved state is too old.") - processed_transactions = defaultdict(list) + processed_transactions = {} processed_transactions.update(state.pop('processed_transactions')) orders_by_id = OrderedDict()