From 3811df78b934d911ed117bfed936dbf343ec256b Mon Sep 17 00:00:00 2001 From: fawce Date: Mon, 22 Apr 2013 15:22:28 -0400 Subject: [PATCH] BUG: Fix grouping of events streamed through blotter. To fix the grouping of events so that (dt, events) ordering is preserved, the tracking of order states needs to change in the following way. Change how order keeps track of dates: - Change order's dt field to reflect modified date. - Add a created field. Change how performance keeps track of orders by: - Map dt to transactions - Map dt to orders - Map order ids to keep track of updated orders. --- zipline/finance/performance.py | 36 ++++++++++++++++++++------------- zipline/finance/slippage.py | 4 ++-- zipline/gens/tradesimulation.py | 10 ++++----- 3 files changed, 29 insertions(+), 21 deletions(-) diff --git a/zipline/finance/performance.py b/zipline/finance/performance.py index 88566461..5beb95b2 100644 --- a/zipline/finance/performance.py +++ b/zipline/finance/performance.py @@ -135,6 +135,7 @@ import math import numpy as np import pandas as pd +from collections import OrderedDict, defaultdict import zipline.protocol as zp import zipline.finance.risk as risk @@ -523,9 +524,10 @@ class PerformancePeriod(object): self.starting_cash = starting_cash self.ending_cash = starting_cash self.keep_transactions = keep_transactions - self.processed_transactions = [] + self.processed_transactions = defaultdict(list) self.keep_orders = keep_orders - self.placed_orders = [] + self.orders_by_modified = defaultdict(list) + self.orders_by_id = OrderedDict() self.cumulative_capital_used = 0.0 self.max_capital_used = 0.0 self.max_leverage = 0.0 @@ -550,8 +552,9 @@ class PerformancePeriod(object): self.starting_cash = self.ending_cash self.period_cash_flow = 0.0 self.pnl = 0.0 - self.processed_transactions = [] - self.placed_orders = [] + self.processed_transactions = defaultdict(list) + self.orders_by_modified = defaultdict(list) + self.orders_by_id = OrderedDict() self.cumulative_capital_used = 0.0 self.max_capital_used = 0.0 self.max_leverage = 0.0 @@ -613,7 +616,13 @@ class PerformancePeriod(object): def record_order(self, order): if self.keep_orders: - self.placed_orders.append(order) + self.orders_by_modified[order.dt].append(order) + # to preserve the order of the orders by modified date + # we delete and add back. (ordered dictionary is sorted by + # first insertion date). + if order.id in self.orders_by_id: + del self.orders_by_id[order.id] + self.orders_by_id[order.id] = order def execute_transaction(self, txn): # Update Position @@ -647,7 +656,7 @@ class PerformancePeriod(object): # add transaction to the list of processed transactions if self.keep_transactions: - self.processed_transactions.append(txn) + self.processed_transactions[txn.dt].append(txn) def round_to_nearest(self, x, base=5): return int(base * round(float(x) / base)) @@ -704,21 +713,20 @@ class PerformancePeriod(object): if dt: # Only include transactions for given dt transactions = [x.to_dict() - for x in self.processed_transactions - if x.dt == dt] + for x in self.processed_transactions[dt]] else: - transactions = [x.to_dict() - for x in self.processed_transactions] + transactions = \ + [y.to_dict() + for x in self.processed_transactions.itervalues() + for y in x] rval['transactions'] = transactions if self.keep_orders: if dt: # only include orders modified as of the given dt. - orders = [x.to_dict() - for x in self.placed_orders - if x.last_modified_dt == dt] + orders = [x.to_dict() for x in self.orders_by_modified[dt]] else: - orders = [x.to_dict() for x in self.placed_orders] + orders = [x.to_dict() for x in self.orders_by_id.itervalues()] rval['orders'] = orders return rval diff --git a/zipline/finance/slippage.py b/zipline/finance/slippage.py index 4887ef27..d0ce0bdf 100644 --- a/zipline/finance/slippage.py +++ b/zipline/finance/slippage.py @@ -222,7 +222,7 @@ class FixedSlippage(object): order.id ) - # mark the last_modified date of the order to match - order.last_modified = event.dt + # mark the date of the order to match the transaction + order.dt = event.dt txns.append(txn) return txns diff --git a/zipline/gens/tradesimulation.py b/zipline/gens/tradesimulation.py index 276210df..ee52981d 100644 --- a/zipline/gens/tradesimulation.py +++ b/zipline/gens/tradesimulation.py @@ -103,12 +103,12 @@ class Blotter(object): txns = self.transact(trade_event, current_orders) for txn in txns: self.orders[txn.order_id].filled += txn.amount - # mark the last_modified date of the order to match - self.orders[txn.order_id].last_modified_dt = txn.dt + # mark the date of the order to match the txn + self.orders[txn.order_id].dt = txn.dt modified_orders = [order for order in self.open_orders[trade_event.sid] - if order.last_modified_dt == trade_event.dt] + if order.dt == trade_event.dt] for order in modified_orders: if not order.open: del self.orders[order.id] @@ -135,7 +135,7 @@ class Order(object): # get a string representation of the uuid. self.id = self.make_id() self.dt = dt - self.last_modified_dt = dt + self.created = dt self.sid = sid self.amount = amount self.filled = filled @@ -165,7 +165,7 @@ class Order(object): check_order_triggers(self, event) if (stop_reached, limit_reached) \ != (self.stop_reached, self.limit_reached): - self.last_modified_dt = event.dt + self.dt = event.dt self.stop_reached = stop_reached self.limit_reached = limit_reached