BUG: Fix grouping of events streamed through blotter.

To fix the grouping of events so that (dt, events) ordering
is preserved, the tracking of order states needs to change
in the following way.

Change how order keeps track of dates:
- Change order's dt field to reflect modified date.
- Add a created field.

Change how performance keeps track of orders by:
- Map dt to transactions
- Map dt to orders
- Map order ids to keep track of updated orders.

zipline/finance/performance.py

@@ -135,6 +135,7 @@ import math
 
 import numpy as np
 import pandas as pd
+from collections import OrderedDict, defaultdict
 
 import zipline.protocol as zp
 import zipline.finance.risk as risk
@@ -523,9 +524,10 @@ class PerformancePeriod(object):
         self.starting_cash = starting_cash
         self.ending_cash = starting_cash
         self.keep_transactions = keep_transactions
-        self.processed_transactions = []
+        self.processed_transactions = defaultdict(list)
         self.keep_orders = keep_orders
-        self.placed_orders = []
+        self.orders_by_modified = defaultdict(list)
+        self.orders_by_id = OrderedDict()
         self.cumulative_capital_used = 0.0
         self.max_capital_used = 0.0
         self.max_leverage = 0.0
@@ -550,8 +552,9 @@ class PerformancePeriod(object):
         self.starting_cash = self.ending_cash
         self.period_cash_flow = 0.0
         self.pnl = 0.0
-        self.processed_transactions = []
-        self.placed_orders = []
+        self.processed_transactions = defaultdict(list)
+        self.orders_by_modified = defaultdict(list)
+        self.orders_by_id = OrderedDict()
         self.cumulative_capital_used = 0.0
         self.max_capital_used = 0.0
         self.max_leverage = 0.0
@@ -613,7 +616,13 @@ class PerformancePeriod(object):
 
     def record_order(self, order):
         if self.keep_orders:
-            self.placed_orders.append(order)
+            self.orders_by_modified[order.dt].append(order)
+            # to preserve the order of the orders by modified date
+            # we delete and add back. (ordered dictionary is sorted by
+            # first insertion date).
+            if order.id in self.orders_by_id:
+                del self.orders_by_id[order.id]
+            self.orders_by_id[order.id] = order
 
     def execute_transaction(self, txn):
         # Update Position
@@ -647,7 +656,7 @@ class PerformancePeriod(object):
 
         # add transaction to the list of processed transactions
         if self.keep_transactions:
-            self.processed_transactions.append(txn)
+            self.processed_transactions[txn.dt].append(txn)
 
     def round_to_nearest(self, x, base=5):
         return int(base * round(float(x) / base))
@@ -704,21 +713,20 @@ class PerformancePeriod(object):
             if dt:
                 # Only include transactions for given dt
                 transactions = [x.to_dict()
-                                for x in self.processed_transactions
-                                if x.dt == dt]
+                                for x in self.processed_transactions[dt]]
             else:
-                transactions = [x.to_dict()
-                                for x in self.processed_transactions]
+                transactions = \
+                    [y.to_dict()
+                     for x in self.processed_transactions.itervalues()
+                     for y in x]
             rval['transactions'] = transactions
 
         if self.keep_orders:
             if dt:
                 # only include orders modified as of the given dt.
-                orders = [x.to_dict()
-                          for x in self.placed_orders
-                          if x.last_modified_dt == dt]
+                orders = [x.to_dict() for x in self.orders_by_modified[dt]]
             else:
-                orders = [x.to_dict() for x in self.placed_orders]
+                orders = [x.to_dict() for x in self.orders_by_id.itervalues()]
             rval['orders'] = orders
 
         return rval

zipline/finance/slippage.py

@@ -222,7 +222,7 @@ class FixedSlippage(object):
                 order.id
             )
 
-            # mark the last_modified date of the order to match
-            order.last_modified = event.dt
+            # mark the date of the order to match the transaction
+            order.dt = event.dt
             txns.append(txn)
         return txns

zipline/gens/tradesimulation.py

@@ -103,12 +103,12 @@ class Blotter(object):
         txns = self.transact(trade_event, current_orders)
         for txn in txns:
             self.orders[txn.order_id].filled += txn.amount
-            # mark the last_modified date of the order to match
-            self.orders[txn.order_id].last_modified_dt = txn.dt
+            # mark the date of the order to match the txn
+            self.orders[txn.order_id].dt = txn.dt
 
         modified_orders = [order for order
                            in self.open_orders[trade_event.sid]
-                           if order.last_modified_dt == trade_event.dt]
+                           if order.dt == trade_event.dt]
         for order in modified_orders:
             if not order.open:
                 del self.orders[order.id]
@@ -135,7 +135,7 @@ class Order(object):
         # get a string representation of the uuid.
         self.id = self.make_id()
         self.dt = dt
-        self.last_modified_dt = dt
+        self.created = dt
         self.sid = sid
         self.amount = amount
         self.filled = filled
@@ -165,7 +165,7 @@ class Order(object):
             check_order_triggers(self, event)
         if (stop_reached, limit_reached) \
                 != (self.stop_reached, self.limit_reached):
-            self.last_modified_dt = event.dt
+            self.dt = event.dt
         self.stop_reached = stop_reached
         self.limit_reached = limit_reached