From 0e3b1e76e82b42733ede1a44e2cf807a081a1756 Mon Sep 17 00:00:00 2001
From: Thomas Wiecki <thomas.wiecki@gmail.com>
Date: Thu, 20 Dec 2012 11:09:32 -0500
Subject: [PATCH 01/13] ENH: batch_transform now supports sid-filtering. DOC:
 Added docs to batch_transform.

---
 tests/test_transforms.py    |  4 ++++
 zipline/test_algorithms.py  | 15 +++++++++++++++
 zipline/transforms/utils.py | 37 +++++++++++++++++++++++++++++++++----
 3 files changed, 52 insertions(+), 4 deletions(-)

diff --git a/tests/test_transforms.py b/tests/test_transforms.py
index f3b2ffe2..a83785e0 100644
--- a/tests/test_transforms.py
+++ b/tests/test_transforms.py
@@ -342,6 +342,10 @@ class TestBatchTransform(TestCase):
             'arbitrary dataframe should contain only "test"'
         )
 
+        for data in algo.history_return_sid_filter[wl:]:
+            self.assertIn(0, data.columns)
+            self.assertNotIn(1, data.columns)
+
         # test overloaded class
         for test_history in [algo.history_return_price_class,
                              algo.history_return_price_decorator]:
diff --git a/zipline/test_algorithms.py b/zipline/test_algorithms.py
index ee3e41a2..7dbc1385 100644
--- a/zipline/test_algorithms.py
+++ b/zipline/test_algorithms.py
@@ -268,6 +268,7 @@ class BatchTransformAlgorithm(TradingAlgorithm):
         self.history_return_args = []
         self.history_return_arbitrary_fields = []
         self.history_return_nan = []
+        self.history_return_sid_filter = []
 
         self.return_price_class = ReturnPriceBatchTransform(
             refresh_period=self.refresh_period,
@@ -305,6 +306,13 @@ class BatchTransformAlgorithm(TradingAlgorithm):
             clean_nans=True
         )
 
+        self.return_sid_filter = return_price_batch_decorator(
+            refresh_period=self.refresh_period,
+            window_length=self.window_length,
+            clean_nans=True,
+            sids=[0]
+        )
+
         self.iter = 0
 
         self.set_slippage(FixedSlippage())
@@ -339,6 +347,13 @@ class BatchTransformAlgorithm(TradingAlgorithm):
 
         self.iter += 1
 
+        # Add a new sid to check that it does not get included
+        extra_sid_data = deepcopy(data)
+        extra_sid_data[1] = extra_sid_data[0]
+        self.history_return_sid_filter.append(
+            self.return_sid_filter.handle_data(extra_sid_data)
+        )
+
 
 class SetPortfolioAlgorithm(TradingAlgorithm):
     """
diff --git a/zipline/transforms/utils.py b/zipline/transforms/utils.py
index 369f8054..5f4c706a 100644
--- a/zipline/transforms/utils.py
+++ b/zipline/transforms/utils.py
@@ -343,7 +343,27 @@ class BatchTransform(EventWindow):
                  func=None,
                  refresh_period=None,
                  window_length=None,
-                 clean_nans=True):
+                 clean_nans=True,
+                 sids=None):
+        """Instantiate new batch_transform object.
+
+        :Arguments:
+            func : python function <optional>
+                If supplied will be called after each refresh_period
+                with the data panel and all args and kwargs supplied
+                to the handle_data() call.
+            refresh_period : int
+                Interval to call batch_transform function.
+            window_length : int
+                How many days the trailing window should have.
+            clean_nans : bool <default=True>
+                Whether to (forward) fill in nans.
+            sids : list <optional>
+                Which sids to include in the moving window.  If not
+                supplied sids will be extracted from incoming
+                events.
+
+        """
 
         super(BatchTransform, self).__init__(True,
                                              window_length=window_length)
@@ -355,6 +375,8 @@ class BatchTransform(EventWindow):
 
         self.clean_nans = clean_nans
 
+        self.sids = sids
+
         self.refresh_period = refresh_period
         self.window_length = window_length
         self.trading_days_since_update = 0
@@ -445,7 +467,13 @@ class BatchTransform(EventWindow):
         """
         # This Panel data structure ultimately gets passed to the
         # user-overloaded get_value() method.
-        sids = set.union(*[set(tick.data.keys()) for tick in self.ticks])
+
+        # If sids are set, use those. Otherwise extract.
+        if self.sids is not None:
+            sids = self.sids
+        else:
+            sids = set.union(*[set(tick.data.keys()) for tick in self.ticks])
+
         dts = [tick.dt for tick in self.ticks]
 
         data = pd.Panel(items=self.field_names, major_axis=dts,
@@ -454,9 +482,10 @@ class BatchTransform(EventWindow):
         # Fill data panel
         for tick in self.ticks:
             dt = tick.dt
-            for sid, fields in tick.data.iteritems():
+            for sid in sids:
+                fields = tick.data[sid]
                 for field_name in self.field_names:
-                        data[field_name][sid].ix[dt] = fields[field_name]
+                    data[field_name][sid].ix[dt] = fields[field_name]
 
         if self.clean_nans:
             # Fills in gaps of missing data during transform

From 2729936affbfaae0535d5e96e3725502e909bf4b Mon Sep 17 00:00:00 2001
From: Thomas Wiecki <thomas.wiecki@gmail.com>
Date: Thu, 20 Dec 2012 11:56:49 -0500
Subject: [PATCH 02/13] ENH: batch_transform now supports field filtering.

---
 tests/test_transforms.py    |  8 ++++++++
 zipline/test_algorithms.py  | 25 +++++++++++++++++++++++++
 zipline/transforms/utils.py | 13 +++++++++----
 3 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/tests/test_transforms.py b/tests/test_transforms.py
index a83785e0..1b2536dc 100644
--- a/tests/test_transforms.py
+++ b/tests/test_transforms.py
@@ -346,6 +346,14 @@ class TestBatchTransform(TestCase):
             self.assertIn(0, data.columns)
             self.assertNotIn(1, data.columns)
 
+        for data in algo.history_return_field_filter[wl:]:
+            self.assertIn('price', data.items)
+            self.assertNotIn('ignore', data.items)
+
+        for data in algo.history_return_field_no_filter[wl:]:
+            self.assertIn('price', data.items)
+            self.assertIn('ignore', data.items)
+
         # test overloaded class
         for test_history in [algo.history_return_price_class,
                              algo.history_return_price_decorator]:
diff --git a/zipline/test_algorithms.py b/zipline/test_algorithms.py
index 7dbc1385..735e3348 100644
--- a/zipline/test_algorithms.py
+++ b/zipline/test_algorithms.py
@@ -269,6 +269,8 @@ class BatchTransformAlgorithm(TradingAlgorithm):
         self.history_return_arbitrary_fields = []
         self.history_return_nan = []
         self.history_return_sid_filter = []
+        self.history_return_field_filter = []
+        self.history_return_field_no_filter = []
 
         self.return_price_class = ReturnPriceBatchTransform(
             refresh_period=self.refresh_period,
@@ -313,6 +315,19 @@ class BatchTransformAlgorithm(TradingAlgorithm):
             sids=[0]
         )
 
+        self.return_field_filter = return_data(
+            refresh_period=self.refresh_period,
+            window_length=self.window_length,
+            clean_nans=True,
+            fields=['price']
+        )
+
+        self.return_field_no_filter = return_data(
+            refresh_period=self.refresh_period,
+            window_length=self.window_length,
+            clean_nans=True
+        )
+
         self.iter = 0
 
         self.set_slippage(FixedSlippage())
@@ -354,6 +369,16 @@ class BatchTransformAlgorithm(TradingAlgorithm):
             self.return_sid_filter.handle_data(extra_sid_data)
         )
 
+        # Add a field to check that it does not get included
+        extra_field_data = deepcopy(data)
+        extra_field_data[0]['ignore'] = extra_sid_data[0]['price']
+        self.history_return_field_filter.append(
+            self.return_field_filter.handle_data(extra_field_data)
+        )
+        self.history_return_field_no_filter.append(
+            self.return_field_no_filter.handle_data(extra_field_data)
+        )
+
 
 class SetPortfolioAlgorithm(TradingAlgorithm):
     """
diff --git a/zipline/transforms/utils.py b/zipline/transforms/utils.py
index 5f4c706a..117fe8c8 100644
--- a/zipline/transforms/utils.py
+++ b/zipline/transforms/utils.py
@@ -344,7 +344,8 @@ class BatchTransform(EventWindow):
                  refresh_period=None,
                  window_length=None,
                  clean_nans=True,
-                 sids=None):
+                 sids=None,
+                 fields=None):
         """Instantiate new batch_transform object.
 
         :Arguments:
@@ -362,7 +363,10 @@ class BatchTransform(EventWindow):
                 Which sids to include in the moving window.  If not
                 supplied sids will be extracted from incoming
                 events.
-
+            fields : list <optional>
+                Which fields to include in the moving window
+                (e.g. 'price'). If not supplied, fields will be
+                extracted from incoming events.
         """
 
         super(BatchTransform, self).__init__(True,
@@ -388,7 +392,7 @@ class BatchTransform(EventWindow):
         self.updated = False
         self.cached = None
 
-        self.field_names = None
+        self.field_names = fields
 
     def handle_data(self, data, *args, **kwargs):
         """
@@ -432,7 +436,8 @@ class BatchTransform(EventWindow):
 
     def handle_add(self, event):
         if not self.last_dt:
-            self.field_names = self._extract_field_names(event)
+            if self.field_names is None:
+                self.field_names = self._extract_field_names(event)
             self.last_dt = event.dt
             return
 

From 5deeb38fb6a73009f19045b5b92b8bb958982695 Mon Sep 17 00:00:00 2001
From: Thomas Wiecki <thomas.wiecki@gmail.com>
Date: Thu, 20 Dec 2012 11:58:23 -0500
Subject: [PATCH 03/13] ENH: sid and field filter kwargs can also be strings or
 ints.

---
 zipline/transforms/utils.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/zipline/transforms/utils.py b/zipline/transforms/utils.py
index 117fe8c8..4e3f72d1 100644
--- a/zipline/transforms/utils.py
+++ b/zipline/transforms/utils.py
@@ -380,6 +380,12 @@ class BatchTransform(EventWindow):
         self.clean_nans = clean_nans
 
         self.sids = sids
+        if isinstance(self.sids, (str, int)):
+            self.sids = [self.sids]
+
+        self.field_names = fields
+        if isinstance(self.field_names, str):
+            self.field_names = [self.field_names]
 
         self.refresh_period = refresh_period
         self.window_length = window_length
@@ -392,8 +398,6 @@ class BatchTransform(EventWindow):
         self.updated = False
         self.cached = None
 
-        self.field_names = fields
-
     def handle_data(self, data, *args, **kwargs):
         """
         New method to handle a data frame as sent to the algorithm's

From d1dace948e3261a0a818d54a376de036459abd68 Mon Sep 17 00:00:00 2001
From: Thomas Wiecki <thomas.wiecki@gmail.com>
Date: Sun, 30 Dec 2012 17:04:01 -0500
Subject: [PATCH 04/13] ENH: Added new kwarg to batch_transform: create_panel.

---
 tests/test_transforms.py    |  5 +++++
 zipline/test_algorithms.py  | 12 +++++++++++-
 zipline/transforms/utils.py | 17 ++++++++++++++---
 3 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/tests/test_transforms.py b/tests/test_transforms.py
index 1b2536dc..1d0197a9 100644
--- a/tests/test_transforms.py
+++ b/tests/test_transforms.py
@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from collections import deque
+
 import pytz
 import numpy as np
 import pandas as pd
@@ -354,6 +356,9 @@ class TestBatchTransform(TestCase):
             self.assertIn('price', data.items)
             self.assertIn('ignore', data.items)
 
+        for data in algo.history_return_ticks[wl:]:
+            self.assertTrue(isinstance(data, deque))
+
         # test overloaded class
         for test_history in [algo.history_return_price_class,
                              algo.history_return_price_decorator]:
diff --git a/zipline/test_algorithms.py b/zipline/test_algorithms.py
index 735e3348..c82edcf8 100644
--- a/zipline/test_algorithms.py
+++ b/zipline/test_algorithms.py
@@ -72,6 +72,7 @@ The algorithm must expose methods:
 
 """
 from copy import deepcopy
+import numpy as np
 
 from zipline.algorithm import TradingAlgorithm
 from zipline.finance.slippage import FixedSlippage
@@ -271,6 +272,7 @@ class BatchTransformAlgorithm(TradingAlgorithm):
         self.history_return_sid_filter = []
         self.history_return_field_filter = []
         self.history_return_field_no_filter = []
+        self.history_return_ticks = []
 
         self.return_price_class = ReturnPriceBatchTransform(
             refresh_period=self.refresh_period,
@@ -328,6 +330,13 @@ class BatchTransformAlgorithm(TradingAlgorithm):
             clean_nans=True
         )
 
+        self.return_ticks = return_data(
+            refresh_period=self.refresh_period,
+            window_length=self.window_length,
+            clean_nans=True,
+            create_panel=False
+        )
+
         self.iter = 0
 
         self.set_slippage(FixedSlippage())
@@ -340,6 +349,8 @@ class BatchTransformAlgorithm(TradingAlgorithm):
         self.history_return_args.append(
             self.return_args_batch.handle_data(
                 data, *self.args, **self.kwargs))
+        self.history_return_ticks.append(
+            self.return_ticks.handle_data(data))
 
         new_data = deepcopy(data)
         for sid in new_data:
@@ -354,7 +365,6 @@ class BatchTransformAlgorithm(TradingAlgorithm):
                 self.return_nan.handle_data(data))
         else:
             nan_data = deepcopy(data)
-            import numpy as np
             for sid in nan_data.iterkeys():
                 nan_data[sid].price = np.nan
             self.history_return_nan.append(
diff --git a/zipline/transforms/utils.py b/zipline/transforms/utils.py
index 4e3f72d1..8182441c 100644
--- a/zipline/transforms/utils.py
+++ b/zipline/transforms/utils.py
@@ -345,7 +345,8 @@ class BatchTransform(EventWindow):
                  window_length=None,
                  clean_nans=True,
                  sids=None,
-                 fields=None):
+                 fields=None,
+                 create_panel=True):
         """Instantiate new batch_transform object.
 
         :Arguments:
@@ -367,6 +368,12 @@ class BatchTransform(EventWindow):
                 Which fields to include in the moving window
                 (e.g. 'price'). If not supplied, fields will be
                 extracted from incoming events.
+            create_panel : bool <default=True>
+                If False, will create a pandas panel every refresh
+                period and pass it to the user-defined function.
+                If True, will pass the underlying deque reference
+                directly to the function which will be significantly
+                faster.
         """
 
         super(BatchTransform, self).__init__(True,
@@ -378,6 +385,7 @@ class BatchTransform(EventWindow):
             self.compute_transform_value = self.get_value
 
         self.clean_nans = clean_nans
+        self.create_panel = create_panel
 
         self.sids = sids
         if isinstance(self.sids, (str, int)):
@@ -528,8 +536,11 @@ class BatchTransform(EventWindow):
             return None
 
         if self.updated:
-            self.cached = self.compute_transform_value(self.get_data(),
-                                                       *args, **kwargs)
+            # Either create new pandas panel or pass ticks dequeue
+            # directly
+            data = self.get_data() if self.create_panel else self.ticks
+            self.cached = self.compute_transform_value(data, *args,
+                                                       **kwargs)
 
         return self.cached
 

From 0f88e4133dfc8e42f639db7f9d31ef53fa7cefd6 Mon Sep 17 00:00:00 2001
From: Thomas Wiecki <thomas.wiecki@gmail.com>
Date: Sun, 30 Dec 2012 17:26:07 -0500
Subject: [PATCH 05/13] ENH: New batch_transform feature: compute_only_full.

---
 tests/test_transforms.py    |  3 +++
 zipline/test_algorithms.py  | 10 +++++++++-
 zipline/transforms/utils.py | 23 +++++++++++++----------
 3 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/tests/test_transforms.py b/tests/test_transforms.py
index 1d0197a9..8b775990 100644
--- a/tests/test_transforms.py
+++ b/tests/test_transforms.py
@@ -359,6 +359,9 @@ class TestBatchTransform(TestCase):
         for data in algo.history_return_ticks[wl:]:
             self.assertTrue(isinstance(data, deque))
 
+        for data in algo.history_return_not_full:
+            self.assertIsNot(data, None)
+
         # test overloaded class
         for test_history in [algo.history_return_price_class,
                              algo.history_return_price_decorator]:
diff --git a/zipline/test_algorithms.py b/zipline/test_algorithms.py
index c82edcf8..b22d2719 100644
--- a/zipline/test_algorithms.py
+++ b/zipline/test_algorithms.py
@@ -273,6 +273,7 @@ class BatchTransformAlgorithm(TradingAlgorithm):
         self.history_return_field_filter = []
         self.history_return_field_no_filter = []
         self.history_return_ticks = []
+        self.history_return_not_full = []
 
         self.return_price_class = ReturnPriceBatchTransform(
             refresh_period=self.refresh_period,
@@ -333,10 +334,15 @@ class BatchTransformAlgorithm(TradingAlgorithm):
         self.return_ticks = return_data(
             refresh_period=self.refresh_period,
             window_length=self.window_length,
-            clean_nans=True,
             create_panel=False
         )
 
+        self.return_not_full = return_data(
+            refresh_period=0,
+            window_length=self.window_length,
+            compute_only_full=False
+        )
+
         self.iter = 0
 
         self.set_slippage(FixedSlippage())
@@ -351,6 +357,8 @@ class BatchTransformAlgorithm(TradingAlgorithm):
                 data, *self.args, **self.kwargs))
         self.history_return_ticks.append(
             self.return_ticks.handle_data(data))
+        self.history_return_not_full.append(
+            self.return_not_full.handle_data(data))
 
         new_data = deepcopy(data)
         for sid in new_data:
diff --git a/zipline/transforms/utils.py b/zipline/transforms/utils.py
index 8182441c..0fd647b5 100644
--- a/zipline/transforms/utils.py
+++ b/zipline/transforms/utils.py
@@ -346,7 +346,9 @@ class BatchTransform(EventWindow):
                  clean_nans=True,
                  sids=None,
                  fields=None,
-                 create_panel=True):
+                 create_panel=True,
+                 compute_only_full=True):
+
         """Instantiate new batch_transform object.
 
         :Arguments:
@@ -374,6 +376,9 @@ class BatchTransform(EventWindow):
                 If True, will pass the underlying deque reference
                 directly to the function which will be significantly
                 faster.
+            compute_only_full : bool <default=True>
+                Only call the user-defined function once the window is
+                full. Returns None if window is not full yet.
         """
 
         super(BatchTransform, self).__init__(True,
@@ -386,6 +391,7 @@ class BatchTransform(EventWindow):
 
         self.clean_nans = clean_nans
         self.create_panel = create_panel
+        self.compute_only_full = compute_only_full
 
         self.sids = sids
         if isinstance(self.sids, (str, int)):
@@ -451,7 +457,6 @@ class BatchTransform(EventWindow):
             if self.field_names is None:
                 self.field_names = self._extract_field_names(event)
             self.last_dt = event.dt
-            return
 
         # update trading day counters
         if self.last_dt.day != event.dt.day:
@@ -459,15 +464,14 @@ class BatchTransform(EventWindow):
             self.trading_days_since_update += 1
             self.trading_days_total += 1
 
-        if (
-            self.trading_days_total >= self.window_length and
-            self.trading_days_since_update >= self.refresh_period
-        ):
+        if self.trading_days_total >= self.window_length:
+            self.full = True
+
+        if self.trading_days_since_update >= self.refresh_period:
             # Setting updated to True will cause get_transform_value()
             # to call the user-defined batch-transform with the most
             # recent datapanel
             self.updated = True
-            self.full = True
             self.trading_days_since_update = 0
         else:
             self.updated = False
@@ -517,8 +521,7 @@ class BatchTransform(EventWindow):
         return data
 
     def handle_remove(self, event):
-        # since an event is expiring, we know the window is full
-        self.full = True
+        pass
 
     def get_value(self, *args, **kwargs):
         raise NotImplementedError(
@@ -532,7 +535,7 @@ class BatchTransform(EventWindow):
         has actually been updated. Otherwise, the previously, cached
         value will be returned.
         """
-        if not self.full:
+        if self.compute_only_full and not self.full:
             return None
 
         if self.updated:

From 48b05397e21d6383e666383134f6c168699fa50d Mon Sep 17 00:00:00 2001
From: Thomas Wiecki <thomas.wiecki@gmail.com>
Date: Thu, 3 Jan 2013 16:46:58 -0500
Subject: [PATCH 06/13] MIN: Changed isinstance check to allow more types.

---
 zipline/transforms/utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/zipline/transforms/utils.py b/zipline/transforms/utils.py
index 0fd647b5..bec3e117 100644
--- a/zipline/transforms/utils.py
+++ b/zipline/transforms/utils.py
@@ -24,6 +24,7 @@ from copy import deepcopy
 from datetime import datetime
 from collections import deque
 from abc import ABCMeta, abstractmethod
+from numbers import Integral
 
 import pandas as pd
 
@@ -394,7 +395,7 @@ class BatchTransform(EventWindow):
         self.compute_only_full = compute_only_full
 
         self.sids = sids
-        if isinstance(self.sids, (str, int)):
+        if isinstance(self.sids, (basestring, Integral)):
             self.sids = [self.sids]
 
         self.field_names = fields

From bf2e8e3586e8d4e985b73c4cbb4df74879d6dbcb Mon Sep 17 00:00:00 2001
From: Thomas Wiecki <thomas.wiecki@gmail.com>
Date: Sun, 30 Dec 2012 21:47:17 -0500
Subject: [PATCH 07/13] DOC: Typo.

---
 zipline/transforms/utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/zipline/transforms/utils.py b/zipline/transforms/utils.py
index bec3e117..e343a40c 100644
--- a/zipline/transforms/utils.py
+++ b/zipline/transforms/utils.py
@@ -372,9 +372,9 @@ class BatchTransform(EventWindow):
                 (e.g. 'price'). If not supplied, fields will be
                 extracted from incoming events.
             create_panel : bool <default=True>
-                If False, will create a pandas panel every refresh
+                If True, will create a pandas panel every refresh
                 period and pass it to the user-defined function.
-                If True, will pass the underlying deque reference
+                If False, will pass the underlying deque reference
                 directly to the function which will be significantly
                 faster.
             compute_only_full : bool <default=True>

From ca9fdcfe84491883829ddcae891278e2da386b7b Mon Sep 17 00:00:00 2001
From: Eddie Hebert <ehebert@quantopian.com>
Date: Fri, 11 Jan 2013 14:50:00 -0500
Subject: [PATCH 08/13] Uses a Portfolio object instead of an ndict.

Gains some performance by using a 'regular' object instead of
an ndict.

Also, directly sets up the values that we return, instead of going in
between with __core_dict and then removing values.

In it's entirety performanc.as_portfolio is the current
highest bottleneck, working on reducing time spent in that function.
---
 zipline/finance/performance.py | 36 ++++++++++++----------------------
 zipline/protocol.py            | 10 ++++++++++
 2 files changed, 22 insertions(+), 24 deletions(-)

diff --git a/zipline/finance/performance.py b/zipline/finance/performance.py
index d8776220..8d0325de 100644
--- a/zipline/finance/performance.py
+++ b/zipline/finance/performance.py
@@ -139,6 +139,7 @@ import math
 from zipline.utils.protocol_utils import ndict
 import zipline.protocol as zp
 import zipline.finance.risk as risk
+from zipline.protocol import Portfolio
 
 log = logbook.Logger('Performance')
 
@@ -543,30 +544,17 @@ class PerformancePeriod(object):
         PerformancePeriod, and in this method we rename some
         fields for usability and remove extraneous fields.
         """
-        portfolio = self.__core_dict()
-        # rename:
-        # ending_cash -> cash
-        # period_open -> backtest_start
-        #
-        # remove:
-        # period_close, starting_value,
-        # cumulative_capital_used, max_leverage, max_capital_used
-        portfolio['cash'] = portfolio['ending_cash']
-        portfolio['start_date'] = portfolio['period_open']
-        portfolio['positions_value'] = portfolio['ending_value']
-
-        del(portfolio['ending_cash'])
-        del(portfolio['period_open'])
-        del(portfolio['period_close'])
-        del(portfolio['starting_value'])
-        del(portfolio['ending_value'])
-        del(portfolio['cumulative_capital_used'])
-        del(portfolio['max_leverage'])
-        del(portfolio['max_capital_used'])
-
-        portfolio['positions'] = self.get_positions()
-
-        return ndict(portfolio)
+        return Portfolio({
+            'capital_used': self.period_capital_used,
+            'starting_cash': self.starting_cash,
+            'portfolio_value': self.ending_cash + self.ending_value,
+            'pnl': self.pnl,
+            'returns': self.returns,
+            'cash': self.ending_cash,
+            'start_date': self.period_open,
+            'positions': self.get_positions(),
+            'positions_value': self.ending_value
+        })
 
     def get_positions(self):
 
diff --git a/zipline/protocol.py b/zipline/protocol.py
index 99d33d39..9e71d3ae 100644
--- a/zipline/protocol.py
+++ b/zipline/protocol.py
@@ -55,3 +55,13 @@ class Event(object):
 
     def __repr__(self):
         return "Event({0})".format(self.__dict__)
+
+
+class Portfolio(object):
+
+    def __init__(self, initial_values=None):
+        if initial_values:
+            self.__dict__ = initial_values
+
+    def __repr__(self):
+        return "Portfolio({0})".format(self.__dict__)

From 1ddfadf5b48876cdcc0ca170786ccdddc0b57fec Mon Sep 17 00:00:00 2001
From: Eddie Hebert <ehebert@quantopian.com>
Date: Fri, 11 Jan 2013 23:30:43 -0500
Subject: [PATCH 09/13] Recycles the portfolio container to be passed to
 handle_data.

The creation of a new portfolio ndict on each call of handle_data
was creating a very high performance overhead.

Instead, we use the same the portfolio object for each event,
and replace the values contained within.
---
 zipline/finance/performance.py | 32 ++++++++++++++++++++------------
 1 file changed, 20 insertions(+), 12 deletions(-)

diff --git a/zipline/finance/performance.py b/zipline/finance/performance.py
index 8d0325de..3543f7ed 100644
--- a/zipline/finance/performance.py
+++ b/zipline/finance/performance.py
@@ -139,7 +139,6 @@ import math
 from zipline.utils.protocol_utils import ndict
 import zipline.protocol as zp
 import zipline.finance.risk as risk
-from zipline.protocol import Portfolio
 
 log = logbook.Logger('Performance')
 
@@ -443,6 +442,11 @@ class PerformancePeriod(object):
 
         self.calculate_performance()
 
+        # An object to recycle via assigning new values
+        # when returning portfolio information.
+        # So as not to avoid creating a new object for each event
+        self._portfolio_store = zp.Portfolio()
+
     def calculate_performance(self):
         self.ending_value = self.calculate_positions_value()
 
@@ -544,17 +548,21 @@ class PerformancePeriod(object):
         PerformancePeriod, and in this method we rename some
         fields for usability and remove extraneous fields.
         """
-        return Portfolio({
-            'capital_used': self.period_capital_used,
-            'starting_cash': self.starting_cash,
-            'portfolio_value': self.ending_cash + self.ending_value,
-            'pnl': self.pnl,
-            'returns': self.returns,
-            'cash': self.ending_cash,
-            'start_date': self.period_open,
-            'positions': self.get_positions(),
-            'positions_value': self.ending_value
-        })
+        # Recycles containing objects' Portfolio object
+        # which is used for returning values.
+        # as_portfolio is called in an inner loop,
+        # so repeated object creation becomes too expensive
+        portfolio = self._portfolio_store
+        portfolio.capital_used = self.period_capital_used,
+        portfolio.starting_cash = self.starting_cash
+        portfolio.portfolio_value = self.ending_cash + self.ending_value
+        portfolio.pnl = self.pnl
+        portfolio.returns = self.returns
+        portfolio.cash = self.ending_cash
+        portfolio.start_date = self.period_open
+        portfolio.positions = self.get_positions()
+        portfolio.positions_value = self.ending_value
+        return portfolio
 
     def get_positions(self):
 

From 34d577d3d7e6f3b854ddb54b525fab684664925a Mon Sep 17 00:00:00 2001
From: Eddie Hebert <ehebert@quantopian.com>
Date: Fri, 11 Jan 2013 23:54:52 -0500
Subject: [PATCH 10/13] Recycles objects for positions.

Instead of creating a new ndict for each position on every event,
we change the values in the object that held the previous position.

The creation of new objects on each event was incurring too much
overhead.

Changes the position type returned by performance module.

For improved speed, changes from ndict to a simple Python object,
since the cost of setting ndict values is too expensive for the
number of times that positions are returned.

Also, changes the containing type of the positions to be dictionary
with the __missing__ overloaded, instead of the ndict that had that
behavior, to reduce the penalty of using ndicts.
---
 zipline/finance/performance.py | 20 ++++++++------------
 zipline/protocol.py            | 20 ++++++++++++++++++++
 2 files changed, 28 insertions(+), 12 deletions(-)

diff --git a/zipline/finance/performance.py b/zipline/finance/performance.py
index 3543f7ed..78130c05 100644
--- a/zipline/finance/performance.py
+++ b/zipline/finance/performance.py
@@ -136,7 +136,6 @@ import datetime
 import pytz
 import math
 
-from zipline.utils.protocol_utils import ndict
 import zipline.protocol as zp
 import zipline.finance.risk as risk
 
@@ -446,6 +445,7 @@ class PerformancePeriod(object):
         # when returning portfolio information.
         # So as not to avoid creating a new object for each event
         self._portfolio_store = zp.Portfolio()
+        self._positions_store = zp.Positions()
 
     def calculate_performance(self):
         self.ending_value = self.calculate_positions_value()
@@ -566,11 +566,15 @@ class PerformancePeriod(object):
 
     def get_positions(self):
 
-        positions = ndict(internal=position_ndict())
+        positions = self._positions_store
 
         for sid, pos in self.positions.iteritems():
-            cur = pos.to_dict()
-            positions[sid] = ndict(cur)
+            if sid not in positions:
+                positions[sid] = zp.Position(sid)
+            position = positions[sid]
+            position.amount = pos.amount
+            position.cost_basis = pos.cost_basis
+            position.last_sale_price = pos.last_sale_price
 
         return positions
 
@@ -588,11 +592,3 @@ class positiondict(dict):
         pos = Position(key)
         self[key] = pos
         return pos
-
-
-class position_ndict(dict):
-
-    def __missing__(self, key):
-        pos = Position(key)
-        self[key] = ndict(pos.to_dict())
-        return pos
diff --git a/zipline/protocol.py b/zipline/protocol.py
index 9e71d3ae..25f1b90b 100644
--- a/zipline/protocol.py
+++ b/zipline/protocol.py
@@ -65,3 +65,23 @@ class Portfolio(object):
 
     def __repr__(self):
         return "Portfolio({0})".format(self.__dict__)
+
+
+class Position(object):
+
+    def __init__(self, sid):
+        self.sid = sid
+        self.amount = 0
+        self.cost_basis = 0.0  # per share
+        self.last_sale_price = 0.0
+
+    def __repr__(self):
+        return "Position({0})".format(self.__dict__)
+
+
+class Positions(dict):
+
+    def __missing__(self, key):
+        pos = Position(key)
+        self[key] = pos
+        return pos

From 3227ba49a44bd8dc412de3f6922a1be8ccde2f4c Mon Sep 17 00:00:00 2001
From: Jonathan Kamens <jik@quantopian.com>
Date: Mon, 14 Jan 2013 11:05:58 -0500
Subject: [PATCH 11/13] Upgrade requests module to 1.1.0

---
 etc/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etc/requirements.txt b/etc/requirements.txt
index fb4a96e6..d3b9d00e 100644
--- a/etc/requirements.txt
+++ b/etc/requirements.txt
@@ -16,4 +16,4 @@ python-dateutil==2.1
 six==1.2.0
 
 # For fetching remote data
-requests==1.0.4
+requests==1.1.0

From e7405d04ad0e7bc24e670dedf82d6f3461294857 Mon Sep 17 00:00:00 2001
From: Eddie Hebert <ehebert@quantopian.com>
Date: Mon, 14 Jan 2013 21:13:45 -0500
Subject: [PATCH 12/13] Rolls over existing PerformancePeriod.

Instead of doing the rollover by creating a new PerformancePeriod,
introduces a `rollover` method that resets the values that need
to be fresh in a new period, and moves the ending values to starting
values, and leaves positions intact.

This isn't a major runtime improvement in of itself, but it does
allow us to more easily keep track of position values from period
to period, which other improvements will use.
---
 tests/test_perf_tracking.py    | 58 +++++++++++++++-------------------
 zipline/finance/performance.py | 39 +++++++++--------------
 2 files changed, 40 insertions(+), 57 deletions(-)

diff --git a/tests/test_perf_tracking.py b/tests/test_perf_tracking.py
index b7619441..b4e41c1d 100644
--- a/tests/test_perf_tracking.py
+++ b/tests/test_perf_tracking.py
@@ -94,7 +94,7 @@ check treasury and benchmark data in findb, and re-run the test."""
         )
 
         txn = factory.create_txn(1, 10.0, 100, self.dt + self.onesec)
-        pp = perf.PerformancePeriod({}, 0.0, 1000.0)
+        pp = perf.PerformancePeriod(1000.0)
 
         pp.execute_transaction(txn)
         for trade in trades:
@@ -165,7 +165,7 @@ single short-sale transaction"""
         trades_1 = trades[:-2]
 
         txn = factory.create_txn(1, 10.0, -100, self.dt + self.onesec)
-        pp = perf.PerformancePeriod({}, 0.0, 1000.0)
+        pp = perf.PerformancePeriod(1000.0)
 
         pp.execute_transaction(txn)
         for trade in trades_1:
@@ -223,68 +223,64 @@ single short-sale transaction"""
         trades_2 = trades[-2:]
 
         #simulate a rollover to a new period
-        pp2 = perf.PerformancePeriod(
-            pp.positions,
-            pp.ending_value,
-            pp.ending_cash
-        )
+        pp.rollover()
 
         for trade in trades_2:
-            pp2.update_last_sale(trade)
+            pp.update_last_sale(trade)
 
-        pp2.calculate_performance()
+        pp.calculate_performance()
 
         self.assertEqual(
-            pp2.period_capital_used,
+            pp.period_capital_used,
             0,
             "capital used should be zero, there were no transactions in \
             performance period"
         )
 
         self.assertEqual(
-            len(pp2.positions),
+            len(pp.positions),
             1,
             "should be just one position"
         )
 
         self.assertEqual(
-            pp2.positions[1].sid,
+            pp.positions[1].sid,
             txn.sid,
             "position should be in security from the transaction"
         )
 
         self.assertEqual(
-            pp2.positions[1].amount,
+            pp.positions[1].amount,
             -100,
             "should have a position of -100 shares"
         )
 
         self.assertEqual(
-            pp2.positions[1].cost_basis,
+            pp.positions[1].cost_basis,
             txn.price,
             "should have a cost basis of 10"
         )
 
         self.assertEqual(
-            pp2.positions[1].last_sale_price,
+            pp.positions[1].last_sale_price,
             trades_2[-1].price,
             "last sale should be price of last trade"
         )
 
         self.assertEqual(
-            pp2.ending_value,
+            pp.ending_value,
             -900,
             "ending value should be price of last trade times number of \
             shares in position")
 
         self.assertEqual(
-            pp2.pnl,
+            pp.pnl,
             200,
             "drop of 2 on -100 shares should be 200"
         )
 
         #now run a performance period encompassing the entire trade sample.
-        ppTotal = perf.PerformancePeriod({}, 0.0, 1000.0)
+        ppTotal = perf.PerformancePeriod(1000.0)
 
         for trade in trades_1:
             ppTotal.update_last_sale(trade)
@@ -364,7 +360,7 @@ trade after cover"""
         )
 
         cover_txn = factory.create_txn(1, 7.0, 100, self.dt + self.onesec * 6)
-        pp = perf.PerformancePeriod({}, 0.0, 1000.0)
+        pp = perf.PerformancePeriod(1000.0)
 
         pp.execute_transaction(short_txn)
         pp.execute_transaction(cover_txn)
@@ -443,7 +439,7 @@ shares in position"
             self.trading_environment
         )
 
-        pp = perf.PerformancePeriod({}, 0.0, 1000.0)
+        pp = perf.PerformancePeriod(1000.0)
 
         for txn in transactions:
             pp.execute_transaction(txn)
@@ -483,33 +479,29 @@ shares in position"
             100,
             trades[-1].dt + self.onesec)
 
-        pp2 = perf.PerformancePeriod(
-            copy.deepcopy(pp.positions),
-            pp.ending_value,
-            pp.ending_cash
-        )
+        pp.rollover()
 
-        pp2.execute_transaction(saleTxn)
-        pp2.update_last_sale(down_tick)
+        pp.execute_transaction(saleTxn)
+        pp.update_last_sale(down_tick)
 
-        pp2.calculate_performance()
+        pp.calculate_performance()
         self.assertEqual(
-            pp2.positions[1].last_sale_price,
+            pp.positions[1].last_sale_price,
             10,
             "should have a last sale of 10, was {val}".format(
-                val=pp2.positions[1].last_sale_price)
+                val=pp.positions[1].last_sale_price)
         )
 
         self.assertEqual(
-            round(pp2.positions[1].cost_basis, 2),
+            round(pp.positions[1].cost_basis, 2),
             11.33,
             "should have a cost basis of 11.33"
         )
 
         #print "second period pnl is {pnl}".format(pnl=pp2.pnl)
-        self.assertEqual(pp2.pnl, -800, "this period goes from +400 to -400")
+        self.assertEqual(pp.pnl, -800, "this period goes from +400 to -400")
 
-        pp3 = perf.PerformancePeriod({}, 0.0, 1000.0)
+        pp3 = perf.PerformancePeriod(1000.0)
 
         transactions.append(saleTxn)
         for txn in transactions:
diff --git a/zipline/finance/performance.py b/zipline/finance/performance.py
index 78130c05..3d1f4641 100644
--- a/zipline/finance/performance.py
+++ b/zipline/finance/performance.py
@@ -179,10 +179,6 @@ class PerformanceTracker(object):
 
         # this performance period will span the entire simulation.
         self.cumulative_performance = PerformancePeriod(
-            # initial positions are empty
-            positiondict(),
-            # initial portfolio positions have zero value
-            0,
             # initial cash is your capital base.
             self.capital_base,
             # the cumulative period will be calculated over the entire test.
@@ -192,10 +188,6 @@ class PerformanceTracker(object):
 
         # this performance period will span just the current market day
         self.todays_performance = PerformancePeriod(
-            # initial positions are empty
-            positiondict(),
-            # initial portfolio positions have zero value
-            0,
             # initial cash is your capital base.
             self.capital_base,
             # the daily period will be calculated for the market day
@@ -313,14 +305,9 @@ Last successful date: %s" % self.market_open)
         self.market_close = self.market_open + self.trading_day
 
         # Roll over positions to current day.
-        self.todays_performance = PerformancePeriod(
-            self.todays_performance.positions,
-            self.todays_performance.ending_value,
-            self.todays_performance.ending_cash,
-            self.market_open,
-            self.market_close,
-            keep_transactions=True
-        )
+        self.todays_performance.rollover()
+        self.todays_performance.period_open = self.market_open
+        self.todays_performance.period_close = self.market_close
 
         return daily_update
 
@@ -410,8 +397,6 @@ class PerformancePeriod(object):
 
     def __init__(
             self,
-            initial_positions,
-            starting_value,
             starting_cash,
             period_open=None,
             period_close=None,
@@ -424,12 +409,8 @@ class PerformancePeriod(object):
         self.period_capital_used = 0.0
         self.pnl = 0.0
         #sid => position object
-        if not isinstance(initial_positions, positiondict):
-            self.positions = positiondict()
-            self.positions.update(initial_positions)
-        else:
-            self.positions = initial_positions
-        self.starting_value = starting_value
+        self.positions = positiondict()
+        self.starting_value = 0.0
         #cash balance at start of period
         self.starting_cash = starting_cash
         self.ending_cash = starting_cash
@@ -447,6 +428,16 @@ class PerformancePeriod(object):
         self._portfolio_store = zp.Portfolio()
         self._positions_store = zp.Positions()
 
+    def rollover(self):
+        self.starting_value = self.ending_value
+        self.starting_cash = self.ending_cash
+        self.period_capital_used = 0.0
+        self.pnl = 0.0
+        self.processed_transactions = []
+        self.cumulative_capital_used = 0.0
+        self.max_capital_used = 0.0
+        self.max_leverage = 0.0
+
     def calculate_performance(self):
         self.ending_value = self.calculate_positions_value()
 

From 018ac679666f7187cb6f9fee667017f5d64750df Mon Sep 17 00:00:00 2001
From: Eddie Hebert <ehebert@quantopian.com>
Date: Mon, 14 Jan 2013 21:27:27 -0500
Subject: [PATCH 13/13] Uses vdot and numpy arrays for position totals.

Gets almost 100x speed up over iterating over the values and
summing up the values in Python.

Farms out the work to numpy and atlas by using the vector dot
product of the amounts and last sale prices.

Adds some wiring of keeping track of an index into the numpy arrays
for each position, so that value can be overwritten as events update
those amounts and sale prices.
---
 zipline/finance/performance.py | 33 ++++++++++++++++++++++++++++-----
 1 file changed, 28 insertions(+), 5 deletions(-)

diff --git a/zipline/finance/performance.py b/zipline/finance/performance.py
index 3d1f4641..ed1e2822 100644
--- a/zipline/finance/performance.py
+++ b/zipline/finance/performance.py
@@ -136,6 +136,8 @@ import datetime
 import pytz
 import math
 
+import numpy as np
+
 import zipline.protocol as zp
 import zipline.finance.risk as risk
 
@@ -420,6 +422,12 @@ class PerformancePeriod(object):
         self.max_capital_used = 0.0
         self.max_leverage = 0.0
 
+        # Maps position to following array indexes
+        self._position_index_map = {}
+        # Arrays for quick calculations of positions value
+        self._position_amounts = np.array([])
+        self._position_last_sale_prices = np.array([])
+
         self.calculate_performance()
 
         # An object to recycle via assigning new values
@@ -438,6 +446,17 @@ class PerformancePeriod(object):
         self.max_capital_used = 0.0
         self.max_leverage = 0.0
 
+    def index_for_position(self, sid):
+        try:
+            index = self._position_index_map[sid]
+        except KeyError:
+            index = len(self._position_index_map)
+            self._position_index_map[sid] = index
+            self._position_amounts = np.append(self._position_amounts, [0])
+            self._position_last_sale_prices = np.append(
+                self._position_last_sale_prices, [0])
+        return index
+
     def calculate_performance(self):
         self.ending_value = self.calculate_positions_value()
 
@@ -454,7 +473,11 @@ class PerformancePeriod(object):
     def execute_transaction(self, txn):
         # Update Position
         # ----------------
-        self.positions[txn.sid].update(txn)
+        position = self.positions[txn.sid]
+        position.update(txn)
+        index = self.index_for_position(txn.sid)
+        self._position_amounts[index] = position.amount
+
         self.period_capital_used += -1 * txn.price * txn.amount
 
         # Max Leverage
@@ -485,15 +508,15 @@ class PerformancePeriod(object):
         return int(base * round(float(x) / base))
 
     def calculate_positions_value(self):
-        mktValue = 0.0
-        for key, pos in self.positions.iteritems():
-            mktValue += pos.currentValue()
-        return mktValue
+        return np.vdot(self._position_amounts, self._position_last_sale_prices)
 
     def update_last_sale(self, event):
         is_trade = event.type == zp.DATASOURCE_TYPE.TRADE
         if event.sid in self.positions and is_trade:
             self.positions[event.sid].last_sale_price = event.price
+            index = self.index_for_position(event.sid)
+            self._position_last_sale_prices[index] = event.price
+
             self.positions[event.sid].last_sale_date = event.dt
 
     def __core_dict(self):