From 236fe92a53ed1537241346330d5c94f1d63df3e3 Mon Sep 17 00:00:00 2001 From: Thomas Wiecki Date: Thu, 6 Jun 2013 11:26:36 -0400 Subject: [PATCH 1/3] ENH: Make RollingPanel update itself if new fields arrive. Before we preinitialized the BT's fields and sids. Thus, no new ones could be added after initialization. This should be fixed now. --- tests/test_batchtransform.py | 9 ++++----- zipline/utils/data.py | 35 ++++++++++++++++++++++++++++++----- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/tests/test_batchtransform.py b/tests/test_batchtransform.py index 17a9b7e3..b2a2c9c8 100644 --- a/tests/test_batchtransform.py +++ b/tests/test_batchtransform.py @@ -43,7 +43,7 @@ def return_price(data): class BatchTransformAlgorithmSetSid(TradingAlgorithm): - def initialize(self, sids): + def initialize(self, sids=None): self.history = [] self.batch_transform = return_price( @@ -111,15 +111,14 @@ class TestChangeOfSids(TestCase): ) def test_all_sids_passed(self): - algo = BatchTransformAlgorithmSetSid(self.sids, - sim_params=self.sim_params) + algo = BatchTransformAlgorithmSetSid(sim_params=self.sim_params) source = DifferentSidSource() algo.run(source) - for df, date in zip(algo.history, source.trading_days): + for i, (df, date) in enumerate(zip(algo.history, source.trading_days)): self.assertEqual(df.index[-1], date, "Newest event doesn't \ match.") - for sid in self.sids: + for sid in self.sids[:i]: self.assertIn(sid, df.columns) last_elem = len(df) - 1 diff --git a/zipline/utils/data.py b/zipline/utils/data.py index 8eead051..a95c1ae2 100644 --- a/zipline/utils/data.py +++ b/zipline/utils/data.py @@ -32,28 +32,53 @@ class RollingPanel(object): Restrictions: major_axis can only be a DatetimeIndex for now """ - def __init__(self, window, items, minor_axis, cap_multiple=2, + def __init__(self, window, items, sids, cap_multiple=2, dtype=np.float64): self.pos = 0 self.window = window self.items = _ensure_index(items) - self.minor_axis = _ensure_index(minor_axis) + self.minor_axis = _ensure_index(sids) self.cap_multiple = cap_multiple self.cap = cap_multiple * window self.dtype = dtype self.index_buf = np.empty(self.cap, dtype='M8[ns]') - self.buffer = pd.Panel(items=items, minor_axis=minor_axis, - major_axis=range(self.cap), - dtype=dtype) + + self.buffer = self._create_buffer() + + def _create_buffer(self): + return pd.Panel(items=self.items, minor_axis=self.minor_axis, + major_axis=range(self.cap), + dtype=self.dtype) + + def _update_buffer(self, frame): + # Drop outdated, nan-filled minors (sids) and items (fields) + non_nan_cols = set(self.buffer.dropna(axis=1).minor_axis) + new_cols = set(frame.columns) + self.minor_axis = _ensure_index(new_cols.union(non_nan_cols)) + + non_nan_items = set(self.buffer.dropna(axis=1).items) + new_items = set(frame.index) + self.items = _ensure_index(new_items.union(non_nan_items)) + + new_buffer = self._create_buffer() + # Copy old values we want to keep + new_buffer.update( + self.buffer.loc[non_nan_items, :, non_nan_cols]) + self.buffer = new_buffer def add_frame(self, tick, frame): """ """ if self.pos == self.cap: self._roll_data() + + if set(frame.columns).difference(set(self.minor_axis)) or \ + set(frame.index).difference(set(self.items)): + self._update_buffer(frame) + self.buffer.values[:, self.pos, :] = frame.ix[self.items].values self.index_buf[self.pos] = tick From 102cddb4cbe7d6f1619acedf1b965d1c38a2d634 Mon Sep 17 00:00:00 2001 From: Thomas Wiecki Date: Thu, 6 Jun 2013 16:38:30 -0400 Subject: [PATCH 2/3] ENH: Use smarter matching for updating RollingPanel. --- zipline/utils/data.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zipline/utils/data.py b/zipline/utils/data.py index a95c1ae2..73b05570 100644 --- a/zipline/utils/data.py +++ b/zipline/utils/data.py @@ -79,7 +79,8 @@ class RollingPanel(object): set(frame.index).difference(set(self.items)): self._update_buffer(frame) - self.buffer.values[:, self.pos, :] = frame.ix[self.items].values + self.buffer.loc[:, self.pos, :] = frame.ix[self.items].T + self.index_buf[self.pos] = tick self.pos += 1 From a7818f853a9709f4a15c97313da8a8c865764918 Mon Sep 17 00:00:00 2001 From: Thomas Wiecki Date: Thu, 6 Jun 2013 16:39:05 -0400 Subject: [PATCH 3/3] DOC: Add note about performance issue when updating. --- zipline/utils/data.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/zipline/utils/data.py b/zipline/utils/data.py index 73b05570..b12fd367 100644 --- a/zipline/utils/data.py +++ b/zipline/utils/data.py @@ -65,8 +65,14 @@ class RollingPanel(object): new_buffer = self._create_buffer() # Copy old values we want to keep + # .update() is pretty slow. Ideally we would be using + # new_buffer.loc[non_nan_items, :, non_nan_cols] = + # but this triggers a bug in Pandas 0.11. Update + # this when 0.12 is released. + # https://github.com/pydata/pandas/issues/3777 new_buffer.update( self.buffer.loc[non_nan_items, :, non_nan_cols]) + self.buffer = new_buffer def add_frame(self, tick, frame):