From 7a0a6f5231ce6991a61ccce1cc68e5f5c11c84ae Mon Sep 17 00:00:00 2001 From: Thomas Wiecki Date: Thu, 6 Dec 2012 12:33:18 -0500 Subject: [PATCH 1/9] BUG: EventWindow now always contains constant number of days. --- tests/test_transforms.py | 8 ++------ zipline/transforms/utils.py | 33 +++++---------------------------- 2 files changed, 7 insertions(+), 34 deletions(-) diff --git a/tests/test_transforms.py b/tests/test_transforms.py index ae340ca8..2d729f87 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -123,11 +123,7 @@ class TestEventWindow(TestCase): # Record the length of the window after each event. lengths.append(len(window.ticks)) - # The window stretches out during the weekend because we wait - # to drop events until the weekend ends. The last window is - # briefly longer because it doesn't complete a full day. The - # window then shrinks once the day completes - assert lengths == [1, 2, 3, 3, 3, 4, 5, 5, 5, 3, 4, 3] + assert lengths == [1, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3] assert window.added == events assert window.removed == events[:-3] @@ -146,7 +142,7 @@ class TestEventWindow(TestCase): # Record the length of the window after each event. lengths.append(len(window.ticks)) - assert lengths == [1, 2, 3, 3, 2] + assert lengths == [1, 2, 2, 2, 2] assert window.added == events assert window.removed == events[:-2] diff --git a/zipline/transforms/utils.py b/zipline/transforms/utils.py index f1e7b551..0050241d 100644 --- a/zipline/transforms/utils.py +++ b/zipline/transforms/utils.py @@ -239,9 +239,6 @@ class EventWindow(object): # adding new ticks. self.handle_add(event) - if self.market_aware: - self.add_new_holidays(event.dt) - # Clear out any expired events. drop_condition changes depending # on whether or not we are running in market_aware mode. # @@ -257,33 +254,13 @@ class EventWindow(object): # behavior for removing ticks. self.handle_remove(popped) - def add_new_holidays(self, newest): - # Add to our tracked window any untracked holidays that are - # older than our newest event. (newest should always be - # self.ticks[-1]) - while len(self.all_holidays) > 0 and self.all_holidays[0] <= newest: - self.cur_holidays.append(self.all_holidays.popleft()) - - def drop_old_holidays(self, oldest): - # Drop from our tracked window any holidays that are older - # than our oldest tracked event. (oldest should always - # be self.ticks[0]) - while len(self.cur_holidays) > 0 and self.cur_holidays[0] < oldest: - self.cur_holidays.popleft() - def out_of_market_window(self, oldest, newest): - self.drop_old_holidays(oldest) - calendar_dates_between = (newest.date() - oldest.date()).days - holidays_between = len(self.cur_holidays) - trading_days_between = calendar_dates_between - holidays_between + # Find number of unique days in window + # Note that this assumes that each day we received an + # event is a trading day. + unique_dts = set([event.dt.date() for event in self.ticks]) - # "Put back" a day if oldest is earlier in its day than newest, - # reflecting the fact that we haven't yet completed the last - # day in the window. - if oldest.time() > newest.time(): - trading_days_between -= 1 - - return trading_days_between >= self.window_length + return len(unique_dts) > self.window_length def out_of_delta(self, oldest, newest): return (newest - oldest) >= self.delta From c69858f8b93c6391d1a712f4260bf60d78ba1fc1 Mon Sep 17 00:00:00 2001 From: Thomas Wiecki Date: Thu, 6 Dec 2012 12:35:46 -0500 Subject: [PATCH 2/9] ENH: Added kwarg to turn off fillna or use different fill method. --- zipline/test_algorithms.py | 24 ++++++++++++------------ zipline/transforms/utils.py | 31 +++++++++++++++++-------------- 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/zipline/test_algorithms.py b/zipline/test_algorithms.py index 8fa80b22..e771ca5e 100644 --- a/zipline/test_algorithms.py +++ b/zipline/test_algorithms.py @@ -271,39 +271,39 @@ class BatchTransformAlgorithm(TradingAlgorithm): self.history_return_arbitrary_fields = [] self.return_price_class = ReturnPriceBatchTransform( - market_aware=False, refresh_period=self.refresh_period, - delta=timedelta(days=self.window_length) + window_length=self.window_length, + fillna=False ) self.return_price_decorator = return_price_batch_decorator( - market_aware=False, refresh_period=self.refresh_period, - delta=timedelta(days=self.window_length) + window_length=self.window_length, + fillna=False ) self.return_args_batch = return_args_batch_decorator( - market_aware=False, refresh_period=self.refresh_period, - delta=timedelta(days=self.window_length) + window_length=self.window_length, + fillna=False ) self.return_price_market_aware = ReturnPriceBatchTransform( - market_aware=True, refresh_period=self.refresh_period, - window_length=self.window_length + window_length=self.window_length, + fillna=False ) self.return_price_more_days_than_refresh = ReturnPriceBatchTransform( - market_aware=True, refresh_period=1, - window_length=3 + window_length=3, + fillna=False ) self.return_arbitrary_fields = return_data( - market_aware=True, refresh_period=1, - window_length=3 + window_length=3, + fillna=False ) self.set_slippage(FixedSlippage()) diff --git a/zipline/transforms/utils.py b/zipline/transforms/utils.py index 0050241d..1785400d 100644 --- a/zipline/transforms/utils.py +++ b/zipline/transforms/utils.py @@ -321,19 +321,21 @@ class BatchTransform(EventWindow): def __init__(self, func=None, refresh_period=None, - market_aware=True, - delta=None, - window_length=None): + window_length=None, + fillna=True, + fill_method='ffill'): - super(BatchTransform, self).__init__(market_aware, - window_length=window_length, - delta=delta) + super(BatchTransform, self).__init__(True, + window_length=window_length) if func is not None: self.compute_transform_value = func else: self.compute_transform_value = self.get_value + self.fillna = fillna + self.fill_method = fill_method + self.refresh_period = refresh_period self.window_length = window_length self.trading_days_since_update = 0 @@ -437,14 +439,15 @@ class BatchTransform(EventWindow): # concatenate different sids into one df df = pd.DataFrame.from_dict(values_per_sid) - # Fills in gaps of missing data during transform of multiple - # stocks. - # e.g. we may be missing minute data because of illiquidity - # of one stock - df = df.fillna(method='ffill') - # Drop any empty rows after the fill. - # This will drop a leading row of N/A - df = df.dropna() + + if self.fillna: + # Fills in gaps of missing data during transform + # of multiple stocks. E.g. we may be missing + # minute data because of illiquidity of one stock + df = df.fillna(method=self.fill_method) + # Drop any empty rows after the fill. + # This will drop a leading row of N/A + df = df.dropna() fields[field_name] = df From 5f6839beeaa068024aa4f1e74b33cfd94331171b Mon Sep 17 00:00:00 2001 From: Thomas Wiecki Date: Thu, 6 Dec 2012 12:36:47 -0500 Subject: [PATCH 3/9] BUG: Refactored batch_transform unittests and fixed some bugs. --- tests/test_transforms.py | 26 +++++++++----------------- zipline/test_algorithms.py | 4 ++-- zipline/transforms/utils.py | 33 +++++++++++++++++---------------- zipline/utils/factory.py | 4 ++-- 4 files changed, 30 insertions(+), 37 deletions(-) diff --git a/tests/test_transforms.py b/tests/test_transforms.py index 2d729f87..cb620d84 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -340,27 +340,19 @@ class TestBatchTransform(TestCase): ) self.assertTrue(all( - field['arbitrary'].values.flatten() == ['test'] * 8), + field['arbitrary'].values.flatten() == + ['test'] * algo.window_length), 'arbitrary dataframe should contain only "test"' ) # test overloaded class for test_history in [algo.history_return_price_class, algo.history_return_price_decorator]: - np.testing.assert_array_equal( - range(2, 8), - test_history[2].values.flatten() - ) - - np.testing.assert_array_equal( - range(2, 8), - test_history[3].values.flatten() - ) - - np.testing.assert_array_equal( - range(4, 12), - test_history[4].values.flatten() - ) + for i in range(3, 6): + np.testing.assert_array_equal( + range(i - algo.window_length + 1, i + 1), + test_history[i].values.flatten() + ) def test_passing_of_args(self): algo = BatchTransformAlgorithm(1, kwarg='str') @@ -371,8 +363,8 @@ class TestBatchTransform(TestCase): expected_item = ((1, ), {'kwarg': 'str'}) self.assertEqual( algo.history_return_args, - [None, None, expected_item, expected_item, - expected_item, expected_item]) + [None, None, None, expected_item, expected_item, + expected_item]) class TestBatchTransformMarketAware(TestCase): diff --git a/zipline/test_algorithms.py b/zipline/test_algorithms.py index e771ca5e..d3959d12 100644 --- a/zipline/test_algorithms.py +++ b/zipline/test_algorithms.py @@ -214,7 +214,6 @@ class TimeoutAlgorithm(TradingAlgorithm): time.sleep(100) pass -from datetime import timedelta from zipline.algorithm import TradingAlgorithm from zipline.transforms import BatchTransform, batch_transform from zipline.transforms import MovingAverage @@ -237,6 +236,7 @@ class TestRegisterTransformAlgorithm(TradingAlgorithm): class ReturnPriceBatchTransform(BatchTransform): def get_value(self, data): + assert data.shape[1] == self.window_length return data.price @@ -257,7 +257,7 @@ def return_data(data, *args, **kwargs): class BatchTransformAlgorithm(TradingAlgorithm): def initialize(self, *args, **kwargs): - self.refresh_period = kwargs.pop('refresh_period', 2) + self.refresh_period = kwargs.pop('refresh_period', 1) self.window_length = kwargs.pop('window_length', 3) self.args = args diff --git a/zipline/transforms/utils.py b/zipline/transforms/utils.py index 1785400d..9fc3c4e2 100644 --- a/zipline/transforms/utils.py +++ b/zipline/transforms/utils.py @@ -345,7 +345,7 @@ class BatchTransform(EventWindow): self.last_dt = None self.updated = False - self.data = None + self.cached = None self.field_names = None @@ -373,20 +373,22 @@ class BatchTransform(EventWindow): # return newly computed or cached value return self.get_transform_value(*args, **kwargs) - def handle_add(self, event): - if not self.last_dt: - self.last_dt = event.dt - return - + def _extract_field_names(self, event): # extract field names from sids (price, volume etc), make sure # every sid has the same fields. sid_keys = [set(sid.keys()) for sid in event.data.itervalues()] assert sid_keys[0] == set.intersection(*sid_keys),\ "Each sid must have the same keys." - if self.field_names is None: - unwanted_fields = set(['portfolio', 'sid', 'dt', 'type', - 'datetime']) - self.field_names = sid_keys[0] - unwanted_fields + + unwanted_fields = set(['portfolio', 'sid', 'dt', 'type', + 'datetime']) + return sid_keys[0] - unwanted_fields + + def handle_add(self, event): + if not self.last_dt: + self.field_names = self._extract_field_names(event) + self.last_dt = event.dt + return # update trading day counters if self.last_dt.day != event.dt.day: @@ -398,13 +400,11 @@ class BatchTransform(EventWindow): self.trading_days_total >= self.window_length and self.trading_days_since_update >= self.refresh_period ): - - # Create datapanel of running event window. - self.data = self.get_data() # Setting updated to True will cause get_transform_value() # to call the user-defined batch-transform with the most # recent datapanel self.updated = True + self.full = True self.trading_days_since_update = 0 else: self.updated = False @@ -427,7 +427,8 @@ class BatchTransform(EventWindow): fields = {} for field_name in self.field_names: - sids = self.ticks[0].data.keys() + # Extract all used sids + sids = set.union(*[set(tick.data.keys()) for tick in self.ticks]) values_per_sid = {} @@ -471,11 +472,11 @@ class BatchTransform(EventWindow): has actually been updated. Otherwise, the previously, cached value will be returned. """ - if self.data is None: + if not self.full: return None if self.updated: - self.cached = self.compute_transform_value(self.data, + self.cached = self.compute_transform_value(self.get_data(), *args, **kwargs) return self.cached diff --git a/zipline/utils/factory.py b/zipline/utils/factory.py index ac2e94b2..dea8c6d0 100644 --- a/zipline/utils/factory.py +++ b/zipline/utils/factory.py @@ -271,9 +271,9 @@ def create_test_df_source(): start = pd.datetime(1990, 1, 3, 0, 0, 0, 0, pytz.utc) end = pd.datetime(1990, 1, 8, 0, 0, 0, 0, pytz.utc) index = pd.DatetimeIndex(start=start, end=end, freq=pd.datetools.day) - x = np.arange(2., len(index) * 2 + 2).reshape((-1, 2)) + x = np.arange(0, len(index)) - df = pd.DataFrame(x, index=index, columns=[0, 1]) + df = pd.DataFrame(x, index=index, columns=[0]) return DataFrameSource(df), df From ba46c6292f57932e21dfe836261cca1caf36a8e7 Mon Sep 17 00:00:00 2001 From: Thomas Wiecki Date: Thu, 6 Dec 2012 14:43:20 -0500 Subject: [PATCH 4/9] ENH: DataPanel is now created more flexible over all sids and all given fields. Added unittest to test for nan-filling. Added backwards filling by default. --- tests/test_transforms.py | 47 ++++++------------------------ zipline/test_algorithms.py | 39 ++++++++++++++++--------- zipline/transforms/utils.py | 57 ++++++++++++++++--------------------- 3 files changed, 58 insertions(+), 85 deletions(-) diff --git a/tests/test_transforms.py b/tests/test_transforms.py index cb620d84..9ed507f2 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -313,23 +313,16 @@ class TestBatchTransform(TestCase): def test_event_window(self): algo = BatchTransformAlgorithm() algo.run(self.source) - - self.assertEqual(algo.history_return_price_class[:2], - [None, None], + wl = algo.window_length + self.assertEqual(algo.history_return_price_class[:wl], + [None] * wl, "First two iterations should return None") - self.assertEqual(algo.history_return_price_decorator[:2], - [None, None], + self.assertEqual(algo.history_return_price_decorator[:wl], + [None] * wl, "First two iterations should return None") - self.assertEqual(algo.history_return_price_market_aware[:2], - [None, None], - "First two iterations should return None") - self.assertEqual(algo.history_return_more_days_than_refresh[:3], - [None, None, None], - "First five iterations should return None") self.assertTrue(isinstance( - algo.history_return_more_days_than_refresh[4], - pd.DataFrame), - "Sixth iteration should not be None" + algo.history_return_price_class[wl + 1], + pd.DataFrame) ) # Test whether arbitrary fields can be added to datapanel @@ -341,7 +334,7 @@ class TestBatchTransform(TestCase): self.assertTrue(all( field['arbitrary'].values.flatten() == - ['test'] * algo.window_length), + [123] * algo.window_length), 'arbitrary dataframe should contain only "test"' ) @@ -365,27 +358,3 @@ class TestBatchTransform(TestCase): algo.history_return_args, [None, None, None, expected_item, expected_item, expected_item]) - - -class TestBatchTransformMarketAware(TestCase): - def setUp(self): - setup_logger(self) - start = pd.datetime(1993, 1, 1, 0, 0, 0, 0, pytz.utc) - end = pd.datetime(1994, 1, 1, 0, 0, 0, 0, pytz.utc) - - self.data = factory.load_from_yahoo(stocks=['AAPL'], - indexes={}, - start=start, end=end) - - def test_event_window(self): - days = 50 - algo = BatchTransformAlgorithm(days=days, refresh_period=days) - algo.run(self.data) - - self.assertEqual(algo.history_return_price_market_aware[:days], - [None] * days, - "First {days} iterations should return None" - .format(days=days)) - self.assertFalse(algo.history_return_price_market_aware[days + 1] - is None, - "Window is contains too many Nones.") diff --git a/zipline/test_algorithms.py b/zipline/test_algorithms.py index d3959d12..21c4b62c 100644 --- a/zipline/test_algorithms.py +++ b/zipline/test_algorithms.py @@ -266,9 +266,8 @@ class BatchTransformAlgorithm(TradingAlgorithm): self.history_return_price_class = [] self.history_return_price_decorator = [] self.history_return_args = [] - self.history_return_price_market_aware = [] - self.history_return_more_days_than_refresh = [] self.history_return_arbitrary_fields = [] + self.history_return_nan = [] self.return_price_class = ReturnPriceBatchTransform( refresh_period=self.refresh_period, @@ -294,18 +293,20 @@ class BatchTransformAlgorithm(TradingAlgorithm): fillna=False ) - self.return_price_more_days_than_refresh = ReturnPriceBatchTransform( - refresh_period=1, - window_length=3, + self.return_arbitrary_fields = return_data( + refresh_period=self.refresh_period, + window_length=self.window_length, fillna=False ) - self.return_arbitrary_fields = return_data( - refresh_period=1, - window_length=3, - fillna=False + self.return_nan = ReturnPriceBatchTransform( + refresh_period=self.refresh_period, + window_length=self.window_length, + fillna=True ) + self.iter = 0 + self.set_slippage(FixedSlippage()) def handle_data(self, data): @@ -316,18 +317,28 @@ class BatchTransformAlgorithm(TradingAlgorithm): self.history_return_args.append( self.return_args_batch.handle_data( data, *self.args, **self.kwargs)) - self.history_return_price_market_aware.append( - self.return_price_market_aware.handle_data(data)) - self.history_return_more_days_than_refresh.append( - self.return_price_more_days_than_refresh.handle_data(data)) new_data = deepcopy(data) for sid in new_data: - new_data[sid]['arbitrary'] = 'test' + new_data[sid]['arbitrary'] = 123 self.history_return_arbitrary_fields.append( self.return_arbitrary_fields.handle_data(new_data)) + # nan every second event price + if self.iter % 2 == 0: + self.history_return_nan.append( + self.return_nan.handle_data(data)) + else: + nan_data = deepcopy(data) + import numpy as np + for sid in nan_data.iterkeys(): + nan_data[sid].price = np.nan + self.history_return_nan.append( + self.return_nan.handle_data(nan_data)) + + self.iter += 1 + class SetPortfolioAlgorithm(TradingAlgorithm): """ diff --git a/zipline/transforms/utils.py b/zipline/transforms/utils.py index 9fc3c4e2..e6f6a251 100644 --- a/zipline/transforms/utils.py +++ b/zipline/transforms/utils.py @@ -322,8 +322,7 @@ class BatchTransform(EventWindow): func=None, refresh_period=None, window_length=None, - fillna=True, - fill_method='ffill'): + fillna=True): super(BatchTransform, self).__init__(True, window_length=window_length) @@ -334,7 +333,6 @@ class BatchTransform(EventWindow): self.compute_transform_value = self.get_value self.fillna = fillna - self.fill_method = fill_method self.refresh_period = refresh_period self.window_length = window_length @@ -381,7 +379,7 @@ class BatchTransform(EventWindow): "Each sid must have the same keys." unwanted_fields = set(['portfolio', 'sid', 'dt', 'type', - 'datetime']) + 'datetime', 'source_id']) return sid_keys[0] - unwanted_fields def handle_add(self, event): @@ -421,38 +419,33 @@ class BatchTransform(EventWindow): """ # This Panel data structure ultimately gets passed to the # user-overloaded get_value() method. - # - # self.ticks contains ndicts with data, dt keys. - # event parameter is an ndict with data, dt keys. - fields = {} + sids = set.union(*[set(tick.data.keys()) for tick in self.ticks]) + dts = [tick.dt for tick in self.ticks] - for field_name in self.field_names: - # Extract all used sids - sids = set.union(*[set(tick.data.keys()) for tick in self.ticks]) + data = pd.Panel(items=self.field_names, major_axis=dts, + minor_axis=sids) - values_per_sid = {} + # Fill data panel + for tick in self.ticks: + dt = tick.dt + for sid, fields in tick.data.iteritems(): + for field_name in self.field_names: + data[field_name][sid].ix[dt] = fields[field_name] - for sid in sids: - values_per_sid[sid] = pd.Series( - {tick.data[sid].dt: tick.data[sid][field_name] - for tick in self.ticks} - ) + if self.fillna: + # Fills in gaps of missing data during transform + # of multiple stocks. E.g. we may be missing + # minute data because of illiquidity of one stock + data = data.fillna(method='ffill') + # Since we already forward filled, this can only + # fill the first value if it was missing. + # It's not wise to drop a complete column via dropna()) + # because of one missing value. + data = data.fillna(method='bfill') - # concatenate different sids into one df - df = pd.DataFrame.from_dict(values_per_sid) - - if self.fillna: - # Fills in gaps of missing data during transform - # of multiple stocks. E.g. we may be missing - # minute data because of illiquidity of one stock - df = df.fillna(method=self.fill_method) - # Drop any empty rows after the fill. - # This will drop a leading row of N/A - df = df.dropna() - - fields[field_name] = df - - data = pd.Panel.from_dict(fields, orient='items') + # Drop any empty rows after the fill. + # This will drop a leading row of N/A + data = data.dropna() return data From dfefaafd03fde46bb4062eed1fb988280de2030b Mon Sep 17 00:00:00 2001 From: Thomas Wiecki Date: Thu, 6 Dec 2012 14:46:33 -0500 Subject: [PATCH 5/9] BUG: Fixed source unittest (regression in testing source). --- tests/test_sources.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_sources.py b/tests/test_sources.py index 1257ac55..837ee459 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -25,11 +25,9 @@ class TestDataFrameSource(TestCase): for expected_dt, expected_price in df.iterrows(): sid0 = source.next() - sid1 = source.next() - assert expected_dt == sid0.dt == sid1.dt + assert expected_dt == sid0.dt assert expected_price[0] == sid0.price - assert expected_price[1] == sid1.price def test_sid_filtering(self): _, df = factory.create_test_df_source() From c7383f6275b3b145fadecd261c28e9344a8e4508 Mon Sep 17 00:00:00 2001 From: Thomas Wiecki Date: Thu, 6 Dec 2012 15:24:01 -0500 Subject: [PATCH 6/9] STY: Removed drop_condition arguments. --- zipline/transforms/utils.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/zipline/transforms/utils.py b/zipline/transforms/utils.py index e6f6a251..9b9bc13e 100644 --- a/zipline/transforms/utils.py +++ b/zipline/transforms/utils.py @@ -241,12 +241,7 @@ class EventWindow(object): # Clear out any expired events. drop_condition changes depending # on whether or not we are running in market_aware mode. - # - # oldest newest - # | | - # V V - while self.drop_condition(self.ticks[0].dt, self.ticks[-1].dt): - + while self.drop_condition(): # popleft removes and returns the oldest tick in self.ticks popped = self.ticks.popleft() @@ -254,7 +249,7 @@ class EventWindow(object): # behavior for removing ticks. self.handle_remove(popped) - def out_of_market_window(self, oldest, newest): + def out_of_market_window(self): # Find number of unique days in window # Note that this assumes that each day we received an # event is a trading day. @@ -262,8 +257,9 @@ class EventWindow(object): return len(unique_dts) > self.window_length - def out_of_delta(self, oldest, newest): - return (newest - oldest) >= self.delta + def out_of_delta(self): + # newest - oldest + return (self.ticks[-1].dt - self.ticks[0].dt) >= self.delta # All event windows expect to receive events with datetime fields # that arrive in sorted order. From a14d702a0da1e541aee0682376d269979be3fdc5 Mon Sep 17 00:00:00 2001 From: Thomas Wiecki Date: Thu, 6 Dec 2012 15:26:49 -0500 Subject: [PATCH 7/9] STY: Replaced hardcoded values with variables. --- tests/test_transforms.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_transforms.py b/tests/test_transforms.py index 9ed507f2..2f1b2d0d 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -341,7 +341,9 @@ class TestBatchTransform(TestCase): # test overloaded class for test_history in [algo.history_return_price_class, algo.history_return_price_decorator]: - for i in range(3, 6): + # starting at window length, the window should contain + # consecutive (of window length) numbers up till the end. + for i in range(algo.window_length, len(test_history)): np.testing.assert_array_equal( range(i - algo.window_length + 1, i + 1), test_history[i].values.flatten() From c46e09f52a84184760377174aaa7efd9d13c2ad8 Mon Sep 17 00:00:00 2001 From: Thomas Wiecki Date: Thu, 6 Dec 2012 15:52:50 -0500 Subject: [PATCH 8/9] BUG: Do not backfill but drop rows with N/A in them. --- zipline/transforms/utils.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/zipline/transforms/utils.py b/zipline/transforms/utils.py index 9b9bc13e..c65644ca 100644 --- a/zipline/transforms/utils.py +++ b/zipline/transforms/utils.py @@ -433,15 +433,10 @@ class BatchTransform(EventWindow): # of multiple stocks. E.g. we may be missing # minute data because of illiquidity of one stock data = data.fillna(method='ffill') - # Since we already forward filled, this can only - # fill the first value if it was missing. - # It's not wise to drop a complete column via dropna()) - # because of one missing value. - data = data.fillna(method='bfill') # Drop any empty rows after the fill. # This will drop a leading row of N/A - data = data.dropna() + data = data.dropna(axis=1) return data From ecb26e9eec76ff7415b4fcc7b072a58f8469c87c Mon Sep 17 00:00:00 2001 From: Thomas Wiecki Date: Thu, 6 Dec 2012 16:07:18 -0500 Subject: [PATCH 9/9] BUG: Can not test for length when dropping nans. --- zipline/test_algorithms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zipline/test_algorithms.py b/zipline/test_algorithms.py index 21c4b62c..99f92a02 100644 --- a/zipline/test_algorithms.py +++ b/zipline/test_algorithms.py @@ -299,7 +299,7 @@ class BatchTransformAlgorithm(TradingAlgorithm): fillna=False ) - self.return_nan = ReturnPriceBatchTransform( + self.return_nan = return_price_batch_decorator( refresh_period=self.refresh_period, window_length=self.window_length, fillna=True