From 6329f477b897f49506834e52d147c8699269513a Mon Sep 17 00:00:00 2001 From: fawce Date: Mon, 4 Mar 2013 06:26:41 -0500 Subject: [PATCH 1/5] removed data_panel option from BatchTransform. - the underlying dequeue shouldn't be modified, so forwarding to the user function is a bit misleading - if we want to provide a dequeue we should consider another class or an EventWindow decorator. --- zipline/transforms/utils.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/zipline/transforms/utils.py b/zipline/transforms/utils.py index 72cc1f4c..a80e1899 100644 --- a/zipline/transforms/utils.py +++ b/zipline/transforms/utils.py @@ -318,7 +318,6 @@ class BatchTransform(EventWindow): clean_nans=True, sids=None, fields=None, - create_panel=True, compute_only_full=True): """Instantiate new batch_transform object. @@ -329,7 +328,7 @@ class BatchTransform(EventWindow): with the data panel and all args and kwargs supplied to the handle_data() call. refresh_period : int - Interval to call batch_transform function. + Interval to wait between advances in the window. window_length : int How many days the trailing window should have. clean_nans : bool @@ -342,12 +341,6 @@ class BatchTransform(EventWindow): Which fields to include in the moving window (e.g. 'price'). If not supplied, fields will be extracted from incoming events. - create_panel : bool - If True, will create a pandas panel every refresh - period and pass it to the user-defined function. - If False, will pass the underlying deque reference - directly to the function which will be significantly - faster. compute_only_full : bool Only call the user-defined function once the window is full. Returns None if window is not full yet. @@ -361,7 +354,6 @@ class BatchTransform(EventWindow): self.compute_transform_value = self.get_value self.clean_nans = clean_nans - self.create_panel = create_panel self.compute_only_full = compute_only_full self.sids = sids @@ -519,11 +511,9 @@ class BatchTransform(EventWindow): return None if self.updated: - # Either create new pandas panel or pass ticks dequeue - # directly - data = self.get_data() if self.create_panel else self.ticks - self.cached = self.compute_transform_value(data, *args, - **kwargs) + # Create new pandas panel + data = self.get_data() + self.cached = self.compute_transform_value(data, *args, **kwargs) return self.cached From a47143099cd4a065cf4521518ef0690bc830fd93 Mon Sep 17 00:00:00 2001 From: fawce Date: Mon, 4 Mar 2013 21:08:15 -0500 Subject: [PATCH 2/5] refining the batch transform interface: - removed use_panel - default for refresh_period is now 0 - refresh_period will only affect the recreation of the datapanel - user's transform method is invoked on every call to batch transform --- zipline/test_algorithms.py | 8 -------- zipline/transforms/utils.py | 13 ++++++++++--- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/zipline/test_algorithms.py b/zipline/test_algorithms.py index 81de1bed..470e9bf8 100644 --- a/zipline/test_algorithms.py +++ b/zipline/test_algorithms.py @@ -342,12 +342,6 @@ class BatchTransformAlgorithm(TradingAlgorithm): clean_nans=True ) - self.return_ticks = return_data( - refresh_period=self.refresh_period, - window_length=self.window_length, - create_panel=False - ) - self.return_not_full = return_data( refresh_period=0, window_length=self.window_length, @@ -372,8 +366,6 @@ class BatchTransformAlgorithm(TradingAlgorithm): self.history_return_args.append( self.return_args_batch.handle_data( data, *self.args, **self.kwargs)) - self.history_return_ticks.append( - self.return_ticks.handle_data(data)) self.history_return_not_full.append( self.return_not_full.handle_data(data)) self.uses_ufunc.handle_data(data) diff --git a/zipline/transforms/utils.py b/zipline/transforms/utils.py index a80e1899..d901a4ae 100644 --- a/zipline/transforms/utils.py +++ b/zipline/transforms/utils.py @@ -313,7 +313,7 @@ class BatchTransform(EventWindow): def __init__(self, func=None, - refresh_period=None, + refresh_period=0, window_length=None, clean_nans=True, sids=None, @@ -368,6 +368,7 @@ class BatchTransform(EventWindow): self.window_length = window_length self.trading_days_since_update = 0 self.trading_days_total = 0 + self.window = None self.full = False self.last_dt = None @@ -512,8 +513,14 @@ class BatchTransform(EventWindow): if self.updated: # Create new pandas panel - data = self.get_data() - self.cached = self.compute_transform_value(data, *args, **kwargs) + self.window = self.get_data() + + if self.window: + self.cached = self.compute_transform_value( + self.window, + *args, + **kwargs + ) return self.cached From 530f1cce557009ebd91f03dbde62e74d499ef304 Mon Sep 17 00:00:00 2001 From: fawce Date: Mon, 4 Mar 2013 22:33:30 -0500 Subject: [PATCH 3/5] added checks for change in optional parameters to trigger transform recalculation. --- zipline/transforms/utils.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/zipline/transforms/utils.py b/zipline/transforms/utils.py index d901a4ae..56212ae4 100644 --- a/zipline/transforms/utils.py +++ b/zipline/transforms/utils.py @@ -375,6 +375,8 @@ class BatchTransform(EventWindow): self.updated = False self.cached = None + self.last_args = None + self.last_kwargs = None # Data panel that provides bar information to fill in the window, # when no bar ticks are available from the data source generator @@ -515,13 +517,18 @@ class BatchTransform(EventWindow): # Create new pandas panel self.window = self.get_data() - if self.window: + args_changed = args != self.last_args + args_changed = args_changed or kwargs != self.last_kwargs + + if self.updated or args_changed: self.cached = self.compute_transform_value( self.window, *args, **kwargs ) + self.last_args = args + self.last_kwargs = kwargs return self.cached def __call__(self, f): From b8144cea2a4fbdbb1339a469cf53226dc25d025d Mon Sep 17 00:00:00 2001 From: fawce Date: Tue, 5 Mar 2013 22:23:43 -0500 Subject: [PATCH 4/5] added tests to ensure: - repeated calls with the same data window do not update batch transform windows. - repeated calls with the same data and same supplemental parameters do not update batch transform results - repeated calls with the same data and different supplemental params do update batch transform results --- zipline/test_algorithms.py | 34 ++++++++++++++++++++++++++++++++++ zipline/transforms/utils.py | 25 ++++++++++++++++--------- 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/zipline/test_algorithms.py b/zipline/test_algorithms.py index 470e9bf8..e4d9b559 100644 --- a/zipline/test_algorithms.py +++ b/zipline/test_algorithms.py @@ -273,6 +273,11 @@ def uses_ufunc(data, *args, **kwargs): return np.log(data) +@batch_transform +def price_multiple(data, multiplier, keyarg=1): + return data.price * multiplier * keyarg + + class BatchTransformAlgorithm(TradingAlgorithm): def initialize(self, *args, **kwargs): self.refresh_period = kwargs.pop('refresh_period', 1) @@ -354,6 +359,12 @@ class BatchTransformAlgorithm(TradingAlgorithm): clean_nans=False ) + self.price_multiple = price_multiple( + refresh_period=self.refresh_period, + window_length=self.window_length, + clean_nans=False + ) + self.iter = 0 self.set_slippage(FixedSlippage()) @@ -370,6 +381,29 @@ class BatchTransformAlgorithm(TradingAlgorithm): self.return_not_full.handle_data(data)) self.uses_ufunc.handle_data(data) + # check that calling transforms with the same arguments + # is idempotent + self.price_multiple.handle_data(data, 1, keyarg=1) + + if self.price_multiple.full: + pre = len(self.price_multiple.ticks) + result1 = self.price_multiple.handle_data(data, 1, keyarg=1) + post = len(self.price_multiple.ticks) + assert pre == post, "batch transform is appending redundant events" + result2 = self.price_multiple.handle_data(data, 1, keyarg=1) + assert result1 is result2, "batch transform is not idempotent" + + # check that calling transform with the same data, but + # different supplemental arguments results in new + # results. + result3 = self.price_multiple.handle_data(data, 2, keyarg=1) + assert result1 is not result3, \ + "batch transform is not updating for new args" + + result4 = self.price_multiple.handle_data(data, 1, keyarg=2) + assert result1 is not result4,\ + "batch transform is not updating for new kwargs" + new_data = deepcopy(data) for sid in new_data: new_data[sid]['arbitrary'] = 123 diff --git a/zipline/transforms/utils.py b/zipline/transforms/utils.py index 56212ae4..cac1c119 100644 --- a/zipline/transforms/utils.py +++ b/zipline/transforms/utils.py @@ -228,8 +228,6 @@ class EventWindow(object): # Subclasses should override handle_add to define behavior for # adding new ticks. self.handle_add(event) - #if len(self.ticks) > self.window_length: - # import nose.tools; nose.tools.set_trace() # Clear out any expired events. # # oldest newest @@ -406,9 +404,19 @@ class BatchTransform(EventWindow): # functionality to zipline if len(v)} - # append data frame to window. update() will call handle_add() and - # handle_remove() appropriately - self.update(event) + # only modify the trailing window if this is + # a new event. This is intended to make handle_data + # idempotent. + if event not in self.ticks: + # append data frame to window. update() will call handle_add() and + # handle_remove() appropriately, and self.updated + # will be modified based on the refresh_period + self.update(event) + else: + # we are recalculating based on an old event, so + # there is no change in the contents of the trailing + # window + self.updated = False # return newly computed or cached value return self.get_transform_value(*args, **kwargs) @@ -449,7 +457,6 @@ class BatchTransform(EventWindow): # to call the user-defined batch-transform with the most # recent datapanel self.updated = True - self.trading_days_since_update = 0 else: self.updated = False @@ -516,10 +523,10 @@ class BatchTransform(EventWindow): if self.updated: # Create new pandas panel self.window = self.get_data() + # reset our counter for refresh_period + self.trading_days_since_update = 0 - args_changed = args != self.last_args - args_changed = args_changed or kwargs != self.last_kwargs - + args_changed = args != self.last_args or kwargs != self.last_kwargs if self.updated or args_changed: self.cached = self.compute_transform_value( self.window, From a12eeb238346d57850127041a8f8285f8272d4e1 Mon Sep 17 00:00:00 2001 From: fawce Date: Wed, 6 Mar 2013 16:49:31 -0500 Subject: [PATCH 5/5] incorporating feedback from @richafrank --- zipline/test_algorithms.py | 14 +++++++------- zipline/transforms/utils.py | 8 ++++++-- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/zipline/test_algorithms.py b/zipline/test_algorithms.py index e4d9b559..0d2a158b 100644 --- a/zipline/test_algorithms.py +++ b/zipline/test_algorithms.py @@ -274,8 +274,8 @@ def uses_ufunc(data, *args, **kwargs): @batch_transform -def price_multiple(data, multiplier, keyarg=1): - return data.price * multiplier * keyarg +def price_multiple(data, multiplier, extra_arg=1): + return data.price * multiplier * extra_arg class BatchTransformAlgorithm(TradingAlgorithm): @@ -383,24 +383,24 @@ class BatchTransformAlgorithm(TradingAlgorithm): # check that calling transforms with the same arguments # is idempotent - self.price_multiple.handle_data(data, 1, keyarg=1) + self.price_multiple.handle_data(data, 1, extra_arg=1) if self.price_multiple.full: pre = len(self.price_multiple.ticks) - result1 = self.price_multiple.handle_data(data, 1, keyarg=1) + result1 = self.price_multiple.handle_data(data, 1, extra_arg=1) post = len(self.price_multiple.ticks) assert pre == post, "batch transform is appending redundant events" - result2 = self.price_multiple.handle_data(data, 1, keyarg=1) + result2 = self.price_multiple.handle_data(data, 1, extra_arg=1) assert result1 is result2, "batch transform is not idempotent" # check that calling transform with the same data, but # different supplemental arguments results in new # results. - result3 = self.price_multiple.handle_data(data, 2, keyarg=1) + result3 = self.price_multiple.handle_data(data, 2, extra_arg=1) assert result1 is not result3, \ "batch transform is not updating for new args" - result4 = self.price_multiple.handle_data(data, 1, keyarg=2) + result4 = self.price_multiple.handle_data(data, 1, extra_arg=2) assert result1 is not result4,\ "batch transform is not updating for new kwargs" diff --git a/zipline/transforms/utils.py b/zipline/transforms/utils.py index cac1c119..7b47d9f7 100644 --- a/zipline/transforms/utils.py +++ b/zipline/transforms/utils.py @@ -520,14 +520,18 @@ class BatchTransform(EventWindow): if self.compute_only_full and not self.full: return None + recalculate_needed = False if self.updated: # Create new pandas panel self.window = self.get_data() # reset our counter for refresh_period self.trading_days_since_update = 0 + recalculate_needed = True + else: + recalculate_needed = \ + args != self.last_args or kwargs != self.last_kwargs - args_changed = args != self.last_args or kwargs != self.last_kwargs - if self.updated or args_changed: + if recalculate_needed: self.cached = self.compute_transform_value( self.window, *args,