From 314b22656facaab9f22695ab5e16dca0385c8738 Mon Sep 17 00:00:00 2001 From: Eddie Hebert Date: Mon, 30 Mar 2015 11:55:40 -0400 Subject: [PATCH] MAINT: Remove left over simple transform code. Remove pieces that are no longer used now that the simple transforms are wrappers around history via the SIDData object. Move window length related pieces into batch_transform, since the rest of the utils module is no longer used. --- tests/test_algorithm.py | 6 - tests/test_exception_handling.py | 16 -- zipline/algorithm.py | 38 +--- zipline/gens/composites.py | 16 -- zipline/transforms/batch_transform.py | 24 ++- zipline/transforms/utils.py | 295 -------------------------- zipline/utils/simfactory.py | 10 - zipline/utils/test_utils.py | 13 -- 8 files changed, 28 insertions(+), 390 deletions(-) delete mode 100644 zipline/transforms/utils.py diff --git a/tests/test_algorithm.py b/tests/test_algorithm.py index 375ba898..d583b665 100644 --- a/tests/test_algorithm.py +++ b/tests/test_algorithm.py @@ -553,9 +553,6 @@ def handle_data(context, data): # placed. self.zipline_test_config['order_count'] = 1 - # self.zipline_test_config['transforms'] = \ - # test_algo.transform_visitor.transforms.values() - zipline = simfactory.create_test_zipline( **self.zipline_test_config) @@ -618,9 +615,6 @@ def handle_data(context, data): # https://www.dropbox.com/s/ulrk2qt0nrtrigb/Volume%20Share%20Worksheet.xlsx self.zipline_test_config['expected_transactions'] = 67 - # self.zipline_test_config['transforms'] = \ - # test_algo.transform_visitor.transforms.values() - zipline = simfactory.create_test_zipline( **self.zipline_test_config) output, _ = assert_single_position(self, zipline) diff --git a/tests/test_exception_handling.py b/tests/test_exception_handling.py index b8bd3d45..505253f3 100644 --- a/tests/test_exception_handling.py +++ b/tests/test_exception_handling.py @@ -24,7 +24,6 @@ from zipline.test_algorithms import ( SetPortfolioAlgorithm, ) from zipline.finance.slippage import FixedSlippage -from zipline.transforms.utils import StatefulTransform from zipline.utils.test_utils import ( @@ -32,7 +31,6 @@ from zipline.utils.test_utils import ( setup_logger, teardown_logger, ExceptionSource, - ExceptionTransform ) DEFAULT_TIMEOUT = 15 # seconds @@ -60,20 +58,6 @@ class ExceptionTestCase(TestCase): with self.assertRaises(ZeroDivisionError): output, _ = drain_zipline(self, zipline) - def test_tranform_exception(self): - exc_tnfm = StatefulTransform(ExceptionTransform) - self.zipline_test_config['transforms'] = [exc_tnfm] - - zipline = simfactory.create_test_zipline( - **self.zipline_test_config - ) - - with self.assertRaises(AssertionError) as ctx: - output, _ = drain_zipline(self, zipline) - - self.assertEqual(str(ctx.exception), - 'An assertion message') - def test_exception_in_handle_data(self): # Simulation # ---------- diff --git a/zipline/algorithm.py b/zipline/algorithm.py index d4097a4e..5f3efa4a 100644 --- a/zipline/algorithm.py +++ b/zipline/algorithm.py @@ -62,13 +62,9 @@ from zipline.finance.slippage import ( SlippageModel, transact_partial ) -from zipline.gens.composites import ( - date_sorted_sources, - sequential_transforms, -) +from zipline.gens.composites import date_sorted_sources from zipline.gens.tradesimulation import AlgorithmSimulator from zipline.sources import DataFrameSource, DataPanelSource -from zipline.transforms.utils import StatefulTransform from zipline.utils.api_support import ZiplineAPI, api_method import zipline.utils.events from zipline.utils.events import ( @@ -143,8 +139,6 @@ class TradingAlgorithm(object): """ self.datetime = None - self.registered_transforms = {} - self.transforms = [] self.sources = [] # List of trading controls to be used to validate orders. @@ -312,8 +306,8 @@ class TradingAlgorithm(object): def _create_data_generator(self, source_filter, sim_params=None): """ - Create a merged data generator using the sources and - transforms attached to this algorithm. + Create a merged data generator using the sources attached to this + algorithm. ::source_filter:: is a method that receives events in date sorted order, and returns True for those events that should be @@ -350,11 +344,8 @@ class TradingAlgorithm(object): if source_filter: date_sorted = filter(source_filter, date_sorted) - with_tnfms = sequential_transforms(date_sorted, - *self.transforms) - with_benchmarks = date_sorted_sources(benchmark_return_source, - with_tnfms) + date_sorted) # Group together events with the same dt field. This depends on the # events already being sorted. @@ -362,8 +353,7 @@ class TradingAlgorithm(object): def _create_generator(self, sim_params, source_filter=None): """ - Create a basic generator setup using the sources and - transforms attached to this algorithm. + Create a basic generator setup using the sources to this algorithm. ::source_filter:: is a method that receives events in date sorted order, and returns True for those events that should be @@ -459,23 +449,11 @@ class TradingAlgorithm(object): self.sim_params.data_frequency, ) - # Create transforms by wrapping them into StatefulTransforms - self.transforms = [] - for namestring, trans_descr in iteritems(self.registered_transforms): - sf = StatefulTransform( - trans_descr['class'], - *trans_descr['args'], - **trans_descr['kwargs'] - ) - sf.namestring = namestring - - self.transforms.append(sf) - # force a reset of the performance tracker, in case # this is a repeat run of the algorithm. self.perf_tracker = None - # create transforms and zipline + # create zipline self.gen = self._create_generator(self.sim_params) with ZiplineAPI(self): @@ -854,10 +832,6 @@ class TradingAlgorithm(object): assert isinstance(sources, list) self.sources = sources - def set_transforms(self, transforms): - assert isinstance(transforms, list) - self.transforms = transforms - # Remain backwards compatibility @property def data_frequency(self): diff --git a/zipline/gens/composites.py b/zipline/gens/composites.py index 390c684b..002da277 100644 --- a/zipline/gens/composites.py +++ b/zipline/gens/composites.py @@ -15,8 +15,6 @@ import heapq -from six.moves import reduce - def _decorate_source(source): for message in source: @@ -33,17 +31,3 @@ def date_sorted_sources(*sources): # Strip out key decoration for _, message in sorted_stream: yield message - - -def sequential_transforms(stream_in, *transforms): - """ - Apply each transform in transforms sequentially to each event in stream_in. - Each transform application will add a new entry indexed to the transform's - hash string. - """ - # Recursively apply all transforms to the stream. - stream_out = reduce(lambda stream, tnfm: tnfm.transform(stream), - transforms, - stream_in) - - return stream_out diff --git a/zipline/transforms/batch_transform.py b/zipline/transforms/batch_transform.py index 5e9ea6cc..ec6f3dc5 100644 --- a/zipline/transforms/batch_transform.py +++ b/zipline/transforms/batch_transform.py @@ -37,8 +37,6 @@ from zipline.protocol import Event from zipline.finance import trading -from . utils import check_window_length - log = logbook.Logger('BatchTransform') func_map = {'open_price': 'first', 'close_price': 'last', @@ -92,6 +90,28 @@ def get_date(mkt_close, d1, d2, d): return d1 +class InvalidWindowLength(Exception): + """ + Error raised when the window length is unusable. + """ + pass + + +def check_window_length(window_length): + """ + Ensure the window length provided to a transform is valid. + """ + if window_length is None: + raise InvalidWindowLength("window_length must be provided") + if not isinstance(window_length, Integral): + raise InvalidWindowLength( + "window_length must be an integer-like number") + if window_length == 0: + raise InvalidWindowLength("window_length must be non-zero") + if window_length < 0: + raise InvalidWindowLength("window_length must be positive") + + class BatchTransform(object): """Base class for batch transforms with a trailing window of variable length. As opposed to pure EventWindows that get a stream diff --git a/zipline/transforms/utils.py b/zipline/transforms/utils.py deleted file mode 100644 index dc0e691d..00000000 --- a/zipline/transforms/utils.py +++ /dev/null @@ -1,295 +0,0 @@ -# -# Copyright 2013 Quantopian, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -""" -Generator versions of transforms. -""" -import logbook - - -from numbers import Integral - -from datetime import datetime -from collections import deque -from abc import ABCMeta, abstractmethod, abstractproperty - -from six import with_metaclass - -from zipline.errors import WrongDataForTransform -from zipline.gens.utils import assert_sort_unframe_protocol, hash_args -from zipline.protocol import DATASOURCE_TYPE -from zipline.finance import trading - -log = logbook.Logger('Transform') - - -class UnsupportedEventWindowFlagValue(Exception): - """ - Error state when an EventWindow option is attempted to be set - to a value that is no longer supported by the library. - - This is to help enforce deprecation of the market_aware and delta flags, - without completely removing it and breaking existing algorithms. - """ - pass - - -class InvalidWindowLength(Exception): - """ - Error raised when the window length is unusable. - """ - pass - - -def check_window_length(window_length): - """ - Ensure the window length provided to a transform is valid. - """ - if window_length is None: - raise InvalidWindowLength("window_length must be provided") - if not isinstance(window_length, Integral): - raise InvalidWindowLength( - "window_length must be an integer-like number") - if window_length == 0: - raise InvalidWindowLength("window_length must be non-zero") - if window_length < 0: - raise InvalidWindowLength("window_length must be positive") - - -class TransformMeta(type): - """ - Metaclass that automatically packages a class inside of - StatefulTransform on initialization. Specifically, if Foo is a - class with its __metaclass__ attribute set to TransformMeta, then - calling Foo(*args, **kwargs) will return StatefulTransform(Foo, - *args, **kwargs) instead of an instance of Foo. (Note that you can - still recover an instance of a "raw" Foo by introspecting the - resulting StatefulTransform's 'state' field.) - """ - - def __call__(cls, *args, **kwargs): - return StatefulTransform(cls, *args, **kwargs) - - -class StatefulTransform(object): - """ - Generic transform generator that takes each message from an - in-stream and passes it to a state object. For each call to - update, the state class must produce a message to be fed - downstream. Any transform class with the FORWARDER class variable - set to true will forward all fields in the original message. - Otherwise only dt, tnfm_id, and tnfm_value are forwarded. - """ - def __init__(self, tnfm_class, *args, **kwargs): - assert hasattr(tnfm_class, 'update'), \ - "Stateful transform requires the class to have an update method" - - # Create an instance of our transform class. - if isinstance(tnfm_class, TransformMeta): - # Classes derived TransformMeta have their __call__ - # attribute overridden. Since this is what is usually - # used to create an instance, we have to delegate the - # responsibility of creating an instance to - # TransformMeta's parent class, which is 'type'. This is - # what is implicitly done behind the scenes by the python - # interpreter for most classes anyway, but here we have to - # be explicit because we've overridden the method that - # usually resolves to our super call. - self.state = super(TransformMeta, tnfm_class).__call__( - *args, **kwargs) - # Normal object instantiation. - else: - self.state = tnfm_class(*args, **kwargs) - # save the window_length of the state for external access. - self.window_length = self.state.window_length - # Create the string associated with this generator's output. - self.namestring = tnfm_class.__name__ + hash_args(*args, **kwargs) - - def get_hash(self): - return self.namestring - - def transform(self, stream_in): - return self._gen(stream_in) - - def _gen(self, stream_in): - # IMPORTANT: Messages may contain pointers that are shared with - # other streams. Transforms that modify their input - # messages should only manipulate copies. - for message in stream_in: - # we only handle TRADE and CUSTOM events. - if hasattr(message, 'type') and \ - message.type not in (DATASOURCE_TYPE.TRADE, - DATASOURCE_TYPE.CUSTOM): - yield message - continue - # allow upstream generators to yield None to avoid - # blocking. - if message is None: - continue - - assert_sort_unframe_protocol(message) - - try: - tnfm_value = self.state.update(message) - except WrongDataForTransform: - # Transform classes should raise WrongDataForTransform if they - # are unable to process the event BEFORE performing any state - # modifications, because we continue the simulation if a - # WrongDataForTransform is raised on a CUSTOM event. - if message.type == DATASOURCE_TYPE.CUSTOM: - # Pass through custom events that are not applicable to - # this transform. - yield message - continue - else: - # If a TRADE event raises a WrongDataForTransform, - # something bad has happend. - raise - - out_message = message - out_message[self.namestring] = tnfm_value - yield out_message - - -class EventWindow(with_metaclass(ABCMeta)): - """ - Abstract base class for transform classes that calculate iterative - metrics on events within a given timedelta. Maintains a list of - events that are within a certain timedelta of the most recent - tick. Calls self.handle_add(event) for each event added to the - window. Calls self.handle_remove(event) for each event removed - from the window. Subclass these methods along with init(*args, - **kwargs) to calculate metrics over the window. - - If the market_aware flag is True, the EventWindow drops old events - based on the number of elapsed trading days between newest and oldest. - Otherwise old events are dropped based on a raw timedelta. - - See zipline/transforms/mavg.py and zipline/transforms/vwap.py for example - implementations of moving average and volume-weighted average - price. - """ - # Mark this as an abstract base class. - - def __init__(self, market_aware=True, window_length=None, delta=None): - - check_window_length(window_length) - self.window_length = window_length - - self.ticks = deque() - - # Only Market-aware mode is now supported. - if not market_aware: - raise UnsupportedEventWindowFlagValue( - "Non-'market aware' mode is no longer supported." - ) - if delta: - raise UnsupportedEventWindowFlagValue( - "delta values are no longer supported." - ) - # Set the behavior for dropping events from the back of the - # event window. - self.drop_condition = self.out_of_market_window - - @abstractmethod - def handle_add(self, event): - raise NotImplementedError() - - @abstractproperty - def fields(self): - raise NotImplementedError() - - @abstractmethod - def handle_remove(self, event): - raise NotImplementedError() - - def __len__(self): - return len(self.ticks) - - def update(self, event): - - self.assert_well_formed(event) - - # Add new event and increment totals. - self.ticks.append(event) - - # Subclasses should override handle_add to define behavior for - # adding new ticks. - self.handle_add(event) - # Clear out any expired events. - # - # oldest newest - # | | - # V V - while self.drop_condition(self.ticks[0].dt, self.ticks[-1].dt): - - # popleft removes and returns the oldest tick in self.ticks - popped = self.ticks.popleft() - - # Subclasses should override handle_remove to define - # behavior for removing ticks. - self.handle_remove(popped) - - def out_of_market_window(self, oldest, newest): - oldest_index = \ - trading.environment.trading_days.searchsorted(oldest) - newest_index = \ - trading.environment.trading_days.searchsorted(newest) - - trading_days_between = newest_index - oldest_index - - # "Put back" a day if oldest is earlier in its day than newest, - # reflecting the fact that we haven't yet completed the last - # day in the window. - if oldest.time() > newest.time(): - trading_days_between -= 1 - - return trading_days_between >= self.window_length - - def assert_well_formed(self, event): - """ - Verify that the supplied event contains all the fields required by this - EventWindow to be processed. - """ - self.check_required_fields(event) - - assert isinstance(event.dt, datetime), \ - "Bad dt in EventWindow:%s" % event - if len(self.ticks) > 0: - # Something is wrong if new event is older than previous. - assert event.dt >= self.ticks[-1].dt, \ - "Events arrived out of order in EventWindow: %s -> %s" % \ - (event, self.ticks[0]) - - def check_required_fields(self, event): - """ - We only allow events with all of our tracked fields. - """ - # All events require a 'dt' field. - if not hasattr(event, 'dt'): - raise WrongDataForTransform( - transform=self.__class__.__name__, - fields=['dt'], - ) - - # Subclasses must implement the 'fields' property to specify other - # required fields. - for field in self.fields: - if field not in event: - raise WrongDataForTransform( - transform=self.__class__.__name__, - fields=self.fields, - ) diff --git a/zipline/utils/simfactory.py b/zipline/utils/simfactory.py index 70fbcc9c..b7137007 100644 --- a/zipline/utils/simfactory.py +++ b/zipline/utils/simfactory.py @@ -23,8 +23,6 @@ def create_test_zipline(**config): Expects an object with a simulate mehod, such as :py:class:`zipline.gens.tradingsimulation.FixedSlippage`. :py:mod:`zipline.finance.trading` - - transforms: optional parameter that provides a list - of StatefulTransform objects. """ assert isinstance(config, dict) sid_list = config.get('sid_list') @@ -87,14 +85,6 @@ def create_test_zipline(**config): test_algo.benchmark_return_source = config.get('benchmark_source', None) - # ------------------- - # Transforms - # ------------------- - - transforms = config.get('transforms', None) - if transforms is not None: - test_algo.set_transforms(transforms) - # ------------------ # generator/simulator sim = test_algo.get_generator() diff --git a/zipline/utils/test_utils.py b/zipline/utils/test_utils.py index e29f5f7c..75faff62 100644 --- a/zipline/utils/test_utils.py +++ b/zipline/utils/test_utils.py @@ -109,19 +109,6 @@ class ExceptionSource(object): 5 / 0 -class ExceptionTransform(object): - - def __init__(self): - self.window_length = 1 - pass - - def get_hash(self): - return "ExceptionTransform" - - def update(self, event): - assert False, "An assertion message" - - @contextmanager def nullctx(): """