From 7bb4e754b04d0549669ee7d9062e6cfb470d12a1 Mon Sep 17 00:00:00 2001 From: scottsanderson Date: Thu, 9 Aug 2012 16:21:49 -0400 Subject: [PATCH 1/3] added logs --- zipline/gens/sort.py | 8 +++++++- zipline/gens/transform.py | 7 ++++++- zipline/protocol.py | 6 ++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/zipline/gens/sort.py b/zipline/gens/sort.py index 9755da74..c76db032 100644 --- a/zipline/gens/sort.py +++ b/zipline/gens/sort.py @@ -1,12 +1,16 @@ """ -Generator version of Feed. +Sorting generator. """ +import logbook + from collections import deque from zipline import ndict from zipline.gens.utils import \ assert_datasource_unframe_protocol, \ assert_sort_protocol +log = logbook.Logger('Sorting') + def date_sort(stream_in, source_ids): """ A generator that takes a generator and a list of source_ids. We @@ -23,6 +27,7 @@ def date_sort(stream_in, source_ids): sources[id] = deque() # Process incoming streams. + log.info('Sorting first message') for message in stream_in: # Incoming messages should be the output of DATASOURCE_UNFRAME. assert_datasource_unframe_protocol(message), \ @@ -46,6 +51,7 @@ def date_sort(stream_in, source_ids): assert len(queue) == 1, "Bad queue in date_sort on exit: %s" % queue assert queue[0].dt == "DONE", \ "Bad last message in date_sort on exit: %s" % queue + log.info('Successfully finished Sorting') def ready(sources): """ diff --git a/zipline/gens/transform.py b/zipline/gens/transform.py index 651c337d..babf4e3d 100644 --- a/zipline/gens/transform.py +++ b/zipline/gens/transform.py @@ -3,6 +3,7 @@ Generator versions of transforms. """ import types import pytz +import logbook from copy import deepcopy from datetime import datetime, timedelta @@ -15,6 +16,8 @@ from zipline.utils.tradingcalendar import trading_days_between from zipline.gens.utils import assert_sort_unframe_protocol, \ assert_transform_protocol, hash_args +log = logbook.Logger('Transform') + class Passthrough(object): FORWARDER = True """ @@ -72,6 +75,7 @@ class StatefulTransform(object): # Create the string associated with this generator's output. self.namestring = tnfm_class.__name__ + hash_args(*args, **kwargs) + log.info('StatefulTransform [%s] initialized' % self.namestring) def get_hash(self): return self.namestring @@ -82,7 +86,7 @@ class StatefulTransform(object): def _gen(self, stream_in): # IMPORTANT: Messages may contain pointers that are shared with # other streams, so we only manipulate copies. - + log.info('Running StatefulTransform [%s]' % self.get_hash()) for message in stream_in: # allow upstream generators to yield None to avoid @@ -143,6 +147,7 @@ class StatefulTransform(object): out_message.dt = message_copy.dt yield out_message + log.info('Finished StatefulTransform [%s]' % self.get_hash()) class EventWindow: """ Abstract base class for transform classes that calculate iterative diff --git a/zipline/protocol.py b/zipline/protocol.py index dd46bd60..fce16403 100644 --- a/zipline/protocol.py +++ b/zipline/protocol.py @@ -654,7 +654,13 @@ def tuple_to_date(date_tuple): dt = dt.replace(microsecond = micros, tzinfo = pytz.utc) return dt +# Datasource type should completely determine the other fields of a +# message with its type. DATASOURCE_TYPE = Enum( + 'AS_TRADED_EQUITY', + 'MERGE', + 'SPLIT', + 'DIVIDEND', 'TRADE', 'EMPTY', 'DONE' From 4bfbaa8c26853f54f794ffb8fed0dee3bbd6bae0 Mon Sep 17 00:00:00 2001 From: scottsanderson Date: Fri, 10 Aug 2012 01:18:55 -0400 Subject: [PATCH 2/3] minor protocol change --- zipline/gens/transform.py | 3 ++- zipline/protocol.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/zipline/gens/transform.py b/zipline/gens/transform.py index babf4e3d..60d0be85 100644 --- a/zipline/gens/transform.py +++ b/zipline/gens/transform.py @@ -105,7 +105,8 @@ class StatefulTransform(object): # FORWARDER flag means we want to keep all original # values, plus append tnfm_id and tnfm_value. Used for # preserving the original event fields when our output - # will be fed into a merge. + # will be fed into a merge. Currently only Passthrough + # uses this flag. if self.forward_all: out_message = message_copy out_message.tnfm_id = self.namestring diff --git a/zipline/protocol.py b/zipline/protocol.py index fce16403..19ee8c80 100644 --- a/zipline/protocol.py +++ b/zipline/protocol.py @@ -658,7 +658,7 @@ def tuple_to_date(date_tuple): # message with its type. DATASOURCE_TYPE = Enum( 'AS_TRADED_EQUITY', - 'MERGE', + 'MERGER', 'SPLIT', 'DIVIDEND', 'TRADE', From 0506baeae59f0272532f90dc7cca3a2d36a64be4 Mon Sep 17 00:00:00 2001 From: fawce Date: Fri, 10 Aug 2012 20:59:53 -0400 Subject: [PATCH 3/3] fixed integration tests --- zipline/gens/tradesimulation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zipline/gens/tradesimulation.py b/zipline/gens/tradesimulation.py index 5e56b4f0..a049a5b8 100644 --- a/zipline/gens/tradesimulation.py +++ b/zipline/gens/tradesimulation.py @@ -87,7 +87,7 @@ class TradeSimulationClient(object): self.sids ) with_portfolio = perf_tracker.transform(with_filled_orders) - + # Pass the messages from perf along with the trading client's # state into the algorithm for simulation. We provide the # trading client so that the algorithm can place new orders @@ -97,7 +97,7 @@ class TradeSimulationClient(object): ordering_client.state, self.algo, ) - + # The algorithm will yield a daily_results message (as # calculated by the performance tracker) at the end of each # day. It will also yield a risk report at the end of the