From cdb383d99935aba8b5b20d2fa17de6d6556fca15 Mon Sep 17 00:00:00 2001 From: Stephen Diehl Date: Thu, 8 Mar 2012 17:18:29 -0500 Subject: [PATCH 1/3] Backtest protocol, ala Bredeche. --- zipline/component.py | 15 +++++++++++++-- zipline/protocol.py | 21 +++++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/zipline/component.py b/zipline/component.py index 4595c192..db9b7bc8 100644 --- a/zipline/component.py +++ b/zipline/component.py @@ -14,7 +14,8 @@ import humanhash from datetime import datetime import zipline.util as qutil -from zipline.protocol import CONTROL_PROTOCOL, COMPONENT_STATE +from zipline.protocol import CONTROL_PROTOCOL, COMPONENT_STATE, \ + COMPONENT_FAILURE, BACKTEST_STATE class Component(object): """ @@ -66,6 +67,7 @@ class Component(object): self.controller = None self.heartbeat_timeout = 2000 self.state_flag = COMPONENT_STATE.OK + self.error_state = COMPONENT_FAILURE.NOFAILURE self.on_done = None self._exception = None @@ -254,8 +256,17 @@ class Component(object): # Internal Maintenance # ---------------------- - def signal_exception(self, exc=None): + def signal_exception(self, exc=None, scope=None): + + if scope == 'algo': + self.error_state = COMPONENT_FAILURE.ALGOEXCEPT + else: + self.error_state = COMPONENT_FAILURE.HOSTEXCEPT + self.state_flag = COMPONENT_STATE.EXCEPTION + # mark the time of failure so we can track the failure + # progogation through the system. + self.stop_tic = time.time() self._exception = exc diff --git a/zipline/protocol.py b/zipline/protocol.py index 89726fe6..22e2547b 100644 --- a/zipline/protocol.py +++ b/zipline/protocol.py @@ -274,6 +274,27 @@ COMPONENT_STATE = Enum( 'EXCEPTION' , # 2 ) +# NOFAILURE - Component is either not running or has not failed +# ALGOEXCEPT - Exception thrown in the given algorithm +# HOSTEXCEPT - Exception thrown on our end. +# INTERRUPT - Manually interuptted by user + +COMPONENT_FAILURE = Enum( + 'NOFAILURE' , + 'ALGOEXCEPT' , + 'HOSTEXCEPT' , + 'INTERRUPT' , +) + +BACKTEST_STATE = Enum( + 'IDLE' , + 'QUEUED' , + 'INPROGRESS' , + 'CANCELLED' , # cancelled ( before natural completion ) + 'EXCEPTION' , # failure ( due to unnatural causes ) + 'DONE' , # done ( naturally completed ) +) + # ================== # Datasource Protocol # ================== From 4df88d5a503d08ac89151500f0d31200fa303a6a Mon Sep 17 00:00:00 2001 From: Stephen Diehl Date: Thu, 8 Mar 2012 18:07:43 -0500 Subject: [PATCH 2/3] Move protocol data structures. --- zipline/protocol.py | 85 +------------------------------------- zipline/protocol_utils.py | 87 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 83 deletions(-) create mode 100644 zipline/protocol_utils.py diff --git a/zipline/protocol.py b/zipline/protocol.py index 22e2547b..7286ce1d 100644 --- a/zipline/protocol.py +++ b/zipline/protocol.py @@ -118,94 +118,13 @@ import msgpack import numbers import datetime import pytz -import copy from collections import namedtuple -import zipline.util as qutil +from protocol_utils import Enum, FrameExceptionFactory, namedict + #import ujson #import ultrajson_numpy -from ctypes import Structure, c_ubyte - -def Enum(*options): - """ - Fast enums are very important when we want really tight zmq - loops. These are probably going to evolve into pure C structs - anyways so might as well get going on that. - """ - class cstruct(Structure): - _fields_ = [(o, c_ubyte) for o in options] - return cstruct(*range(len(options))) - -def FrameExceptionFactory(name): - """ - Exception factory with a closure around the frame class name. - """ - class InvalidFrame(Exception): - def __init__(self, got): - self.got = got - - def __str__(self): - return "Invalid {framecls} Frame: {got}".format( - framecls = name, - got = self.got, - ) - - return InvalidFrame - -class namedict(object): - """ - So that you can use:: - - foo.BAR - -- or -- - foo['BAR'] - - For more complex structs use collections.namedtuple: - """ - - def __init__(self, dct=None): - if(dct): - self.__dict__.update(dct) - - def __setitem__(self, key, value): - """ - Required for use by pymongo as_class parameter to find. - """ - if(key == '_id'): - self.__dict__['id'] = value - else: - self.__dict__[key] = value - - def __getitem__(self, key): - return self.__dict__[key] - - def keys(self): - return self.__dict__.keys() - - def as_dict(self): - # shallow copy is O(n) - return copy.copy(self.__dict__) - - def delete(self, key): - del(self.__dict__[key]) - - def merge(self, other_nd): - assert isinstance(other_nd, namedict) - self.__dict__.update(other_nd.__dict__) - - def __repr__(self): - return "namedict: " + str(self.__dict__) - - def __eq__(self, other): - # !!!!!!!!!!!!!!!!!!!! - # !!!! DANGEROUS !!!!! - # !!!!!!!!!!!!!!!!!!!! - return other != None and self.__dict__ == other.__dict__ - - def has_attr(self, name): - return self.__dict__.has_key(name) - # ================ # Control Protocol # ================ diff --git a/zipline/protocol_utils.py b/zipline/protocol_utils.py new file mode 100644 index 00000000..0e6cb858 --- /dev/null +++ b/zipline/protocol_utils.py @@ -0,0 +1,87 @@ +import copy +from ctypes import Structure, c_ubyte + +def Enum(*options): + """ + Fast enums are very important when we want really tight zmq + loops. These are probably going to evolve into pure C structs + anyways so might as well get going on that. + """ + class cstruct(Structure): + _fields_ = [(o, c_ubyte) for o in options] + return cstruct(*range(len(options))) + +def FrameExceptionFactory(name): + """ + Exception factory with a closure around the frame class name. + """ + class InvalidFrame(Exception): + def __init__(self, got): + self.got = got + + def __str__(self): + return "Invalid {framecls} Frame: {got}".format( + framecls = name, + got = self.got, + ) + + return InvalidFrame + +class namedict(object): + """ + + Namedicts are dict like objects that have fields accessible by attribute lookup + as well as being indexable and iterable:: + + HEARTBEAT_PROTOCOL = namedict({ + 'REQ' : b'\x01', + 'REP' : b'\x02', + }) + + HEARTBEAT_PROTOCOL.REQ # syntactic sugar + HEARTBEAT_PROTOCOL.REP # oh suga suga + + For more complex structs use collections.namedtuple: + """ + + def __init__(self, dct=None): + if(dct): + self.__dict__.update(dct) + + def __setitem__(self, key, value): + """ + Required for use by pymongo as_class parameter to find. + """ + if(key == '_id'): + self.__dict__['id'] = value + else: + self.__dict__[key] = value + + def __getitem__(self, key): + return self.__dict__[key] + + def keys(self): + return self.__dict__.keys() + + def as_dict(self): + # shallow copy is O(n) + return copy.copy(self.__dict__) + + def delete(self, key): + del(self.__dict__[key]) + + def merge(self, other_nd): + assert isinstance(other_nd, namedict) + self.__dict__.update(other_nd.__dict__) + + def __repr__(self): + return "namedict: " + str(self.__dict__) + + def __eq__(self, other): + # !!!!!!!!!!!!!!!!!!!! + # !!!! DANGEROUS !!!!! + # !!!!!!!!!!!!!!!!!!!! + return other != None and self.__dict__ == other.__dict__ + + def has_attr(self, name): + return self.__dict__.has_key(name) From aba06d40e988c6648f9466f9a3b1a60431ed88b3 Mon Sep 17 00:00:00 2001 From: Stephen Diehl Date: Tue, 13 Mar 2012 11:03:37 -0400 Subject: [PATCH 3/3] Added zmq debug console for great good. --- zipline/zmq_utils.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 zipline/zmq_utils.py diff --git a/zipline/zmq_utils.py b/zipline/zmq_utils.py new file mode 100644 index 00000000..e7c09047 --- /dev/null +++ b/zipline/zmq_utils.py @@ -0,0 +1,16 @@ +import gevent +from gevent_zeromq import zmq + +def ZmqConsole(sock_typ, socket_addr, sock_conn=None, context=None): + + context = context or zmq.Context.instance() + socket = context.socket(zmq.PULL) + socket.connect('tcp://127.0.0.1:3141') + + def console(): + while True: + msg = socket.recv() + print msg + import pdb; pdb.set_trace() + + return gevent.spawn(console)