import inspect import os import zmq import sys import time import itertools import logbook from setproctitle import setproctitle from signal import SIGHUP, SIGINT from collections import OrderedDict, Counter from zipline.protocol import CONTROL_PROTOCOL, CONTROL_FRAME, \ CONTROL_UNFRAME, CONTROL_STATES, INVALID_CONTROL_FRAME \ from zipline.utils.protocol_utils import ndict INIT, SOURCES_READY, RUNNING, TERMINATE = CONTROL_STATES CONTROLLER_TRANSITIONS = frozenset([ (-1 , INIT), (INIT , SOURCES_READY), (SOURCES_READY , RUNNING), (INIT , TERMINATE), # pseudo failure mode (SOURCES_READY , TERMINATE), # pseudo failure mode (RUNNING , TERMINATE), ]) class UnknownChatter(Exception): def __init__(self, name): self.named = name def __str__(self): return """Component calling itself "%s" talking on unexpected channel""" % self.named log = logbook.Logger('Controller') # The scalars determining the timing of the monitor behavior for # the system. PARAMETERS = ndict(dict( # time Monitor will wait for a heartbeat, in seconds GENERATIONAL_PERIOD = 10, # time Component will wait for GO and for a heartbeat before # timing out. MAX_COMPONENT_WAIT = 20, ALLOWED_SKIPPED_HEARTBEATS = 10, ALLOWED_INVALID_HEARTBEATS = 3, PRESTART_HEARBEATS = 3, SOURCES_START_HEARTBEATS = 3, SYSTEM_TIMEOUT = 50, )) class Controller(object): """ A N to M messaging system for inter component communication. :param pub_socket: Socket to publish messages, the starting point of :func message_listener: . :param route_socket: Socket to listen for status updates for the individual components. :func message_sender: . """ # Turn on debug for verbose logging of the system. debug = True period = PARAMETERS.GENERATIONAL_PERIOD def __init__(self, pub_socket, route_socket): self.nosignals = False self.context = None self.zmq = None self.zmq_poller = None self.running = False self.alive = False self.tracked = set() self.finished = set() self.responses = set() self.ctime = 0 self.tic = time.time() self.freeform = False self._state = -1 self.associated = [] self.pub_socket = pub_socket self.route_socket = route_socket self.error_replay = OrderedDict() self.missed_beats = Counter() # if we are inside a test, we want to skip signalling # back to the parent process. self.inside_test = 'nose' in inspect.stack()[-1][1] def init_zmq(self): self.zmq = zmq self.context = self.zmq.Context() self.zmq_poller = self.zmq.Poller return def manage(self, topology): """ Give the controller a set set of components to manage and a set of state transitions for the entire system. """ # A freeform topology is where we heartbeat with anything # that shows up. if topology == 'freeform': self.freeform = True self.topology = frozenset([]) else: self.freeform = False self.topology = frozenset(topology) self.alive = True @property def state(self): #log.info('returned %s' % self._state) return self._state @state.setter def state(self, new): old = self._state if (old, new) in CONTROLLER_TRANSITIONS: self._state = new log.info("State Transition : %s -> %s" % (old, self._state)) else: raise RuntimeError("Invalid State Transition : %s -> %s" %(old, new)) def run(self): self.running = True self.init_zmq() setproctitle('Monitor') self.state = CONTROL_STATES.INIT # Interpreter SIDE EFFECT # ----------------------- # The last breathe of the interpreter will assume that we've # failed unless we specify otherwise. log.info('registering exit function') sys.exitfunc = self.signal_interrupt # We overload this if ( and only if ) the topology exits # cleanly. This prevents failure modes where the monitor # dies. try: return self._poll() # use a python loop except KeyboardInterrupt: log.info('Shutdown event loop') def log_status(self): """ Snapshot of the tracked components at every period. """ #log.info("Tracking component : %s" % ([c for c in self.tracked],)) pass def replay_errors(self): """ Replay the errors in the order they were reported to the controller. """ return [ a for a in sorted(self.replay_errors.keys())] # ------------- # Publications # ------------- def send_go(self): go_frame = CONTROL_FRAME( CONTROL_PROTOCOL.GO, '' ) self.pub.send(go_frame) def send_heart(self): if not self.running: return heartbeat_frame = CONTROL_FRAME( CONTROL_PROTOCOL.HEARTBEAT, str(self.ctime) ) self.pub.send(heartbeat_frame) def send_hardkill(self): if not self.running: return kill_frame = CONTROL_FRAME( CONTROL_PROTOCOL.KILL, '' ) self.pub.send(kill_frame) def send_softkill(self): if not self.running: return soft_frame = CONTROL_FRAME( CONTROL_PROTOCOL.SHUTDOWN, '' ) self.pub.send(soft_frame) # ----------- # Event Loops # ----------- def _poll(self): assert self.route_socket assert self.pub_socket assert self.topology,\ """"Must define topology to monitor, call setup_controller() on your Zipline. """ # -- Publish -- # ============= self.pub = self.context.socket(self.zmq.PUB) self.pub.bind(self.pub_socket) self.pub.setsockopt(zmq.LINGER, 0) # -- Router -- # ============= self.router = self.context.socket(self.zmq.ROUTER) self.router.bind(self.route_socket) self.router.setsockopt(zmq.LINGER, 0) poller = self.zmq.Poller() poller.register(self.router, self.zmq.POLLIN) #poller.register(self.cancel, self.zmq.POLLIN) self.associated += [self.pub, self.router] # TODO: actually do this self.state = CONTROL_STATES.SOURCES_READY self.state = CONTROL_STATES.RUNNING buffer = [] # =================== # Heartbeat Iteration # =================== for i in itertools.count(0): self.log_status() # Reset the responses for this cycle self.responses = set() # broadcast the heartbeat packet self.ctime = time.time() self.send_heart() # ============== # Hearbeat Cycle # ============== initializing = len(self.tracked) == 0 and len(self.finished) == 0 # Wait the responses while self.alive: socks = dict(poller.poll(0)) tic = time.time() if socks.get(self.router) == self.zmq.POLLIN: rawmessage = self.router.recv() if rawmessage: buffer.append(rawmessage) try: if not self.router.getsockopt(self.zmq.RCVMORE): self.handle_recv(buffer[:]) buffer = [] except INVALID_CONTROL_FRAME: log.error('Invalid frame', rawmessage) pass # We break out of this loop if the time between # sending and receiving the heartbeat is more # than our poll period. if tic - self.ctime > self.period: log.info("heartbeat loop timedout: %s" % (tic - self.ctime)) log.info(repr(self.responses)) break # if this is the first time heartbeating, break # out early if we get everything tracked no need # to hold out for the full heartbeat. if initializing and not self.freeform: if len(self.responses) == len(self.topology): log.info("breaking out of initial heartbeat") break # ================ # Heartbeat Stats # ================ complete = self.beat() # ================ # Topology Status # ================ # Has the entire topology told us its DONE done = len(self.finished) == len(self.topology) # Has the entire topology shown up to the party complete = len(self.tracked) == len(self.topology) if complete: self.send_go() log.info('Heartbeat (%s, %s)' % (done, complete)) # ================ # Exit Strategies # ================ # Will also fall out of loop when done, if using # non-freeform topology if done: log.info('Entire topology exited cleanly') self.shutdown() # Noop exit func #sys.exitfunc = lambda: None # Send SIGHUP to buritto self.signal_hangup() if not self.alive: log.info('Breaking out of Monitor Loop') break def signal_hangup(self): """ A clean exit, inform the burrito ( and arbiter ) that we're good. The topology exited cleanly and we can prove it. """ if self.inside_test: log.warning("Skipping SIGHUP because we're in a nosetest") return ppid = os.getppid() log.warning("Sending SIGHUP") os.kill(ppid, SIGHUP) def signal_interrupt(self): """ Send a SIGINT in the error mode that the monitor's interpreter exits. If the monitor dies the system is considered a failure. """ ppid = os.getpid() os.kill(ppid, SIGINT) def beat(self): """ The tracking logic of the system. It's the "stethoscope" that inspects to the heartbeats in a generation and infers the state of the system from the responses. """ # These the set overloaded operations # A & B ~ set.intersection # A - B ~ set.difference # * good - Components we are currently tracking and who just sent # us back the right response. # * bad - Components we are currently tracking but who did not # send us back a response. # * new - Components we haven't heard from yet, but sent back the # right response. # * finished - Components we were tracking but have now # finished, when this set goes to zero this # triggers the end of the topology. good = self.tracked & self.responses bad = self.tracked - good - self.finished new = self.responses - good - self.finished missing = self.topology - self.tracked - self.finished for component in new: self.new(component) if self.debug: log.info('New component %r' % component) for component in bad: self.fail(component) for component in missing: if self.debug: log.info('Missing component %r' % component) if self.debug: for component in self.tracked: if component not in self.topology: log.info('Uninvited component %r' % component) # -------------- # Init Handlers # -------------- def new_universal(self): pass # The various "states of being that a component can inform us # of def new(self, component): if self.state is CONTROL_STATES.TERMINATE: return if component in self.finished: #log.info("Got heartbeat from supposedly finished component") return log.info('Now Tracking "%s" ' % component) universal = self.new_universal init_handlers = {} if component in (self.topology - self.finished) or self.freeform: init_handlers.get(component, universal)() self.tracked.add(component) else: # Some sort of socket collision has occurred, this is # a very bad failure mode. raise UnknownChatter(component) # ------------------ # Epic Fail Handling # ------------------ def fail_universal(self): # TODO: this requires higher order functionality log.error('System in exception state, shutting down') self.shutdown() def fail(self, component): if self.state is CONTROL_STATES.TERMINATE: return universal = self.fail_universal fail_handlers = { } if component in (self.topology - self.finished) or self.freeform: log.warning('Component "%s" missed heartbeat' % component) self.tracked.remove(component) fail_handlers.get(component, universal)() # ------------------- # Completion Handling # ------------------- def done(self, component): self.finished.add(component) self.tracked.discard(component) log.info('Component "%s" finished.' % component) # -------------- # Error Handling # -------------- def exception_universal(self): """ Shutdown the system on failure. """ log.error('System in exception state, shutting down') self.kill() def exception(self, component, failure): universal = self.exception_universal exception_handlers = { } if component in self.topology or self.freeform: self.error_replay[(component, time.time())] = failure log.error('Component in exception state: %s' % component) exception_handlers.get(component, universal)() else: raise UnknownChatter(component) # ----------------- # Protocol Handling # ----------------- def handle_recv(self, msg): """ Check for proper framing at the transport layer. Seperates the proper frames from anything else that might be coming over the wire. """ identity = msg[0] # identity of the socket id, status = CONTROL_UNFRAME(msg[1]) # I'm alive, condemned to be a free process in the cold # cold dark absurd Zipline universe. if id is CONTROL_PROTOCOL.READY: self.responses.add(identity) return # The heartbeat love song between a component and the # controller if id is CONTROL_PROTOCOL.OK: if status == str(self.ctime): # Go to your bosom; knock there, and ask your heart what # it doth know... self.responses.add(identity) elif float(status) < self.ctime: # False face must hide what the false heart doth know. log.warning('Delayed heartbeat received: %s' % msg) elif float(status) > self.ctime: # Pre-emptive heartbeat from the component # log.info("pre-emptive pong: %s" % msg) self.responses.add(identity) else: # Otherwise its something weird and we don't know # what to do so just say so, probably line noise # from ZeroMQ # What's in a name? that which we call a rose... log.error("Weird heartbeat packet happened: %s" % msg) return # A component is telling us it failed, and how if id is CONTROL_PROTOCOL.EXCEPTION: self.exception(identity, status) return # A component is telling us its done with work and won't # be talking to us anymore if id is CONTROL_PROTOCOL.DONE: self.done(identity) return # ------------------- # Hooks for Endpoints # ------------------- # These are all connects so no complex allocation logic is # needed. Dealers and Subscribers can all come and go as a # function of time without impacting flow of the whole # system. def message_sender(self, identity, context = None): """ Spin off a socket used for sending messages to this controller. """ if not context: context = self.zmq.Context.instance() s = context.socket(zmq.DEALER) s.setsockopt(zmq.IDENTITY, identity) s.connect(self.route_socket) self.associated.append(s) return s def message_listener(self, context = None): """ Spin off a socket used for receiving messages from this controller. """ if not context: context = self.zmq.Context.instance() s = context.socket(zmq.SUB) s.connect(self.pub_socket) s.setsockopt(zmq.SUBSCRIBE, '') self.associated.append(s) return s def do_error_replay(self): for (component, time), error in self.error_replay.iteritems(): log.info('Component Log for -- %s --:\n%s' % (component, error)) def kill(self): if self.state is CONTROL_STATES.TERMINATE: return log.info('Hard Shutdown') self.send_hardkill() self.state = CONTROL_STATES.TERMINATE self.alive = False def shutdown(self): if self.state is CONTROL_STATES.TERMINATE: return log.info('Soft Shutdown') self.send_softkill() self.state = CONTROL_STATES.TERMINATE self.alive = False