""" Commonly used messaging components. """ import datetime from collections import Counter import zipline.util as qutil from zipline.component import Component import zipline.protocol as zp from zipline.protocol import CONTROL_PROTOCOL, COMPONENT_TYPE, \ COMPONENT_STATE, CONTROL_FRAME, CONTROL_UNFRAME class ComponentHost(Component): """ Components that can launch multiple sub-components, synchronize their start, and then wait for all components to be finished. """ def __init__(self, addresses): Component.__init__(self) self.addresses = addresses self.running = False self.init() def init(self): assert hasattr(self, 'zmq_flavor'), \ """ You must specify a flavor of ZeroMQ for all ComponentHost subclasses. """ # Component Registry, keyed by get_id # ---------------------- self.components = {} # ---------------------- # Internal Registry, keyed by guid self._components = {} # ---------------------- self.sync_register = {} self.timeout = datetime.timedelta(seconds=60) self.feed = Feed() self.merge = Merge() self.passthrough = PassthroughTransform() self.controller = None #register the feed and the merge self.register_components([self.feed, self.merge, self.passthrough]) def register_controller(self, controller): """ Add the given components to the registry. Establish communication with them. """ if self.controller != None: raise Exception("There can be only one!") self.controller = controller self.controller.zmq_flavor = self.zmq_flavor # Propogate the controller to all the subcomponents for component in self.components.itervalues(): component.controller = controller def register_components(self, components): """ Add the given components to the registry. Establish communication with them. """ assert isinstance(components, list) for component in components: component.addresses = self.addresses component.controller = self.controller # Hosts share their zmq flavor with hosted components component.zmq_flavor = self.zmq_flavor self._components[component.guid] = component self.components[component.get_id] = component self.sync_register[component.get_id] = datetime.datetime.utcnow() if isinstance(component, DataSource): self.feed.add_source(component.get_id) if isinstance(component, BaseTransform): self.merge.add_source(component.get_id) def unregister_component(self, component_id): del self.components[component_id] del self.sync_register[component_id] def setup_sync(self): """ Setup the sync socket and poller. ( Bind ) """ qutil.LOGGER.debug("Connecting sync server.") self.sync_socket = self.context.socket(self.zmq.REP) self.sync_socket.bind(self.addresses['sync_address']) self.sync_poller = self.zmq_poller() self.sync_poller.register(self.sync_socket, self.zmq.POLLIN) self.sockets.append(self.sync_socket) def open(self): for component in self.components.values(): self.launch_component(component) self.launch_controller() def is_running(self): """ DEPRECATED, left in for compatability for now. """ cur_time = datetime.datetime.utcnow() if len(self.components) == 0: qutil.LOGGER.info("Component register is empty.") return False return True def loop(self, lockstep=True): while self.is_running(): # wait for synchronization request at start, and DONE at end. # don't timeout. socks = dict(self.sync_poller.poll()) if self.sync_socket in socks and socks[self.sync_socket] == self.zmq.POLLIN: msg = self.sync_socket.recv() try: parts = msg.split(':') sync_id, status = parts except ValueError as exc: self.signal_exception(exc) if status == str(CONTROL_PROTOCOL.DONE): # TODO: other way around #qutil.LOGGER.debug("{id} is DONE".format(id=sync_id)) self.unregister_component(sync_id) self.state_flag = COMPONENT_STATE.DONE else: self.sync_register[sync_id] = datetime.datetime.utcnow() #qutil.LOGGER.info("confirmed {id}".format(id=msg)) # send synchronization reply self.sync_socket.send('ack', self.zmq.NOBLOCK) # ------------------ # Simulation Control # ------------------ def launch_controller(self, controller): raise NotImplementedError def launch_component(self, component): raise NotImplementedError def teardown_component(self, component): raise NotImplementedError class Feed(Component): """ Connects to N PULL sockets, publishing all messages received to a PUB socket. Published messages are guaranteed to be in chronological order based on message property dt. Expects to be instantiated in one execution context (thread, process, etc) and run in another. """ def __init__(self): Component.__init__(self) self.sent_count = 0 self.received_count = 0 self.draining = False self.ds_finished_counter = 0 # Depending on the size of this, might want to use a data # structure with better asymptotics. self.data_buffer = {} # source_id -> integer count self.sent_counters = Counter() self.recv_counters = Counter() def init(self): pass @property def get_id(self): return "FEED" @property def get_type(self): return COMPONENT_TYPE.CONDUIT # ------------- # Core Methods # ------------- def open(self): self.pull_socket = self.bind_data() self.feed_socket = self.bind_feed() def do_work(self): # wait for synchronization reply from the host socks = dict(self.poll.poll(self.heartbeat_timeout)) # TODO: Abstract this out, maybe on base component if self.control_in in socks and socks[self.control_in] == self.zmq.POLLIN: msg = self.control_in.recv() event, payload = CONTROL_UNFRAME(msg) # -- Heartbeat -- if event == CONTROL_PROTOCOL.HEARTBEAT: # Heart outgoing heartbeat_frame = CONTROL_FRAME( CONTROL_PROTOCOL.OK, payload ) self.control_out.send(heartbeat_frame) # -- Soft Kill -- elif event == CONTROL_PROTOCOL.SHUTDOWN: self.done() self.shutdown() # -- Hard Kill -- elif event == CONTROL_PROTOCOL.KILL: self.kill() if self.pull_socket in socks and socks[self.pull_socket] == self.zmq.POLLIN: message = self.pull_socket.recv() if message == str(CONTROL_PROTOCOL.DONE): self.ds_finished_counter += 1 if len(self.data_buffer) == self.ds_finished_counter: #drain any remaining messages in the buffer qutil.LOGGER.debug("draining feed") self.drain() self.signal_done() else: try: event = self.unframe(message) # deserialization error except zp.INVALID_DATASOURCE_FRAME as exc: return self.signal_exception(exc) try: self.append(event) self.send_next() # Invalid message except zp.INVALID_DATASOURCE_FRAME as exc: return self.signal_exception(exc) def unframe(self, msg): return zp.DATASOURCE_UNFRAME(msg) def frame(self, event): return zp.FEED_FRAME(event) # ------------- # Flow Control # ------------- def drain(self): """ Send all messages in the buffer. """ self.draining = True while self.pending_messages() > 0: self.send_next() def send_next(self): """ Send the (chronologically) next message in the buffer. """ if not (self.is_full() or self.draining): return event = self.next() if(event != None): self.feed_socket.send(self.frame(event), self.zmq.NOBLOCK) self.sent_counters[event.source_id] += 1 self.sent_count += 1 def append(self, event): """ Add an event to the buffer for the source specified by source_id. """ self.data_buffer[event.source_id].append(event) self.recv_counters[event.source_id] += 1 self.received_count += 1 def next(self): """ Get the next message in chronological order. """ if not(self.is_full() or self.draining): return cur_source = None earliest_source = None earliest_event = None #iterate over the queues of events from all sources #(1 queue per datasource) for events in self.data_buffer.values(): if len(events) == 0: continue cur_source = events first_in_list = events[0] if first_in_list.dt == None: #this is a filler event, discard events.pop(0) continue if (earliest_event == None) or (first_in_list.dt <= earliest_event.dt): earliest_event = first_in_list earliest_source = cur_source if earliest_event != None: return earliest_source.pop(0) def is_full(self): """ Indicates whether the buffer has messages in buffer for all un-DONE, blocking sources. """ for source_id, events in self.data_buffer.iteritems(): if len(events) == 0: return False return True def pending_messages(self): """ Returns the count of all events from all sources in the buffer. """ total = 0 for events in self.data_buffer.values(): total += len(events) return total def add_source(self, source_id): """ Add a data source to the buffer. """ self.data_buffer[source_id] = [] def __len__(self): """ Buffer's length is same as internal map holding separate sorted arrays of events keyed by source id. """ return len(self.data_buffer) class Merge(Feed): """ Merges multiple streams of events into single messages. """ def __init__(self): Feed.__init__(self) self.init() def init(self): pass @property def get_id(self): return "MERGE" @property def get_type(self): return COMPONENT_TYPE.CONDUIT def open(self): self.pull_socket = self.bind_merge() self.feed_socket = self.bind_result() def next(self): """Get the next merged message from the feed buffer.""" if not (self.is_full() or self.draining): return if self.pending_messages() == 0: return # #get the raw event from the passthrough transform. result = self.data_buffer[zp.TRANSFORM_TYPE.PASSTHROUGH].pop(0).PASSTHROUGH for source, events in self.data_buffer.iteritems(): if source == zp.TRANSFORM_TYPE.PASSTHROUGH: continue if len(events) > 0: cur = events.pop(0) result.merge(cur) return result def unframe(self, msg): return zp.TRANSFORM_UNFRAME(msg) def frame(self, event): return zp.MERGE_FRAME(event) def append(self, event): """ :param event: a namedict with one entry. key is the name of the transform, value is the transformed value. Add an event to the buffer for the source specified by source_id. """ self.data_buffer[event.keys()[0]].append(event) self.received_count += 1 class BaseTransform(Component): """ Top level execution entry point for the transform - connects to the feed socket to subscribe to events - connects to the result socket (most oftened bound by a TransformsMerge) to PUSH transforms - processes all messages received from feed, until DONE message received - pushes all transforms - sends DONE to result socket, closes all sockets and context Parent class for feed transforms. Subclass and override transform method to create a new derived value from the combined feed. """ def __init__(self, name): Component.__init__(self) self.state = { 'name': name } self.init() def init(self): pass @property def get_id(self): return self.state['name'] @property def get_type(self): return COMPONENT_TYPE.CONDUIT def open(self): """ Establishes zmq connections. """ #create the feed. self.feed_socket = self.connect_feed() #create the result PUSH self.result_socket = self.connect_merge() def do_work(self): """ Loops until feed's DONE message is received: - receive an event from the data feed - call transform (subclass' method) on event - send the transformed event """ socks = dict(self.poll.poll(self.heartbeat_timeout)) # TODO: Abstract this out, maybe on base component if self.control_in in socks and socks[self.control_in] == self.zmq.POLLIN: msg = self.control_in.recv() event, payload = CONTROL_UNFRAME(msg) # -- Heartbeat -- if event == CONTROL_PROTOCOL.HEARTBEAT: # Heart outgoing heartbeat_frame = CONTROL_FRAME( CONTROL_PROTOCOL.OK, payload ) self.control_out.send(heartbeat_frame) # -- Soft Kill -- elif event == CONTROL_PROTOCOL.SHUTDOWN: self.done() self.shutdown() # -- Hard Kill -- elif event == CONTROL_PROTOCOL.KILL: self.kill() if self.feed_socket in socks and socks[self.feed_socket] == self.zmq.POLLIN: message = self.feed_socket.recv() if message == str(CONTROL_PROTOCOL.DONE): self.signal_done() return try: event = self.unframe(message) except zp.INVALID_FEED_FRAME as exc: return self.signal_exception(exc) try: cur_state = self.transform(event) # This is overloaded, so it can fail in all sorts of # unknown ways. Its best to catch it in the # Transformer itself. except Exception as exc: return self.signal_exception(exc) try: transform_frame = self.frame(cur_state) except zp.INVALID_TRANSFORM_FRAME as exc: return self.signal_exception(exc) self.result_socket.send(transform_frame, self.zmq.NOBLOCK) def frame(self, cur_state): return zp.TRANSFORM_FRAME(cur_state['name'], cur_state['value']) def unframe(self, msg): return zp.FEED_UNFRAME(msg) def transform(self, event): """ Must return the transformed value as a map with:: {name:"name of new transform", value: "value of new field"} Transforms run in parallel and results are merged into a single map, so transform names must be unique. Best practice is to use the self.state object initialized from the transform configuration, and only set the transformed value:: self.state['value'] = transformed_value """ raise NotImplementedError class PassthroughTransform(BaseTransform): """ A bypass transform which is also an identity transform:: +-------+ +---| f |---> +-------+ +------id-------> """ def __init__(self): BaseTransform.__init__(self, "PASSTHROUGH") self.init() def init(self): pass @property def get_type(self): return COMPONENT_TYPE.CONDUIT #TODO, could save some cycles by skipping the _UNFRAME call and just setting value to original msg string. def transform(self, event): return {'name':zp.TRANSFORM_TYPE.PASSTHROUGH, 'value': zp.FEED_FRAME(event) } class DataSource(Component): """ Baseclass for data sources. Subclass and implement send_all - usually this means looping through all records in a store, converting to a dict, and calling send(map). Every datasource has a dict property to hold filters:: - key -- name of the filter, e.g. SID - value -- a primitive representing the filter. e.g. a list of ints. Modify the datasource's filters via the set_filter(name, value) """ def __init__(self, source_id): Component.__init__(self) self.id = source_id self.init() self.filter = {} def init(self): self.cur_event = None def set_filter(self, name, value): self.filter[name] = value @property def get_id(self): return self.id @property def get_type(self): return COMPONENT_TYPE.SOURCE def open(self): self.data_socket = self.connect_data() def send(self, event): """ Emit data. """ assert isinstance(event, zp.namedict) event['source_id'] = self.get_id event['type'] = self.get_type try: ds_frame = self.frame(event) except zp.INVALID_DATASOURCE_FRAME as exc: return self.signal_exception(exc) self.data_socket.send(ds_frame) def frame(self, event): return zp.DATASOURCE_FRAME(event)