mirror of
https://github.com/wassname/catalyst.git
synced 2026-06-29 22:32:30 +08:00
Stephen Diehl
108 lines
2.8 KiB
Python
108 lines
2.8 KiB
Python
import logbook
|
|
from collections import Counter
|
|
|
|
from zipline.components.aggregator import Aggregate, \
|
|
AGGREGATE_STATES, AGGREGATE_TRANSITIONS
|
|
import zipline.protocol as zp
|
|
|
|
log = logbook.Logger('Feed')
|
|
|
|
# =========
|
|
# Component
|
|
# =========
|
|
|
|
class Feed(Aggregate):
|
|
"""
|
|
Connects to N PULL sockets, publishing all messages received to a
|
|
PUB socket. Published messages are guaranteed to be in chronological
|
|
order based on message property dt. Expects to be instantiated in
|
|
one execution context (thread, process, etc) and run in another.
|
|
"""
|
|
|
|
states = list(AGGREGATE_STATES)
|
|
transitions = AGGREGATE_TRANSITIONS
|
|
initial_state = -1
|
|
|
|
def init(self):
|
|
self.sent_count = 0
|
|
self.received_count = 0
|
|
self.ds_finished_counter = 0
|
|
|
|
self.data_buffer = {}
|
|
|
|
# source_id -> integer count
|
|
self.sent_counters = Counter()
|
|
self.recv_counters = Counter()
|
|
|
|
self.state = AGGREGATE_STATES.INIT
|
|
|
|
@property
|
|
def get_id(self):
|
|
return "FEED"
|
|
|
|
@property
|
|
def draining(self):
|
|
return self.state == AGGREGATE_STATES.DRAINING
|
|
|
|
# -------
|
|
# Sockets
|
|
# -------
|
|
|
|
def open(self):
|
|
self.pull_socket = self.bind_data()
|
|
self.feed_socket = self.bind_feed()
|
|
|
|
# -------
|
|
# Framing
|
|
# -------
|
|
|
|
def unframe(self, msg):
|
|
return zp.DATASOURCE_UNFRAME(msg)
|
|
|
|
def frame(self, event):
|
|
return zp.FEED_FRAME(event)
|
|
|
|
# -------------
|
|
# Flow Control
|
|
# -------------
|
|
|
|
def append(self, event):
|
|
"""
|
|
Add an event to the buffer for the source specified by
|
|
source_id.
|
|
"""
|
|
self.data_buffer[event.source_id].append(event)
|
|
self.recv_counters[event.source_id] += 1
|
|
self.received_count += 1
|
|
|
|
def next(self):
|
|
"""
|
|
Get the next message in chronological order.
|
|
"""
|
|
|
|
# is_full and draining defined in aggregator
|
|
if not(self.is_full() or self.draining):
|
|
return
|
|
|
|
cur_source = None
|
|
earliest_source = None
|
|
earliest_event = None
|
|
#iterate over the queues of events from all sources
|
|
#(1 queue per datasource)
|
|
for events in self.data_buffer.itervalues():
|
|
if len(events) == 0:
|
|
continue
|
|
cur_source = events
|
|
first_in_list = events[0]
|
|
if first_in_list.dt == None:
|
|
#this is a filler event, discard
|
|
events.pop(0)
|
|
continue
|
|
|
|
if (earliest_event == None) or (first_in_list.dt <= earliest_event.dt):
|
|
earliest_event = first_in_list
|
|
earliest_source = cur_source
|
|
|
|
if earliest_event != None:
|
|
return earliest_source.pop(0)
|