diff --git a/config.py b/config.py new file mode 100644 index 00000000..584e41b3 --- /dev/null +++ b/config.py @@ -0,0 +1,43 @@ +import json + +class Config(object): + """ Name/Value configuration object with type-safe accessors and json serialization/deserialization.""" + + def __init__(self, props): + self.store = props + + def __setitem__(self,key,value): + self.store[key] = value + + def __getitem__(self, key): + if self.store.has_key(key): + return self.store[key] + + def __getattr__(self,attrname): + if self.store.has_key(attrname): + return self.store[attrname] + else: + raise AttributeError("No attribute named {name}".format(name=attrname)) + + def get_integer(self, name, default=0): + return self.get_value(name, default, type(1)) + + def get_string(self, name, default=''): + return self.get_value(name, default, type('')) + + def get_float(self, name, default=0.0): + return self.get_value(name, default, type(1.0)) + + def get_value(self, name, default, expected_type): + if(self.store.has_key(name)): + val = self.store[name] + if isinstance(val, expected_type): + return val + else: + return default + + def to_json(self): + return json.dumps(self.store) + + def from_json(self, json_string): + self.store = json.loads(json_string) \ No newline at end of file diff --git a/data/feed.py b/data/feed.py index 22a2910f..d8ec5afe 100644 --- a/data/feed.py +++ b/data/feed.py @@ -1,67 +1,110 @@ from data.sources.equity import * +import time +import logging class DataFeed(object): - def __init__(self, db, logger): - self.logger = logger + def __init__(self, db, subscriber_count): + self.logger = logging.getLogger() self.db = db self.data_workers = {} self.data_address = "tcp://127.0.0.1:{port}".format(port=10101) self.sync_address = "tcp://127.0.0.1:{port}".format(port=10102) + self.feed_address = "tcp://127.0.0.1:{port}".format(port=10103) + self.data_buffer = {} #source_id -> [] + self.subscriber_count = subscriber_count def start_data_workers(self): """Start a sub-process for each datasource.""" - # Socket to receive signals - syncservice = self.context.socket(zmq.REP) - syncservice.bind(self.sync_address) - - - emt1 = EquityMinuteTrades(133, self.db, self.data_address, self.sync_address, 1, self.logger) + emt1 = EquityMinuteTrades(133, self.db, self.data_address, self.sync_address, 1) self.data_workers[1] = emt1 + self.data_buffer[1] = [] emt1.start() - emt2 = EquityMinuteTrades(134, self.db, self.data_address, self.sync_address, 2, self.logger) + emt2 = EquityMinuteTrades(134, self.db, self.data_address, self.sync_address, 2) self.data_workers[2] = emt1 + self.data_buffer[2] = [] emt2.start() - workers = 0 - while workers < len(self.data_workers): + self.logger.info("ds processes launched") + + def sync_clients(self): + # Socket to receive signals + self.logger.info("waiting for all datasources and clients to be ready") + self.syncservice = self.context.socket(zmq.REP) + self.syncservice.bind(self.sync_address) + + subscribers = 0 + while subscribers < (self.subscriber_count + len(self.data_workers)): + self.logger.info("sync'ing {count}".format(count=subscribers)) # wait for synchronization request - msg = syncservice.recv() + msg = self.syncservice.recv() # send synchronization reply - syncservice.send('') - workers += 1 + self.syncservice.send('') + subscribers += 1 - syncservice.close() - - self.logger.info("{count} ds processes launched".format(count=workers)) + self.syncservice.close() + self.logger.info("sync'd all datasources and clients") def run(self): # Prepare our context and sockets self.context = zmq.Context() + counter = 0 + self.start_data_workers() + #create the data sink. Based on http://zguide.zeromq.org/py:tasksink2 #see: http://zguide.zeromq.org/py:taskwork2 self.data_socket = self.context.socket(zmq.PULL) self.data_socket.bind(self.data_address) - counter = 0 + #create the feed + self.feed_socket = self.context.socket(zmq.PUSH) + self.feed_socket.bind(self.feed_address) - self.start_data_workers() + #wait for all feed subscribers + self.sync_clients() + + self.logger.info("entering feed loop on {addr}".format(addr=self.data_address)) while True: message = self.data_socket.recv() event = json.loads(message) - #self.logger.info(message) - counter = counter + 1 - if(event['type'] == "DONE"): - source = event['s'] + #self.logger.info(" count " + str(counter) + " - " + str(event['dt'])) + if(event["type"] == "DONE"): + self.logger.info("DONE") + source = event[u's'] if(self.data_workers.has_key(source)): del(self.data_workers[source]) if(len(self.data_workers) == 0): break + else: + self.data_buffer[event[u's']].append(event) + counter = counter + 1 + self.send_earliest_event() + + #drain any remaining messages in the buffer + self.send_earliest_event(drain=True) + self.logger.info("Collected {n} messages".format(n=counter)) self.data_socket.close() - self.context.term() \ No newline at end of file + self.feed_socket.close() + self.context.term() + + def send_earliest_event(self, drain=False): + earliest = None + next_source = None + while True: #send messages as long as we have >0 messages from each source + for source, events in self.data_buffer.iteritems(): + if(not drain and len(events) == 0 and self.data_workers.has_key(source)): + #there's no way to know that we have the next message + return + if(len(events) > 0 and (earliest == None or earliest > events[0])): + earliest = events[0]['dt'] + next_source = source + + + event = self.data_buffer[next_source].pop(0) + self.feed_socket.send(json.dumps(event)) \ No newline at end of file diff --git a/data/sources/equity.py b/data/sources/equity.py index 87cfb00e..90962e34 100644 --- a/data/sources/equity.py +++ b/data/sources/equity.py @@ -6,6 +6,7 @@ import json import pytz import copy import multiprocessing +import logging from pymongo import ASCENDING, DESCENDING from backtest.util import * @@ -13,13 +14,13 @@ from backtest.util import * class EquityMinuteTrades(object): - def __init__(self, sid, db, data_address, sync_address, source_id, logger): + def __init__(self, sid, db, data_address, sync_address, source_id): self.sid = sid self.db = db self.source_id = source_id - self.logger = logger - self.data_address = data_address + self.logger = logging.getLogger() self.sync_address = sync_address + self.data_address = data_address self.logger.info("data address is {ds}".format(ds=data_address)) self.cur_event = None @@ -29,9 +30,18 @@ class EquityMinuteTrades(object): self.proc.start() def run(self): - self.logger.info("starting data source:{sid}".format(sid=self.sid)) + self.logger.info("starting data source:{sid} on {addr}".format(sid=self.sid, addr=self.data_address)) self.context = zmq.Context() + #synchronize with feed + sync_socket = self.context.socket(zmq.REQ) + sync_socket.connect(self.sync_address) + # send a synchronization request to the feed + sync_socket.send('') + # wait for synchronization reply from the feed + sync_socket.recv() + sync_socket.close() + #create the data sink. Based on http://zguide.zeromq.org/py:tasksink2 self.data_socket = self.context.socket(zmq.PUSH) self.data_socket.connect(self.data_address) @@ -42,16 +52,6 @@ class EquityMinuteTrades(object): slave_ok=True) self.logger.info("found {count} events".format(count=eventQS.count())) - #synchronize with feed - syncclient = self.context.socket(zmq.REQ) - syncclient.connect(self.sync_address) - - # send a synchronization request - syncclient.send('') - # wait for synchronization reply - syncclient.recv() - - syncclient.close() for doc in eventQS: doc_dt = doc['dt'].replace(tzinfo = pytz.utc) diff --git a/data/transforms.py b/data/transforms.py new file mode 100644 index 00000000..285176bc --- /dev/null +++ b/data/transforms.py @@ -0,0 +1,150 @@ +import zmq +import logging +import datetime +import json +import config +import multiprocessing + +class Transform(object): + """Parent class for feed transforms. Subclass to create a new derived value from the combined feed.""" + + def __init__(self, feed_address, result_address, sync_address, config_dict): + """ + feed_address - zmq socket address, Transform will CONNECT a PULL socket and receive messages until "DONE" is received. + result_address - zmq socket address, Transform will CONNECT a PUSH socket and send messaes until feed_socket receives "DONE" + sync_address - zmq socket address, Transform will CONNECT a REQ socket and send/receive one message before entering feed loop + config - must be a config.Config object with at least an entry for 'name':string value + """ + self.logger = logging.getLogger() + self.feed_address = feed_address + self.result_address = result_address + self.sync_address = sync_address + self.config = config.Config(config_dict) + self.name = self.config.get_string('name') + self.state = {} + self.state['name'] = self.name + + def run(self): + self.context = zmq.Context() + + self.logger.info("starting {name} transform".format(name = self.name)) + #create the feed PULL. + self.feed_socket = self.context.socket(zmq.PULL) + self.feed_socket.connect(self.feed_address) + + #create the result PUSH + self.result_socket = self.context.socket(zmq.PUSH) + self.result_socket.connect(self.result_address) + + self.logger.info("sync'ing feed from {name}".format(name = self.name)) + #synchronize with feed + sync_socket = self.context.socket(zmq.REQ) + sync_socket.connect(self.sync_address) + # send a synchronization request to the feed + sync_socket.send('') + # wait for synchronization reply from the feed + sync_socket.recv() + sync_socket.close() + + self.logger.info("starting {name} event loop".format(name = self.name)) + + while True: + message = self.feed_socket.recv() + self.logger.info("got feed message at {name}".format(name=self.name)) + if(message == "DONE"): + break; + event = json.loads(message) + cur_state = update(event) + + self.result_socket.send(json.dumps(cur_state)) + self.logger.info("sent message from {name}".format(name=self.name)) + + def update(self, event): + return {} + +class Merge(Transform): + """ Merge data feed and array of transform feeds into a single result vector. + PULL from feed + PULL from child transforms + PUSH to client + + """ + + def __init__(self, feed_address, result_address, sync_address, props): + """ + config - must have an entry for 'transforms':array of dicts, which are convertedto configs. + """ + Transform.__init__(self, feed_address, result_address, sync_address, props) + self.transform_address = "tcp://127.0.0.1:{port}".format(port=10104) + self.transform_socket = None + self.create_transforms(self.config.transforms) + + + def create_transforms(self, configs): + self.transforms = {} + for props in configs: + class_name = props['class'] + if(class_name == 'MovingAverage'): + mavg = MovingAverage(self.feed_address, self.transform_address, self.sync_address, props) + self.transforms[mavg.name] = mavg + + for name, transform in self.transforms.iteritems(): + self.logger.info("starting {name}".format(name=name)) + proc = multiprocessing.Process(target=transform.run) + proc.start() + + def get_socket(self): + + if(self.transform_socket == None): + #create the feed PULL. + self.transform_socket = self.context.socket(zmq.PULL) + self.transform_socket.connect(self.transform_address) + + def update(self, event): + + state = {} + state['feed'] = event + + count = 0 + while count < len(transforms): + message = get_socket().recv + data = json.loads(message) + state[data['name']] = data + + return state + + + +class MovingAverage(Transform): + + def __init__(self, feed_address, result_address, sync_address, props): + Transform.__init__(self, feed_address, result_address, sync_address, props) + self.events = [] + + self.window = datetime.timedelta(days = self.config.get_integer('days'), + seconds = self.config.get_integer('seconds'), + microseconds = self.config.get_integer('microseconds'), + milliseconds = self.config.get_integer('milliseconds'), + minutes = self.config.get_integer('minutes'), + hours = self.config.get_integer('hours'), + weeks = self.config.get_integer('weeks')) + + + + + def update(self, event): + self.events.append(event) + + #filter the event list to the window length. + self.events = [x for x in self.events if (x.dt - curTick.dt) <= self.window] + + if(len(self.events) == 0): + return 0.0 + + total = 0.0 + for event in self.events: + total += event['price'] + + self.average = total/len(self.events) + + self.state['avg'] = self.average \ No newline at end of file diff --git a/qbt.py b/qbt.py index 9baa616d..1835b931 100644 --- a/qbt.py +++ b/qbt.py @@ -48,79 +48,20 @@ class Backtest(object): def __init__(self, db, logger): self.logger = logger self.db = db - self.data_workers = {} + self.feed = DataFeed(db, logger) - - def start_data_workers(self): - """Start a sub-process for each datasource.""" - - emt1 = EquityMinuteTrades(133, self.db, DATA_SINK_PORT, 1, self.logger) - data_workers[1] = emt1 - emt1.start() - #emt2 = EquityMinuteTrades(134, self.db, self.data_socket, self.controller_socket, 2, self.logger) - #emt2.start() - self.logger.info("ds processes launched") + def start_feed(self): + proc1 = multiprocessing.Process(target=feed.run) + proc1.start() def run(self): # Prepare our context and sockets self.context = zmq.Context() - #create the data sink. Based on http://zguide.zeromq.org/py:tasksink2 - #see: http://zguide.zeromq.org/py:taskwork2 - self.data_socket = "tcp://127.0.0.1:{port}".format(port=DATA_SINK_PORT) - self.data_sink = self.context.socket(zmq.PULL) - self.data_sink.bind(self.data_socket) + #create the feed sink. + self.feed_address = self.feed.data_address + self.feed_socket = self.context.connect(self.feed_address) + self.feed_socket.connect(zmq.PULL) - #create the controller publishing socket. - self.controller_socket = "tcp://127.0.0.1:{port}".format(port=CONTROLLER_PORT) - #self.controller = self.context.socket(zmq.PUB) - #self.controller.bind(self.controller_socket) - - - #create the merged dataset feed socket - self.data_feed = self.context.socket(zmq.PUSH) - self.data_feed.bind("tcp://127.0.0.1:{port}".format(port=DATA_FEED_PORT)) - - self.last_event_time = None - self.data_workers = [] - - last_dt = None - event_q = [] - #data workers start in independent processes, connecting to the data_socket - self.start_data_workers() - while True: - try: - #self.logger.info("about to receive") - message = self.data_sink.recv() - event = json.loads(message) - last_dt = event['dt'] - - event_q.append(event) - event_q = sorted(event_q, key=lambda event: event['dt']) - cur_event = event_q.pop(0) - self.data_feed.send(json.dumps(cur_event)) - - #this signals loop at datasource should proceed... all sources will send all events as fast as possible. - self.data_sink.send(str(last_dt)) - except zmq.ZMQError as err: - - #EAGAIN indicates recv doesn't have messages now, and datasources are sending - #so only throw if we have an error other than EAGAIN. - if err.errno != zmq.EAGAIN: - raise err #real error, throw back to caller - #self.logger.info("we received an error") - break - - #no events in q at this point means we've processed all the data in all the sources! - #if(len(event_q) == 0): - # return - - #event_q = sorted(event_q, key=lambda event: event['dt']) - # we have the most recent message from each source, so we can only - # be sure the first uninterrupted streak of messages from the same source - # in the queue are most recent. - #while len(event_q) > 0: - # cur_event = event_q.pop(0) - # self.data_feed.send(json.dumps(cur_event)) - # last_dt = event_q[0]['dt'] + \ No newline at end of file diff --git a/qbt_client.py b/qbt_client.py index a0e522b5..b9819fc9 100644 --- a/qbt_client.py +++ b/qbt_client.py @@ -1,20 +1,35 @@ import copy import multiprocessing import zmq +import logging from backtest.util import * class BacktestClient(object): - def __init__(self,feed_port, logger): + def __init__(self,feed_address, sync_address): self.context = zmq.Context() - - self.data_feed = self.context.socket(zmq.PULL) - self.data_feed.connect("tcp://localhost:{port}".format(port=feed_port)) - self.logger = logger + self.logger = logging.getLogger() + self.feed_address = feed_address + self.sync_address = sync_address def run(self): self.logger.info("running the client") + self.data_feed = self.context.socket(zmq.PULL) + self.data_feed.connect(self.feed_address) + + #synchronize with feed + sync_socket = self.context.socket(zmq.REQ) + sync_socket.connect(self.sync_address) + # send a synchronization request to the feed + sync_socket.send('') + # wait for synchronization reply from the feed + sync_socket.recv() + sync_socket.close() + + counter = 0 while True: + counter += 1 msg = self.data_feed.recv() - self.logger.info(msg) \ No newline at end of file + self.logger.info("received {n} messages".format(n=counter)) + \ No newline at end of file diff --git a/test.py b/test.py index a85c345d..025d726a 100644 --- a/test.py +++ b/test.py @@ -3,21 +3,45 @@ import logging import tornado from data.sources.equity import * from data.feed import * +from data.transforms import Merge, MovingAverage + from qbt_server import connect_db +from qbt_client import BacktestClient def datafeed(): connection, db = connect_db() logger = logging.getLogger() - feed = DataFeed(db, logger) - logger.info("starting the feed") - feed.run() + feed = DataFeed(db, 1) #one merge, two moving averages. + feed_proc = multiprocessing.Process(target=feed.run) + feed_proc.start() + #config = {} + #config['name'] = '**merged feed**' + #config['transforms'] = [{'name':'mavg1', 'class':'MovingAverage', 'hours':1},{'name':'mavg2', 'class':'MovingAverage', 'hours':2}] + + #result_address = "tcp://127.0.0.1:20202" + + #mavg = MovingAverage(feed.feed_address, result_address, feed.sync_address, config['transforms'][0]) + #mavg.run() + + #merger = Merge(feed.feed_address, result_address, feed.sync_address, config) + #merger_proc = multiprocessing.Process(target=merger.run) + #merger_proc.start() + + client = BacktestClient(feed.feed_address, feed.sync_address) + client.run() + +def pubsub(): + proc1 = multiprocessing.Process(target=sub) + proc2 = multiprocessing.Process(target=pub) + proc1.start() + proc2.start() def sub(): context = zmq.Context() - controller = context.socket(zmq.SUB) + controller = context.socket(zmq.PULL) controller.connect("tcp://127.0.0.1:10099") - controller.setsockopt(zmq.SUBSCRIBE, '') + #controller.setsockopt(zmq.SUBSCRIBE, '') while True: try: message = controller.recv() @@ -28,7 +52,7 @@ def sub(): def pub(): context = zmq.Context() - controller = context.socket(zmq.PUB) + controller = context.socket(zmq.PUSH) controller.bind("tcp://127.0.0.1:10099") while True: controller.send("HELLO3") diff --git a/test2.py b/test2.py deleted file mode 100644 index 9e378aa4..00000000 --- a/test2.py +++ /dev/null @@ -1,11 +0,0 @@ -import zmq - -def main(): - context = zmq.Context() - controller = context.socket(zmq.PUB) - controller.bind("tcp://127.0.0.1:10099") - while True: - controller.send("HELLO3") - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/test3.py b/test3.py deleted file mode 100644 index 7f89c9c1..00000000 --- a/test3.py +++ /dev/null @@ -1,12 +0,0 @@ -import multiprocessing -import test -import test2 - -def main(): - proc1 = multiprocessing.Process(target=test2.main) - proc1.start() - proc2 = multiprocessing.Process(target=test.main) - proc2.start() - -if __name__ == "__main__": - main() \ No newline at end of file