synchronized data sources send events to feed.

2026-09-10 11:50:32 +08:00 · 2012-01-15 14:53:33 -05:00
parent abce64914c
commit ddaefb9e1b
4 changed files with 165 additions and 90 deletions
@@ -0,0 +1,67 @@
+
+from data.sources.equity import *
+
+class DataFeed(object):
+    
+    def __init__(self, db, logger):
+        self.logger = logger
+        self.db = db
+        self.data_workers = {}
+        self.data_address = "tcp://127.0.0.1:{port}".format(port=10101)
+        self.sync_address = "tcp://127.0.0.1:{port}".format(port=10102)
+        
+    def start_data_workers(self):
+        """Start a sub-process for each datasource."""
+        
+        # Socket to receive signals
+        syncservice = self.context.socket(zmq.REP)
+        syncservice.bind(self.sync_address)
+        
+        
+        emt1 = EquityMinuteTrades(133, self.db, self.data_address, self.sync_address, 1, self.logger)
+        self.data_workers[1] = emt1
+        emt1.start()
+        emt2 = EquityMinuteTrades(134, self.db, self.data_address, self.sync_address, 2, self.logger)
+        self.data_workers[2] = emt1
+        emt2.start()
+        
+        workers = 0
+        while workers < len(self.data_workers):
+            # wait for synchronization request
+            msg = syncservice.recv()
+            # send synchronization reply
+            syncservice.send('')
+            workers += 1
+        
+        syncservice.close()
+        
+        self.logger.info("{count} ds processes launched".format(count=workers))
+       
+    def run(self):   
+        # Prepare our context and sockets
+        self.context = zmq.Context()
+        
+        #create the data sink. Based on http://zguide.zeromq.org/py:tasksink2 
+        #see: http://zguide.zeromq.org/py:taskwork2
+        self.data_socket = self.context.socket(zmq.PULL)
+        self.data_socket.bind(self.data_address)
+        
+        counter = 0
+        
+        self.start_data_workers()
+        
+        while True:
+            message = self.data_socket.recv()
+            event = json.loads(message)
+            #self.logger.info(message)
+            counter = counter + 1
+            if(event['type'] == "DONE"):
+                source = event['s']
+                if(self.data_workers.has_key(source)):
+                    del(self.data_workers[source])
+                if(len(self.data_workers) == 0):
+                    break
+                
+        self.logger.info("Collected {n} messages".format(n=counter))
+        self.data_socket.close()
+        self.context.term()