eliminated tornado and pymongo, more package refactoring, test passes

qsim/util.py

@@ -0,0 +1,140 @@
+"""
+Small classes to assist with db access, timezone calculations, and so on.
+"""
+
+import datetime
+import pytz
+import json
+import logging
+import uuid
+import zmq
+
+logger = logging.getLogger('QSimLogger')
+            
+def parse_date(dt_str):
+    """parse strings according to the same format as generated by format_date"""
+    if(dt_str == None):
+        return None
+    parts = dt_str.split(".")
+    dt = datetime.datetime.strptime(parts[0], '%Y/%m/%d-%H:%M:%S').replace(microsecond=int(parts[1]+"000")).replace(tzinfo = pytz.utc)
+    return dt
+    
+def format_date(dt):
+    """Format the date into a date with millesecond resolution and string/alphabetical sorting that is equivalent to datetime sorting"""
+    if(dt == None):
+        return None
+    dt_str = dt.strftime('%Y/%m/%d-%H:%M:%S') + "." + str(dt.microsecond / 1000)
+    return dt_str
+    
+    
+class ParallelBuffer(object):
+    """ holds several queues of events by key, allows retrieval in date order or by merging"""
+    def __init__(self, key_list):
+        self.out_socket = None
+        self.sent_count = 0
+        self.received_count = 0
+        self.draining = False
+        self.data_buffer = {}
+        for key in key_list:
+            self.data_buffer[key] = []
+    
+    def __len__(self):
+        return len(self.data_buffer)        
+    
+    def append(self, key, value):
+        self.data_buffer[key].append(value)
+        self.received_count += 1
+    
+    def next(self):
+        if(not(self.is_full() or self.draining)):
+            return
+            
+        cur = None
+        earliest = None
+        for source, events in self.data_buffer.iteritems():
+            if len(events) == 0:
+                continue
+            cur = events
+            if(earliest == None) or (cur[0]['dt'] <= earliest[0]['dt']):
+                earliest = cur
+        
+        if(earliest != None):
+            return earliest.pop(0)
+        
+    def is_full(self):
+        for source, events in self.data_buffer.iteritems():
+            if (len(events) == 0):
+                return False
+        return True
+    
+    def pending_messages(self):
+        total = 0
+        for source, events in self.data_buffer.iteritems():
+            total += len(events)
+        return total       
+        
+    def drain(self):
+        self.draining = True
+        while(self.pending_messages() > 0):
+            self.send_next()
+            
+    def send_next(self):
+        if(not(self.is_full() or self.draining)):
+            return
+            
+        event = self.next()
+        if(event != None):        
+            self.out_socket.send(json.dumps(event))
+            self.sent_count += 1   
+    
+    
+class MergedParallelBuffer(ParallelBuffer):
+    
+    def __init__(self, keys):
+        ParallelBuffer.__init__(self, keys)
+        self.feed = []
+        self.data_buffer["feed"] = self.feed
+    
+    def next(self):
+        if(not(self.is_full() or self.draining)):
+            return
+        
+        result = self.feed.pop(0)
+        for source, events in self.data_buffer.iteritems():
+            if(source == "feed"):
+                continue
+            if(len(events) > 0):
+                cur = events.pop(0)
+                result[source] = cur['value']
+        return result
+        
+        
+class FeedSync(object):
+    
+    def __init__(self, feed, name):
+        self.feed = feed
+        self.id = "{name}-{id}".format(name=name, id=uuid.uuid1())
+        self.feed.register_sync(self.id)
+        self.logger = logger
+        #self.logger.info("registered {id} with feed".format(id=self.id))
+        
+    def confirm(self):
+        context = zmq.Context()
+        #synchronize with feed
+        sync_socket = context.socket(zmq.REQ)
+        sync_socket.connect(self.feed.sync_address)
+        # send a synchronization request to the feed
+        sync_socket.send(self.id)
+        # wait for synchronization reply from the feed
+        sync_socket.recv()
+        sync_socket.close()
+        context.term()
+        self.logger.info("sync'd feed from {id}".format(id = self.id))
+        
+
+def configure_logging(loglevel=logging.DEBUG):  
+    logger.setLevel(loglevel)
+    handler = logging.handlers.RotatingFileHandler("/tmp/{lfn}.log".format(lfn="qsim-log"), maxBytes=10*1024*1024, backupCount=5)  
+    handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(filename)s %(funcName)s - %(message)s","%Y-%m-%d %H:%M:%S %Z")) 
+    logger.addHandler(handler) 
+    logger.info("logging started...")