eliminated tornado and pymongo, more package refactoring, test passes

qsim/data/equity.py

@@ -1,16 +1,13 @@
 import datetime
 import zmq
-import pymongo
-import pymongo.json_util
 import json
 import pytz
 import copy
 import multiprocessing
 import logging
 import random
-from pymongo import ASCENDING, DESCENDING
 
-import qsim.simulator.backtest.util as qutil
+import qsim.util as qutil
 
 class DataSource(object):
     def __init__(self, feed, source_id):
@@ -59,32 +56,7 @@ class DataSource(object):
         self.data_socket.close()
         self.context.term()
         self.logger.info("finished processing data source")
-
-class EquityMinuteTrades(DataSource):
-    
-    def __init__(self, sid, feed, source_id):
-        self.sid = sid
-        self.connection, self.db    = qutil.connect_db()
-        DataSource.__init__(self, feed, source_id)
-               
-     
-    def send_all(self):   
-        eventQS = self.db.equity.trades.minute.find(fields=["sid","price","volume","dt"],
-                                     spec={"sid":self.sid},
-                                     sort=[("dt",ASCENDING)],
-                                     slave_ok=True)
-        self.logger.info("found {count} events".format(count=eventQS.count()))
-        
-        for doc in eventQS:
-            doc_dt = doc['dt'].replace(tzinfo = pytz.utc)
-            doc_dt_str = qutil.format_date(doc_dt)
-            event = copy.copy(doc)
-            event['dt'] = doc_dt_str
-            del(event['_id'])
-            self.send(event)
-            
-        
-        
+       
 class RandomEquityTrades(DataSource):
     
     def __init__(self, sid, feed, source_id, count):

qsim/simulator/feed.py

@@ -1,6 +1,6 @@
 
-import qsim.simulator.data.sources.equity as qequity
-import qsim.simulator.backtest.util as qutil
+import qsim.data.equity as qequity
+import qsim.util as qutil
 import zmq
 import time
 import logging

qsim/simulator/qbt.py

@@ -34,8 +34,8 @@ import copy
 import multiprocessing
 import zmq
 
-import qsim.backtest.util as qutil
-import qsim.data.sources.equity as qequity
+import qsim.util as qutil
+import qsim.data.equity as qequity
 
 
 

qsim/simulator/qbt_client.py

@@ -4,7 +4,7 @@ import zmq
 import logging
 import json
 
-import qsim.simulator.backtest.util as qutil
+import qsim.util as qutil
 
 class TestClient(object):
     

qsim/test/test_messaging.py

@@ -4,9 +4,10 @@ import logging
 import tornado
 import multiprocessing
 
-from qsim.simulator.data.feed import DataFeed
-from qsim.transforms.transforms import MergedTransformsFeed, MovingAverage
-import qsim.simulator.backtest.util as qutil
+from qsim.simulator.feed import DataFeed
+from qsim.transforms.base import MergedTransformsFeed
+from qsim.transforms.technical import MovingAverage
+import qsim.util as qutil
 
 from qsim.simulator.qbt_client import TestClient
 

qsim/transforms/base.py

@@ -4,10 +4,11 @@ import datetime
 import json
 import copy
 import multiprocessing
-import qsim.simulator.backtest.util as qutil
+import qsim.util as qutil
 import qsim.simulator.config as config
+
 class Transform(object):
-    """Parent class for feed transforms. Subclass to create a new derived value from the combined feed."""
+    """Parent class for feed transforms. Subclass and override transform method to create a new derived value from the combined feed."""
     
     def __init__(self, feed, config_dict, result_address):
         """
@@ -74,46 +75,15 @@ class Transform(object):
         self.context.term()
         
     def transform(self, event):
+        """ Must return the transformed value as a map with {name:"name of new transform", value: "value of new field"}
+            Transforms run in parallel and results are merged into a single map, so transform names must be unique. 
+            Best practice is to use the self.state object initialized from the transform configuration, and only set the
+            transformed value:
+                self.state['value'] = transformed_value
+        """
         return {}
                     
         
-class MovingAverage(Transform):
-    
-    def __init__(self, feed, props, result_address): 
-        Transform.__init__(self, feed, props, result_address)
-        self.events = []
-        
-        self.window = datetime.timedelta(days           = self.config.get_integer('days'), 
-                                        seconds         = self.config.get_integer('seconds'), 
-                                        microseconds    = self.config.get_integer('microseconds'), 
-                                        milliseconds    = self.config.get_integer('milliseconds'),
-                                        minutes         = self.config.get_integer('minutes'),
-                                        hours           = self.config.get_integer('hours'),
-                                        weeks           = self.config.get_integer('weeks'))
-    
-        
-  
-        
-    def transform(self, event):
-        self.events.append(event)
-        
-        #filter the event list to the window length.
-        self.events = [x for x in self.events if (qutil.parse_date(x['dt']) - qutil.parse_date(event['dt'])) <= self.window]
-        
-        if(len(self.events) == 0):
-            return 0.0
-            
-        total = 0.0
-        for event in self.events:
-            total += event['price']
-        
-        self.average = total/len(self.events)
-        
-        self.state['value'] = self.average
-        
-        return self.state
-        
-        
 class MergedTransformsFeed(Transform):
     """ Merge data feed and array of transform feeds into a single result vector.
         PULL from feed

qsim/transforms/technical.py

@@ -0,0 +1,38 @@
+import qsim.transforms.base as base
+
+class MovingAverage(base.Transform):
+    
+    def __init__(self, feed, props, result_address): 
+        base.Transform.__init__(self, feed, props, result_address)
+        self.events = []
+        
+        self.window = datetime.timedelta(days           = self.config.get_integer('days'), 
+                                        seconds         = self.config.get_integer('seconds'), 
+                                        microseconds    = self.config.get_integer('microseconds'), 
+                                        milliseconds    = self.config.get_integer('milliseconds'),
+                                        minutes         = self.config.get_integer('minutes'),
+                                        hours           = self.config.get_integer('hours'),
+                                        weeks           = self.config.get_integer('weeks'))
+    
+        
+  
+        
+    def transform(self, event):
+        self.events.append(event)
+        
+        #filter the event list to the window length.
+        self.events = [x for x in self.events if (qutil.parse_date(x['dt']) - qutil.parse_date(event['dt'])) <= self.window]
+        
+        if(len(self.events) == 0):
+            return 0.0
+            
+        total = 0.0
+        for event in self.events:
+            total += event['price']
+        
+        self.average = total/len(self.events)
+        
+        self.state['value'] = self.average
+        
+        return self.state
+        

qsim/util.py

@@ -8,42 +8,8 @@ import json
 import logging
 import uuid
 import zmq
-import pymongo
-from tornado.options import define, options
 
 logger = logging.getLogger('QSimLogger')
-
-class DocWrap():
-    """
-        Provides attribute access style on top of dictionary results from pymongo. 
-        Allows you to access result['field'] as result.field.
-        Aliases result['_id'] to result.id.
-        
-    """
-    def __init__(self, store=None):
-        if(store == None):
-            self.store = {}
-        else:
-            self.store = store.copy()
-        if(self.store.has_key('_id')):
-            self.store['id'] = self.store['_id']
-            del(self.store['_id'])
-        
-    def __setitem__(self,key,value):
-        if(key == '_id'):
-            self.store['id'] = value
-        else:
-            self.store[key] = value
-        
-    def __getitem__(self, key):
-        if self.store.has_key(key):
-            return self.store[key]
-            
-    def __getattr__(self,attrname):
-        if self.store.has_key(attrname):
-            return self.store[attrname]
-        else:
-            raise AttributeError("No attribute named {name}".format(name=attrname))
             
 def parse_date(dt_str):
     """parse strings according to the same format as generated by format_date"""
@@ -165,22 +131,7 @@ class FeedSync(object):
         context.term()
         self.logger.info("sync'd feed from {id}".format(id = self.id))
         
-        
-define("user_email", default="qbt@quantopian.com", help="email address for qbt user")
-define("password", default="foobar", help="password for qbt user")
-define("port", default=8888, help="run the qbt on the given port", type=int)
-define("mongodb_host", default="127.0.0.1", help="mongodb host address")
-define("mongodb_port", default=27017, help="connect to the mongodb on the given port", type=int)
-define("mongodb_dbname", default="qbt", help="database name")
-define("mongodb_user", default="qbt", help="database user")
-define("mongodb_password", default="qbt", help="database password")
 
-def connect_db():
-    connection = pymongo.Connection(options.mongodb_host, options.mongodb_port)
-    db = connection[options.mongodb_dbname]
-    db.authenticate(options.mongodb_user, options.mongodb_password)
-    return connection, db
-    
 def configure_logging(loglevel=logging.DEBUG):  
     logger.setLevel(loglevel)
     handler = logging.handlers.RotatingFileHandler("/tmp/{lfn}.log".format(lfn="qsim-log"), maxBytes=10*1024*1024, backupCount=5)