From 9078bf922bff56984aff5d11636d1708a75ba221 Mon Sep 17 00:00:00 2001
From: fawce <fawcett@gmail.com>
Date: Tue, 7 Feb 2012 23:13:41 -0500
Subject: [PATCH] added qsim top level package, added rolling log file, test
 passes

---
 pylint.out                            |   1 +
 qsim/simulator/backtest/util.py       |  30 +++++-
 qsim/simulator/data/feed.py           |  12 ++-
 qsim/simulator/data/sources/equity.py |  16 ++-
 qsim/simulator/host_settings.py       |  14 ---
 qsim/simulator/qbt.py                 |   4 +-
 qsim/simulator/qbt_client.py          |   6 +-
 qsim/simulator/qbt_server.py          | 148 --------------------------
 qsim/test/test_messaging.py           |  15 ++-
 qsim/transforms/transforms.py         |   8 +-
 scripts/dev_setup.py                  |   7 +-
 scripts/jenkins.sh                    |   6 +-
 scripts/requirements_dev.txt          |  29 +++--
 13 files changed, 90 insertions(+), 206 deletions(-)
 create mode 100644 pylint.out
 delete mode 100644 qsim/simulator/host_settings.py
 delete mode 100644 qsim/simulator/qbt_server.py

diff --git a/pylint.out b/pylint.out
new file mode 100644
index 00000000..b6feaade
--- /dev/null
+++ b/pylint.out
@@ -0,0 +1 @@
+__init__.py:1: [F] error while code parsing: Unable to load file '__init__.py' ([Errno 2] No such file or directory: '__init__.py')
diff --git a/qsim/simulator/backtest/util.py b/qsim/simulator/backtest/util.py
index f4ec3652..d5455ac3 100644
--- a/qsim/simulator/backtest/util.py
+++ b/qsim/simulator/backtest/util.py
@@ -8,6 +8,9 @@ import json
 import logging
 import uuid
 import zmq
+from tornado.options import define, options
+
+logger = logging.getLogger('QSimLogger')
 
 class DocWrap():
     """
@@ -145,7 +148,7 @@ class FeedSync(object):
         self.feed = feed
         self.id = "{name}-{id}".format(name=name, id=uuid.uuid1())
         self.feed.register_sync(self.id)
-        self.logger = logging.getLogger()
+        self.logger = logger
         #self.logger.info("registered {id} with feed".format(id=self.id))
         
     def confirm(self):
@@ -159,4 +162,27 @@ class FeedSync(object):
         sync_socket.recv()
         sync_socket.close()
         context.term()
-        self.logger.info("sync'd feed from {id}".format(id = self.id))
\ No newline at end of file
+        self.logger.info("sync'd feed from {id}".format(id = self.id))
+        
+        
+define("user_email", default="qbt@quantopian.com", help="email address for qbt user")
+define("password", default="foobar", help="password for qbt user")
+define("port", default=8888, help="run the qbt on the given port", type=int)
+define("mongodb_host", default="127.0.0.1", help="mongodb host address")
+define("mongodb_port", default=27017, help="connect to the mongodb on the given port", type=int)
+define("mongodb_dbname", default="qbt", help="database name")
+define("mongodb_user", default="qbt", help="database user")
+define("mongodb_password", default="qbt", help="database password")
+
+def connect_db():
+    connection = pymongo.Connection(options.mongodb_host, options.mongodb_port)
+    db = connection[options.mongodb_dbname]
+    db.authenticate(options.mongodb_user, options.mongodb_password)
+    return connection, db
+    
+def configure_logging(loglevel=logging.DEBUG):  
+    logger.setLevel(loglevel)
+    handler = logging.handlers.RotatingFileHandler("/tmp/{lfn}.log".format(lfn="qsim-log"), maxBytes=10*1024*1024, backupCount=5)  
+    handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(filename)s %(funcName)s - %(message)s","%Y-%m-%d %H:%M:%S %Z")) 
+    logger.addHandler(handler) 
+    logger.info("logging started...")   
\ No newline at end of file
diff --git a/qsim/simulator/data/feed.py b/qsim/simulator/data/feed.py
index 1f84d4ea..5ccd0ff8 100644
--- a/qsim/simulator/data/feed.py
+++ b/qsim/simulator/data/feed.py
@@ -1,13 +1,15 @@
 
-from simulator.data.sources.equity import *
-from simulator.backtest.util import *
+import qsim.simulator.data.sources.equity as qequity
+import qsim.simulator.backtest.util as qutil
+import zmq
 import time
 import logging
+import json
 
 class DataFeed(object):
     
     def __init__(self, config):
-        self.logger = logging.getLogger()
+        self.logger = qutil.logger
         
         self.data_address = "tcp://127.0.0.1:{port}".format(port=10101)
         self.sync_address = "tcp://127.0.0.1:{port}".format(port=10102)
@@ -22,10 +24,10 @@ class DataFeed(object):
                 emt = EquityMinuteTrades(info['sid'], self, name)
                 self.data_workers[name] = emt
             elif(info['class'] == "RandomEquityTrades"):
-                ret = RandomEquityTrades(info['sid'], self, name, info['count'])
+                ret = qequity.RandomEquityTrades(info['sid'], self, name, info['count'])
                 self.data_workers[name] = ret
                 
-        self.data_buffer = ParallelBuffer(self.data_workers.keys())
+        self.data_buffer = qutil.ParallelBuffer(self.data_workers.keys())
              
     def start_data_workers(self):
         """Start a sub-process for each datasource.""" 
diff --git a/qsim/simulator/data/sources/equity.py b/qsim/simulator/data/sources/equity.py
index a3ef1d62..4790a0dc 100644
--- a/qsim/simulator/data/sources/equity.py
+++ b/qsim/simulator/data/sources/equity.py
@@ -10,16 +10,14 @@ import logging
 import random
 from pymongo import ASCENDING, DESCENDING
 
-from simulator.backtest.util import *
-
-from simulator.qbt_server import * #connect_db
+import qsim.simulator.backtest.util as qutil
 
 class DataSource(object):
     def __init__(self, feed, source_id):
         self.source_id              = source_id
-        self.logger                 = logging.getLogger()
+        self.logger                 = qutil.logger
         self.feed                   = feed
-        self.sync                   = FeedSync(self.feed, str(source_id))
+        self.sync                   = qutil.FeedSync(self.feed, str(source_id))
         self.data_address           = self.feed.data_address
         self.logger.info("data address is {ds}".format(ds=self.feed.data_address))        
         self.cur_event = None
@@ -46,7 +44,7 @@ class DataSource(object):
             self.send_all()
             self.close()    
         except Exception as err:
-            self.logger.error(err, "Unexpected failure running datasource - {name}.".format(name=name))
+            self.logger.exception("Unexpected failure running datasource - {name}.".format(name=self.source_id))
             
     def send(self, event):
         event['s'] = self.source_id             
@@ -66,7 +64,7 @@ class EquityMinuteTrades(DataSource):
     
     def __init__(self, sid, feed, source_id):
         self.sid = sid
-        self.connection, self.db    = connect_db()
+        self.connection, self.db    = qutil.connect_db()
         DataSource.__init__(self, feed, source_id)
                
      
@@ -79,7 +77,7 @@ class EquityMinuteTrades(DataSource):
         
         for doc in eventQS:
             doc_dt = doc['dt'].replace(tzinfo = pytz.utc)
-            doc_dt_str = format_date(doc_dt)
+            doc_dt_str = qutil.format_date(doc_dt)
             event = copy.copy(doc)
             event['dt'] = doc_dt_str
             del(event['_id'])
@@ -101,7 +99,7 @@ class RandomEquityTrades(DataSource):
         
         for i in range(self.count):
             price = price + random.uniform(-0.05,0.05)
-            event = {'sid':self.sid, 'dt':format_date(trade_start + (minute * i)),'price':price, 'volume':random.randrange(100,10000,100)}
+            event = {'sid':self.sid, 'dt':qutil.format_date(trade_start + (minute * i)),'price':price, 'volume':random.randrange(100,10000,100)}
             self.send(event)
           
    
diff --git a/qsim/simulator/host_settings.py b/qsim/simulator/host_settings.py
deleted file mode 100644
index 23c9efdc..00000000
--- a/qsim/simulator/host_settings.py
+++ /dev/null
@@ -1,14 +0,0 @@
-class Config(object):
-   def __init__(self, dct):
-       self.__dict__.update(dct)
-
-mongo_conn_args = Config({
-   'mongodb_host'         :  'claire.mongohq.com',
-   'mongodb_port'         :  10087,
-   'mongodb_dbname'       :  'quantodata-staging',
-   'mongodb_user'         :  'quantopian',
-   'mongodb_password'     :  'quantopian',
-})
-
-root_url = 'http://localhost:8000'
-ws_url = 'ws://localhost:8001'
diff --git a/qsim/simulator/qbt.py b/qsim/simulator/qbt.py
index 1835b931..a4bba52e 100644
--- a/qsim/simulator/qbt.py
+++ b/qsim/simulator/qbt.py
@@ -34,8 +34,8 @@ import copy
 import multiprocessing
 import zmq
 
-from backtest.util import *
-from data.sources.equity import *
+import qsim.backtest.util as qutil
+import qsim.data.sources.equity as qequity
 
 
 
diff --git a/qsim/simulator/qbt_client.py b/qsim/simulator/qbt_client.py
index eda31fb6..67847902 100644
--- a/qsim/simulator/qbt_client.py
+++ b/qsim/simulator/qbt_client.py
@@ -4,15 +4,15 @@ import zmq
 import logging
 import json
 
-from backtest.util import *
+import qsim.simulator.backtest.util as qutil
 
 class TestClient(object):
     
     def __init__(self,feed, address, bind=False):
-        self.logger         = logging.getLogger()
+        self.logger         = qutil.logger
         self.feed           = feed
         self.address        = address
-        self.sync           = FeedSync(feed, "testclient")
+        self.sync           = qutil.FeedSync(feed, "testclient")
         self.bind           = bind
         self.received_count = 0
         
diff --git a/qsim/simulator/qbt_server.py b/qsim/simulator/qbt_server.py
deleted file mode 100644
index 13482fc8..00000000
--- a/qsim/simulator/qbt_server.py
+++ /dev/null
@@ -1,148 +0,0 @@
-import tornado.auth
-import tornado.httpserver
-import tornado.ioloop
-from tornado.options import define, options
-import tornado.web
-import pymongo
-import bson
-import hashlib
-import base64
-import uuid
-import os
-import logging
-import datetime
-import multiprocessing
-
-from qbt import *
-from qbt_client import * 
-
-logger = logging.getLogger()
-
-define("port", default=8888, help="run the qbt on the given port", type=int)
-define("mongodb_host", default="127.0.0.1", help="mongodb host address")
-define("mongodb_port", default=27017, help="connect to the mongodb on the given port", type=int)
-define("mongodb_dbname", default="qbt", help="database name")
-define("mongodb_user", default="qbt", help="database user")
-define("mongodb_password", default="qbt", help="database password")
-
-HASH_ALGO = 'sha256'
-
-def connect_db():
-    connection = pymongo.Connection(options.mongodb_host, options.mongodb_port)
-    db = connection[options.mongodb_dbname]
-    db.authenticate(options.mongodb_user, options.mongodb_password)
-    return connection, db
-    
-def encrypt_password(salt, password):
-    if(salt == None):
-        h1 = hashlib.new(HASH_ALGO)
-        h1.update(str(datetime.datetime.utcnow())+"--"+password)
-        salt = h1.hexdigest()
-        
-    h2 = hashlib.new(HASH_ALGO)
-    h2.update(salt+"--"+password)
-    encrypted_password = h2.hexdigest()
-    
-    return salt, encrypted_password
-
-class Application(tornado.web.Application):
-    def __init__(self):
-        handlers = [
-            (r"/", MainHandler),
-            (r"/login", LoginHandler),
-            (r"/backtest", BacktestHandler)
-        ]
-        settings = dict(
-            template_path=os.path.join(os.path.dirname(__file__), "templates"),
-            static_path=os.path.join(os.path.dirname(__file__), "static"),
-            xsrf_cookies=False,
-            cookie_secret=base64.b64encode(uuid.uuid4().bytes + uuid.uuid4().bytes),
-            login_url="/login",
-            #autoescape=None,
-            #debug=True,
-        )
-        tornado.web.Application.__init__(self, handlers, **settings)
-
-        # Have one global connection to the blog DB across all handlers
-        self.connection, self.db = connect_db()
-
-
-class BaseHandler(tornado.web.RequestHandler):
-    @property
-    def db(self):
-        return self.application.db
-
-    def get_current_user(self):
-        return "fawce"
-        user_id = self.get_secure_cookie(u"user_id")
-        logger.info("looking up user with id: {id}".format(id=user_id))
-        if not user_id: return None
-        #get user record by id
-        users = self.db.users.find(spec={"_id":bson.ObjectId(user_id)}, limit=1)
-        if(users.count > 0):
-            return users[0]
-        return None
-
-class MainHandler(BaseHandler):
-    @tornado.web.authenticated
-    def get(self):
-        self.write("Hello, world. Try launching a <a href='/backtest'>backtest</a>.")
-                         
-class LoginHandler(BaseHandler):
-    def get(self):
-        self.write('<html><body><form action="/login" method="post">'
-                   'Name: <input type="text" name="user_name">'
-                   'pass: <input type="password" name="password">'
-                   '<input type="submit" value="Sign in">'
-                   '</form></body></html>')
-
-    def post(self):  
-        self.authenticate(self.get_argument("user_name"),self.get_argument("password"))
-        self.redirect("/")
-
-    def authenticate(self, username, password):
-        h = hashlib.new(HASH_ALGO)
-        #find user record by username.
-        users = self.db.users.find(spec={"email":username}, limit=1)
-        if(users.count > 0):
-            user_record = users[0]
-        else:
-            logger.debug("no user with name: {username}", username=username)
-            return
-
-        
-        #calculate password hash
-        salt, encrypted_password = encrypt_password(user_record['salt'], password)
-        if (user_record['encrypted_password'] == encrypted_password):
-            #we have a match, so set the secure cookie to the salt
-            logger.debug("setting user_id cookie to {id}".format(id=user_record['_id']))
-            self.set_secure_cookie(u"user_id", unicode(user_record['_id']))
-
-class BacktestHandler(BaseHandler):
-    @tornado.web.authenticated
-    def get(self):
-        self.write('<html><body><form action="/backtest" method="post">'
-                   '<input type="submit" value="Launch">'
-                   '</form></body></html>')
-    @tornado.web.authenticated
-    def post(self):
-        
-        bt = Backtest(self.db, logger)
-        #btc = BacktestClient(DATA_FEED_PORT, logger)
-        bt_proc = multiprocessing.Process(target=bt.run)
-        #btc_proc = multiprocessing.Process(target=btc.run)
-        bt_proc.start()
-        #btc_proc.start()
-
-
-
-    
-def main():
-    tornado.options.parse_command_line()
-    http_server = tornado.httpserver.HTTPServer(Application())
-    http_server.listen(options.port)
-    tornado.ioloop.IOLoop.instance().start()
-
-
-if __name__ == "__main__":
-    main()
diff --git a/qsim/test/test_messaging.py b/qsim/test/test_messaging.py
index 12d547b6..d0158fe8 100644
--- a/qsim/test/test_messaging.py
+++ b/qsim/test/test_messaging.py
@@ -2,16 +2,21 @@ import unittest2 as unittest
 import zmq
 import logging
 import tornado
-from simulator.data.sources.equity import *
-from simulator.data.feed import *
-from transforms.transforms import MergedTransformsFeed, MovingAverage
+import multiprocessing
 
-from simulator.qbt_client import TestClient
+from qsim.simulator.data.feed import DataFeed
+from qsim.transforms.transforms import MergedTransformsFeed, MovingAverage
+import qsim.simulator.backtest.util as qutil
+
+from qsim.simulator.qbt_client import TestClient
 
 
 class MessagingTestCase(unittest.TestCase):
 
     def setUp(self):
+        tornado.options.parse_command_line()
+        qutil.configure_logging()
+        qutil.logger.info("testing...")
         self.total_data_count = 800
         self.feed_config = {'emt1':{'sid':133, 'class':'RandomEquityTrades', 'count':400},
                             'emt2':{'sid':134, 'class':'RandomEquityTrades', 'count':400}}
@@ -23,7 +28,7 @@ class MessagingTestCase(unittest.TestCase):
         self.config['transforms'] = [{'name':'mavg1', 'class':'MovingAverage', 'hours':1},{'name':'mavg2', 'class':'MovingAverage', 'hours':2}]  
 
     def test_client(self):  
-        #subscribe a client to the transformed feed
+        #subscribe a client to the multiplexed feed
         client = TestClient(self.feed, self.feed.feed_address)
     
         feed_proc = multiprocessing.Process(target=self.feed.run)
diff --git a/qsim/transforms/transforms.py b/qsim/transforms/transforms.py
index b578fd16..65204aa7 100644
--- a/qsim/transforms/transforms.py
+++ b/qsim/transforms/transforms.py
@@ -4,8 +4,8 @@ import datetime
 import json
 import copy
 import multiprocessing
-from simulator.backtest.util import *
-import simulator.config as config
+import qsim.simulator.backtest.util as qutil
+import qsim.simulator.config as config
 class Transform(object):
     """Parent class for feed transforms. Subclass to create a new derived value from the combined feed."""
     
@@ -18,7 +18,7 @@ class Transform(object):
             server          - if True, transform will bind to the result address (and act as a server), if False it will connect. The
                               the last transform in a series should be server=True so that clients can connect.
         """
-        self.logger             = logging.getLogger()
+        self.logger             = qutil.logger
         self.feed               = feed
         self.feed_address       = feed.feed_address
         self.result_address     = result_address
@@ -98,7 +98,7 @@ class MovingAverage(Transform):
         self.events.append(event)
         
         #filter the event list to the window length.
-        self.events = [x for x in self.events if (parse_date(x['dt']) - parse_date(event['dt'])) <= self.window]
+        self.events = [x for x in self.events if (qutil.parse_date(x['dt']) - qutil.parse_date(event['dt'])) <= self.window]
         
         if(len(self.events) == 0):
             return 0.0
diff --git a/scripts/dev_setup.py b/scripts/dev_setup.py
index 29c085b6..898c4442 100644
--- a/scripts/dev_setup.py
+++ b/scripts/dev_setup.py
@@ -13,16 +13,13 @@ import logging
 import datetime
 import random
 
-import simulator.qbt_server as qbt_server
+import qsim.simulator.backtest.util as qutil
 
 MINUTE_COUNT=390
 
-define("user_email", default="qbt@quantopian.com", help="email address for qbt user")
-define("password", default="foobar", help="password for qbt user")
-
 def db_main():
     tornado.options.parse_command_line()
-    connection, db = qbt_server.connect_db()
+    connection, db = qutil.connect_db()
     
     #create a user for testing
     salt, encrypted_password = qbt_server.encrypt_password(None, options.password)
diff --git a/scripts/jenkins.sh b/scripts/jenkins.sh
index 7ee0136a..31fb09dc 100755
--- a/scripts/jenkins.sh
+++ b/scripts/jenkins.sh
@@ -11,15 +11,15 @@ source /usr/local/bin/virtualenvwrapper.sh
 #create the scientific python virtualenv and copy to provide qsim base
 mkvirtualenv --no-site-packages scientific_base
 workon scientific_base
-./ordered_pip.sh requirements_sci.txt
+./scripts/ordered_pip.sh ./scripts/requirements_sci.txt
 deactivate
 #re-base qsim
 #rmvirtualenv qsim
 cpvirtualenv scientific_base qsim  
 
 workon qsim
-./scripts/ordered_pip.sh requirements.txt
-./scripts/ordered_pip.sh requirements_dev.txt
+./scripts/ordered_pip.sh ./scripts/requirements.txt
+./scripts/ordered_pip.sh ./scripts/requirements_dev.txt
 
 #setup the local mongodb
 python ./scripts/dev_setup.py 
diff --git a/scripts/requirements_dev.txt b/scripts/requirements_dev.txt
index a22d34ae..dc3a0691 100644
--- a/scripts/requirements_dev.txt
+++ b/scripts/requirements_dev.txt
@@ -1,14 +1,31 @@
+ipython==0.12
+
 # For debugger
 fancycompleter==0.2
 pyrepl==0.8.2
 Pygments==1.4
 pdbpp==0.7.2
 
-ipython==0.12
 
-# For unit tests
+# Testing
+unittest2
 nose==1.1.2
-unittest2==0.5.1
-requests==0.10.1
-nosexcover
-pylint
+coverage==3.5.1
+mock==0.7.2
+nosexcover==1.0.7
+pylint==0.25.1
+
+# Documentation
+docutils==0.8.1
+Sphinx==1.1.2
+
+# Task running
+Paver==1.0.5
+Paved==0.3
+
+# Testing
+nose==1.1.2
+coverage==3.5.1
+mock==0.7.2
+nosexcover==1.0.7
+pylint==0.25.1
\ No newline at end of file