reorganized packages, test passing again

simulator/backtest/util.py

@@ -0,0 +1,162 @@
+"""
+Small classes to assist with db access, timezone calculations, and so on.
+"""
+
+import datetime
+import pytz
+import json
+import logging
+import uuid
+import zmq
+
+class DocWrap():
+    """
+        Provides attribute access style on top of dictionary results from pymongo. 
+        Allows you to access result['field'] as result.field.
+        Aliases result['_id'] to result.id.
+        
+    """
+    def __init__(self, store=None):
+        if(store == None):
+            self.store = {}
+        else:
+            self.store = store.copy()
+        if(self.store.has_key('_id')):
+            self.store['id'] = self.store['_id']
+            del(self.store['_id'])
+        
+    def __setitem__(self,key,value):
+        if(key == '_id'):
+            self.store['id'] = value
+        else:
+            self.store[key] = value
+        
+    def __getitem__(self, key):
+        if self.store.has_key(key):
+            return self.store[key]
+            
+    def __getattr__(self,attrname):
+        if self.store.has_key(attrname):
+            return self.store[attrname]
+        else:
+            raise AttributeError("No attribute named {name}".format(name=attrname))
+            
+def parse_date(dt_str):
+    """parse strings according to the same format as generated by format_date"""
+    if(dt_str == None):
+        return None
+    parts = dt_str.split(".")
+    dt = datetime.datetime.strptime(parts[0], '%Y/%m/%d-%H:%M:%S').replace(microsecond=int(parts[1]+"000")).replace(tzinfo = pytz.utc)
+    return dt
+    
+def format_date(dt):
+    """Format the date into a date with millesecond resolution and string/alphabetical sorting that is equivalent to datetime sorting"""
+    if(dt == None):
+        return None
+    dt_str = dt.strftime('%Y/%m/%d-%H:%M:%S') + "." + str(dt.microsecond / 1000)
+    return dt_str
+    
+    
+class ParallelBuffer(object):
+    """ holds several queues of events by key, allows retrieval in date order or by merging"""
+    def __init__(self, key_list):
+        self.out_socket = None
+        self.sent_count = 0
+        self.received_count = 0
+        self.draining = False
+        self.data_buffer = {}
+        for key in key_list:
+            self.data_buffer[key] = []
+    
+    def __len__(self):
+        return len(self.data_buffer)        
+    
+    def append(self, key, value):
+        self.data_buffer[key].append(value)
+        self.received_count += 1
+    
+    def next(self):
+        if(not(self.is_full() or self.draining)):
+            return
+            
+        cur = None
+        earliest = None
+        for source, events in self.data_buffer.iteritems():
+            if len(events) == 0:
+                continue
+            cur = events
+            if(earliest == None) or (cur[0]['dt'] <= earliest[0]['dt']):
+                earliest = cur
+        
+        if(earliest != None):
+            return earliest.pop(0)
+        
+    def is_full(self):
+        for source, events in self.data_buffer.iteritems():
+            if (len(events) == 0):
+                return False
+        return True
+    
+    def pending_messages(self):
+        total = 0
+        for source, events in self.data_buffer.iteritems():
+            total += len(events)
+        return total       
+        
+    def drain(self):
+        self.draining = True
+        while(self.pending_messages() > 0):
+            self.send_next()
+            
+    def send_next(self):
+        if(not(self.is_full() or self.draining)):
+            return
+            
+        event = self.next()
+        if(event != None):        
+            self.out_socket.send(json.dumps(event))
+            self.sent_count += 1   
+    
+    
+class MergedParallelBuffer(ParallelBuffer):
+    
+    def __init__(self, keys):
+        ParallelBuffer.__init__(self, keys)
+        self.feed = []
+        self.data_buffer["feed"] = self.feed
+    
+    def next(self):
+        if(not(self.is_full() or self.draining)):
+            return
+        
+        result = self.feed.pop(0)
+        for source, events in self.data_buffer.iteritems():
+            if(source == "feed"):
+                continue
+            if(len(events) > 0):
+                cur = events.pop(0)
+                result[source] = cur['value']
+        return result
+        
+        
+class FeedSync(object):
+    
+    def __init__(self, feed, name):
+        self.feed = feed
+        self.id = "{name}-{id}".format(name=name, id=uuid.uuid1())
+        self.feed.register_sync(self.id)
+        self.logger = logging.getLogger()
+        #self.logger.info("registered {id} with feed".format(id=self.id))
+        
+    def confirm(self):
+        context = zmq.Context()
+        #synchronize with feed
+        sync_socket = context.socket(zmq.REQ)
+        sync_socket.connect(self.feed.sync_address)
+        # send a synchronization request to the feed
+        sync_socket.send(self.id)
+        # wait for synchronization reply from the feed
+        sync_socket.recv()
+        sync_socket.close()
+        context.term()
+        self.logger.info("sync'd feed from {id}".format(id = self.id))

simulator/config.py

@@ -0,0 +1,43 @@
+import json
+
+class Config(object):
+    """ Name/Value configuration object with type-safe accessors and json serialization/deserialization."""   
+    
+    def __init__(self, props):
+        self.store = props
+        
+    def __setitem__(self,key,value):
+        self.store[key] = value
+        
+    def __getitem__(self, key):
+        if self.store.has_key(key):
+            return self.store[key]
+            
+    def __getattr__(self,attrname):
+        if self.store.has_key(attrname):
+            return self.store[attrname]
+        else:
+            raise AttributeError("No attribute named {name}".format(name=attrname))
+        
+    def get_integer(self, name, default=0):
+        return self.get_value(name, default, type(1))
+        
+    def get_string(self, name, default=''):
+        return self.get_value(name, default, type(''))
+        
+    def get_float(self, name, default=0.0):
+        return self.get_value(name, default, type(1.0))
+    
+    def get_value(self, name, default, expected_type):
+        if(self.store.has_key(name)):
+                val = self.store[name]
+                if isinstance(val, expected_type):
+                    return val
+        else:
+            return default
+                    
+    def to_json(self):
+        return json.dumps(self.store)
+    
+    def from_json(self, json_string):
+        self.store = json.loads(json_string)

simulator/data/feed.py

@@ -0,0 +1,117 @@
+
+from simulator.data.sources.equity import *
+from simulator.backtest.util import *
+import time
+import logging
+
+class DataFeed(object):
+    
+    def __init__(self, config):
+        self.logger = logging.getLogger()
+        
+        self.data_address = "tcp://127.0.0.1:{port}".format(port=10101)
+        self.sync_address = "tcp://127.0.0.1:{port}".format(port=10102)
+        self.feed_address = "tcp://127.0.0.1:{port}".format(port=10103)
+        
+        self.client_register = {}
+        
+        self.data_workers = {}
+        self.config = config
+        for name, info in config.iteritems():
+            if(info['class'] == "EquityMinuteTrades"):
+                emt = EquityMinuteTrades(info['sid'], self, name)
+                self.data_workers[name] = emt
+            elif(info['class'] == "RandomEquityTrades"):
+                ret = RandomEquityTrades(info['sid'], self, name, info['count'])
+                self.data_workers[name] = ret
+                
+        self.data_buffer = ParallelBuffer(self.data_workers.keys())
+             
+    def start_data_workers(self):
+        """Start a sub-process for each datasource.""" 
+        for source_id, source in self.data_workers.iteritems():
+            self.logger.info("starting {id}".format(id=source_id))
+            source.start()
+        self.logger.info("ds processes launched")
+        
+    def register_sync(self, sync_id):
+        self.client_register[sync_id] = "UNCONFIRMED"
+        
+    def registration_complete(self):
+        for sync_id, status in self.client_register.iteritems():
+            if status == "UNCONFIRMED":
+                return False
+        
+        return True
+    
+    def sync_clients(self):
+        # Socket to receive signals
+        self.logger.info("waiting for all datasources and clients to be ready")
+        self.syncservice = self.context.socket(zmq.REP)
+        self.syncservice.bind(self.sync_address) 
+        
+        while not self.registration_complete():
+            # wait for synchronization request
+            msg = self.syncservice.recv()
+            self.client_register[msg] = "CONFIRMED"
+            #self.logger.info("confirmed {id}".format(id=msg))
+            # send synchronization reply
+            self.syncservice.send('CONFIRMED')
+        
+        self.syncservice.close()
+        self.logger.info("sync'd all datasources and clients")
+       
+    def run(self):   
+        # Prepare our context and sockets
+        self.context = zmq.Context()
+        
+        ds_finished_counter = 0
+             
+        #create the data sink. Based on http://zguide.zeromq.org/py:tasksink2 
+        #see: http://zguide.zeromq.org/py:taskwork2
+        self.data_socket = self.context.socket(zmq.PULL)
+        self.data_socket.bind(self.data_address)
+        
+        #create the feed
+        self.feed_socket = self.context.socket(zmq.PUB)
+        self.feed_socket.bind(self.feed_address)
+        
+        self.data_buffer.out_socket = self.feed_socket
+        
+        #start the data source workers
+        self.start_data_workers()
+        
+        #wait for all feed subscribers
+        self.sync_clients()
+        
+        self.logger.info("entering feed loop on {addr}".format(addr=self.data_address))
+        
+        while True:
+            message = self.data_socket.recv()
+            event = json.loads(message)
+            if(event["type"] == "DONE"):
+                ds_finished_counter += 1
+                if(len(self.data_workers) == ds_finished_counter):
+                    break
+            else:
+                self.data_buffer.append(event[u's'], event)
+                self.data_buffer.send_next()
+            
+                
+        #drain any remaining messages in the buffer
+        self.data_buffer.drain()
+        
+        #send the DONE message
+        self.feed_socket.send("DONE")
+        self.logger.info("received {n} messages, sent {m} messages".format(n=self.data_buffer.received_count, m=self.data_buffer.sent_count))
+        self.data_socket.close()
+        self.feed_socket.close()
+        self.context.term()
+        
+            
+       
+        
+        
+
+        
+            

simulator/data/sources/equity.py

@@ -0,0 +1,110 @@
+import datetime
+import zmq
+import pymongo
+import pymongo.json_util
+import json
+import pytz
+import copy
+import multiprocessing
+import logging
+import random
+from pymongo import ASCENDING, DESCENDING
+
+from simulator.backtest.util import *
+
+from simulator.qbt_server import * #connect_db
+
+class DataSource(object):
+    def __init__(self, feed, source_id):
+        self.source_id              = source_id
+        self.logger                 = logging.getLogger()
+        self.feed                   = feed
+        self.sync                   = FeedSync(self.feed, str(source_id))
+        self.data_address           = self.feed.data_address
+        self.logger.info("data address is {ds}".format(ds=self.feed.data_address))        
+        self.cur_event = None
+
+    def start(self):
+        self.proc = multiprocessing.Process(target=self.run)
+        self.proc.start()
+        
+        
+    def open(self):    
+        self.logger.info("starting data source:{source_id} on {addr}".format(source_id=self.source_id, addr=self.feed.data_address))
+        self.context = zmq.Context()
+        
+        #create the data sink. Based on http://zguide.zeromq.org/py:tasksink2 
+        self.data_socket = self.context.socket(zmq.PUSH)
+        self.data_socket.connect(self.data_address)
+        
+        #signal we are ready
+        self.sync.confirm()
+    
+    def run(self):    
+        try:    
+            self.open()
+            self.send_all()
+            self.close()    
+        except Exception as err:
+            self.logger.error(err, "Unexpected failure running datasource - {name}.".format(name=name))
+            
+    def send(self, event):
+        event['s'] = self.source_id             
+        event['type'] = 'event'
+        self.data_socket.send(json.dumps(event))
+    
+    def close(self):
+        done_msg = {}
+        done_msg['type'] = 'DONE'
+        done_msg['s'] = self.source_id
+        self.data_socket.send(json.dumps(done_msg))   
+        self.data_socket.close()
+        self.context.term()
+        self.logger.info("finished processing data source")
+
+class EquityMinuteTrades(DataSource):
+    
+    def __init__(self, sid, feed, source_id):
+        self.sid = sid
+        self.connection, self.db    = connect_db()
+        DataSource.__init__(self, feed, source_id)
+               
+     
+    def send_all(self):   
+        eventQS = self.db.equity.trades.minute.find(fields=["sid","price","volume","dt"],
+                                     spec={"sid":self.sid},
+                                     sort=[("dt",ASCENDING)],
+                                     slave_ok=True)
+        self.logger.info("found {count} events".format(count=eventQS.count()))
+        
+        for doc in eventQS:
+            doc_dt = doc['dt'].replace(tzinfo = pytz.utc)
+            doc_dt_str = format_date(doc_dt)
+            event = copy.copy(doc)
+            event['dt'] = doc_dt_str
+            del(event['_id'])
+            self.send(event)
+            
+        
+        
+class RandomEquityTrades(DataSource):
+    
+    def __init__(self, sid, feed, source_id, count):
+        DataSource.__init__(self, feed, source_id)
+        self.count = count
+        self.sid = sid
+        
+    def send_all(self):
+        trade_start = datetime.datetime.now()
+        minute = datetime.timedelta(minutes=1)
+        price = random.uniform(5.0,50.0)
+        
+        for i in range(self.count):
+            price = price + random.uniform(-0.05,0.05)
+            event = {'sid':self.sid, 'dt':format_date(trade_start + (minute * i)),'price':price, 'volume':random.randrange(100,10000,100)}
+            self.send(event)
+          
+   
+       
+        
+        

simulator/host_settings.py

@@ -0,0 +1,14 @@
+class Config(object):
+   def __init__(self, dct):
+       self.__dict__.update(dct)
+
+mongo_conn_args = Config({
+   'mongodb_host'         :  'claire.mongohq.com',
+   'mongodb_port'         :  10087,
+   'mongodb_dbname'       :  'quantodata-staging',
+   'mongodb_user'         :  'quantopian',
+   'mongodb_password'     :  'quantopian',
+})
+
+root_url = 'http://localhost:8000'
+ws_url = 'ws://localhost:8001'

simulator/qbt.py

@@ -0,0 +1,67 @@
+"""
+QBT - Quantopian Backtest
+====================================
+
+qbt runs backtests using multiple processes and zeromq messaging for communication and coordination. 
+
+Backtest is the primary process. It maintains both server and client sockets:
+zmq sockets for internal processing::
+
+    - data sink,        ZMQ.REQ. Port = port_start + 1
+        - backtest will connect to socket, and then spawn one process per datasource, passing the data sink url as a startup arg. Each
+          datasource process will bind to the socket, and start processing 
+        - backtest is responsible for merging the data events from all sources into a serialized stream and relaying it to the 
+          aggregators, merging agg results, and transmitting consolidated stream to event feed.
+    - agg source,       ZMQ.PUSH. Port = port_start + 2
+    - agg sink,         ZMQ.PULL. Port = port_start + 3
+    - control source,   ZMQ.PUB.  Port = port_start + 4
+        - all child processes must subscribe to this socket. Control commands:
+            - START -- begin processing
+            - TIME  -- current simulated time in backtest
+            - KILL  -- exit immediately
+            
+zmq sockets for backtest clients:
+=================================
+    - orders sink,      ZMQ.RESP. Port = port_start + 5
+        - backtest will connect (can you bind?) to this socket and await orders from the client. Order data will be processed against the streaming datafeed.
+    - event feed,       ZMQ.RESP. Port = port_start + 6
+        - backtest will bind to this socket and respond to requests from client for more data. Response data will be the queue of events that
+          transpired since the last request.
+           
+    
+"""
+import copy
+import multiprocessing
+import zmq
+
+from backtest.util import *
+from data.sources.equity import *
+
+
+
+CONTROLLER_PORT     =  9000
+DATA_SINK_PORT      = 10000
+DATA_FEED_PORT      = 30000
+
+class Backtest(object):
+    
+    def __init__(self, db, logger):
+        self.logger = logger
+        self.db = db
+        self.feed = DataFeed(db, logger)
+        
+    def start_feed(self):
+        proc1 = multiprocessing.Process(target=feed.run)
+        proc1.start()
+       
+    def run(self):   
+        # Prepare our context and sockets
+        self.context = zmq.Context()
+        
+        #create the feed sink. 
+        self.feed_address = self.feed.data_address
+        self.feed_socket = self.context.connect(self.feed_address)
+        self.feed_socket.connect(zmq.PULL)
+        
+
+      

simulator/qbt_client.py

@@ -0,0 +1,55 @@
+import copy
+import multiprocessing
+import zmq
+import logging
+import json
+
+from backtest.util import *
+
+class TestClient(object):
+    
+    def __init__(self,feed, address, bind=False):
+        self.logger         = logging.getLogger()
+        self.feed           = feed
+        self.address        = address
+        self.sync           = FeedSync(feed, "testclient")
+        self.bind           = bind
+        self.received_count = 0
+        
+    def run(self):
+        
+        self.logger.info("running the client")
+        self.context = zmq.Context()
+        
+        self.data_feed = self.context.socket(zmq.PULL)
+
+        if(self.bind):
+            self.logger.info("binding to {address}".format(address=self.address))
+            self.data_feed.bind(self.address)
+        else:
+            self.logger.info("connecting to {address}".format(address=self.address))
+            self.data_feed.connect(self.address)
+        
+        self.sync.confirm()
+        
+        self.logger.info("Starting the client loop")
+
+        prev_dt = None
+        while True:
+            msg = self.data_feed.recv()
+            if(msg == "DONE"):
+                self.logger.info("DONE!")
+                break
+            self.received_count += 1
+            event = json.loads(msg)
+            if(prev_dt != None):
+                if(not event['dt'] >= prev_dt):
+                    raise Exception("message arrived out of order: {date} after {prev}".format(date=event['dt'], prev=prev_dt))
+            
+            prev_dt = event['dt']
+            if(self.received_count % 100 == 0):
+                self.logger.info("received {n} messages".format(n=self.received_count))
+            
+        self.logger.info("received {n} messages".format(n=self.received_count))
+        self.data_feed.close()
+        self.context.term()

simulator/qbt_server.py

@@ -0,0 +1,148 @@
+import tornado.auth
+import tornado.httpserver
+import tornado.ioloop
+from tornado.options import define, options
+import tornado.web
+import pymongo
+import bson
+import hashlib
+import base64
+import uuid
+import os
+import logging
+import datetime
+import multiprocessing
+
+from qbt import *
+from qbt_client import * 
+
+logger = logging.getLogger()
+
+define("port", default=8888, help="run the qbt on the given port", type=int)
+define("mongodb_host", default="127.0.0.1", help="mongodb host address")
+define("mongodb_port", default=27017, help="connect to the mongodb on the given port", type=int)
+define("mongodb_dbname", default="qbt", help="database name")
+define("mongodb_user", default="qbt", help="database user")
+define("mongodb_password", default="qbt", help="database password")
+
+HASH_ALGO = 'sha256'
+
+def connect_db():
+    connection = pymongo.Connection(options.mongodb_host, options.mongodb_port)
+    db = connection[options.mongodb_dbname]
+    db.authenticate(options.mongodb_user, options.mongodb_password)
+    return connection, db
+    
+def encrypt_password(salt, password):
+    if(salt == None):
+        h1 = hashlib.new(HASH_ALGO)
+        h1.update(str(datetime.datetime.utcnow())+"--"+password)
+        salt = h1.hexdigest()
+        
+    h2 = hashlib.new(HASH_ALGO)
+    h2.update(salt+"--"+password)
+    encrypted_password = h2.hexdigest()
+    
+    return salt, encrypted_password
+
+class Application(tornado.web.Application):
+    def __init__(self):
+        handlers = [
+            (r"/", MainHandler),
+            (r"/login", LoginHandler),
+            (r"/backtest", BacktestHandler)
+        ]
+        settings = dict(
+            template_path=os.path.join(os.path.dirname(__file__), "templates"),
+            static_path=os.path.join(os.path.dirname(__file__), "static"),
+            xsrf_cookies=False,
+            cookie_secret=base64.b64encode(uuid.uuid4().bytes + uuid.uuid4().bytes),
+            login_url="/login",
+            #autoescape=None,
+            #debug=True,
+        )
+        tornado.web.Application.__init__(self, handlers, **settings)
+
+        # Have one global connection to the blog DB across all handlers
+        self.connection, self.db = connect_db()
+
+
+class BaseHandler(tornado.web.RequestHandler):
+    @property
+    def db(self):
+        return self.application.db
+
+    def get_current_user(self):
+        return "fawce"
+        user_id = self.get_secure_cookie(u"user_id")
+        logger.info("looking up user with id: {id}".format(id=user_id))
+        if not user_id: return None
+        #get user record by id
+        users = self.db.users.find(spec={"_id":bson.ObjectId(user_id)}, limit=1)
+        if(users.count > 0):
+            return users[0]
+        return None
+
+class MainHandler(BaseHandler):
+    @tornado.web.authenticated
+    def get(self):
+        self.write("Hello, world. Try launching a <a href='/backtest'>backtest</a>.")
+                         
+class LoginHandler(BaseHandler):
+    def get(self):
+        self.write('<html><body><form action="/login" method="post">'
+                   'Name: <input type="text" name="user_name">'
+                   'pass: <input type="password" name="password">'
+                   '<input type="submit" value="Sign in">'
+                   '</form></body></html>')
+
+    def post(self):  
+        self.authenticate(self.get_argument("user_name"),self.get_argument("password"))
+        self.redirect("/")
+
+    def authenticate(self, username, password):
+        h = hashlib.new(HASH_ALGO)
+        #find user record by username.
+        users = self.db.users.find(spec={"email":username}, limit=1)
+        if(users.count > 0):
+            user_record = users[0]
+        else:
+            logger.debug("no user with name: {username}", username=username)
+            return
+
+        
+        #calculate password hash
+        salt, encrypted_password = encrypt_password(user_record['salt'], password)
+        if (user_record['encrypted_password'] == encrypted_password):
+            #we have a match, so set the secure cookie to the salt
+            logger.debug("setting user_id cookie to {id}".format(id=user_record['_id']))
+            self.set_secure_cookie(u"user_id", unicode(user_record['_id']))
+
+class BacktestHandler(BaseHandler):
+    @tornado.web.authenticated
+    def get(self):
+        self.write('<html><body><form action="/backtest" method="post">'
+                   '<input type="submit" value="Launch">'
+                   '</form></body></html>')
+    @tornado.web.authenticated
+    def post(self):
+        
+        bt = Backtest(self.db, logger)
+        #btc = BacktestClient(DATA_FEED_PORT, logger)
+        bt_proc = multiprocessing.Process(target=bt.run)
+        #btc_proc = multiprocessing.Process(target=btc.run)
+        bt_proc.start()
+        #btc_proc.start()
+
+
+
+    
+def main():
+    tornado.options.parse_command_line()
+    http_server = tornado.httpserver.HTTPServer(Application())
+    http_server.listen(options.port)
+    tornado.ioloop.IOLoop.instance().start()
+
+
+if __name__ == "__main__":
+    main()