mirror of
https://github.com/wassname/catalyst.git
synced 2026-08-04 12:45:06 +08:00
eliminated tornado and pymongo, more package refactoring, test passes
This commit is contained in:
fawce
@@ -1,189 +0,0 @@
|
||||
"""
|
||||
Small classes to assist with db access, timezone calculations, and so on.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import pytz
|
||||
import json
|
||||
import logging
|
||||
import uuid
|
||||
import zmq
|
||||
import pymongo
|
||||
from tornado.options import define, options
|
||||
|
||||
logger = logging.getLogger('QSimLogger')
|
||||
|
||||
class DocWrap():
|
||||
"""
|
||||
Provides attribute access style on top of dictionary results from pymongo.
|
||||
Allows you to access result['field'] as result.field.
|
||||
Aliases result['_id'] to result.id.
|
||||
|
||||
"""
|
||||
def __init__(self, store=None):
|
||||
if(store == None):
|
||||
self.store = {}
|
||||
else:
|
||||
self.store = store.copy()
|
||||
if(self.store.has_key('_id')):
|
||||
self.store['id'] = self.store['_id']
|
||||
del(self.store['_id'])
|
||||
|
||||
def __setitem__(self,key,value):
|
||||
if(key == '_id'):
|
||||
self.store['id'] = value
|
||||
else:
|
||||
self.store[key] = value
|
||||
|
||||
def __getitem__(self, key):
|
||||
if self.store.has_key(key):
|
||||
return self.store[key]
|
||||
|
||||
def __getattr__(self,attrname):
|
||||
if self.store.has_key(attrname):
|
||||
return self.store[attrname]
|
||||
else:
|
||||
raise AttributeError("No attribute named {name}".format(name=attrname))
|
||||
|
||||
def parse_date(dt_str):
|
||||
"""parse strings according to the same format as generated by format_date"""
|
||||
if(dt_str == None):
|
||||
return None
|
||||
parts = dt_str.split(".")
|
||||
dt = datetime.datetime.strptime(parts[0], '%Y/%m/%d-%H:%M:%S').replace(microsecond=int(parts[1]+"000")).replace(tzinfo = pytz.utc)
|
||||
return dt
|
||||
|
||||
def format_date(dt):
|
||||
"""Format the date into a date with millesecond resolution and string/alphabetical sorting that is equivalent to datetime sorting"""
|
||||
if(dt == None):
|
||||
return None
|
||||
dt_str = dt.strftime('%Y/%m/%d-%H:%M:%S') + "." + str(dt.microsecond / 1000)
|
||||
return dt_str
|
||||
|
||||
|
||||
class ParallelBuffer(object):
|
||||
""" holds several queues of events by key, allows retrieval in date order or by merging"""
|
||||
def __init__(self, key_list):
|
||||
self.out_socket = None
|
||||
self.sent_count = 0
|
||||
self.received_count = 0
|
||||
self.draining = False
|
||||
self.data_buffer = {}
|
||||
for key in key_list:
|
||||
self.data_buffer[key] = []
|
||||
|
||||
def __len__(self):
|
||||
return len(self.data_buffer)
|
||||
|
||||
def append(self, key, value):
|
||||
self.data_buffer[key].append(value)
|
||||
self.received_count += 1
|
||||
|
||||
def next(self):
|
||||
if(not(self.is_full() or self.draining)):
|
||||
return
|
||||
|
||||
cur = None
|
||||
earliest = None
|
||||
for source, events in self.data_buffer.iteritems():
|
||||
if len(events) == 0:
|
||||
continue
|
||||
cur = events
|
||||
if(earliest == None) or (cur[0]['dt'] <= earliest[0]['dt']):
|
||||
earliest = cur
|
||||
|
||||
if(earliest != None):
|
||||
return earliest.pop(0)
|
||||
|
||||
def is_full(self):
|
||||
for source, events in self.data_buffer.iteritems():
|
||||
if (len(events) == 0):
|
||||
return False
|
||||
return True
|
||||
|
||||
def pending_messages(self):
|
||||
total = 0
|
||||
for source, events in self.data_buffer.iteritems():
|
||||
total += len(events)
|
||||
return total
|
||||
|
||||
def drain(self):
|
||||
self.draining = True
|
||||
while(self.pending_messages() > 0):
|
||||
self.send_next()
|
||||
|
||||
def send_next(self):
|
||||
if(not(self.is_full() or self.draining)):
|
||||
return
|
||||
|
||||
event = self.next()
|
||||
if(event != None):
|
||||
self.out_socket.send(json.dumps(event))
|
||||
self.sent_count += 1
|
||||
|
||||
|
||||
class MergedParallelBuffer(ParallelBuffer):
|
||||
|
||||
def __init__(self, keys):
|
||||
ParallelBuffer.__init__(self, keys)
|
||||
self.feed = []
|
||||
self.data_buffer["feed"] = self.feed
|
||||
|
||||
def next(self):
|
||||
if(not(self.is_full() or self.draining)):
|
||||
return
|
||||
|
||||
result = self.feed.pop(0)
|
||||
for source, events in self.data_buffer.iteritems():
|
||||
if(source == "feed"):
|
||||
continue
|
||||
if(len(events) > 0):
|
||||
cur = events.pop(0)
|
||||
result[source] = cur['value']
|
||||
return result
|
||||
|
||||
|
||||
class FeedSync(object):
|
||||
|
||||
def __init__(self, feed, name):
|
||||
self.feed = feed
|
||||
self.id = "{name}-{id}".format(name=name, id=uuid.uuid1())
|
||||
self.feed.register_sync(self.id)
|
||||
self.logger = logger
|
||||
#self.logger.info("registered {id} with feed".format(id=self.id))
|
||||
|
||||
def confirm(self):
|
||||
context = zmq.Context()
|
||||
#synchronize with feed
|
||||
sync_socket = context.socket(zmq.REQ)
|
||||
sync_socket.connect(self.feed.sync_address)
|
||||
# send a synchronization request to the feed
|
||||
sync_socket.send(self.id)
|
||||
# wait for synchronization reply from the feed
|
||||
sync_socket.recv()
|
||||
sync_socket.close()
|
||||
context.term()
|
||||
self.logger.info("sync'd feed from {id}".format(id = self.id))
|
||||
|
||||
|
||||
define("user_email", default="qbt@quantopian.com", help="email address for qbt user")
|
||||
define("password", default="foobar", help="password for qbt user")
|
||||
define("port", default=8888, help="run the qbt on the given port", type=int)
|
||||
define("mongodb_host", default="127.0.0.1", help="mongodb host address")
|
||||
define("mongodb_port", default=27017, help="connect to the mongodb on the given port", type=int)
|
||||
define("mongodb_dbname", default="qbt", help="database name")
|
||||
define("mongodb_user", default="qbt", help="database user")
|
||||
define("mongodb_password", default="qbt", help="database password")
|
||||
|
||||
def connect_db():
|
||||
connection = pymongo.Connection(options.mongodb_host, options.mongodb_port)
|
||||
db = connection[options.mongodb_dbname]
|
||||
db.authenticate(options.mongodb_user, options.mongodb_password)
|
||||
return connection, db
|
||||
|
||||
def configure_logging(loglevel=logging.DEBUG):
|
||||
logger.setLevel(loglevel)
|
||||
handler = logging.handlers.RotatingFileHandler("/tmp/{lfn}.log".format(lfn="qsim-log"), maxBytes=10*1024*1024, backupCount=5)
|
||||
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(filename)s %(funcName)s - %(message)s","%Y-%m-%d %H:%M:%S %Z"))
|
||||
logger.addHandler(handler)
|
||||
logger.info("logging started...")
|
||||
@@ -1,108 +0,0 @@
|
||||
import datetime
|
||||
import zmq
|
||||
import pymongo
|
||||
import pymongo.json_util
|
||||
import json
|
||||
import pytz
|
||||
import copy
|
||||
import multiprocessing
|
||||
import logging
|
||||
import random
|
||||
from pymongo import ASCENDING, DESCENDING
|
||||
|
||||
import qsim.simulator.backtest.util as qutil
|
||||
|
||||
class DataSource(object):
|
||||
def __init__(self, feed, source_id):
|
||||
self.source_id = source_id
|
||||
self.logger = qutil.logger
|
||||
self.feed = feed
|
||||
self.sync = qutil.FeedSync(self.feed, str(source_id))
|
||||
self.data_address = self.feed.data_address
|
||||
self.logger.info("data address is {ds}".format(ds=self.feed.data_address))
|
||||
self.cur_event = None
|
||||
|
||||
def start(self):
|
||||
self.proc = multiprocessing.Process(target=self.run)
|
||||
self.proc.start()
|
||||
|
||||
|
||||
def open(self):
|
||||
self.logger.info("starting data source:{source_id} on {addr}".format(source_id=self.source_id, addr=self.feed.data_address))
|
||||
self.context = zmq.Context()
|
||||
|
||||
#create the data sink. Based on http://zguide.zeromq.org/py:tasksink2
|
||||
self.data_socket = self.context.socket(zmq.PUSH)
|
||||
self.data_socket.connect(self.data_address)
|
||||
|
||||
#signal we are ready
|
||||
self.sync.confirm()
|
||||
|
||||
def run(self):
|
||||
try:
|
||||
self.open()
|
||||
self.send_all()
|
||||
self.close()
|
||||
except Exception as err:
|
||||
self.logger.exception("Unexpected failure running datasource - {name}.".format(name=self.source_id))
|
||||
|
||||
def send(self, event):
|
||||
event['s'] = self.source_id
|
||||
event['type'] = 'event'
|
||||
self.data_socket.send(json.dumps(event))
|
||||
|
||||
def close(self):
|
||||
done_msg = {}
|
||||
done_msg['type'] = 'DONE'
|
||||
done_msg['s'] = self.source_id
|
||||
self.data_socket.send(json.dumps(done_msg))
|
||||
self.data_socket.close()
|
||||
self.context.term()
|
||||
self.logger.info("finished processing data source")
|
||||
|
||||
class EquityMinuteTrades(DataSource):
|
||||
|
||||
def __init__(self, sid, feed, source_id):
|
||||
self.sid = sid
|
||||
self.connection, self.db = qutil.connect_db()
|
||||
DataSource.__init__(self, feed, source_id)
|
||||
|
||||
|
||||
def send_all(self):
|
||||
eventQS = self.db.equity.trades.minute.find(fields=["sid","price","volume","dt"],
|
||||
spec={"sid":self.sid},
|
||||
sort=[("dt",ASCENDING)],
|
||||
slave_ok=True)
|
||||
self.logger.info("found {count} events".format(count=eventQS.count()))
|
||||
|
||||
for doc in eventQS:
|
||||
doc_dt = doc['dt'].replace(tzinfo = pytz.utc)
|
||||
doc_dt_str = qutil.format_date(doc_dt)
|
||||
event = copy.copy(doc)
|
||||
event['dt'] = doc_dt_str
|
||||
del(event['_id'])
|
||||
self.send(event)
|
||||
|
||||
|
||||
|
||||
class RandomEquityTrades(DataSource):
|
||||
|
||||
def __init__(self, sid, feed, source_id, count):
|
||||
DataSource.__init__(self, feed, source_id)
|
||||
self.count = count
|
||||
self.sid = sid
|
||||
|
||||
def send_all(self):
|
||||
trade_start = datetime.datetime.now()
|
||||
minute = datetime.timedelta(minutes=1)
|
||||
price = random.uniform(5.0,50.0)
|
||||
|
||||
for i in range(self.count):
|
||||
price = price + random.uniform(-0.05,0.05)
|
||||
event = {'sid':self.sid, 'dt':qutil.format_date(trade_start + (minute * i)),'price':price, 'volume':random.randrange(100,10000,100)}
|
||||
self.send(event)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
|
||||
import qsim.simulator.data.sources.equity as qequity
|
||||
import qsim.simulator.backtest.util as qutil
|
||||
import qsim.data.equity as qequity
|
||||
import qsim.util as qutil
|
||||
import zmq
|
||||
import time
|
||||
import logging
|
||||
@@ -34,8 +34,8 @@ import copy
|
||||
import multiprocessing
|
||||
import zmq
|
||||
|
||||
import qsim.backtest.util as qutil
|
||||
import qsim.data.sources.equity as qequity
|
||||
import qsim.util as qutil
|
||||
import qsim.data.equity as qequity
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ import zmq
|
||||
import logging
|
||||
import json
|
||||
|
||||
import qsim.simulator.backtest.util as qutil
|
||||
import qsim.util as qutil
|
||||
|
||||
class TestClient(object):
|
||||
|
||||
|
||||
Reference in New Issue
Block a user