wassname/catalyst
1
0
Fork 0
mirror of https://github.com/wassname/catalyst.git synced 2026-08-01 12:20:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

added qsim top level package, added rolling log file, test passes

Browse Source
This commit is contained in:
fawce
2012-02-07 23:13:41 -05:00
parent cca65808d5
commit 9078bf922b
13 changed files with 90 additions and 206 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pylint.out
+1
View File
@@ -0,0 +1 @@
__init__.py:1: [F] error while code parsing: Unable to load file '__init__.py' ([Errno 2] No such file or directory: '__init__.py')
qsim/simulator/backtest/util.py
+28 -2
View File
@@ -8,6 +8,9 @@ import json
import logging
import uuid
import zmq
from tornado.options import define, options
logger = logging.getLogger('QSimLogger')
class DocWrap():
"""
@@ -145,7 +148,7 @@ class FeedSync(object):
self.feed = feed
self.id = "{name}-{id}".format(name=name, id=uuid.uuid1())
self.feed.register_sync(self.id)
self.logger = logging.getLogger()
self.logger = logger
#self.logger.info("registered {id} with feed".format(id=self.id))
def confirm(self):
@@ -159,4 +162,27 @@ class FeedSync(object):
sync_socket.recv()
sync_socket.close()
context.term()
self.logger.info("sync'd feed from {id}".format(id = self.id))
self.logger.info("sync'd feed from {id}".format(id = self.id))
define("user_email", default="qbt@quantopian.com", help="email address for qbt user")
define("password", default="foobar", help="password for qbt user")
define("port", default=8888, help="run the qbt on the given port", type=int)
define("mongodb_host", default="127.0.0.1", help="mongodb host address")
define("mongodb_port", default=27017, help="connect to the mongodb on the given port", type=int)
define("mongodb_dbname", default="qbt", help="database name")
define("mongodb_user", default="qbt", help="database user")
define("mongodb_password", default="qbt", help="database password")
def connect_db():
connection = pymongo.Connection(options.mongodb_host, options.mongodb_port)
db = connection[options.mongodb_dbname]
db.authenticate(options.mongodb_user, options.mongodb_password)
return connection, db
def configure_logging(loglevel=logging.DEBUG):
logger.setLevel(loglevel)
handler = logging.handlers.RotatingFileHandler("/tmp/{lfn}.log".format(lfn="qsim-log"), maxBytes=10*1024*1024, backupCount=5)
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(filename)s %(funcName)s - %(message)s","%Y-%m-%d %H:%M:%S %Z"))
logger.addHandler(handler)
logger.info("logging started...")
qsim/simulator/data/feed.py
+7 -5
View File
@@ -1,13 +1,15 @@
from simulator.data.sources.equity import *
from simulator.backtest.util import *
import qsim.simulator.data.sources.equity as qequity
import qsim.simulator.backtest.util as qutil
import zmq
import time
import logging
import json
class DataFeed(object):
def __init__(self, config):
self.logger = logging.getLogger()
self.logger = qutil.logger
self.data_address = "tcp://127.0.0.1:{port}".format(port=10101)
self.sync_address = "tcp://127.0.0.1:{port}".format(port=10102)
@@ -22,10 +24,10 @@ class DataFeed(object):
emt = EquityMinuteTrades(info['sid'], self, name)
self.data_workers[name] = emt
elif(info['class'] == "RandomEquityTrades"):
ret = RandomEquityTrades(info['sid'], self, name, info['count'])
ret = qequity.RandomEquityTrades(info['sid'], self, name, info['count'])
self.data_workers[name] = ret
self.data_buffer = ParallelBuffer(self.data_workers.keys())
self.data_buffer = qutil.ParallelBuffer(self.data_workers.keys())
def start_data_workers(self):
"""Start a sub-process for each datasource."""
qsim/simulator/data/sources/equity.py
+7 -9
View File
@@ -10,16 +10,14 @@ import logging
import random
from pymongo import ASCENDING, DESCENDING
from simulator.backtest.util import *
from simulator.qbt_server import * #connect_db
import qsim.simulator.backtest.util as qutil
class DataSource(object):
def __init__(self, feed, source_id):
self.source_id = source_id
self.logger = logging.getLogger()
self.logger = qutil.logger
self.feed = feed
self.sync = FeedSync(self.feed, str(source_id))
self.sync = qutil.FeedSync(self.feed, str(source_id))
self.data_address = self.feed.data_address
self.logger.info("data address is {ds}".format(ds=self.feed.data_address))
self.cur_event = None
@@ -46,7 +44,7 @@ class DataSource(object):
self.send_all()
self.close()
except Exception as err:
self.logger.error(err, "Unexpected failure running datasource - {name}.".format(name=name))
self.logger.exception("Unexpected failure running datasource - {name}.".format(name=self.source_id))
def send(self, event):
event['s'] = self.source_id
@@ -66,7 +64,7 @@ class EquityMinuteTrades(DataSource):
def __init__(self, sid, feed, source_id):
self.sid = sid
self.connection, self.db = connect_db()
self.connection, self.db = qutil.connect_db()
DataSource.__init__(self, feed, source_id)
@@ -79,7 +77,7 @@ class EquityMinuteTrades(DataSource):
for doc in eventQS:
doc_dt = doc['dt'].replace(tzinfo = pytz.utc)
doc_dt_str = format_date(doc_dt)
doc_dt_str = qutil.format_date(doc_dt)
event = copy.copy(doc)
event['dt'] = doc_dt_str
del(event['_id'])
@@ -101,7 +99,7 @@ class RandomEquityTrades(DataSource):
for i in range(self.count):
price = price + random.uniform(-0.05,0.05)
event = {'sid':self.sid, 'dt':format_date(trade_start + (minute * i)),'price':price, 'volume':random.randrange(100,10000,100)}
event = {'sid':self.sid, 'dt':qutil.format_date(trade_start + (minute * i)),'price':price, 'volume':random.randrange(100,10000,100)}
self.send(event)
qsim/simulator/host_settings.py
-14
View File
@@ -1,14 +0,0 @@
class Config(object):
def __init__(self, dct):
self.__dict__.update(dct)
mongo_conn_args = Config({
'mongodb_host' : 'claire.mongohq.com',
'mongodb_port' : 10087,
'mongodb_dbname' : 'quantodata-staging',
'mongodb_user' : 'quantopian',
'mongodb_password' : 'quantopian',
})
root_url = 'http://localhost:8000'
ws_url = 'ws://localhost:8001'
qsim/simulator/qbt.py
+2 -2
View File
@@ -34,8 +34,8 @@ import copy
import multiprocessing
import zmq
from backtest.util import *
from data.sources.equity import *
import qsim.backtest.util as qutil
import qsim.data.sources.equity as qequity
qsim/simulator/qbt_client.py
+3 -3
View File
@@ -4,15 +4,15 @@ import zmq
import logging
import json
from backtest.util import *
import qsim.simulator.backtest.util as qutil
class TestClient(object):
def __init__(self,feed, address, bind=False):
self.logger = logging.getLogger()
self.logger = qutil.logger
self.feed = feed
self.address = address
self.sync = FeedSync(feed, "testclient")
self.sync = qutil.FeedSync(feed, "testclient")
self.bind = bind
self.received_count = 0
qsim/simulator/qbt_server.py
-148
View File
@@ -1,148 +0,0 @@
import tornado.auth
import tornado.httpserver
import tornado.ioloop
from tornado.options import define, options
import tornado.web
import pymongo
import bson
import hashlib
import base64
import uuid
import os
import logging
import datetime
import multiprocessing
from qbt import *
from qbt_client import *
logger = logging.getLogger()
define("port", default=8888, help="run the qbt on the given port", type=int)
define("mongodb_host", default="127.0.0.1", help="mongodb host address")
define("mongodb_port", default=27017, help="connect to the mongodb on the given port", type=int)
define("mongodb_dbname", default="qbt", help="database name")
define("mongodb_user", default="qbt", help="database user")
define("mongodb_password", default="qbt", help="database password")
HASH_ALGO = 'sha256'
def connect_db():
connection = pymongo.Connection(options.mongodb_host, options.mongodb_port)
db = connection[options.mongodb_dbname]
db.authenticate(options.mongodb_user, options.mongodb_password)
return connection, db
def encrypt_password(salt, password):
if(salt == None):
h1 = hashlib.new(HASH_ALGO)
h1.update(str(datetime.datetime.utcnow())+"--"+password)
salt = h1.hexdigest()
h2 = hashlib.new(HASH_ALGO)
h2.update(salt+"--"+password)
encrypted_password = h2.hexdigest()
return salt, encrypted_password
class Application(tornado.web.Application):
def __init__(self):
handlers = [
(r"/", MainHandler),
(r"/login", LoginHandler),
(r"/backtest", BacktestHandler)
]
settings = dict(
template_path=os.path.join(os.path.dirname(__file__), "templates"),
static_path=os.path.join(os.path.dirname(__file__), "static"),
xsrf_cookies=False,
cookie_secret=base64.b64encode(uuid.uuid4().bytes + uuid.uuid4().bytes),
login_url="/login",
#autoescape=None,
#debug=True,
)
tornado.web.Application.__init__(self, handlers, **settings)
# Have one global connection to the blog DB across all handlers
self.connection, self.db = connect_db()
class BaseHandler(tornado.web.RequestHandler):
@property
def db(self):
return self.application.db
def get_current_user(self):
return "fawce"
user_id = self.get_secure_cookie(u"user_id")
logger.info("looking up user with id: {id}".format(id=user_id))
if not user_id: return None
#get user record by id
users = self.db.users.find(spec={"_id":bson.ObjectId(user_id)}, limit=1)
if(users.count > 0):
return users[0]
return None
class MainHandler(BaseHandler):
@tornado.web.authenticated
def get(self):
self.write("Hello, world. Try launching a <a href='/backtest'>backtest</a>.")
class LoginHandler(BaseHandler):
def get(self):
self.write('<html><body><form action="/login" method="post">'
'Name: <input type="text" name="user_name">'
'pass: <input type="password" name="password">'
'<input type="submit" value="Sign in">'
'</form></body></html>')
def post(self):
self.authenticate(self.get_argument("user_name"),self.get_argument("password"))
self.redirect("/")
def authenticate(self, username, password):
h = hashlib.new(HASH_ALGO)
#find user record by username.
users = self.db.users.find(spec={"email":username}, limit=1)
if(users.count > 0):
user_record = users[0]
else:
logger.debug("no user with name: {username}", username=username)
return
#calculate password hash
salt, encrypted_password = encrypt_password(user_record['salt'], password)
if (user_record['encrypted_password'] == encrypted_password):
#we have a match, so set the secure cookie to the salt
logger.debug("setting user_id cookie to {id}".format(id=user_record['_id']))
self.set_secure_cookie(u"user_id", unicode(user_record['_id']))
class BacktestHandler(BaseHandler):
@tornado.web.authenticated
def get(self):
self.write('<html><body><form action="/backtest" method="post">'
'<input type="submit" value="Launch">'
'</form></body></html>')
@tornado.web.authenticated
def post(self):
bt = Backtest(self.db, logger)
#btc = BacktestClient(DATA_FEED_PORT, logger)
bt_proc = multiprocessing.Process(target=bt.run)
#btc_proc = multiprocessing.Process(target=btc.run)
bt_proc.start()
#btc_proc.start()
def main():
tornado.options.parse_command_line()
http_server = tornado.httpserver.HTTPServer(Application())
http_server.listen(options.port)
tornado.ioloop.IOLoop.instance().start()
if __name__ == "__main__":
main()
qsim/test/test_messaging.py
+10 -5
View File
@@ -2,16 +2,21 @@ import unittest2 as unittest
import zmq
import logging
import tornado
from simulator.data.sources.equity import *
from simulator.data.feed import *
from transforms.transforms import MergedTransformsFeed, MovingAverage
import multiprocessing
from simulator.qbt_client import TestClient
from qsim.simulator.data.feed import DataFeed
from qsim.transforms.transforms import MergedTransformsFeed, MovingAverage
import qsim.simulator.backtest.util as qutil
from qsim.simulator.qbt_client import TestClient
class MessagingTestCase(unittest.TestCase):
def setUp(self):
tornado.options.parse_command_line()
qutil.configure_logging()
qutil.logger.info("testing...")
self.total_data_count = 800
self.feed_config = {'emt1':{'sid':133, 'class':'RandomEquityTrades', 'count':400},
'emt2':{'sid':134, 'class':'RandomEquityTrades', 'count':400}}
@@ -23,7 +28,7 @@ class MessagingTestCase(unittest.TestCase):
self.config['transforms'] = [{'name':'mavg1', 'class':'MovingAverage', 'hours':1},{'name':'mavg2', 'class':'MovingAverage', 'hours':2}]
def test_client(self):
#subscribe a client to the transformed feed
#subscribe a client to the multiplexed feed
client = TestClient(self.feed, self.feed.feed_address)
feed_proc = multiprocessing.Process(target=self.feed.run)
qsim/transforms/transforms.py
+4 -4
View File
@@ -4,8 +4,8 @@ import datetime
import json
import copy
import multiprocessing
from simulator.backtest.util import *
import simulator.config as config
import qsim.simulator.backtest.util as qutil
import qsim.simulator.config as config
class Transform(object):
"""Parent class for feed transforms. Subclass to create a new derived value from the combined feed."""
@@ -18,7 +18,7 @@ class Transform(object):
server - if True, transform will bind to the result address (and act as a server), if False it will connect. The
the last transform in a series should be server=True so that clients can connect.
"""
self.logger = logging.getLogger()
self.logger = qutil.logger
self.feed = feed
self.feed_address = feed.feed_address
self.result_address = result_address
@@ -98,7 +98,7 @@ class MovingAverage(Transform):
self.events.append(event)
#filter the event list to the window length.
self.events = [x for x in self.events if (parse_date(x['dt']) - parse_date(event['dt'])) <= self.window]
self.events = [x for x in self.events if (qutil.parse_date(x['dt']) - qutil.parse_date(event['dt'])) <= self.window]
if(len(self.events) == 0):
return 0.0
scripts/dev_setup.py
+2 -5
View File
@@ -13,16 +13,13 @@ import logging
import datetime
import random
import simulator.qbt_server as qbt_server
import qsim.simulator.backtest.util as qutil
MINUTE_COUNT=390
define("user_email", default="qbt@quantopian.com", help="email address for qbt user")
define("password", default="foobar", help="password for qbt user")
def db_main():
tornado.options.parse_command_line()
connection, db = qbt_server.connect_db()
connection, db = qutil.connect_db()
#create a user for testing
salt, encrypted_password = qbt_server.encrypt_password(None, options.password)
scripts/jenkins.sh
+3 -3
View File
@@ -11,15 +11,15 @@ source /usr/local/bin/virtualenvwrapper.sh
#create the scientific python virtualenv and copy to provide qsim base
mkvirtualenv --no-site-packages scientific_base
workon scientific_base
./ordered_pip.sh requirements_sci.txt
./scripts/ordered_pip.sh ./scripts/requirements_sci.txt
deactivate
#re-base qsim
#rmvirtualenv qsim
cpvirtualenv scientific_base qsim
workon qsim
./scripts/ordered_pip.sh requirements.txt
./scripts/ordered_pip.sh requirements_dev.txt
./scripts/ordered_pip.sh ./scripts/requirements.txt
./scripts/ordered_pip.sh ./scripts/requirements_dev.txt
#setup the local mongodb
python ./scripts/dev_setup.py
scripts/requirements_dev.txt
+23 -6
View File
@@ -1,14 +1,31 @@
ipython==0.12
# For debugger
fancycompleter==0.2
pyrepl==0.8.2
Pygments==1.4
pdbpp==0.7.2
ipython==0.12
# For unit tests
# Testing
unittest2
nose==1.1.2
unittest2==0.5.1
requests==0.10.1
nosexcover
pylint
coverage==3.5.1
mock==0.7.2
nosexcover==1.0.7
pylint==0.25.1
# Documentation
docutils==0.8.1
Sphinx==1.1.2
# Task running
Paver==1.0.5
Paved==0.3
# Testing
nose==1.1.2
coverage==3.5.1
mock==0.7.2
nosexcover==1.0.7
pylint==0.25.1