wassname/catalyst
1
0
Fork 0
mirror of https://github.com/wassname/catalyst.git synced 2026-06-28 16:44:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

eliminated tornado and pymongo, more package refactoring, test passes

Browse Source
This commit is contained in:
fawce
2012-02-08 13:51:39 -05:00
parent c0fff214d7
commit 2eebae2ba9
15 changed files with 59 additions and 230 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+1 -53
View File
@@ -5,63 +5,11 @@
Install the necessary python libraries. Because there are dependencies between the various C packages that don't seem to be handled by ```pip install -r```. So, we provide a helper to install the libraries as listed in requirements.txt and requirements_dev.txt:
```
./ordered_pip.sh requirements_sci.txt #go get coffee, this will compile a heap of C/C++ code
./ordered_pip.sh requirements.txt
./ordered_pip.sh requirements_dev.txt
```
Navigate to your mongodb installation and start your db server:
```
./bin/mongodb
```
Create a development database with sample data, will create one qbt user:
```
python qbt_data_bootstrap.py --user_email=... --password=...
```
To run the qbt against a local mongodb instance, navigate to the source dir and run
```
python qbt_server.py --mongodb_dbname=qbt
```
To see all the available options:
```
python qbt_server.py --help
```
or
```
python qbt_data_bootstrap.py --help
```
qbt uses tornado to accept synchronous requests for backtesting sessions.
The client of a backtesting session first invokes the _backtest_ endpoint:
http://serverip/backtest?startdate=<>&enddate=<>...
qbt will respond with a json object describing the session:
- backtest id, to be referenced in all further requests
- zeromq connection information for the event stream
A backtesting session is comprised of:
- REST endpoint to request orders
- an event stream delivered via zeromq
## Pre-requisites
You need to have the tornado and pymongo eggs installed:
easy_install tornado pymongo pyzmq
You need to have mongodb installed and running. Find your system at http://www.mongodb.org/downloads and set it up.
You need to have zeromq installed - http://www.zeromq.org/intro:get-the-software.
### Tooling hints
dev_setup.py
-45
View File
@@ -1,45 +0,0 @@
import tornado.auth
import tornado.httpserver
import tornado.ioloop
from tornado.options import define, options
import tornado.web
import pymongo
import bson
import hashlib
import base64
import uuid
import os
import logging
import datetime
import random
import qsim.simulator.backtest.util as qutil
MINUTE_COUNT=390
def db_main():
tornado.options.parse_command_line()
connection, db = qutil.connect_db()
#create one mythical company
if not db.company_info.find_one({'sid':133}):
db.company_info.insert({'sid':133, "exchange" : "NEW YORK STOCK EXCHANGE", "symbol" : "JHF", "first date" : "01/04/1993", "last date" : "10/01/2008", "sid" : 133, "industry code" : "130A", "company name" : "JACK INC"})
#create one mythical company
if not db.company_info.find_one({'sid':134}):
db.company_info.insert({'sid':134, "exchange" : "NEW YORK STOCK EXCHANGE", "symbol" : "RCF", "first date" : "01/04/1993", "last date" : "10/01/2008", "sid" : 134, "industry code" : "130A", "company name" : "ROCCO INC"})
#create minute equity data collection and populate with a day of random data
prices = {133:25.0,134:45.0} #sid, initial price.
if not db.equity.trades.minute.find().count() == MINUTE_COUNT * len(prices):
db.equity.trades.minute.drop()
trade_start = datetime.datetime.now()
minute = datetime.timedelta(minutes=1)
for i in range(MINUTE_COUNT):
for sid,price in prices.iteritems():
price = price + random.uniform(-0.05,0.05)
db.equity.trades.minute.insert({'sid':sid, 'dt':trade_start + (minute * i),'price':price, 'volume':random.randrange(100,10000,100)})
if __name__ == "__main__":
db_main()
pylint.out
-1
View File
@@ -1 +0,0 @@
__init__.py:1: [F] error while code parsing: Unable to load file '__init__.py' ([Errno 2] No such file or directory: '__init__.py')
qsim/simulator/backtest/__init__.py → qsim/data/__init__.py
View File
qsim/simulator/data/sources/equity.py → qsim/data/equity.py
+2 -30
View File
@@ -1,16 +1,13 @@
import datetime
import zmq
import pymongo
import pymongo.json_util
import json
import pytz
import copy
import multiprocessing
import logging
import random
from pymongo import ASCENDING, DESCENDING
import qsim.simulator.backtest.util as qutil
import qsim.util as qutil
class DataSource(object):
def __init__(self, feed, source_id):
@@ -59,32 +56,7 @@ class DataSource(object):
self.data_socket.close()
self.context.term()
self.logger.info("finished processing data source")
class EquityMinuteTrades(DataSource):
def __init__(self, sid, feed, source_id):
self.sid = sid
self.connection, self.db = qutil.connect_db()
DataSource.__init__(self, feed, source_id)
def send_all(self):
eventQS = self.db.equity.trades.minute.find(fields=["sid","price","volume","dt"],
spec={"sid":self.sid},
sort=[("dt",ASCENDING)],
slave_ok=True)
self.logger.info("found {count} events".format(count=eventQS.count()))
for doc in eventQS:
doc_dt = doc['dt'].replace(tzinfo = pytz.utc)
doc_dt_str = qutil.format_date(doc_dt)
event = copy.copy(doc)
event['dt'] = doc_dt_str
del(event['_id'])
self.send(event)
class RandomEquityTrades(DataSource):
def __init__(self, sid, feed, source_id, count):
qsim/simulator/data/__init__.py → qsim/messaging/__init__.py
View File
qsim/simulator/data/sources/__init__.py
View File
qsim/simulator/data/feed.py → qsim/simulator/feed.py
+2 -2
View File
@@ -1,6 +1,6 @@
import qsim.simulator.data.sources.equity as qequity
import qsim.simulator.backtest.util as qutil
import qsim.data.equity as qequity
import qsim.util as qutil
import zmq
import time
import logging
qsim/simulator/qbt.py
+2 -2
View File
@@ -34,8 +34,8 @@ import copy
import multiprocessing
import zmq
import qsim.backtest.util as qutil
import qsim.data.sources.equity as qequity
import qsim.util as qutil
import qsim.data.equity as qequity
qsim/simulator/qbt_client.py
+1 -1
View File
@@ -4,7 +4,7 @@ import zmq
import logging
import json
import qsim.simulator.backtest.util as qutil
import qsim.util as qutil
class TestClient(object):
qsim/test/test_messaging.py
+4 -3
View File
@@ -4,9 +4,10 @@ import logging
import tornado
import multiprocessing
from qsim.simulator.data.feed import DataFeed
from qsim.transforms.transforms import MergedTransformsFeed, MovingAverage
import qsim.simulator.backtest.util as qutil
from qsim.simulator.feed import DataFeed
from qsim.transforms.base import MergedTransformsFeed
from qsim.transforms.technical import MovingAverage
import qsim.util as qutil
from qsim.simulator.qbt_client import TestClient
qsim/transforms/transforms.py → qsim/transforms/base.py
+9 -39
View File
@@ -4,10 +4,11 @@ import datetime
import json
import copy
import multiprocessing
import qsim.simulator.backtest.util as qutil
import qsim.util as qutil
import qsim.simulator.config as config
class Transform(object):
"""Parent class for feed transforms. Subclass to create a new derived value from the combined feed."""
"""Parent class for feed transforms. Subclass and override transform method to create a new derived value from the combined feed."""
def __init__(self, feed, config_dict, result_address):
"""
@@ -74,46 +75,15 @@ class Transform(object):
self.context.term()
def transform(self, event):
""" Must return the transformed value as a map with {name:"name of new transform", value: "value of new field"}
Transforms run in parallel and results are merged into a single map, so transform names must be unique.
Best practice is to use the self.state object initialized from the transform configuration, and only set the
transformed value:
self.state['value'] = transformed_value
"""
return {}
class MovingAverage(Transform):
def __init__(self, feed, props, result_address):
Transform.__init__(self, feed, props, result_address)
self.events = []
self.window = datetime.timedelta(days = self.config.get_integer('days'),
seconds = self.config.get_integer('seconds'),
microseconds = self.config.get_integer('microseconds'),
milliseconds = self.config.get_integer('milliseconds'),
minutes = self.config.get_integer('minutes'),
hours = self.config.get_integer('hours'),
weeks = self.config.get_integer('weeks'))
def transform(self, event):
self.events.append(event)
#filter the event list to the window length.
self.events = [x for x in self.events if (qutil.parse_date(x['dt']) - qutil.parse_date(event['dt'])) <= self.window]
if(len(self.events) == 0):
return 0.0
total = 0.0
for event in self.events:
total += event['price']
self.average = total/len(self.events)
self.state['value'] = self.average
return self.state
class MergedTransformsFeed(Transform):
""" Merge data feed and array of transform feeds into a single result vector.
PULL from feed
qsim/transforms/technical.py
+38
View File
@@ -0,0 +1,38 @@
import qsim.transforms.base as base
class MovingAverage(base.Transform):
def __init__(self, feed, props, result_address):
base.Transform.__init__(self, feed, props, result_address)
self.events = []
self.window = datetime.timedelta(days = self.config.get_integer('days'),
seconds = self.config.get_integer('seconds'),
microseconds = self.config.get_integer('microseconds'),
milliseconds = self.config.get_integer('milliseconds'),
minutes = self.config.get_integer('minutes'),
hours = self.config.get_integer('hours'),
weeks = self.config.get_integer('weeks'))
def transform(self, event):
self.events.append(event)
#filter the event list to the window length.
self.events = [x for x in self.events if (qutil.parse_date(x['dt']) - qutil.parse_date(event['dt'])) <= self.window]
if(len(self.events) == 0):
return 0.0
total = 0.0
for event in self.events:
total += event['price']
self.average = total/len(self.events)
self.state['value'] = self.average
return self.state
qsim/simulator/backtest/util.py → qsim/util.py
-49
View File
@@ -8,42 +8,8 @@ import json
import logging
import uuid
import zmq
import pymongo
from tornado.options import define, options
logger = logging.getLogger('QSimLogger')
class DocWrap():
"""
Provides attribute access style on top of dictionary results from pymongo.
Allows you to access result['field'] as result.field.
Aliases result['_id'] to result.id.
"""
def __init__(self, store=None):
if(store == None):
self.store = {}
else:
self.store = store.copy()
if(self.store.has_key('_id')):
self.store['id'] = self.store['_id']
del(self.store['_id'])
def __setitem__(self,key,value):
if(key == '_id'):
self.store['id'] = value
else:
self.store[key] = value
def __getitem__(self, key):
if self.store.has_key(key):
return self.store[key]
def __getattr__(self,attrname):
if self.store.has_key(attrname):
return self.store[attrname]
else:
raise AttributeError("No attribute named {name}".format(name=attrname))
def parse_date(dt_str):
"""parse strings according to the same format as generated by format_date"""
@@ -165,22 +131,7 @@ class FeedSync(object):
context.term()
self.logger.info("sync'd feed from {id}".format(id = self.id))
define("user_email", default="qbt@quantopian.com", help="email address for qbt user")
define("password", default="foobar", help="password for qbt user")
define("port", default=8888, help="run the qbt on the given port", type=int)
define("mongodb_host", default="127.0.0.1", help="mongodb host address")
define("mongodb_port", default=27017, help="connect to the mongodb on the given port", type=int)
define("mongodb_dbname", default="qbt", help="database name")
define("mongodb_user", default="qbt", help="database user")
define("mongodb_password", default="qbt", help="database password")
def connect_db():
connection = pymongo.Connection(options.mongodb_host, options.mongodb_port)
db = connection[options.mongodb_dbname]
db.authenticate(options.mongodb_user, options.mongodb_password)
return connection, db
def configure_logging(loglevel=logging.DEBUG):
logger.setLevel(loglevel)
handler = logging.handlers.RotatingFileHandler("/tmp/{lfn}.log".format(lfn="qsim-log"), maxBytes=10*1024*1024, backupCount=5)
scripts/requirements.txt
-5
View File
@@ -1,8 +1,3 @@
tornado
#data source related
pymongo==2.1.1
#zeromq related
pyzmq==2.1.11