From 2eebae2ba90a330e44f996b404fe551a8e46bd2b Mon Sep 17 00:00:00 2001 From: fawce Date: Wed, 8 Feb 2012 13:51:39 -0500 Subject: [PATCH] eliminated tornado and pymongo, more package refactoring, test passes --- README.md | 54 +------------------ dev_setup.py | 45 ---------------- pylint.out | 1 - qsim/{simulator/backtest => data}/__init__.py | 0 .../data/sources => data}/equity.py | 32 +---------- .../{simulator/data => messaging}/__init__.py | 0 qsim/simulator/data/sources/__init__.py | 0 qsim/simulator/{data => }/feed.py | 4 +- qsim/simulator/qbt.py | 4 +- qsim/simulator/qbt_client.py | 2 +- qsim/test/test_messaging.py | 7 +-- qsim/transforms/{transforms.py => base.py} | 48 ++++------------- qsim/transforms/technical.py | 38 +++++++++++++ qsim/{simulator/backtest => }/util.py | 49 ----------------- scripts/requirements.txt | 5 -- 15 files changed, 59 insertions(+), 230 deletions(-) delete mode 100644 dev_setup.py delete mode 100644 pylint.out rename qsim/{simulator/backtest => data}/__init__.py (100%) rename qsim/{simulator/data/sources => data}/equity.py (71%) rename qsim/{simulator/data => messaging}/__init__.py (100%) delete mode 100644 qsim/simulator/data/sources/__init__.py rename qsim/simulator/{data => }/feed.py (97%) rename qsim/transforms/{transforms.py => base.py} (82%) create mode 100644 qsim/transforms/technical.py rename qsim/{simulator/backtest => }/util.py (70%) diff --git a/README.md b/README.md index f5a627d8..cbf54ecc 100644 --- a/README.md +++ b/README.md @@ -5,63 +5,11 @@ Install the necessary python libraries. Because there are dependencies between the various C packages that don't seem to be handled by ```pip install -r```. So, we provide a helper to install the libraries as listed in requirements.txt and requirements_dev.txt: ``` + ./ordered_pip.sh requirements_sci.txt #go get coffee, this will compile a heap of C/C++ code ./ordered_pip.sh requirements.txt ./ordered_pip.sh requirements_dev.txt ``` -Navigate to your mongodb installation and start your db server: - -``` - ./bin/mongodb -``` - - -Create a development database with sample data, will create one qbt user: - -``` - python qbt_data_bootstrap.py --user_email=... --password=... -``` - -To run the qbt against a local mongodb instance, navigate to the source dir and run - -``` - python qbt_server.py --mongodb_dbname=qbt -``` - -To see all the available options: - -``` - python qbt_server.py --help -``` - -or - -``` - python qbt_data_bootstrap.py --help -``` - - -qbt uses tornado to accept synchronous requests for backtesting sessions. -The client of a backtesting session first invokes the _backtest_ endpoint: -http://serverip/backtest?startdate=<>&enddate=<>... - -qbt will respond with a json object describing the session: - -- backtest id, to be referenced in all further requests -- zeromq connection information for the event stream - -A backtesting session is comprised of: - -- REST endpoint to request orders -- an event stream delivered via zeromq - -## Pre-requisites -You need to have the tornado and pymongo eggs installed: - - easy_install tornado pymongo pyzmq - -You need to have mongodb installed and running. Find your system at http://www.mongodb.org/downloads and set it up. - You need to have zeromq installed - http://www.zeromq.org/intro:get-the-software. ### Tooling hints diff --git a/dev_setup.py b/dev_setup.py deleted file mode 100644 index 096082b8..00000000 --- a/dev_setup.py +++ /dev/null @@ -1,45 +0,0 @@ -import tornado.auth -import tornado.httpserver -import tornado.ioloop -from tornado.options import define, options -import tornado.web -import pymongo -import bson -import hashlib -import base64 -import uuid -import os -import logging -import datetime -import random - -import qsim.simulator.backtest.util as qutil - -MINUTE_COUNT=390 - -def db_main(): - tornado.options.parse_command_line() - connection, db = qutil.connect_db() - - #create one mythical company - if not db.company_info.find_one({'sid':133}): - db.company_info.insert({'sid':133, "exchange" : "NEW YORK STOCK EXCHANGE", "symbol" : "JHF", "first date" : "01/04/1993", "last date" : "10/01/2008", "sid" : 133, "industry code" : "130A", "company name" : "JACK INC"}) - - #create one mythical company - if not db.company_info.find_one({'sid':134}): - db.company_info.insert({'sid':134, "exchange" : "NEW YORK STOCK EXCHANGE", "symbol" : "RCF", "first date" : "01/04/1993", "last date" : "10/01/2008", "sid" : 134, "industry code" : "130A", "company name" : "ROCCO INC"}) - - #create minute equity data collection and populate with a day of random data - prices = {133:25.0,134:45.0} #sid, initial price. - if not db.equity.trades.minute.find().count() == MINUTE_COUNT * len(prices): - db.equity.trades.minute.drop() - trade_start = datetime.datetime.now() - minute = datetime.timedelta(minutes=1) - - for i in range(MINUTE_COUNT): - for sid,price in prices.iteritems(): - price = price + random.uniform(-0.05,0.05) - db.equity.trades.minute.insert({'sid':sid, 'dt':trade_start + (minute * i),'price':price, 'volume':random.randrange(100,10000,100)}) - -if __name__ == "__main__": - db_main() diff --git a/pylint.out b/pylint.out deleted file mode 100644 index b6feaade..00000000 --- a/pylint.out +++ /dev/null @@ -1 +0,0 @@ -__init__.py:1: [F] error while code parsing: Unable to load file '__init__.py' ([Errno 2] No such file or directory: '__init__.py') diff --git a/qsim/simulator/backtest/__init__.py b/qsim/data/__init__.py similarity index 100% rename from qsim/simulator/backtest/__init__.py rename to qsim/data/__init__.py diff --git a/qsim/simulator/data/sources/equity.py b/qsim/data/equity.py similarity index 71% rename from qsim/simulator/data/sources/equity.py rename to qsim/data/equity.py index 4790a0dc..790046d8 100644 --- a/qsim/simulator/data/sources/equity.py +++ b/qsim/data/equity.py @@ -1,16 +1,13 @@ import datetime import zmq -import pymongo -import pymongo.json_util import json import pytz import copy import multiprocessing import logging import random -from pymongo import ASCENDING, DESCENDING -import qsim.simulator.backtest.util as qutil +import qsim.util as qutil class DataSource(object): def __init__(self, feed, source_id): @@ -59,32 +56,7 @@ class DataSource(object): self.data_socket.close() self.context.term() self.logger.info("finished processing data source") - -class EquityMinuteTrades(DataSource): - - def __init__(self, sid, feed, source_id): - self.sid = sid - self.connection, self.db = qutil.connect_db() - DataSource.__init__(self, feed, source_id) - - - def send_all(self): - eventQS = self.db.equity.trades.minute.find(fields=["sid","price","volume","dt"], - spec={"sid":self.sid}, - sort=[("dt",ASCENDING)], - slave_ok=True) - self.logger.info("found {count} events".format(count=eventQS.count())) - - for doc in eventQS: - doc_dt = doc['dt'].replace(tzinfo = pytz.utc) - doc_dt_str = qutil.format_date(doc_dt) - event = copy.copy(doc) - event['dt'] = doc_dt_str - del(event['_id']) - self.send(event) - - - + class RandomEquityTrades(DataSource): def __init__(self, sid, feed, source_id, count): diff --git a/qsim/simulator/data/__init__.py b/qsim/messaging/__init__.py similarity index 100% rename from qsim/simulator/data/__init__.py rename to qsim/messaging/__init__.py diff --git a/qsim/simulator/data/sources/__init__.py b/qsim/simulator/data/sources/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/qsim/simulator/data/feed.py b/qsim/simulator/feed.py similarity index 97% rename from qsim/simulator/data/feed.py rename to qsim/simulator/feed.py index 5ccd0ff8..e3256df4 100644 --- a/qsim/simulator/data/feed.py +++ b/qsim/simulator/feed.py @@ -1,6 +1,6 @@ -import qsim.simulator.data.sources.equity as qequity -import qsim.simulator.backtest.util as qutil +import qsim.data.equity as qequity +import qsim.util as qutil import zmq import time import logging diff --git a/qsim/simulator/qbt.py b/qsim/simulator/qbt.py index a4bba52e..98f1a21c 100644 --- a/qsim/simulator/qbt.py +++ b/qsim/simulator/qbt.py @@ -34,8 +34,8 @@ import copy import multiprocessing import zmq -import qsim.backtest.util as qutil -import qsim.data.sources.equity as qequity +import qsim.util as qutil +import qsim.data.equity as qequity diff --git a/qsim/simulator/qbt_client.py b/qsim/simulator/qbt_client.py index 67847902..c86fd024 100644 --- a/qsim/simulator/qbt_client.py +++ b/qsim/simulator/qbt_client.py @@ -4,7 +4,7 @@ import zmq import logging import json -import qsim.simulator.backtest.util as qutil +import qsim.util as qutil class TestClient(object): diff --git a/qsim/test/test_messaging.py b/qsim/test/test_messaging.py index d2d86b2d..46cd6f14 100644 --- a/qsim/test/test_messaging.py +++ b/qsim/test/test_messaging.py @@ -4,9 +4,10 @@ import logging import tornado import multiprocessing -from qsim.simulator.data.feed import DataFeed -from qsim.transforms.transforms import MergedTransformsFeed, MovingAverage -import qsim.simulator.backtest.util as qutil +from qsim.simulator.feed import DataFeed +from qsim.transforms.base import MergedTransformsFeed +from qsim.transforms.technical import MovingAverage +import qsim.util as qutil from qsim.simulator.qbt_client import TestClient diff --git a/qsim/transforms/transforms.py b/qsim/transforms/base.py similarity index 82% rename from qsim/transforms/transforms.py rename to qsim/transforms/base.py index 65204aa7..013b704b 100644 --- a/qsim/transforms/transforms.py +++ b/qsim/transforms/base.py @@ -4,10 +4,11 @@ import datetime import json import copy import multiprocessing -import qsim.simulator.backtest.util as qutil +import qsim.util as qutil import qsim.simulator.config as config + class Transform(object): - """Parent class for feed transforms. Subclass to create a new derived value from the combined feed.""" + """Parent class for feed transforms. Subclass and override transform method to create a new derived value from the combined feed.""" def __init__(self, feed, config_dict, result_address): """ @@ -74,46 +75,15 @@ class Transform(object): self.context.term() def transform(self, event): + """ Must return the transformed value as a map with {name:"name of new transform", value: "value of new field"} + Transforms run in parallel and results are merged into a single map, so transform names must be unique. + Best practice is to use the self.state object initialized from the transform configuration, and only set the + transformed value: + self.state['value'] = transformed_value + """ return {} -class MovingAverage(Transform): - - def __init__(self, feed, props, result_address): - Transform.__init__(self, feed, props, result_address) - self.events = [] - - self.window = datetime.timedelta(days = self.config.get_integer('days'), - seconds = self.config.get_integer('seconds'), - microseconds = self.config.get_integer('microseconds'), - milliseconds = self.config.get_integer('milliseconds'), - minutes = self.config.get_integer('minutes'), - hours = self.config.get_integer('hours'), - weeks = self.config.get_integer('weeks')) - - - - - def transform(self, event): - self.events.append(event) - - #filter the event list to the window length. - self.events = [x for x in self.events if (qutil.parse_date(x['dt']) - qutil.parse_date(event['dt'])) <= self.window] - - if(len(self.events) == 0): - return 0.0 - - total = 0.0 - for event in self.events: - total += event['price'] - - self.average = total/len(self.events) - - self.state['value'] = self.average - - return self.state - - class MergedTransformsFeed(Transform): """ Merge data feed and array of transform feeds into a single result vector. PULL from feed diff --git a/qsim/transforms/technical.py b/qsim/transforms/technical.py new file mode 100644 index 00000000..8c2d409a --- /dev/null +++ b/qsim/transforms/technical.py @@ -0,0 +1,38 @@ +import qsim.transforms.base as base + +class MovingAverage(base.Transform): + + def __init__(self, feed, props, result_address): + base.Transform.__init__(self, feed, props, result_address) + self.events = [] + + self.window = datetime.timedelta(days = self.config.get_integer('days'), + seconds = self.config.get_integer('seconds'), + microseconds = self.config.get_integer('microseconds'), + milliseconds = self.config.get_integer('milliseconds'), + minutes = self.config.get_integer('minutes'), + hours = self.config.get_integer('hours'), + weeks = self.config.get_integer('weeks')) + + + + + def transform(self, event): + self.events.append(event) + + #filter the event list to the window length. + self.events = [x for x in self.events if (qutil.parse_date(x['dt']) - qutil.parse_date(event['dt'])) <= self.window] + + if(len(self.events) == 0): + return 0.0 + + total = 0.0 + for event in self.events: + total += event['price'] + + self.average = total/len(self.events) + + self.state['value'] = self.average + + return self.state + \ No newline at end of file diff --git a/qsim/simulator/backtest/util.py b/qsim/util.py similarity index 70% rename from qsim/simulator/backtest/util.py rename to qsim/util.py index 80741e68..08c49e87 100644 --- a/qsim/simulator/backtest/util.py +++ b/qsim/util.py @@ -8,42 +8,8 @@ import json import logging import uuid import zmq -import pymongo -from tornado.options import define, options logger = logging.getLogger('QSimLogger') - -class DocWrap(): - """ - Provides attribute access style on top of dictionary results from pymongo. - Allows you to access result['field'] as result.field. - Aliases result['_id'] to result.id. - - """ - def __init__(self, store=None): - if(store == None): - self.store = {} - else: - self.store = store.copy() - if(self.store.has_key('_id')): - self.store['id'] = self.store['_id'] - del(self.store['_id']) - - def __setitem__(self,key,value): - if(key == '_id'): - self.store['id'] = value - else: - self.store[key] = value - - def __getitem__(self, key): - if self.store.has_key(key): - return self.store[key] - - def __getattr__(self,attrname): - if self.store.has_key(attrname): - return self.store[attrname] - else: - raise AttributeError("No attribute named {name}".format(name=attrname)) def parse_date(dt_str): """parse strings according to the same format as generated by format_date""" @@ -165,22 +131,7 @@ class FeedSync(object): context.term() self.logger.info("sync'd feed from {id}".format(id = self.id)) - -define("user_email", default="qbt@quantopian.com", help="email address for qbt user") -define("password", default="foobar", help="password for qbt user") -define("port", default=8888, help="run the qbt on the given port", type=int) -define("mongodb_host", default="127.0.0.1", help="mongodb host address") -define("mongodb_port", default=27017, help="connect to the mongodb on the given port", type=int) -define("mongodb_dbname", default="qbt", help="database name") -define("mongodb_user", default="qbt", help="database user") -define("mongodb_password", default="qbt", help="database password") -def connect_db(): - connection = pymongo.Connection(options.mongodb_host, options.mongodb_port) - db = connection[options.mongodb_dbname] - db.authenticate(options.mongodb_user, options.mongodb_password) - return connection, db - def configure_logging(loglevel=logging.DEBUG): logger.setLevel(loglevel) handler = logging.handlers.RotatingFileHandler("/tmp/{lfn}.log".format(lfn="qsim-log"), maxBytes=10*1024*1024, backupCount=5) diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 83b1ea93..ef46d31b 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -1,8 +1,3 @@ -tornado - -#data source related -pymongo==2.1.1 - #zeromq related pyzmq==2.1.11