diff --git a/coverage.xml b/coverage.xml
new file mode 100644
index 00000000..d2179403
--- /dev/null
+++ b/coverage.xml
@@ -0,0 +1,477 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/docs/index.rst b/docs/index.rst
index 6629f7e8..5cd17a3b 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -3,9 +3,12 @@
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
-Welcome to QSim's documentation!
+Quantopian Simulator: QSim
================================
+Qsim runs backtests using asynchronous components and zeromq messaging for communication and coordination.
+
+
Contents:
.. toctree::
@@ -13,6 +16,7 @@ Contents:
notes.rst
modules.rst
+ messaging.rst
diff --git a/qsim/simulator/qbt.py b/docs/messaging.rst
similarity index 64%
rename from qsim/simulator/qbt.py
rename to docs/messaging.rst
index 45247a4d..5ade0428 100644
--- a/qsim/simulator/qbt.py
+++ b/docs/messaging.rst
@@ -1,7 +1,3 @@
-"""
-QBT - Quantopian Backtest
-====================================
-
qbt runs backtests using multiple processes and zeromq messaging for communication and coordination.
Backtest is the primary process. It maintains both server and client sockets:
@@ -27,41 +23,4 @@ zmq sockets for backtest clients:
- event feed, ZMQ.RESP. Port = port_start + 6
- backtest will bind to this socket and respond to requests from client for more data. Response data will be the queue of events that
transpired since the last request.
-
-
-"""
-import copy
-import multiprocessing
-import zmq
-
-import qsim.util as qutil
-import qsim.simulator.sources as sources
-
-
-
-CONTROLLER_PORT = 9000
-DATA_SINK_PORT = 10000
-DATA_FEED_PORT = 30000
-
-class Backtest(object):
-
- def __init__(self, db, logger):
- qutil.logger = logger
- self.db = db
- self.feed = DataFeed(db, logger)
-
- def start_feed(self):
- proc1 = multiprocessing.Process(target=feed.run)
- proc1.start()
-
- def run(self):
- # Prepare our context and sockets
- self.context = zmq.Context()
-
- #create the feed sink.
- self.feed_address = self.feed.data_address
- self.feed_socket = self.context.connect(self.feed_address)
- self.feed_socket.connect(zmq.PULL)
-
-
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/etc/jenkins.sh b/etc/jenkins.sh
index 56659efd..b45cab15 100755
--- a/etc/jenkins.sh
+++ b/etc/jenkins.sh
@@ -31,6 +31,7 @@ paver apidocs html
nosetests
#run pylint checks
+cp ./pylint.rcfile /mnt/jenkins/.pylintrc #default location for config file...
pylint -f parseable qsim | tee pylint.out
#run sloccount analysis
diff --git a/nosetests.xml b/nosetests.xml
index 45278cc6..8557c4d9 100644
--- a/nosetests.xml
+++ b/nosetests.xml
@@ -1 +1 @@
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/pylint.rcfile b/pylint.rcfile
new file mode 100644
index 00000000..b682c7fb
--- /dev/null
+++ b/pylint.rcfile
@@ -0,0 +1,238 @@
+[MASTER]
+
+# Specify a configuration file.
+#rcfile=
+
+# Python code to execute, usually for sys.path manipulation such as
+# pygtk.require().
+#init-hook=
+
+# Profiled execution.
+profile=no
+
+# Add to the black list. It should be a base name, not a
+# path. You may set this option multiple times.
+ignore=CVS
+
+# Pickle collected data for later comparisons.
+persistent=yes
+
+# List of plugins (as comma separated values of python modules names) to load,
+# usually to register additional checkers.
+load-plugins=
+
+
+[MESSAGES CONTROL]
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time.
+#enable=
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once).
+#disable=
+
+
+[REPORTS]
+
+# Set the output format. Available formats are text, parseable, colorized, msvs
+# (visual studio) and html
+output-format=text
+
+# Include message's id in output
+include-ids=no
+
+# Put messages in a separate file for each module / package specified on the
+# command line instead of printing them on stdout. Reports (if any) will be
+# written in a file name "pylint_global.[txt|html]".
+files-output=no
+
+# Tells whether to display a full report or only the messages
+reports=yes
+
+# Python expression which should return a note less than 10 (10 is the highest
+# note). You have access to the variables errors warning, statement which
+# respectively contain the number of errors / warnings messages and the total
+# number of statements analyzed. This is used by the global evaluation report
+# (RP0004).
+evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+
+# Add a comment according to your evaluation note. This is used by the global
+# evaluation report (RP0004).
+comment=no
+
+
+[BASIC]
+
+# Required attributes for module, separated by a comma
+required-attributes=
+
+# List of builtins function names that should not be used, separated by a comma
+bad-functions=map,filter,apply,input
+
+# Regular expression which should only match correct module names
+module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
+
+# Regular expression which should only match correct module level names
+const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$
+
+# Regular expression which should only match correct class names
+class-rgx=[A-Z_][a-zA-Z0-9]+$
+
+# Regular expression which should only match correct function names
+function-rgx=[a-z_][a-z0-9_]{2,30}$
+
+# Regular expression which should only match correct method names
+method-rgx=[a-z_][a-z0-9_]{2,30}$|setUp
+
+# Regular expression which should only match correct instance attribute names
+attr-rgx=[a-z_][a-z0-9_]{2,30}$
+
+# Regular expression which should only match correct argument names
+argument-rgx=[a-z_][a-z0-9_]{2,30}$
+
+# Regular expression which should only match correct variable names
+variable-rgx=[a-z_][a-z0-9_]{2,30}$
+
+# Regular expression which should only match correct list comprehension /
+# generator expression variable names
+inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
+
+# Good variable names which should always be accepted, separated by a comma
+good-names=i,j,k,ex,Run,_
+
+# Bad variable names which should always be refused, separated by a comma
+bad-names=foo,bar,baz,toto,tutu,tata
+
+# Regular expression which should only match functions or classes name which do
+# not require a docstring
+no-docstring-rgx=__.*__
+
+
+[FORMAT]
+
+# Maximum number of characters on a single line.
+max-line-length=125
+
+# Maximum number of lines in a module
+max-module-lines=1000
+
+# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
+# tab).
+indent-string=' '
+
+
+[MISCELLANEOUS]
+
+# List of note tags to take in consideration, separated by a comma.
+notes=FIXME,XXX,TODO
+
+
+[SIMILARITIES]
+
+# Minimum lines number of a similarity.
+min-similarity-lines=4
+
+# Ignore comments when computing similarities.
+ignore-comments=yes
+
+# Ignore docstrings when computing similarities.
+ignore-docstrings=yes
+
+
+[TYPECHECK]
+
+# Tells whether missing members accessed in mixin class should be ignored. A
+# mixin class is detected if its name ends with "mixin" (case insensitive).
+ignore-mixin-members=yes
+
+# List of classes names for which member attributes should not be checked
+# (useful for classes with attributes dynamically set).
+ignored-classes=SQLObject
+
+# When zope mode is activated, add a predefined set of Zope acquired attributes
+# to generated-members.
+zope=no
+
+# List of members which are set dynamically and missed by pylint inference
+# system, and so shouldn't trigger E0201 when accessed.
+generated-members=REQUEST,acl_users,aq_parent
+
+
+[VARIABLES]
+
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
+
+# A regular expression matching the beginning of the name of dummy variables
+# (i.e. not used).
+dummy-variables-rgx=_|dummy
+
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid to define new builtins when possible.
+additional-builtins=
+
+
+[CLASSES]
+
+# List of interface methods to ignore, separated by a comma. This is used for
+# instance to not check methods defines in Zope's Interface base class.
+ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by
+
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,__new__,setUp
+
+
+[DESIGN]
+
+# Maximum number of arguments for function / method
+max-args=5
+
+# Argument names that match this expression will be ignored. Default to name
+# with leading underscore
+ignored-argument-names=_.*
+
+# Maximum number of locals for function / method body
+max-locals=15
+
+# Maximum number of return / yield for function / method body
+max-returns=6
+
+# Maximum number of branch for function / method body
+max-branchs=12
+
+# Maximum number of statements in function / method body
+max-statements=50
+
+# Maximum number of parents for a class (see R0901).
+max-parents=7
+
+# Maximum number of attributes for a class (see R0902).
+max-attributes=7
+
+# Minimum number of public methods for a class (see R0903).
+min-public-methods=2
+
+# Maximum number of public methods for a class (see R0904).
+max-public-methods=20
+
+
+[IMPORTS]
+
+# Deprecated modules which should not be used, separated by a comma
+deprecated-modules=regsub,string,TERMIOS,Bastion,rexec
+
+# Create a graph of every (i.e. internal and external) dependencies in the
+# given file (report RP0402 must not be disabled)
+import-graph=
+
+# Create a graph of external dependencies in the given file (report RP0402 must
+# not be disabled)
+ext-import-graph=
+
+# Create a graph of internal dependencies in the given file (report RP0402 must
+# not be disabled)
+int-import-graph=
diff --git a/qsim/messaging.py b/qsim/messaging.py
index 5b512b6a..2e6e50cd 100644
--- a/qsim/messaging.py
+++ b/qsim/messaging.py
@@ -5,7 +5,7 @@ import json
import uuid
import zmq
-import util as qutil
+import qsim.util as qutil
class ParallelBuffer(object):
""" holds several queues of events by key, allows retrieval in date order
@@ -20,13 +20,16 @@ class ParallelBuffer(object):
self.data_buffer[key] = []
def __len__(self):
+ """buffer's length is same as internal map holding separate sorted arrays of events keyed by source id"""
return len(self.data_buffer)
- def append(self, key, value):
- self.data_buffer[key].append(value)
+ def append(self, source_id, value):
+ """add an event to the buffer for the source specified by source_id"""
+ self.data_buffer[source_id].append(value)
self.received_count += 1
def next(self):
+ """Get the next message in chronological order"""
if(not(self.is_full() or self.draining)):
return
@@ -43,12 +46,14 @@ class ParallelBuffer(object):
return earliest.pop(0)
def is_full(self):
+ """indicates whether the buffer has messages in buffer for all un-DONE sources"""
for source, events in self.data_buffer.iteritems():
if (len(events) == 0):
return False
return True
def pending_messages(self):
+ """"""
total = 0
for source, events in self.data_buffer.iteritems():
total += len(events)
diff --git a/qsim/test/test_messaging.py b/qsim/test/test_messaging.py
index 6636812c..7e2e794a 100644
--- a/qsim/test/test_messaging.py
+++ b/qsim/test/test_messaging.py
@@ -1,7 +1,9 @@
+"""
+Test suite for the messaging infrastructure of QSim.
+"""
+#don't worry about excessive public methods pylint: disable=R0904
+
import unittest2 as unittest
-import zmq
-import logging
-import tornado
import multiprocessing
from qsim.simulator.feed import DataFeed
@@ -12,9 +14,12 @@ import qsim.util as qutil
from qsim.test.client import TestClient
-class MessagingTestCase(unittest.TestCase):
+class MessagingTestCase(unittest.TestCase):
+ """Tests the message passing: datasources -> feed -> transforms -> merge -> client"""
def setUp(self):
+ """generate some config objects for the datafeed, sources, and transforms."""
+
qutil.configure_logging()
qutil.logger.info("testing...")
self.total_data_count = 800
@@ -25,9 +30,12 @@ class MessagingTestCase(unittest.TestCase):
self.config = {}
self.config['name'] = '**merged feed**'
- self.config['transforms'] = [{'name':'mavg1', 'class':'MovingAverage', 'hours':1},{'name':'mavg2', 'class':'MovingAverage', 'hours':2}]
+ self.config['transforms'] = [{'name':'mavg1', 'class':'MovingAverage', 'hours':1},
+ {'name':'mavg2', 'class':'MovingAverage', 'hours':2}]
- def test_client(self):
+ def test_client(self):
+ """directly connect the test client to the feed, using two random data sources"""
+
#subscribe a client to the multiplexed feed
client = TestClient(self.feed, self.feed.feed_address)
@@ -36,11 +44,18 @@ class MessagingTestCase(unittest.TestCase):
client.run()
- self.assertEqual(self.feed.data_buffer.pending_messages(), 0, "The feed should be drained of all messages, found {n} remaining.".format(n=self.feed.data_buffer.pending_messages()))
- self.assertEqual(self.total_data_count, client.received_count, "The client should have received ({n}) the same number of messages as the feed sent ({m}).".format(n=client.received_count, m=self.total_data_count))
+ self.assertEqual(self.feed.data_buffer.pending_messages(), 0,
+ "The feed should be drained of all messages, found {n} remaining."
+ .format(n=self.feed.data_buffer.pending_messages()))
+ self.assertEqual(self.total_data_count, client.received_count,
+ "The client should have received ({n}) the same number of messages as the feed sent ({m})."
+ .format(n=client.received_count, m=self.total_data_count))
def dtest_moving_average_to_client(self):
+ """2 datasources -> feed -> moving average transform -> testclient
+ verify message count at client."""
+
mavg = MovingAverage(self.feed, self.config['transforms'][0], result_address="tcp://127.0.0.1:20202")
mavg_proc = multiprocessing.Process(target=mavg.run)
mavg_proc.start()
@@ -52,9 +67,14 @@ class MessagingTestCase(unittest.TestCase):
client.run()
self.assertEqual(self.feed.data_buffer.pending_messages(), 0, "The feed should be drained of all messages.")
- self.assertEqual(self.total_data_count, client.received_count, "The client should have received the same number of messages as the feed sent.")
+ self.assertEqual(self.total_data_count, client.received_count,
+ "The client should have received the same number of messages as the feed sent.")
def dtest_merged_to_client(self):
+ """
+ 2 datasources -> feed -> 2 moving average transforms -> transform merge -> testclient
+ verify message count at client.
+ """
merger = MergedTransformsFeed(self.feed, self.config)
merger_proc = multiprocessing.Process(target=merger.run)
merger_proc.start()
@@ -66,5 +86,6 @@ class MessagingTestCase(unittest.TestCase):
client.run()
self.assertEqual(self.feed.data_buffer.pending_messages(), 0, "The feed should be drained of all messages.")
- self.assertEqual(self.total_data_count, client.received_count, "The client should have received the same number of messages as the feed sent.")
+ self.assertEqual(self.total_data_count, client.received_count,
+ "The client should have received the same number of messages as the feed sent.")
diff --git a/sloccount.sc b/sloccount.sc
new file mode 100644
index 00000000..e69de29b