wassname/catalyst
1
0
Fork 0
mirror of https://github.com/wassname/catalyst.git synced 2026-07-19 11:22:06 +08:00
Code Issues Packages Projects Releases Wiki Activity

documentation on simulator, delinting, refactoring

Browse Source
This commit is contained in:
fawce
2012-02-10 17:02:01 -05:00
parent d3d3ef833c
commit 227eb35ee2
18 changed files with 194 additions and 580 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+2
View File
@@ -32,6 +32,8 @@ var
sdist
develop-eggs
.installed.cfg
coverage.xml
nosetests.xml
*.swp
*.swo
coverage.xml
-477
View File
@@ -1,477 +0,0 @@
<?xml version="1.0" ?>
<!DOCTYPE coverage
SYSTEM 'http://cobertura.sourceforge.net/xml/coverage-03.dtd'>
<coverage branch-rate="0" line-rate="0.3704" timestamp="1328803087680" version="3.5.1">
<!-- Generated by coverage.py: http://nedbatchelder.com/code/coverage -->
<packages>
<package branch-rate="0" complexity="0" line-rate="0.5146" name="qsim">
<classes>
<class branch-rate="0" complexity="0" filename="qsim/__init__.py" line-rate="0" name="__init__">
<methods/>
<lines/>
</class>
<class branch-rate="0" complexity="0" filename="qsim/messaging.py" line-rate="0.5" name="messaging">
<methods/>
<lines>
<line hits="1" number="4"/>
<line hits="1" number="5"/>
<line hits="1" number="6"/>
<line hits="1" number="8"/>
<line hits="1" number="10"/>
<line hits="1" number="13"/>
<line hits="1" number="14"/>
<line hits="1" number="15"/>
<line hits="1" number="16"/>
<line hits="1" number="17"/>
<line hits="1" number="18"/>
<line hits="1" number="19"/>
<line hits="1" number="20"/>
<line hits="1" number="22"/>
<line hits="0" number="23"/>
<line hits="1" number="25"/>
<line hits="0" number="26"/>
<line hits="0" number="27"/>
<line hits="1" number="29"/>
<line hits="0" number="30"/>
<line hits="0" number="31"/>
<line hits="0" number="33"/>
<line hits="0" number="34"/>
<line hits="0" number="35"/>
<line hits="0" number="36"/>
<line hits="0" number="37"/>
<line hits="0" number="38"/>
<line hits="0" number="39"/>
<line hits="0" number="40"/>
<line hits="0" number="42"/>
<line hits="0" number="43"/>
<line hits="1" number="45"/>
<line hits="0" number="46"/>
<line hits="0" number="47"/>
<line hits="0" number="48"/>
<line hits="0" number="49"/>
<line hits="1" number="51"/>
<line hits="1" number="52"/>
<line hits="1" number="53"/>
<line hits="1" number="54"/>
<line hits="1" number="55"/>
<line hits="1" number="57"/>
<line hits="0" number="58"/>
<line hits="0" number="59"/>
<line hits="0" number="60"/>
<line hits="1" number="62"/>
<line hits="0" number="63"/>
<line hits="0" number="64"/>
<line hits="0" number="66"/>
<line hits="0" number="67"/>
<line hits="0" number="68"/>
<line hits="0" number="69"/>
<line hits="1" number="72"/>
<line hits="1" number="77"/>
<line hits="0" number="78"/>
<line hits="0" number="79"/>
<line hits="0" number="80"/>
<line hits="1" number="82"/>
<line hits="0" number="84"/>
<line hits="0" number="85"/>
<line hits="0" number="87"/>
<line hits="0" number="88"/>
<line hits="0" number="89"/>
<line hits="0" number="90"/>
<line hits="0" number="91"/>
<line hits="0" number="92"/>
<line hits="0" number="93"/>
<line hits="0" number="94"/>
<line hits="1" number="97"/>
<line hits="1" number="103"/>
<line hits="1" number="104"/>
<line hits="1" number="105"/>
<line hits="1" number="106"/>
<line hits="1" number="109"/>
<line hits="1" number="111"/>
<line hits="1" number="113"/>
<line hits="1" number="114"/>
<line hits="1" number="116"/>
<line hits="1" number="118"/>
<line hits="1" number="119"/>
<line hits="1" number="120"/>
<line hits="1" number="121"/>
</lines>
</class>
<class branch-rate="0" complexity="0" filename="qsim/util.py" line-rate="0.5714" name="util">
<methods/>
<lines>
<line hits="1" number="6"/>
<line hits="1" number="7"/>
<line hits="1" number="8"/>
<line hits="1" number="11"/>
<line hits="1" number="13"/>
<line hits="0" number="15"/>
<line hits="0" number="16"/>
<line hits="0" number="17"/>
<line hits="0" number="18"/>
<line hits="0" number="19"/>
<line hits="1" number="21"/>
<line hits="0" number="23"/>
<line hits="0" number="24"/>
<line hits="0" number="25"/>
<line hits="0" number="26"/>
<line hits="1" number="28"/>
<line hits="1" number="29"/>
<line hits="1" number="30"/>
<line hits="1" number="31"/>
<line hits="1" number="32"/>
<line hits="1" number="33"/>
</lines>
</class>
</classes>
</package>
<package branch-rate="0" complexity="0" line-rate="0.4231" name="qsim.simulator">
<classes>
<class branch-rate="0" complexity="0" filename="qsim/simulator/__init__.py" line-rate="0" name="__init__">
<methods/>
<lines/>
</class>
<class branch-rate="0" complexity="0" filename="qsim/simulator/config.py" line-rate="0.4138" name="config">
<methods/>
<lines>
<line hits="1" number="2"/>
<line hits="1" number="4"/>
<line hits="1" number="7"/>
<line hits="0" number="8"/>
<line hits="1" number="10"/>
<line hits="0" number="11"/>
<line hits="1" number="13"/>
<line hits="0" number="14"/>
<line hits="0" number="15"/>
<line hits="1" number="17"/>
<line hits="0" number="18"/>
<line hits="0" number="19"/>
<line hits="0" number="21"/>
<line hits="1" number="23"/>
<line hits="0" number="25"/>
<line hits="1" number="27"/>
<line hits="0" number="29"/>
<line hits="1" number="31"/>
<line hits="0" number="33"/>
<line hits="1" number="35"/>
<line hits="0" number="40"/>
<line hits="0" number="41"/>
<line hits="0" number="42"/>
<line hits="0" number="43"/>
<line hits="0" number="45"/>
<line hits="1" number="47"/>
<line hits="0" number="49"/>
<line hits="1" number="51"/>
<line hits="0" number="53"/>
</lines>
</class>
<class branch-rate="0" complexity="0" filename="qsim/simulator/feed.py" line-rate="0.3889" name="feed">
<methods/>
<lines>
<line hits="1" number="2"/>
<line hits="1" number="3"/>
<line hits="1" number="4"/>
<line hits="1" number="5"/>
<line hits="1" number="6"/>
<line hits="1" number="7"/>
<line hits="1" number="8"/>
<line hits="1" number="10"/>
<line hits="1" number="12"/>
<line hits="1" number="13"/>
<line hits="1" number="15"/>
<line hits="1" number="16"/>
<line hits="1" number="17"/>
<line hits="1" number="19"/>
<line hits="1" number="21"/>
<line hits="1" number="22"/>
<line hits="1" number="23"/>
<line hits="1" number="24"/>
<line hits="0" number="25"/>
<line hits="0" number="26"/>
<line hits="1" number="27"/>
<line hits="1" number="28"/>
<line hits="1" number="29"/>
<line hits="1" number="31"/>
<line hits="1" number="33"/>
<line hits="0" number="35"/>
<line hits="0" number="36"/>
<line hits="0" number="37"/>
<line hits="0" number="38"/>
<line hits="1" number="40"/>
<line hits="1" number="41"/>
<line hits="1" number="43"/>
<line hits="0" number="44"/>
<line hits="0" number="45"/>
<line hits="0" number="46"/>
<line hits="0" number="48"/>
<line hits="1" number="50"/>
<line hits="0" number="52"/>
<line hits="0" number="53"/>
<line hits="0" number="54"/>
<line hits="0" number="56"/>
<line hits="0" number="58"/>
<line hits="0" number="59"/>
<line hits="0" number="62"/>
<line hits="0" number="64"/>
<line hits="0" number="65"/>
<line hits="1" number="67"/>
<line hits="0" number="69"/>
<line hits="0" number="71"/>
<line hits="0" number="75"/>
<line hits="0" number="76"/>
<line hits="0" number="79"/>
<line hits="0" number="80"/>
<line hits="0" number="82"/>
<line hits="0" number="85"/>
<line hits="0" number="88"/>
<line hits="0" number="90"/>
<line hits="0" number="92"/>
<line hits="0" number="93"/>
<line hits="0" number="94"/>
<line hits="0" number="95"/>
<line hits="0" number="96"/>
<line hits="0" number="97"/>
<line hits="0" number="98"/>
<line hits="0" number="100"/>
<line hits="0" number="101"/>
<line hits="0" number="105"/>
<line hits="0" number="108"/>
<line hits="0" number="109"/>
<line hits="0" number="110"/>
<line hits="0" number="111"/>
<line hits="0" number="112"/>
</lines>
</class>
<class branch-rate="0" complexity="0" filename="qsim/simulator/sources.py" line-rate="0.4727" name="sources">
<methods/>
<lines>
<line hits="1" number="4"/>
<line hits="1" number="5"/>
<line hits="1" number="6"/>
<line hits="1" number="7"/>
<line hits="1" number="8"/>
<line hits="1" number="10"/>
<line hits="1" number="11"/>
<line hits="1" number="13"/>
<line hits="1" number="19"/>
<line hits="1" number="20"/>
<line hits="1" number="21"/>
<line hits="1" number="22"/>
<line hits="1" number="23"/>
<line hits="1" number="24"/>
<line hits="1" number="26"/>
<line hits="0" number="28"/>
<line hits="0" number="29"/>
<line hits="1" number="32"/>
<line hits="0" number="34"/>
<line hits="0" number="38"/>
<line hits="0" number="41"/>
<line hits="0" number="42"/>
<line hits="0" number="45"/>
<line hits="0" number="46"/>
<line hits="1" number="48"/>
<line hits="0" number="50"/>
<line hits="0" number="51"/>
<line hits="0" number="52"/>
<line hits="1" number="54"/>
<line hits="0" number="56"/>
<line hits="1" number="58"/>
<line hits="0" number="64"/>
<line hits="0" number="65"/>
<line hits="0" number="66"/>
<line hits="1" number="68"/>
<line hits="0" number="72"/>
<line hits="0" number="73"/>
<line hits="0" number="74"/>
<line hits="0" number="75"/>
<line hits="0" number="76"/>
<line hits="0" number="77"/>
<line hits="0" number="78"/>
<line hits="1" number="80"/>
<line hits="1" number="83"/>
<line hits="1" number="84"/>
<line hits="1" number="85"/>
<line hits="1" number="86"/>
<line hits="1" number="88"/>
<line hits="0" number="89"/>
<line hits="0" number="90"/>
<line hits="0" number="91"/>
<line hits="0" number="93"/>
<line hits="0" number="94"/>
<line hits="0" number="95"/>
<line hits="0" number="99"/>
</lines>
</class>
</classes>
</package>
<package branch-rate="0" complexity="0" line-rate="0.2123" name="qsim.transforms">
<classes>
<class branch-rate="0" complexity="0" filename="qsim/transforms/__init__.py" line-rate="0" name="__init__">
<methods/>
<lines/>
</class>
<class branch-rate="0" complexity="0" filename="qsim/transforms/core.py" line-rate="0.2353" name="core">
<methods/>
<lines>
<line hits="1" number="20"/>
<line hits="1" number="21"/>
<line hits="1" number="22"/>
<line hits="1" number="23"/>
<line hits="1" number="24"/>
<line hits="1" number="27"/>
<line hits="1" number="31"/>
<line hits="0" number="41"/>
<line hits="0" number="42"/>
<line hits="0" number="43"/>
<line hits="0" number="44"/>
<line hits="0" number="45"/>
<line hits="0" number="46"/>
<line hits="0" number="47"/>
<line hits="0" number="48"/>
<line hits="0" number="49"/>
<line hits="1" number="51"/>
<line hits="0" number="59"/>
<line hits="0" number="60"/>
<line hits="0" number="61"/>
<line hits="1" number="63"/>
<line hits="0" number="67"/>
<line hits="0" number="69"/>
<line hits="0" number="72"/>
<line hits="0" number="73"/>
<line hits="0" number="74"/>
<line hits="0" number="77"/>
<line hits="0" number="78"/>
<line hits="1" number="80"/>
<line hits="0" number="87"/>
<line hits="0" number="88"/>
<line hits="0" number="90"/>
<line hits="0" number="91"/>
<line hits="0" number="92"/>
<line hits="0" number="93"/>
<line hits="0" number="94"/>
<line hits="0" number="95"/>
<line hits="0" number="96"/>
<line hits="0" number="97"/>
<line hits="0" number="98"/>
<line hits="0" number="99"/>
<line hits="0" number="100"/>
<line hits="0" number="101"/>
<line hits="0" number="102"/>
<line hits="1" number="104"/>
<line hits="0" number="108"/>
<line hits="0" number="110"/>
<line hits="0" number="111"/>
<line hits="0" number="112"/>
<line hits="1" number="114"/>
<line hits="0" number="121"/>
</lines>
</class>
<class branch-rate="0" complexity="0" filename="qsim/transforms/merge.py" line-rate="0.1711" name="merge">
<methods/>
<lines>
<line hits="1" number="1"/>
<line hits="1" number="2"/>
<line hits="1" number="3"/>
<line hits="1" number="5"/>
<line hits="1" number="6"/>
<line hits="1" number="7"/>
<line hits="1" number="8"/>
<line hits="1" number="10"/>
<line hits="1" number="18"/>
<line hits="0" number="23"/>
<line hits="0" number="24"/>
<line hits="0" number="25"/>
<line hits="0" number="26"/>
<line hits="1" number="29"/>
<line hits="0" number="35"/>
<line hits="0" number="36"/>
<line hits="0" number="37"/>
<line hits="0" number="38"/>
<line hits="0" number="39"/>
<line hits="0" number="40"/>
<line hits="0" number="42"/>
<line hits="0" number="43"/>
<line hits="0" number="44"/>
<line hits="0" number="46"/>
<line hits="0" number="47"/>
<line hits="0" number="48"/>
<line hits="1" number="50"/>
<line hits="0" number="54"/>
<line hits="0" number="56"/>
<line hits="0" number="58"/>
<line hits="0" number="59"/>
<line hits="0" number="60"/>
<line hits="0" number="63"/>
<line hits="0" number="64"/>
<line hits="0" number="67"/>
<line hits="0" number="68"/>
<line hits="0" number="69"/>
<line hits="0" number="72"/>
<line hits="0" number="73"/>
<line hits="0" number="74"/>
<line hits="0" number="76"/>
<line hits="0" number="77"/>
<line hits="0" number="78"/>
<line hits="0" number="79"/>
<line hits="0" number="81"/>
<line hits="1" number="83"/>
<line hits="0" number="87"/>
<line hits="0" number="88"/>
<line hits="1" number="90"/>
<line hits="0" number="97"/>
<line hits="0" number="98"/>
<line hits="0" number="99"/>
<line hits="0" number="101"/>
<line hits="0" number="102"/>
<line hits="0" number="103"/>
<line hits="0" number="104"/>
<line hits="0" number="105"/>
<line hits="0" number="107"/>
<line hits="0" number="108"/>
<line hits="0" number="109"/>
<line hits="0" number="111"/>
<line hits="0" number="112"/>
<line hits="0" number="113"/>
<line hits="0" number="114"/>
<line hits="0" number="115"/>
<line hits="0" number="117"/>
<line hits="0" number="118"/>
<line hits="0" number="119"/>
<line hits="0" number="121"/>
<line hits="0" number="122"/>
<line hits="0" number="124"/>
<line hits="0" number="126"/>
<line hits="0" number="127"/>
<line hits="0" number="130"/>
<line hits="0" number="133"/>
<line hits="0" number="134"/>
</lines>
</class>
<class branch-rate="0" complexity="0" filename="qsim/transforms/technical.py" line-rate="0.3158" name="technical">
<methods/>
<lines>
<line hits="1" number="7"/>
<line hits="1" number="8"/>
<line hits="1" number="10"/>
<line hits="1" number="12"/>
<line hits="1" number="18"/>
<line hits="0" number="19"/>
<line hits="0" number="20"/>
<line hits="0" number="22"/>
<line hits="1" number="33"/>
<line hits="0" number="36"/>
<line hits="0" number="39"/>
<line hits="0" number="41"/>
<line hits="0" number="42"/>
<line hits="0" number="44"/>
<line hits="0" number="45"/>
<line hits="0" number="46"/>
<line hits="0" number="48"/>
<line hits="0" number="50"/>
<line hits="0" number="52"/>
</lines>
</class>
</classes>
</package>
</packages>
</coverage>
docs/qsim.rst
+75 -3
View File
@@ -1,6 +1,81 @@
============
qsim Package
============
Simulator is the heart of QSim, and the primary access point for creating, launching, and tracking simulations. You can find it in :py:class:`~qsim.core.Simulator`
Simulator Sub-Components
==========================
Each simulation contains numerous subcomponents, each operating asynchronously from all others, and communicating
via zeromq.
DataSources
--------------------
A DataSource represents a historical event record, which will be played back during simulation. A simulation may have one or more DataSources, which will be combined in DataFeed. Generally, datasources read records from a persistent store (db, csv file, remote service), format the messages for downstream simulation components, and send them to a PUSH socket. See the base class for all datasources :py:class:`~qsim.sources.DataSource`
DataFeed
--------------------
All simulations start with a collection of :py:class:`DataSources <qsim.sources.DataSource>`, which need to be fed to an algorithm. Each :py:class:`~qsim.sources.DataSource`can contain events of differing content (trades, quotes, corporate event) and frequency (quarterly, intraday). To simplify the process of managing the data sources, :py:class:`~qsim.core.DataFeed` can receive events from multiple :py:class:`DataSources <qsim.sources.DataSource>` and combine them into a serial chronological stream.
Transforms
--------------------
Often, an algorithm will require a running calculation on top of a :py:class:`~qsim.sources.DataSource`, or on the consolidated feed. A simple example is a technical indicator or a moving average. Transforms can be described in :py:class:`~qsim.core.Simulator`'s configuration. Subclass :py:class:`~qsim.transforms.core.Transform` to add your own Transform. Transforms must hold their own state between events, and serialize their current values into messages.
Data Alignment
--------------------
Like Datasources, Transforms have differing frequencies and results. Simulator manages the results of parallel transforms and **aligns** transform results with the raw DataFeed. Client algorithms simply receive a map of data, which includes the current event and all the transformed values.
QSim API
===========================
:mod:`qsim` Package
-------------------
.. automodule:: qsim.__init__
:members:
:undoc-members:
:show-inheritance:
:mod:`config` Module
--------------------
.. automodule:: qsim.config
:members:
:undoc-members:
:show-inheritance:
:mod:`core` Module
------------------
.. automodule:: qsim.core
:members:
:undoc-members:
:show-inheritance:
:mod:`messaging` Module
-----------------------
.. automodule:: qsim.messaging
:members:
:undoc-members:
:show-inheritance:
:mod:`sources` Module
---------------------
.. automodule:: qsim.sources
:members:
:undoc-members:
:show-inheritance:
:mod:`util` Module
------------------
@@ -14,9 +89,6 @@ Subpackages
.. toctree::
qsim.data
qsim.messaging
qsim.simulator
qsim.test
qsim.transforms
docs/qsim.simulator.rst
-27
View File
@@ -1,27 +0,0 @@
simulator Package
=================
:mod:`config` Module
--------------------
.. automodule:: qsim.simulator.config
:members:
:undoc-members:
:show-inheritance:
:mod:`feed` Module
------------------
.. automodule:: qsim.simulator.feed
:members:
:undoc-members:
:show-inheritance:
:mod:`qbt` Module
-----------------
.. automodule:: qsim.simulator.qbt
:members:
:undoc-members:
:show-inheritance:
docs/qsim.transforms.rst
+10 -2
View File
@@ -1,10 +1,18 @@
transforms Package
==================
:mod:`base` Module
:mod:`core` Module
------------------
.. automodule:: qsim.transforms.base
.. automodule:: qsim.transforms.core
:members:
:undoc-members:
:show-inheritance:
:mod:`merge` Module
-------------------
.. automodule:: qsim.transforms.merge
:members:
:undoc-members:
:show-inheritance:
nosetests.xml
-1
View File
@@ -1 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?><testsuite name="nosetests" tests="1" errors="0" failures="0" skip="0"><testcase classname="qsim.test.test_messaging.MessagingTestCase" name="test_client" time="0.408" /></testsuite>
pylint.rcfile
+5 -5
View File
@@ -92,10 +92,10 @@ method-rgx=[a-z_][a-z0-9_]{2,30}$|setUp
attr-rgx=[a-z_][a-z0-9_]{2,30}$
# Regular expression which should only match correct argument names
argument-rgx=[a-z_][a-z0-9_]{2,30}$
argument-rgx=[a-z_][a-z0-9_]{1,30}$
# Regular expression which should only match correct variable names
variable-rgx=[a-z_][a-z0-9_]{2,30}$
variable-rgx=[a-z_][a-z0-9_]{1,30}$
# Regular expression which should only match correct list comprehension /
# generator expression variable names
@@ -115,7 +115,7 @@ no-docstring-rgx=__.*__
[FORMAT]
# Maximum number of characters on a single line.
max-line-length=125
max-line-length=130
# Maximum number of lines in a module
max-module-lines=1000
@@ -211,10 +211,10 @@ max-statements=50
max-parents=7
# Maximum number of attributes for a class (see R0902).
max-attributes=7
max-attributes=15
# Minimum number of public methods for a class (see R0903).
min-public-methods=2
min-public-methods=1
# Maximum number of public methods for a class (see R0904).
max-public-methods=20
qsim/__init__.py
+3
View File
@@ -0,0 +1,3 @@
"""
QSim provides asynchronous simulation of historic data streams, simulated trade execution, and data stream transformations.
"""
qsim/simulator/config.py → qsim/config.py
+4 -3
View File
@@ -1,4 +1,5 @@
"""Tools for managing configuration data for sources and transforms."""
import json
class Config(object):
@@ -38,9 +39,9 @@ class Config(object):
if the property is missing, or is not of the right type, return default.
"""
if(self.store.has_key(name)):
val = self.store[name]
if isinstance(val, expected_type):
return val
val = self.store[name]
if isinstance(val, expected_type):
return val
else:
return default
qsim/simulator/feed.py → qsim/core.py
+38 -12
View File
@@ -1,5 +1,8 @@
"""
Provides simulated data feed services.
"""
import qsim.simulator.sources as sources
import qsim.sources as sources
import qsim.util as qutil
import qsim.messaging as qmsg
import zmq
@@ -7,10 +10,31 @@ import time
import logging
import json
class DataFeed(object):
class Simulator(object):
"""
Simulator is the heart of QSim. The beating heart...
"""
def __init__(self, config):
qutil.logger = qutil.logger
"""
:config: a qsim.config.Config object that contains configuration information for all datasources, all transforms, and all
client algorithms that simulator should create.
"""
self.config = config
def launch(self):
"""
Create all components specified in config.
"""
pass
class DataFeed(object):
"""DataFeed is the heart of a simulation. It is initialized with a configuration for """
def __init__(self, config):
qutil.LOGGER = qutil.LOGGER
self.data_address = "tcp://127.0.0.1:{port}".format(port=10101)
self.sync_address = "tcp://127.0.0.1:{port}".format(port=10102)
@@ -33,9 +57,9 @@ class DataFeed(object):
def start_data_workers(self):
"""Start a sub-process for each datasource."""
for source_id, source in self.data_workers.iteritems():
qutil.logger.info("starting {id}".format(id=source_id))
qutil.LOGGER.info("starting {id}".format(id=source_id))
source.start()
qutil.logger.info("ds processes launched")
qutil.LOGGER.info("ds processes launched")
def register_sync(self, sync_id):
self.client_register[sync_id] = "UNCONFIRMED"
@@ -49,7 +73,7 @@ class DataFeed(object):
def sync_clients(self):
# Socket to receive signals
qutil.logger.info("waiting for all datasources and clients to be ready")
qutil.LOGGER.info("waiting for all datasources and clients to be ready")
self.syncservice = self.context.socket(zmq.REP)
self.syncservice.bind(self.sync_address)
@@ -57,12 +81,12 @@ class DataFeed(object):
# wait for synchronization request
msg = self.syncservice.recv()
self.client_register[msg] = "CONFIRMED"
#qutil.logger.info("confirmed {id}".format(id=msg))
#qutil.LOGGER.info("confirmed {id}".format(id=msg))
# send synchronization reply
self.syncservice.send('CONFIRMED')
self.syncservice.close()
qutil.logger.info("sync'd all datasources and clients")
qutil.LOGGER.info("sync'd all datasources and clients")
def run(self):
# Prepare our context and sockets
@@ -87,7 +111,7 @@ class DataFeed(object):
#wait for all feed subscribers
self.sync_clients()
qutil.logger.info("entering feed loop on {addr}".format(addr=self.data_address))
qutil.LOGGER.info("entering feed loop on {addr}".format(addr=self.data_address))
while True:
message = self.data_socket.recv()
@@ -106,14 +130,16 @@ class DataFeed(object):
#send the DONE message
self.feed_socket.send("DONE")
qutil.logger.info("received {n} messages, sent {m} messages".format(n=self.data_buffer.received_count, m=self.data_buffer.sent_count))
qutil.LOGGER.info("received {n} messages, sent {m} messages".format(n=self.data_buffer.received_count,
m=self.data_buffer.sent_count))
self.data_socket.close()
self.feed_socket.close()
self.context.term()
qsim/messaging.py
+14 -12
View File
@@ -35,7 +35,7 @@ class ParallelBuffer(object):
cur = None
earliest = None
for source, events in self.data_buffer.iteritems():
for events in self.data_buffer.values():
if len(events) == 0:
continue
cur = events
@@ -47,29 +47,31 @@ class ParallelBuffer(object):
def is_full(self):
"""indicates whether the buffer has messages in buffer for all un-DONE sources"""
for source, events in self.data_buffer.iteritems():
for events in self.data_buffer.values():
if (len(events) == 0):
return False
return True
def pending_messages(self):
""""""
"""returns the count of all events from all sources in the buffer"""
total = 0
for source, events in self.data_buffer.iteritems():
for events in self.data_buffer.values():
total += len(events)
return total
return total
def drain(self):
"""send all messages in the buffer"""
self.draining = True
while(self.pending_messages() > 0):
self.send_next()
def send_next(self):
"""send the (chronologically) next message in the buffer."""
if(not(self.is_full() or self.draining)):
return
event = self.next()
if(event != None):
if(event != None):
self.out_socket.send(json.dumps(event))
self.sent_count += 1
@@ -107,9 +109,9 @@ class FeedSync(object):
def __init__(self, feed, name):
self.feed = feed
self.id = "{name}-{id}".format(name=name, id=uuid.uuid1())
self.feed.register_sync(self.id)
#qutil.logger.info("registered {id} with feed".format(id=self.id))
self.sync_id = "{name}-{id}".format(name=name, id=uuid.uuid1())
self.feed.register_sync(self.sync_id)
#qutil.LOGGER.info("registered {id} with feed".format(id=self.sync_id))
def confirm(self):
"""Confirm readiness with the DataFeed."""
@@ -118,10 +120,10 @@ class FeedSync(object):
sync_socket = context.socket(zmq.REQ)
sync_socket.connect(self.feed.sync_address)
# send a synchronization request to the feed
sync_socket.send(self.id)
sync_socket.send(self.sync_id)
# wait for synchronization reply from the feed
sync_socket.recv()
sync_socket.close()
context.term()
qutil.logger.info("sync'd feed from {id}".format(id = self.id))
qutil.LOGGER.info("sync'd feed from {id}".format(id = self.sync_id))
qsim/simulator/__init__.py
View File
qsim/simulator/sources.py → qsim/sources.py
+3 -3
View File
@@ -1,5 +1,5 @@
"""
Provides data handlers that can push messages to a qsim.simulator.DataFeed
Provides data handlers that can push messages to a qsim.core.DataFeed
"""
import datetime
import zmq
@@ -31,7 +31,7 @@ class DataSource(object):
def open(self):
"""create zmq context and socket"""
qutil.logger.info(
qutil.LOGGER.info(
"starting data source:{source_id} on {addr}"
.format(source_id=self.source_id, addr=self.feed.data_address))
@@ -75,7 +75,7 @@ class DataSource(object):
self.data_socket.send(json.dumps(done_msg))
self.data_socket.close()
self.context.term()
qutil.logger.info("finished processing data source")
qutil.LOGGER.info("finished processing data source")
class RandomEquityTrades(DataSource):
"""Generates a random stream of trades for testing."""
qsim/test/client.py
+7 -7
View File
@@ -18,27 +18,27 @@ class TestClient(object):
def run(self):
qutil.logger.info("running the client")
qutil.LOGGER.info("running the client")
self.context = zmq.Context()
self.data_feed = self.context.socket(zmq.PULL)
if(self.bind):
qutil.logger.info("binding to {address}".format(address=self.address))
qutil.LOGGER.info("binding to {address}".format(address=self.address))
self.data_feed.bind(self.address)
else:
qutil.logger.info("connecting to {address}".format(address=self.address))
qutil.LOGGER.info("connecting to {address}".format(address=self.address))
self.data_feed.connect(self.address)
self.sync.confirm()
qutil.logger.info("Starting the client loop")
qutil.LOGGER.info("Starting the client loop")
prev_dt = None
while True:
msg = self.data_feed.recv()
if(msg == "DONE"):
qutil.logger.info("DONE!")
qutil.LOGGER.info("DONE!")
break
self.received_count += 1
event = json.loads(msg)
@@ -48,8 +48,8 @@ class TestClient(object):
prev_dt = event['dt']
if(self.received_count % 100 == 0):
qutil.logger.info("received {n} messages".format(n=self.received_count))
qutil.LOGGER.info("received {n} messages".format(n=self.received_count))
qutil.logger.info("received {n} messages".format(n=self.received_count))
qutil.LOGGER.info("received {n} messages".format(n=self.received_count))
self.data_feed.close()
self.context.term()
qsim/test/test_messaging.py
+2 -2
View File
@@ -6,7 +6,7 @@ Test suite for the messaging infrastructure of QSim.
import unittest2 as unittest
import multiprocessing
from qsim.simulator.feed import DataFeed
from qsim.core import DataFeed
from qsim.transforms.merge import MergedTransformsFeed
from qsim.transforms.technical import MovingAverage
import qsim.util as qutil
@@ -21,7 +21,7 @@ class MessagingTestCase(unittest.TestCase):
"""generate some config objects for the datafeed, sources, and transforms."""
qutil.configure_logging()
qutil.logger.info("testing...")
qutil.LOGGER.info("testing...")
self.total_data_count = 800
self.feed_config = {'emt1':{'sid':133, 'class':'RandomEquityTrades', 'count':400},
'emt2':{'sid':134, 'class':'RandomEquityTrades', 'count':400}}
qsim/transforms/core.py
+6 -6
View File
@@ -3,7 +3,7 @@ Transforms
==========
Transforms provide re-useable components for stream processing. All
Transforms expect to receive data events from qsim.simulator.feed.DataFeed
Transforms expect to receive data events from qsim.core.DataFeed
asynchronously via zeromq. Each transform is designed to run in independent
process space, independently of all other transforms, to allow for parallel
computation.
@@ -21,7 +21,7 @@ import zmq
import json
import qsim.messaging as qmsg
import qsim.util as qutil
import qsim.simulator.config as config
import qsim.config as config
class BaseTransform(object):
@@ -66,7 +66,7 @@ class BaseTransform(object):
"""
self.context = zmq.Context()
qutil.logger.info("starting {name} transform".
qutil.LOGGER.info("starting {name} transform".
format(name = self.state['name']))
#create the feed SUB.
self.feed_socket = self.context.socket(zmq.SUB)
@@ -84,13 +84,13 @@ class BaseTransform(object):
- call transform (subclass' method) on event
- send the transformed event
"""
qutil.logger.info("starting {name} event loop".format(name = self.state['name']))
qutil.LOGGER.info("starting {name} event loop".format(name = self.state['name']))
self.sync.confirm()
while True:
message = self.feed_socket.recv()
if(message == "DONE"):
qutil.logger.info("{name} received the Done message from the feed".format(name=self.state['name']))
qutil.LOGGER.info("{name} received the Done message from the feed".format(name=self.state['name']))
self.result_socket.send("DONE")
break;
self.received_count += 1
@@ -105,7 +105,7 @@ class BaseTransform(object):
"""
Shut down zmq resources.
"""
qutil.logger.info("Transform {name} recieved {r} and sent {s}".format(name=self.state['name'], r=self.received_count, s=self.sent_count))
qutil.LOGGER.info("Transform {name} recieved {r} and sent {s}".format(name=self.state['name'], r=self.received_count, s=self.sent_count))
self.feed_socket.close()
self.result_socket.close()
qsim/transforms/merge.py
+7 -7
View File
@@ -53,7 +53,7 @@ class MergedTransformsFeed(BaseTransform):
ready with the DataFeed at the conclusion."""
self.context = zmq.Context()
qutil.logger.info("starting {name} transform".format(name = self.state['name']))
qutil.LOGGER.info("starting {name} transform".format(name = self.state['name']))
#create the feed SUB.
self.feed_socket = self.context.socket(zmq.SUB)
self.feed_socket.connect(self.feed.feed_address)
@@ -74,7 +74,7 @@ class MergedTransformsFeed(BaseTransform):
self.poller.register(self.transform_socket, zmq.POLLIN)
for name, transform in self.transforms.iteritems():
qutil.logger.info("starting {name}".format(name=name))
qutil.LOGGER.info("starting {name}".format(name=name))
proc = multiprocessing.Process(target=transform.run)
proc.start()
@@ -101,7 +101,7 @@ class MergedTransformsFeed(BaseTransform):
if self.feed_socket in socks and socks[self.feed_socket] == zmq.POLLIN:
message = self.feed_socket.recv()
if(message == "DONE"):
qutil.logger.info("finished receiving feed to merge")
qutil.LOGGER.info("finished receiving feed to merge")
done_count += 1
else:
self.received_count += 1
@@ -111,7 +111,7 @@ class MergedTransformsFeed(BaseTransform):
if self.transform_socket in socks and socks[self.transform_socket] == zmq.POLLIN:
t_message = self.transform_socket.recv()
if(t_message == "DONE"):
qutil.logger.info("finished receiving a transform to merge")
qutil.LOGGER.info("finished receiving a transform to merge")
done_count += 1
else:
self.received_count += 1
@@ -123,13 +123,13 @@ class MergedTransformsFeed(BaseTransform):
self.data_buffer.send_next()
qutil.logger.info("Transform {name} received {r} and sent {s}".format(name=self.state['name'], r=self.data_buffer.received_count, s=self.data_buffer.sent_count))
qutil.logger.info("about to drain {n} messages from merger's buffer".format(n=self.data_buffer.pending_messages()))
qutil.LOGGER.info("Transform {name} received {r} and sent {s}".format(name=self.state['name'], r=self.data_buffer.received_count, s=self.data_buffer.sent_count))
qutil.LOGGER.info("about to drain {n} messages from merger's buffer".format(n=self.data_buffer.pending_messages()))
#drain any remaining messages in the buffer
self.data_buffer.drain()
#signal to client that we're done
self.result_socket.send("DONE")
qutil.logger.info("Transform {name} received {r} and sent {s}".format(name=self.state['name'], r=self.data_buffer.received_count, s=self.data_buffer.sent_count))
qutil.LOGGER.info("Transform {name} received {r} and sent {s}".format(name=self.state['name'], r=self.data_buffer.received_count, s=self.data_buffer.sent_count))
qsim/util.py
+18 -13
View File
@@ -1,5 +1,5 @@
"""
Small classes to assist with timezone calculations, logger configuration,
Small classes to assist with timezone calculations, LOGGER configuration,
and other common operations.
"""
@@ -8,26 +8,31 @@ import pytz
import logging
logger = logging.getLogger('QSimLogger')
LOGGER = logging.getLogger('QSimLogger')
def parse_date(dt_str):
"""parse strings according to the same format as generated by format_date"""
if(dt_str == None):
return None
parts = dt_str.split(".")
dt = datetime.datetime.strptime(parts[0], '%Y/%m/%d-%H:%M:%S').replace(microsecond=int(parts[1]+"000")).replace(tzinfo = pytz.utc)
dt = datetime.datetime.strptime(parts[0], '%Y/%m/%d-%H:%M:%S').replace(microsecond=int(parts[1]+"000"), tzinfo = pytz.utc)
return dt
def format_date(dt):
"""Format the date into a date with millesecond resolution and string/alphabetical sorting that is equivalent to datetime sorting"""
"""Format the date into a date with millesecond resolution and string/alphabetical
sorting that is equivalent to datetime sorting"""
if(dt == None):
return None
dt_str = dt.strftime('%Y/%m/%d-%H:%M:%S') + "." + str(dt.microsecond / 1000)
return dt_str
def configure_logging(loglevel=logging.DEBUG):
logger.setLevel(loglevel)
handler = logging.handlers.RotatingFileHandler("/tmp/{lfn}.log".format(lfn="qsim-log"), maxBytes=10*1024*1024, backupCount=5)
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(filename)s %(funcName)s - %(message)s","%Y-%m-%d %H:%M:%S %Z"))
logger.addHandler(handler)
logger.info("logging started...")
def configure_logging(loglevel=logging.DEBUG):
"""configures qsim.util.LOGGER to write a rotating file (10M per file, 5 files) to /var/log/qsim.log"""
LOGGER.setLevel(loglevel)
handler = logging.handlers.RotatingFileHandler(
"/var/log/qsim/{lfn}.log".format(lfn="qsim"),
maxBytes=10*1024*1024, backupCount=5)
handler.setFormatter(logging.Formatter(
"%(asctime)s %(levelname)s %(filename)s %(funcName)s - %(message)s","%Y-%m-%d %H:%M:%S %Z"))
LOGGER.addHandler(handler)
LOGGER.info("logging started...")