From 227eb35ee2f3e0373acedabe7272cf5804061800 Mon Sep 17 00:00:00 2001 From: fawce Date: Fri, 10 Feb 2012 17:02:01 -0500 Subject: [PATCH] documentation on simulator, delinting, refactoring --- .gitignore | 2 + coverage.xml | 477 ---------------------------- docs/qsim.rst | 78 ++++- docs/qsim.simulator.rst | 27 -- docs/qsim.transforms.rst | 12 +- nosetests.xml | 1 - pylint.rcfile | 10 +- qsim/__init__.py | 3 + qsim/{simulator => }/config.py | 7 +- qsim/{simulator/feed.py => core.py} | 50 ++- qsim/messaging.py | 26 +- qsim/simulator/__init__.py | 0 qsim/{simulator => }/sources.py | 6 +- qsim/test/client.py | 14 +- qsim/test/test_messaging.py | 4 +- qsim/transforms/core.py | 12 +- qsim/transforms/merge.py | 14 +- qsim/util.py | 31 +- 18 files changed, 194 insertions(+), 580 deletions(-) delete mode 100644 coverage.xml delete mode 100644 docs/qsim.simulator.rst delete mode 100644 nosetests.xml rename qsim/{simulator => }/config.py (93%) rename qsim/{simulator/feed.py => core.py} (73%) delete mode 100644 qsim/simulator/__init__.py rename qsim/{simulator => }/sources.py (94%) diff --git a/.gitignore b/.gitignore index 64c74f01..6d9cb863 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,8 @@ var sdist develop-eggs .installed.cfg +coverage.xml +nosetests.xml *.swp *.swo diff --git a/coverage.xml b/coverage.xml deleted file mode 100644 index d2179403..00000000 --- a/coverage.xml +++ /dev/null @@ -1,477 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/qsim.rst b/docs/qsim.rst index c1751ad5..f1bb8fb1 100644 --- a/docs/qsim.rst +++ b/docs/qsim.rst @@ -1,6 +1,81 @@ +============ qsim Package ============ +Simulator is the heart of QSim, and the primary access point for creating, launching, and tracking simulations. You can find it in :py:class:`~qsim.core.Simulator` + +Simulator Sub-Components +========================== + +Each simulation contains numerous subcomponents, each operating asynchronously from all others, and communicating +via zeromq. + +DataSources +-------------------- + +A DataSource represents a historical event record, which will be played back during simulation. A simulation may have one or more DataSources, which will be combined in DataFeed. Generally, datasources read records from a persistent store (db, csv file, remote service), format the messages for downstream simulation components, and send them to a PUSH socket. See the base class for all datasources :py:class:`~qsim.sources.DataSource` + +DataFeed +-------------------- + +All simulations start with a collection of :py:class:`DataSources `, which need to be fed to an algorithm. Each :py:class:`~qsim.sources.DataSource`can contain events of differing content (trades, quotes, corporate event) and frequency (quarterly, intraday). To simplify the process of managing the data sources, :py:class:`~qsim.core.DataFeed` can receive events from multiple :py:class:`DataSources ` and combine them into a serial chronological stream. + +Transforms +-------------------- + +Often, an algorithm will require a running calculation on top of a :py:class:`~qsim.sources.DataSource`, or on the consolidated feed. A simple example is a technical indicator or a moving average. Transforms can be described in :py:class:`~qsim.core.Simulator`'s configuration. Subclass :py:class:`~qsim.transforms.core.Transform` to add your own Transform. Transforms must hold their own state between events, and serialize their current values into messages. + + +Data Alignment +-------------------- + +Like Datasources, Transforms have differing frequencies and results. Simulator manages the results of parallel transforms and **aligns** transform results with the raw DataFeed. Client algorithms simply receive a map of data, which includes the current event and all the transformed values. + + + +QSim API +=========================== + +:mod:`qsim` Package +------------------- + +.. automodule:: qsim.__init__ + :members: + :undoc-members: + :show-inheritance: + +:mod:`config` Module +-------------------- + +.. automodule:: qsim.config + :members: + :undoc-members: + :show-inheritance: + +:mod:`core` Module +------------------ + +.. automodule:: qsim.core + :members: + :undoc-members: + :show-inheritance: + +:mod:`messaging` Module +----------------------- + +.. automodule:: qsim.messaging + :members: + :undoc-members: + :show-inheritance: + +:mod:`sources` Module +--------------------- + +.. automodule:: qsim.sources + :members: + :undoc-members: + :show-inheritance: + :mod:`util` Module ------------------ @@ -14,9 +89,6 @@ Subpackages .. toctree:: - qsim.data - qsim.messaging - qsim.simulator qsim.test qsim.transforms diff --git a/docs/qsim.simulator.rst b/docs/qsim.simulator.rst deleted file mode 100644 index 5066af95..00000000 --- a/docs/qsim.simulator.rst +++ /dev/null @@ -1,27 +0,0 @@ -simulator Package -================= - -:mod:`config` Module --------------------- - -.. automodule:: qsim.simulator.config - :members: - :undoc-members: - :show-inheritance: - -:mod:`feed` Module ------------------- - -.. automodule:: qsim.simulator.feed - :members: - :undoc-members: - :show-inheritance: - -:mod:`qbt` Module ------------------ - -.. automodule:: qsim.simulator.qbt - :members: - :undoc-members: - :show-inheritance: - diff --git a/docs/qsim.transforms.rst b/docs/qsim.transforms.rst index 3aa83e56..6b8b05c9 100644 --- a/docs/qsim.transforms.rst +++ b/docs/qsim.transforms.rst @@ -1,10 +1,18 @@ transforms Package ================== -:mod:`base` Module +:mod:`core` Module ------------------ -.. automodule:: qsim.transforms.base +.. automodule:: qsim.transforms.core + :members: + :undoc-members: + :show-inheritance: + +:mod:`merge` Module +------------------- + +.. automodule:: qsim.transforms.merge :members: :undoc-members: :show-inheritance: diff --git a/nosetests.xml b/nosetests.xml deleted file mode 100644 index 8557c4d9..00000000 --- a/nosetests.xml +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/pylint.rcfile b/pylint.rcfile index b682c7fb..55e39e25 100644 --- a/pylint.rcfile +++ b/pylint.rcfile @@ -92,10 +92,10 @@ method-rgx=[a-z_][a-z0-9_]{2,30}$|setUp attr-rgx=[a-z_][a-z0-9_]{2,30}$ # Regular expression which should only match correct argument names -argument-rgx=[a-z_][a-z0-9_]{2,30}$ +argument-rgx=[a-z_][a-z0-9_]{1,30}$ # Regular expression which should only match correct variable names -variable-rgx=[a-z_][a-z0-9_]{2,30}$ +variable-rgx=[a-z_][a-z0-9_]{1,30}$ # Regular expression which should only match correct list comprehension / # generator expression variable names @@ -115,7 +115,7 @@ no-docstring-rgx=__.*__ [FORMAT] # Maximum number of characters on a single line. -max-line-length=125 +max-line-length=130 # Maximum number of lines in a module max-module-lines=1000 @@ -211,10 +211,10 @@ max-statements=50 max-parents=7 # Maximum number of attributes for a class (see R0902). -max-attributes=7 +max-attributes=15 # Minimum number of public methods for a class (see R0903). -min-public-methods=2 +min-public-methods=1 # Maximum number of public methods for a class (see R0904). max-public-methods=20 diff --git a/qsim/__init__.py b/qsim/__init__.py index e69de29b..f47dbf47 100644 --- a/qsim/__init__.py +++ b/qsim/__init__.py @@ -0,0 +1,3 @@ +""" +QSim provides asynchronous simulation of historic data streams, simulated trade execution, and data stream transformations. +""" \ No newline at end of file diff --git a/qsim/simulator/config.py b/qsim/config.py similarity index 93% rename from qsim/simulator/config.py rename to qsim/config.py index c460d60d..fa2b9a9f 100644 --- a/qsim/simulator/config.py +++ b/qsim/config.py @@ -1,4 +1,5 @@ """Tools for managing configuration data for sources and transforms.""" + import json class Config(object): @@ -38,9 +39,9 @@ class Config(object): if the property is missing, or is not of the right type, return default. """ if(self.store.has_key(name)): - val = self.store[name] - if isinstance(val, expected_type): - return val + val = self.store[name] + if isinstance(val, expected_type): + return val else: return default diff --git a/qsim/simulator/feed.py b/qsim/core.py similarity index 73% rename from qsim/simulator/feed.py rename to qsim/core.py index 086f151b..1edd8e76 100644 --- a/qsim/simulator/feed.py +++ b/qsim/core.py @@ -1,5 +1,8 @@ +""" +Provides simulated data feed services. +""" -import qsim.simulator.sources as sources +import qsim.sources as sources import qsim.util as qutil import qsim.messaging as qmsg import zmq @@ -7,10 +10,31 @@ import time import logging import json -class DataFeed(object): +class Simulator(object): + """ + Simulator is the heart of QSim. The beating heart... + """ def __init__(self, config): - qutil.logger = qutil.logger + """ + :config: a qsim.config.Config object that contains configuration information for all datasources, all transforms, and all + client algorithms that simulator should create. + """ + self.config = config + + def launch(self): + """ + Create all components specified in config. + """ + pass + + + +class DataFeed(object): + """DataFeed is the heart of a simulation. It is initialized with a configuration for """ + + def __init__(self, config): + qutil.LOGGER = qutil.LOGGER self.data_address = "tcp://127.0.0.1:{port}".format(port=10101) self.sync_address = "tcp://127.0.0.1:{port}".format(port=10102) @@ -33,9 +57,9 @@ class DataFeed(object): def start_data_workers(self): """Start a sub-process for each datasource.""" for source_id, source in self.data_workers.iteritems(): - qutil.logger.info("starting {id}".format(id=source_id)) + qutil.LOGGER.info("starting {id}".format(id=source_id)) source.start() - qutil.logger.info("ds processes launched") + qutil.LOGGER.info("ds processes launched") def register_sync(self, sync_id): self.client_register[sync_id] = "UNCONFIRMED" @@ -49,7 +73,7 @@ class DataFeed(object): def sync_clients(self): # Socket to receive signals - qutil.logger.info("waiting for all datasources and clients to be ready") + qutil.LOGGER.info("waiting for all datasources and clients to be ready") self.syncservice = self.context.socket(zmq.REP) self.syncservice.bind(self.sync_address) @@ -57,12 +81,12 @@ class DataFeed(object): # wait for synchronization request msg = self.syncservice.recv() self.client_register[msg] = "CONFIRMED" - #qutil.logger.info("confirmed {id}".format(id=msg)) + #qutil.LOGGER.info("confirmed {id}".format(id=msg)) # send synchronization reply self.syncservice.send('CONFIRMED') self.syncservice.close() - qutil.logger.info("sync'd all datasources and clients") + qutil.LOGGER.info("sync'd all datasources and clients") def run(self): # Prepare our context and sockets @@ -87,7 +111,7 @@ class DataFeed(object): #wait for all feed subscribers self.sync_clients() - qutil.logger.info("entering feed loop on {addr}".format(addr=self.data_address)) + qutil.LOGGER.info("entering feed loop on {addr}".format(addr=self.data_address)) while True: message = self.data_socket.recv() @@ -106,14 +130,16 @@ class DataFeed(object): #send the DONE message self.feed_socket.send("DONE") - qutil.logger.info("received {n} messages, sent {m} messages".format(n=self.data_buffer.received_count, m=self.data_buffer.sent_count)) + qutil.LOGGER.info("received {n} messages, sent {m} messages".format(n=self.data_buffer.received_count, + m=self.data_buffer.sent_count)) self.data_socket.close() self.feed_socket.close() self.context.term() - - + + + diff --git a/qsim/messaging.py b/qsim/messaging.py index 2e6e50cd..4e47d1b2 100644 --- a/qsim/messaging.py +++ b/qsim/messaging.py @@ -35,7 +35,7 @@ class ParallelBuffer(object): cur = None earliest = None - for source, events in self.data_buffer.iteritems(): + for events in self.data_buffer.values(): if len(events) == 0: continue cur = events @@ -47,29 +47,31 @@ class ParallelBuffer(object): def is_full(self): """indicates whether the buffer has messages in buffer for all un-DONE sources""" - for source, events in self.data_buffer.iteritems(): + for events in self.data_buffer.values(): if (len(events) == 0): return False return True def pending_messages(self): - """""" + """returns the count of all events from all sources in the buffer""" total = 0 - for source, events in self.data_buffer.iteritems(): + for events in self.data_buffer.values(): total += len(events) - return total + return total def drain(self): + """send all messages in the buffer""" self.draining = True while(self.pending_messages() > 0): self.send_next() def send_next(self): + """send the (chronologically) next message in the buffer.""" if(not(self.is_full() or self.draining)): return - + event = self.next() - if(event != None): + if(event != None): self.out_socket.send(json.dumps(event)) self.sent_count += 1 @@ -107,9 +109,9 @@ class FeedSync(object): def __init__(self, feed, name): self.feed = feed - self.id = "{name}-{id}".format(name=name, id=uuid.uuid1()) - self.feed.register_sync(self.id) - #qutil.logger.info("registered {id} with feed".format(id=self.id)) + self.sync_id = "{name}-{id}".format(name=name, id=uuid.uuid1()) + self.feed.register_sync(self.sync_id) + #qutil.LOGGER.info("registered {id} with feed".format(id=self.sync_id)) def confirm(self): """Confirm readiness with the DataFeed.""" @@ -118,10 +120,10 @@ class FeedSync(object): sync_socket = context.socket(zmq.REQ) sync_socket.connect(self.feed.sync_address) # send a synchronization request to the feed - sync_socket.send(self.id) + sync_socket.send(self.sync_id) # wait for synchronization reply from the feed sync_socket.recv() sync_socket.close() context.term() - qutil.logger.info("sync'd feed from {id}".format(id = self.id)) + qutil.LOGGER.info("sync'd feed from {id}".format(id = self.sync_id)) \ No newline at end of file diff --git a/qsim/simulator/__init__.py b/qsim/simulator/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/qsim/simulator/sources.py b/qsim/sources.py similarity index 94% rename from qsim/simulator/sources.py rename to qsim/sources.py index aa7ccc64..90c2b63c 100644 --- a/qsim/simulator/sources.py +++ b/qsim/sources.py @@ -1,5 +1,5 @@ """ -Provides data handlers that can push messages to a qsim.simulator.DataFeed +Provides data handlers that can push messages to a qsim.core.DataFeed """ import datetime import zmq @@ -31,7 +31,7 @@ class DataSource(object): def open(self): """create zmq context and socket""" - qutil.logger.info( + qutil.LOGGER.info( "starting data source:{source_id} on {addr}" .format(source_id=self.source_id, addr=self.feed.data_address)) @@ -75,7 +75,7 @@ class DataSource(object): self.data_socket.send(json.dumps(done_msg)) self.data_socket.close() self.context.term() - qutil.logger.info("finished processing data source") + qutil.LOGGER.info("finished processing data source") class RandomEquityTrades(DataSource): """Generates a random stream of trades for testing.""" diff --git a/qsim/test/client.py b/qsim/test/client.py index 0efb074b..ec5ca471 100644 --- a/qsim/test/client.py +++ b/qsim/test/client.py @@ -18,27 +18,27 @@ class TestClient(object): def run(self): - qutil.logger.info("running the client") + qutil.LOGGER.info("running the client") self.context = zmq.Context() self.data_feed = self.context.socket(zmq.PULL) if(self.bind): - qutil.logger.info("binding to {address}".format(address=self.address)) + qutil.LOGGER.info("binding to {address}".format(address=self.address)) self.data_feed.bind(self.address) else: - qutil.logger.info("connecting to {address}".format(address=self.address)) + qutil.LOGGER.info("connecting to {address}".format(address=self.address)) self.data_feed.connect(self.address) self.sync.confirm() - qutil.logger.info("Starting the client loop") + qutil.LOGGER.info("Starting the client loop") prev_dt = None while True: msg = self.data_feed.recv() if(msg == "DONE"): - qutil.logger.info("DONE!") + qutil.LOGGER.info("DONE!") break self.received_count += 1 event = json.loads(msg) @@ -48,8 +48,8 @@ class TestClient(object): prev_dt = event['dt'] if(self.received_count % 100 == 0): - qutil.logger.info("received {n} messages".format(n=self.received_count)) + qutil.LOGGER.info("received {n} messages".format(n=self.received_count)) - qutil.logger.info("received {n} messages".format(n=self.received_count)) + qutil.LOGGER.info("received {n} messages".format(n=self.received_count)) self.data_feed.close() self.context.term() \ No newline at end of file diff --git a/qsim/test/test_messaging.py b/qsim/test/test_messaging.py index 7e2e794a..b641e743 100644 --- a/qsim/test/test_messaging.py +++ b/qsim/test/test_messaging.py @@ -6,7 +6,7 @@ Test suite for the messaging infrastructure of QSim. import unittest2 as unittest import multiprocessing -from qsim.simulator.feed import DataFeed +from qsim.core import DataFeed from qsim.transforms.merge import MergedTransformsFeed from qsim.transforms.technical import MovingAverage import qsim.util as qutil @@ -21,7 +21,7 @@ class MessagingTestCase(unittest.TestCase): """generate some config objects for the datafeed, sources, and transforms.""" qutil.configure_logging() - qutil.logger.info("testing...") + qutil.LOGGER.info("testing...") self.total_data_count = 800 self.feed_config = {'emt1':{'sid':133, 'class':'RandomEquityTrades', 'count':400}, 'emt2':{'sid':134, 'class':'RandomEquityTrades', 'count':400}} diff --git a/qsim/transforms/core.py b/qsim/transforms/core.py index bd1ed398..7e2f5dbb 100644 --- a/qsim/transforms/core.py +++ b/qsim/transforms/core.py @@ -3,7 +3,7 @@ Transforms ========== Transforms provide re-useable components for stream processing. All -Transforms expect to receive data events from qsim.simulator.feed.DataFeed +Transforms expect to receive data events from qsim.core.DataFeed asynchronously via zeromq. Each transform is designed to run in independent process space, independently of all other transforms, to allow for parallel computation. @@ -21,7 +21,7 @@ import zmq import json import qsim.messaging as qmsg import qsim.util as qutil -import qsim.simulator.config as config +import qsim.config as config class BaseTransform(object): @@ -66,7 +66,7 @@ class BaseTransform(object): """ self.context = zmq.Context() - qutil.logger.info("starting {name} transform". + qutil.LOGGER.info("starting {name} transform". format(name = self.state['name'])) #create the feed SUB. self.feed_socket = self.context.socket(zmq.SUB) @@ -84,13 +84,13 @@ class BaseTransform(object): - call transform (subclass' method) on event - send the transformed event """ - qutil.logger.info("starting {name} event loop".format(name = self.state['name'])) + qutil.LOGGER.info("starting {name} event loop".format(name = self.state['name'])) self.sync.confirm() while True: message = self.feed_socket.recv() if(message == "DONE"): - qutil.logger.info("{name} received the Done message from the feed".format(name=self.state['name'])) + qutil.LOGGER.info("{name} received the Done message from the feed".format(name=self.state['name'])) self.result_socket.send("DONE") break; self.received_count += 1 @@ -105,7 +105,7 @@ class BaseTransform(object): """ Shut down zmq resources. """ - qutil.logger.info("Transform {name} recieved {r} and sent {s}".format(name=self.state['name'], r=self.received_count, s=self.sent_count)) + qutil.LOGGER.info("Transform {name} recieved {r} and sent {s}".format(name=self.state['name'], r=self.received_count, s=self.sent_count)) self.feed_socket.close() self.result_socket.close() diff --git a/qsim/transforms/merge.py b/qsim/transforms/merge.py index 6406a9cb..e60d1e95 100644 --- a/qsim/transforms/merge.py +++ b/qsim/transforms/merge.py @@ -53,7 +53,7 @@ class MergedTransformsFeed(BaseTransform): ready with the DataFeed at the conclusion.""" self.context = zmq.Context() - qutil.logger.info("starting {name} transform".format(name = self.state['name'])) + qutil.LOGGER.info("starting {name} transform".format(name = self.state['name'])) #create the feed SUB. self.feed_socket = self.context.socket(zmq.SUB) self.feed_socket.connect(self.feed.feed_address) @@ -74,7 +74,7 @@ class MergedTransformsFeed(BaseTransform): self.poller.register(self.transform_socket, zmq.POLLIN) for name, transform in self.transforms.iteritems(): - qutil.logger.info("starting {name}".format(name=name)) + qutil.LOGGER.info("starting {name}".format(name=name)) proc = multiprocessing.Process(target=transform.run) proc.start() @@ -101,7 +101,7 @@ class MergedTransformsFeed(BaseTransform): if self.feed_socket in socks and socks[self.feed_socket] == zmq.POLLIN: message = self.feed_socket.recv() if(message == "DONE"): - qutil.logger.info("finished receiving feed to merge") + qutil.LOGGER.info("finished receiving feed to merge") done_count += 1 else: self.received_count += 1 @@ -111,7 +111,7 @@ class MergedTransformsFeed(BaseTransform): if self.transform_socket in socks and socks[self.transform_socket] == zmq.POLLIN: t_message = self.transform_socket.recv() if(t_message == "DONE"): - qutil.logger.info("finished receiving a transform to merge") + qutil.LOGGER.info("finished receiving a transform to merge") done_count += 1 else: self.received_count += 1 @@ -123,13 +123,13 @@ class MergedTransformsFeed(BaseTransform): self.data_buffer.send_next() - qutil.logger.info("Transform {name} received {r} and sent {s}".format(name=self.state['name'], r=self.data_buffer.received_count, s=self.data_buffer.sent_count)) - qutil.logger.info("about to drain {n} messages from merger's buffer".format(n=self.data_buffer.pending_messages())) + qutil.LOGGER.info("Transform {name} received {r} and sent {s}".format(name=self.state['name'], r=self.data_buffer.received_count, s=self.data_buffer.sent_count)) + qutil.LOGGER.info("about to drain {n} messages from merger's buffer".format(n=self.data_buffer.pending_messages())) #drain any remaining messages in the buffer self.data_buffer.drain() #signal to client that we're done self.result_socket.send("DONE") - qutil.logger.info("Transform {name} received {r} and sent {s}".format(name=self.state['name'], r=self.data_buffer.received_count, s=self.data_buffer.sent_count)) + qutil.LOGGER.info("Transform {name} received {r} and sent {s}".format(name=self.state['name'], r=self.data_buffer.received_count, s=self.data_buffer.sent_count)) diff --git a/qsim/util.py b/qsim/util.py index 53d54572..9b94203f 100644 --- a/qsim/util.py +++ b/qsim/util.py @@ -1,5 +1,5 @@ """ -Small classes to assist with timezone calculations, logger configuration, +Small classes to assist with timezone calculations, LOGGER configuration, and other common operations. """ @@ -8,26 +8,31 @@ import pytz import logging -logger = logging.getLogger('QSimLogger') - +LOGGER = logging.getLogger('QSimLogger') + def parse_date(dt_str): """parse strings according to the same format as generated by format_date""" if(dt_str == None): return None parts = dt_str.split(".") - dt = datetime.datetime.strptime(parts[0], '%Y/%m/%d-%H:%M:%S').replace(microsecond=int(parts[1]+"000")).replace(tzinfo = pytz.utc) + dt = datetime.datetime.strptime(parts[0], '%Y/%m/%d-%H:%M:%S').replace(microsecond=int(parts[1]+"000"), tzinfo = pytz.utc) return dt - + def format_date(dt): - """Format the date into a date with millesecond resolution and string/alphabetical sorting that is equivalent to datetime sorting""" + """Format the date into a date with millesecond resolution and string/alphabetical + sorting that is equivalent to datetime sorting""" if(dt == None): return None dt_str = dt.strftime('%Y/%m/%d-%H:%M:%S') + "." + str(dt.microsecond / 1000) return dt_str - -def configure_logging(loglevel=logging.DEBUG): - logger.setLevel(loglevel) - handler = logging.handlers.RotatingFileHandler("/tmp/{lfn}.log".format(lfn="qsim-log"), maxBytes=10*1024*1024, backupCount=5) - handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(filename)s %(funcName)s - %(message)s","%Y-%m-%d %H:%M:%S %Z")) - logger.addHandler(handler) - logger.info("logging started...") \ No newline at end of file + +def configure_logging(loglevel=logging.DEBUG): + """configures qsim.util.LOGGER to write a rotating file (10M per file, 5 files) to /var/log/qsim.log""" + LOGGER.setLevel(loglevel) + handler = logging.handlers.RotatingFileHandler( + "/var/log/qsim/{lfn}.log".format(lfn="qsim"), + maxBytes=10*1024*1024, backupCount=5) + handler.setFormatter(logging.Formatter( + "%(asctime)s %(levelname)s %(filename)s %(funcName)s - %(message)s","%Y-%m-%d %H:%M:%S %Z")) + LOGGER.addHandler(handler) + LOGGER.info("logging started...") \ No newline at end of file