wassname/catalyst
1
0
Fork 0
mirror of https://github.com/wassname/catalyst.git synced 2026-09-11 12:00:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

exceptions tests passing

Browse Source
This commit is contained in:
fawce
2012-08-07 13:16:42 -04:00
parent 5aba8bfc28
commit 804bcb4e0c
3 changed files with 72 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_exception_handling.py
+51 -65
View File
@@ -52,18 +52,14 @@ class ExceptionTestCase(TestCase):
**self.zipline_test_config **self.zipline_test_config
) )
output, _ = drain_zipline(self, zipline) output, _ = drain_zipline(self, zipline)
self.assertEqual(len(output), 1) self.assertEqual(len(output), 2)
self.assertEqual(output[-1]['prefix'], 'EXCEPTION') self.assertEqual(output[-1]['prefix'], 'EXCEPTION')
payload = output[-1]['payload'] payload = output[-1]['payload']
self.assertTrue(payload['date']) self.assertTrue(payload['date'])
del payload['date'] del payload['date']
check(self, payload, INITIALIZE_TB) check(self, payload, INITIALIZE_TB)
self.assertTrue(zipline.sim.ready())
self.assertFalse(zipline.sim.exception)
def test_exception_in_handle_data(self): def test_exception_in_handle_data(self):
# Simulation # Simulation
# ---------- # ----------
self.zipline_test_config['algorithm'] = \ self.zipline_test_config['algorithm'] = \
@@ -77,15 +73,12 @@ class ExceptionTestCase(TestCase):
) )
output, _ = drain_zipline(self, zipline) output, _ = drain_zipline(self, zipline)
self.assertEqual(len(output), 3)
self.assertEqual(len(output), 1)
self.assertEqual(output[-1]['prefix'], 'EXCEPTION') self.assertEqual(output[-1]['prefix'], 'EXCEPTION')
payload = output[-1]['payload'] payload = output[-1]['payload']
self.assertTrue(payload['date']) self.assertTrue(payload['date'])
del payload['date'] del payload['date']
check(self, payload, HANDLE_DATA_TB) check(self, payload, HANDLE_DATA_TB)
self.assertTrue(zipline.sim.ready())
self.assertFalse(zipline.sim.exception)
def test_zerodivision_exception_in_handle_data(self): def test_zerodivision_exception_in_handle_data(self):
@@ -101,14 +94,12 @@ class ExceptionTestCase(TestCase):
) )
output, _ = drain_zipline(self, zipline) output, _ = drain_zipline(self, zipline)
self.assertEqual(len(output), 5) self.assertEqual(len(output), 6)
self.assertEqual(output[-1]['prefix'], 'EXCEPTION') self.assertEqual(output[-1]['prefix'], 'EXCEPTION')
payload = output[-1]['payload'] payload = output[-1]['payload']
self.assertTrue(payload['date']) self.assertTrue(payload['date'])
del payload['date'] del payload['date']
check(self, payload, ZERO_DIV_TB) check(self, payload, ZERO_DIV_TB)
self.assertTrue(zipline.sim.ready())
self.assertFalse(zipline.sim.exception)
# TODO: # TODO:
# - define more zipline failure modes: exception in other # - define more zipline failure modes: exception in other
@@ -120,21 +111,12 @@ class ExceptionTestCase(TestCase):
INITIALIZE_TB =\ INITIALIZE_TB =\
{'message': 'Algo exception in initialize', {'message': 'Algo exception in initialize',
'name': 'Exception', 'name': 'Exception',
'stack': [{'filename': '/zipline/core/component.py', 'line': 'self._run()', 'lineno': 210, 'method': 'run'}, 'stack': [{'filename': '/zipline/lines.py', 'line': 'for event in self.gen:', 'lineno': 152, 'method': 'stream_results'},
{'filename': '/zipline/core/component.py', 'line': 'self.loop()', 'lineno': 201, 'method': '_run'}, {'filename': '/zipline/gens/tradesimulation.py', 'line': 'self.algo,', 'lineno': 93, 'method': 'simulate'},
{'filename': '/zipline/core/component.py', 'line': 'self.do_work()', 'lineno': 241, 'method': 'loop'}, {'filename': '/zipline/gens/tradesimulation.py',
{'filename': '/zipline/components/tradesimulation.py', 'line': 'self.algo.initialize()',
'line': 'self.initialize_algo()', 'lineno': 123,
'lineno': 91, 'method': '__init__'},
'method': 'do_work'},
{'filename': '/zipline/components/tradesimulation.py',
'line': 'self.do_op(self.algorithm.initialize)',
'lineno': 74,
'method': 'initialize_algo'},
{'filename': '/zipline/components/tradesimulation.py',
'line': 'callable_op(*args, **kwargs)',
'lineno': 194,
'method': 'do_op'},
{'filename': '/zipline/test_algorithms.py', {'filename': '/zipline/test_algorithms.py',
'line': 'raise Exception("Algo exception in initialize")', 'line': 'raise Exception("Algo exception in initialize")',
'lineno': 166, 'lineno': 166,
@@ -143,25 +125,27 @@ INITIALIZE_TB =\
HANDLE_DATA_TB =\ HANDLE_DATA_TB =\
{'message': 'Algo exception in handle_data', {'message': 'Algo exception in handle_data',
'name': 'Exception', 'name': 'Exception',
'stack': [{'filename': '/zipline/core/component.py', 'line': 'self._run()', 'lineno': 210, 'method': 'run'}, 'stack': [{'filename': '/zipline/lines.py', 'line': 'for event in self.gen:', 'lineno': 152, 'method': 'stream_results'},
{'filename': '/zipline/core/component.py', 'line': 'self.loop()', 'lineno': 201, 'method': '_run'}, {'filename': '/zipline/gens/tradesimulation.py',
{'filename': '/zipline/core/component.py', 'line': 'self.do_work()', 'lineno': 241, 'method': 'loop'}, 'line': 'for message in algo_results:',
{'filename': '/zipline/components/tradesimulation.py', 'lineno': 100,
'line': 'self.process_event(event)', 'method': 'simulate'},
'lineno': 110, {'filename': '/zipline/gens/tradesimulation.py',
'method': 'do_work'}, 'line': 'return self.__generator.next()',
{'filename': '/zipline/components/tradesimulation.py', 'lineno': 144,
'line': 'self.run_algorithm()', 'method': 'next'},
'lineno': 158, {'filename': '/zipline/gens/tradesimulation.py',
'method': 'process_event'}, 'line': 'self.update_current_snapshot(event)',
{'filename': '/zipline/components/tradesimulation.py', 'lineno': 199,
'line': 'self.do_op(self.algorithm.handle_data, data)', 'method': '_gen'},
'lineno': 180, {'filename': '/zipline/gens/tradesimulation.py',
'method': 'run_algorithm'}, 'line': 'self.simulate_current_snapshot()',
{'filename': '/zipline/components/tradesimulation.py', 'lineno': 221,
'line': 'callable_op(*args, **kwargs)', 'method': 'update_current_snapshot'},
'lineno': 194, {'filename': '/zipline/gens/tradesimulation.py',
'method': 'do_op'}, 'line': 'self.algo.handle_data(self.universe)',
'lineno': 246,
'method': 'simulate_current_snapshot'},
{'filename': '/zipline/test_algorithms.py', {'filename': '/zipline/test_algorithms.py',
'line': 'raise Exception("Algo exception in handle_data")', 'line': 'raise Exception("Algo exception in handle_data")',
'lineno': 187, 'lineno': 187,
@@ -170,23 +154,25 @@ HANDLE_DATA_TB =\
ZERO_DIV_TB= \ ZERO_DIV_TB= \
{'message': 'integer division or modulo by zero', {'message': 'integer division or modulo by zero',
'name': 'ZeroDivisionError', 'name': 'ZeroDivisionError',
'stack': [{'filename': '/zipline/core/component.py', 'line': 'self._run()', 'lineno': 210, 'method': 'run'}, 'stack': [{'filename': '/zipline/lines.py', 'line': 'for event in self.gen:', 'lineno': 152, 'method': 'stream_results'},
{'filename': '/zipline/core/component.py', 'line': 'self.loop()', 'lineno': 201, 'method': '_run'}, {'filename': '/zipline/gens/tradesimulation.py',
{'filename': '/zipline/core/component.py', 'line': 'self.do_work()', 'lineno': 241, 'method': 'loop'}, 'line': 'for message in algo_results:',
{'filename': '/zipline/components/tradesimulation.py', 'lineno': 100,
'line': 'self.process_event(event)', 'method': 'simulate'},
'lineno': 110, {'filename': '/zipline/gens/tradesimulation.py',
'method': 'do_work'}, 'line': 'return self.__generator.next()',
{'filename': '/zipline/components/tradesimulation.py', 'lineno': 144,
'line': 'self.run_algorithm()', 'method': 'next'},
'lineno': 158, {'filename': '/zipline/gens/tradesimulation.py',
'method': 'process_event'}, 'line': 'self.update_current_snapshot(event)',
{'filename': '/zipline/components/tradesimulation.py', 'lineno': 199,
'line': 'self.do_op(self.algorithm.handle_data, data)', 'method': '_gen'},
'lineno': 180, {'filename': '/zipline/gens/tradesimulation.py',
'method': 'run_algorithm'}, 'line': 'self.simulate_current_snapshot()',
{'filename': '/zipline/components/tradesimulation.py', 'lineno': 221,
'line': 'callable_op(*args, **kwargs)', 'method': 'update_current_snapshot'},
'lineno': 194, {'filename': '/zipline/gens/tradesimulation.py',
'method': 'do_op'}, 'line': 'self.algo.handle_data(self.universe)',
'lineno': 246,
'method': 'simulate_current_snapshot'},
{'filename': '/zipline/test_algorithms.py', 'line': '5/0', 'lineno': 218, 'method': 'handle_data'}]} {'filename': '/zipline/test_algorithms.py', 'line': '5/0', 'lineno': 218, 'method': 'handle_data'}]}
zipline/lines.py
+17 -19
View File
@@ -99,10 +99,10 @@ class SimulatedTrading(object):
self.date_sorted = date_sorted_sources(*sources) self.date_sorted = date_sorted_sources(*sources)
self.transforms = transforms self.transforms = transforms
self.transforms.extend(StatefulTransform(Passthrough)) self.transforms.append(StatefulTransform(Passthrough))
self.merged = merged_transforms(self.date_sorted, *self.transforms) self.merged = merged_transforms(self.date_sorted, *self.transforms)
self.trading_client = tsc(algorithm, environment, style) self.trading_client = tsc(algorithm, environment, style)
self.gen = self.trading_client.simluate(self.merged) self.gen = self.trading_client.simulate(self.merged)
self.results_uri = results_socket_uri self.results_uri = results_socket_uri
self.results_socket = None self.results_socket = None
self.context = context self.context = context
@@ -111,6 +111,7 @@ class SimulatedTrading(object):
# optional process if we fork simulate into an # optional process if we fork simulate into an
# independent process. # independent process.
self.proc = None self.proc = None
self.logger = Logger(sim_id)
def simulate(self, blocking=True): def simulate(self, blocking=True):
@@ -122,7 +123,7 @@ class SimulatedTrading(object):
return self.fork_and_sim() return self.fork_and_sim()
def fork_and_sim(self): def fork_and_sim(self):
self.proc = multiprocessing.Process(self.run_gen) self.proc = multiprocessing.Process(target=self.run_gen)
self.proc.start() self.proc.start()
return self.proc return self.proc
@@ -133,15 +134,16 @@ class SimulatedTrading(object):
def inject_event_data(record): def inject_event_data(record):
# Record the simulation time. # Record the simulation time.
record.extra['algo_dt'] = self.current_dt #record.extra['algo_dt'] = self.current_dt
pass
data_injector = Processor(inject_event_data) data_injector = Processor(inject_event_data)
log_pipeline = NestedSetup([self.zmq_out,data_injector]) log_pipeline = NestedSetup([self.zmq_out,data_injector])
with log_pipeline.threadbound(), self.stdout_capture(self.logger, ''): with log_pipeline.threadbound(), self.stdout_capture(self.logger, ''):
self.drain_gen() self.stream_results()
# if no log socket, just run the algo normally # if no log socket, just run the algo normally
else: else:
self.drain_gen() self.stream_results()
def stream_results(self): def stream_results(self):
assert self.results_socket, \ assert self.results_socket, \
@@ -153,7 +155,8 @@ class SimulatedTrading(object):
else: else:
msg = zp.RISK_FRAME(event) msg = zp.RISK_FRAME(event)
self.results_socket.send(msg) self.results_socket.send(msg)
self.signal_done()
self.signal_done()
except Exception as exc: except Exception as exc:
self.handle_exception(exc) self.handle_exception(exc)
finally: finally:
@@ -186,10 +189,8 @@ class SimulatedTrading(object):
""" """
exc_type, exc_value, exc_traceback = sys.exc_info() exc_type, exc_value, exc_traceback = sys.exc_info()
log.exception("Unexpected error in run for {id}.".format(id=self.sim_id))
try: try:
log.info('{id} sending exception to monitor'\ log.exception('{id} sending exception to result stream.'\
.format(id=self.sim_id)) .format(id=self.sim_id))
msg = zp.EXCEPTION_FRAME( msg = zp.EXCEPTION_FRAME(
exc_traceback, exc_traceback,
@@ -197,11 +198,7 @@ class SimulatedTrading(object):
exc_value.message exc_value.message
) )
exception_frame = zp.CONTROL_FRAME( self.results_socket.send(msg)
zp.CONTROL_PROTOCOL.EXCEPTION,
msg
)
self.results_socket.send(exception_frame)
except: except:
log.exception("Exception while reporting simulation exception.") log.exception("Exception while reporting simulation exception.")
@@ -214,9 +211,6 @@ class SimulatedTrading(object):
sock = self.context.socket(zmq.PUSH) sock = self.context.socket(zmq.PUSH)
sock.connect(self.results_uri) sock.connect(self.results_uri)
self.results_socket = sock self.results_socket = sock
self.sockets.append(sock)
self.results_socket = sock
self.setup_logging() self.setup_logging()
def setup_logging(self, socket = None): def setup_logging(self, socket = None):
@@ -231,6 +225,10 @@ class SimulatedTrading(object):
# in run_algorithm. The class provides a generator. # in run_algorithm. The class provides a generator.
self.stdout_capture = stdout_only_pipe self.stdout_capture = stdout_only_pipe
def join(self):
if self.proc:
self.proc.join()
@staticmethod @staticmethod
def create_test_zipline(**config): def create_test_zipline(**config):
""" """
@@ -333,8 +331,8 @@ class SimulatedTrading(object):
test_algo, test_algo,
trading_environment, trading_environment,
simulation_style, simulation_style,
zmq_context,
results_socket_uri, results_socket_uri,
zmq_context,
simulation_id) simulation_id)
#------------------- #-------------------
zipline/utils/test_utils.py
+4 -5
View File
@@ -65,10 +65,10 @@ def drain_zipline(test, zipline):
assert test.ctx, "method expects a valid zmq context" assert test.ctx, "method expects a valid zmq context"
assert test.zipline_test_config, "method expects a valid test config" assert test.zipline_test_config, "method expects a valid test config"
assert isinstance(test.zipline_test_config, dict) assert isinstance(test.zipline_test_config, dict)
assert test.zipline_test_config['results_socket'], \ assert test.zipline_test_config['results_socket_uri'], \
"need to specify a socket address for logs/perf/risk" "need to specify a socket address for logs/perf/risk"
test.receiver = create_receiver( test.receiver = create_receiver(
test.zipline_test_config['results_socket'], test.zipline_test_config['results_socket_uri'],
test.ctx test.ctx
) )
# Bind and connect are asynch, so allow time for bind before # Bind and connect are asynch, so allow time for bind before
@@ -76,13 +76,12 @@ def drain_zipline(test, zipline):
time.sleep(1) time.sleep(1)
# start the simulation # start the simulation
zipline.simulate(blocking=False) zipline.simulate(blocking=True)
output, transaction_count = drain_receiver(test.receiver) output, transaction_count = drain_receiver(test.receiver)
# some processes will exit after the message stream is # some processes will exit after the message stream is
# finished. We block here to avoid collisions with subsequent # finished. We block here to avoid collisions with subsequent
# ziplines. # ziplines.
for process in zipline.sim.subprocesses: zipline.join()
process.join()
return output, transaction_count return output, transaction_count