From 1c91c7287bf66e276a97c1bcc2419dd5b0f5e00b Mon Sep 17 00:00:00 2001 From: Thomas Wiecki Date: Mon, 28 May 2012 12:02:11 -0400 Subject: [PATCH 01/10] Moved cycle import out of function. --- tests/test_optimize.py | 5 ++--- zipline/optimize/factory.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_optimize.py b/tests/test_optimize.py index b3ff7b06..391cec51 100644 --- a/tests/test_optimize.py +++ b/tests/test_optimize.py @@ -110,7 +110,7 @@ class TestUpDown(TestCase): def test_concavity_of_returns(self): - """verify concave relationship between of free parameter and + """verify concave relationship between free parameter and returns in certain region around the max. Moreover, establishes that the max returns is at the correct value (i.e. 0). @@ -169,7 +169,7 @@ class TestUpDown(TestCase): idx[0] -= 1 idx[1] += 1 - @skip + #@skip def test_optimize(self): """verify that gradient descent (Powell's method) can find the optimal free parameter under which the BuySellAlgorithm produces @@ -200,7 +200,6 @@ class TestUpDown(TestCase): self.zipline_test_config['environment'] = trading_environment zipline = SimulatedTrading.create_test_zipline(**self.zipline_test_config) zipline.simulate(blocking=True) - zipline.shutdown() #function is getting minimized, so have to return negative cum returns. return -zipline.get_cumulative_performance()['returns'] diff --git a/zipline/optimize/factory.py b/zipline/optimize/factory.py index dc8295ce..fbb32f40 100644 --- a/zipline/optimize/factory.py +++ b/zipline/optimize/factory.py @@ -12,9 +12,9 @@ from zipline.finance.sources import SpecificEquityTrades from zipline.optimize.algorithms import BuySellAlgorithm from zipline.lines import SimulatedTrading from copy import deepcopy +from itertools import cycle def create_updown_trade_source(sid, trade_count, trading_environment, start_price, amplitude): - from itertools import cycle volume = 1000 events = [] price = start_price-amplitude/2. From 23fb5fa0837bbed249506e15aa41d02a8b921df6 Mon Sep 17 00:00:00 2001 From: fawce Date: Mon, 28 May 2012 15:25:44 -0400 Subject: [PATCH 02/10] cleanup --- zipline/finance/risk.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/zipline/finance/risk.py b/zipline/finance/risk.py index f54fcab9..af94b284 100644 --- a/zipline/finance/risk.py +++ b/zipline/finance/risk.py @@ -308,7 +308,10 @@ class RiskMetrics(): message = "no rate for end date = {dt} and term = {term}. Check \ that date doesn't exceed treasury history range." - message = message.format(dt=self.end_date,term=self.treasury_duration) + message = message.format( + dt=self.end_date, + term=self.treasury_duration + ) raise Exception(message) From 13ccf169f13d3c0844ddff043f3ae2c1ef3d3c30 Mon Sep 17 00:00:00 2001 From: fawce Date: Mon, 28 May 2012 16:06:41 -0400 Subject: [PATCH 03/10] encoding epoch as an int, rather than float. --- zipline/utils/date_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zipline/utils/date_utils.py b/zipline/utils/date_utils.py index 3ac53bdd..801315b8 100644 --- a/zipline/utils/date_utils.py +++ b/zipline/utils/date_utils.py @@ -44,10 +44,10 @@ def EPOCH(utc_datetime): delta = utc_datetime - UNIX_EPOCH seconds = delta.total_seconds() ms = seconds * 1000 - return ms + return int(ms) def UN_EPOCH(ms_since_epoch): - seconds_since_epoch = ms_since_epoch / 1000 + seconds_since_epoch = float(ms_since_epoch) / 1000.0 delta = timedelta(seconds = seconds_since_epoch) dt = UNIX_EPOCH + delta return dt From 069dbd1afe0080e87ddcfc03085c4e05afc9a3ec Mon Sep 17 00:00:00 2001 From: fawce Date: Mon, 28 May 2012 21:32:23 -0400 Subject: [PATCH 04/10] removed protocol cruft from transaction transform days. --- tests/test_perf_tracking.py | 2 +- tests/test_protocol.py | 21 ++++-------- zipline/finance/trading.py | 11 +++---- zipline/protocol.py | 64 ------------------------------------- zipline/utils/date_utils.py | 3 +- 5 files changed, 13 insertions(+), 88 deletions(-) diff --git a/tests/test_perf_tracking.py b/tests/test_perf_tracking.py index 06581434..d979473d 100644 --- a/tests/test_perf_tracking.py +++ b/tests/test_perf_tracking.py @@ -552,7 +552,7 @@ shares in position" }) else: txn = None - event[zp.TRANSFORM_TYPE.TRANSACTION] = txn + event['TRANSACTION'] = txn perf_tracker.process_event(event) #we skip two trades, to test case of None transaction diff --git a/tests/test_protocol.py b/tests/test_protocol.py index d26c2feb..a61ab246 100644 --- a/tests/test_protocol.py +++ b/tests/test_protocol.py @@ -1,5 +1,5 @@ """ -Test the FRAME/UNFRAME functions in the sequence expected from ziplines. +Test the FRAME/UNFRAME functions in the sequence expected from ziplines. """ import pytz @@ -36,10 +36,10 @@ class ProtocolTestCase(TestCase): one_day_td = timedelta(days=1) trades = factory.create_trade_history( - sid, - price, - volume, - one_day_td, + sid, + price, + volume, + one_day_td, self.trading_environment ) @@ -92,7 +92,7 @@ class ProtocolTestCase(TestCase): self.assertEqual(order.sid, 133) self.assertEqual(order.amount, 100) self.assertEqual(order.dt, now) - + #order datasource datasource frames the order order_event = zp.ndict({ "sid" : order.sid, @@ -118,12 +118,3 @@ class ProtocolTestCase(TestCase): 'price' : 10.0, 'commission' : 0.50 }) - - #frame that transaction - txn_msg = zp.TRANSFORM_FRAME(zp.TRANSFORM_TYPE.TRANSACTION, txn) - - #unframe - recovered_tx = zp.TRANSFORM_UNFRAME(txn_msg).TRANSACTION - self.assertEqual(recovered_tx.sid, 133) - self.assertEqual(recovered_tx.amount, 100) - diff --git a/zipline/finance/trading.py b/zipline/finance/trading.py index 6b5ac1f4..2d378574 100644 --- a/zipline/finance/trading.py +++ b/zipline/finance/trading.py @@ -168,12 +168,11 @@ for orders: def create_transaction(self, sid, amount, price, dt, direction): self.txn_count += 1 - txn = {'sid' : sid, - 'amount' : int(amount), - 'dt' : dt, - 'price' : price, - 'commission' : self.commission * amount * direction, - 'source_id' : zp.FINANCE_COMPONENT.TRANSACTION_SIM + txn = {'sid' : sid, + 'amount' : int(amount), + 'dt' : dt, + 'price' : price, + 'commission' : self.commission * amount * direction } return zp.ndict(txn) diff --git a/zipline/protocol.py b/zipline/protocol.py index 8db194cb..7828cc06 100644 --- a/zipline/protocol.py +++ b/zipline/protocol.py @@ -342,8 +342,6 @@ def TRANSFORM_FRAME(name, value): assert isinstance(name, basestring) if value == None: return msgpack.dumps(tuple([name, TRANSFORM_TYPE.EMPTY])) - if(name == TRANSFORM_TYPE.TRANSACTION): - value = TRANSACTION_FRAME(value) return msgpack.dumps(tuple([name, value])) def TRANSFORM_UNFRAME(msg): @@ -359,8 +357,6 @@ def TRANSFORM_UNFRAME(msg): assert isinstance(name, basestring) if(name == TRANSFORM_TYPE.PASSTHROUGH): value = FEED_UNFRAME(value) - elif(name == TRANSFORM_TYPE.TRANSACTION): - value = TRANSACTION_UNFRAME(value) return ndict({name : value}) except TypeError: @@ -382,11 +378,6 @@ def MERGE_FRAME(event): """ assert isinstance(event, ndict) PACK_DATE(event) - if(event.has_attr(TRANSFORM_TYPE.TRANSACTION)): - if(event.TRANSACTION == None): - event.TRANSACTION = TRANSFORM_TYPE.EMPTY - else: - event.TRANSACTION = TRANSACTION_FRAME(event.TRANSACTION) payload = event.as_dict() return msgpack.dumps(payload) @@ -396,11 +387,6 @@ def MERGE_UNFRAME(msg): #TODO: anything we can do to assert more about the content of the dict? assert isinstance(payload, dict) payload = ndict(payload) - if(payload.has_attr(TRANSFORM_TYPE.TRANSACTION)): - if(payload.TRANSACTION == TRANSFORM_TYPE.EMPTY): - payload.TRANSACTION = None - else: - payload.TRANSACTION = TRANSACTION_UNFRAME(payload.TRANSACTION) UNPACK_DATE(payload) return payload except TypeError: @@ -501,54 +487,6 @@ def ORDER_UNFRAME(msg): except ValueError: raise INVALID_ORDER_FRAME(msg) - -# ----------------------- -# TRANSACTIONS -# ----------------------- -# -# - Should only be called from inside TRANSFORM_(UN)FRAME. - - - -def TRANSACTION_FRAME(event): - assert isinstance(event, ndict) - assert isinstance(event.sid, int) - assert isinstance(event.price, numbers.Real) - assert isinstance(event.commission, numbers.Real) - assert isinstance(event.amount, int) - PACK_DATE(event) - return msgpack.dumps(tuple([ - event.sid, - event.price, - event.amount, - event.commission, - event.dt - ])) - -def TRANSACTION_UNFRAME(msg): - try: - sid, price, amount, commission, dt = msgpack.loads(msg) - - assert isinstance(sid, int) - assert isinstance(price, numbers.Real) - assert isinstance(commission, numbers.Real) - assert isinstance(amount, int) - rval = ndict({ - 'sid' : sid, - 'price' : price, - 'amount' : amount, - 'commission' : commission, - 'dt' : dt - }) - - UNPACK_DATE(rval) - return rval - except TypeError: - raise INVALID_TRADE_FRAME(msg) - except ValueError: - raise INVALID_TRADE_FRAME(msg) - - # ----------------------- # ORDERS # ----------------------- @@ -742,7 +680,6 @@ ORDER_PROTOCOL = Enum( #Transform type needs to be a ndict to facilitate merging. TRANSFORM_TYPE = ndict({ - 'TRANSACTION' : 'TRANSACTION', #needed? 'PASSTHROUGH' : 'PASSTHROUGH', 'EMPTY' : '' }) @@ -752,7 +689,6 @@ FINANCE_COMPONENT = namelookup({ 'TRADING_CLIENT' : 'TRADING_CLIENT', 'PORTFOLIO_CLIENT' : 'PORTFOLIO_CLIENT', 'ORDER_SOURCE' : 'ORDER_SOURCE', - 'TRANSACTION_SIM' : 'TRANSACTION_SIM' }) diff --git a/zipline/utils/date_utils.py b/zipline/utils/date_utils.py index 801315b8..131dd61b 100644 --- a/zipline/utils/date_utils.py +++ b/zipline/utils/date_utils.py @@ -47,8 +47,7 @@ def EPOCH(utc_datetime): return int(ms) def UN_EPOCH(ms_since_epoch): - seconds_since_epoch = float(ms_since_epoch) / 1000.0 - delta = timedelta(seconds = seconds_since_epoch) + delta = timedelta(milliseconds = ms_since_epoch) dt = UNIX_EPOCH + delta return dt From 433170df9bcb1e8f51fa26418a94e0ef53682783 Mon Sep 17 00:00:00 2001 From: fawce Date: Mon, 28 May 2012 21:39:54 -0400 Subject: [PATCH 05/10] removed old order protocol cruft. --- tests/test_finance.py | 1 - tests/test_protocol.py | 43 ------------ zipline/components/tradesimulation.py | 3 - zipline/protocol.py | 98 --------------------------- 4 files changed, 145 deletions(-) diff --git a/tests/test_finance.py b/tests/test_finance.py index 3676bea9..bbceac25 100644 --- a/tests/test_finance.py +++ b/tests/test_finance.py @@ -438,7 +438,6 @@ class FinanceTestCase(TestCase): { 'sid' : sid, 'amount' : order_amount * alternator**i, - 'type' : zp.DATASOURCE_TYPE.ORDER, 'dt' : order_date }) diff --git a/tests/test_protocol.py b/tests/test_protocol.py index a61ab246..c90b09dc 100644 --- a/tests/test_protocol.py +++ b/tests/test_protocol.py @@ -75,46 +75,3 @@ class ProtocolTestCase(TestCase): event.delete('helloworld') self.assertEqual(zp.ndict(trade), event) - - @timed(DEFAULT_TIMEOUT) - def test_order_protocol(self): - #client places an order - now = datetime.utcnow().replace(tzinfo=pytz.utc) - order = zp.ndict({ - 'dt':now, - 'sid':133, - 'amount':100 - }) - order_msg = zp.ORDER_FRAME(order) - - #order datasource receives - order = zp.ORDER_UNFRAME(order_msg) - self.assertEqual(order.sid, 133) - self.assertEqual(order.amount, 100) - self.assertEqual(order.dt, now) - - #order datasource datasource frames the order - order_event = zp.ndict({ - "sid" : order.sid, - "amount" : order.amount, - "dt" : order.dt, - "source_id" : zp.FINANCE_COMPONENT.ORDER_SOURCE, - "type" : zp.DATASOURCE_TYPE.ORDER - }) - - - order_ds_msg = zp.DATASOURCE_FRAME(order_event) - - #transaction transform unframes - recovered_order = zp.DATASOURCE_UNFRAME(order_ds_msg) - - self.assertEqual(now, recovered_order.dt) - - #create a transaction from the order - txn = zp.ndict({ - 'sid' : recovered_order.sid, - 'amount' : recovered_order.amount, - 'dt' : recovered_order.dt, - 'price' : 10.0, - 'commission' : 0.50 - }) diff --git a/zipline/components/tradesimulation.py b/zipline/components/tradesimulation.py index 06aa7272..827a8d00 100644 --- a/zipline/components/tradesimulation.py +++ b/zipline/components/tradesimulation.py @@ -150,9 +150,6 @@ class TradeSimulationClient(Component): self.perf.log_order(order) self.txn_sim.add_open_order(order) - def signal_order_done(self): - self.order_socket.send(str(zp.ORDER_PROTOCOL.DONE)) - def queue_event(self, event): if self.event_queue == None: self.event_queue = [] diff --git a/zipline/protocol.py b/zipline/protocol.py index 7828cc06..8ff24389 100644 --- a/zipline/protocol.py +++ b/zipline/protocol.py @@ -246,12 +246,6 @@ def DATASOURCE_FRAME(event): event.source_id, TRADE_FRAME(event) ])) - elif(event.type == DATASOURCE_TYPE.ORDER): - return msgpack.dumps(tuple([ - event.type, - event.source_id, - ORDER_SOURCE_FRAME(event) - ])) else: raise INVALID_DATASOURCE_FRAME(str(event)) @@ -286,8 +280,6 @@ def DATASOURCE_UNFRAME(msg): child_value = ndict({'dt':None}) elif(ds_type == DATASOURCE_TYPE.TRADE): child_value = TRADE_UNFRAME(payload) - elif(ds_type == DATASOURCE_TYPE.ORDER): - child_value = ORDER_SOURCE_UNFRAME(payload) else: raise INVALID_DATASOURCE_FRAME(msg) @@ -395,12 +387,6 @@ def MERGE_UNFRAME(msg): raise INVALID_MERGE_FRAME(msg) -# ----------------------- -# Finance Protocol -# ----------------------- -INVALID_ORDER_FRAME = FrameExceptionFactory('ORDER') -INVALID_TRADE_FRAME = FrameExceptionFactory('TRADE') - # ----------------------- # Trades # ----------------------- @@ -454,83 +440,6 @@ def TRADE_UNFRAME(msg): except ValueError: raise INVALID_TRADE_FRAME(msg) -# ----------------------- -# Orders -# ----------------------- -# - from client to order source - -def ORDER_FRAME(order): - assert isinstance(order.sid, int) - assert isinstance(order.amount, int) #no partial shares... - PACK_DATE(order) - return msgpack.dumps(tuple([ - order.sid, - order.amount, - order.dt - ])) - - -def ORDER_UNFRAME(msg): - try: - sid, amount, dt = msgpack.loads(msg) - assert isinstance(sid, int) - assert isinstance(amount, int) - rval = ndict({ - 'sid':sid, - 'amount':amount, - 'dt':dt - }) - UNPACK_DATE(rval) - return rval - except TypeError: - raise INVALID_ORDER_FRAME(msg) - except ValueError: - raise INVALID_ORDER_FRAME(msg) - -# ----------------------- -# ORDERS -# ----------------------- -# -# - from order source to feed -# - should only be called from inside DATASOURCE_(UN)FRAME - - -def ORDER_SOURCE_FRAME(event): - assert isinstance(event.sid, int) - assert isinstance(event.amount, int) #no partial shares... - assert isinstance(event.source_id, basestring) - assert event.type == DATASOURCE_TYPE.ORDER - PACK_DATE(event) - return msgpack.dumps(tuple([ - event.sid, - event.amount, - event.dt, - event.source_id, - event.type - ])) - - -def ORDER_SOURCE_UNFRAME(msg): - try: - sid, amount, dt, source_id, source_type = msgpack.loads(msg) - event = ndict({ - "sid" : sid, - "amount" : amount, - "dt" : dt, - "source_id" : source_id, - "type" : source_type - }) - assert isinstance(sid, int) - assert isinstance(amount, int) - assert isinstance(source_id, basestring) - assert isinstance(source_type, int) - UNPACK_DATE(event) - return event - except TypeError: - raise INVALID_ORDER_FRAME(msg) - except ValueError: - raise INVALID_ORDER_FRAME(msg) - # ----------------------- # Performance and Risk # ----------------------- @@ -667,16 +576,10 @@ def tuple_to_date(date_tuple): return dt DATASOURCE_TYPE = Enum( - 'ORDER', 'TRADE', 'EMPTY', ) -ORDER_PROTOCOL = Enum( - 'DONE', - 'BREAK', -) - #Transform type needs to be a ndict to facilitate merging. TRANSFORM_TYPE = ndict({ @@ -688,7 +591,6 @@ TRANSFORM_TYPE = ndict({ FINANCE_COMPONENT = namelookup({ 'TRADING_CLIENT' : 'TRADING_CLIENT', 'PORTFOLIO_CLIENT' : 'PORTFOLIO_CLIENT', - 'ORDER_SOURCE' : 'ORDER_SOURCE', }) From b8d0a8e4e08a21638e22b69f7fa57e4038d4ab6b Mon Sep 17 00:00:00 2001 From: fawce Date: Mon, 28 May 2012 22:13:43 -0400 Subject: [PATCH 06/10] cleanup --- zipline/protocol.py | 1 - 1 file changed, 1 deletion(-) diff --git a/zipline/protocol.py b/zipline/protocol.py index 8ff24389..7081291f 100644 --- a/zipline/protocol.py +++ b/zipline/protocol.py @@ -495,7 +495,6 @@ def convert_transactions(transactions): for txn in transactions: txn['date'] = EPOCH(txn['dt']) del(txn['dt']) - del(txn['source_id']) results.append(txn) return results From 736bbe51af9711d97c3f573b53070085b6d92394 Mon Sep 17 00:00:00 2001 From: Stephen Diehl Date: Tue, 29 May 2012 00:38:23 -0400 Subject: [PATCH 07/10] Misc changes. --- zipline/core/interpreter.py | 19 +++++++++++++++---- zipline/utils/date_utils.py | 5 +++++ zipline/utils/protocol_utils.py | 2 +- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/zipline/core/interpreter.py b/zipline/core/interpreter.py index f36f5661..6bcd4eed 100644 --- a/zipline/core/interpreter.py +++ b/zipline/core/interpreter.py @@ -3,6 +3,7 @@ import yaml import argparse import fileinput from cStringIO import StringIO +from zipline.utils.date_utils import EPOCH, date_to_datetime def interpret(args): print 'Reading {ifile}'.format(ifile=args.file) @@ -13,7 +14,7 @@ def interpret(args): metadata = StringIO() algorithm = StringIO() - for line in fileinput.input(sys.argv[1]): + for line in fileinput.input(args.file): if line.startswith('---'): if metastart: metastart = False @@ -45,18 +46,28 @@ def interpret(args): except StopIteration: raise RuntimeError("No metadata in file.") - start = meta['start'] - end = meta['end'] + algocode = algorithm.getvalue() + + start = meta['start_date'] + end = meta['end_date'] + + meta['start_date'] = EPOCH(date_to_datetime(start)) + meta['end_date'] = EPOCH(date_to_datetime(end)) + meta['algocode'] = algocode print end - start ns = {} - exec(algorithm.getvalue()) in ns + + # -- Sanity check -- + exec(algocode) in ns assert ns['initialize'] assert ns['get_sid_filter'] assert ns['handle_data'] + return algocode, meta + def main(): parser = argparse.ArgumentParser() parser.add_argument('file', metavar='file', help='Algorithm file.') diff --git a/zipline/utils/date_utils.py b/zipline/utils/date_utils.py index 3ac53bdd..b79c2a13 100644 --- a/zipline/utils/date_utils.py +++ b/zipline/utils/date_utils.py @@ -128,3 +128,8 @@ if __name__ == '__main__': for day in trading_days(now, now30): print day print time.time() - tic + +def date_to_datetime(t): + dt = datetime.fromordinal(t.toordinal()) + dt = dt.replace(tzinfo = pytz.utc) + return dt diff --git a/zipline/utils/protocol_utils.py b/zipline/utils/protocol_utils.py index c0b7ad16..7149a7c1 100644 --- a/zipline/utils/protocol_utils.py +++ b/zipline/utils/protocol_utils.py @@ -141,7 +141,7 @@ class ndict(MutableMapping): self.__internal.update(other_nd.__internal) def __repr__(self): - return "namedict: " + str(self.__internal) + return "ndict(%s)" % str(self.__internal) # Faster dictionary comparison? #def __eq__(self, other): From 45409132b20c067dfde8c90ff1b1fa7823f1d8b1 Mon Sep 17 00:00:00 2001 From: Thomas Wiecki Date: Tue, 29 May 2012 14:37:43 -0400 Subject: [PATCH 08/10] Refactored unittests. zipline creation is now happening in a function in factory.py instead of each unittest. --- tests/test_optimize.py | 88 ++++++++++++------------------------- zipline/optimize/factory.py | 25 +++++++---- 2 files changed, 46 insertions(+), 67 deletions(-) diff --git a/tests/test_optimize.py b/tests/test_optimize.py index 391cec51..99390df9 100644 --- a/tests/test_optimize.py +++ b/tests/test_optimize.py @@ -14,6 +14,7 @@ from zipline.utils.logger import configure_logging from zipline.core.devsimulator import AddressAllocator, Simulator from zipline.optimize.algorithms import BuySellAlgorithm +from zipline.optimize.factory import create_predictable_zipline from zipline.finance.trading import TradingEnvironment from zipline.lines import SimulatedTrading from zipline.finance.trading import SIMULATION_STYLE @@ -53,20 +54,18 @@ class TestUpDown(TestCase): base_price = 50 amplitude = 6 offset = 0 - self.zipline_test_config['order_count'] = trade_count - 1 - self.zipline_test_config['trade_count'] = trade_count - self.zipline_test_config['simulation_style'] = \ - SIMULATION_STYLE.FIXED_SLIPPAGE - trading_environment = factory.create_trading_environment() - source = create_updown_trade_source(sid, - trade_count, - trading_environment, - base_price, - amplitude + zipline, config = create_predictable_zipline( + self.zipline_test_config, + sid=sid, + amplitude=amplitude, + base_price=base_price, + offset=offset, + trade_count=5, + simulate=False ) - prices = np.array([event.price for event in source.event_list]) + prices = np.array([event.price for event in config['trade_source'].event_list]) max_price_idx = np.where(prices==prices.max())[0] min_price_idx = np.where(prices==prices.min())[0] self.assertTrue(np.all(max_price_idx % 2 == 1), @@ -82,15 +81,10 @@ class TestUpDown(TestCase): "Minimum price does not equal expected maximum price." ) - algo = BuySellAlgorithm(sid, 100, 0) - - self.zipline_test_config['trade_source'] = source - self.zipline_test_config['algorithm'] = algo - self.zipline_test_config['environment'] = trading_environment - - zipline = SimulatedTrading.create_test_zipline(**self.zipline_test_config) zipline.simulate(blocking=True) + algo = config['algorithm'] + orders = np.asarray(algo.orders) max_order_idx = np.where(orders==orders.max())[0] min_order_idx = np.where(orders==orders.min())[0] @@ -108,7 +102,6 @@ class TestUpDown(TestCase): "Algorithm did not sell when price was going to increase." ) - def test_concavity_of_returns(self): """verify concave relationship between free parameter and returns in certain region around the max. Moreover, @@ -121,10 +114,6 @@ class TestUpDown(TestCase): sid = 133 amplitude = 30 base_price = 50 - self.zipline_test_config['order_count'] = trade_count - 1 - self.zipline_test_config['trade_count'] = trade_count - self.zipline_test_config['simulation_style'] = \ - SIMULATION_STYLE.FIXED_SLIPPAGE #test whether return-function is concave wrt repeats. test_offsets = np.arange(-9, 9, 1.) @@ -133,21 +122,16 @@ class TestUpDown(TestCase): compound_returns = np.empty(len(test_offsets)) ziplines = [] - for i, test_offset in enumerate(test_offsets): - trading_environment = factory.create_trading_environment() - source = create_updown_trade_source(sid, - trade_count, - trading_environment, - base_price, - amplitude + for i, offset in enumerate(test_offsets): + zipline, config = create_predictable_zipline( + self.zipline_test_config, + sid=sid, + amplitude=amplitude, + base_price=base_price, + offset=offset, + trade_count=trade_count, + simulate=True ) - - algo = BuySellAlgorithm(sid, 100, test_offset) - self.zipline_test_config['algorithm'] = algo - self.zipline_test_config['trade_source'] = source - self.zipline_test_config['environment'] = trading_environment - zipline = SimulatedTrading.create_test_zipline(**self.zipline_test_config) - zipline.simulate(blocking=True) ziplines.append(zipline) compound_returns[i] = zipline.get_cumulative_performance()['returns'] @@ -177,29 +161,15 @@ class TestUpDown(TestCase): """ def simulate(offset): - #generate events - trade_count = 3 - sid = 133 - amplitude = 10 - base_price = 50 - self.zipline_test_config['order_count'] = trade_count - 1 - self.zipline_test_config['trade_count'] = trade_count - self.zipline_test_config['simulation_style'] = \ - SIMULATION_STYLE.FIXED_SLIPPAGE - trading_environment = factory.create_trading_environment() - source = create_updown_trade_source(sid, - trade_count, - trading_environment, - base_price, - amplitude + zipline, config = create_predictable_zipline( + self.zipline_test_config, + sid=133, + amplitude=10, + base_price=50, + offset=offset, + trade_count=5, + simulate=True ) - - algo = BuySellAlgorithm(sid, 100, offset) - self.zipline_test_config['algorithm'] = algo - self.zipline_test_config['trade_source'] = source - self.zipline_test_config['environment'] = trading_environment - zipline = SimulatedTrading.create_test_zipline(**self.zipline_test_config) - zipline.simulate(blocking=True) #function is getting minimized, so have to return negative cum returns. return -zipline.get_cumulative_performance()['returns'] diff --git a/zipline/optimize/factory.py b/zipline/optimize/factory.py index fbb32f40..a752a013 100644 --- a/zipline/optimize/factory.py +++ b/zipline/optimize/factory.py @@ -11,6 +11,8 @@ from zipline.utils.factory import get_next_trading_dt, create_trading_environmen from zipline.finance.sources import SpecificEquityTrades from zipline.optimize.algorithms import BuySellAlgorithm from zipline.lines import SimulatedTrading +from zipline.finance.trading import SIMULATION_STYLE + from copy import deepcopy from itertools import cycle @@ -47,20 +49,27 @@ def create_updown_trade_source(sid, trade_count, trading_environment, start_pric return source -def create_predictable_zipline(config, sid=133, amplitude=10, base_price=50, offset=0): - config = deepcopy(config) +def create_predictable_zipline(config, sid=133, amplitude=10, base_price=50, offset=0, trade_count=3, simulate=True): + #config = deepcopy(config) trading_environment = create_trading_environment() source = create_updown_trade_source(sid, - config['trade_count'], + trade_count, trading_environment, base_price, amplitude) - algo = BuySellAlgorithm(sid, 100, offset) - config['algorithm'] = algo + if 'algorithm' not in config: + config['algorithm'] = BuySellAlgorithm(sid, 100, offset) + + config['order_count'] = trade_count - 1 + config['trade_count'] = trade_count config['trade_source'] = source config['environment'] = trading_environment - zipline = SimulatedTrading.create_test_zipline(**config) - zipline.simulate(blocking=True) + config['simulation_style'] = SIMULATION_STYLE.FIXED_SLIPPAGE - return zipline + zipline = SimulatedTrading.create_test_zipline(**config) + + if simulate: + zipline.simulate(blocking=True) + + return zipline, config From 954ad8afb31fa57565e003ea511fe5dc92876a1d Mon Sep 17 00:00:00 2001 From: Stephen Diehl Date: Tue, 29 May 2012 17:09:08 -0400 Subject: [PATCH 09/10] Added setproctitle and C extension doc. --- docs/extensions.rst | 103 +++++++++++++++++++++++++++++++++++++++++++ etc/requirements.txt | 3 ++ 2 files changed, 106 insertions(+) create mode 100644 docs/extensions.rst diff --git a/docs/extensions.rst b/docs/extensions.rst new file mode 100644 index 00000000..2903a8f4 --- /dev/null +++ b/docs/extensions.rst @@ -0,0 +1,103 @@ +.. highlight:: cython + +Philosophy +========== + +Use judgement when to use C extensions. Debugging them potentially +has a time cost of t=infinity, they can segfault, and may not be +debugabble by anyone else. Simply put 90% of the time its not worth it +and construction of extensions be must informed by scientific profiling. +Listen to your inner Knuth. + +Writing C Extensions +==================== + +Caveats aside, C Extensions can be in two forms: + +- C +- Cython + +Cython is a superset of Python which compiles into C. The code it +produces is generally not human readable. + +Reference: http://docs.cython.org/ + +C is well, C. You manage your own memory and interface with Python.h . +If you need raw performance or need to interface with other C libraries +this is often the best approach. Of course this requires that you +be very careful to tend to memory and and Python's internal garbage +collection. + +Reference: http://docs.python.org/c-api/ + +One can write C++ extensions, but please don't. + +One could also embed Assembly in C and thus in Python, but again please +don't. + +Compilers +========= + +Compatability + + - Do not use Clang + - Do not use GCC-LLVM + +Use standard GCC >= 4.6 from gnu.org, otherwise extensions will have +undefined behavior and will not be portable. + +Also make sure to code against Python 2.7 and numpy 1.6.1 header +files. If using Cython have it auto figure out the paths to ensurable +portability. + +Building +======== + +In pavement.py :: + + example = Extension( + "zipline/example", ["zipline/example.pyx"], + ) + +If you need Numpy:: + + example = Extension( + "zipline/example", ["zipline/example.pyx"], + include_dirs=[np.get_include()], + ) + +To build in development :: + + $ paver build_ext --inplace + +Pure C +====== + +.. highlight :: c + #include "Python.h" + +Releasing the GIL +================= + +:: + from libc.stdio cimport printf + + with nogil: + # in here you allowed to do whatever you like so long as + # you do not touch Python objects. This really should + # only be used to interface with other C libraries. + + printf("hello, world\n"); + +Debugging +========= + +Compile with debug symbols and use gdb and valgrind. It sucks but its +really the only way. + +Vim +=== + +For syntax highlighting in Vim:: + + :set syntax=pyrex diff --git a/etc/requirements.txt b/etc/requirements.txt index caa0d0c6..236073f7 100644 --- a/etc/requirements.txt +++ b/etc/requirements.txt @@ -10,3 +10,6 @@ gevent-zeromq==0.2.2 # Packaging distribute==0.6.27 setuptools==0.6c11 + +# Unix +setproctitle==1.1.6 From ef763f28d0587ac12100645b9400c8b52ee5143b Mon Sep 17 00:00:00 2001 From: fawce Date: Tue, 29 May 2012 21:20:16 -0400 Subject: [PATCH 10/10] added a created date to the risk report. --- zipline/finance/risk.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/zipline/finance/risk.py b/zipline/finance/risk.py index af94b284..f4bd736f 100644 --- a/zipline/finance/risk.py +++ b/zipline/finance/risk.py @@ -41,7 +41,7 @@ import datetime import math import numpy as np import numpy.linalg as la -import zipline.protocol as zp +from zipline.utils.date_utils import epoch_now LOGGER = logging.getLogger('ZiplineLogger') @@ -331,6 +331,7 @@ class RiskReport(): self.algorithm_returns = algorithm_returns self.trading_environment = trading_environment self.exceeded_max_loss = exceeded_max_loss + self.created = epoch_now() if len(self.algorithm_returns) == 0: start_date = self.trading_environment.period_start @@ -364,7 +365,8 @@ class RiskReport(): 'three_month' : [x.to_dict() for x in self.three_month_periods], 'six_month' : [x.to_dict() for x in self.six_month_periods], 'twelve_month' : [x.to_dict() for x in self.year_periods], - 'exceeded_max_loss' : self.exceeded_max_loss + 'exceeded_max_loss' : self.exceeded_max_loss, + 'created' : self.created } def periodsInRange(self, months_per, start, end):