From 069dbd1afe0080e87ddcfc03085c4e05afc9a3ec Mon Sep 17 00:00:00 2001 From: fawce Date: Mon, 28 May 2012 21:32:23 -0400 Subject: [PATCH 1/3] removed protocol cruft from transaction transform days. --- tests/test_perf_tracking.py | 2 +- tests/test_protocol.py | 21 ++++-------- zipline/finance/trading.py | 11 +++---- zipline/protocol.py | 64 ------------------------------------- zipline/utils/date_utils.py | 3 +- 5 files changed, 13 insertions(+), 88 deletions(-) diff --git a/tests/test_perf_tracking.py b/tests/test_perf_tracking.py index 06581434..d979473d 100644 --- a/tests/test_perf_tracking.py +++ b/tests/test_perf_tracking.py @@ -552,7 +552,7 @@ shares in position" }) else: txn = None - event[zp.TRANSFORM_TYPE.TRANSACTION] = txn + event['TRANSACTION'] = txn perf_tracker.process_event(event) #we skip two trades, to test case of None transaction diff --git a/tests/test_protocol.py b/tests/test_protocol.py index d26c2feb..a61ab246 100644 --- a/tests/test_protocol.py +++ b/tests/test_protocol.py @@ -1,5 +1,5 @@ """ -Test the FRAME/UNFRAME functions in the sequence expected from ziplines. +Test the FRAME/UNFRAME functions in the sequence expected from ziplines. """ import pytz @@ -36,10 +36,10 @@ class ProtocolTestCase(TestCase): one_day_td = timedelta(days=1) trades = factory.create_trade_history( - sid, - price, - volume, - one_day_td, + sid, + price, + volume, + one_day_td, self.trading_environment ) @@ -92,7 +92,7 @@ class ProtocolTestCase(TestCase): self.assertEqual(order.sid, 133) self.assertEqual(order.amount, 100) self.assertEqual(order.dt, now) - + #order datasource datasource frames the order order_event = zp.ndict({ "sid" : order.sid, @@ -118,12 +118,3 @@ class ProtocolTestCase(TestCase): 'price' : 10.0, 'commission' : 0.50 }) - - #frame that transaction - txn_msg = zp.TRANSFORM_FRAME(zp.TRANSFORM_TYPE.TRANSACTION, txn) - - #unframe - recovered_tx = zp.TRANSFORM_UNFRAME(txn_msg).TRANSACTION - self.assertEqual(recovered_tx.sid, 133) - self.assertEqual(recovered_tx.amount, 100) - diff --git a/zipline/finance/trading.py b/zipline/finance/trading.py index 6b5ac1f4..2d378574 100644 --- a/zipline/finance/trading.py +++ b/zipline/finance/trading.py @@ -168,12 +168,11 @@ for orders: def create_transaction(self, sid, amount, price, dt, direction): self.txn_count += 1 - txn = {'sid' : sid, - 'amount' : int(amount), - 'dt' : dt, - 'price' : price, - 'commission' : self.commission * amount * direction, - 'source_id' : zp.FINANCE_COMPONENT.TRANSACTION_SIM + txn = {'sid' : sid, + 'amount' : int(amount), + 'dt' : dt, + 'price' : price, + 'commission' : self.commission * amount * direction } return zp.ndict(txn) diff --git a/zipline/protocol.py b/zipline/protocol.py index 8db194cb..7828cc06 100644 --- a/zipline/protocol.py +++ b/zipline/protocol.py @@ -342,8 +342,6 @@ def TRANSFORM_FRAME(name, value): assert isinstance(name, basestring) if value == None: return msgpack.dumps(tuple([name, TRANSFORM_TYPE.EMPTY])) - if(name == TRANSFORM_TYPE.TRANSACTION): - value = TRANSACTION_FRAME(value) return msgpack.dumps(tuple([name, value])) def TRANSFORM_UNFRAME(msg): @@ -359,8 +357,6 @@ def TRANSFORM_UNFRAME(msg): assert isinstance(name, basestring) if(name == TRANSFORM_TYPE.PASSTHROUGH): value = FEED_UNFRAME(value) - elif(name == TRANSFORM_TYPE.TRANSACTION): - value = TRANSACTION_UNFRAME(value) return ndict({name : value}) except TypeError: @@ -382,11 +378,6 @@ def MERGE_FRAME(event): """ assert isinstance(event, ndict) PACK_DATE(event) - if(event.has_attr(TRANSFORM_TYPE.TRANSACTION)): - if(event.TRANSACTION == None): - event.TRANSACTION = TRANSFORM_TYPE.EMPTY - else: - event.TRANSACTION = TRANSACTION_FRAME(event.TRANSACTION) payload = event.as_dict() return msgpack.dumps(payload) @@ -396,11 +387,6 @@ def MERGE_UNFRAME(msg): #TODO: anything we can do to assert more about the content of the dict? assert isinstance(payload, dict) payload = ndict(payload) - if(payload.has_attr(TRANSFORM_TYPE.TRANSACTION)): - if(payload.TRANSACTION == TRANSFORM_TYPE.EMPTY): - payload.TRANSACTION = None - else: - payload.TRANSACTION = TRANSACTION_UNFRAME(payload.TRANSACTION) UNPACK_DATE(payload) return payload except TypeError: @@ -501,54 +487,6 @@ def ORDER_UNFRAME(msg): except ValueError: raise INVALID_ORDER_FRAME(msg) - -# ----------------------- -# TRANSACTIONS -# ----------------------- -# -# - Should only be called from inside TRANSFORM_(UN)FRAME. - - - -def TRANSACTION_FRAME(event): - assert isinstance(event, ndict) - assert isinstance(event.sid, int) - assert isinstance(event.price, numbers.Real) - assert isinstance(event.commission, numbers.Real) - assert isinstance(event.amount, int) - PACK_DATE(event) - return msgpack.dumps(tuple([ - event.sid, - event.price, - event.amount, - event.commission, - event.dt - ])) - -def TRANSACTION_UNFRAME(msg): - try: - sid, price, amount, commission, dt = msgpack.loads(msg) - - assert isinstance(sid, int) - assert isinstance(price, numbers.Real) - assert isinstance(commission, numbers.Real) - assert isinstance(amount, int) - rval = ndict({ - 'sid' : sid, - 'price' : price, - 'amount' : amount, - 'commission' : commission, - 'dt' : dt - }) - - UNPACK_DATE(rval) - return rval - except TypeError: - raise INVALID_TRADE_FRAME(msg) - except ValueError: - raise INVALID_TRADE_FRAME(msg) - - # ----------------------- # ORDERS # ----------------------- @@ -742,7 +680,6 @@ ORDER_PROTOCOL = Enum( #Transform type needs to be a ndict to facilitate merging. TRANSFORM_TYPE = ndict({ - 'TRANSACTION' : 'TRANSACTION', #needed? 'PASSTHROUGH' : 'PASSTHROUGH', 'EMPTY' : '' }) @@ -752,7 +689,6 @@ FINANCE_COMPONENT = namelookup({ 'TRADING_CLIENT' : 'TRADING_CLIENT', 'PORTFOLIO_CLIENT' : 'PORTFOLIO_CLIENT', 'ORDER_SOURCE' : 'ORDER_SOURCE', - 'TRANSACTION_SIM' : 'TRANSACTION_SIM' }) diff --git a/zipline/utils/date_utils.py b/zipline/utils/date_utils.py index 801315b8..131dd61b 100644 --- a/zipline/utils/date_utils.py +++ b/zipline/utils/date_utils.py @@ -47,8 +47,7 @@ def EPOCH(utc_datetime): return int(ms) def UN_EPOCH(ms_since_epoch): - seconds_since_epoch = float(ms_since_epoch) / 1000.0 - delta = timedelta(seconds = seconds_since_epoch) + delta = timedelta(milliseconds = ms_since_epoch) dt = UNIX_EPOCH + delta return dt From 433170df9bcb1e8f51fa26418a94e0ef53682783 Mon Sep 17 00:00:00 2001 From: fawce Date: Mon, 28 May 2012 21:39:54 -0400 Subject: [PATCH 2/3] removed old order protocol cruft. --- tests/test_finance.py | 1 - tests/test_protocol.py | 43 ------------ zipline/components/tradesimulation.py | 3 - zipline/protocol.py | 98 --------------------------- 4 files changed, 145 deletions(-) diff --git a/tests/test_finance.py b/tests/test_finance.py index 3676bea9..bbceac25 100644 --- a/tests/test_finance.py +++ b/tests/test_finance.py @@ -438,7 +438,6 @@ class FinanceTestCase(TestCase): { 'sid' : sid, 'amount' : order_amount * alternator**i, - 'type' : zp.DATASOURCE_TYPE.ORDER, 'dt' : order_date }) diff --git a/tests/test_protocol.py b/tests/test_protocol.py index a61ab246..c90b09dc 100644 --- a/tests/test_protocol.py +++ b/tests/test_protocol.py @@ -75,46 +75,3 @@ class ProtocolTestCase(TestCase): event.delete('helloworld') self.assertEqual(zp.ndict(trade), event) - - @timed(DEFAULT_TIMEOUT) - def test_order_protocol(self): - #client places an order - now = datetime.utcnow().replace(tzinfo=pytz.utc) - order = zp.ndict({ - 'dt':now, - 'sid':133, - 'amount':100 - }) - order_msg = zp.ORDER_FRAME(order) - - #order datasource receives - order = zp.ORDER_UNFRAME(order_msg) - self.assertEqual(order.sid, 133) - self.assertEqual(order.amount, 100) - self.assertEqual(order.dt, now) - - #order datasource datasource frames the order - order_event = zp.ndict({ - "sid" : order.sid, - "amount" : order.amount, - "dt" : order.dt, - "source_id" : zp.FINANCE_COMPONENT.ORDER_SOURCE, - "type" : zp.DATASOURCE_TYPE.ORDER - }) - - - order_ds_msg = zp.DATASOURCE_FRAME(order_event) - - #transaction transform unframes - recovered_order = zp.DATASOURCE_UNFRAME(order_ds_msg) - - self.assertEqual(now, recovered_order.dt) - - #create a transaction from the order - txn = zp.ndict({ - 'sid' : recovered_order.sid, - 'amount' : recovered_order.amount, - 'dt' : recovered_order.dt, - 'price' : 10.0, - 'commission' : 0.50 - }) diff --git a/zipline/components/tradesimulation.py b/zipline/components/tradesimulation.py index 06aa7272..827a8d00 100644 --- a/zipline/components/tradesimulation.py +++ b/zipline/components/tradesimulation.py @@ -150,9 +150,6 @@ class TradeSimulationClient(Component): self.perf.log_order(order) self.txn_sim.add_open_order(order) - def signal_order_done(self): - self.order_socket.send(str(zp.ORDER_PROTOCOL.DONE)) - def queue_event(self, event): if self.event_queue == None: self.event_queue = [] diff --git a/zipline/protocol.py b/zipline/protocol.py index 7828cc06..8ff24389 100644 --- a/zipline/protocol.py +++ b/zipline/protocol.py @@ -246,12 +246,6 @@ def DATASOURCE_FRAME(event): event.source_id, TRADE_FRAME(event) ])) - elif(event.type == DATASOURCE_TYPE.ORDER): - return msgpack.dumps(tuple([ - event.type, - event.source_id, - ORDER_SOURCE_FRAME(event) - ])) else: raise INVALID_DATASOURCE_FRAME(str(event)) @@ -286,8 +280,6 @@ def DATASOURCE_UNFRAME(msg): child_value = ndict({'dt':None}) elif(ds_type == DATASOURCE_TYPE.TRADE): child_value = TRADE_UNFRAME(payload) - elif(ds_type == DATASOURCE_TYPE.ORDER): - child_value = ORDER_SOURCE_UNFRAME(payload) else: raise INVALID_DATASOURCE_FRAME(msg) @@ -395,12 +387,6 @@ def MERGE_UNFRAME(msg): raise INVALID_MERGE_FRAME(msg) -# ----------------------- -# Finance Protocol -# ----------------------- -INVALID_ORDER_FRAME = FrameExceptionFactory('ORDER') -INVALID_TRADE_FRAME = FrameExceptionFactory('TRADE') - # ----------------------- # Trades # ----------------------- @@ -454,83 +440,6 @@ def TRADE_UNFRAME(msg): except ValueError: raise INVALID_TRADE_FRAME(msg) -# ----------------------- -# Orders -# ----------------------- -# - from client to order source - -def ORDER_FRAME(order): - assert isinstance(order.sid, int) - assert isinstance(order.amount, int) #no partial shares... - PACK_DATE(order) - return msgpack.dumps(tuple([ - order.sid, - order.amount, - order.dt - ])) - - -def ORDER_UNFRAME(msg): - try: - sid, amount, dt = msgpack.loads(msg) - assert isinstance(sid, int) - assert isinstance(amount, int) - rval = ndict({ - 'sid':sid, - 'amount':amount, - 'dt':dt - }) - UNPACK_DATE(rval) - return rval - except TypeError: - raise INVALID_ORDER_FRAME(msg) - except ValueError: - raise INVALID_ORDER_FRAME(msg) - -# ----------------------- -# ORDERS -# ----------------------- -# -# - from order source to feed -# - should only be called from inside DATASOURCE_(UN)FRAME - - -def ORDER_SOURCE_FRAME(event): - assert isinstance(event.sid, int) - assert isinstance(event.amount, int) #no partial shares... - assert isinstance(event.source_id, basestring) - assert event.type == DATASOURCE_TYPE.ORDER - PACK_DATE(event) - return msgpack.dumps(tuple([ - event.sid, - event.amount, - event.dt, - event.source_id, - event.type - ])) - - -def ORDER_SOURCE_UNFRAME(msg): - try: - sid, amount, dt, source_id, source_type = msgpack.loads(msg) - event = ndict({ - "sid" : sid, - "amount" : amount, - "dt" : dt, - "source_id" : source_id, - "type" : source_type - }) - assert isinstance(sid, int) - assert isinstance(amount, int) - assert isinstance(source_id, basestring) - assert isinstance(source_type, int) - UNPACK_DATE(event) - return event - except TypeError: - raise INVALID_ORDER_FRAME(msg) - except ValueError: - raise INVALID_ORDER_FRAME(msg) - # ----------------------- # Performance and Risk # ----------------------- @@ -667,16 +576,10 @@ def tuple_to_date(date_tuple): return dt DATASOURCE_TYPE = Enum( - 'ORDER', 'TRADE', 'EMPTY', ) -ORDER_PROTOCOL = Enum( - 'DONE', - 'BREAK', -) - #Transform type needs to be a ndict to facilitate merging. TRANSFORM_TYPE = ndict({ @@ -688,7 +591,6 @@ TRANSFORM_TYPE = ndict({ FINANCE_COMPONENT = namelookup({ 'TRADING_CLIENT' : 'TRADING_CLIENT', 'PORTFOLIO_CLIENT' : 'PORTFOLIO_CLIENT', - 'ORDER_SOURCE' : 'ORDER_SOURCE', }) From b8d0a8e4e08a21638e22b69f7fa57e4038d4ab6b Mon Sep 17 00:00:00 2001 From: fawce Date: Mon, 28 May 2012 22:13:43 -0400 Subject: [PATCH 3/3] cleanup --- zipline/protocol.py | 1 - 1 file changed, 1 deletion(-) diff --git a/zipline/protocol.py b/zipline/protocol.py index 8ff24389..7081291f 100644 --- a/zipline/protocol.py +++ b/zipline/protocol.py @@ -495,7 +495,6 @@ def convert_transactions(transactions): for txn in transactions: txn['date'] = EPOCH(txn['dt']) del(txn['dt']) - del(txn['source_id']) results.append(txn) return results