From e4a0c7af7fb45b1051dcc53db5e2c340a4830700 Mon Sep 17 00:00:00 2001 From: fawce Date: Tue, 1 May 2012 15:25:04 -0400 Subject: [PATCH 1/4] added initialize method to the algorithm protocol. --- zipline/finance/trading.py | 4 +++- zipline/test/algorithms.py | 16 ++++++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/zipline/finance/trading.py b/zipline/finance/trading.py index 03e45fa8..6e3d92a9 100644 --- a/zipline/finance/trading.py +++ b/zipline/finance/trading.py @@ -59,8 +59,10 @@ class TradeSimulationClient(qmsg.Component): :py:mod:`zipline.test.algorithm` """ self.algorithm = algorithm - #register the trading_client's order method with the algorithm + # register the trading_client's order method with the algorithm self.algorithm.set_order(self.order) + # ask the algorithm to initialize + self.algorithm.initialize() def open(self): self.result_feed = self.connect_result() diff --git a/zipline/test/algorithms.py b/zipline/test/algorithms.py index 17873d27..b3743709 100644 --- a/zipline/test/algorithms.py +++ b/zipline/test/algorithms.py @@ -9,6 +9,9 @@ are provided below. The algorithm must expose methods: + - initialize: method that takes no args, no returns. Simply called to + enable the algorithm to set any internal state needed. + - get_sid_filter: method that takes no args, and returns a list of valid sids. List must have a length between 1 and 10. If None is returned the filter will block all events. @@ -18,7 +21,7 @@ The algorithm must expose methods: +-----------------+--------------+----------------+--------------------+ | | SID(133) | SID(134) | SID(135) | - +=================+==============+=====================================+ + +=================+==============+================+====================+ | price | $10.10 | $22.50 | $13.37 | +-----------------+--------------+----------------+--------------------+ | volume | 10,000 | 5,000 | 50,000 | @@ -61,6 +64,9 @@ class TestAlgorithm(): self.order = None self.frame_count = 0 self.portfolio = None + + def initialize(self): + pass def set_order(self, order_callable): self.order = order_callable @@ -94,7 +100,10 @@ class HeavyBuyAlgorithm(): self.order = None self.frame_count = 0 self.portfolio = None - + + def initialize(self): + pass + def set_order(self, order_callable): self.order = order_callable @@ -114,6 +123,9 @@ class NoopAlgorithm(object): """ Dolce fa niente. """ + + def initialize(self): + pass def set_order(self, order_callable): pass From a60154dd6b59272dac0d2a7b5f1a969b0336f137 Mon Sep 17 00:00:00 2001 From: fawce Date: Tue, 1 May 2012 16:00:53 -0400 Subject: [PATCH 2/4] intersticial commit for transforms --- zipline/finance/transforms.py | 61 +++++++++++++++++++++++++++++++++++ zipline/messaging.py | 10 +++--- 2 files changed, 66 insertions(+), 5 deletions(-) create mode 100644 zipline/finance/transforms.py diff --git a/zipline/finance/transforms.py b/zipline/finance/transforms.py new file mode 100644 index 00000000..4d32f138 --- /dev/null +++ b/zipline/finance/transforms.py @@ -0,0 +1,61 @@ +from datetime import timedelta +from itertools import ifilter +from collections import defaultdict + +from zipline.messaging import BaseTransform + +class VWAPTransform(BaseTransform): + + def init(self, daycount=3): + self.daycount = daycount + + +class DailyVWAP: + """A class that tracks the volume weighted average price + based on tick updates.""" + def __init__(self, daycount=3): + self.ticks = [] + self.dropped_ticks = [] + self.flux = 0.0 + self.volume = 0 + self.lastTick = None + self.vwap = 0.0 + self.delta = timedelta(days=daycount) + + def update(self, event): + + self.ticks.append(event) + flux, volume = self.calculate_flux([event]) + self.flux += flux + self.volume += volume + + self.last_date = event['dt'] + self.first_date = self.last_date - self.delta + #use a list comprehension to filter the ticks to those within + #desired day range. The dt properties are full datetime objects + #and provide overloads for arithmetic operations. + self.dropped_ticks = [] + for tick in self.ticks: + if tick['dt'] < self.first_date: + self.dropped_ticks.append(tick) + + slice_index = len(self.dropped_ticks) + self.ticks = self.ticks[slice_index:] + + dropped_flux, dropped_volume = self.calculate_flux(self.dropped_ticks) + + self.flux -= dropped_flux + self.volume -= dropped_volume + + if(self.volume != 0): + self.vwap = self.flux / self.volume + else: + self.vwap = None + + def calculate_flux(self, ticks): + flux = 0.0 + volume = 0 + for tick in ticks: + flux += tick['volume'] * tick['price'] + volume += tick['volume'] + return flux, volume \ No newline at end of file diff --git a/zipline/messaging.py b/zipline/messaging.py index fd1875c1..e1011071 100644 --- a/zipline/messaging.py +++ b/zipline/messaging.py @@ -440,14 +440,14 @@ class BaseTransform(Component): method to create a new derived value from the combined feed. """ - def __init__(self, name): + def __init__(self, name, **kwargs): Component.__init__(self) self.state = { 'name': name } - self.init() + self.init(**kwargs) def init(self): pass @@ -564,11 +564,11 @@ class PassthroughTransform(BaseTransform): """ - def __init__(self): + def __init__(self, **kwargs): BaseTransform.__init__(self, "PASSTHROUGH") - self.init() + self.init(**kwargs) - def init(self): + def init(self, **kwargs): pass @property From 4da50156e734fa0e59713ebfc64233ba7d387299 Mon Sep 17 00:00:00 2001 From: fawce Date: Wed, 2 May 2012 11:10:03 -0400 Subject: [PATCH 3/4] added a test to mix long/short orders --- zipline/test/test_finance.py | 101 +++++++++++++++++++++++++---------- 1 file changed, 72 insertions(+), 29 deletions(-) diff --git a/zipline/test/test_finance.py b/zipline/test/test_finance.py index 763d8024..0876e19c 100644 --- a/zipline/test/test_finance.py +++ b/zipline/test/test_finance.py @@ -368,33 +368,54 @@ class FinanceTestCase(TestCase): # same scenario, but short sales. params2 = { - 'trade_count':100, - 'trade_amount':100, - 'trade_delay': timedelta(minutes=5), - 'trade_interval': timedelta(days=1), - 'order_count':3, - 'order_amount':1000, - 'order_interval': timedelta(minutes=30), + 'trade_count' : 100, + 'trade_amount' : 100, + 'trade_delay' : timedelta(minutes=5), + 'trade_interval' : timedelta(days=1), + 'order_count' : 3, + 'order_amount' :-1000, + 'order_interval' : timedelta(minutes=30), # because we placed an orders totaling less than 25% of one trade # the simulator should produce just one transaction. - 'expected_txn_count' : 1, - 'expected_txn_volume' : 25 + 'expected_txn_count' : 1, + 'expected_txn_volume' : -25 } self.transaction_sim(**params2) - - + + @timed(DEFAULT_TIMEOUT) + def test_alternating_long_short(self): + # create a scenario where we alternate buys and sells + params1 = { + 'trade_count' : int(6.5 * 60 * 4), + 'trade_amount' : 100, + 'trade_interval' : timedelta(minutes=1), + 'order_count' : 4, + 'order_amount' : 10, + 'order_interval' : timedelta(hours=24), + 'alternate' : True, + 'complete_fill' : True, + 'expected_txn_count' : 4, + 'expected_txn_volume' : 0 #equal buys and sells + } + self.transaction_sim(**params1) def transaction_sim(self, **params): - trade_count = params['trade_count'] - trade_amount = params['trade_amount'] - trade_interval = params['trade_interval'] - trade_delay = params.get('trade_delay') - order_count = params['order_count'] - order_amount = params['order_amount'] - order_interval = params['order_interval'] - expected_txn_count = params['expected_txn_count'] + trade_count = params['trade_count'] + trade_amount = params['trade_amount'] + trade_interval = params['trade_interval'] + trade_delay = params.get('trade_delay') + order_count = params['order_count'] + order_amount = params['order_amount'] + order_interval = params['order_interval'] + expected_txn_count = params['expected_txn_count'] expected_txn_volume = params['expected_txn_volume'] + # optional parameters + # --------------------- + # if present, alternate between long and short sales + alternate = params.get('alternate') + # if present, expect transaction amounts to match orders exactly. + complete_fill = params.get('complete_fill') trading_environment = factory.create_trading_environment() trade_sim = TransactionSimulator() @@ -411,17 +432,31 @@ class FinanceTestCase(TestCase): trading_environment ) - for i in range(order_count): + if alternate: + alternator = -1 + else: + alternator = 1 + + order_date = start_date + for i in xrange(order_count): order = namedict( { - 'sid':sid, - 'amount':order_amount, - 'type':zp.DATASOURCE_TYPE.ORDER, - 'dt' : start_date + i * order_interval + 'sid' : sid, + 'amount' : order_amount * alternator**i, + 'type' : zp.DATASOURCE_TYPE.ORDER, + 'dt' : order_date }) trade_sim.add_open_order(order) - + + order_date = order_date + order_interval + # move after market orders to just after market next + # market open. + if order_date.hour >= 21: + if order_date.minute >= 00: + order_date = order_date + timedelta(days=1) + order_date = order_date.replace(hour=14, minute=30) + # there should now be one open order list stored under the sid oo = trade_sim.open_orders self.assertEqual(len(oo), 1) @@ -429,9 +464,10 @@ class FinanceTestCase(TestCase): order_list = oo[sid] self.assertEqual(order_count, len(order_list)) - for order in order_list: + for i in xrange(order_count): + order = order_list[i] self.assertEqual(order.sid, sid) - self.assertEqual(order.amount, order_amount) + self.assertEqual(order.amount, order_amount * alternator**i) tracker = PerformanceTracker(trading_environment) @@ -450,10 +486,17 @@ class FinanceTestCase(TestCase): trade.TRANSACTION = None tracker.process_event(trade) - + + if complete_fill: + self.assertEqual(len(transactions), len(order_list)) + total_volume = 0 - for txn in transactions: + for i in xrange(len(transactions)): + txn = transactions[i] total_volume += txn.amount + if complete_fill: + order = order_list[i] + self.assertEqual(order.amount, txn.amount) self.assertEqual(total_volume, expected_txn_volume) self.assertEqual(len(transactions), expected_txn_count) From 9b42a1e63fc73ff6a9187d5f179bd2109df197f2 Mon Sep 17 00:00:00 2001 From: fawce Date: Fri, 4 May 2012 13:32:29 -0400 Subject: [PATCH 4/4] draft of a vwap transform. --- zipline/finance/transforms.py | 6 +++ zipline/lines.py | 95 ++++++++++++++++------------------- 2 files changed, 49 insertions(+), 52 deletions(-) diff --git a/zipline/finance/transforms.py b/zipline/finance/transforms.py index 4d32f138..cdfcbdc8 100644 --- a/zipline/finance/transforms.py +++ b/zipline/finance/transforms.py @@ -8,7 +8,13 @@ class VWAPTransform(BaseTransform): def init(self, daycount=3): self.daycount = daycount + self.by_sid = defaultdict(DailyVWAP) + def transform(self, event): + cur = self.by_sid(event.sid) + cur.update(event) + self.state['value'] = cur.vwap + return self.state class DailyVWAP: """A class that tracks the volume weighted average price diff --git a/zipline/lines.py b/zipline/lines.py index 56ec2b9a..26d01f67 100644 --- a/zipline/lines.py +++ b/zipline/lines.py @@ -4,9 +4,7 @@ messaging. All ziplines follow a general topology of parallel sources, datetimestamp serialization, parallel transformations, and finally sinks. Furthermore, many ziplines have common needs. For example, all trade simulations require a -:py:class:`~zipline.finance.trading.TradeSimulationClient`, an -:py:class:`~zipline.finance.trading.OrderSource`, and a -:py:class:`~zipline.finance.trading.TransactionSimulator` (a transform). +:py:class:`~zipline.finance.trading.TradeSimulationClient`. To establish best practices and minimize code replication, the lines module provides complete zipline topologies. You can extend any zipline without @@ -17,56 +15,49 @@ before invoking simulate. Here is a diagram of the SimulatedTrading zipline: - - +----------------------+ +------------------------+ - +-->| Orders DataSource | | (DataSource added | - | | Integrates algo | | via add_source) | - | | orders into history | | | - | +--------------------+-+ +-+----------------------+ - | | | - | | | - | v v - | +---------+ - | | Feed | - | +-+------++ - | | | - | | | - | v v - | +----------------------+ +----------------------+ - | | Transaction | | | - | | Transform simulates | | (Transforms added | - | | trades based on | | via add_transform) | - | | orders from algo. | | | - | +-------------------+--+ +-+--------------------+ - | | | - | | | - | v v - | +------------+ - | | Merge | - | +------+-----+ - | | - | | - | V - | +--------------------------------+ - | | | - | | TradingSimulationClient | - | orders | tracks performance and | - +---------------+ provides API to algorithm. | - | | - +---------------------+----------+ - ^ | - | orders | frames - | | - | v - +---------+-----------------------+ - | | - | Algorithm added via | - | __init__. | - | | - | | - | | - +---------------------------------+ + +----------------------+ +------------------------+ + | Trade History | | (DataSource added | + | | | via add_source) | + | | | | + +--------------------+-+ +-+----------------------+ + | | + | | + v v + +---------+ + | Feed | (ensures events are serialized + +-+------++ in chronological order) + | | + | | + v v + +----------------------+ +----------------------+ + | (Transforms added | | (Transforms added | + | via add_transform) | | via add_transform) | + +-------------------+--+ +-+--------------------+ + | | + | | + v v + +------------+ + | Merge | (combines original event and + +------+-----+ transforms into one vector) + | + | + V + +---------------+ +--------------------------------+ + | Risk and Perf | | | + | Tracker | | TradingSimulationClient | + +---------------+ | tracks performance and | + ^ Trades and | provides API to algorithm. | + | simulated | | + | transactions +--+------------------+----------+ + | | ^ | + +---------------------+ | orders | frames + | | + | v + +---------------------------------+ + | Algorithm added via | + | __init__. | + +---------------------------------+ """ import mock