From e6b3e8e7b13450d68301a54e92d7068cf910cec5 Mon Sep 17 00:00:00 2001 From: fawce Date: Thu, 27 Sep 2012 23:09:14 -0400 Subject: [PATCH 1/4] adding independent commission modeling. --- zipline/finance/commissions.py | 19 +++++++++++++++++++ zipline/finance/slippage.py | 21 +++++++++++++-------- 2 files changed, 32 insertions(+), 8 deletions(-) create mode 100644 zipline/finance/commissions.py diff --git a/zipline/finance/commissions.py b/zipline/finance/commissions.py new file mode 100644 index 00000000..63ae2962 --- /dev/null +++ b/zipline/finance/commissions.py @@ -0,0 +1,19 @@ +class PerShareCommission(object): + + def __init__(self, cost=0.03): + self.cost = cost + + + def calculate(self, transaction): + return self.cost, abs(transaction.amount * self.cost) + +class PerTradeCommission(object): + + def __init__(self, cost=5.0): + self.cost = cost + + def calculate(self, transaction): + if transaction.amount == 0: + return 0.0, 0.0 + + return abs(self.cost / transaction.amount), self.cost diff --git a/zipline/finance/slippage.py b/zipline/finance/slippage.py index 363e70ad..a3c8ada1 100644 --- a/zipline/finance/slippage.py +++ b/zipline/finance/slippage.py @@ -1,24 +1,31 @@ import pytz import math -from datetime import timedelta import zipline.protocol as zp -def create_transaction(sid, amount, price, dt, direction, commission): +from zipline.finance.commissions import PerShareCommission + +def create_transaction(sid, amount, price, dt, commission): + txn = {'sid' : sid, 'amount' : int(amount), 'dt' : dt, 'price' : price, - 'commission' : commission * amount * direction } - return zp.ndict(txn) + + transaction = zp.ndict(txn) + per_share, total_commission = commission.calculate(transaction) + transaction.price = transaction.price + per_share + transaction.commission = total_commission + return transaction class VolumeShareSlippage(object): def __init__(self, volume_limit=.25, price_impact=0.1, - commission=0.03): + commission=PerShareCommission()): + self.volume_limit = volume_limit self.price_impact = price_impact self.commission = commission @@ -83,13 +90,12 @@ class VolumeShareSlippage(object): simulated_amount, event.price + simulated_impact, dt.replace(tzinfo = pytz.utc), - direction, self.commission ) class FixedSlippage(object): - def __init__(self, spread=0.0, commission=0.0): + def __init__(self, spread=0.0, commission=PerShareCommission()): """ Use the fixed slippage model, which will just add/subtract a specified spread spread/2 will be added on buys and subtracted on sells per share @@ -119,7 +125,6 @@ class FixedSlippage(object): amount, event.price + (self.spread/2.0 * direction), event.dt, - direction, self.commission ) From 1caefbff43e2c383d16002304cdb36af1dfb50ec Mon Sep 17 00:00:00 2001 From: fawce Date: Fri, 28 Sep 2012 22:58:37 -0400 Subject: [PATCH 2/4] tests are passing for independent commission model --- tests/test_exception_handling.py | 16 +-- tests/test_finance.py | 3 +- tests/test_transforms.py | 15 +++ zipline/algorithm.py | 10 +- .../finance/{commissions.py => commission.py} | 4 +- zipline/finance/slippage.py | 37 +++--- zipline/finance/trading.py | 20 +-- zipline/gens/tradesimulation.py | 96 ++++++++------- zipline/lines.py | 114 +----------------- zipline/test_algorithms.py | 26 ++-- zipline/utils/factory.py | 2 + zipline/utils/simfactory.py | 113 +++++++++++++++++ 12 files changed, 241 insertions(+), 215 deletions(-) rename zipline/finance/{commissions.py => commission.py} (85%) create mode 100644 zipline/utils/simfactory.py diff --git a/tests/test_exception_handling.py b/tests/test_exception_handling.py index be8117ce..f950dd07 100644 --- a/tests/test_exception_handling.py +++ b/tests/test_exception_handling.py @@ -2,6 +2,7 @@ from unittest2 import TestCase from collections import defaultdict +import zipline.utils.simfactory as simfactory from zipline.test_algorithms import ExceptionAlgorithm, DivByZeroAlgorithm, \ InitializeTimeoutAlgorithm, TooMuchProcessingAlgorithm from zipline.finance.slippage import FixedSlippage @@ -9,6 +10,7 @@ from zipline.lines import SimulatedTrading from zipline.gens.transform import StatefulTransform from zipline.utils.timeout import TimeoutException + from zipline.utils.test_utils import ( drain_zipline, setup_logger, @@ -37,7 +39,7 @@ class ExceptionTestCase(TestCase): def test_datasource_exception(self): self.zipline_test_config['trade_source'] = ExceptionSource() - zipline = SimulatedTrading.create_test_zipline( + zipline = simfactory.create_test_zipline( **self.zipline_test_config ) @@ -54,7 +56,7 @@ class ExceptionTestCase(TestCase): exc_tnfm = StatefulTransform(ExceptionTransform) self.zipline_test_config['transforms'] = [exc_tnfm] - zipline = SimulatedTrading.create_test_zipline( + zipline = simfactory.create_test_zipline( **self.zipline_test_config ) @@ -72,7 +74,7 @@ class ExceptionTestCase(TestCase): self.zipline_test_config['sid'] ) - zipline = SimulatedTrading.create_test_zipline( + zipline = simfactory.create_test_zipline( **self.zipline_test_config ) @@ -90,7 +92,7 @@ class ExceptionTestCase(TestCase): self.zipline_test_config['sid'] ) - zipline = SimulatedTrading.create_test_zipline( + zipline = simfactory.create_test_zipline( **self.zipline_test_config ) @@ -108,7 +110,7 @@ class ExceptionTestCase(TestCase): self.zipline_test_config['sid'] ) - zipline = SimulatedTrading.create_test_zipline( + zipline = simfactory.create_test_zipline( **self.zipline_test_config ) @@ -125,7 +127,7 @@ class ExceptionTestCase(TestCase): self.zipline_test_config['sid'] ) - zipline = SimulatedTrading.create_test_zipline( + zipline = simfactory.create_test_zipline( **self.zipline_test_config ) @@ -140,7 +142,7 @@ class ExceptionTestCase(TestCase): TooMuchProcessingAlgorithm( self.zipline_test_config['sid'] ) - zipline = SimulatedTrading.create_test_zipline( + zipline = simfactory.create_test_zipline( **self.zipline_test_config ) diff --git a/tests/test_finance.py b/tests/test_finance.py index 8d3ab408..d7f15943 100644 --- a/tests/test_finance.py +++ b/tests/test_finance.py @@ -10,6 +10,7 @@ from collections import defaultdict from nose.tools import timed import zipline.utils.factory as factory +import zipline.utils.simfactory as simfactory from zipline.finance.trading import TradingEnvironment from zipline.lines import SimulatedTrading @@ -111,7 +112,7 @@ class FinanceTestCase(TestCase): #provide enough trades to ensure all orders are filled. self.zipline_test_config['order_count'] = 100 self.zipline_test_config['trade_count'] = 200 - zipline = SimulatedTrading.create_test_zipline(**self.zipline_test_config) + zipline = simfactory.create_test_zipline(**self.zipline_test_config) assert_single_position(self, zipline) # TODO: write tests for short sales diff --git a/tests/test_transforms.py b/tests/test_transforms.py index 4f6cae0f..62b98b54 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -307,6 +307,21 @@ class BatchTransformTestCase(TestCase): self.assertTrue(np.all(test_history[3].values.flatten() == range(4, 10))) self.assertTrue(np.all(test_history[4].values.flatten() == range(6, 14))) + np.testing.assert_array_equal( + range(4, 10), + test_history[2].values.flatten() + ) + + np.testing.assert_array_equal( + range(4, 10), + test_history[3].values.flatten() + ) + + np.testing.assert_array_equal( + range(6, 14), + test_history[4].values.flatten() + ) + def test_passing_of_args(self): algo = BatchTransformAlgorithm([0, 1], 1, kwarg='str') self.assertEqual(algo.args, (1,)) diff --git a/zipline/algorithm.py b/zipline/algorithm.py index f54ba0cb..bf696b46 100644 --- a/zipline/algorithm.py +++ b/zipline/algorithm.py @@ -5,7 +5,8 @@ from zipline.gens.tradegens import DataFrameSource from zipline.utils.factory import create_trading_environment from zipline.gens.transform import StatefulTransform from zipline.lines import SimulatedTrading -from zipline.finance.slippage import FixedSlippage +from zipline.finance.slippage import FixedSlippage, simulate_method_factory +from zipline.finance.commission import PerShare class TradingAlgorithm(object): @@ -74,7 +75,7 @@ class TradingAlgorithm(object): transforms, self, environment, - FixedSlippage() + simulate_method_factory(FixedSlippage(), PerShare(0.0)) ) def run(self, source, start=None, end=None): @@ -179,8 +180,5 @@ class TradingAlgorithm(object): def initialize(self, *args, **kwargs): pass - def set_slippage_override(self, slippage_callable): + def set_simulate_override(self, slippage_callable): pass - - - diff --git a/zipline/finance/commissions.py b/zipline/finance/commission.py similarity index 85% rename from zipline/finance/commissions.py rename to zipline/finance/commission.py index 63ae2962..38883a65 100644 --- a/zipline/finance/commissions.py +++ b/zipline/finance/commission.py @@ -1,4 +1,4 @@ -class PerShareCommission(object): +class PerShare(object): def __init__(self, cost=0.03): self.cost = cost @@ -7,7 +7,7 @@ class PerShareCommission(object): def calculate(self, transaction): return self.cost, abs(transaction.amount * self.cost) -class PerTradeCommission(object): +class PerTrade(object): def __init__(self, cost=5.0): self.cost = cost diff --git a/zipline/finance/slippage.py b/zipline/finance/slippage.py index a3c8ada1..707fdd94 100644 --- a/zipline/finance/slippage.py +++ b/zipline/finance/slippage.py @@ -3,32 +3,39 @@ import math import zipline.protocol as zp -from zipline.finance.commissions import PerShareCommission -def create_transaction(sid, amount, price, dt, commission): +def simulate_method_factory(slippage, commission): + + def simulate(open_orders, events): + transaction = slippage.simulate(open_orders, events) + if transaction: + per_share, total_commission = commission.calculate(transaction) + transaction.price = transaction.price + per_share + transaction.commission = total_commission + return transaction + + return simulate + +def create_transaction(sid, amount, price, dt): txn = {'sid' : sid, - 'amount' : int(amount), - 'dt' : dt, - 'price' : price, + 'amount' : int(amount), + 'dt' : dt, + 'price' : price, } transaction = zp.ndict(txn) - per_share, total_commission = commission.calculate(transaction) - transaction.price = transaction.price + per_share - transaction.commission = total_commission return transaction + class VolumeShareSlippage(object): def __init__(self, volume_limit=.25, - price_impact=0.1, - commission=PerShareCommission()): + price_impact=0.1): self.volume_limit = volume_limit self.price_impact = price_impact - self.commission = commission def simulate(self, event, open_orders): @@ -90,19 +97,16 @@ class VolumeShareSlippage(object): simulated_amount, event.price + simulated_impact, dt.replace(tzinfo = pytz.utc), - self.commission ) class FixedSlippage(object): - def __init__(self, spread=0.0, commission=PerShareCommission()): + def __init__(self, spread=0.0): """ Use the fixed slippage model, which will just add/subtract a specified spread spread/2 will be added on buys and subtracted on sells per share - commission will be charged per share """ self.spread = spread - self.commission = commission def simulate(self, event, open_orders): if event.sid in open_orders: @@ -124,8 +128,7 @@ class FixedSlippage(object): event.sid, amount, event.price + (self.spread/2.0 * direction), - event.dt, - self.commission + event.dt ) open_orders[event.sid] = [] diff --git a/zipline/finance/trading.py b/zipline/finance/trading.py index 4258a4ff..4fc94aef 100644 --- a/zipline/finance/trading.py +++ b/zipline/finance/trading.py @@ -5,18 +5,24 @@ import datetime from collections import defaultdict import zipline.protocol as zp -from zipline.finance.slippage import VolumeShareSlippage, FixedSlippage +from zipline.finance.slippage import ( + VolumeShareSlippage, + simulate_method_factory + ) +from zipline.finance.commission import PerShare log = logbook.Logger('Transaction Simulator') class TransactionSimulator(object): - def __init__(self, slippage=None): - if slippage: - assert isinstance(slippage, (VolumeShareSlippage, FixedSlippage)) - self.slippage = slippage + def __init__(self, simulate=None): + if simulate: + self.simulate = simulate else: - self.slippage = VolumeShareSlippage() + self.simulate = simulate_method_factory( + VolumeShareSlippage(), + PerShare() + ) self.open_orders = defaultdict(list) @@ -37,7 +43,7 @@ class TransactionSimulator(object): event.TRANSACTION = None # We only fill transactions on trade events. if event.type == zp.DATASOURCE_TYPE.TRADE: - event.TRANSACTION = self.slippage.simulate(event, self.open_orders) + event.TRANSACTION = self.simulate(event, self.open_orders) return event diff --git a/zipline/gens/tradesimulation.py b/zipline/gens/tradesimulation.py index 5b5cec52..8287536a 100644 --- a/zipline/gens/tradesimulation.py +++ b/zipline/gens/tradesimulation.py @@ -214,66 +214,64 @@ class AlgorithmSimulator(object): """ Main generator work loop. """ - # Capture any output of this generator to stdout and pipe it - # to a logbook interface. Also inject the current algo - # snapshot time to any log record generated. - #with self.processor.threadbound(), self.stdout_capture(Logger('Print'),''): - # Call user's initialize method with a timeout (only if # initialize wasn't called already). if not getattr(self.algo, 'initialized', False): with Timeout(INIT_TIMEOUT, message="Call to initialize timed out"): self.algo.initialize() - # Group together events with the same dt field. This depends on the - # events already being sorted. - for date, snapshot in groupby(stream_in, attrgetter('dt')): - # Set the simulation date to be the first event we see. - # This should only occur once, at the start of the test. - if self.simulation_dt == None: - self.simulation_dt = date + # inject the current algo + # snapshot time to any log record generated. + with self.processor.threadbound(): + # Group together events with the same dt field. This depends on the + # events already being sorted. + for date, snapshot in groupby(stream_in, attrgetter('dt')): + # Set the simulation date to be the first event we see. + # This should only occur once, at the start of the test. + if self.simulation_dt == None: + self.simulation_dt = date - # Done message has the risk report, so we yield before exiting. - if date == 'DONE': - for event in snapshot: - yield event.perf_message - raise StopIteration() - - # We're still in the warmup period. Use the event to - # update our universe, but don't yield any perf messages, - # and don't send a snapshot to handle_data. - elif date < self.algo_start: - for event in snapshot: - del event['perf_message'] - self.update_universe(event) - - # The algo has taken so long to process events that - # its simulated time is later than the event time. - # Update the universe and yield any perf messages - # encountered, but don't call handle_data. - elif date < self.simulation_dt: - for event in snapshot: - # Only yield if we have something interesting to say. - if event.perf_message != None: + # Done message has the risk report, so we yield before exiting. + if date == 'DONE': + for event in snapshot: yield event.perf_message - # Delete the message before updating so we don't send it - # to the user. - del event['perf_message'] - self.update_universe(event) + raise StopIteration() - # Regular snapshot. Update the universe and send a snapshot - # to handle data. - else: - for event in snapshot: - # Only yield if we have something interesting to say. - if event.perf_message != None: - yield event.perf_message - del event['perf_message'] + # We're still in the warmup period. Use the event to + # update our universe, but don't yield any perf messages, + # and don't send a snapshot to handle_data. + elif date < self.algo_start: + for event in snapshot: + del event['perf_message'] + self.update_universe(event) - self.update_universe(event) + # The algo has taken so long to process events that + # its simulated time is later than the event time. + # Update the universe and yield any perf messages + # encountered, but don't call handle_data. + elif date < self.simulation_dt: + for event in snapshot: + # Only yield if we have something interesting to say. + if event.perf_message != None: + yield event.perf_message + # Delete the message before updating so we don't send it + # to the user. + del event['perf_message'] + self.update_universe(event) - # Send the current state of the universe to the user's algo. - self.simulate_snapshot(date) + # Regular snapshot. Update the universe and send a snapshot + # to handle data. + else: + for event in snapshot: + # Only yield if we have something interesting to say. + if event.perf_message != None: + yield event.perf_message + del event['perf_message'] + + self.update_universe(event) + + # Send the current state of the universe to the user's algo. + self.simulate_snapshot(date) def update_universe(self, event): """ diff --git a/zipline/lines.py b/zipline/lines.py index a8fedc56..e29d8cfa 100644 --- a/zipline/lines.py +++ b/zipline/lines.py @@ -61,13 +61,11 @@ before invoking simulate. """ from zipline.utils import factory - from zipline.gens.composites import ( date_sorted_sources, sequential_transforms ) from zipline.gens.tradesimulation import TradeSimulationClient as tsc -from zipline.finance.slippage import FixedSlippage from logbook import Logger @@ -81,7 +79,7 @@ class SimulatedTrading(object): transforms, algorithm, environment, - slippage): + sim_method=None): """ @sources - an iterable of iterables These iterables must yield ndicts that contain: @@ -108,7 +106,7 @@ class SimulatedTrading(object): # Formerly merged_transforms. self.with_tnfms = sequential_transforms(self.date_sorted, *self.transforms) - self.trading_client = tsc(algorithm, environment, slippage) + self.trading_client = tsc(algorithm, environment, sim_method) self.gen = self.trading_client.simulate(self.with_tnfms) def __iter__(self): @@ -116,111 +114,3 @@ class SimulatedTrading(object): def next(self): return self.gen.next() - - @staticmethod - def create_test_zipline(**config): - """ - :param config: A configuration object that is a dict with: - - - environment - a \ - :py:class:`zipline.finance.trading.TradingEnvironment` - - sid - an integer, which will be used as the security ID. - - order_count - the number of orders the test algo will place, - defaults to 100 - - order_amount - the number of shares per order, defaults to 100 - - trade_count - the number of trades to simulate, defaults to 101 - to ensure all orders are processed. - - algorithm - optional parameter providing an algorithm. defaults - to :py:class:`zipline.test.algorithms.TestAlgorithm` - - trade_source - optional parameter to specify trades, if present. - If not present :py:class:`zipline.sources.SpecificEquityTrades` - is the source, with daily frequency in trades. - - slippage: optional parameter that configures the - :py:class:`zipline.gens.tradingsimulation.TransactionSimulator`. Expects - an object with a simulate mehod, such as - :py:class:`zipline.gens.tradingsimulation.FixedSlippage`. - :py:mod:`zipline.finance.trading` - - transforms: optional parameter that provides a list - of StatefulTransform objects. - """ - from zipline.test_algorithms import TestAlgorithm - - assert isinstance(config, dict) - sid_list = config.get('sid_list') - if not sid_list: - sid = config.get('sid') - sid_list = [sid] - - concurrent_trades = config.get('concurrent_trades', False) - - #-------------------- - # Trading Environment - #-------------------- - if 'environment' in config: - trading_environment = config['environment'] - else: - trading_environment = factory.create_trading_environment() - - if 'order_count' in config: - order_count = config['order_count'] - else: - order_count = 100 - - if 'order_amount' in config: - order_amount = config['order_amount'] - else: - order_amount = 100 - - if 'trade_count' in config: - trade_count = config['trade_count'] - else: - # to ensure all orders are filled, we provide one more - # trade than order - trade_count = 101 - - slippage = config.get('slippage', FixedSlippage()) - - #------------------- - # Trade Source - #------------------- - if 'trade_source' in config: - trade_source = config['trade_source'] - else: - trade_source = factory.create_daily_trade_source( - sid_list, - trade_count, - trading_environment, - concurrent=concurrent_trades - ) - - #------------------- - # Transforms - #------------------- - transforms = config.get('transforms', []) - - #------------------- - # Create the Algo - #------------------- - if 'algorithm' in config: - test_algo = config['algorithm'] - else: - test_algo = TestAlgorithm( - sid, - order_amount, - order_count - ) - - #------------------- - # Simulation - #------------------- - - sim = SimulatedTrading( - [trade_source], - transforms, - test_algo, - trading_environment, - slippage, - ) - #------------------- - - return sim diff --git a/zipline/test_algorithms.py b/zipline/test_algorithms.py index a8bc6ab4..6974b248 100644 --- a/zipline/test_algorithms.py +++ b/zipline/test_algorithms.py @@ -44,8 +44,8 @@ The algorithm must expose methods: self.Portfolio[sid(133)]['cost_basis'] - - set_slippage_override: method that accepts a callable. Will - be set as the value of the set_slippage_override method of + - set_simulate_override: method that accepts a callable. Will + be set as the value of the set_simulate_override method of the trading_client. This allows an algorithm to change the slippage model used to predict transactions based on orders and trade events. @@ -97,7 +97,7 @@ class TestAlgorithm(): def get_sid_filter(self): return self.sid_filter - def set_slippage_override(self, slippage_callable): + def set_simulate_override(self, txn_sim_callable): pass @@ -138,7 +138,7 @@ class HeavyBuyAlgorithm(): def get_sid_filter(self): return [self.sid] - def set_slippage_override(self, slippage_callable): + def set_simulate_override(self, txn_sim_callable): pass class NoopAlgorithm(object): @@ -164,7 +164,7 @@ class NoopAlgorithm(object): def get_sid_filter(self): return [] - def set_slippage_override(self, slippage_callable): + def set_simulate_override(self, txn_sim_callable): pass class ExceptionAlgorithm(object): @@ -210,7 +210,7 @@ class ExceptionAlgorithm(object): else: return [self.sid] - def set_slippage_override(self, slippage_callable): + def set_simulate_override(self, txn_sim_callable): pass class DivByZeroAlgorithm(): @@ -240,7 +240,7 @@ class DivByZeroAlgorithm(): def get_sid_filter(self): return [self.sid] - def set_slippage_override(self, slippage_callable): + def set_simulate_override(self, txn_sim_callable): pass class InitializeTimeoutAlgorithm(): @@ -269,7 +269,7 @@ class InitializeTimeoutAlgorithm(): def get_sid_filter(self): return [self.sid] - def set_slippage_override(self, slippage_callable): + def set_simulate_override(self, txn_sim_callable): pass class TooMuchProcessingAlgorithm(): @@ -297,7 +297,7 @@ class TooMuchProcessingAlgorithm(): def get_sid_filter(self): return [self.sid] - def set_slippage_override(self, slippage_callable): + def set_simulate_override(self, txn_sim_callable): pass class TimeoutAlgorithm(): @@ -327,7 +327,7 @@ class TimeoutAlgorithm(): def get_sid_filter(self): return [self.sid] - def set_slippage_override(self, slippage_callable): + def set_simulate_override(self, txn_sim_callable): pass class TestPrintAlgorithm(): @@ -354,7 +354,7 @@ class TestPrintAlgorithm(): def get_sid_filter(self): return [self.sid] - def set_slippage_override(self, slippage_callable): + def set_simulate_override(self, txn_sim_callable): pass class TestLoggingAlgorithm(): @@ -381,7 +381,7 @@ class TestLoggingAlgorithm(): def get_sid_filter(self): return [self.sid] - def set_slippage_override(self, slippage_callable): + def set_simulate_override(self, txn_sim_callable): pass @@ -447,5 +447,3 @@ class BatchTransformAlgorithm(TradingAlgorithm): self.history_return_price_class.append(self.return_price_class.handle_data(data)) self.history_return_price_decorator.append(self.return_price_decorator.handle_data(data)) self.history_return_args.append(self.return_args_batch.handle_data(data, *self.args, **self.kwargs)) - - diff --git a/zipline/utils/factory.py b/zipline/utils/factory.py index 7517fff5..c8a2311b 100644 --- a/zipline/utils/factory.py +++ b/zipline/utils/factory.py @@ -244,3 +244,5 @@ def create_test_df_source(): df = pd.DataFrame(x, index=index, columns=[0, 1]) return DataFrameSource(df), df + + diff --git a/zipline/utils/simfactory.py b/zipline/utils/simfactory.py new file mode 100644 index 00000000..8866c2bb --- /dev/null +++ b/zipline/utils/simfactory.py @@ -0,0 +1,113 @@ +import zipline.utils.factory as factory + +from zipline.test_algorithms import TestAlgorithm +from zipline.lines import SimulatedTrading +from zipline.finance.slippage import FixedSlippage, simulate_method_factory +from zipline.finance.commission import PerShare + +def create_test_zipline(**config): + """ + :param config: A configuration object that is a dict with: + + - environment - a \ + :py:class:`zipline.finance.trading.TradingEnvironment` + - sid - an integer, which will be used as the security ID. + - order_count - the number of orders the test algo will place, + defaults to 100 + - order_amount - the number of shares per order, defaults to 100 + - trade_count - the number of trades to simulate, defaults to 101 + to ensure all orders are processed. + - algorithm - optional parameter providing an algorithm. defaults + to :py:class:`zipline.test.algorithms.TestAlgorithm` + - trade_source - optional parameter to specify trades, if present. + If not present :py:class:`zipline.sources.SpecificEquityTrades` + is the source, with daily frequency in trades. + - slippage: optional parameter that configures the + :py:class:`zipline.gens.tradingsimulation.TransactionSimulator`. Expects + an object with a simulate mehod, such as + :py:class:`zipline.gens.tradingsimulation.FixedSlippage`. + :py:mod:`zipline.finance.trading` + - transforms: optional parameter that provides a list + of StatefulTransform objects. + """ + assert isinstance(config, dict) + sid_list = config.get('sid_list') + if not sid_list: + sid = config.get('sid') + sid_list = [sid] + + concurrent_trades = config.get('concurrent_trades', False) + + #-------------------- + # Trading Environment + #-------------------- + if 'environment' in config: + trading_environment = config['environment'] + else: + trading_environment = factory.create_trading_environment() + + if 'order_count' in config: + order_count = config['order_count'] + else: + order_count = 100 + + if 'order_amount' in config: + order_amount = config['order_amount'] + else: + order_amount = 100 + + if 'trade_count' in config: + trade_count = config['trade_count'] + else: + # to ensure all orders are filled, we provide one more + # trade than order + trade_count = 101 + + slippage = config.get('slippage', FixedSlippage()) + commission = PerShare() + sim_method = simulate_method_factory(slippage, commission) + + #------------------- + # Trade Source + #------------------- + if 'trade_source' in config: + trade_source = config['trade_source'] + else: + trade_source = factory.create_daily_trade_source( + sid_list, + trade_count, + trading_environment, + concurrent=concurrent_trades + ) + + #------------------- + # Transforms + #------------------- + transforms = config.get('transforms', []) + + #------------------- + # Create the Algo + #------------------- + if 'algorithm' in config: + test_algo = config['algorithm'] + else: + test_algo = TestAlgorithm( + sid, + order_amount, + order_count + ) + + #------------------- + # Simulation + #------------------- + + sim = SimulatedTrading( + [trade_source], + transforms, + test_algo, + trading_environment, + sim_method + ) + #------------------- + + return sim From ebde40ab057202550fd2549bc0451b94a3fbc3c7 Mon Sep 17 00:00:00 2001 From: fawce Date: Fri, 28 Sep 2012 23:19:39 -0400 Subject: [PATCH 3/4] patched up tradesim from master. --- zipline/gens/tradesimulation.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/zipline/gens/tradesimulation.py b/zipline/gens/tradesimulation.py index 8287536a..b9d3ccc6 100644 --- a/zipline/gens/tradesimulation.py +++ b/zipline/gens/tradesimulation.py @@ -9,7 +9,6 @@ from zipline.utils.timeout import Heartbeat, Timeout from zipline.finance.trading import TransactionSimulator from zipline.finance.performance import PerformanceTracker -from zipline.utils.log_utils import stdout_only_pipe from zipline.gens.utils import hash_args log = Logger('Trade Simulation') @@ -53,14 +52,14 @@ class TradeSimulationClient(object): is sent to the algo. """ - def __init__(self, algo, environment, slippage): + def __init__(self, algo, environment, txn_sim): self.algo = algo self.sids = algo.get_sid_filter() self.environment = environment - self.slippage = slippage + self.txn_sim = txn_sim - self.ordering_client = TransactionSimulator(self.slippage) + self.ordering_client = TransactionSimulator(self.txn_sim) self.perf_tracker = PerformanceTracker(self.environment, self.sids) self.algo_start = self.environment.first_open @@ -132,7 +131,7 @@ class AlgorithmSimulator(object): self.algo.set_logger(self.algolog) # Porived user algorithm with slippage override. - self.algo.set_slippage_override(self.override_slippage) + self.algo.set_simulate_override(self.simulate_override) # Handler for heartbeats during calls to handle_data. def log_heartbeats(beat_count, stackframe): @@ -174,11 +173,7 @@ class AlgorithmSimulator(object): record.extra['algo_dt'] = self.snapshot_dt self.processor = Processor(inject_algo_dt) - # Single_use generator that uses the @contextmanager decorator - # to monkey patch sys.stdout with a logbook interface. - self.stdout_capture = stdout_only_pipe - - def override_slippage(self, slippage): + def simulate_override(self, slippage): self.order_book.slippage = slippage def order(self, sid, amount): From 7a8697f0e5b0493c07c384075167e4b991de2b2b Mon Sep 17 00:00:00 2001 From: fawce Date: Sun, 30 Sep 2012 22:12:00 -0400 Subject: [PATCH 4/4] updated based on PR feedback. --- tests/test_transforms.py | 4 ---- zipline/algorithm.py | 6 +++--- zipline/finance/commission.py | 26 ++++++++++++++++++++++++++ zipline/finance/slippage.py | 25 +++++++++++++++---------- zipline/finance/trading.py | 12 ++++++------ zipline/gens/tradesimulation.py | 24 +++++++++++++++--------- zipline/test_algorithms.py | 24 ++++++++++++------------ zipline/utils/simfactory.py | 6 +++--- 8 files changed, 80 insertions(+), 47 deletions(-) diff --git a/tests/test_transforms.py b/tests/test_transforms.py index 62b98b54..3e05436b 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -303,10 +303,6 @@ class BatchTransformTestCase(TestCase): # test overloaded class for test_history in [algo.history_return_price_class, algo.history_return_price_decorator]: - self.assertTrue(np.all(test_history[2].values.flatten() == range(4, 10))) - self.assertTrue(np.all(test_history[3].values.flatten() == range(4, 10))) - self.assertTrue(np.all(test_history[4].values.flatten() == range(6, 14))) - np.testing.assert_array_equal( range(4, 10), test_history[2].values.flatten() diff --git a/zipline/algorithm.py b/zipline/algorithm.py index bf696b46..23cde1d7 100644 --- a/zipline/algorithm.py +++ b/zipline/algorithm.py @@ -5,7 +5,7 @@ from zipline.gens.tradegens import DataFrameSource from zipline.utils.factory import create_trading_environment from zipline.gens.transform import StatefulTransform from zipline.lines import SimulatedTrading -from zipline.finance.slippage import FixedSlippage, simulate_method_factory +from zipline.finance.slippage import FixedSlippage, transact_partial from zipline.finance.commission import PerShare @@ -75,7 +75,7 @@ class TradingAlgorithm(object): transforms, self, environment, - simulate_method_factory(FixedSlippage(), PerShare(0.0)) + transact_partial(FixedSlippage(), PerShare(0.0)) ) def run(self, source, start=None, end=None): @@ -180,5 +180,5 @@ class TradingAlgorithm(object): def initialize(self, *args, **kwargs): pass - def set_simulate_override(self, slippage_callable): + def set_transact_setter(self, transact_setter): pass diff --git a/zipline/finance/commission.py b/zipline/finance/commission.py index 38883a65..442117ec 100644 --- a/zipline/finance/commission.py +++ b/zipline/finance/commission.py @@ -1,18 +1,44 @@ class PerShare(object): + """ + Calculates a commission for a transaction based on a per + share cost. + """ def __init__(self, cost=0.03): + """ + Cost parameter is the cost of a trade per-share. $0.03 + means three cents per share, which is a very conservative + (quite high) for per share costs. + """ self.cost = cost def calculate(self, transaction): + """ + returns a tuple of: + (per share commission, total transaction commission) + """ return self.cost, abs(transaction.amount * self.cost) class PerTrade(object): + """ + Calculates a commission for a transaction based on a per + trade cost. + """ def __init__(self, cost=5.0): + """ + Cost parameter is the cost of a trade, regardless of + share count. $5.00 per trade is fairly typical of + discount brokers. + """ self.cost = cost def calculate(self, transaction): + """ + returns a tuple of: + (per share commission, total transaction commission) + """ if transaction.amount == 0: return 0.0, 0.0 diff --git a/zipline/finance/slippage.py b/zipline/finance/slippage.py index 707fdd94..be6e4f5b 100644 --- a/zipline/finance/slippage.py +++ b/zipline/finance/slippage.py @@ -1,20 +1,25 @@ import pytz import math +from functools import partial + import zipline.protocol as zp +def transact_stub(slippage, commission, open_orders, events): + """ + This is intended to be wrapped in a partial, so that the + slippage and commission models can be enclosed. + """ + transaction = slippage.simulate(open_orders, events) + if transaction: + per_share, total_commission = commission.calculate(transaction) + transaction.price = transaction.price + per_share + transaction.commission = total_commission + return transaction -def simulate_method_factory(slippage, commission): - def simulate(open_orders, events): - transaction = slippage.simulate(open_orders, events) - if transaction: - per_share, total_commission = commission.calculate(transaction) - transaction.price = transaction.price + per_share - transaction.commission = total_commission - return transaction - - return simulate +def transact_partial(slippage, commission): + return partial(transact_stub, slippage, commission) def create_transaction(sid, amount, price, dt): diff --git a/zipline/finance/trading.py b/zipline/finance/trading.py index 4fc94aef..dc13bef9 100644 --- a/zipline/finance/trading.py +++ b/zipline/finance/trading.py @@ -7,7 +7,7 @@ from collections import defaultdict import zipline.protocol as zp from zipline.finance.slippage import ( VolumeShareSlippage, - simulate_method_factory + transact_partial ) from zipline.finance.commission import PerShare @@ -15,11 +15,11 @@ log = logbook.Logger('Transaction Simulator') class TransactionSimulator(object): - def __init__(self, simulate=None): - if simulate: - self.simulate = simulate + def __init__(self, transact=None): + if transact != None: + self.transact = transact else: - self.simulate = simulate_method_factory( + self.transact = transact_partial( VolumeShareSlippage(), PerShare() ) @@ -43,7 +43,7 @@ class TransactionSimulator(object): event.TRANSACTION = None # We only fill transactions on trade events. if event.type == zp.DATASOURCE_TYPE.TRADE: - event.TRANSACTION = self.simulate(event, self.open_orders) + event.TRANSACTION = self.transact(event, self.open_orders) return event diff --git a/zipline/gens/tradesimulation.py b/zipline/gens/tradesimulation.py index b9d3ccc6..70e067d1 100644 --- a/zipline/gens/tradesimulation.py +++ b/zipline/gens/tradesimulation.py @@ -52,14 +52,14 @@ class TradeSimulationClient(object): is sent to the algo. """ - def __init__(self, algo, environment, txn_sim): + def __init__(self, algo, environment, transact): self.algo = algo self.sids = algo.get_sid_filter() self.environment = environment - self.txn_sim = txn_sim + self.transact = transact - self.ordering_client = TransactionSimulator(self.txn_sim) + self.ordering_client = TransactionSimulator(self.transact) self.perf_tracker = PerformanceTracker(self.environment, self.sids) self.algo_start = self.environment.first_open @@ -130,8 +130,10 @@ class AlgorithmSimulator(object): self.algolog = Logger("AlgoLog") self.algo.set_logger(self.algolog) - # Porived user algorithm with slippage override. - self.algo.set_simulate_override(self.simulate_override) + # Provide user algorithm with a setter for the transact + # method (method that constructs transactions based on + # open orders and trade events). + self.algo.set_transact_setter(self.set_transact) # Handler for heartbeats during calls to handle_data. def log_heartbeats(beat_count, stackframe): @@ -173,13 +175,17 @@ class AlgorithmSimulator(object): record.extra['algo_dt'] = self.snapshot_dt self.processor = Processor(inject_algo_dt) - def simulate_override(self, slippage): - self.order_book.slippage = slippage + def set_transact(self, transact): + """ + Set the method that will be called to create a + transaction from open orders and trade events. + """ + self.order_book.transact = transact def order(self, sid, amount): """ Closure to pass into the user's algo to allow placing orders - into the txn_sim's dict of open orders. + into the transaction simulator's dict of open orders. """ assert sid in self.sids, "Order on invalid sid: %i" % sid order = ndict({ @@ -230,7 +236,7 @@ class AlgorithmSimulator(object): if date == 'DONE': for event in snapshot: yield event.perf_message - raise StopIteration() + raise StopIteration # We're still in the warmup period. Use the event to # update our universe, but don't yield any perf messages, diff --git a/zipline/test_algorithms.py b/zipline/test_algorithms.py index 6974b248..ab54b15a 100644 --- a/zipline/test_algorithms.py +++ b/zipline/test_algorithms.py @@ -44,8 +44,8 @@ The algorithm must expose methods: self.Portfolio[sid(133)]['cost_basis'] - - set_simulate_override: method that accepts a callable. Will - be set as the value of the set_simulate_override method of + - set_transact_setter: method that accepts a callable. Will + be set as the value of the set_transact_setter method of the trading_client. This allows an algorithm to change the slippage model used to predict transactions based on orders and trade events. @@ -97,7 +97,7 @@ class TestAlgorithm(): def get_sid_filter(self): return self.sid_filter - def set_simulate_override(self, txn_sim_callable): + def set_transact_setter(self, txn_sim_callable): pass @@ -138,7 +138,7 @@ class HeavyBuyAlgorithm(): def get_sid_filter(self): return [self.sid] - def set_simulate_override(self, txn_sim_callable): + def set_transact_setter(self, txn_sim_callable): pass class NoopAlgorithm(object): @@ -164,7 +164,7 @@ class NoopAlgorithm(object): def get_sid_filter(self): return [] - def set_simulate_override(self, txn_sim_callable): + def set_transact_setter(self, txn_sim_callable): pass class ExceptionAlgorithm(object): @@ -210,7 +210,7 @@ class ExceptionAlgorithm(object): else: return [self.sid] - def set_simulate_override(self, txn_sim_callable): + def set_transact_setter(self, txn_sim_callable): pass class DivByZeroAlgorithm(): @@ -240,7 +240,7 @@ class DivByZeroAlgorithm(): def get_sid_filter(self): return [self.sid] - def set_simulate_override(self, txn_sim_callable): + def set_transact_setter(self, txn_sim_callable): pass class InitializeTimeoutAlgorithm(): @@ -269,7 +269,7 @@ class InitializeTimeoutAlgorithm(): def get_sid_filter(self): return [self.sid] - def set_simulate_override(self, txn_sim_callable): + def set_transact_setter(self, txn_sim_callable): pass class TooMuchProcessingAlgorithm(): @@ -297,7 +297,7 @@ class TooMuchProcessingAlgorithm(): def get_sid_filter(self): return [self.sid] - def set_simulate_override(self, txn_sim_callable): + def set_transact_setter(self, txn_sim_callable): pass class TimeoutAlgorithm(): @@ -327,7 +327,7 @@ class TimeoutAlgorithm(): def get_sid_filter(self): return [self.sid] - def set_simulate_override(self, txn_sim_callable): + def set_transact_setter(self, txn_sim_callable): pass class TestPrintAlgorithm(): @@ -354,7 +354,7 @@ class TestPrintAlgorithm(): def get_sid_filter(self): return [self.sid] - def set_simulate_override(self, txn_sim_callable): + def set_transact_setter(self, txn_sim_callable): pass class TestLoggingAlgorithm(): @@ -381,7 +381,7 @@ class TestLoggingAlgorithm(): def get_sid_filter(self): return [self.sid] - def set_simulate_override(self, txn_sim_callable): + def set_transact_setter(self, txn_sim_callable): pass diff --git a/zipline/utils/simfactory.py b/zipline/utils/simfactory.py index 8866c2bb..ba2d5abb 100644 --- a/zipline/utils/simfactory.py +++ b/zipline/utils/simfactory.py @@ -2,7 +2,7 @@ import zipline.utils.factory as factory from zipline.test_algorithms import TestAlgorithm from zipline.lines import SimulatedTrading -from zipline.finance.slippage import FixedSlippage, simulate_method_factory +from zipline.finance.slippage import FixedSlippage, transact_partial from zipline.finance.commission import PerShare def create_test_zipline(**config): @@ -65,7 +65,7 @@ def create_test_zipline(**config): slippage = config.get('slippage', FixedSlippage()) commission = PerShare() - sim_method = simulate_method_factory(slippage, commission) + transact_method = transact_partial(slippage, commission) #------------------- # Trade Source @@ -106,7 +106,7 @@ def create_test_zipline(**config): transforms, test_algo, trading_environment, - sim_method + transact_method ) #-------------------