diff --git a/tests/test_exception_handling.py b/tests/test_exception_handling.py index be8117ce..f950dd07 100644 --- a/tests/test_exception_handling.py +++ b/tests/test_exception_handling.py @@ -2,6 +2,7 @@ from unittest2 import TestCase from collections import defaultdict +import zipline.utils.simfactory as simfactory from zipline.test_algorithms import ExceptionAlgorithm, DivByZeroAlgorithm, \ InitializeTimeoutAlgorithm, TooMuchProcessingAlgorithm from zipline.finance.slippage import FixedSlippage @@ -9,6 +10,7 @@ from zipline.lines import SimulatedTrading from zipline.gens.transform import StatefulTransform from zipline.utils.timeout import TimeoutException + from zipline.utils.test_utils import ( drain_zipline, setup_logger, @@ -37,7 +39,7 @@ class ExceptionTestCase(TestCase): def test_datasource_exception(self): self.zipline_test_config['trade_source'] = ExceptionSource() - zipline = SimulatedTrading.create_test_zipline( + zipline = simfactory.create_test_zipline( **self.zipline_test_config ) @@ -54,7 +56,7 @@ class ExceptionTestCase(TestCase): exc_tnfm = StatefulTransform(ExceptionTransform) self.zipline_test_config['transforms'] = [exc_tnfm] - zipline = SimulatedTrading.create_test_zipline( + zipline = simfactory.create_test_zipline( **self.zipline_test_config ) @@ -72,7 +74,7 @@ class ExceptionTestCase(TestCase): self.zipline_test_config['sid'] ) - zipline = SimulatedTrading.create_test_zipline( + zipline = simfactory.create_test_zipline( **self.zipline_test_config ) @@ -90,7 +92,7 @@ class ExceptionTestCase(TestCase): self.zipline_test_config['sid'] ) - zipline = SimulatedTrading.create_test_zipline( + zipline = simfactory.create_test_zipline( **self.zipline_test_config ) @@ -108,7 +110,7 @@ class ExceptionTestCase(TestCase): self.zipline_test_config['sid'] ) - zipline = SimulatedTrading.create_test_zipline( + zipline = simfactory.create_test_zipline( **self.zipline_test_config ) @@ -125,7 +127,7 @@ class ExceptionTestCase(TestCase): self.zipline_test_config['sid'] ) - zipline = SimulatedTrading.create_test_zipline( + zipline = simfactory.create_test_zipline( **self.zipline_test_config ) @@ -140,7 +142,7 @@ class ExceptionTestCase(TestCase): TooMuchProcessingAlgorithm( self.zipline_test_config['sid'] ) - zipline = SimulatedTrading.create_test_zipline( + zipline = simfactory.create_test_zipline( **self.zipline_test_config ) diff --git a/tests/test_finance.py b/tests/test_finance.py index 8d3ab408..d7f15943 100644 --- a/tests/test_finance.py +++ b/tests/test_finance.py @@ -10,6 +10,7 @@ from collections import defaultdict from nose.tools import timed import zipline.utils.factory as factory +import zipline.utils.simfactory as simfactory from zipline.finance.trading import TradingEnvironment from zipline.lines import SimulatedTrading @@ -111,7 +112,7 @@ class FinanceTestCase(TestCase): #provide enough trades to ensure all orders are filled. self.zipline_test_config['order_count'] = 100 self.zipline_test_config['trade_count'] = 200 - zipline = SimulatedTrading.create_test_zipline(**self.zipline_test_config) + zipline = simfactory.create_test_zipline(**self.zipline_test_config) assert_single_position(self, zipline) # TODO: write tests for short sales diff --git a/tests/test_transforms.py b/tests/test_transforms.py index 4f6cae0f..62b98b54 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -307,6 +307,21 @@ class BatchTransformTestCase(TestCase): self.assertTrue(np.all(test_history[3].values.flatten() == range(4, 10))) self.assertTrue(np.all(test_history[4].values.flatten() == range(6, 14))) + np.testing.assert_array_equal( + range(4, 10), + test_history[2].values.flatten() + ) + + np.testing.assert_array_equal( + range(4, 10), + test_history[3].values.flatten() + ) + + np.testing.assert_array_equal( + range(6, 14), + test_history[4].values.flatten() + ) + def test_passing_of_args(self): algo = BatchTransformAlgorithm([0, 1], 1, kwarg='str') self.assertEqual(algo.args, (1,)) diff --git a/zipline/algorithm.py b/zipline/algorithm.py index f54ba0cb..bf696b46 100644 --- a/zipline/algorithm.py +++ b/zipline/algorithm.py @@ -5,7 +5,8 @@ from zipline.gens.tradegens import DataFrameSource from zipline.utils.factory import create_trading_environment from zipline.gens.transform import StatefulTransform from zipline.lines import SimulatedTrading -from zipline.finance.slippage import FixedSlippage +from zipline.finance.slippage import FixedSlippage, simulate_method_factory +from zipline.finance.commission import PerShare class TradingAlgorithm(object): @@ -74,7 +75,7 @@ class TradingAlgorithm(object): transforms, self, environment, - FixedSlippage() + simulate_method_factory(FixedSlippage(), PerShare(0.0)) ) def run(self, source, start=None, end=None): @@ -179,8 +180,5 @@ class TradingAlgorithm(object): def initialize(self, *args, **kwargs): pass - def set_slippage_override(self, slippage_callable): + def set_simulate_override(self, slippage_callable): pass - - - diff --git a/zipline/finance/commissions.py b/zipline/finance/commission.py similarity index 85% rename from zipline/finance/commissions.py rename to zipline/finance/commission.py index 63ae2962..38883a65 100644 --- a/zipline/finance/commissions.py +++ b/zipline/finance/commission.py @@ -1,4 +1,4 @@ -class PerShareCommission(object): +class PerShare(object): def __init__(self, cost=0.03): self.cost = cost @@ -7,7 +7,7 @@ class PerShareCommission(object): def calculate(self, transaction): return self.cost, abs(transaction.amount * self.cost) -class PerTradeCommission(object): +class PerTrade(object): def __init__(self, cost=5.0): self.cost = cost diff --git a/zipline/finance/slippage.py b/zipline/finance/slippage.py index a3c8ada1..707fdd94 100644 --- a/zipline/finance/slippage.py +++ b/zipline/finance/slippage.py @@ -3,32 +3,39 @@ import math import zipline.protocol as zp -from zipline.finance.commissions import PerShareCommission -def create_transaction(sid, amount, price, dt, commission): +def simulate_method_factory(slippage, commission): + + def simulate(open_orders, events): + transaction = slippage.simulate(open_orders, events) + if transaction: + per_share, total_commission = commission.calculate(transaction) + transaction.price = transaction.price + per_share + transaction.commission = total_commission + return transaction + + return simulate + +def create_transaction(sid, amount, price, dt): txn = {'sid' : sid, - 'amount' : int(amount), - 'dt' : dt, - 'price' : price, + 'amount' : int(amount), + 'dt' : dt, + 'price' : price, } transaction = zp.ndict(txn) - per_share, total_commission = commission.calculate(transaction) - transaction.price = transaction.price + per_share - transaction.commission = total_commission return transaction + class VolumeShareSlippage(object): def __init__(self, volume_limit=.25, - price_impact=0.1, - commission=PerShareCommission()): + price_impact=0.1): self.volume_limit = volume_limit self.price_impact = price_impact - self.commission = commission def simulate(self, event, open_orders): @@ -90,19 +97,16 @@ class VolumeShareSlippage(object): simulated_amount, event.price + simulated_impact, dt.replace(tzinfo = pytz.utc), - self.commission ) class FixedSlippage(object): - def __init__(self, spread=0.0, commission=PerShareCommission()): + def __init__(self, spread=0.0): """ Use the fixed slippage model, which will just add/subtract a specified spread spread/2 will be added on buys and subtracted on sells per share - commission will be charged per share """ self.spread = spread - self.commission = commission def simulate(self, event, open_orders): if event.sid in open_orders: @@ -124,8 +128,7 @@ class FixedSlippage(object): event.sid, amount, event.price + (self.spread/2.0 * direction), - event.dt, - self.commission + event.dt ) open_orders[event.sid] = [] diff --git a/zipline/finance/trading.py b/zipline/finance/trading.py index 4258a4ff..4fc94aef 100644 --- a/zipline/finance/trading.py +++ b/zipline/finance/trading.py @@ -5,18 +5,24 @@ import datetime from collections import defaultdict import zipline.protocol as zp -from zipline.finance.slippage import VolumeShareSlippage, FixedSlippage +from zipline.finance.slippage import ( + VolumeShareSlippage, + simulate_method_factory + ) +from zipline.finance.commission import PerShare log = logbook.Logger('Transaction Simulator') class TransactionSimulator(object): - def __init__(self, slippage=None): - if slippage: - assert isinstance(slippage, (VolumeShareSlippage, FixedSlippage)) - self.slippage = slippage + def __init__(self, simulate=None): + if simulate: + self.simulate = simulate else: - self.slippage = VolumeShareSlippage() + self.simulate = simulate_method_factory( + VolumeShareSlippage(), + PerShare() + ) self.open_orders = defaultdict(list) @@ -37,7 +43,7 @@ class TransactionSimulator(object): event.TRANSACTION = None # We only fill transactions on trade events. if event.type == zp.DATASOURCE_TYPE.TRADE: - event.TRANSACTION = self.slippage.simulate(event, self.open_orders) + event.TRANSACTION = self.simulate(event, self.open_orders) return event diff --git a/zipline/gens/tradesimulation.py b/zipline/gens/tradesimulation.py index 5b5cec52..8287536a 100644 --- a/zipline/gens/tradesimulation.py +++ b/zipline/gens/tradesimulation.py @@ -214,66 +214,64 @@ class AlgorithmSimulator(object): """ Main generator work loop. """ - # Capture any output of this generator to stdout and pipe it - # to a logbook interface. Also inject the current algo - # snapshot time to any log record generated. - #with self.processor.threadbound(), self.stdout_capture(Logger('Print'),''): - # Call user's initialize method with a timeout (only if # initialize wasn't called already). if not getattr(self.algo, 'initialized', False): with Timeout(INIT_TIMEOUT, message="Call to initialize timed out"): self.algo.initialize() - # Group together events with the same dt field. This depends on the - # events already being sorted. - for date, snapshot in groupby(stream_in, attrgetter('dt')): - # Set the simulation date to be the first event we see. - # This should only occur once, at the start of the test. - if self.simulation_dt == None: - self.simulation_dt = date + # inject the current algo + # snapshot time to any log record generated. + with self.processor.threadbound(): + # Group together events with the same dt field. This depends on the + # events already being sorted. + for date, snapshot in groupby(stream_in, attrgetter('dt')): + # Set the simulation date to be the first event we see. + # This should only occur once, at the start of the test. + if self.simulation_dt == None: + self.simulation_dt = date - # Done message has the risk report, so we yield before exiting. - if date == 'DONE': - for event in snapshot: - yield event.perf_message - raise StopIteration() - - # We're still in the warmup period. Use the event to - # update our universe, but don't yield any perf messages, - # and don't send a snapshot to handle_data. - elif date < self.algo_start: - for event in snapshot: - del event['perf_message'] - self.update_universe(event) - - # The algo has taken so long to process events that - # its simulated time is later than the event time. - # Update the universe and yield any perf messages - # encountered, but don't call handle_data. - elif date < self.simulation_dt: - for event in snapshot: - # Only yield if we have something interesting to say. - if event.perf_message != None: + # Done message has the risk report, so we yield before exiting. + if date == 'DONE': + for event in snapshot: yield event.perf_message - # Delete the message before updating so we don't send it - # to the user. - del event['perf_message'] - self.update_universe(event) + raise StopIteration() - # Regular snapshot. Update the universe and send a snapshot - # to handle data. - else: - for event in snapshot: - # Only yield if we have something interesting to say. - if event.perf_message != None: - yield event.perf_message - del event['perf_message'] + # We're still in the warmup period. Use the event to + # update our universe, but don't yield any perf messages, + # and don't send a snapshot to handle_data. + elif date < self.algo_start: + for event in snapshot: + del event['perf_message'] + self.update_universe(event) - self.update_universe(event) + # The algo has taken so long to process events that + # its simulated time is later than the event time. + # Update the universe and yield any perf messages + # encountered, but don't call handle_data. + elif date < self.simulation_dt: + for event in snapshot: + # Only yield if we have something interesting to say. + if event.perf_message != None: + yield event.perf_message + # Delete the message before updating so we don't send it + # to the user. + del event['perf_message'] + self.update_universe(event) - # Send the current state of the universe to the user's algo. - self.simulate_snapshot(date) + # Regular snapshot. Update the universe and send a snapshot + # to handle data. + else: + for event in snapshot: + # Only yield if we have something interesting to say. + if event.perf_message != None: + yield event.perf_message + del event['perf_message'] + + self.update_universe(event) + + # Send the current state of the universe to the user's algo. + self.simulate_snapshot(date) def update_universe(self, event): """ diff --git a/zipline/lines.py b/zipline/lines.py index a8fedc56..e29d8cfa 100644 --- a/zipline/lines.py +++ b/zipline/lines.py @@ -61,13 +61,11 @@ before invoking simulate. """ from zipline.utils import factory - from zipline.gens.composites import ( date_sorted_sources, sequential_transforms ) from zipline.gens.tradesimulation import TradeSimulationClient as tsc -from zipline.finance.slippage import FixedSlippage from logbook import Logger @@ -81,7 +79,7 @@ class SimulatedTrading(object): transforms, algorithm, environment, - slippage): + sim_method=None): """ @sources - an iterable of iterables These iterables must yield ndicts that contain: @@ -108,7 +106,7 @@ class SimulatedTrading(object): # Formerly merged_transforms. self.with_tnfms = sequential_transforms(self.date_sorted, *self.transforms) - self.trading_client = tsc(algorithm, environment, slippage) + self.trading_client = tsc(algorithm, environment, sim_method) self.gen = self.trading_client.simulate(self.with_tnfms) def __iter__(self): @@ -116,111 +114,3 @@ class SimulatedTrading(object): def next(self): return self.gen.next() - - @staticmethod - def create_test_zipline(**config): - """ - :param config: A configuration object that is a dict with: - - - environment - a \ - :py:class:`zipline.finance.trading.TradingEnvironment` - - sid - an integer, which will be used as the security ID. - - order_count - the number of orders the test algo will place, - defaults to 100 - - order_amount - the number of shares per order, defaults to 100 - - trade_count - the number of trades to simulate, defaults to 101 - to ensure all orders are processed. - - algorithm - optional parameter providing an algorithm. defaults - to :py:class:`zipline.test.algorithms.TestAlgorithm` - - trade_source - optional parameter to specify trades, if present. - If not present :py:class:`zipline.sources.SpecificEquityTrades` - is the source, with daily frequency in trades. - - slippage: optional parameter that configures the - :py:class:`zipline.gens.tradingsimulation.TransactionSimulator`. Expects - an object with a simulate mehod, such as - :py:class:`zipline.gens.tradingsimulation.FixedSlippage`. - :py:mod:`zipline.finance.trading` - - transforms: optional parameter that provides a list - of StatefulTransform objects. - """ - from zipline.test_algorithms import TestAlgorithm - - assert isinstance(config, dict) - sid_list = config.get('sid_list') - if not sid_list: - sid = config.get('sid') - sid_list = [sid] - - concurrent_trades = config.get('concurrent_trades', False) - - #-------------------- - # Trading Environment - #-------------------- - if 'environment' in config: - trading_environment = config['environment'] - else: - trading_environment = factory.create_trading_environment() - - if 'order_count' in config: - order_count = config['order_count'] - else: - order_count = 100 - - if 'order_amount' in config: - order_amount = config['order_amount'] - else: - order_amount = 100 - - if 'trade_count' in config: - trade_count = config['trade_count'] - else: - # to ensure all orders are filled, we provide one more - # trade than order - trade_count = 101 - - slippage = config.get('slippage', FixedSlippage()) - - #------------------- - # Trade Source - #------------------- - if 'trade_source' in config: - trade_source = config['trade_source'] - else: - trade_source = factory.create_daily_trade_source( - sid_list, - trade_count, - trading_environment, - concurrent=concurrent_trades - ) - - #------------------- - # Transforms - #------------------- - transforms = config.get('transforms', []) - - #------------------- - # Create the Algo - #------------------- - if 'algorithm' in config: - test_algo = config['algorithm'] - else: - test_algo = TestAlgorithm( - sid, - order_amount, - order_count - ) - - #------------------- - # Simulation - #------------------- - - sim = SimulatedTrading( - [trade_source], - transforms, - test_algo, - trading_environment, - slippage, - ) - #------------------- - - return sim diff --git a/zipline/test_algorithms.py b/zipline/test_algorithms.py index a8bc6ab4..6974b248 100644 --- a/zipline/test_algorithms.py +++ b/zipline/test_algorithms.py @@ -44,8 +44,8 @@ The algorithm must expose methods: self.Portfolio[sid(133)]['cost_basis'] - - set_slippage_override: method that accepts a callable. Will - be set as the value of the set_slippage_override method of + - set_simulate_override: method that accepts a callable. Will + be set as the value of the set_simulate_override method of the trading_client. This allows an algorithm to change the slippage model used to predict transactions based on orders and trade events. @@ -97,7 +97,7 @@ class TestAlgorithm(): def get_sid_filter(self): return self.sid_filter - def set_slippage_override(self, slippage_callable): + def set_simulate_override(self, txn_sim_callable): pass @@ -138,7 +138,7 @@ class HeavyBuyAlgorithm(): def get_sid_filter(self): return [self.sid] - def set_slippage_override(self, slippage_callable): + def set_simulate_override(self, txn_sim_callable): pass class NoopAlgorithm(object): @@ -164,7 +164,7 @@ class NoopAlgorithm(object): def get_sid_filter(self): return [] - def set_slippage_override(self, slippage_callable): + def set_simulate_override(self, txn_sim_callable): pass class ExceptionAlgorithm(object): @@ -210,7 +210,7 @@ class ExceptionAlgorithm(object): else: return [self.sid] - def set_slippage_override(self, slippage_callable): + def set_simulate_override(self, txn_sim_callable): pass class DivByZeroAlgorithm(): @@ -240,7 +240,7 @@ class DivByZeroAlgorithm(): def get_sid_filter(self): return [self.sid] - def set_slippage_override(self, slippage_callable): + def set_simulate_override(self, txn_sim_callable): pass class InitializeTimeoutAlgorithm(): @@ -269,7 +269,7 @@ class InitializeTimeoutAlgorithm(): def get_sid_filter(self): return [self.sid] - def set_slippage_override(self, slippage_callable): + def set_simulate_override(self, txn_sim_callable): pass class TooMuchProcessingAlgorithm(): @@ -297,7 +297,7 @@ class TooMuchProcessingAlgorithm(): def get_sid_filter(self): return [self.sid] - def set_slippage_override(self, slippage_callable): + def set_simulate_override(self, txn_sim_callable): pass class TimeoutAlgorithm(): @@ -327,7 +327,7 @@ class TimeoutAlgorithm(): def get_sid_filter(self): return [self.sid] - def set_slippage_override(self, slippage_callable): + def set_simulate_override(self, txn_sim_callable): pass class TestPrintAlgorithm(): @@ -354,7 +354,7 @@ class TestPrintAlgorithm(): def get_sid_filter(self): return [self.sid] - def set_slippage_override(self, slippage_callable): + def set_simulate_override(self, txn_sim_callable): pass class TestLoggingAlgorithm(): @@ -381,7 +381,7 @@ class TestLoggingAlgorithm(): def get_sid_filter(self): return [self.sid] - def set_slippage_override(self, slippage_callable): + def set_simulate_override(self, txn_sim_callable): pass @@ -447,5 +447,3 @@ class BatchTransformAlgorithm(TradingAlgorithm): self.history_return_price_class.append(self.return_price_class.handle_data(data)) self.history_return_price_decorator.append(self.return_price_decorator.handle_data(data)) self.history_return_args.append(self.return_args_batch.handle_data(data, *self.args, **self.kwargs)) - - diff --git a/zipline/utils/factory.py b/zipline/utils/factory.py index 7517fff5..c8a2311b 100644 --- a/zipline/utils/factory.py +++ b/zipline/utils/factory.py @@ -244,3 +244,5 @@ def create_test_df_source(): df = pd.DataFrame(x, index=index, columns=[0, 1]) return DataFrameSource(df), df + + diff --git a/zipline/utils/simfactory.py b/zipline/utils/simfactory.py new file mode 100644 index 00000000..8866c2bb --- /dev/null +++ b/zipline/utils/simfactory.py @@ -0,0 +1,113 @@ +import zipline.utils.factory as factory + +from zipline.test_algorithms import TestAlgorithm +from zipline.lines import SimulatedTrading +from zipline.finance.slippage import FixedSlippage, simulate_method_factory +from zipline.finance.commission import PerShare + +def create_test_zipline(**config): + """ + :param config: A configuration object that is a dict with: + + - environment - a \ + :py:class:`zipline.finance.trading.TradingEnvironment` + - sid - an integer, which will be used as the security ID. + - order_count - the number of orders the test algo will place, + defaults to 100 + - order_amount - the number of shares per order, defaults to 100 + - trade_count - the number of trades to simulate, defaults to 101 + to ensure all orders are processed. + - algorithm - optional parameter providing an algorithm. defaults + to :py:class:`zipline.test.algorithms.TestAlgorithm` + - trade_source - optional parameter to specify trades, if present. + If not present :py:class:`zipline.sources.SpecificEquityTrades` + is the source, with daily frequency in trades. + - slippage: optional parameter that configures the + :py:class:`zipline.gens.tradingsimulation.TransactionSimulator`. Expects + an object with a simulate mehod, such as + :py:class:`zipline.gens.tradingsimulation.FixedSlippage`. + :py:mod:`zipline.finance.trading` + - transforms: optional parameter that provides a list + of StatefulTransform objects. + """ + assert isinstance(config, dict) + sid_list = config.get('sid_list') + if not sid_list: + sid = config.get('sid') + sid_list = [sid] + + concurrent_trades = config.get('concurrent_trades', False) + + #-------------------- + # Trading Environment + #-------------------- + if 'environment' in config: + trading_environment = config['environment'] + else: + trading_environment = factory.create_trading_environment() + + if 'order_count' in config: + order_count = config['order_count'] + else: + order_count = 100 + + if 'order_amount' in config: + order_amount = config['order_amount'] + else: + order_amount = 100 + + if 'trade_count' in config: + trade_count = config['trade_count'] + else: + # to ensure all orders are filled, we provide one more + # trade than order + trade_count = 101 + + slippage = config.get('slippage', FixedSlippage()) + commission = PerShare() + sim_method = simulate_method_factory(slippage, commission) + + #------------------- + # Trade Source + #------------------- + if 'trade_source' in config: + trade_source = config['trade_source'] + else: + trade_source = factory.create_daily_trade_source( + sid_list, + trade_count, + trading_environment, + concurrent=concurrent_trades + ) + + #------------------- + # Transforms + #------------------- + transforms = config.get('transforms', []) + + #------------------- + # Create the Algo + #------------------- + if 'algorithm' in config: + test_algo = config['algorithm'] + else: + test_algo = TestAlgorithm( + sid, + order_amount, + order_count + ) + + #------------------- + # Simulation + #------------------- + + sim = SimulatedTrading( + [trade_source], + transforms, + test_algo, + trading_environment, + sim_method + ) + #------------------- + + return sim