Merge pull request #124 from quantopian/commissions

Independent Commissions
This commit is contained in:
fawce
2012-09-30 19:47:37 -07:00
12 changed files with 308 additions and 230 deletions
+9 -7
View File
@@ -2,6 +2,7 @@
from unittest2 import TestCase
from collections import defaultdict
import zipline.utils.simfactory as simfactory
from zipline.test_algorithms import ExceptionAlgorithm, DivByZeroAlgorithm, \
InitializeTimeoutAlgorithm, TooMuchProcessingAlgorithm
from zipline.finance.slippage import FixedSlippage
@@ -9,6 +10,7 @@ from zipline.lines import SimulatedTrading
from zipline.gens.transform import StatefulTransform
from zipline.utils.timeout import TimeoutException
from zipline.utils.test_utils import (
drain_zipline,
setup_logger,
@@ -37,7 +39,7 @@ class ExceptionTestCase(TestCase):
def test_datasource_exception(self):
self.zipline_test_config['trade_source'] = ExceptionSource()
zipline = SimulatedTrading.create_test_zipline(
zipline = simfactory.create_test_zipline(
**self.zipline_test_config
)
@@ -54,7 +56,7 @@ class ExceptionTestCase(TestCase):
exc_tnfm = StatefulTransform(ExceptionTransform)
self.zipline_test_config['transforms'] = [exc_tnfm]
zipline = SimulatedTrading.create_test_zipline(
zipline = simfactory.create_test_zipline(
**self.zipline_test_config
)
@@ -72,7 +74,7 @@ class ExceptionTestCase(TestCase):
self.zipline_test_config['sid']
)
zipline = SimulatedTrading.create_test_zipline(
zipline = simfactory.create_test_zipline(
**self.zipline_test_config
)
@@ -90,7 +92,7 @@ class ExceptionTestCase(TestCase):
self.zipline_test_config['sid']
)
zipline = SimulatedTrading.create_test_zipline(
zipline = simfactory.create_test_zipline(
**self.zipline_test_config
)
@@ -108,7 +110,7 @@ class ExceptionTestCase(TestCase):
self.zipline_test_config['sid']
)
zipline = SimulatedTrading.create_test_zipline(
zipline = simfactory.create_test_zipline(
**self.zipline_test_config
)
@@ -125,7 +127,7 @@ class ExceptionTestCase(TestCase):
self.zipline_test_config['sid']
)
zipline = SimulatedTrading.create_test_zipline(
zipline = simfactory.create_test_zipline(
**self.zipline_test_config
)
@@ -140,7 +142,7 @@ class ExceptionTestCase(TestCase):
TooMuchProcessingAlgorithm(
self.zipline_test_config['sid']
)
zipline = SimulatedTrading.create_test_zipline(
zipline = simfactory.create_test_zipline(
**self.zipline_test_config
)
+2 -1
View File
@@ -10,6 +10,7 @@ from collections import defaultdict
from nose.tools import timed
import zipline.utils.factory as factory
import zipline.utils.simfactory as simfactory
from zipline.finance.trading import TradingEnvironment
from zipline.lines import SimulatedTrading
@@ -111,7 +112,7 @@ class FinanceTestCase(TestCase):
#provide enough trades to ensure all orders are filled.
self.zipline_test_config['order_count'] = 100
self.zipline_test_config['trade_count'] = 200
zipline = SimulatedTrading.create_test_zipline(**self.zipline_test_config)
zipline = simfactory.create_test_zipline(**self.zipline_test_config)
assert_single_position(self, zipline)
# TODO: write tests for short sales
+14 -3
View File
@@ -303,9 +303,20 @@ class BatchTransformTestCase(TestCase):
# test overloaded class
for test_history in [algo.history_return_price_class, algo.history_return_price_decorator]:
self.assertTrue(np.all(test_history[2].values.flatten() == range(4, 10)))
self.assertTrue(np.all(test_history[3].values.flatten() == range(4, 10)))
self.assertTrue(np.all(test_history[4].values.flatten() == range(6, 14)))
np.testing.assert_array_equal(
range(4, 10),
test_history[2].values.flatten()
)
np.testing.assert_array_equal(
range(4, 10),
test_history[3].values.flatten()
)
np.testing.assert_array_equal(
range(6, 14),
test_history[4].values.flatten()
)
def test_passing_of_args(self):
algo = BatchTransformAlgorithm([0, 1], 1, kwarg='str')
+4 -6
View File
@@ -5,7 +5,8 @@ from zipline.gens.tradegens import DataFrameSource
from zipline.utils.factory import create_trading_environment
from zipline.gens.transform import StatefulTransform
from zipline.lines import SimulatedTrading
from zipline.finance.slippage import FixedSlippage
from zipline.finance.slippage import FixedSlippage, transact_partial
from zipline.finance.commission import PerShare
class TradingAlgorithm(object):
@@ -74,7 +75,7 @@ class TradingAlgorithm(object):
transforms,
self,
environment,
FixedSlippage()
transact_partial(FixedSlippage(), PerShare(0.0))
)
def run(self, source, start=None, end=None):
@@ -179,8 +180,5 @@ class TradingAlgorithm(object):
def initialize(self, *args, **kwargs):
pass
def set_slippage_override(self, slippage_callable):
def set_transact_setter(self, transact_setter):
pass
+45
View File
@@ -0,0 +1,45 @@
class PerShare(object):
"""
Calculates a commission for a transaction based on a per
share cost.
"""
def __init__(self, cost=0.03):
"""
Cost parameter is the cost of a trade per-share. $0.03
means three cents per share, which is a very conservative
(quite high) for per share costs.
"""
self.cost = cost
def calculate(self, transaction):
"""
returns a tuple of:
(per share commission, total transaction commission)
"""
return self.cost, abs(transaction.amount * self.cost)
class PerTrade(object):
"""
Calculates a commission for a transaction based on a per
trade cost.
"""
def __init__(self, cost=5.0):
"""
Cost parameter is the cost of a trade, regardless of
share count. $5.00 per trade is fairly typical of
discount brokers.
"""
self.cost = cost
def calculate(self, transaction):
"""
returns a tuple of:
(per share commission, total transaction commission)
"""
if transaction.amount == 0:
return 0.0, 0.0
return abs(self.cost / transaction.amount), self.cost
+31 -18
View File
@@ -1,27 +1,46 @@
import pytz
import math
from datetime import timedelta
from functools import partial
import zipline.protocol as zp
def create_transaction(sid, amount, price, dt, direction, commission):
def transact_stub(slippage, commission, open_orders, events):
"""
This is intended to be wrapped in a partial, so that the
slippage and commission models can be enclosed.
"""
transaction = slippage.simulate(open_orders, events)
if transaction:
per_share, total_commission = commission.calculate(transaction)
transaction.price = transaction.price + per_share
transaction.commission = total_commission
return transaction
def transact_partial(slippage, commission):
return partial(transact_stub, slippage, commission)
def create_transaction(sid, amount, price, dt):
txn = {'sid' : sid,
'amount' : int(amount),
'dt' : dt,
'price' : price,
'commission' : commission * amount * direction
'amount' : int(amount),
'dt' : dt,
'price' : price,
}
return zp.ndict(txn)
transaction = zp.ndict(txn)
return transaction
class VolumeShareSlippage(object):
def __init__(self,
volume_limit=.25,
price_impact=0.1,
commission=0.03):
price_impact=0.1):
self.volume_limit = volume_limit
self.price_impact = price_impact
self.commission = commission
def simulate(self, event, open_orders):
@@ -83,20 +102,16 @@ class VolumeShareSlippage(object):
simulated_amount,
event.price + simulated_impact,
dt.replace(tzinfo = pytz.utc),
direction,
self.commission
)
class FixedSlippage(object):
def __init__(self, spread=0.0, commission=0.0):
def __init__(self, spread=0.0):
"""
Use the fixed slippage model, which will just add/subtract a specified spread
spread/2 will be added on buys and subtracted on sells per share
commission will be charged per share
"""
self.spread = spread
self.commission = commission
def simulate(self, event, open_orders):
if event.sid in open_orders:
@@ -118,9 +133,7 @@ class FixedSlippage(object):
event.sid,
amount,
event.price + (self.spread/2.0 * direction),
event.dt,
direction,
self.commission
event.dt
)
open_orders[event.sid] = []
+13 -7
View File
@@ -5,18 +5,24 @@ import datetime
from collections import defaultdict
import zipline.protocol as zp
from zipline.finance.slippage import VolumeShareSlippage, FixedSlippage
from zipline.finance.slippage import (
VolumeShareSlippage,
transact_partial
)
from zipline.finance.commission import PerShare
log = logbook.Logger('Transaction Simulator')
class TransactionSimulator(object):
def __init__(self, slippage=None):
if slippage:
assert isinstance(slippage, (VolumeShareSlippage, FixedSlippage))
self.slippage = slippage
def __init__(self, transact=None):
if transact != None:
self.transact = transact
else:
self.slippage = VolumeShareSlippage()
self.transact = transact_partial(
VolumeShareSlippage(),
PerShare()
)
self.open_orders = defaultdict(list)
@@ -37,7 +43,7 @@ class TransactionSimulator(object):
event.TRANSACTION = None
# We only fill transactions on trade events.
if event.type == zp.DATASOURCE_TYPE.TRADE:
event.TRANSACTION = self.slippage.simulate(event, self.open_orders)
event.TRANSACTION = self.transact(event, self.open_orders)
return event
+61 -62
View File
@@ -9,7 +9,6 @@ from zipline.utils.timeout import Heartbeat, Timeout
from zipline.finance.trading import TransactionSimulator
from zipline.finance.performance import PerformanceTracker
from zipline.utils.log_utils import stdout_only_pipe
from zipline.gens.utils import hash_args
log = Logger('Trade Simulation')
@@ -53,14 +52,14 @@ class TradeSimulationClient(object):
is sent to the algo.
"""
def __init__(self, algo, environment, slippage):
def __init__(self, algo, environment, transact):
self.algo = algo
self.sids = algo.get_sid_filter()
self.environment = environment
self.slippage = slippage
self.transact = transact
self.ordering_client = TransactionSimulator(self.slippage)
self.ordering_client = TransactionSimulator(self.transact)
self.perf_tracker = PerformanceTracker(self.environment, self.sids)
self.algo_start = self.environment.first_open
@@ -131,8 +130,10 @@ class AlgorithmSimulator(object):
self.algolog = Logger("AlgoLog")
self.algo.set_logger(self.algolog)
# Porived user algorithm with slippage override.
self.algo.set_slippage_override(self.override_slippage)
# Provide user algorithm with a setter for the transact
# method (method that constructs transactions based on
# open orders and trade events).
self.algo.set_transact_setter(self.set_transact)
# Handler for heartbeats during calls to handle_data.
def log_heartbeats(beat_count, stackframe):
@@ -174,17 +175,17 @@ class AlgorithmSimulator(object):
record.extra['algo_dt'] = self.snapshot_dt
self.processor = Processor(inject_algo_dt)
# Single_use generator that uses the @contextmanager decorator
# to monkey patch sys.stdout with a logbook interface.
self.stdout_capture = stdout_only_pipe
def override_slippage(self, slippage):
self.order_book.slippage = slippage
def set_transact(self, transact):
"""
Set the method that will be called to create a
transaction from open orders and trade events.
"""
self.order_book.transact = transact
def order(self, sid, amount):
"""
Closure to pass into the user's algo to allow placing orders
into the txn_sim's dict of open orders.
into the transaction simulator's dict of open orders.
"""
assert sid in self.sids, "Order on invalid sid: %i" % sid
order = ndict({
@@ -214,66 +215,64 @@ class AlgorithmSimulator(object):
"""
Main generator work loop.
"""
# Capture any output of this generator to stdout and pipe it
# to a logbook interface. Also inject the current algo
# snapshot time to any log record generated.
#with self.processor.threadbound(), self.stdout_capture(Logger('Print'),''):
# Call user's initialize method with a timeout (only if
# initialize wasn't called already).
if not getattr(self.algo, 'initialized', False):
with Timeout(INIT_TIMEOUT, message="Call to initialize timed out"):
self.algo.initialize()
# Group together events with the same dt field. This depends on the
# events already being sorted.
for date, snapshot in groupby(stream_in, attrgetter('dt')):
# Set the simulation date to be the first event we see.
# This should only occur once, at the start of the test.
if self.simulation_dt == None:
self.simulation_dt = date
# inject the current algo
# snapshot time to any log record generated.
with self.processor.threadbound():
# Group together events with the same dt field. This depends on the
# events already being sorted.
for date, snapshot in groupby(stream_in, attrgetter('dt')):
# Set the simulation date to be the first event we see.
# This should only occur once, at the start of the test.
if self.simulation_dt == None:
self.simulation_dt = date
# Done message has the risk report, so we yield before exiting.
if date == 'DONE':
for event in snapshot:
yield event.perf_message
raise StopIteration()
# We're still in the warmup period. Use the event to
# update our universe, but don't yield any perf messages,
# and don't send a snapshot to handle_data.
elif date < self.algo_start:
for event in snapshot:
del event['perf_message']
self.update_universe(event)
# The algo has taken so long to process events that
# its simulated time is later than the event time.
# Update the universe and yield any perf messages
# encountered, but don't call handle_data.
elif date < self.simulation_dt:
for event in snapshot:
# Only yield if we have something interesting to say.
if event.perf_message != None:
# Done message has the risk report, so we yield before exiting.
if date == 'DONE':
for event in snapshot:
yield event.perf_message
# Delete the message before updating so we don't send it
# to the user.
del event['perf_message']
self.update_universe(event)
raise StopIteration
# Regular snapshot. Update the universe and send a snapshot
# to handle data.
else:
for event in snapshot:
# Only yield if we have something interesting to say.
if event.perf_message != None:
yield event.perf_message
del event['perf_message']
# We're still in the warmup period. Use the event to
# update our universe, but don't yield any perf messages,
# and don't send a snapshot to handle_data.
elif date < self.algo_start:
for event in snapshot:
del event['perf_message']
self.update_universe(event)
self.update_universe(event)
# The algo has taken so long to process events that
# its simulated time is later than the event time.
# Update the universe and yield any perf messages
# encountered, but don't call handle_data.
elif date < self.simulation_dt:
for event in snapshot:
# Only yield if we have something interesting to say.
if event.perf_message != None:
yield event.perf_message
# Delete the message before updating so we don't send it
# to the user.
del event['perf_message']
self.update_universe(event)
# Send the current state of the universe to the user's algo.
self.simulate_snapshot(date)
# Regular snapshot. Update the universe and send a snapshot
# to handle data.
else:
for event in snapshot:
# Only yield if we have something interesting to say.
if event.perf_message != None:
yield event.perf_message
del event['perf_message']
self.update_universe(event)
# Send the current state of the universe to the user's algo.
self.simulate_snapshot(date)
def update_universe(self, event):
"""
+2 -112
View File
@@ -61,13 +61,11 @@ before invoking simulate.
"""
from zipline.utils import factory
from zipline.gens.composites import (
date_sorted_sources,
sequential_transforms
)
from zipline.gens.tradesimulation import TradeSimulationClient as tsc
from zipline.finance.slippage import FixedSlippage
from logbook import Logger
@@ -81,7 +79,7 @@ class SimulatedTrading(object):
transforms,
algorithm,
environment,
slippage):
sim_method=None):
"""
@sources - an iterable of iterables
These iterables must yield ndicts that contain:
@@ -108,7 +106,7 @@ class SimulatedTrading(object):
# Formerly merged_transforms.
self.with_tnfms = sequential_transforms(self.date_sorted,
*self.transforms)
self.trading_client = tsc(algorithm, environment, slippage)
self.trading_client = tsc(algorithm, environment, sim_method)
self.gen = self.trading_client.simulate(self.with_tnfms)
def __iter__(self):
@@ -116,111 +114,3 @@ class SimulatedTrading(object):
def next(self):
return self.gen.next()
@staticmethod
def create_test_zipline(**config):
"""
:param config: A configuration object that is a dict with:
- environment - a \
:py:class:`zipline.finance.trading.TradingEnvironment`
- sid - an integer, which will be used as the security ID.
- order_count - the number of orders the test algo will place,
defaults to 100
- order_amount - the number of shares per order, defaults to 100
- trade_count - the number of trades to simulate, defaults to 101
to ensure all orders are processed.
- algorithm - optional parameter providing an algorithm. defaults
to :py:class:`zipline.test.algorithms.TestAlgorithm`
- trade_source - optional parameter to specify trades, if present.
If not present :py:class:`zipline.sources.SpecificEquityTrades`
is the source, with daily frequency in trades.
- slippage: optional parameter that configures the
:py:class:`zipline.gens.tradingsimulation.TransactionSimulator`. Expects
an object with a simulate mehod, such as
:py:class:`zipline.gens.tradingsimulation.FixedSlippage`.
:py:mod:`zipline.finance.trading`
- transforms: optional parameter that provides a list
of StatefulTransform objects.
"""
from zipline.test_algorithms import TestAlgorithm
assert isinstance(config, dict)
sid_list = config.get('sid_list')
if not sid_list:
sid = config.get('sid')
sid_list = [sid]
concurrent_trades = config.get('concurrent_trades', False)
#--------------------
# Trading Environment
#--------------------
if 'environment' in config:
trading_environment = config['environment']
else:
trading_environment = factory.create_trading_environment()
if 'order_count' in config:
order_count = config['order_count']
else:
order_count = 100
if 'order_amount' in config:
order_amount = config['order_amount']
else:
order_amount = 100
if 'trade_count' in config:
trade_count = config['trade_count']
else:
# to ensure all orders are filled, we provide one more
# trade than order
trade_count = 101
slippage = config.get('slippage', FixedSlippage())
#-------------------
# Trade Source
#-------------------
if 'trade_source' in config:
trade_source = config['trade_source']
else:
trade_source = factory.create_daily_trade_source(
sid_list,
trade_count,
trading_environment,
concurrent=concurrent_trades
)
#-------------------
# Transforms
#-------------------
transforms = config.get('transforms', [])
#-------------------
# Create the Algo
#-------------------
if 'algorithm' in config:
test_algo = config['algorithm']
else:
test_algo = TestAlgorithm(
sid,
order_amount,
order_count
)
#-------------------
# Simulation
#-------------------
sim = SimulatedTrading(
[trade_source],
transforms,
test_algo,
trading_environment,
slippage,
)
#-------------------
return sim
+12 -14
View File
@@ -44,8 +44,8 @@ The algorithm must expose methods:
self.Portfolio[sid(133)]['cost_basis']
- set_slippage_override: method that accepts a callable. Will
be set as the value of the set_slippage_override method of
- set_transact_setter: method that accepts a callable. Will
be set as the value of the set_transact_setter method of
the trading_client. This allows an algorithm to change the
slippage model used to predict transactions based on orders
and trade events.
@@ -97,7 +97,7 @@ class TestAlgorithm():
def get_sid_filter(self):
return self.sid_filter
def set_slippage_override(self, slippage_callable):
def set_transact_setter(self, txn_sim_callable):
pass
@@ -138,7 +138,7 @@ class HeavyBuyAlgorithm():
def get_sid_filter(self):
return [self.sid]
def set_slippage_override(self, slippage_callable):
def set_transact_setter(self, txn_sim_callable):
pass
class NoopAlgorithm(object):
@@ -164,7 +164,7 @@ class NoopAlgorithm(object):
def get_sid_filter(self):
return []
def set_slippage_override(self, slippage_callable):
def set_transact_setter(self, txn_sim_callable):
pass
class ExceptionAlgorithm(object):
@@ -210,7 +210,7 @@ class ExceptionAlgorithm(object):
else:
return [self.sid]
def set_slippage_override(self, slippage_callable):
def set_transact_setter(self, txn_sim_callable):
pass
class DivByZeroAlgorithm():
@@ -240,7 +240,7 @@ class DivByZeroAlgorithm():
def get_sid_filter(self):
return [self.sid]
def set_slippage_override(self, slippage_callable):
def set_transact_setter(self, txn_sim_callable):
pass
class InitializeTimeoutAlgorithm():
@@ -269,7 +269,7 @@ class InitializeTimeoutAlgorithm():
def get_sid_filter(self):
return [self.sid]
def set_slippage_override(self, slippage_callable):
def set_transact_setter(self, txn_sim_callable):
pass
class TooMuchProcessingAlgorithm():
@@ -297,7 +297,7 @@ class TooMuchProcessingAlgorithm():
def get_sid_filter(self):
return [self.sid]
def set_slippage_override(self, slippage_callable):
def set_transact_setter(self, txn_sim_callable):
pass
class TimeoutAlgorithm():
@@ -327,7 +327,7 @@ class TimeoutAlgorithm():
def get_sid_filter(self):
return [self.sid]
def set_slippage_override(self, slippage_callable):
def set_transact_setter(self, txn_sim_callable):
pass
class TestPrintAlgorithm():
@@ -354,7 +354,7 @@ class TestPrintAlgorithm():
def get_sid_filter(self):
return [self.sid]
def set_slippage_override(self, slippage_callable):
def set_transact_setter(self, txn_sim_callable):
pass
class TestLoggingAlgorithm():
@@ -381,7 +381,7 @@ class TestLoggingAlgorithm():
def get_sid_filter(self):
return [self.sid]
def set_slippage_override(self, slippage_callable):
def set_transact_setter(self, txn_sim_callable):
pass
@@ -447,5 +447,3 @@ class BatchTransformAlgorithm(TradingAlgorithm):
self.history_return_price_class.append(self.return_price_class.handle_data(data))
self.history_return_price_decorator.append(self.return_price_decorator.handle_data(data))
self.history_return_args.append(self.return_args_batch.handle_data(data, *self.args, **self.kwargs))
+2
View File
@@ -244,3 +244,5 @@ def create_test_df_source():
df = pd.DataFrame(x, index=index, columns=[0, 1])
return DataFrameSource(df), df
+113
View File
@@ -0,0 +1,113 @@
import zipline.utils.factory as factory
from zipline.test_algorithms import TestAlgorithm
from zipline.lines import SimulatedTrading
from zipline.finance.slippage import FixedSlippage, transact_partial
from zipline.finance.commission import PerShare
def create_test_zipline(**config):
"""
:param config: A configuration object that is a dict with:
- environment - a \
:py:class:`zipline.finance.trading.TradingEnvironment`
- sid - an integer, which will be used as the security ID.
- order_count - the number of orders the test algo will place,
defaults to 100
- order_amount - the number of shares per order, defaults to 100
- trade_count - the number of trades to simulate, defaults to 101
to ensure all orders are processed.
- algorithm - optional parameter providing an algorithm. defaults
to :py:class:`zipline.test.algorithms.TestAlgorithm`
- trade_source - optional parameter to specify trades, if present.
If not present :py:class:`zipline.sources.SpecificEquityTrades`
is the source, with daily frequency in trades.
- slippage: optional parameter that configures the
:py:class:`zipline.gens.tradingsimulation.TransactionSimulator`. Expects
an object with a simulate mehod, such as
:py:class:`zipline.gens.tradingsimulation.FixedSlippage`.
:py:mod:`zipline.finance.trading`
- transforms: optional parameter that provides a list
of StatefulTransform objects.
"""
assert isinstance(config, dict)
sid_list = config.get('sid_list')
if not sid_list:
sid = config.get('sid')
sid_list = [sid]
concurrent_trades = config.get('concurrent_trades', False)
#--------------------
# Trading Environment
#--------------------
if 'environment' in config:
trading_environment = config['environment']
else:
trading_environment = factory.create_trading_environment()
if 'order_count' in config:
order_count = config['order_count']
else:
order_count = 100
if 'order_amount' in config:
order_amount = config['order_amount']
else:
order_amount = 100
if 'trade_count' in config:
trade_count = config['trade_count']
else:
# to ensure all orders are filled, we provide one more
# trade than order
trade_count = 101
slippage = config.get('slippage', FixedSlippage())
commission = PerShare()
transact_method = transact_partial(slippage, commission)
#-------------------
# Trade Source
#-------------------
if 'trade_source' in config:
trade_source = config['trade_source']
else:
trade_source = factory.create_daily_trade_source(
sid_list,
trade_count,
trading_environment,
concurrent=concurrent_trades
)
#-------------------
# Transforms
#-------------------
transforms = config.get('transforms', [])
#-------------------
# Create the Algo
#-------------------
if 'algorithm' in config:
test_algo = config['algorithm']
else:
test_algo = TestAlgorithm(
sid,
order_amount,
order_count
)
#-------------------
# Simulation
#-------------------
sim = SimulatedTrading(
[trade_source],
transforms,
test_algo,
trading_environment,
transact_method
)
#-------------------
return sim