wassname/catalyst
1
0
Fork 0
mirror of https://github.com/wassname/catalyst.git synced 2026-07-06 05:14:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:quantopian/zipline

Browse Source
This commit is contained in:
Thomas Wiecki
2012-05-30 08:42:12 -04:00
parent 1a0416cfdb 46ca4531f6
commit 8171a5d8f4
14 changed files with 197 additions and 313 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/extensions.rst
+103
View File
@@ -0,0 +1,103 @@
.. highlight:: cython
Philosophy
==========
Use judgement when to use C extensions. Debugging them potentially
has a time cost of t=infinity, they can segfault, and may not be
debugabble by anyone else. Simply put 90% of the time its not worth it
and construction of extensions be must informed by scientific profiling.
Listen to your inner Knuth.
Writing C Extensions
====================
Caveats aside, C Extensions can be in two forms:
- C
- Cython
Cython is a superset of Python which compiles into C. The code it
produces is generally not human readable.
Reference: http://docs.cython.org/
C is well, C. You manage your own memory and interface with Python.h .
If you need raw performance or need to interface with other C libraries
this is often the best approach. Of course this requires that you
be very careful to tend to memory and and Python's internal garbage
collection.
Reference: http://docs.python.org/c-api/
One can write C++ extensions, but please don't.
One could also embed Assembly in C and thus in Python, but again please
don't.
Compilers
=========
Compatability
- Do not use Clang
- Do not use GCC-LLVM
Use standard GCC >= 4.6 from gnu.org, otherwise extensions will have
undefined behavior and will not be portable.
Also make sure to code against Python 2.7 and numpy 1.6.1 header
files. If using Cython have it auto figure out the paths to ensurable
portability.
Building
========
In pavement.py ::
example = Extension(
"zipline/example", ["zipline/example.pyx"],
)
If you need Numpy::
example = Extension(
"zipline/example", ["zipline/example.pyx"],
include_dirs=[np.get_include()],
)
To build in development ::
$ paver build_ext --inplace
Pure C
======
.. highlight :: c
#include "Python.h"
Releasing the GIL
=================
::
from libc.stdio cimport printf
with nogil:
# in here you allowed to do whatever you like so long as
# you do not touch Python objects. This really should
# only be used to interface with other C libraries.
printf("hello, world\n");
Debugging
=========
Compile with debug symbols and use gdb and valgrind. It sucks but its
really the only way.
Vim
===
For syntax highlighting in Vim::
:set syntax=pyrex
etc/requirements.txt
+3
View File
@@ -10,3 +10,6 @@ gevent-zeromq==0.2.2
# Packaging
distribute==0.6.27
setuptools==0.6c11
# Unix
setproctitle==1.1.6
tests/test_finance.py
-1
View File
@@ -438,7 +438,6 @@ class FinanceTestCase(TestCase):
{
'sid' : sid,
'amount' : order_amount * alternator**i,
'type' : zp.DATASOURCE_TYPE.ORDER,
'dt' : order_date
})
tests/test_optimize.py
+31 -62
View File
@@ -14,6 +14,7 @@ from zipline.utils.logger import configure_logging
from zipline.core.devsimulator import AddressAllocator, Simulator
from zipline.optimize.algorithms import BuySellAlgorithm
from zipline.optimize.factory import create_predictable_zipline
from zipline.finance.trading import TradingEnvironment
from zipline.lines import SimulatedTrading
from zipline.finance.trading import SIMULATION_STYLE
@@ -53,20 +54,18 @@ class TestUpDown(TestCase):
base_price = 50
amplitude = 6
offset = 0
self.zipline_test_config['order_count'] = trade_count - 1
self.zipline_test_config['trade_count'] = trade_count
self.zipline_test_config['simulation_style'] = \
SIMULATION_STYLE.FIXED_SLIPPAGE
trading_environment = factory.create_trading_environment()
source = create_updown_trade_source(sid,
trade_count,
trading_environment,
base_price,
amplitude
zipline, config = create_predictable_zipline(
self.zipline_test_config,
sid=sid,
amplitude=amplitude,
base_price=base_price,
offset=offset,
trade_count=5,
simulate=False
)
prices = np.array([event.price for event in source.event_list])
prices = np.array([event.price for event in config['trade_source'].event_list])
max_price_idx = np.where(prices==prices.max())[0]
min_price_idx = np.where(prices==prices.min())[0]
self.assertTrue(np.all(max_price_idx % 2 == 1),
@@ -82,15 +81,10 @@ class TestUpDown(TestCase):
"Minimum price does not equal expected maximum price."
)
algo = BuySellAlgorithm(sid, 100, 0)
self.zipline_test_config['trade_source'] = source
self.zipline_test_config['algorithm'] = algo
self.zipline_test_config['environment'] = trading_environment
zipline = SimulatedTrading.create_test_zipline(**self.zipline_test_config)
zipline.simulate(blocking=True)
algo = config['algorithm']
orders = np.asarray(algo.orders)
max_order_idx = np.where(orders==orders.max())[0]
min_order_idx = np.where(orders==orders.min())[0]
@@ -108,9 +102,8 @@ class TestUpDown(TestCase):
"Algorithm did not sell when price was going to increase."
)
def test_concavity_of_returns(self):
"""verify concave relationship between of free parameter and
"""verify concave relationship between free parameter and
returns in certain region around the max. Moreover,
establishes that the max returns is at the correct value
(i.e. 0).
@@ -121,10 +114,6 @@ class TestUpDown(TestCase):
sid = 133
amplitude = 30
base_price = 50
self.zipline_test_config['order_count'] = trade_count - 1
self.zipline_test_config['trade_count'] = trade_count
self.zipline_test_config['simulation_style'] = \
SIMULATION_STYLE.FIXED_SLIPPAGE
#test whether return-function is concave wrt repeats.
test_offsets = np.arange(-9, 9, 1.)
@@ -133,21 +122,16 @@ class TestUpDown(TestCase):
compound_returns = np.empty(len(test_offsets))
ziplines = []
for i, test_offset in enumerate(test_offsets):
trading_environment = factory.create_trading_environment()
source = create_updown_trade_source(sid,
trade_count,
trading_environment,
base_price,
amplitude
for i, offset in enumerate(test_offsets):
zipline, config = create_predictable_zipline(
self.zipline_test_config,
sid=sid,
amplitude=amplitude,
base_price=base_price,
offset=offset,
trade_count=trade_count,
simulate=True
)
algo = BuySellAlgorithm(sid, 100, test_offset)
self.zipline_test_config['algorithm'] = algo
self.zipline_test_config['trade_source'] = source
self.zipline_test_config['environment'] = trading_environment
zipline = SimulatedTrading.create_test_zipline(**self.zipline_test_config)
zipline.simulate(blocking=True)
ziplines.append(zipline)
compound_returns[i] = zipline.get_cumulative_performance()['returns']
@@ -169,7 +153,7 @@ class TestUpDown(TestCase):
idx[0] -= 1
idx[1] += 1
@skip
#@skip
def test_optimize(self):
"""verify that gradient descent (Powell's method) can find
the optimal free parameter under which the BuySellAlgorithm produces
@@ -177,30 +161,15 @@ class TestUpDown(TestCase):
"""
def simulate(offset):
#generate events
trade_count = 3
sid = 133
amplitude = 10
base_price = 50
self.zipline_test_config['order_count'] = trade_count - 1
self.zipline_test_config['trade_count'] = trade_count
self.zipline_test_config['simulation_style'] = \
SIMULATION_STYLE.FIXED_SLIPPAGE
trading_environment = factory.create_trading_environment()
source = create_updown_trade_source(sid,
trade_count,
trading_environment,
base_price,
amplitude
zipline, config = create_predictable_zipline(
self.zipline_test_config,
sid=133,
amplitude=10,
base_price=50,
offset=offset,
trade_count=5,
simulate=True
)
algo = BuySellAlgorithm(sid, 100, offset)
self.zipline_test_config['algorithm'] = algo
self.zipline_test_config['trade_source'] = source
self.zipline_test_config['environment'] = trading_environment
zipline = SimulatedTrading.create_test_zipline(**self.zipline_test_config)
zipline.simulate(blocking=True)
zipline.shutdown()
#function is getting minimized, so have to return negative cum returns.
return -zipline.get_cumulative_performance()['returns']
tests/test_perf_tracking.py
+1 -1
View File
@@ -552,7 +552,7 @@ shares in position"
})
else:
txn = None
event[zp.TRANSFORM_TYPE.TRANSACTION] = txn
event['TRANSACTION'] = txn
perf_tracker.process_event(event)
#we skip two trades, to test case of None transaction
tests/test_protocol.py
+5 -57
View File
@@ -1,5 +1,5 @@
"""
Test the FRAME/UNFRAME functions in the sequence expected from ziplines.
Test the FRAME/UNFRAME functions in the sequence expected from ziplines.
"""
import pytz
@@ -36,10 +36,10 @@ class ProtocolTestCase(TestCase):
one_day_td = timedelta(days=1)
trades = factory.create_trade_history(
sid,
price,
volume,
one_day_td,
sid,
price,
volume,
one_day_td,
self.trading_environment
)
@@ -75,55 +75,3 @@ class ProtocolTestCase(TestCase):
event.delete('helloworld')
self.assertEqual(zp.ndict(trade), event)
@timed(DEFAULT_TIMEOUT)
def test_order_protocol(self):
#client places an order
now = datetime.utcnow().replace(tzinfo=pytz.utc)
order = zp.ndict({
'dt':now,
'sid':133,
'amount':100
})
order_msg = zp.ORDER_FRAME(order)
#order datasource receives
order = zp.ORDER_UNFRAME(order_msg)
self.assertEqual(order.sid, 133)
self.assertEqual(order.amount, 100)
self.assertEqual(order.dt, now)
#order datasource datasource frames the order
order_event = zp.ndict({
"sid" : order.sid,
"amount" : order.amount,
"dt" : order.dt,
"source_id" : zp.FINANCE_COMPONENT.ORDER_SOURCE,
"type" : zp.DATASOURCE_TYPE.ORDER
})
order_ds_msg = zp.DATASOURCE_FRAME(order_event)
#transaction transform unframes
recovered_order = zp.DATASOURCE_UNFRAME(order_ds_msg)
self.assertEqual(now, recovered_order.dt)
#create a transaction from the order
txn = zp.ndict({
'sid' : recovered_order.sid,
'amount' : recovered_order.amount,
'dt' : recovered_order.dt,
'price' : 10.0,
'commission' : 0.50
})
#frame that transaction
txn_msg = zp.TRANSFORM_FRAME(zp.TRANSFORM_TYPE.TRANSACTION, txn)
#unframe
recovered_tx = zp.TRANSFORM_UNFRAME(txn_msg).TRANSACTION
self.assertEqual(recovered_tx.sid, 133)
self.assertEqual(recovered_tx.amount, 100)
zipline/components/tradesimulation.py
-3
View File
@@ -151,9 +151,6 @@ class TradeSimulationClient(Component):
self.perf.log_order(order)
self.txn_sim.add_open_order(order)
def signal_order_done(self):
self.order_socket.send(str(zp.ORDER_PROTOCOL.DONE))
def queue_event(self, event):
if self.event_queue == None:
self.event_queue = []
zipline/core/interpreter.py
+15 -4
View File
@@ -3,6 +3,7 @@ import yaml
import argparse
import fileinput
from cStringIO import StringIO
from zipline.utils.date_utils import EPOCH, date_to_datetime
def interpret(args):
print 'Reading {ifile}'.format(ifile=args.file)
@@ -13,7 +14,7 @@ def interpret(args):
metadata = StringIO()
algorithm = StringIO()
for line in fileinput.input(sys.argv[1]):
for line in fileinput.input(args.file):
if line.startswith('---'):
if metastart:
metastart = False
@@ -45,18 +46,28 @@ def interpret(args):
except StopIteration:
raise RuntimeError("No metadata in file.")
start = meta['start']
end = meta['end']
algocode = algorithm.getvalue()
start = meta['start_date']
end = meta['end_date']
meta['start_date'] = EPOCH(date_to_datetime(start))
meta['end_date'] = EPOCH(date_to_datetime(end))
meta['algocode'] = algocode
print end - start
ns = {}
exec(algorithm.getvalue()) in ns
# -- Sanity check --
exec(algocode) in ns
assert ns['initialize']
assert ns['get_sid_filter']
assert ns['handle_data']
return algocode, meta
def main():
parser = argparse.ArgumentParser()
parser.add_argument('file', metavar='file', help='Algorithm file.')
zipline/finance/risk.py
+8 -3
View File
@@ -41,7 +41,7 @@ import datetime
import math
import numpy as np
import numpy.linalg as la
import zipline.protocol as zp
from zipline.utils.date_utils import epoch_now
LOGGER = logging.getLogger('ZiplineLogger')
@@ -308,7 +308,10 @@ class RiskMetrics():
message = "no rate for end date = {dt} and term = {term}. Check \
that date doesn't exceed treasury history range."
message = message.format(dt=self.end_date,term=self.treasury_duration)
message = message.format(
dt=self.end_date,
term=self.treasury_duration
)
raise Exception(message)
@@ -328,6 +331,7 @@ class RiskReport():
self.algorithm_returns = algorithm_returns
self.trading_environment = trading_environment
self.exceeded_max_loss = exceeded_max_loss
self.created = epoch_now()
if len(self.algorithm_returns) == 0:
start_date = self.trading_environment.period_start
@@ -361,7 +365,8 @@ class RiskReport():
'three_month' : [x.to_dict() for x in self.three_month_periods],
'six_month' : [x.to_dict() for x in self.six_month_periods],
'twelve_month' : [x.to_dict() for x in self.year_periods],
'exceeded_max_loss' : self.exceeded_max_loss
'exceeded_max_loss' : self.exceeded_max_loss,
'created' : self.created
}
def periodsInRange(self, months_per, start, end):
zipline/finance/trading.py
+5 -6
View File
@@ -168,12 +168,11 @@ for orders:
def create_transaction(self, sid, amount, price, dt, direction):
self.txn_count += 1
txn = {'sid' : sid,
'amount' : int(amount),
'dt' : dt,
'price' : price,
'commission' : self.commission * amount * direction,
'source_id' : zp.FINANCE_COMPONENT.TRANSACTION_SIM
txn = {'sid' : sid,
'amount' : int(amount),
'dt' : dt,
'price' : price,
'commission' : self.commission * amount * direction
}
return zp.ndict(txn)
zipline/optimize/factory.py
+18 -9
View File
@@ -11,10 +11,12 @@ from zipline.utils.factory import get_next_trading_dt, create_trading_environmen
from zipline.finance.sources import SpecificEquityTrades
from zipline.optimize.algorithms import BuySellAlgorithm
from zipline.lines import SimulatedTrading
from zipline.finance.trading import SIMULATION_STYLE
from copy import deepcopy
from itertools import cycle
def create_updown_trade_source(sid, trade_count, trading_environment, start_price, amplitude):
from itertools import cycle
volume = 1000
events = []
price = start_price-amplitude/2.
@@ -47,20 +49,27 @@ def create_updown_trade_source(sid, trade_count, trading_environment, start_pric
return source
def create_predictable_zipline(config, sid=133, amplitude=10, base_price=50, offset=0):
config = deepcopy(config)
def create_predictable_zipline(config, sid=133, amplitude=10, base_price=50, offset=0, trade_count=3, simulate=True):
#config = deepcopy(config)
trading_environment = create_trading_environment()
source = create_updown_trade_source(sid,
config['trade_count'],
trade_count,
trading_environment,
base_price,
amplitude)
algo = BuySellAlgorithm(sid, 100, offset)
config['algorithm'] = algo
if 'algorithm' not in config:
config['algorithm'] = BuySellAlgorithm(sid, 100, offset)
config['order_count'] = trade_count - 1
config['trade_count'] = trade_count
config['trade_source'] = source
config['environment'] = trading_environment
zipline = SimulatedTrading.create_test_zipline(**config)
zipline.simulate(blocking=True)
config['simulation_style'] = SIMULATION_STYLE.FIXED_SLIPPAGE
return zipline
zipline = SimulatedTrading.create_test_zipline(**config)
if simulate:
zipline.simulate(blocking=True)
return zipline, config
zipline/protocol.py
-163
View File
@@ -246,12 +246,6 @@ def DATASOURCE_FRAME(event):
event.source_id,
TRADE_FRAME(event)
]))
elif(event.type == DATASOURCE_TYPE.ORDER):
return msgpack.dumps(tuple([
event.type,
event.source_id,
ORDER_SOURCE_FRAME(event)
]))
else:
raise INVALID_DATASOURCE_FRAME(str(event))
@@ -286,8 +280,6 @@ def DATASOURCE_UNFRAME(msg):
child_value = ndict({'dt':None})
elif(ds_type == DATASOURCE_TYPE.TRADE):
child_value = TRADE_UNFRAME(payload)
elif(ds_type == DATASOURCE_TYPE.ORDER):
child_value = ORDER_SOURCE_UNFRAME(payload)
else:
raise INVALID_DATASOURCE_FRAME(msg)
@@ -342,8 +334,6 @@ def TRANSFORM_FRAME(name, value):
assert isinstance(name, basestring)
if value == None:
return msgpack.dumps(tuple([name, TRANSFORM_TYPE.EMPTY]))
if(name == TRANSFORM_TYPE.TRANSACTION):
value = TRANSACTION_FRAME(value)
return msgpack.dumps(tuple([name, value]))
def TRANSFORM_UNFRAME(msg):
@@ -359,8 +349,6 @@ def TRANSFORM_UNFRAME(msg):
assert isinstance(name, basestring)
if(name == TRANSFORM_TYPE.PASSTHROUGH):
value = FEED_UNFRAME(value)
elif(name == TRANSFORM_TYPE.TRANSACTION):
value = TRANSACTION_UNFRAME(value)
return ndict({name : value})
except TypeError:
@@ -382,11 +370,6 @@ def MERGE_FRAME(event):
"""
assert isinstance(event, ndict)
PACK_DATE(event)
if(event.has_attr(TRANSFORM_TYPE.TRANSACTION)):
if(event.TRANSACTION == None):
event.TRANSACTION = TRANSFORM_TYPE.EMPTY
else:
event.TRANSACTION = TRANSACTION_FRAME(event.TRANSACTION)
payload = event.as_dict()
return msgpack.dumps(payload)
@@ -396,11 +379,6 @@ def MERGE_UNFRAME(msg):
#TODO: anything we can do to assert more about the content of the dict?
assert isinstance(payload, dict)
payload = ndict(payload)
if(payload.has_attr(TRANSFORM_TYPE.TRANSACTION)):
if(payload.TRANSACTION == TRANSFORM_TYPE.EMPTY):
payload.TRANSACTION = None
else:
payload.TRANSACTION = TRANSACTION_UNFRAME(payload.TRANSACTION)
UNPACK_DATE(payload)
return payload
except TypeError:
@@ -409,12 +387,6 @@ def MERGE_UNFRAME(msg):
raise INVALID_MERGE_FRAME(msg)
# -----------------------
# Finance Protocol
# -----------------------
INVALID_ORDER_FRAME = FrameExceptionFactory('ORDER')
INVALID_TRADE_FRAME = FrameExceptionFactory('TRADE')
# -----------------------
# Trades
# -----------------------
@@ -468,131 +440,6 @@ def TRADE_UNFRAME(msg):
except ValueError:
raise INVALID_TRADE_FRAME(msg)
# -----------------------
# Orders
# -----------------------
# - from client to order source
def ORDER_FRAME(order):
assert isinstance(order.sid, int)
assert isinstance(order.amount, int) #no partial shares...
PACK_DATE(order)
return msgpack.dumps(tuple([
order.sid,
order.amount,
order.dt
]))
def ORDER_UNFRAME(msg):
try:
sid, amount, dt = msgpack.loads(msg)
assert isinstance(sid, int)
assert isinstance(amount, int)
rval = ndict({
'sid':sid,
'amount':amount,
'dt':dt
})
UNPACK_DATE(rval)
return rval
except TypeError:
raise INVALID_ORDER_FRAME(msg)
except ValueError:
raise INVALID_ORDER_FRAME(msg)
# -----------------------
# TRANSACTIONS
# -----------------------
#
# - Should only be called from inside TRANSFORM_(UN)FRAME.
def TRANSACTION_FRAME(event):
assert isinstance(event, ndict)
assert isinstance(event.sid, int)
assert isinstance(event.price, numbers.Real)
assert isinstance(event.commission, numbers.Real)
assert isinstance(event.amount, int)
PACK_DATE(event)
return msgpack.dumps(tuple([
event.sid,
event.price,
event.amount,
event.commission,
event.dt
]))
def TRANSACTION_UNFRAME(msg):
try:
sid, price, amount, commission, dt = msgpack.loads(msg)
assert isinstance(sid, int)
assert isinstance(price, numbers.Real)
assert isinstance(commission, numbers.Real)
assert isinstance(amount, int)
rval = ndict({
'sid' : sid,
'price' : price,
'amount' : amount,
'commission' : commission,
'dt' : dt
})
UNPACK_DATE(rval)
return rval
except TypeError:
raise INVALID_TRADE_FRAME(msg)
except ValueError:
raise INVALID_TRADE_FRAME(msg)
# -----------------------
# ORDERS
# -----------------------
#
# - from order source to feed
# - should only be called from inside DATASOURCE_(UN)FRAME
def ORDER_SOURCE_FRAME(event):
assert isinstance(event.sid, int)
assert isinstance(event.amount, int) #no partial shares...
assert isinstance(event.source_id, basestring)
assert event.type == DATASOURCE_TYPE.ORDER
PACK_DATE(event)
return msgpack.dumps(tuple([
event.sid,
event.amount,
event.dt,
event.source_id,
event.type
]))
def ORDER_SOURCE_UNFRAME(msg):
try:
sid, amount, dt, source_id, source_type = msgpack.loads(msg)
event = ndict({
"sid" : sid,
"amount" : amount,
"dt" : dt,
"source_id" : source_id,
"type" : source_type
})
assert isinstance(sid, int)
assert isinstance(amount, int)
assert isinstance(source_id, basestring)
assert isinstance(source_type, int)
UNPACK_DATE(event)
return event
except TypeError:
raise INVALID_ORDER_FRAME(msg)
except ValueError:
raise INVALID_ORDER_FRAME(msg)
# -----------------------
# Performance and Risk
# -----------------------
@@ -648,7 +495,6 @@ def convert_transactions(transactions):
for txn in transactions:
txn['date'] = EPOCH(txn['dt'])
del(txn['dt'])
del(txn['source_id'])
results.append(txn)
return results
@@ -729,20 +575,13 @@ def tuple_to_date(date_tuple):
return dt
DATASOURCE_TYPE = Enum(
'ORDER',
'TRADE',
'EMPTY',
)
ORDER_PROTOCOL = Enum(
'DONE',
'BREAK',
)
#Transform type needs to be a ndict to facilitate merging.
TRANSFORM_TYPE = ndict({
'TRANSACTION' : 'TRANSACTION', #needed?
'PASSTHROUGH' : 'PASSTHROUGH',
'EMPTY' : ''
})
@@ -751,8 +590,6 @@ TRANSFORM_TYPE = ndict({
FINANCE_COMPONENT = namelookup({
'TRADING_CLIENT' : 'TRADING_CLIENT',
'PORTFOLIO_CLIENT' : 'PORTFOLIO_CLIENT',
'ORDER_SOURCE' : 'ORDER_SOURCE',
'TRANSACTION_SIM' : 'TRANSACTION_SIM'
})
zipline/utils/date_utils.py
+7 -3
View File
@@ -44,11 +44,10 @@ def EPOCH(utc_datetime):
delta = utc_datetime - UNIX_EPOCH
seconds = delta.total_seconds()
ms = seconds * 1000
return ms
return int(ms)
def UN_EPOCH(ms_since_epoch):
seconds_since_epoch = ms_since_epoch / 1000
delta = timedelta(seconds = seconds_since_epoch)
delta = timedelta(milliseconds = ms_since_epoch)
dt = UNIX_EPOCH + delta
return dt
@@ -128,3 +127,8 @@ if __name__ == '__main__':
for day in trading_days(now, now30):
print day
print time.time() - tic
def date_to_datetime(t):
dt = datetime.fromordinal(t.toordinal())
dt = dt.replace(tzinfo = pytz.utc)
return dt
zipline/utils/protocol_utils.py
+1 -1
View File
@@ -141,7 +141,7 @@ class ndict(MutableMapping):
self.__internal.update(other_nd.__internal)
def __repr__(self):
return "namedict: " + str(self.__internal)
return "ndict(%s)" % str(self.__internal)
# Faster dictionary comparison?
#def __eq__(self, other):