Removes optimize directory and optimize unittests which live on a separate branch yet. Was not functional yet and should thus not be in the main branch.

This commit is contained in:
Thomas Wiecki
2012-10-22 08:51:30 -04:00
parent b421d805b4
commit d468d05ba4
6 changed files with 0 additions and 650 deletions
-168
View File
@@ -1,168 +0,0 @@
#
# Copyright 2012 Quantopian, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for the zipline.finance package"""
from unittest2 import TestCase, skip
from collections import defaultdict
import numpy as np
from zipline.optimize.factory import create_predictable_zipline
from zipline.utils.test_utils import setup_logger, teardown_logger
class TestUpDown(TestCase):
"""This unittest verifies that the BuySellAlgorithm in
combination with the UpDownSource are suitable for usage in an
optimization framework.
"""
leased_sockets = defaultdict(list)
def setUp(self):
self.zipline_test_config = {
'sid': [0],
'trade_count': 5,
'amplitude': 30,
'base_price': 50
}
setup_logger(self)
def tearDown(self):
teardown_logger(self)
@skip
def test_source_and_orders(self):
"""verify that UpDownSource is having the correct
behavior and that BuySellAlgorithm places the buy/sell
orders at the right time. Moreover, establishes that
UpDownSource and BuySellAlgorithm interact correctly."
"""
algo, config = create_predictable_zipline(
self.zipline_test_config,
offset=0
)
#extract arguments
base_price = self.zipline_test_config['base_price']
amplitude = self.zipline_test_config['amplitude']
prices = config['trade_source'][0].values
max_price_idx = np.where(prices == prices.max())[0]
min_price_idx = np.where(prices == prices.min())[0]
self.assertTrue(np.all(max_price_idx % 2 == 1),
"Maximum prices are not periodic."
)
self.assertTrue(np.all(min_price_idx % 2 == 0),
"Minimum prices are not periodic."
)
self.assertEqual(prices.max(), base_price + amplitude / 2.,
"Maximum price does not equal expected maximum price."
)
self.assertEqual(prices.min(), base_price - amplitude / 2.,
"Minimum price does not equal expected maximum price."
)
stats = algo.run(config['trade_source'])
self.assertTrue(len(stats) != 0)
orders = np.asarray(algo.orders)
max_order_idx = np.where(orders == orders.max())[0]
min_order_idx = np.where(orders == orders.min())[0]
self.assertTrue(np.all(max_order_idx % 2 == 1),
"Maximum orders are not periodic."
)
self.assertTrue(np.all(min_order_idx % 2 == 0),
"Minimum orders are not periodic."
)
self.assertTrue(np.all(max_order_idx == max_price_idx),
"Algorithm did not buy when price was going to drop."
)
self.assertTrue(np.all(min_order_idx == min_price_idx),
"Algorithm did not sell when price was going to increase."
)
@skip
def test_concavity_of_returns(self):
"""verify concave relationship between free parameter and
returns in certain region around the max. Moreover,
establishes that the max returns is at the correct value
(i.e. 0).
"""
test_offsets = np.arange(-9, 9, 1.)
#maximum value is expect to be at center, create boolean mask
#for later extraction
supposed_max = np.zeros(len(test_offsets), dtype=bool)
supposed_max[len(test_offsets) // 2] = True
compound_returns = np.empty(len(test_offsets))
ziplines = []
for i, offset in enumerate(test_offsets):
algo, config = create_predictable_zipline(
self.zipline_test_config,
offset=offset,
)
results = algo.run(config['trade_source'])
ziplines.append(algo)
compound_returns[i] = results.returns.sum()
self.assertTrue(np.all(
compound_returns[supposed_max] >
compound_returns[np.logical_not(supposed_max)]),
"Maximum compound returns are not where they are supposed to be."
)
# test for concavity
max_idx = np.where(supposed_max)[0][0]
idx = np.array([max_idx, max_idx])
for i in range((len(test_offsets) - 1) / 2):
# going outwards, returns must decrease
self.assertTrue(compound_returns[idx[0] - 1] <
compound_returns[idx[0]],
"Compound returns are not convex."
)
self.assertTrue(compound_returns[idx[1] + 1] <
compound_returns[idx[1]],
"Compound returns are not convex."
)
idx[0] -= 1
idx[1] += 1
@skip
def test_optimize(self):
"""verify that gradient descent (Powell's method) can find
the optimal free parameter under which the BuySellAlgorithm produces
maximum returns.
"""
def simulate(offset):
zipline, config = create_predictable_zipline(
self.zipline_test_config,
offset=offset,
)
# function is getting minimized,
# so have to return negative cum returns.
return -zipline.get_cumulative_performance()['returns']
from scipy import optimize
opt = optimize.fmin_powell(simulate, 1.5)
np.testing.assert_almost_equal(opt, 0, 5)
-19
View File
@@ -1,19 +0,0 @@
#
# Copyright 2012 Quantopian, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Thomas's parameter optimization library.
"""
-49
View File
@@ -1,49 +0,0 @@
#
# Copyright 2012 Quantopian, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from logbook import Logger
from zipline.algorithm import TradingAlgorithm
logger = Logger('Algo')
class BuySellAlgorithm(TradingAlgorithm):
"""Algorithm that buys and sells alternatingly. The amount for
each order can be specified. In addition, an offset that will
quadratically reduce the amount that will be bought can be
specified.
This algorithm is used to test the parameter optimization
framework. If combined with the UpDown trade source, an offset of
0 will produce maximum returns.
"""
def initialize(self, amount=100, offset=0):
self.amount = amount
self.buy_or_sell = -1
self.offset = offset
self.orders = []
def handle_data(self, data):
order_size = self.buy_or_sell * (self.amount - (self.offset ** 2))
self.order(self.sids[0], order_size)
logger.debug("ordering" + str(order_size))
#sell next time around.
self.buy_or_sell *= -1
self.orders.append(order_size)
-257
View File
@@ -1,257 +0,0 @@
#
# Copyright 2012 Quantopian, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# WARNING: This file is still work in progress and contains rather
# random code snippets.
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from zipline.transforms import MovingAverage
from zipline.algorithm import TradingAlgorithm
from zipline.transforms import batch_transform
class DMA(TradingAlgorithm):
"""Dual Moving Average algorithm.
"""
def initialize(self, amount=100, short_window=20, long_window=40):
self.amount = amount
self.events = 0
self.invested = {}
for sid in self.sids:
self.invested[sid] = False
self.add_transform(MovingAverage, 'short_mavg', ['price'],
market_aware=True,
days=short_window)
self.add_transform(MovingAverage, 'long_mavg', ['price'],
market_aware=True,
days=long_window)
def handle_data(self, data):
self.events += 1
for sid in self.sids:
# access transforms via their user-defined tag
if (data[sid].short_mavg['price'] >
data[sid].long_mavg['price']) \
and not self.invested[sid]:
self.order(sid, self.amount)
self.invested[sid] = True
elif (data[sid].short_mavg['price'] <
data[sid].long_mavg['price']) \
and self.invested[sid]:
self.order(sid, -self.amount)
self.invested[sid] = False
class DualMovingAverage(TradingAlgorithm):
"""Dual Moving Average algorithm.
"""
def initialize(self, short_window=200, long_window=400):
self.short_mavg = []
self.long_mavg = []
self.invested = False
self.add_transform(MovingAverage, 'short_mavg', ['price'],
market_aware=True,
days=short_window)
self.add_transform(MovingAverage, 'long_mavg', ['price'],
market_aware=True,
days=long_window)
def handle_data(self, data):
self.short_mavg.append(data['AAPL'].short_mavg['price'])
self.long_mavg.append(data['AAPL'].long_mavg['price'])
if (data['AAPL'].short_mavg['price'] >
data['AAPL'].long_mavg['price']) and not self.invested:
self.order('AAPL', 100)
self.invested = True
elif (data['AAPL'].short_mavg['price'] <
data['AAPL'].long_mavg['price']) and self.invested:
self.order('AAPL', -100)
self.invested = False
def load_close_px(indexes=None, stocks=None):
from pandas.io.data import DataReader
import pytz
from collections import OrderedDict
if indexes is None:
indexes = {'SPX': '^GSPC'}
if stocks is None:
stocks = ['AAPL', 'GE', 'IBM', 'MSFT', 'XOM', 'AA', 'JNJ', 'PEP', 'KO']
start = pd.datetime(1990, 1, 1, 0, 0, 0, 0, pytz.utc)
end = pd.datetime(2000, 1, 1, 0, 0, 0, 0, pytz.utc)
data = OrderedDict()
for stock in stocks:
print stock
stkd = DataReader(stock, 'yahoo', start, end).sort_index()
data[stock] = stkd
for name, ticker in indexes.iteritems():
print name
stkd = DataReader(ticker, 'yahoo', start, end).sort_index()
data[name] = stkd
df = pd.DataFrame({key: d['Close'] for key, d in data.iteritems()})
df.index = df.index.tz_localize(pytz.utc)
df.save('close_px.dat')
return df
def run((short_window, long_window)):
data = pd.load('close_px.dat')
#data = load_close_px()
myalgo = DMA([0, 1],
amount=100,
short_window=short_window,
long_window=long_window)
stats = myalgo.run(data)
stats['sw'] = short_window
stats['lw'] = long_window
return stats
def explore_params():
sws, lws = np.mgrid[10:20:5, 10:20:5]
stats_all = map(run, zip(sws.flatten(), lws.flatten()))
stats = pd.concat(stats_all)
returns = stats.groupby(['sw', 'lw']).sum()
plt.contourf(sws, lws, returns.returns.reshape(sws.shape))
plt.xlabel('Short window length')
plt.ylabel('Long window length')
plt.savefig('DMA_contour.png')
plt.show()
def get_opt_holdings_qp(univ_rets, track_rets):
from cvxopt import matrix
from cvxopt.solvers import qp
# set up the QP for CVXOPT
# .5 x' P x + q'x
# P = 2 * R'R
# q = - 2 * bmk'R
R = univ_rets.values
b = track_rets.values
P = matrix(2 * np.dot(R.T, R))
q = matrix(-2 * np.dot(R.T, b))
result = qp(P, q)
if result['status'] != 'optimal':
raise Exception('optimum not reached by QP')
return pd.Series(np.array(result['x']).ravel(), index=univ_rets.columns)
def opt_portfolio(cov, budget, min_return):
from cvxopt import matrix
from cvxopt.solvers import qp
n = len(cov)
cov = matrix(2 * cov)
q = matrix(np.zeros(n))
h = matrix(budget) # G*x < h
# coneqp
result = qp(cov, q, h=h)
if result['status'] != 'optimal':
raise Exception('optimum not reached by QP')
return pd.Series(np.array(result['x']).ravel())
def calc_te(weights, univ_rets, track_rets):
port_rets = (univ_rets * weights).sum(1)
return (port_rets - track_rets).std()
def plot_returns(port_returns, bmk_returns):
plt.figure()
cum_port = ((1 + port_returns).cumprod() - 1)
cum_bmk = ((1 + bmk_returns).cumprod() - 1)
# cum_port = port_returns.cumsum()
# cum_bmk = bmk_returns.cumsum()
cum_port.plot(label='Portfolio returns')
cum_bmk.plot(label='Benchmark')
plt.title('Portfolio performance')
plt.legend(loc='best')
#print run((10, 20))
import statsmodels.api as sm
@batch_transform
def ols_transform(data, spreads):
p0 = data.price['PEP']
p1 = sm.add_constant(data.price['KO'])
beta, intercept = sm.OLS(p0, p1).fit().params
spread = (data.price['PEP'] - (beta * data.price['KO'] + intercept))[-1]
if len(spreads) > 10:
z_score = (spread - np.mean(spreads[-10:])) / np.std(spreads[-10:])
else:
z_score = np.nan
spreads.append(spread)
return z_score
class Pairtrade(TradingAlgorithm):
def initialize(self):
self.spreads = []
self.invested = False
self.ols_transform = ols_transform(refresh_period=10, days=10)
def handle_data(self, data):
zscore = self.ols_transform.handle_data(data, self.spreads)
if zscore == np.nan:
return
if zscore >= 2.0 and not self.invested:
self.order('PEP', int(100 / data['PEP'].price))
self.order('KO', -int(100 / data['KO'].price))
elif zscore <= -2.0 and not self.invested:
self.order('KO', -int(100 / data['KO'].price))
self.order('PEP', int(100 / data['PEP'].price))
elif abs(zscore) < .5 and self.invested:
pass
def run_pairtrade():
data = load_close_px()
data.save('close_px.dat')
#data = pd.load('close_px.dat')
myalgo = Pairtrade()
stats = myalgo.run(data)
return stats
-155
View File
@@ -1,155 +0,0 @@
#
# Copyright 2012 Quantopian, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Factory functions to prepare useful data for optimize tests.
Author: Thomas V. Wiecki (thomas.wiecki@gmail.com), 2012
"""
from datetime import timedelta
from zipline.utils.protocol_utils import ndict
import zipline.protocol as zp
from zipline.utils.factory import (
get_next_trading_dt,
create_trading_environment
)
from zipline.sources import SpecificEquityTrades
from zipline.optimize.algorithms import BuySellAlgorithm
from zipline.finance.slippage import FixedSlippage
from copy import copy
from itertools import cycle
def create_updown_trade_source(sid, trade_count, trading_environment,
base_price, amplitude):
"""Create the updown trade source. This source emits events with
the price going up and down by the same amount in each
iteration. The trade source is thus perfectly predictable. This is
used for a test case for the optimization code.
:Arguments:
sid : int
SID of stock to create.
trade_count : int
How many trade events to create (will also influence order count)
trading_environment : TradeEnvironment object
The trading environment to use
(see zipline.factory.create_trading_environment)
base_price : int
The average price that each iteration will hover around.
amplitude : int
How much the price will go up and down each iteration.
:Returns:
source : SpecificEquityTrades
The trade source emitting up down events.
"""
volume = 1000
events = []
price = base_price - amplitude / 2.
cur = trading_environment.first_open
one_day = timedelta(minutes=1)
#create iterator to cycle through up and down phases
change = cycle([1, -1])
for i in xrange(trade_count + 2):
cur = get_next_trading_dt(cur, one_day, trading_environment)
event = ndict({
"type": zp.DATASOURCE_TYPE.TRADE,
"sid": sid,
"price": price,
"volume": volume,
"dt": cur,
})
events.append(event)
price += change.next() * amplitude
trading_environment.period_end = cur
source = SpecificEquityTrades(events)
return source
def create_predictable_zipline(config, offset=0, simulate=True):
"""Create a test zipline object as specified by config. The
zipline will use the UpDown tradesource which is perfectly
predictable.
Trade source parameters can be specified inside the config object.
:Trade source arguments:
config['sid'] : int
SID of stock to create.
config['amplitude'] : int (default 10)
How much the price will go up and down each iteration.
config['base_price'] : int (default 50)
The average price that each iteration will hover around.
config['trade_count'] : int (default 3)
How many trade events to create (will also influence order count)
If not specified, the BuySellAlgorithm is used by default. This
can be changed by setting config['algorithm'].
:Arguments:
offset : int (default 0)
The offset parameter specifies how much the BuySellAlgorithm will
order each iteration and is a negative quadratic centered around
0. Thus, any deviations from 0 will lead to less buy and sell
orders each iteration and ultimately to less compound returns.
simulate : bool (default True)
Whether to call .simulate(blocking=True) on the created zipline
argument.
:Returns:
zipline : class zipline
created zipline object
config : dict
the config dict used to create the zipline
"""
config = copy(config)
sid = config['sid']
# remove
amplitude = config.pop('amplitude', 10)
base_price = config.pop('base_price', 50)
trade_count = config.pop('trade_count', 3)
trading_environment = create_trading_environment()
source = create_updown_trade_source(sid,
trade_count,
trading_environment,
base_price,
amplitude)
if 'algorithm' not in config:
algorithm = BuySellAlgorithm(sids=[sid], amount=100, offset=offset)
config['order_count'] = trade_count - 1
config['trade_count'] = trade_count
config['trade_source'] = source
config['environment'] = trading_environment
config['slippage'] = FixedSlippage()
config['devel'] = True
return algorithm, config
-2
View File
@@ -1,2 +0,0 @@
#!/bin/bash
python -m cProfile -o example.prof example.py