diff --git a/tests/test_optimize.py b/tests/test_optimize.py deleted file mode 100644 index c5da3a20..00000000 --- a/tests/test_optimize.py +++ /dev/null @@ -1,168 +0,0 @@ -# -# Copyright 2012 Quantopian, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Tests for the zipline.finance package""" -from unittest2 import TestCase, skip -from collections import defaultdict - -import numpy as np - -from zipline.optimize.factory import create_predictable_zipline - -from zipline.utils.test_utils import setup_logger, teardown_logger - - -class TestUpDown(TestCase): - """This unittest verifies that the BuySellAlgorithm in - combination with the UpDownSource are suitable for usage in an - optimization framework. - - """ - leased_sockets = defaultdict(list) - - def setUp(self): - self.zipline_test_config = { - 'sid': [0], - 'trade_count': 5, - 'amplitude': 30, - 'base_price': 50 - } - setup_logger(self) - - def tearDown(self): - teardown_logger(self) - - @skip - def test_source_and_orders(self): - """verify that UpDownSource is having the correct - behavior and that BuySellAlgorithm places the buy/sell - orders at the right time. Moreover, establishes that - UpDownSource and BuySellAlgorithm interact correctly." - - """ - - algo, config = create_predictable_zipline( - self.zipline_test_config, - offset=0 - ) - - #extract arguments - base_price = self.zipline_test_config['base_price'] - amplitude = self.zipline_test_config['amplitude'] - - prices = config['trade_source'][0].values - max_price_idx = np.where(prices == prices.max())[0] - min_price_idx = np.where(prices == prices.min())[0] - self.assertTrue(np.all(max_price_idx % 2 == 1), - "Maximum prices are not periodic." - ) - self.assertTrue(np.all(min_price_idx % 2 == 0), - "Minimum prices are not periodic." - ) - self.assertEqual(prices.max(), base_price + amplitude / 2., - "Maximum price does not equal expected maximum price." - ) - self.assertEqual(prices.min(), base_price - amplitude / 2., - "Minimum price does not equal expected maximum price." - ) - - stats = algo.run(config['trade_source']) - - self.assertTrue(len(stats) != 0) - - orders = np.asarray(algo.orders) - max_order_idx = np.where(orders == orders.max())[0] - min_order_idx = np.where(orders == orders.min())[0] - - self.assertTrue(np.all(max_order_idx % 2 == 1), - "Maximum orders are not periodic." - ) - self.assertTrue(np.all(min_order_idx % 2 == 0), - "Minimum orders are not periodic." - ) - self.assertTrue(np.all(max_order_idx == max_price_idx), - "Algorithm did not buy when price was going to drop." - ) - self.assertTrue(np.all(min_order_idx == min_price_idx), - "Algorithm did not sell when price was going to increase." - ) - - @skip - def test_concavity_of_returns(self): - """verify concave relationship between free parameter and - returns in certain region around the max. Moreover, - establishes that the max returns is at the correct value - (i.e. 0). - - """ - test_offsets = np.arange(-9, 9, 1.) - #maximum value is expect to be at center, create boolean mask - #for later extraction - supposed_max = np.zeros(len(test_offsets), dtype=bool) - supposed_max[len(test_offsets) // 2] = True - - compound_returns = np.empty(len(test_offsets)) - ziplines = [] - for i, offset in enumerate(test_offsets): - algo, config = create_predictable_zipline( - self.zipline_test_config, - offset=offset, - ) - results = algo.run(config['trade_source']) - ziplines.append(algo) - - compound_returns[i] = results.returns.sum() - - self.assertTrue(np.all( - compound_returns[supposed_max] > - compound_returns[np.logical_not(supposed_max)]), - "Maximum compound returns are not where they are supposed to be." - ) - - # test for concavity - max_idx = np.where(supposed_max)[0][0] - idx = np.array([max_idx, max_idx]) - for i in range((len(test_offsets) - 1) / 2): - # going outwards, returns must decrease - self.assertTrue(compound_returns[idx[0] - 1] < - compound_returns[idx[0]], - "Compound returns are not convex." - ) - self.assertTrue(compound_returns[idx[1] + 1] < - compound_returns[idx[1]], - "Compound returns are not convex." - ) - idx[0] -= 1 - idx[1] += 1 - - @skip - def test_optimize(self): - """verify that gradient descent (Powell's method) can find - the optimal free parameter under which the BuySellAlgorithm produces - maximum returns. - - """ - def simulate(offset): - zipline, config = create_predictable_zipline( - self.zipline_test_config, - offset=offset, - ) - # function is getting minimized, - # so have to return negative cum returns. - return -zipline.get_cumulative_performance()['returns'] - - from scipy import optimize - opt = optimize.fmin_powell(simulate, 1.5) - np.testing.assert_almost_equal(opt, 0, 5) diff --git a/zipline/optimize/__init__.py b/zipline/optimize/__init__.py deleted file mode 100644 index 38f8d1ea..00000000 --- a/zipline/optimize/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -# -# Copyright 2012 Quantopian, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -""" -Thomas's parameter optimization library. -""" diff --git a/zipline/optimize/algorithms.py b/zipline/optimize/algorithms.py deleted file mode 100644 index 3f2efe40..00000000 --- a/zipline/optimize/algorithms.py +++ /dev/null @@ -1,49 +0,0 @@ -# -# Copyright 2012 Quantopian, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from logbook import Logger -from zipline.algorithm import TradingAlgorithm - -logger = Logger('Algo') - - -class BuySellAlgorithm(TradingAlgorithm): - """Algorithm that buys and sells alternatingly. The amount for - each order can be specified. In addition, an offset that will - quadratically reduce the amount that will be bought can be - specified. - - This algorithm is used to test the parameter optimization - framework. If combined with the UpDown trade source, an offset of - 0 will produce maximum returns. - - """ - - def initialize(self, amount=100, offset=0): - self.amount = amount - self.buy_or_sell = -1 - self.offset = offset - self.orders = [] - - def handle_data(self, data): - order_size = self.buy_or_sell * (self.amount - (self.offset ** 2)) - self.order(self.sids[0], order_size) - logger.debug("ordering" + str(order_size)) - - #sell next time around. - self.buy_or_sell *= -1 - - self.orders.append(order_size) diff --git a/zipline/optimize/example.py b/zipline/optimize/example.py deleted file mode 100644 index d9f9f4cd..00000000 --- a/zipline/optimize/example.py +++ /dev/null @@ -1,257 +0,0 @@ -# -# Copyright 2012 Quantopian, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# WARNING: This file is still work in progress and contains rather -# random code snippets. - -import pandas as pd - -import numpy as np - -import matplotlib.pyplot as plt -from zipline.transforms import MovingAverage -from zipline.algorithm import TradingAlgorithm -from zipline.transforms import batch_transform - - -class DMA(TradingAlgorithm): - """Dual Moving Average algorithm. - """ - def initialize(self, amount=100, short_window=20, long_window=40): - self.amount = amount - self.events = 0 - - self.invested = {} - for sid in self.sids: - self.invested[sid] = False - - self.add_transform(MovingAverage, 'short_mavg', ['price'], - market_aware=True, - days=short_window) - - self.add_transform(MovingAverage, 'long_mavg', ['price'], - market_aware=True, - days=long_window) - - def handle_data(self, data): - self.events += 1 - - for sid in self.sids: - # access transforms via their user-defined tag - if (data[sid].short_mavg['price'] > - data[sid].long_mavg['price']) \ - and not self.invested[sid]: - self.order(sid, self.amount) - self.invested[sid] = True - elif (data[sid].short_mavg['price'] < - data[sid].long_mavg['price']) \ - and self.invested[sid]: - self.order(sid, -self.amount) - self.invested[sid] = False - - -class DualMovingAverage(TradingAlgorithm): - """Dual Moving Average algorithm. - """ - def initialize(self, short_window=200, long_window=400): - self.short_mavg = [] - self.long_mavg = [] - - self.invested = False - - self.add_transform(MovingAverage, 'short_mavg', ['price'], - market_aware=True, - days=short_window) - - self.add_transform(MovingAverage, 'long_mavg', ['price'], - market_aware=True, - days=long_window) - - def handle_data(self, data): - self.short_mavg.append(data['AAPL'].short_mavg['price']) - self.long_mavg.append(data['AAPL'].long_mavg['price']) - - if (data['AAPL'].short_mavg['price'] > - data['AAPL'].long_mavg['price']) and not self.invested: - self.order('AAPL', 100) - self.invested = True - elif (data['AAPL'].short_mavg['price'] < - data['AAPL'].long_mavg['price']) and self.invested: - self.order('AAPL', -100) - self.invested = False - - -def load_close_px(indexes=None, stocks=None): - from pandas.io.data import DataReader - import pytz - from collections import OrderedDict - - if indexes is None: - indexes = {'SPX': '^GSPC'} - if stocks is None: - stocks = ['AAPL', 'GE', 'IBM', 'MSFT', 'XOM', 'AA', 'JNJ', 'PEP', 'KO'] - - start = pd.datetime(1990, 1, 1, 0, 0, 0, 0, pytz.utc) - end = pd.datetime(2000, 1, 1, 0, 0, 0, 0, pytz.utc) - - data = OrderedDict() - - for stock in stocks: - print stock - stkd = DataReader(stock, 'yahoo', start, end).sort_index() - data[stock] = stkd - - for name, ticker in indexes.iteritems(): - print name - stkd = DataReader(ticker, 'yahoo', start, end).sort_index() - data[name] = stkd - - df = pd.DataFrame({key: d['Close'] for key, d in data.iteritems()}) - - df.index = df.index.tz_localize(pytz.utc) - - df.save('close_px.dat') - return df - - -def run((short_window, long_window)): - data = pd.load('close_px.dat') - #data = load_close_px() - myalgo = DMA([0, 1], - amount=100, - short_window=short_window, - long_window=long_window) - stats = myalgo.run(data) - stats['sw'] = short_window - stats['lw'] = long_window - return stats - - -def explore_params(): - sws, lws = np.mgrid[10:20:5, 10:20:5] - - stats_all = map(run, zip(sws.flatten(), lws.flatten())) - stats = pd.concat(stats_all) - returns = stats.groupby(['sw', 'lw']).sum() - - plt.contourf(sws, lws, returns.returns.reshape(sws.shape)) - plt.xlabel('Short window length') - plt.ylabel('Long window length') - plt.savefig('DMA_contour.png') - plt.show() - - -def get_opt_holdings_qp(univ_rets, track_rets): - from cvxopt import matrix - from cvxopt.solvers import qp - # set up the QP for CVXOPT - # .5 x' P x + q'x - # P = 2 * R'R - # q = - 2 * bmk'R - R = univ_rets.values - b = track_rets.values - P = matrix(2 * np.dot(R.T, R)) - q = matrix(-2 * np.dot(R.T, b)) - result = qp(P, q) - if result['status'] != 'optimal': - raise Exception('optimum not reached by QP') - return pd.Series(np.array(result['x']).ravel(), index=univ_rets.columns) - - -def opt_portfolio(cov, budget, min_return): - from cvxopt import matrix - from cvxopt.solvers import qp - n = len(cov) - cov = matrix(2 * cov) - q = matrix(np.zeros(n)) - - h = matrix(budget) # G*x < h - # coneqp - result = qp(cov, q, h=h) - if result['status'] != 'optimal': - raise Exception('optimum not reached by QP') - - return pd.Series(np.array(result['x']).ravel()) - - -def calc_te(weights, univ_rets, track_rets): - port_rets = (univ_rets * weights).sum(1) - return (port_rets - track_rets).std() - - -def plot_returns(port_returns, bmk_returns): - plt.figure() - cum_port = ((1 + port_returns).cumprod() - 1) - cum_bmk = ((1 + bmk_returns).cumprod() - 1) - # cum_port = port_returns.cumsum() - # cum_bmk = bmk_returns.cumsum() - cum_port.plot(label='Portfolio returns') - cum_bmk.plot(label='Benchmark') - plt.title('Portfolio performance') - plt.legend(loc='best') - -#print run((10, 20)) - -import statsmodels.api as sm - - -@batch_transform -def ols_transform(data, spreads): - p0 = data.price['PEP'] - p1 = sm.add_constant(data.price['KO']) - beta, intercept = sm.OLS(p0, p1).fit().params - - spread = (data.price['PEP'] - (beta * data.price['KO'] + intercept))[-1] - - if len(spreads) > 10: - z_score = (spread - np.mean(spreads[-10:])) / np.std(spreads[-10:]) - else: - z_score = np.nan - - spreads.append(spread) - - return z_score - - -class Pairtrade(TradingAlgorithm): - def initialize(self): - self.spreads = [] - self.invested = False - self.ols_transform = ols_transform(refresh_period=10, days=10) - - def handle_data(self, data): - zscore = self.ols_transform.handle_data(data, self.spreads) - - if zscore == np.nan: - return - - if zscore >= 2.0 and not self.invested: - self.order('PEP', int(100 / data['PEP'].price)) - self.order('KO', -int(100 / data['KO'].price)) - elif zscore <= -2.0 and not self.invested: - self.order('KO', -int(100 / data['KO'].price)) - self.order('PEP', int(100 / data['PEP'].price)) - elif abs(zscore) < .5 and self.invested: - pass - - -def run_pairtrade(): - data = load_close_px() - data.save('close_px.dat') - #data = pd.load('close_px.dat') - myalgo = Pairtrade() - stats = myalgo.run(data) - return stats diff --git a/zipline/optimize/factory.py b/zipline/optimize/factory.py deleted file mode 100644 index d7fb5b21..00000000 --- a/zipline/optimize/factory.py +++ /dev/null @@ -1,155 +0,0 @@ -# -# Copyright 2012 Quantopian, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -""" -Factory functions to prepare useful data for optimize tests. - -Author: Thomas V. Wiecki (thomas.wiecki@gmail.com), 2012 -""" -from datetime import timedelta - -from zipline.utils.protocol_utils import ndict -import zipline.protocol as zp - -from zipline.utils.factory import ( - get_next_trading_dt, - create_trading_environment -) -from zipline.sources import SpecificEquityTrades -from zipline.optimize.algorithms import BuySellAlgorithm -from zipline.finance.slippage import FixedSlippage - -from copy import copy -from itertools import cycle - - -def create_updown_trade_source(sid, trade_count, trading_environment, - base_price, amplitude): - """Create the updown trade source. This source emits events with - the price going up and down by the same amount in each - iteration. The trade source is thus perfectly predictable. This is - used for a test case for the optimization code. - - :Arguments: - sid : int - SID of stock to create. - trade_count : int - How many trade events to create (will also influence order count) - trading_environment : TradeEnvironment object - The trading environment to use - (see zipline.factory.create_trading_environment) - base_price : int - The average price that each iteration will hover around. - amplitude : int - How much the price will go up and down each iteration. - - :Returns: - source : SpecificEquityTrades - The trade source emitting up down events. - """ - volume = 1000 - events = [] - price = base_price - amplitude / 2. - - cur = trading_environment.first_open - one_day = timedelta(minutes=1) - - #create iterator to cycle through up and down phases - change = cycle([1, -1]) - - for i in xrange(trade_count + 2): - cur = get_next_trading_dt(cur, one_day, trading_environment) - - event = ndict({ - "type": zp.DATASOURCE_TYPE.TRADE, - "sid": sid, - "price": price, - "volume": volume, - "dt": cur, - }) - - events.append(event) - - price += change.next() * amplitude - - trading_environment.period_end = cur - - source = SpecificEquityTrades(events) - - return source - - -def create_predictable_zipline(config, offset=0, simulate=True): - """Create a test zipline object as specified by config. The - zipline will use the UpDown tradesource which is perfectly - predictable. - - Trade source parameters can be specified inside the config object. - - :Trade source arguments: - config['sid'] : int - SID of stock to create. - config['amplitude'] : int (default 10) - How much the price will go up and down each iteration. - config['base_price'] : int (default 50) - The average price that each iteration will hover around. - config['trade_count'] : int (default 3) - How many trade events to create (will also influence order count) - - If not specified, the BuySellAlgorithm is used by default. This - can be changed by setting config['algorithm']. - - :Arguments: - offset : int (default 0) - The offset parameter specifies how much the BuySellAlgorithm will - order each iteration and is a negative quadratic centered around - 0. Thus, any deviations from 0 will lead to less buy and sell - orders each iteration and ultimately to less compound returns. - simulate : bool (default True) - Whether to call .simulate(blocking=True) on the created zipline - argument. - - :Returns: - zipline : class zipline - created zipline object - config : dict - the config dict used to create the zipline - """ - config = copy(config) - sid = config['sid'] - # remove - amplitude = config.pop('amplitude', 10) - base_price = config.pop('base_price', 50) - trade_count = config.pop('trade_count', 3) - - trading_environment = create_trading_environment() - source = create_updown_trade_source(sid, - trade_count, - trading_environment, - base_price, - amplitude) - - if 'algorithm' not in config: - algorithm = BuySellAlgorithm(sids=[sid], amount=100, offset=offset) - - config['order_count'] = trade_count - 1 - config['trade_count'] = trade_count - config['trade_source'] = source - config['environment'] = trading_environment - config['slippage'] = FixedSlippage() - config['devel'] = True - - return algorithm, config diff --git a/zipline/optimize/profile.sh b/zipline/optimize/profile.sh deleted file mode 100755 index 0e0b602b..00000000 --- a/zipline/optimize/profile.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -python -m cProfile -o example.prof example.py