From f1ce833995301ec12b73708dbaa2452727cd1811 Mon Sep 17 00:00:00 2001 From: Thomas Wiecki Date: Mon, 14 May 2012 18:02:40 -0400 Subject: [PATCH] Moved over files from optimize and incorporated into new infrastructure. Temporarily fixed control_out bug. get_id() bug remains however. --- zipline/component.py | 3 +- zipline/optimize/__init__.py | 0 zipline/optimize/algorithms.py | 49 +++++++++ zipline/optimize/factory.py | 65 ++++++++++++ zipline/sources.py | 18 ++-- zipline/test/test_optimize.py | 185 +++++++++++++++++++++++++++++++++ 6 files changed, 312 insertions(+), 8 deletions(-) create mode 100644 zipline/optimize/__init__.py create mode 100644 zipline/optimize/algorithms.py create mode 100644 zipline/optimize/factory.py create mode 100644 zipline/test/test_optimize.py diff --git a/zipline/component.py b/zipline/component.py index d82c8fb9..f522e946 100644 --- a/zipline/component.py +++ b/zipline/component.py @@ -103,6 +103,7 @@ class Component(object): """ pass + # ------------ # Core Methods # ------------ @@ -243,7 +244,7 @@ class Component(object): self.receive_sync_ack() # blocking self.confirmed = True - + def runtime(self): if self.ready() and self.start_tic and self.stop_tic: return self.stop_tic - self.start_tic diff --git a/zipline/optimize/__init__.py b/zipline/optimize/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/zipline/optimize/algorithms.py b/zipline/optimize/algorithms.py new file mode 100644 index 00000000..d0c84b60 --- /dev/null +++ b/zipline/optimize/algorithms.py @@ -0,0 +1,49 @@ +class BuySellAlgorithm(): + """Algorithm that buys and sells alternatingly. The amount for + each order can be specified. In addition, an offset that will + quadratically reduce the amount that will be bought can be + specified. + + This algorithm is used to test the parameter optimization + framework. If combined with the UpDown trade source, an offset of + 0 will produce maximum returns. + + """ + + def __init__(self, sid, amount, offset): + self.sid = sid + self.amount = amount + self.incr = 0 + self.done = False + self.order = None + self.frame_count = 0 + self.portfolio = None + self.buy_or_sell = -1 + self.offset = offset + self.orders = [] + self.prices = [] + + def initialize(self): + pass + + def set_order(self, order_callable): + self.order = order_callable + + def set_portfolio(self, portfolio): + self.portfolio = portfolio + + def handle_data(self, frame): + order_size = self.buy_or_sell * (self.amount - (self.offset**2)) + self.order(self.sid, order_size) + + #sell next time around. + self.buy_or_sell *= -1 + + self.orders.append(order_size) + #self.prices.append(frame['price']) + + self.frame_count += 1 + self.incr += 1 + + def get_sid_filter(self): + return [self.sid] diff --git a/zipline/optimize/factory.py b/zipline/optimize/factory.py new file mode 100644 index 00000000..6b4ba8ad --- /dev/null +++ b/zipline/optimize/factory.py @@ -0,0 +1,65 @@ +""" +Factory functions to prepare useful data for optimize tests. + +Author: Thomas V. Wiecki (thomas.wiecki@gmail.com), 2012 +""" +from datetime import datetime, timedelta + +import zipline.protocol as zp + +from zipline.test.factory import get_next_trading_dt +from zipline.sources import SpecificEquityTrades +from zipline.optimize.algorithms import BuySellAlgorithm +from zipline.lines import SimulatedTrading + +def create_updown_trade_source(sid, trade_count, trading_environment, start_price, amplitude): + from itertools import cycle + volume = 1000 + events = [] + price = start_price-amplitude/2. + + cur = trading_environment.first_open + one_day = timedelta(days = 1) + + #create iterator to cycle through up and down phases + change = cycle([1,-1]) + + for i in xrange(trade_count + 2): + cur = get_next_trading_dt(cur, one_day, trading_environment) + + event = zp.namedict({ + "type" : zp.DATASOURCE_TYPE.TRADE, + "sid" : sid, + "price" : price, + "volume" : volume, + "dt" : cur, + }) + + events.append(event) + + price += change.next()*amplitude + + trading_environment.period_end = cur + + source = SpecificEquityTrades(sid, events) + + return source + + +def create_predictable_zipline(config, sid=133, amplitude=10, base_price=50, offset=0): + config = deepcopy(config) + trading_environment = create_trading_environment() + source = create_updown_trade_source(sid, + config['trade_count'], + trading_environment, + base_price, + amplitude) + + algo = RegularIntervalBuySellAlgorithm(sid, 100, offset) + config['algorithm'] = algo + config['trade_source'] = source + config['environment'] = trading_environment + zipline = SimulatedTrading.create_test_zipline(**config) + zipline.simulate(blocking=True) + + return zipline diff --git a/zipline/sources.py b/zipline/sources.py index bf08644c..513f0b6a 100644 --- a/zipline/sources.py +++ b/zipline/sources.py @@ -4,6 +4,7 @@ Provides data handlers that can push messages to a zipline.core.DataFeed import datetime import random import pytz +from mock import Mock import zipline.messaging as zm import zipline.protocol as zp @@ -18,9 +19,9 @@ class TradeDataSource(zm.DataSource): :py:func: `zipline.protocol.TRADE_FRAME` :rtype: None """ - + event.source_id = self.get_id - if event.sid in self.filter['SID']: + if event.sid in self.filter['SID']: message = zp.DATASOURCE_FRAME(event) else: blank = zp.namedict({ @@ -28,9 +29,9 @@ class TradeDataSource(zm.DataSource): "source_id" : self.get_id }) message = zp.DATASOURCE_FRAME(blank) - + self.data_socket.send(message) - + class RandomEquityTrades(TradeDataSource): """ @@ -67,7 +68,7 @@ class RandomEquityTrades(TradeDataSource): }) self.send(event) self.incr += 1 - + class SpecificEquityTrades(TradeDataSource): @@ -77,7 +78,7 @@ class SpecificEquityTrades(TradeDataSource): def __init__(self, source_id, event_list): """ - :param event_list: should be a chronologically ordered list of + :param event_list: should be a chronologically ordered list of dictionaries in the following form: event = { @@ -91,10 +92,13 @@ class SpecificEquityTrades(TradeDataSource): self.event_list = event_list self.count = 0 + # TODO temporary hack + self.control_out = Mock() + def get_type(self): zp.COMPONENT_TYPE.SOURCE - + def do_work(self): if(len(self.event_list) == 0): self.signal_done() diff --git a/zipline/test/test_optimize.py b/zipline/test/test_optimize.py new file mode 100644 index 00000000..076909d4 --- /dev/null +++ b/zipline/test/test_optimize.py @@ -0,0 +1,185 @@ +"""Tests for the zipline.finance package""" +import unittest +from unittest2 import TestCase +from nose.tools import timed +from collections import defaultdict +from datetime import datetime, timedelta + +import numpy as np + +from zipline.optimize.factory import create_updown_trade_source +import zipline.test.factory as factory +import zipline.util as qutil + +from zipline.simulator import AddressAllocator, Simulator +from zipline.optimize.algorithms import BuySellAlgorithm +from zipline.finance.trading import TradingEnvironment +from zipline.lines import SimulatedTrading +from zipline.finance.trading import SIMULATION_STYLE + +DEFAULT_TIMEOUT = 15 # seconds +EXTENDED_TIMEOUT = 90 + +allocator = AddressAllocator(1000) + +class FinanceTestCase(TestCase): + + leased_sockets = defaultdict(list) + + def setUp(self): + qutil.configure_logging() + self.zipline_test_config = { + 'allocator':allocator, + 'sid':133 + } + + @timed(DEFAULT_TIMEOUT) + def test_buysell(self): + #generate events + trade_count = 50 + sid = 133 + base_price = 50 + amplitude = 6 + offset = 0 + self.zipline_test_config['order_count'] = trade_count - 1 + self.zipline_test_config['trade_count'] = trade_count + self.zipline_test_config['simulation_style'] = \ + SIMULATION_STYLE.FIXED_SLIPPAGE + + trading_environment = factory.create_trading_environment() + source = factory.create_updown_trade_source(sid, + trade_count, + trading_environment, + base_price, + amplitude + ) + + prices = np.array([event.price for event in source.event_list]) + max_price_idx = np.where(prices==prices.max())[0] + min_price_idx = np.where(prices==prices.min())[0] + self.assertTrue(np.all(max_price_idx % 2 == 1), + "Maximum prices are not periodic." + ) + self.assertTrue(np.all(min_price_idx % 2 == 0), + "Minimum prices are not periodic." + ) + self.assertEqual(prices.max(), base_price+amplitude/2., + "Maximum price does not equal expected maximum price." + ) + self.assertEqual(prices.min(), base_price-amplitude/2., + "Minimum price does not equal expected maximum price." + ) + + algo = BuySellAlgorithm(sid, 100, 0) + + self.zipline_test_config['trade_source'] = source + self.zipline_test_config['algorithm'] = algo + self.zipline_test_config['environment'] = trading_environment + + zipline = SimulatedTrading.create_test_zipline(**self.zipline_test_config) + zipline.simulate(blocking=True) + + orders = np.asarray(algo.orders) + max_order_idx = np.where(orders==orders.max())[0] + min_order_idx = np.where(orders==orders.min())[0] + + self.assertTrue(np.all(max_order_idx % 2 == 1), + "Maximum orders are not periodic." + ) + self.assertTrue(np.all(min_order_idx % 2 == 0), + "Minimum orders are not periodic." + ) + self.assertTrue(np.all(max_order_idx == max_price_idx), + "Algorithm did not buy when price was going to drop." + ) + self.assertTrue(np.all(min_order_idx == min_price_idx), + "Algorithm did not sell when price was going to increase." + ) + + def test_buysell_concave(self): + #generate events + trade_count = 6 + sid = 133 + amplitude = 30 + base_price = 50 + self.zipline_test_config['order_count'] = trade_count - 1 + self.zipline_test_config['trade_count'] = trade_count + self.zipline_test_config['simulation_style'] = \ + SIMULATION_STYLE.FIXED_SLIPPAGE + + #test whether return-function is concave wrt repeats. + test_offsets = np.arange(-9, 9, 1.) + supposed_max = np.zeros(len(test_offsets), dtype=bool) + supposed_max[len(test_offsets) // 2] = True + + compound_returns = np.empty(len(test_offsets)) + ziplines = [] + for i, test_offset in enumerate(test_offsets): + trading_environment = factory.create_trading_environment() + source = factory.create_updown_trade_source(sid, + trade_count, + trading_environment, + base_price, + amplitude + ) + + algo = BuySellAlgorithm(sid, 100, test_offset) + self.zipline_test_config['algorithm'] = algo + self.zipline_test_config['trade_source'] = source + self.zipline_test_config['environment'] = trading_environment + zipline = SimulatedTrading.create_test_zipline(**self.zipline_test_config) + zipline.simulate(blocking=True) + ziplines.append(zipline) + compound_returns[i] = zipline.get_cumulative_performance()['returns'] + + self.assertTrue(np.all(compound_returns[supposed_max] > compound_returns[np.logical_not(supposed_max)]), + "Maximum compound returns are not where they are supposed to be." + ) + + # test for concavity + max_idx = np.where(supposed_max)[0][0] + idx = np.array([max_idx, max_idx]) + for i in range((len(test_offsets)-1)/2): + # going outwards, returns must decrease + self.assertTrue(compound_returns[idx[0]-1] < compound_returns[idx[0]], + "Compound returns are not convex." + ) + self.assertTrue(compound_returns[idx[1]+1] < compound_returns[idx[1]], + "Compound returns are not convex." + ) + idx[0] -= 1 + idx[1] += 1 + + + def test_optimize(self): + def simulate(offset): + #generate events + trade_count = 3 + sid = 133 + amplitude = 10 + base_price = 50 + self.zipline_test_config['order_count'] = trade_count - 1 + self.zipline_test_config['trade_count'] = trade_count + self.zipline_test_config['simulation_style'] = \ + SIMULATION_STYLE.FIXED_SLIPPAGE + trading_environment = factory.create_trading_environment() + source = create_updown_trade_source(sid, + trade_count, + trading_environment, + base_price, + amplitude + ) + + algo = BuySellAlgorithm(sid, 100, offset) + self.zipline_test_config['algorithm'] = algo + self.zipline_test_config['trade_source'] = source + self.zipline_test_config['environment'] = trading_environment + zipline = SimulatedTrading.create_test_zipline(**self.zipline_test_config) + zipline.simulate(blocking=True) + zipline.shutdown() + #function is getting minimized, so have return negative. + return -zipline.get_cumulative_performance()['returns'] + + from scipy import optimize + opt = optimize.fmin_powell(simulate, 1.5) + np.testing.assert_almost_equal(opt, 0, 5)