diff --git a/etc/requirements.txt b/etc/requirements.txt index c3d82d5b..ac475f3e 100644 --- a/etc/requirements.txt +++ b/etc/requirements.txt @@ -17,3 +17,6 @@ six==1.2.0 # For fetching remote data requests==1.1.0 + +# For remaining sane when coping with dates +Delorean==0.1.6 diff --git a/tests/test_algorithm.py b/tests/test_algorithm.py index 621c45ad..be953be2 100644 --- a/tests/test_algorithm.py +++ b/tests/test_algorithm.py @@ -29,17 +29,17 @@ from zipline.transforms import MovingAverage class TestRecordAlgorithm(TestCase): def setUp(self): - self.trading_environment = factory.create_trading_environment() + self.sim_params = factory.create_simulation_parameters() trade_history = factory.create_trade_history( 133, [10.0, 10.0, 11.0, 11.0], [100, 100, 100, 300], timedelta(days=1), - self.trading_environment + self.sim_params ) self.source = SpecificEquityTrades(event_list=trade_history) self.df_source, self.df = \ - factory.create_test_df_source(self.trading_environment) + factory.create_test_df_source(self.sim_params) def test_record_incr(self): algo = RecordAlgorithm() @@ -51,7 +51,7 @@ class TestRecordAlgorithm(TestCase): class TestTransformAlgorithm(TestCase): def setUp(self): setup_logger(self) - self.trading_environment = factory.create_trading_environment() + self.sim_params = factory.create_simulation_parameters() setup_logger(self) trade_history = factory.create_trade_history( @@ -59,35 +59,48 @@ class TestTransformAlgorithm(TestCase): [10.0, 10.0, 11.0, 11.0], [100, 100, 100, 300], timedelta(days=1), - self.trading_environment + self.sim_params ) self.source = SpecificEquityTrades(event_list=trade_history) self.df_source, self.df = \ - factory.create_test_df_source(self.trading_environment) + factory.create_test_df_source(self.sim_params) self.panel_source, self.panel = \ - factory.create_test_panel_source(self.trading_environment) + factory.create_test_panel_source(self.sim_params) def test_source_as_input(self): - algo = TestRegisterTransformAlgorithm(sids=[133]) + algo = TestRegisterTransformAlgorithm( + self.sim_params, + sids=[133] + ) algo.run(self.source) self.assertEqual(len(algo.sources), 1) assert isinstance(algo.sources[0], SpecificEquityTrades) def test_multi_source_as_input_no_start_end(self): - algo = TestRegisterTransformAlgorithm(sids=[133]) + algo = TestRegisterTransformAlgorithm( + self.sim_params, + sids=[133] + ) + with self.assertRaises(AssertionError): algo.run([self.source, self.df_source]) def test_multi_source_as_input(self): - algo = TestRegisterTransformAlgorithm(sids=[0, 1, 133]) + algo = TestRegisterTransformAlgorithm( + self.sim_params, + sids=[0, 1, 133] + ) algo.run([self.source, self.df_source], start=self.df.index[0], end=self.df.index[-1]) self.assertEqual(len(algo.sources), 2) def test_df_as_input(self): - algo = TestRegisterTransformAlgorithm(sids=[0, 1]) + algo = TestRegisterTransformAlgorithm( + self.sim_params, + sids=[0, 1] + ) algo.run(self.df) assert isinstance(algo.sources[0], DataFrameSource) @@ -97,14 +110,22 @@ class TestTransformAlgorithm(TestCase): assert isinstance(algo.sources[0], DataPanelSource) def test_run_twice(self): - algo = TestRegisterTransformAlgorithm(sids=[0, 1]) + algo = TestRegisterTransformAlgorithm( + self.sim_params, + sids=[0, 1] + ) + res1 = algo.run(self.df) res2 = algo.run(self.df) np.testing.assert_array_equal(res1, res2) def test_transform_registered(self): - algo = TestRegisterTransformAlgorithm(sids=[133]) + algo = TestRegisterTransformAlgorithm( + self.sim_params, + sids=[133] + ) + algo.run(self.source) assert 'mavg' in algo.registered_transforms assert algo.registered_transforms['mavg']['args'] == (['price'],) @@ -113,15 +134,24 @@ class TestTransformAlgorithm(TestCase): assert algo.registered_transforms['mavg']['class'] is MovingAverage def test_data_frequency_setting(self): - algo = TestRegisterTransformAlgorithm(data_frequency='daily') + algo = TestRegisterTransformAlgorithm( + self.sim_params, + data_frequency='daily' + ) self.assertEqual(algo.data_frequency, 'daily') self.assertEqual(algo.annualizer, 250) - algo = TestRegisterTransformAlgorithm(data_frequency='minute') + algo = TestRegisterTransformAlgorithm( + self.sim_params, + data_frequency='minute' + ) self.assertEqual(algo.data_frequency, 'minute') self.assertEqual(algo.annualizer, 250 * 6 * 60) - algo = TestRegisterTransformAlgorithm(data_frequency='minute', - annualizer=10) + algo = TestRegisterTransformAlgorithm( + self.sim_params, + data_frequency='minute', + annualizer=10 + ) self.assertEqual(algo.data_frequency, 'minute') self.assertEqual(algo.annualizer, 10) diff --git a/tests/test_algorithm_gen.py b/tests/test_algorithm_gen.py index 9da2fbb0..6de78158 100644 --- a/tests/test_algorithm_gen.py +++ b/tests/test_algorithm_gen.py @@ -84,20 +84,42 @@ class AlgorithmGeneratorTestCase(TestCase): Ensure the pipeline of generators are in sync, at least as far as their current dates. """ - algo = TestAlgo(self) - trading_environment = factory.create_trading_environment( + sim_params = factory.create_simulation_parameters( start=datetime(2011, 7, 30, tzinfo=pytz.utc), end=datetime(2012, 7, 30, tzinfo=pytz.utc) ) + algo = TestAlgo(self, sim_params=sim_params) trade_source = factory.create_daily_trade_source( [8229], 200, - trading_environment + sim_params ) algo.set_sources([trade_source]) - gen = algo.get_generator(trading_environment) + gen = algo.get_generator() self.assertTrue(list(gen)) self.assertTrue(algo.slippage.latest_date) self.assertTrue(algo.latest_date) + + @timed(DEFAULT_TIMEOUT) + def test_progress(self): + """ + Ensure the pipeline of generators are in sync, at least as far as + their current dates. + """ + sim_params = factory.create_simulation_parameters( + start=datetime(2008, 1, 1, tzinfo=pytz.utc), + end=datetime(2008, 1, 5, tzinfo=pytz.utc) + ) + algo = TestAlgo(self, sim_params=sim_params) + trade_source = factory.create_daily_trade_source( + [8229], + 3, + sim_params + ) + algo.set_sources([trade_source]) + + gen = algo.get_generator() + results = list(gen) + self.assertEqual(results[-2]['progress'], 1.0) diff --git a/tests/test_exception_handling.py b/tests/test_exception_handling.py index be226d81..bb6263ff 100644 --- a/tests/test_exception_handling.py +++ b/tests/test_exception_handling.py @@ -16,6 +16,8 @@ from unittest import TestCase import zipline.utils.simfactory as simfactory +import zipline.utils.factory as factory + from zipline.test_algorithms import ( ExceptionAlgorithm, DivByZeroAlgorithm, @@ -83,7 +85,8 @@ class ExceptionTestCase(TestCase): self.zipline_test_config['algorithm'] = \ ExceptionAlgorithm( 'handle_data', - self.zipline_test_config['sid'] + self.zipline_test_config['sid'], + sim_params=factory.create_simulation_parameters() ) zipline = simfactory.create_test_zipline( @@ -102,7 +105,8 @@ class ExceptionTestCase(TestCase): # ---------- self.zipline_test_config['algorithm'] = \ DivByZeroAlgorithm( - self.zipline_test_config['sid'] + self.zipline_test_config['sid'], + sim_params=factory.create_simulation_parameters() ) zipline = simfactory.create_test_zipline( @@ -124,7 +128,8 @@ class ExceptionTestCase(TestCase): # ---------- self.zipline_test_config['algorithm'] = \ SetPortfolioAlgorithm( - self.zipline_test_config['sid'] + self.zipline_test_config['sid'], + sim_params=factory.create_simulation_parameters() ) zipline = simfactory.create_test_zipline( diff --git a/tests/test_finance.py b/tests/test_finance.py index af4fc91d..bb0c3f52 100644 --- a/tests/test_finance.py +++ b/tests/test_finance.py @@ -28,7 +28,9 @@ from nose.tools import timed import zipline.utils.factory as factory import zipline.utils.simfactory as simfactory -from zipline.finance.trading import TradingEnvironment +import zipline.finance.trading as trading +from zipline.finance.trading import SimulationParameters + from zipline.finance.performance import PerformanceTracker from zipline.utils.protocol_utils import ndict from zipline.finance.trading import TransactionSimulator @@ -56,11 +58,11 @@ class FinanceTestCase(TestCase): @timed(DEFAULT_TIMEOUT) def test_factory_daily(self): - trading_environment = factory.create_trading_environment() + sim_params = factory.create_simulation_parameters() trade_source = factory.create_daily_trade_source( [133], 200, - trading_environment + sim_params ) prev = None for trade in trade_source: @@ -70,16 +72,6 @@ class FinanceTestCase(TestCase): @timed(DEFAULT_TIMEOUT) def test_trading_environment(self): - benchmark_returns, treasury_curves = \ - factory.load_market_data() - - env = TradingEnvironment( - benchmark_returns, - treasury_curves, - period_start=datetime(2008, 1, 1, tzinfo=pytz.utc), - period_end=datetime(2008, 12, 31, tzinfo=pytz.utc), - capital_base=100000, - ) #holidays taken from: http://www.nyse.com/press/1191407641943.html new_years = datetime(2008, 1, 1, tzinfo=pytz.utc) mlk_day = datetime(2008, 1, 21, tzinfo=pytz.utc) @@ -107,23 +99,27 @@ class FinanceTestCase(TestCase): ] for holiday in holidays: - self.assertTrue(not env.is_trading_day(holiday)) + self.assertTrue(not trading.environment.is_trading_day(holiday)) first_trading_day = datetime(2008, 1, 2, tzinfo=pytz.utc) last_trading_day = datetime(2008, 12, 31, tzinfo=pytz.utc) workdays = [first_trading_day, last_trading_day] for workday in workdays: - self.assertTrue(env.is_trading_day(workday)) + self.assertTrue(trading.environment.is_trading_day(workday)) + + def test_simulation_parameters(self): + env = SimulationParameters( + period_start=datetime(2008, 1, 1, tzinfo=pytz.utc), + period_end=datetime(2008, 12, 31, tzinfo=pytz.utc), + capital_base=100000, + ) self.assertTrue(env.last_close.month == 12) self.assertTrue(env.last_close.day == 31) @timed(DEFAULT_TIMEOUT) - def test_trading_environment_days_in_period(self): - - benchmark_returns, treasury_curves = \ - factory.load_market_data() + def test_sim_params_days_in_period(self): # January 2008 # Su Mo Tu We Th Fr Sa @@ -133,9 +129,7 @@ class FinanceTestCase(TestCase): # 20 21 22 23 24 25 26 # 27 28 29 30 31 - env = TradingEnvironment( - benchmark_returns, - treasury_curves, + env = SimulationParameters( period_start=datetime(2007, 12, 31, tzinfo=pytz.utc), period_end=datetime(2008, 1, 7, tzinfo=pytz.utc), capital_base=100000, @@ -154,10 +148,9 @@ class FinanceTestCase(TestCase): ) num_expected_trading_days = 5 - self.assertEquals(num_expected_trading_days, env.days_in_period) np.testing.assert_array_equal(expected_trading_days, - env.period_trading_days) + env.trading_days.tolist()) @timed(EXTENDED_TIMEOUT) def test_full_zipline(self): @@ -288,18 +281,18 @@ class FinanceTestCase(TestCase): complete_fill = params.get('complete_fill') sid = 1 - trading_environment = factory.create_trading_environment() + sim_params = factory.create_simulation_parameters() trade_sim = TransactionSimulator() price = [10.1] * trade_count volume = [100] * trade_count - start_date = trading_environment.first_open + start_date = sim_params.first_open generated_trades = factory.create_trade_history( sid, price, volume, trade_interval, - trading_environment + sim_params ) if alternate: @@ -337,7 +330,7 @@ class FinanceTestCase(TestCase): self.assertEqual(order.sid, sid) self.assertEqual(order.amount, order_amount * alternator ** i) - tracker = PerformanceTracker(trading_environment) + tracker = PerformanceTracker(sim_params) # this approximates the loop inside TradingSimulationClient transactions = [] diff --git a/tests/test_perf_tracking.py b/tests/test_perf_tracking.py index ebafcbfa..b9187130 100644 --- a/tests/test_perf_tracking.py +++ b/tests/test_perf_tracking.py @@ -27,9 +27,8 @@ import zipline.finance.performance as perf from zipline.utils.protocol_utils import ndict from zipline.gens.composites import date_sorted_sources -from zipline.finance.trading import TradingEnvironment -from zipline.utils.factory import create_random_trading_environment - +from zipline.finance.trading import SimulationParameters +from zipline.utils.factory import create_random_simulation_parameters onesec = datetime.timedelta(seconds=1) oneday = datetime.timedelta(days=1) @@ -40,10 +39,10 @@ class TestDividendPerformance(unittest.TestCase): def setUp(self): - self.trading_environment, self.dt, self.end_dt = \ - create_random_trading_environment() + self.sim_params, self.dt, self.end_dt = \ + create_random_simulation_parameters() - self.trading_environment.capital_base = 10e3 + self.sim_params.capital_base = 10e3 def test_long_position_receives_dividend(self): #post some trades in the market @@ -52,7 +51,7 @@ class TestDividendPerformance(unittest.TestCase): [10, 10, 10, 10, 10], [100, 100, 100, 100, 100], oneday, - self.trading_environment + self.sim_params ) dividend = factory.create_dividend( @@ -71,7 +70,7 @@ class TestDividendPerformance(unittest.TestCase): txn = factory.create_txn(1, 10.0, 100, events[0].dt) events[0].TRANSACTION = txn events.insert(1, dividend) - perf_tracker = perf.PerformanceTracker(self.trading_environment) + perf_tracker = perf.PerformanceTracker(self.sim_params) transformed_events = list(perf_tracker.transform( ((event.dt, [event]) for event in events)) ) @@ -114,7 +113,7 @@ class TestDividendPerformance(unittest.TestCase): [10, 10, 10, 10, 10], [100, 100, 100, 100, 100], oneday, - self.trading_environment + self.sim_params ) dividend = factory.create_dividend( @@ -128,7 +127,7 @@ class TestDividendPerformance(unittest.TestCase): events.insert(1, dividend) txn = factory.create_txn(1, 10.0, 100, events[3].dt) events[3].TRANSACTION = txn - perf_tracker = perf.PerformanceTracker(self.trading_environment) + perf_tracker = perf.PerformanceTracker(self.sim_params) transformed_events = list(perf_tracker.transform( ((event.dt, [event]) for event in events)) ) @@ -162,7 +161,7 @@ class TestDividendPerformance(unittest.TestCase): [10, 10, 10, 10, 10], [100, 100, 100, 100, 100], oneday, - self.trading_environment + self.sim_params ) dividend = factory.create_dividend( @@ -178,7 +177,7 @@ class TestDividendPerformance(unittest.TestCase): sell_txn = factory.create_txn(1, 10.0, -100, events[2].dt) events[2].TRANSACTION = sell_txn events.insert(1, dividend) - perf_tracker = perf.PerformanceTracker(self.trading_environment) + perf_tracker = perf.PerformanceTracker(self.sim_params) transformed_events = list(perf_tracker.transform( ((event.dt, [event]) for event in events)) ) @@ -212,7 +211,7 @@ class TestDividendPerformance(unittest.TestCase): [10, 10, 10, 10, 10, 10], [100, 100, 100, 100, 100, 100], oneday, - self.trading_environment + self.sim_params ) dividend = factory.create_dividend( @@ -228,7 +227,7 @@ class TestDividendPerformance(unittest.TestCase): sell_txn = factory.create_txn(1, 10.0, -100, events[2].dt) events[2].TRANSACTION = sell_txn events.insert(1, dividend) - perf_tracker = perf.PerformanceTracker(self.trading_environment) + perf_tracker = perf.PerformanceTracker(self.sim_params) transformed_events = list(perf_tracker.transform( ((event.dt, [event]) for event in events)) ) @@ -262,7 +261,7 @@ class TestDividendPerformance(unittest.TestCase): [10, 10, 10, 10, 10], [100, 100, 100, 100, 100], oneday, - self.trading_environment + self.sim_params ) dividend = factory.create_dividend( @@ -276,7 +275,7 @@ class TestDividendPerformance(unittest.TestCase): buy_txn = factory.create_txn(1, 10.0, 100, events[1].dt) events[1].TRANSACTION = buy_txn events.insert(1, dividend) - perf_tracker = perf.PerformanceTracker(self.trading_environment) + perf_tracker = perf.PerformanceTracker(self.sim_params) transformed_events = list(perf_tracker.transform( ((event.dt, [event]) for event in events)) ) @@ -313,7 +312,7 @@ class TestDividendPerformance(unittest.TestCase): [10, 10, 10, 10, 10], [100, 100, 100, 100, 100], oneday, - self.trading_environment + self.sim_params ) dividend = factory.create_dividend( @@ -327,7 +326,7 @@ class TestDividendPerformance(unittest.TestCase): txn = factory.create_txn(1, 10.0, -100, self.dt+oneday) events[0].TRANSACTION = txn events.insert(0, dividend) - perf_tracker = perf.PerformanceTracker(self.trading_environment) + perf_tracker = perf.PerformanceTracker(self.sim_params) transformed_events = list(perf_tracker.transform( ((event.dt, [event]) for event in events)) ) @@ -361,7 +360,7 @@ class TestDividendPerformance(unittest.TestCase): [10, 10, 10, 10, 10], [100, 100, 100, 100, 100], oneday, - self.trading_environment + self.sim_params ) dividend = factory.create_dividend( @@ -373,7 +372,7 @@ class TestDividendPerformance(unittest.TestCase): ) events.insert(1, dividend) - perf_tracker = perf.PerformanceTracker(self.trading_environment) + perf_tracker = perf.PerformanceTracker(self.sim_params) transformed_events = list(perf_tracker.transform( ((event.dt, [event]) for event in events)) ) @@ -404,8 +403,8 @@ class TestDividendPerformance(unittest.TestCase): class TestPositionPerformance(unittest.TestCase): def setUp(self): - self.trading_environment, self.dt, self.end_dt = \ - create_random_trading_environment() + self.sim_params, self.dt, self.end_dt = \ + create_random_simulation_parameters() def test_long_position(self): """ @@ -418,7 +417,7 @@ class TestPositionPerformance(unittest.TestCase): [10, 10, 10, 11], [100, 100, 100, 100], onesec, - self.trading_environment + self.sim_params ) txn = factory.create_txn(1, 10.0, 100, self.dt + onesec) @@ -487,7 +486,7 @@ single short-sale transaction""" [10, 10, 10, 11, 10, 9], [100, 100, 100, 100, 100, 100], onesec, - self.trading_environment + self.sim_params ) trades_1 = trades[:-2] @@ -677,7 +676,7 @@ trade after cover""" [10, 10, 10, 11, 9, 8, 7, 8, 9, 10], [100, 100, 100, 100, 100, 100, 100, 100, 100, 100], onesec, - self.trading_environment + self.sim_params ) short_txn = factory.create_txn( @@ -756,7 +755,7 @@ shares in position" [10, 11, 11, 12], [100, 100, 100, 100], onesec, - self.trading_environment + self.sim_params ) transactions = factory.create_txn_history( @@ -764,7 +763,7 @@ shares in position" [10, 11, 11, 12], [100, 100, 100, 100], onesec, - self.trading_environment + self.sim_params ) pp = perf.PerformancePeriod(1000.0) @@ -914,12 +913,7 @@ class TestPerformanceTracker(unittest.TestCase): volume = [100] * trade_count trade_time_increment = datetime.timedelta(days=1) - benchmark_returns, treasury_curves = \ - factory.load_market_data() - - trading_environment = TradingEnvironment( - benchmark_returns, - treasury_curves, + sim_params = SimulationParameters( period_start=start_dt, period_end=end_dt ) @@ -929,7 +923,7 @@ class TestPerformanceTracker(unittest.TestCase): price_list, volume, trade_time_increment, - trading_environment, + sim_params, source_id="factory1" ) @@ -941,7 +935,7 @@ class TestPerformanceTracker(unittest.TestCase): price2_list, volume, trade_time_increment, - trading_environment, + sim_params, source_id="factory2" ) # 'middle' start of 3 depends on number of days == 7 @@ -962,19 +956,19 @@ class TestPerformanceTracker(unittest.TestCase): del trade_history[-days_to_delete.end:] del trade_history2[-days_to_delete.end:] - trading_environment.first_open = \ - trading_environment.calculate_first_open() - trading_environment.last_close = \ - trading_environment.calculate_last_close() - trading_environment.capital_base = 1000.0 - trading_environment.frame_index = [ + sim_params.first_open = \ + sim_params.calculate_first_open() + sim_params.last_close = \ + sim_params.calculate_last_close() + sim_params.capital_base = 1000.0 + sim_params.frame_index = [ 'sid', 'volume', 'dt', 'price', 'changed'] perf_tracker = perf.PerformanceTracker( - trading_environment + sim_params ) events = date_sorted_sources(trade_history, trade_history2) @@ -1007,7 +1001,7 @@ class TestPerformanceTracker(unittest.TestCase): perf_tracker.cumulative_risk_metrics.end_date) self.assertEqual(len(perf_messages), - trading_environment.days_in_period) + sim_params.days_in_period) def event_with_txn(self, event, no_txn_dt): #create a transaction for all but diff --git a/tests/test_risk.py b/tests/test_risk.py index 5b3e08ef..e62a4a9d 100644 --- a/tests/test_risk.py +++ b/tests/test_risk.py @@ -20,7 +20,7 @@ import pytz import zipline.finance.risk as risk from zipline.utils import factory -from zipline.finance.trading import TradingEnvironment +from zipline.finance.trading import SimulationParameters class Risk(unittest.TestCase): @@ -37,12 +37,7 @@ class Risk(unittest.TestCase): end_date = datetime.datetime( year=2006, month=12, day=31, tzinfo=pytz.utc) - self.benchmark_returns, self.treasury_curves = \ - factory.load_market_data() - - self.trading_env = TradingEnvironment( - self.benchmark_returns, - self.treasury_curves, + self.sim_params = SimulationParameters( period_start=start_date, period_end=end_date ) @@ -54,12 +49,12 @@ class Risk(unittest.TestCase): self.algo_returns_06 = factory.create_returns_from_list( RETURNS, - self.trading_env + self.sim_params ) self.metrics_06 = risk.RiskReport( self.algo_returns_06, - self.trading_env + self.sim_params ) start_08 = datetime.datetime( @@ -76,9 +71,7 @@ class Risk(unittest.TestCase): day=31, tzinfo=pytz.utc ) - self.trading_env08 = TradingEnvironment( - self.benchmark_returns, - self.treasury_curves, + self.sim_params08 = SimulationParameters( period_start=start_08, period_end=end_08 ) @@ -88,24 +81,23 @@ class Risk(unittest.TestCase): def test_factory(self): returns = [0.1] * 100 - r_objects = factory.create_returns_from_list(returns, self.trading_env) + r_objects = factory.create_returns_from_list(returns, self.sim_params) self.assertTrue(r_objects[-1].date <= datetime.datetime( year=2006, month=12, day=31, tzinfo=pytz.utc)) def test_drawdown(self): returns = factory.create_returns_from_list( - [1.0, -0.5, 0.8, .17, 1.0, -0.1, -0.45], self.trading_env) + [1.0, -0.5, 0.8, .17, 1.0, -0.1, -0.45], self.sim_params) #200, 100, 180, 210.6, 421.2, 379.8, 208.494 metrics = risk.RiskMetricsBatch(returns[0].date, returns[-1].date, - returns, - self.trading_env) + returns) self.assertEqual(metrics.max_drawdown, 0.505) def test_benchmark_returns_06(self): - returns = factory.create_returns_from_range(self.trading_env) - metrics = risk.RiskReport(returns, self.trading_env) + returns = factory.create_returns_from_range(self.sim_params) + metrics = risk.RiskReport(returns, self.sim_params) self.assertEqual([round(x.benchmark_period_returns, 4) for x in metrics.month_periods], [0.0255, @@ -146,16 +138,16 @@ class Risk(unittest.TestCase): [0.1407]) def test_trading_days_06(self): - returns = factory.create_returns_from_range(self.trading_env) - metrics = risk.RiskReport(returns, self.trading_env) + returns = factory.create_returns_from_range(self.sim_params) + metrics = risk.RiskReport(returns, self.sim_params) self.assertEqual([x.trading_days for x in metrics.year_periods], [251]) self.assertEqual([x.trading_days for x in metrics.month_periods], [20, 19, 23, 19, 22, 22, 20, 23, 20, 22, 21, 20]) def test_benchmark_volatility_06(self): - returns = factory.create_returns_from_range(self.trading_env) - metrics = risk.RiskReport(returns, self.trading_env) + returns = factory.create_returns_from_range(self.sim_params) + metrics = risk.RiskReport(returns, self.sim_params) self.assertEqual([round(x.benchmark_volatility, 3) for x in metrics.month_periods], [0.031, @@ -588,8 +580,8 @@ class Risk(unittest.TestCase): [0.0000399]) def test_benchmark_returns_08(self): - returns = factory.create_returns_from_range(self.trading_env08) - metrics = risk.RiskReport(returns, self.trading_env08) + returns = factory.create_returns_from_range(self.sim_params08) + metrics = risk.RiskReport(returns, self.sim_params08) monthly = [round(x.benchmark_period_returns, 3) for x in metrics.month_periods] @@ -636,8 +628,8 @@ class Risk(unittest.TestCase): [-0.353]) def test_trading_days_08(self): - returns = factory.create_returns_from_range(self.trading_env08) - metrics = risk.RiskReport(returns, self.trading_env08) + returns = factory.create_returns_from_range(self.sim_params08) + metrics = risk.RiskReport(returns, self.sim_params08) self.assertEqual([x.trading_days for x in metrics.year_periods], [253]) @@ -645,8 +637,8 @@ class Risk(unittest.TestCase): [21, 20, 20, 22, 21, 21, 22, 21, 21, 23, 19, 22]) def test_benchmark_volatility_08(self): - returns = factory.create_returns_from_range(self.trading_env08) - metrics = risk.RiskReport(returns, self.trading_env08) + returns = factory.create_returns_from_range(self.sim_params08) + metrics = risk.RiskReport(returns, self.sim_params08) self.assertEqual([round(x.benchmark_volatility, 3) for x in metrics.month_periods], [0.069, @@ -692,8 +684,8 @@ class Risk(unittest.TestCase): [0.391]) def test_treasury_returns_06(self): - returns = factory.create_returns_from_range(self.trading_env) - metrics = risk.RiskReport(returns, self.trading_env) + returns = factory.create_returns_from_range(self.sim_params) + metrics = risk.RiskReport(returns, self.sim_params) self.assertEqual([round(x.treasury_period_return, 4) for x in metrics.month_periods], [0.0037, @@ -752,16 +744,14 @@ class Risk(unittest.TestCase): #1992 and 1996 were leap years total_days = 365 * 5 + 2 end = start + datetime.timedelta(days=total_days) - trading_env90s = TradingEnvironment( - self.benchmark_returns, - self.treasury_curves, + sim_params90s = SimulationParameters( period_start=start, period_end=end ) - returns = factory.create_returns(total_days, trading_env90s) + returns = factory.create_returns(total_days, sim_params90s) returns = returns[:-10] # truncate the returns series to end mid-month - metrics = risk.RiskReport(returns, trading_env90s) + metrics = risk.RiskReport(returns, sim_params90s) total_months = 60 self.check_metrics(metrics, total_months, start) @@ -773,8 +763,8 @@ class Risk(unittest.TestCase): # and i think this func is [start,end) ld = calendar.leapdays(start_date.year, start_date.year + years + 1) - returns = factory.create_returns(365 * years + ld, self.trading_env08) - metrics = risk.RiskReport(returns, self.trading_env) + returns = factory.create_returns(365 * years + ld, self.sim_params08) + metrics = risk.RiskReport(returns, self.sim_params) total_months = years * 12 self.check_metrics(metrics, total_months, start_date) diff --git a/tests/test_risk_compare_batch_iterative.py b/tests/test_risk_compare_batch_iterative.py index 4eec911e..d98fbe94 100644 --- a/tests/test_risk_compare_batch_iterative.py +++ b/tests/test_risk_compare_batch_iterative.py @@ -21,9 +21,9 @@ import pytz import numpy as np import zipline.finance.risk as risk -from zipline.utils import factory from zipline.finance.trading import TradingEnvironment +import zipline.finance.trading as trading from test_risk import RETURNS @@ -43,22 +43,13 @@ class RiskCompareIterativeToBatch(unittest.TestCase): tzinfo=pytz.utc) self.end_date = datetime.datetime( year=2006, month=12, day=31, tzinfo=pytz.utc) - self.benchmark_returns, self.treasury_curves = \ - factory.load_market_data() - - self.trading_env = TradingEnvironment( - self.benchmark_returns, - self.treasury_curves, - period_start=self.start_date, - period_end=self.end_date, - capital_base=1000.0 - ) + # setup the default trading environment + trading.environment = TradingEnvironment() self.oneday = datetime.timedelta(days=1) def test_risk_metrics_returns(self): - risk_metrics_refactor = risk.RiskMetricsIterative( - self.start_date, self.trading_env) + risk_metrics_refactor = risk.RiskMetricsIterative(self.start_date) todays_date = self.start_date @@ -73,15 +64,14 @@ class RiskCompareIterativeToBatch(unittest.TestCase): # Move forward day counter to next trading day todays_date += self.oneday - while not self.trading_env.is_trading_day(todays_date): + while not trading.environment.is_trading_day(todays_date): todays_date += self.oneday try: risk_metrics_original = risk.RiskMetricsBatch( start_date=self.start_date, end_date=todays_date, - returns=cur_returns, - trading_environment=self.trading_env + returns=cur_returns ) except Exception as e: #assert that when original raises exception, same diff --git a/tests/test_tradingcalendar.py b/tests/test_tradingcalendar.py index 0a9b7e03..6c29c077 100644 --- a/tests/test_tradingcalendar.py +++ b/tests/test_tradingcalendar.py @@ -17,10 +17,30 @@ from unittest import TestCase from zipline.utils import tradingcalendar import pytz import datetime +from zipline.finance.trading import TradingEnvironment class TestTradingCalendar(TestCase): + def test_calendar_vs_environment(self): + """ + test_calendar_vs_environment checks whether the + historical data from yahoo matches our rule based system. + handy, if not canonical, reference: + http://www.chronos-st.org/NYSE_Observed_Holidays-1885-Present.html + """ + + env = TradingEnvironment() + env_start_index = \ + env.trading_days.searchsorted(tradingcalendar.start) + env_days = env.trading_days[env_start_index:] + diff = env_days - tradingcalendar.trading_days + self.assertEqual( + len(diff), + 0, + "{diff} should be empty".format(diff=diff) + ) + def test_newyears(self): """ Check whether tradingcalendar contains certain dates. diff --git a/tests/test_transforms.py b/tests/test_transforms.py index c2f17051..aff06de5 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -61,6 +61,8 @@ class NoopEventWindow(EventWindow): class TestEventWindow(TestCase): def setUp(self): + self.sim_params = factory.create_simulation_parameters() + setup_logger(self) self.monday = datetime(2012, 7, 9, 16, tzinfo=pytz.utc) @@ -126,7 +128,7 @@ class TestEventWindow(TestCase): class TestFinanceTransforms(TestCase): def setUp(self): - self.trading_environment = factory.create_trading_environment() + self.sim_params = factory.create_simulation_parameters() setup_logger(self) trade_history = factory.create_trade_history( @@ -134,7 +136,7 @@ class TestFinanceTransforms(TestCase): [10.0, 10.0, 11.0, 11.0], [100, 100, 100, 300], timedelta(days=1), - self.trading_environment + self.sim_params ) self.source = SpecificEquityTrades(event_list=trade_history) @@ -142,7 +144,6 @@ class TestFinanceTransforms(TestCase): self.log_handler.pop_application() def test_vwap(self): - vwap = MovingVWAP( market_aware=True, window_length=2 @@ -186,7 +187,7 @@ class TestFinanceTransforms(TestCase): [10.0, 15.0, 13.0, 12.0, 13.0], [100, 100, 100, 300, 100], timedelta(days=1), - self.trading_environment + self.sim_params ) self.source = SpecificEquityTrades(event_list=trade_history) @@ -247,7 +248,7 @@ class TestFinanceTransforms(TestCase): [10.0, 15.0, 13.0, 12.0], [100, 100, 100, 100], timedelta(days=1), - self.trading_environment + self.sim_params ) stddev = MovingStandardDev( @@ -283,8 +284,13 @@ class TestFinanceTransforms(TestCase): class TestBatchTransform(TestCase): def setUp(self): + self.sim_params = factory.create_simulation_parameters( + start=datetime(1990, 1, 1, tzinfo=pytz.utc), + end=datetime(1990, 1, 8, tzinfo=pytz.utc) + ) setup_logger(self) - self.source, self.df = factory.create_test_df_source() + self.source, self.df = \ + factory.create_test_df_source(self.sim_params) def test_event_window(self): algo = BatchTransformAlgorithm() @@ -361,12 +367,15 @@ class TestBatchTransform(TestCase): self.assertEqual( algo.history_return_args, [ + # 1990-01-01 - market holiday, no event + # 1990-01-02 - window not full + None, # 1990-01-03 - window not full None, - # 1990-01-04 - window not full - None, - # 1990-01-05 - window not full, 3rd event + # 1990-01-04 - window not full, 3rd event None, + # 1990-01-05 - window now full + expected_item, # 1990-01-08 - window now full expected_item ]) diff --git a/zipline/algorithm.py b/zipline/algorithm.py index 839812ff..b4005946 100644 --- a/zipline/algorithm.py +++ b/zipline/algorithm.py @@ -24,7 +24,7 @@ from itertools import groupby from operator import attrgetter from zipline.sources import DataFrameSource, DataPanelSource -from zipline.utils.factory import create_trading_environment +from zipline.utils.factory import create_simulation_parameters from zipline.transforms.utils import StatefulTransform from zipline.finance.slippage import ( VolumeShareSlippage, @@ -107,6 +107,8 @@ class TradingAlgorithm(object): # set the capital base self.capital_base = kwargs.get('capital_base', DEFAULT_CAPITAL_BASE) + self.sim_params = kwargs.pop('sim_params', None) + # an algorithm subclass needs to set initialized to True when # it is fully initialized. self.initialized = False @@ -114,7 +116,7 @@ class TradingAlgorithm(object): # call to user-defined constructor method self.initialize(*args, **kwargs) - def _create_generator(self, environment): + def _create_generator(self, sim_params): """ Create a basic generator setup using the sources and transforms attached to this algorithm. @@ -127,20 +129,20 @@ class TradingAlgorithm(object): # Group together events with the same dt field. This depends on the # events already being sorted. self.grouped_by_date = groupby(self.with_alias_dt, attrgetter('dt')) - self.trading_client = tsc(self, environment) + self.trading_client = tsc(self, sim_params) transact_method = transact_partial(self.slippage, self.commission) self.set_transact(transact_method) return self.trading_client.simulate(self.grouped_by_date) - def get_generator(self, environment): + def get_generator(self): """ Override this method to add new logic to the construction of the generator. Overrides can use the _create_generator method to get a standard construction generator. """ - return self._create_generator(environment) + return self._create_generator(self.sim_params) def initialize(self, *args, **kwargs): pass @@ -190,6 +192,13 @@ class TradingAlgorithm(object): else: self.sources = source + if not self.sim_params: + self.sim_params = create_simulation_parameters( + start=start, + end=end, + capital_base=self.capital_base + ) + # Create transforms by wrapping them into StatefulTransforms self.transforms = [] for namestring, trans_descr in self.registered_transforms.iteritems(): @@ -202,14 +211,8 @@ class TradingAlgorithm(object): self.transforms.append(sf) - environment = create_trading_environment( - start=start, - end=end, - capital_base=self.capital_base - ) - # create transforms and zipline - self.gen = self._create_generator(environment) + self.gen = self._create_generator(self.sim_params) # loop through simulated_trading, each iteration returns a # perf ndict diff --git a/zipline/data/benchmarks.py b/zipline/data/benchmarks.py index e8be5db4..919b0f92 100644 --- a/zipline/data/benchmarks.py +++ b/zipline/data/benchmarks.py @@ -33,7 +33,7 @@ from loader_utils import Mapping from zipline.finance.risk import DailyReturn _BENCHMARK_MAPPING = { - # Need to add 'symbol' and GSPC as a constant + # Need to add 'symbol' 'volume': (int, 'Volume'), 'open': (float, 'Open'), 'close': (float, 'Close'), @@ -50,13 +50,12 @@ def benchmark_mappings(): in _BENCHMARK_MAPPING.iteritems()} -def get_raw_benchmark_data(start_date, end_date): +def get_raw_benchmark_data(start_date, end_date, symbol): # create benchmark files # ^GSPC 19500103 params = { - # the s&p 500 - 's': '^GSPC', + 's': symbol, # end_date month, zero indexed 'd': end_date.month - 1, # end_date day str(int(todate[6:8])) #day @@ -79,14 +78,14 @@ def get_raw_benchmark_data(start_date, end_date): return csv.DictReader(StringIO(res.content)) -def get_benchmark_data(): +def get_benchmark_data(symbol): """ - Benchmarks from Yahoo's GSPC source. + Benchmarks from Yahoo. """ start_date = datetime(year=1950, month=1, day=3) end_date = datetime.utcnow() - raw_benchmark_data = get_raw_benchmark_data(start_date, end_date) + raw_benchmark_data = get_raw_benchmark_data(start_date, end_date, symbol) # Reverse data so we can load it in reverse chron order. benchmarks_source = reversed(list(raw_benchmark_data)) @@ -95,11 +94,11 @@ def get_benchmark_data(): return source_to_records(mappings, benchmarks_source) -def get_benchmark_returns(): +def get_benchmark_returns(symbol): benchmark_returns = [] - for data_point in get_benchmark_data(): + for data_point in get_benchmark_data(symbol): returns = (data_point['close'] - data_point['open']) / \ data_point['open'] daily_return = DailyReturn(date=data_point['date'], returns=returns) diff --git a/zipline/data/loader.py b/zipline/data/loader.py index f957b247..4ebe4a23 100644 --- a/zipline/data/loader.py +++ b/zipline/data/loader.py @@ -16,12 +16,18 @@ import os from os.path import expanduser - import msgpack +from collections import OrderedDict + from treasuries import get_treasury_data from benchmarks import get_benchmark_returns +from zipline.utils.date_utils import tuple_to_date +import zipline.finance.risk as risk +from operator import attrgetter + + # TODO: Make this path customizable. DATA_PATH = os.path.join( expanduser("~"), @@ -63,19 +69,72 @@ def dump_treasury_curves(): tr_fp.write(msgpack.dumps(tr_data)) -def dump_benchmarks(): +def dump_benchmarks(symbol): """ Dumps data to be used with zipline. Puts source treasury and data into zipline. """ benchmark_data = [] - for daily_return in get_benchmark_returns(): + for daily_return in get_benchmark_returns(symbol): date_as_tuple = daily_return.date.timetuple()[0:6] + \ (daily_return.date.microsecond,) # Not ideal but massaging data into expected format benchmark = (date_as_tuple, daily_return.returns) benchmark_data.append(benchmark) - with get_datafile('benchmark.msgpack', mode='wb') as bmark_fp: + with get_datafile(get_benchmark_filename(symbol), mode='wb') as bmark_fp: bmark_fp.write(msgpack.dumps(benchmark_data)) + + +def get_benchmark_filename(symbol): + return "%s_benchmark.msgpack" % symbol + + +def load_market_data(bm_symbol='^GSPC'): + try: + fp_bm = get_datafile(get_benchmark_filename(bm_symbol), "rb") + except IOError: + print """ +data msgpacks aren't distribute with source. +Fetching data from Yahoo Finance. +""".strip() + dump_benchmarks(bm_symbol) + fp_bm = get_datafile(get_benchmark_filename(bm_symbol), "rb") + + bm_list = msgpack.loads(fp_bm.read()) + bm_returns = [] + for packed_date, returns in bm_list: + event_dt = tuple_to_date(packed_date) + + daily_return = risk.DailyReturn(date=event_dt, returns=returns) + bm_returns.append(daily_return) + + fp_bm.close() + + bm_returns = sorted(bm_returns, key=attrgetter('date')) + + try: + fp_tr = get_datafile('treasury_curves.msgpack', "rb") + except IOError: + print """ +data msgpacks aren't distribute with source. +Fetching data from data.treasury.gov +""".strip() + dump_treasury_curves() + fp_tr = get_datafile('treasury_curves.msgpack', "rb") + + tr_list = msgpack.loads(fp_tr.read()) + tr_curves = {} + for packed_date, curve in tr_list: + tr_dt = tuple_to_date(packed_date) + #tr_dt = tr_dt.replace(hour=0, minute=0, second=0, tzinfo=pytz.utc) + tr_curves[tr_dt] = curve + + fp_tr.close() + + tr_curves = OrderedDict(sorted( + ((dt, c) for dt, c in tr_curves.iteritems()), + key=lambda t: t[0])) + + return bm_returns, tr_curves diff --git a/zipline/finance/performance.py b/zipline/finance/performance.py index 54fd82c7..10306b68 100644 --- a/zipline/finance/performance.py +++ b/zipline/finance/performance.py @@ -141,6 +141,7 @@ import numpy as np import zipline.protocol as zp import zipline.finance.risk as risk +import zipline.finance.trading as trading log = logbook.Logger('Performance') @@ -157,28 +158,28 @@ class PerformanceTracker(object): """ - def __init__(self, trading_environment): + def __init__(self, sim_params): - self.trading_environment = trading_environment - self.trading_day = datetime.timedelta(hours=6, minutes=30) + self.sim_params = sim_params self.started_at = datetime.datetime.utcnow().replace(tzinfo=pytz.utc) - self.period_start = self.trading_environment.period_start - self.period_end = self.trading_environment.period_end - self.last_close = self.trading_environment.last_close - self.market_open = self.trading_environment.first_open - self.market_close = self.market_open + self.trading_day + self.period_start = self.sim_params.period_start + self.period_end = self.sim_params.period_end + self.last_close = self.sim_params.last_close + first_day = self.sim_params.first_open + self.market_open, self.market_close = \ + trading.environment.get_open_and_close(first_day) self.progress = 0.0 - self.total_days = self.trading_environment.days_in_period + self.total_days = self.sim_params.days_in_period # one indexed so that we reach 100% self.day_count = 0.0 - self.capital_base = self.trading_environment.capital_base + self.capital_base = self.sim_params.capital_base self.returns = [] self.txn_count = 0 self.event_count = 0 self.last_dict = None - self.cumulative_risk_metrics = risk.RiskMetricsIterative( - self.period_start, self.trading_environment) + self.cumulative_risk_metrics = \ + risk.RiskMetricsIterative(self.period_start) # this performance period will span the entire simulation. self.cumulative_performance = PerformancePeriod( @@ -203,7 +204,7 @@ class PerformanceTracker(object): def __repr__(self): return "%s(%r)" % ( self.__class__.__name__, - {'trading_environment': self.trading_environment}) + {'simulation parameters': self.sim_params}) def transform(self, stream_in): """ @@ -249,7 +250,7 @@ class PerformanceTracker(object): if event.type == zp.DATASOURCE_TYPE.TRADE: messages = [] - while event.dt > self.market_close: + while event.dt > self.market_close and event.dt < self.last_close: messages.append(self.handle_market_close()) if event.TRANSACTION: @@ -275,7 +276,6 @@ class PerformanceTracker(object): return messages def handle_market_close(self): - # add the return results from today to the list of DailyReturn objects. todays_date = self.market_close.replace(hour=0, minute=0, second=0) todays_return_obj = risk.DailyReturn( @@ -305,17 +305,8 @@ class PerformanceTracker(object): return daily_update #move the market day markers forward - next_open = self.trading_environment.next_trading_day(self.market_open) - if next_open is None: - raise Exception( - "Attempt to backtest beyond available history. \ -Last successful date: %s" % self.market_open) - - # next_open is a midnight date, but we want the time too - self.market_open = next_open.replace(hour=self.market_open.hour, - minute=self.market_open.minute, - second=self.market_open.second) - self.market_close = self.market_open + self.trading_day + self.market_open, self.market_close = \ + trading.environment.next_open_and_close(self.market_open) # Roll over positions to current day. self.todays_performance.rollover() @@ -350,14 +341,11 @@ Last successful date: %s" % self.market_open) log_msg = "Simulated {n} trading days out of {m}." log.info(log_msg.format(n=int(self.day_count), m=self.total_days)) log.info("first open: {d}".format( - d=self.trading_environment.first_open)) + d=self.sim_params.first_open)) log.info("last close: {d}".format( - d=self.trading_environment.last_close)) + d=self.sim_params.last_close)) - self.risk_report = risk.RiskReport( - self.returns, - self.trading_environment - ) + self.risk_report = risk.RiskReport(self.returns, self.sim_params) risk_dict = self.risk_report.to_dict() return perf_messages, risk_dict diff --git a/zipline/finance/risk.py b/zipline/finance/risk.py index ccf2481b..1a6a46e0 100644 --- a/zipline/finance/risk.py +++ b/zipline/finance/risk.py @@ -63,8 +63,11 @@ from collections import OrderedDict import bisect import numpy as np import numpy.linalg as la + +import zipline.finance.trading as trading from zipline.utils.date_utils import epoch_now + log = logbook.Logger('Risk') @@ -110,20 +113,19 @@ class DailyReturn(object): class RiskMetricsBase(object): - def __init__(self, start_date, end_date, returns, trading_environment): + def __init__(self, start_date, end_date, returns): - self.treasury_curves = trading_environment.treasury_curves + self.treasury_curves = trading.environment.treasury_curves assert isinstance(self.treasury_curves, OrderedDict), \ "Treasury curves must be an OrderedDict" self.start_date = start_date self.end_date = end_date - self.trading_environment = trading_environment self.algorithm_period_returns, self.algorithm_returns = \ self.calculate_period_returns(returns) benchmark_returns = [ - x for x in self.trading_environment.benchmark_returns + x for x in trading.environment.benchmark_returns if x.date >= returns[0].date and x.date <= returns[-1].date ] @@ -227,7 +229,7 @@ class RiskMetricsBase(object): x.returns for x in daily_returns if x.date >= self.start_date and x.date <= self.end_date and - self.trading_environment.is_trading_day(x.date) + trading.environment.is_trading_day(x.date) ] period_returns = 1.0 @@ -427,7 +429,7 @@ that date doesn't exceed treasury history range." raise Exception(message) def search_day_distance(self, dt): - tdd = self.trading_environment.trading_day_distance(dt, self.end_date) + tdd = trading.environment.trading_day_distance(dt, self.end_date) if tdd is None: return None assert tdd >= 0 @@ -457,11 +459,10 @@ class RiskMetricsIterative(RiskMetricsBase): Call update() method on each dt to update the metrics. """ - def __init__(self, start_date, trading_environment): - self.treasury_curves = trading_environment.treasury_curves + def __init__(self, start_date): + self.treasury_curves = trading.environment.treasury_curves self.start_date = start_date self.end_date = start_date - self.trading_environment = trading_environment self.compounded_log_returns = [] self.moving_avg = [] @@ -484,12 +485,12 @@ class RiskMetricsIterative(RiskMetricsBase): self.trading_days = 0 self.all_benchmark_returns = [ - x for x in self.trading_environment.benchmark_returns + x for x in trading.environment.benchmark_returns if x.date >= self.start_date ] def update(self, market_close, returns_in_period): - if self.trading_environment.is_trading_day(self.end_date): + if trading.environment.is_trading_day(self.end_date): self.algorithm_returns.append(returns_in_period) self.benchmark_returns.append( self.all_benchmark_returns.pop(0).returns) @@ -711,7 +712,7 @@ class RiskReport(object): def __init__( self, algorithm_returns, - trading_environment, + sim_params ): """ algorithm_returns needs to be a list of daily_return objects @@ -719,12 +720,12 @@ class RiskReport(object): """ self.algorithm_returns = algorithm_returns - self.trading_environment = trading_environment + self.sim_params = sim_params self.created = epoch_now() if len(self.algorithm_returns) == 0: - start_date = self.trading_environment.period_start - end_date = self.trading_environment.period_end + start_date = self.sim_params.period_start + end_date = self.sim_params.period_end else: start_date = self.algorithm_returns[0].date end_date = self.algorithm_returns[-1].date @@ -778,8 +779,7 @@ class RiskReport(object): cur_period_metrics = RiskMetricsBatch( start_date=cur_start, end_date=cur_end, - returns=self.algorithm_returns, - trading_environment=self.trading_environment + returns=self.algorithm_returns ) ends.append(cur_period_metrics) diff --git a/zipline/finance/trading.py b/zipline/finance/trading.py index cfaf0b02..7f1b92af 100644 --- a/zipline/finance/trading.py +++ b/zipline/finance/trading.py @@ -13,22 +13,27 @@ # See the License for the specific language governing permissions and # limitations under the License. +import bisect import pytz import logbook import datetime from collections import defaultdict, OrderedDict -import bisect +from delorean import Delorean +from pandas import DatetimeIndex import zipline.protocol as zp from zipline.finance.slippage import ( VolumeShareSlippage, transact_partial ) + from zipline.finance.commission import PerShare log = logbook.Logger('Transaction Simulator') +environment = None + class TransactionSimulator(object): @@ -60,107 +65,31 @@ class TradingEnvironment(object): def __init__( self, - benchmark_returns, - treasury_curves, - period_start=None, - period_end=None, - capital_base=None + load=None, + bm_symbol='^GSPC', + exchange_tz="US/Eastern" ): self.trading_day_map = OrderedDict() - self.treasury_curves = treasury_curves - self.benchmark_returns = benchmark_returns - self.period_start = period_start - self.period_end = period_end - self.capital_base = capital_base + self.bm_symbol = bm_symbol + if not load: + from zipline.data.loader import load_market_data + load = load_market_data + + self.benchmark_returns, self.treasury_curves = \ + load(self.bm_symbol) + self._period_trading_days = None + self._trading_days_series = None + self.full_trading_day = datetime.timedelta(hours=6, minutes=30) + self.exchange_tz = exchange_tz - assert self.period_start <= self.period_end, \ - "Period start falls after period end." - - for bm in benchmark_returns: + for bm in self.benchmark_returns: self.trading_day_map[bm.date] = bm self.first_trading_day = next(self.trading_day_map.iterkeys()) self.last_trading_day = next(reversed(self.trading_day_map)) - assert self.period_start <= self.last_trading_day, \ - "Period start falls after the last known trading day." - assert self.period_end >= self.first_trading_day, \ - "Period end falls before the first known trading day." - - self.first_open = self.calculate_first_open() - self.last_close = self.calculate_last_close() - - self.prior_day_open = self.calculate_prior_day_open() - - def __repr__(self): - return "%s(%r)" % ( - self.__class__.__name__, - {'first_open': self.first_open, - 'last_close': self.last_close - }) - - def calculate_first_open(self): - """ - Finds the first trading day on or after self.period_start. - """ - first_open = self.period_start - one_day = datetime.timedelta(days=1) - - while not self.is_trading_day(first_open): - first_open = first_open + one_day - - first_open = self.set_NYSE_time(first_open, 9, 30) - return first_open - - def calculate_prior_day_open(self): - """ - Finds the first trading day open that falls at least a day - before period_start. - """ - one_day = datetime.timedelta(days=1) - first_open = self.period_start - one_day - - if first_open <= self.first_trading_day: - log.warn("Cannot calculate prior day open.") - return self.period_start - - while not self.is_trading_day(first_open): - first_open = first_open - one_day - - first_open = self.set_NYSE_time(first_open, 9, 30) - return first_open - - def calculate_last_close(self): - """ - Finds the last trading day on or before self.period_end - """ - last_close = self.period_end - one_day = datetime.timedelta(days=1) - - while not self.is_trading_day(last_close): - last_close = last_close - one_day - - last_close = self.set_NYSE_time(last_close, 16, 00) - - return last_close - - #TODO: add other exchanges and timezones... - def set_NYSE_time(self, dt, hour, minute): - naive = datetime.datetime( - year=dt.year, - month=dt.month, - day=dt.day - ) - local = pytz.timezone('US/Eastern') - local_dt = naive.replace(tzinfo=local) - # set the clock to the opening bell in NYC time. - local_dt = local_dt.replace(hour=hour, minute=minute) - # convert to UTC - utc_dt = local_dt.astimezone(pytz.utc) - return utc_dt - def normalize_date(self, test_date): return datetime.datetime( year=test_date.year, @@ -181,18 +110,17 @@ class TradingEnvironment(object): return self._period_trading_days @property - def days_in_period(self): - """return the number of trading days within the period [start, end)""" - return len(self.period_trading_days) + def trading_days(self): + if self._trading_days_series is None: + self._trading_days_series = \ + DatetimeIndex(self.trading_day_map.iterkeys()) + return self._trading_days_series def is_market_hours(self, test_date): if not self.is_trading_day(test_date): return False - mkt_open = self.set_NYSE_time(test_date, 9, 30) - #TODO: half days? - mkt_close = self.set_NYSE_time(test_date, 16, 00) - + mkt_open, mkt_close = self.get_open_and_close(test_date) return test_date >= mkt_open and test_date <= mkt_close def is_trading_day(self, test_date): @@ -210,6 +138,46 @@ class TradingEnvironment(object): return None + def next_open_and_close(self, start_date): + """ + Given the start_date, returns the next open and close of + the market. + """ + next_open = self.next_trading_day(start_date) + + if next_open is None: + raise Exception( + "Attempt to backtest beyond available history. \ +Last successful date: %s" % self.market_open) + + return self.get_open_and_close(next_open) + + def get_open_and_close(self, next_open): + + # creating a naive datetime with the correct hour, + # minute, and date. this will allow us to use Delorean to + # shift the time between EST and UTC. + next_open = next_open.replace( + hour=9, + minute=30, + second=0, + tzinfo=None + ) + # create a new Delorean with the next_open naive date and + # the correct timezone for the exchange. + open_delorean = Delorean(next_open, "US/Eastern") + open_utc = open_delorean.shift("UTC").datetime + + market_open = open_utc + market_close = market_open + self.get_trading_day_duration(open_utc) + + return market_open, market_close + + def get_trading_day_duration(self, trading_day): + # TODO: make a list of half-days and modify the + # calculation of market close to reflect them. + return self.full_trading_day + def trading_day_distance(self, first_date, second_date): first_date = self.normalize_date(first_date) second_date = self.normalize_date(second_date) @@ -224,3 +192,98 @@ class TradingEnvironment(object): return None return j - i + + def get_index(self, dt): + ndt = self.normalize_date(dt) + return self.trading_days.searchsorted(ndt) + + +class SimulationParameters(object): + def __init__(self, period_start, period_end, + capital_base=10e3): + + # raise and exception if the global environment is not + # set. + global environment + if not environment: + environment = TradingEnvironment() + + self.period_start = period_start + self.period_end = period_end + self.capital_base = capital_base + self.first_open = self.calculate_first_open() + self.last_close = self.calculate_last_close() + start_index = \ + environment.get_index(self.first_open) + end_index = environment.get_index(self.last_close) + + # take an inclusive slice of the environment's + # trading_days. + self.trading_days = \ + environment.trading_days[start_index:end_index+1] + + self.prior_day_open = self.calculate_prior_day_open() + + assert self.period_start <= self.period_end, \ + "Period start falls after period end." + + assert self.period_start <= environment.last_trading_day, \ + "Period start falls after the last known trading day." + assert self.period_end >= environment.first_trading_day, \ + "Period end falls before the first known trading day." + + def calculate_first_open(self): + """ + Finds the first trading day on or after self.period_start. + """ + first_open = self.period_start + one_day = datetime.timedelta(days=1) + + while not environment.is_trading_day(first_open): + first_open = first_open + one_day + + mkt_open, _ = environment.get_open_and_close(first_open) + return mkt_open + + def calculate_prior_day_open(self): + """ + Finds the first trading day open that falls at least a day + before period_start. + """ + one_day = datetime.timedelta(days=1) + first_open = self.period_start - one_day + + if first_open <= environment.first_trading_day: + log.warn("Cannot calculate prior day open.") + return self.period_start + + while not environment.is_trading_day(first_open): + first_open = first_open - one_day + + mkt_open, _ = environment.get_open_and_close(first_open) + return mkt_open + + def calculate_last_close(self): + """ + Finds the last trading day on or before self.period_end + """ + last_close = self.period_end + one_day = datetime.timedelta(days=1) + + while not environment.is_trading_day(last_close): + last_close = last_close - one_day + + _, mkt_close = environment.get_open_and_close(last_close) + return mkt_close + + @property + def days_in_period(self): + """return the number of trading days within the period [start, end)""" + return len(self.trading_days) + + def __repr__(self): + return "%s(%r)" % ( + self.__class__.__name__, + {'first_open': self.first_open, + 'last_close': self.last_close + }) diff --git a/zipline/transforms/mavg.py b/zipline/transforms/mavg.py index 3dbba340..3dd7bcc9 100644 --- a/zipline/transforms/mavg.py +++ b/zipline/transforms/mavg.py @@ -30,6 +30,7 @@ class MovingAverage(object): def __init__(self, fields='price', market_aware=True, window_length=None, delta=None): + if isinstance(fields, basestring): fields = [fields] self.fields = fields diff --git a/zipline/transforms/returns.py b/zipline/transforms/returns.py index e8772439..6f0c9c67 100644 --- a/zipline/transforms/returns.py +++ b/zipline/transforms/returns.py @@ -38,7 +38,9 @@ class Returns(object): return tracker.returns def _create(self): - return ReturnsFromPriorClose(self.window_length) + return ReturnsFromPriorClose( + self.window_length + ) class ReturnsFromPriorClose(object): diff --git a/zipline/transforms/utils.py b/zipline/transforms/utils.py index f45b1a0f..11e7cece 100644 --- a/zipline/transforms/utils.py +++ b/zipline/transforms/utils.py @@ -29,8 +29,8 @@ from numbers import Integral import pandas as pd from zipline.protocol import Event, DATASOURCE_TYPE -from zipline.utils import tradingcalendar from zipline.gens.utils import assert_sort_unframe_protocol, hash_args +import zipline.finance.trading as trading log = logbook.Logger('Transform') @@ -228,7 +228,8 @@ class EventWindow(object): # Subclasses should override handle_add to define behavior for # adding new ticks. self.handle_add(event) - + #if len(self.ticks) > self.window_length: + # import nose.tools; nose.tools.set_trace() # Clear out any expired events. # # oldest newest @@ -244,8 +245,10 @@ class EventWindow(object): self.handle_remove(popped) def out_of_market_window(self, oldest, newest): - oldest_index = tradingcalendar.trading_days.searchsorted(oldest) - newest_index = tradingcalendar.trading_days.searchsorted(newest) + oldest_index = \ + trading.environment.trading_days.searchsorted(oldest) + newest_index = \ + trading.environment.trading_days.searchsorted(newest) trading_days_between = newest_index - oldest_index @@ -350,8 +353,7 @@ class BatchTransform(EventWindow): full. Returns None if window is not full yet. """ - super(BatchTransform, self).__init__(True, - window_length=window_length) + super(BatchTransform, self).__init__(True, window_length=window_length) if func is not None: self.compute_transform_value = func diff --git a/zipline/utils/factory.py b/zipline/utils/factory.py index 0c11c461..066ba414 100644 --- a/zipline/utils/factory.py +++ b/zipline/utils/factory.py @@ -18,9 +18,7 @@ Factory functions to prepare useful data for tests. """ import pytz -import msgpack import random -from operator import attrgetter from collections import OrderedDict import pandas as pd @@ -29,98 +27,37 @@ import numpy as np from datetime import datetime, timedelta import zipline.finance.risk as risk -from zipline.utils.date_utils import tuple_to_date from zipline.protocol import Event, DATASOURCE_TYPE from zipline.sources import (SpecificEquityTrades, DataFrameSource, DataPanelSource) from zipline.gens.utils import create_trade -from zipline.finance.trading import TradingEnvironment -from zipline.data.loader import ( - get_datafile, - dump_benchmarks, - dump_treasury_curves -) +from zipline.finance.trading import SimulationParameters, TradingEnvironment +import zipline.finance.trading as trading -def load_market_data(): - try: - fp_bm = get_datafile('benchmark.msgpack', "rb") - except IOError: - print """ -data msgpacks aren't distribute with source. -Fetching data from Yahoo Finance. -""".strip() - dump_benchmarks() - fp_bm = get_datafile('benchmark.msgpack', "rb") - - bm_list = msgpack.loads(fp_bm.read()) - bm_returns = [] - for packed_date, returns in bm_list: - event_dt = tuple_to_date(packed_date) - #event_dt = event_dt.replace( - # hour=0, - # minute=0, - # second=0, - # tzinfo=pytz.utc - #) - - daily_return = risk.DailyReturn(date=event_dt, returns=returns) - bm_returns.append(daily_return) - - fp_bm.close() - - bm_returns = sorted(bm_returns, key=attrgetter('date')) - - try: - fp_tr = get_datafile('treasury_curves.msgpack', "rb") - except IOError: - print """ -data msgpacks aren't distribute with source. -Fetching data from data.treasury.gov -""".strip() - dump_treasury_curves() - fp_tr = get_datafile('treasury_curves.msgpack', "rb") - - tr_list = msgpack.loads(fp_tr.read()) - tr_curves = {} - for packed_date, curve in tr_list: - tr_dt = tuple_to_date(packed_date) - #tr_dt = tr_dt.replace(hour=0, minute=0, second=0, tzinfo=pytz.utc) - tr_curves[tr_dt] = curve - - fp_tr.close() - - tr_curves = OrderedDict(sorted( - ((dt, c) for dt, c in tr_curves.iteritems()), - key=lambda t: t[0])) - - return bm_returns, tr_curves - - -def create_trading_environment(year=2006, start=None, end=None, - capital_base=float("1.0e5")): +def create_simulation_parameters(year=2006, start=None, end=None, + capital_base=float("1.0e5") + ): """Construct a complete environment with reasonable defaults""" - benchmark_returns, treasury_curves = load_market_data() - + trading.environment = TradingEnvironment() if start is None: start = datetime(year, 1, 1, tzinfo=pytz.utc) if end is None: end = datetime(year, 12, 31, tzinfo=pytz.utc) - trading_environment = TradingEnvironment( - benchmark_returns, - treasury_curves, + sim_params = SimulationParameters( period_start=start, period_end=end, - capital_base=capital_base + capital_base=capital_base, ) - return trading_environment + return sim_params -def create_random_trading_environment(): - benchmark_returns, treasury_curves = load_market_data() +def create_random_simulation_parameters(): + trading.environment = TradingEnvironment() + treasury_curves = trading.environment.treasury_curves for n in range(100): @@ -141,35 +78,33 @@ def create_random_trading_environment(): failed to find a suitable daterange after 100 attempts. please double check treasury and benchmark data in findb, and re-run the test.""" - trading_environment = TradingEnvironment( - benchmark_returns, - treasury_curves, + sim_params = SimulationParameters( period_start=start_dt, period_end=end_dt ) - return trading_environment, start_dt, end_dt + return sim_params, start_dt, end_dt -def get_next_trading_dt(current, interval, trading_calendar): +def get_next_trading_dt(current, interval): next = current while True: next = next + interval - if trading_calendar.is_market_hours(next): + if trading.environment.is_market_hours(next): break return next -def create_trade_history(sid, prices, amounts, interval, trading_calendar, +def create_trade_history(sid, prices, amounts, interval, sim_params, source_id="test_factory"): trades = [] - current = trading_calendar.first_open + current = sim_params.first_open for price, amount in zip(prices, amounts): trade = create_trade(sid, price, amount, current, source_id) trades.append(trade) - current = get_next_trading_dt(current, interval, trading_calendar) + current = get_next_trading_dt(current, interval) assert len(trades) == len(prices) return trades @@ -199,115 +134,115 @@ def create_txn(sid, price, amount, datetime): return txn -def create_txn_history(sid, priceList, amtList, interval, trading_calendar): +def create_txn_history(sid, priceList, amtList, interval, sim_params): txns = [] - current = trading_calendar.first_open + current = sim_params.first_open for price, amount in zip(priceList, amtList): - current = get_next_trading_dt(current, interval, trading_calendar) + current = get_next_trading_dt(current, interval) txns.append(create_txn(sid, price, amount, current)) current = current + interval return txns -def create_returns(daycount, trading_calendar): +def create_returns(daycount, sim_params): """ For the given number of calendar (not trading) days return all the trading days between start and start + daycount. """ test_range = [] - current = trading_calendar.first_open + current = sim_params.first_open one_day = timedelta(days=1) for day in range(daycount): current = current + one_day - if trading_calendar.is_trading_day(current): + if trading.environment.is_trading_day(current): r = risk.DailyReturn(current, random.random()) test_range.append(r) return test_range -def create_returns_from_range(trading_calendar): - current = trading_calendar.first_open - end = trading_calendar.last_close +def create_returns_from_range(sim_params): + current = sim_params.first_open + end = sim_params.last_close one_day = timedelta(days=1) test_range = [] while current <= end: r = risk.DailyReturn(current, random.random()) test_range.append(r) - current = get_next_trading_dt(current, one_day, trading_calendar) + current = get_next_trading_dt(current, one_day) return test_range -def create_returns_from_list(returns, trading_calendar): - current = trading_calendar.first_open +def create_returns_from_list(returns, sim_params): + current = sim_params.first_open one_day = timedelta(days=1) test_range = [] #sometimes the range starts with a non-trading day. - if not trading_calendar.is_trading_day(current): - current = get_next_trading_dt(current, one_day, trading_calendar) + if not trading.environment.is_trading_day(current): + current = get_next_trading_dt(current, one_day) for return_val in returns: r = risk.DailyReturn(current, return_val) test_range.append(r) - current = get_next_trading_dt(current, one_day, trading_calendar) + current = get_next_trading_dt(current, one_day) return test_range -def create_daily_trade_source(sids, trade_count, trading_environment, +def create_daily_trade_source(sids, trade_count, sim_params, concurrent=False): """ creates trade_count trades for each sid in sids list. - first trade will be on trading_environment.period_start, and daily + first trade will be on sim_params.period_start, and daily thereafter for each sid. Thus, two sids should result in two trades per day. - Important side-effect: trading_environment.period_end will be modified + Important side-effect: sim_params.period_end will be modified to match the day of the final trade. """ return create_trade_source( sids, trade_count, timedelta(days=1), - trading_environment, + sim_params, concurrent=concurrent ) -def create_minutely_trade_source(sids, trade_count, trading_environment, +def create_minutely_trade_source(sids, trade_count, sim_params, concurrent=False): """ creates trade_count trades for each sid in sids list. - first trade will be on trading_environment.period_start, and every minute + first trade will be on sim_params.period_start, and every minute thereafter for each sid. Thus, two sids should result in two trades per minute. - Important side-effect: trading_environment.period_end will be modified + Important side-effect: sim_params.period_end will be modified to match the day of the final trade. """ return create_trade_source( sids, trade_count, timedelta(minutes=1), - trading_environment, + sim_params, concurrent=concurrent ) def create_trade_source(sids, trade_count, - trade_time_increment, trading_environment, + trade_time_increment, sim_params, concurrent=False): args = tuple() kwargs = { 'count': trade_count, 'sids': sids, - 'start': trading_environment.first_open, + 'start': sim_params.first_open, 'delta': trade_time_increment, 'filter': sids, 'concurrent': concurrent @@ -316,19 +251,24 @@ def create_trade_source(sids, trade_count, # TODO: do we need to set the trading environment's end to same dt as # the last trade in the history? - #trading_environment.period_end = trade_history[-1].dt + #sim_params.period_end = trade_history[-1].dt return source -def create_test_df_source(trading_calendar=None): - start = trading_calendar.first_open \ - if trading_calendar else pd.datetime(1990, 1, 3, 0, 0, 0, 0, pytz.utc) +def create_test_df_source(sim_params=None): - end = trading_calendar.last_close \ - if trading_calendar else pd.datetime(1990, 1, 8, 0, 0, 0, 0, pytz.utc) + if sim_params: + index = sim_params.trading_days + else: + start = pd.datetime(1990, 1, 3, 0, 0, 0, 0, pytz.utc) + end = pd.datetime(1990, 1, 8, 0, 0, 0, 0, pytz.utc) + index = pd.DatetimeIndex( + start=start, + end=end, + freq=pd.datetools.BDay() + ) - index = pd.DatetimeIndex(start=start, end=end, freq=pd.datetools.BDay()) x = np.arange(0, len(index)) df = pd.DataFrame(x, index=index, columns=[0]) @@ -336,12 +276,12 @@ def create_test_df_source(trading_calendar=None): return DataFrameSource(df), df -def create_test_panel_source(trading_calendar=None): - start = trading_calendar.first_open \ - if trading_calendar else pd.datetime(1990, 1, 3, 0, 0, 0, 0, pytz.utc) +def create_test_panel_source(sim_params=None): + start = sim_params.first_open \ + if sim_params else pd.datetime(1990, 1, 3, 0, 0, 0, 0, pytz.utc) - end = trading_calendar.last_close \ - if trading_calendar else pd.datetime(1990, 1, 8, 0, 0, 0, 0, pytz.utc) + end = sim_params.last_close \ + if sim_params else pd.datetime(1990, 1, 8, 0, 0, 0, 0, pytz.utc) index = pd.DatetimeIndex(start=start, end=end, freq=pd.datetools.day) price = np.arange(0, len(index)) diff --git a/zipline/utils/simfactory.py b/zipline/utils/simfactory.py index 025628eb..2355745a 100644 --- a/zipline/utils/simfactory.py +++ b/zipline/utils/simfactory.py @@ -7,8 +7,6 @@ def create_test_zipline(**config): """ :param config: A configuration object that is a dict with: - - environment - a \ - :py:class:`zipline.finance.trading.TradingEnvironment` - sid - an integer, which will be used as the security ID. - order_count - the number of orders the test algo will place, defaults to 100 @@ -36,14 +34,6 @@ def create_test_zipline(**config): concurrent_trades = config.get('concurrent_trades', False) - #-------------------- - # Trading Environment - #-------------------- - if 'environment' in config: - trading_environment = config['environment'] - else: - trading_environment = factory.create_trading_environment() - if 'order_count' in config: order_count = config['order_count'] else: @@ -70,7 +60,8 @@ def create_test_zipline(**config): test_algo = TestAlgorithm( sid, order_amount, - order_count + order_count, + sim_params=factory.create_simulation_parameters() ) #------------------- @@ -82,7 +73,7 @@ def create_test_zipline(**config): trade_source = factory.create_daily_trade_source( sid_list, trade_count, - trading_environment, + test_algo.sim_params, concurrent=concurrent_trades ) @@ -105,6 +96,6 @@ def create_test_zipline(**config): # ------------------ # generator/simulator - sim = test_algo.get_generator(trading_environment) + sim = test_algo.get_generator() return sim diff --git a/zipline/utils/test_utils.py b/zipline/utils/test_utils.py index 54f4fa63..f9ebf7e0 100644 --- a/zipline/utils/test_utils.py +++ b/zipline/utils/test_utils.py @@ -26,9 +26,6 @@ def check_dict(test, a, b, label): test.assertTrue(isinstance(a, dict)) test.assertTrue(isinstance(b, dict)) for key in a.keys(): - # ignore the extra fields used by dictshield - if key in ['progress']: - continue test.assertTrue(key in a, "missing key at: " + label + "." + key) test.assertTrue(key in b, "missing key at: " + label + "." + key) diff --git a/zipline/utils/tradingcalendar.py b/zipline/utils/tradingcalendar.py index 46fbbf71..08d63fda 100644 --- a/zipline/utils/tradingcalendar.py +++ b/zipline/utils/tradingcalendar.py @@ -61,7 +61,7 @@ def get_non_trading_days(start, end): bymonth=1, byweekday=(rrule.MO(+3)), cache=True, - dtstart=start, + dtstart=datetime(1998, 1, 1, tzinfo=pytz.utc), until=end ) non_trading_rules.append(mlk_day)