wassname/catalyst
1
0
Fork 0
mirror of https://github.com/wassname/catalyst.git synced 2026-07-15 11:22:18 +08:00
Code Issues Packages Projects Releases Wiki Activity

Refactoring of TradingEnvironment to isolate the global state: index symbol and exchange timezone. Parameters that define the simulation (start, end, and capital base) were put in a new class, SimulationParameters.

Browse Source 
Global state for the financial simulation environment is accessed through the
zipline.finance.trading module, which now contains a module variable:
environment.

Parameters are passed into an algorithm as a keyword argument, sim_params.
SimulationParameters creates a trading day index for the test period that
can be used to find trading days, calculate distance between trading days,
and other common operations. The sim params index is just selected from the
global state.

================

Details:

    - adding delorean to the requirements.
    - made index symbol a parameter for loading the benchmark data. changed
    messagepack storage to be symbol specific.
    - ported risk, performance, algorithm, transforms, batch transforms
    and associated tests to use simulation parameters and global environment
    - factory and sim factory use global state and sim params
    - factory method parameter names now reflect the class expected
This commit is contained in:
fawce
2013-02-18 10:24:32 -05:00
committed by Eddie Hebert
parent 8f37b794e5
commit 2c7355a0dc
23 changed files with 576 additions and 475 deletions
Download Patch File Download Diff File Expand all files Collapse all files
etc/requirements.txt
+3
View File
@@ -17,3 +17,6 @@ six==1.2.0
# For fetching remote data
requests==1.1.0
# For remaining sane when coping with dates
Delorean==0.1.6
tests/test_algorithm.py
+47 -17
View File
@@ -29,17 +29,17 @@ from zipline.transforms import MovingAverage
class TestRecordAlgorithm(TestCase):
def setUp(self):
self.trading_environment = factory.create_trading_environment()
self.sim_params = factory.create_simulation_parameters()
trade_history = factory.create_trade_history(
133,
[10.0, 10.0, 11.0, 11.0],
[100, 100, 100, 300],
timedelta(days=1),
self.trading_environment
self.sim_params
)
self.source = SpecificEquityTrades(event_list=trade_history)
self.df_source, self.df = \
factory.create_test_df_source(self.trading_environment)
factory.create_test_df_source(self.sim_params)
def test_record_incr(self):
algo = RecordAlgorithm()
@@ -51,7 +51,7 @@ class TestRecordAlgorithm(TestCase):
class TestTransformAlgorithm(TestCase):
def setUp(self):
setup_logger(self)
self.trading_environment = factory.create_trading_environment()
self.sim_params = factory.create_simulation_parameters()
setup_logger(self)
trade_history = factory.create_trade_history(
@@ -59,35 +59,48 @@ class TestTransformAlgorithm(TestCase):
[10.0, 10.0, 11.0, 11.0],
[100, 100, 100, 300],
timedelta(days=1),
self.trading_environment
self.sim_params
)
self.source = SpecificEquityTrades(event_list=trade_history)
self.df_source, self.df = \
factory.create_test_df_source(self.trading_environment)
factory.create_test_df_source(self.sim_params)
self.panel_source, self.panel = \
factory.create_test_panel_source(self.trading_environment)
factory.create_test_panel_source(self.sim_params)
def test_source_as_input(self):
algo = TestRegisterTransformAlgorithm(sids=[133])
algo = TestRegisterTransformAlgorithm(
self.sim_params,
sids=[133]
)
algo.run(self.source)
self.assertEqual(len(algo.sources), 1)
assert isinstance(algo.sources[0], SpecificEquityTrades)
def test_multi_source_as_input_no_start_end(self):
algo = TestRegisterTransformAlgorithm(sids=[133])
algo = TestRegisterTransformAlgorithm(
self.sim_params,
sids=[133]
)
with self.assertRaises(AssertionError):
algo.run([self.source, self.df_source])
def test_multi_source_as_input(self):
algo = TestRegisterTransformAlgorithm(sids=[0, 1, 133])
algo = TestRegisterTransformAlgorithm(
self.sim_params,
sids=[0, 1, 133]
)
algo.run([self.source, self.df_source],
start=self.df.index[0], end=self.df.index[-1])
self.assertEqual(len(algo.sources), 2)
def test_df_as_input(self):
algo = TestRegisterTransformAlgorithm(sids=[0, 1])
algo = TestRegisterTransformAlgorithm(
self.sim_params,
sids=[0, 1]
)
algo.run(self.df)
assert isinstance(algo.sources[0], DataFrameSource)
@@ -97,14 +110,22 @@ class TestTransformAlgorithm(TestCase):
assert isinstance(algo.sources[0], DataPanelSource)
def test_run_twice(self):
algo = TestRegisterTransformAlgorithm(sids=[0, 1])
algo = TestRegisterTransformAlgorithm(
self.sim_params,
sids=[0, 1]
)
res1 = algo.run(self.df)
res2 = algo.run(self.df)
np.testing.assert_array_equal(res1, res2)
def test_transform_registered(self):
algo = TestRegisterTransformAlgorithm(sids=[133])
algo = TestRegisterTransformAlgorithm(
self.sim_params,
sids=[133]
)
algo.run(self.source)
assert 'mavg' in algo.registered_transforms
assert algo.registered_transforms['mavg']['args'] == (['price'],)
@@ -113,15 +134,24 @@ class TestTransformAlgorithm(TestCase):
assert algo.registered_transforms['mavg']['class'] is MovingAverage
def test_data_frequency_setting(self):
algo = TestRegisterTransformAlgorithm(data_frequency='daily')
algo = TestRegisterTransformAlgorithm(
self.sim_params,
data_frequency='daily'
)
self.assertEqual(algo.data_frequency, 'daily')
self.assertEqual(algo.annualizer, 250)
algo = TestRegisterTransformAlgorithm(data_frequency='minute')
algo = TestRegisterTransformAlgorithm(
self.sim_params,
data_frequency='minute'
)
self.assertEqual(algo.data_frequency, 'minute')
self.assertEqual(algo.annualizer, 250 * 6 * 60)
algo = TestRegisterTransformAlgorithm(data_frequency='minute',
annualizer=10)
algo = TestRegisterTransformAlgorithm(
self.sim_params,
data_frequency='minute',
annualizer=10
)
self.assertEqual(algo.data_frequency, 'minute')
self.assertEqual(algo.annualizer, 10)
tests/test_algorithm_gen.py
+26 -4
View File
@@ -84,20 +84,42 @@ class AlgorithmGeneratorTestCase(TestCase):
Ensure the pipeline of generators are in sync, at least as far as
their current dates.
"""
algo = TestAlgo(self)
trading_environment = factory.create_trading_environment(
sim_params = factory.create_simulation_parameters(
start=datetime(2011, 7, 30, tzinfo=pytz.utc),
end=datetime(2012, 7, 30, tzinfo=pytz.utc)
)
algo = TestAlgo(self, sim_params=sim_params)
trade_source = factory.create_daily_trade_source(
[8229],
200,
trading_environment
sim_params
)
algo.set_sources([trade_source])
gen = algo.get_generator(trading_environment)
gen = algo.get_generator()
self.assertTrue(list(gen))
self.assertTrue(algo.slippage.latest_date)
self.assertTrue(algo.latest_date)
@timed(DEFAULT_TIMEOUT)
def test_progress(self):
"""
Ensure the pipeline of generators are in sync, at least as far as
their current dates.
"""
sim_params = factory.create_simulation_parameters(
start=datetime(2008, 1, 1, tzinfo=pytz.utc),
end=datetime(2008, 1, 5, tzinfo=pytz.utc)
)
algo = TestAlgo(self, sim_params=sim_params)
trade_source = factory.create_daily_trade_source(
[8229],
3,
sim_params
)
algo.set_sources([trade_source])
gen = algo.get_generator()
results = list(gen)
self.assertEqual(results[-2]['progress'], 1.0)
tests/test_exception_handling.py
+8 -3
View File
@@ -16,6 +16,8 @@
from unittest import TestCase
import zipline.utils.simfactory as simfactory
import zipline.utils.factory as factory
from zipline.test_algorithms import (
ExceptionAlgorithm,
DivByZeroAlgorithm,
@@ -83,7 +85,8 @@ class ExceptionTestCase(TestCase):
self.zipline_test_config['algorithm'] = \
ExceptionAlgorithm(
'handle_data',
self.zipline_test_config['sid']
self.zipline_test_config['sid'],
sim_params=factory.create_simulation_parameters()
)
zipline = simfactory.create_test_zipline(
@@ -102,7 +105,8 @@ class ExceptionTestCase(TestCase):
# ----------
self.zipline_test_config['algorithm'] = \
DivByZeroAlgorithm(
self.zipline_test_config['sid']
self.zipline_test_config['sid'],
sim_params=factory.create_simulation_parameters()
)
zipline = simfactory.create_test_zipline(
@@ -124,7 +128,8 @@ class ExceptionTestCase(TestCase):
# ----------
self.zipline_test_config['algorithm'] = \
SetPortfolioAlgorithm(
self.zipline_test_config['sid']
self.zipline_test_config['sid'],
sim_params=factory.create_simulation_parameters()
)
zipline = simfactory.create_test_zipline(
tests/test_finance.py
+21 -28
View File
@@ -28,7 +28,9 @@ from nose.tools import timed
import zipline.utils.factory as factory
import zipline.utils.simfactory as simfactory
from zipline.finance.trading import TradingEnvironment
import zipline.finance.trading as trading
from zipline.finance.trading import SimulationParameters
from zipline.finance.performance import PerformanceTracker
from zipline.utils.protocol_utils import ndict
from zipline.finance.trading import TransactionSimulator
@@ -56,11 +58,11 @@ class FinanceTestCase(TestCase):
@timed(DEFAULT_TIMEOUT)
def test_factory_daily(self):
trading_environment = factory.create_trading_environment()
sim_params = factory.create_simulation_parameters()
trade_source = factory.create_daily_trade_source(
[133],
200,
trading_environment
sim_params
)
prev = None
for trade in trade_source:
@@ -70,16 +72,6 @@ class FinanceTestCase(TestCase):
@timed(DEFAULT_TIMEOUT)
def test_trading_environment(self):
benchmark_returns, treasury_curves = \
factory.load_market_data()
env = TradingEnvironment(
benchmark_returns,
treasury_curves,
period_start=datetime(2008, 1, 1, tzinfo=pytz.utc),
period_end=datetime(2008, 12, 31, tzinfo=pytz.utc),
capital_base=100000,
)
#holidays taken from: http://www.nyse.com/press/1191407641943.html
new_years = datetime(2008, 1, 1, tzinfo=pytz.utc)
mlk_day = datetime(2008, 1, 21, tzinfo=pytz.utc)
@@ -107,23 +99,27 @@ class FinanceTestCase(TestCase):
]
for holiday in holidays:
self.assertTrue(not env.is_trading_day(holiday))
self.assertTrue(not trading.environment.is_trading_day(holiday))
first_trading_day = datetime(2008, 1, 2, tzinfo=pytz.utc)
last_trading_day = datetime(2008, 12, 31, tzinfo=pytz.utc)
workdays = [first_trading_day, last_trading_day]
for workday in workdays:
self.assertTrue(env.is_trading_day(workday))
self.assertTrue(trading.environment.is_trading_day(workday))
def test_simulation_parameters(self):
env = SimulationParameters(
period_start=datetime(2008, 1, 1, tzinfo=pytz.utc),
period_end=datetime(2008, 12, 31, tzinfo=pytz.utc),
capital_base=100000,
)
self.assertTrue(env.last_close.month == 12)
self.assertTrue(env.last_close.day == 31)
@timed(DEFAULT_TIMEOUT)
def test_trading_environment_days_in_period(self):
benchmark_returns, treasury_curves = \
factory.load_market_data()
def test_sim_params_days_in_period(self):
# January 2008
# Su Mo Tu We Th Fr Sa
@@ -133,9 +129,7 @@ class FinanceTestCase(TestCase):
# 20 21 22 23 24 25 26
# 27 28 29 30 31
env = TradingEnvironment(
benchmark_returns,
treasury_curves,
env = SimulationParameters(
period_start=datetime(2007, 12, 31, tzinfo=pytz.utc),
period_end=datetime(2008, 1, 7, tzinfo=pytz.utc),
capital_base=100000,
@@ -154,10 +148,9 @@ class FinanceTestCase(TestCase):
)
num_expected_trading_days = 5
self.assertEquals(num_expected_trading_days, env.days_in_period)
np.testing.assert_array_equal(expected_trading_days,
env.period_trading_days)
env.trading_days.tolist())
@timed(EXTENDED_TIMEOUT)
def test_full_zipline(self):
@@ -288,18 +281,18 @@ class FinanceTestCase(TestCase):
complete_fill = params.get('complete_fill')
sid = 1
trading_environment = factory.create_trading_environment()
sim_params = factory.create_simulation_parameters()
trade_sim = TransactionSimulator()
price = [10.1] * trade_count
volume = [100] * trade_count
start_date = trading_environment.first_open
start_date = sim_params.first_open
generated_trades = factory.create_trade_history(
sid,
price,
volume,
trade_interval,
trading_environment
sim_params
)
if alternate:
@@ -337,7 +330,7 @@ class FinanceTestCase(TestCase):
self.assertEqual(order.sid, sid)
self.assertEqual(order.amount, order_amount * alternator ** i)
tracker = PerformanceTracker(trading_environment)
tracker = PerformanceTracker(sim_params)
# this approximates the loop inside TradingSimulationClient
transactions = []
tests/test_perf_tracking.py
+37 -43
View File
@@ -27,9 +27,8 @@ import zipline.finance.performance as perf
from zipline.utils.protocol_utils import ndict
from zipline.gens.composites import date_sorted_sources
from zipline.finance.trading import TradingEnvironment
from zipline.utils.factory import create_random_trading_environment
from zipline.finance.trading import SimulationParameters
from zipline.utils.factory import create_random_simulation_parameters
onesec = datetime.timedelta(seconds=1)
oneday = datetime.timedelta(days=1)
@@ -40,10 +39,10 @@ class TestDividendPerformance(unittest.TestCase):
def setUp(self):
self.trading_environment, self.dt, self.end_dt = \
create_random_trading_environment()
self.sim_params, self.dt, self.end_dt = \
create_random_simulation_parameters()
self.trading_environment.capital_base = 10e3
self.sim_params.capital_base = 10e3
def test_long_position_receives_dividend(self):
#post some trades in the market
@@ -52,7 +51,7 @@ class TestDividendPerformance(unittest.TestCase):
[10, 10, 10, 10, 10],
[100, 100, 100, 100, 100],
oneday,
self.trading_environment
self.sim_params
)
dividend = factory.create_dividend(
@@ -71,7 +70,7 @@ class TestDividendPerformance(unittest.TestCase):
txn = factory.create_txn(1, 10.0, 100, events[0].dt)
events[0].TRANSACTION = txn
events.insert(1, dividend)
perf_tracker = perf.PerformanceTracker(self.trading_environment)
perf_tracker = perf.PerformanceTracker(self.sim_params)
transformed_events = list(perf_tracker.transform(
((event.dt, [event]) for event in events))
)
@@ -114,7 +113,7 @@ class TestDividendPerformance(unittest.TestCase):
[10, 10, 10, 10, 10],
[100, 100, 100, 100, 100],
oneday,
self.trading_environment
self.sim_params
)
dividend = factory.create_dividend(
@@ -128,7 +127,7 @@ class TestDividendPerformance(unittest.TestCase):
events.insert(1, dividend)
txn = factory.create_txn(1, 10.0, 100, events[3].dt)
events[3].TRANSACTION = txn
perf_tracker = perf.PerformanceTracker(self.trading_environment)
perf_tracker = perf.PerformanceTracker(self.sim_params)
transformed_events = list(perf_tracker.transform(
((event.dt, [event]) for event in events))
)
@@ -162,7 +161,7 @@ class TestDividendPerformance(unittest.TestCase):
[10, 10, 10, 10, 10],
[100, 100, 100, 100, 100],
oneday,
self.trading_environment
self.sim_params
)
dividend = factory.create_dividend(
@@ -178,7 +177,7 @@ class TestDividendPerformance(unittest.TestCase):
sell_txn = factory.create_txn(1, 10.0, -100, events[2].dt)
events[2].TRANSACTION = sell_txn
events.insert(1, dividend)
perf_tracker = perf.PerformanceTracker(self.trading_environment)
perf_tracker = perf.PerformanceTracker(self.sim_params)
transformed_events = list(perf_tracker.transform(
((event.dt, [event]) for event in events))
)
@@ -212,7 +211,7 @@ class TestDividendPerformance(unittest.TestCase):
[10, 10, 10, 10, 10, 10],
[100, 100, 100, 100, 100, 100],
oneday,
self.trading_environment
self.sim_params
)
dividend = factory.create_dividend(
@@ -228,7 +227,7 @@ class TestDividendPerformance(unittest.TestCase):
sell_txn = factory.create_txn(1, 10.0, -100, events[2].dt)
events[2].TRANSACTION = sell_txn
events.insert(1, dividend)
perf_tracker = perf.PerformanceTracker(self.trading_environment)
perf_tracker = perf.PerformanceTracker(self.sim_params)
transformed_events = list(perf_tracker.transform(
((event.dt, [event]) for event in events))
)
@@ -262,7 +261,7 @@ class TestDividendPerformance(unittest.TestCase):
[10, 10, 10, 10, 10],
[100, 100, 100, 100, 100],
oneday,
self.trading_environment
self.sim_params
)
dividend = factory.create_dividend(
@@ -276,7 +275,7 @@ class TestDividendPerformance(unittest.TestCase):
buy_txn = factory.create_txn(1, 10.0, 100, events[1].dt)
events[1].TRANSACTION = buy_txn
events.insert(1, dividend)
perf_tracker = perf.PerformanceTracker(self.trading_environment)
perf_tracker = perf.PerformanceTracker(self.sim_params)
transformed_events = list(perf_tracker.transform(
((event.dt, [event]) for event in events))
)
@@ -313,7 +312,7 @@ class TestDividendPerformance(unittest.TestCase):
[10, 10, 10, 10, 10],
[100, 100, 100, 100, 100],
oneday,
self.trading_environment
self.sim_params
)
dividend = factory.create_dividend(
@@ -327,7 +326,7 @@ class TestDividendPerformance(unittest.TestCase):
txn = factory.create_txn(1, 10.0, -100, self.dt+oneday)
events[0].TRANSACTION = txn
events.insert(0, dividend)
perf_tracker = perf.PerformanceTracker(self.trading_environment)
perf_tracker = perf.PerformanceTracker(self.sim_params)
transformed_events = list(perf_tracker.transform(
((event.dt, [event]) for event in events))
)
@@ -361,7 +360,7 @@ class TestDividendPerformance(unittest.TestCase):
[10, 10, 10, 10, 10],
[100, 100, 100, 100, 100],
oneday,
self.trading_environment
self.sim_params
)
dividend = factory.create_dividend(
@@ -373,7 +372,7 @@ class TestDividendPerformance(unittest.TestCase):
)
events.insert(1, dividend)
perf_tracker = perf.PerformanceTracker(self.trading_environment)
perf_tracker = perf.PerformanceTracker(self.sim_params)
transformed_events = list(perf_tracker.transform(
((event.dt, [event]) for event in events))
)
@@ -404,8 +403,8 @@ class TestDividendPerformance(unittest.TestCase):
class TestPositionPerformance(unittest.TestCase):
def setUp(self):
self.trading_environment, self.dt, self.end_dt = \
create_random_trading_environment()
self.sim_params, self.dt, self.end_dt = \
create_random_simulation_parameters()
def test_long_position(self):
"""
@@ -418,7 +417,7 @@ class TestPositionPerformance(unittest.TestCase):
[10, 10, 10, 11],
[100, 100, 100, 100],
onesec,
self.trading_environment
self.sim_params
)
txn = factory.create_txn(1, 10.0, 100, self.dt + onesec)
@@ -487,7 +486,7 @@ single short-sale transaction"""
[10, 10, 10, 11, 10, 9],
[100, 100, 100, 100, 100, 100],
onesec,
self.trading_environment
self.sim_params
)
trades_1 = trades[:-2]
@@ -677,7 +676,7 @@ trade after cover"""
[10, 10, 10, 11, 9, 8, 7, 8, 9, 10],
[100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
onesec,
self.trading_environment
self.sim_params
)
short_txn = factory.create_txn(
@@ -756,7 +755,7 @@ shares in position"
[10, 11, 11, 12],
[100, 100, 100, 100],
onesec,
self.trading_environment
self.sim_params
)
transactions = factory.create_txn_history(
@@ -764,7 +763,7 @@ shares in position"
[10, 11, 11, 12],
[100, 100, 100, 100],
onesec,
self.trading_environment
self.sim_params
)
pp = perf.PerformancePeriod(1000.0)
@@ -914,12 +913,7 @@ class TestPerformanceTracker(unittest.TestCase):
volume = [100] * trade_count
trade_time_increment = datetime.timedelta(days=1)
benchmark_returns, treasury_curves = \
factory.load_market_data()
trading_environment = TradingEnvironment(
benchmark_returns,
treasury_curves,
sim_params = SimulationParameters(
period_start=start_dt,
period_end=end_dt
)
@@ -929,7 +923,7 @@ class TestPerformanceTracker(unittest.TestCase):
price_list,
volume,
trade_time_increment,
trading_environment,
sim_params,
source_id="factory1"
)
@@ -941,7 +935,7 @@ class TestPerformanceTracker(unittest.TestCase):
price2_list,
volume,
trade_time_increment,
trading_environment,
sim_params,
source_id="factory2"
)
# 'middle' start of 3 depends on number of days == 7
@@ -962,19 +956,19 @@ class TestPerformanceTracker(unittest.TestCase):
del trade_history[-days_to_delete.end:]
del trade_history2[-days_to_delete.end:]
trading_environment.first_open = \
trading_environment.calculate_first_open()
trading_environment.last_close = \
trading_environment.calculate_last_close()
trading_environment.capital_base = 1000.0
trading_environment.frame_index = [
sim_params.first_open = \
sim_params.calculate_first_open()
sim_params.last_close = \
sim_params.calculate_last_close()
sim_params.capital_base = 1000.0
sim_params.frame_index = [
'sid',
'volume',
'dt',
'price',
'changed']
perf_tracker = perf.PerformanceTracker(
trading_environment
sim_params
)
events = date_sorted_sources(trade_history, trade_history2)
@@ -1007,7 +1001,7 @@ class TestPerformanceTracker(unittest.TestCase):
perf_tracker.cumulative_risk_metrics.end_date)
self.assertEqual(len(perf_messages),
trading_environment.days_in_period)
sim_params.days_in_period)
def event_with_txn(self, event, no_txn_dt):
#create a transaction for all but
tests/test_risk.py
+27 -37
View File
@@ -20,7 +20,7 @@ import pytz
import zipline.finance.risk as risk
from zipline.utils import factory
from zipline.finance.trading import TradingEnvironment
from zipline.finance.trading import SimulationParameters
class Risk(unittest.TestCase):
@@ -37,12 +37,7 @@ class Risk(unittest.TestCase):
end_date = datetime.datetime(
year=2006, month=12, day=31, tzinfo=pytz.utc)
self.benchmark_returns, self.treasury_curves = \
factory.load_market_data()
self.trading_env = TradingEnvironment(
self.benchmark_returns,
self.treasury_curves,
self.sim_params = SimulationParameters(
period_start=start_date,
period_end=end_date
)
@@ -54,12 +49,12 @@ class Risk(unittest.TestCase):
self.algo_returns_06 = factory.create_returns_from_list(
RETURNS,
self.trading_env
self.sim_params
)
self.metrics_06 = risk.RiskReport(
self.algo_returns_06,
self.trading_env
self.sim_params
)
start_08 = datetime.datetime(
@@ -76,9 +71,7 @@ class Risk(unittest.TestCase):
day=31,
tzinfo=pytz.utc
)
self.trading_env08 = TradingEnvironment(
self.benchmark_returns,
self.treasury_curves,
self.sim_params08 = SimulationParameters(
period_start=start_08,
period_end=end_08
)
@@ -88,24 +81,23 @@ class Risk(unittest.TestCase):
def test_factory(self):
returns = [0.1] * 100
r_objects = factory.create_returns_from_list(returns, self.trading_env)
r_objects = factory.create_returns_from_list(returns, self.sim_params)
self.assertTrue(r_objects[-1].date <=
datetime.datetime(
year=2006, month=12, day=31, tzinfo=pytz.utc))
def test_drawdown(self):
returns = factory.create_returns_from_list(
[1.0, -0.5, 0.8, .17, 1.0, -0.1, -0.45], self.trading_env)
[1.0, -0.5, 0.8, .17, 1.0, -0.1, -0.45], self.sim_params)
#200, 100, 180, 210.6, 421.2, 379.8, 208.494
metrics = risk.RiskMetricsBatch(returns[0].date,
returns[-1].date,
returns,
self.trading_env)
returns)
self.assertEqual(metrics.max_drawdown, 0.505)
def test_benchmark_returns_06(self):
returns = factory.create_returns_from_range(self.trading_env)
metrics = risk.RiskReport(returns, self.trading_env)
returns = factory.create_returns_from_range(self.sim_params)
metrics = risk.RiskReport(returns, self.sim_params)
self.assertEqual([round(x.benchmark_period_returns, 4)
for x in metrics.month_periods],
[0.0255,
@@ -146,16 +138,16 @@ class Risk(unittest.TestCase):
[0.1407])
def test_trading_days_06(self):
returns = factory.create_returns_from_range(self.trading_env)
metrics = risk.RiskReport(returns, self.trading_env)
returns = factory.create_returns_from_range(self.sim_params)
metrics = risk.RiskReport(returns, self.sim_params)
self.assertEqual([x.trading_days for x in metrics.year_periods],
[251])
self.assertEqual([x.trading_days for x in metrics.month_periods],
[20, 19, 23, 19, 22, 22, 20, 23, 20, 22, 21, 20])
def test_benchmark_volatility_06(self):
returns = factory.create_returns_from_range(self.trading_env)
metrics = risk.RiskReport(returns, self.trading_env)
returns = factory.create_returns_from_range(self.sim_params)
metrics = risk.RiskReport(returns, self.sim_params)
self.assertEqual([round(x.benchmark_volatility, 3)
for x in metrics.month_periods],
[0.031,
@@ -588,8 +580,8 @@ class Risk(unittest.TestCase):
[0.0000399])
def test_benchmark_returns_08(self):
returns = factory.create_returns_from_range(self.trading_env08)
metrics = risk.RiskReport(returns, self.trading_env08)
returns = factory.create_returns_from_range(self.sim_params08)
metrics = risk.RiskReport(returns, self.sim_params08)
monthly = [round(x.benchmark_period_returns, 3)
for x in metrics.month_periods]
@@ -636,8 +628,8 @@ class Risk(unittest.TestCase):
[-0.353])
def test_trading_days_08(self):
returns = factory.create_returns_from_range(self.trading_env08)
metrics = risk.RiskReport(returns, self.trading_env08)
returns = factory.create_returns_from_range(self.sim_params08)
metrics = risk.RiskReport(returns, self.sim_params08)
self.assertEqual([x.trading_days for x in metrics.year_periods],
[253])
@@ -645,8 +637,8 @@ class Risk(unittest.TestCase):
[21, 20, 20, 22, 21, 21, 22, 21, 21, 23, 19, 22])
def test_benchmark_volatility_08(self):
returns = factory.create_returns_from_range(self.trading_env08)
metrics = risk.RiskReport(returns, self.trading_env08)
returns = factory.create_returns_from_range(self.sim_params08)
metrics = risk.RiskReport(returns, self.sim_params08)
self.assertEqual([round(x.benchmark_volatility, 3)
for x in metrics.month_periods],
[0.069,
@@ -692,8 +684,8 @@ class Risk(unittest.TestCase):
[0.391])
def test_treasury_returns_06(self):
returns = factory.create_returns_from_range(self.trading_env)
metrics = risk.RiskReport(returns, self.trading_env)
returns = factory.create_returns_from_range(self.sim_params)
metrics = risk.RiskReport(returns, self.sim_params)
self.assertEqual([round(x.treasury_period_return, 4)
for x in metrics.month_periods],
[0.0037,
@@ -752,16 +744,14 @@ class Risk(unittest.TestCase):
#1992 and 1996 were leap years
total_days = 365 * 5 + 2
end = start + datetime.timedelta(days=total_days)
trading_env90s = TradingEnvironment(
self.benchmark_returns,
self.treasury_curves,
sim_params90s = SimulationParameters(
period_start=start,
period_end=end
)
returns = factory.create_returns(total_days, trading_env90s)
returns = factory.create_returns(total_days, sim_params90s)
returns = returns[:-10] # truncate the returns series to end mid-month
metrics = risk.RiskReport(returns, trading_env90s)
metrics = risk.RiskReport(returns, sim_params90s)
total_months = 60
self.check_metrics(metrics, total_months, start)
@@ -773,8 +763,8 @@ class Risk(unittest.TestCase):
# and i think this func is [start,end)
ld = calendar.leapdays(start_date.year,
start_date.year + years + 1)
returns = factory.create_returns(365 * years + ld, self.trading_env08)
metrics = risk.RiskReport(returns, self.trading_env)
returns = factory.create_returns(365 * years + ld, self.sim_params08)
metrics = risk.RiskReport(returns, self.sim_params)
total_months = years * 12
self.check_metrics(metrics, total_months, start_date)
tests/test_risk_compare_batch_iterative.py
+6 -16
View File
@@ -21,9 +21,9 @@ import pytz
import numpy as np
import zipline.finance.risk as risk
from zipline.utils import factory
from zipline.finance.trading import TradingEnvironment
import zipline.finance.trading as trading
from test_risk import RETURNS
@@ -43,22 +43,13 @@ class RiskCompareIterativeToBatch(unittest.TestCase):
tzinfo=pytz.utc)
self.end_date = datetime.datetime(
year=2006, month=12, day=31, tzinfo=pytz.utc)
self.benchmark_returns, self.treasury_curves = \
factory.load_market_data()
self.trading_env = TradingEnvironment(
self.benchmark_returns,
self.treasury_curves,
period_start=self.start_date,
period_end=self.end_date,
capital_base=1000.0
)
# setup the default trading environment
trading.environment = TradingEnvironment()
self.oneday = datetime.timedelta(days=1)
def test_risk_metrics_returns(self):
risk_metrics_refactor = risk.RiskMetricsIterative(
self.start_date, self.trading_env)
risk_metrics_refactor = risk.RiskMetricsIterative(self.start_date)
todays_date = self.start_date
@@ -73,15 +64,14 @@ class RiskCompareIterativeToBatch(unittest.TestCase):
# Move forward day counter to next trading day
todays_date += self.oneday
while not self.trading_env.is_trading_day(todays_date):
while not trading.environment.is_trading_day(todays_date):
todays_date += self.oneday
try:
risk_metrics_original = risk.RiskMetricsBatch(
start_date=self.start_date,
end_date=todays_date,
returns=cur_returns,
trading_environment=self.trading_env
returns=cur_returns
)
except Exception as e:
#assert that when original raises exception, same
tests/test_tradingcalendar.py
+20
View File
@@ -17,10 +17,30 @@ from unittest import TestCase
from zipline.utils import tradingcalendar
import pytz
import datetime
from zipline.finance.trading import TradingEnvironment
class TestTradingCalendar(TestCase):
def test_calendar_vs_environment(self):
"""
test_calendar_vs_environment checks whether the
historical data from yahoo matches our rule based system.
handy, if not canonical, reference:
http://www.chronos-st.org/NYSE_Observed_Holidays-1885-Present.html
"""
env = TradingEnvironment()
env_start_index = \
env.trading_days.searchsorted(tradingcalendar.start)
env_days = env.trading_days[env_start_index:]
diff = env_days - tradingcalendar.trading_days
self.assertEqual(
len(diff),
0,
"{diff} should be empty".format(diff=diff)
)
def test_newyears(self):
"""
Check whether tradingcalendar contains certain dates.
tests/test_transforms.py
+18 -9
View File
@@ -61,6 +61,8 @@ class NoopEventWindow(EventWindow):
class TestEventWindow(TestCase):
def setUp(self):
self.sim_params = factory.create_simulation_parameters()
setup_logger(self)
self.monday = datetime(2012, 7, 9, 16, tzinfo=pytz.utc)
@@ -126,7 +128,7 @@ class TestEventWindow(TestCase):
class TestFinanceTransforms(TestCase):
def setUp(self):
self.trading_environment = factory.create_trading_environment()
self.sim_params = factory.create_simulation_parameters()
setup_logger(self)
trade_history = factory.create_trade_history(
@@ -134,7 +136,7 @@ class TestFinanceTransforms(TestCase):
[10.0, 10.0, 11.0, 11.0],
[100, 100, 100, 300],
timedelta(days=1),
self.trading_environment
self.sim_params
)
self.source = SpecificEquityTrades(event_list=trade_history)
@@ -142,7 +144,6 @@ class TestFinanceTransforms(TestCase):
self.log_handler.pop_application()
def test_vwap(self):
vwap = MovingVWAP(
market_aware=True,
window_length=2
@@ -186,7 +187,7 @@ class TestFinanceTransforms(TestCase):
[10.0, 15.0, 13.0, 12.0, 13.0],
[100, 100, 100, 300, 100],
timedelta(days=1),
self.trading_environment
self.sim_params
)
self.source = SpecificEquityTrades(event_list=trade_history)
@@ -247,7 +248,7 @@ class TestFinanceTransforms(TestCase):
[10.0, 15.0, 13.0, 12.0],
[100, 100, 100, 100],
timedelta(days=1),
self.trading_environment
self.sim_params
)
stddev = MovingStandardDev(
@@ -283,8 +284,13 @@ class TestFinanceTransforms(TestCase):
class TestBatchTransform(TestCase):
def setUp(self):
self.sim_params = factory.create_simulation_parameters(
start=datetime(1990, 1, 1, tzinfo=pytz.utc),
end=datetime(1990, 1, 8, tzinfo=pytz.utc)
)
setup_logger(self)
self.source, self.df = factory.create_test_df_source()
self.source, self.df = \
factory.create_test_df_source(self.sim_params)
def test_event_window(self):
algo = BatchTransformAlgorithm()
@@ -361,12 +367,15 @@ class TestBatchTransform(TestCase):
self.assertEqual(
algo.history_return_args,
[
# 1990-01-01 - market holiday, no event
# 1990-01-02 - window not full
None,
# 1990-01-03 - window not full
None,
# 1990-01-04 - window not full
None,
# 1990-01-05 - window not full, 3rd event
# 1990-01-04 - window not full, 3rd event
None,
# 1990-01-05 - window now full
expected_item,
# 1990-01-08 - window now full
expected_item
])
zipline/algorithm.py
+15 -12
View File
@@ -24,7 +24,7 @@ from itertools import groupby
from operator import attrgetter
from zipline.sources import DataFrameSource, DataPanelSource
from zipline.utils.factory import create_trading_environment
from zipline.utils.factory import create_simulation_parameters
from zipline.transforms.utils import StatefulTransform
from zipline.finance.slippage import (
VolumeShareSlippage,
@@ -107,6 +107,8 @@ class TradingAlgorithm(object):
# set the capital base
self.capital_base = kwargs.get('capital_base', DEFAULT_CAPITAL_BASE)
self.sim_params = kwargs.pop('sim_params', None)
# an algorithm subclass needs to set initialized to True when
# it is fully initialized.
self.initialized = False
@@ -114,7 +116,7 @@ class TradingAlgorithm(object):
# call to user-defined constructor method
self.initialize(*args, **kwargs)
def _create_generator(self, environment):
def _create_generator(self, sim_params):
"""
Create a basic generator setup using the sources and
transforms attached to this algorithm.
@@ -127,20 +129,20 @@ class TradingAlgorithm(object):
# Group together events with the same dt field. This depends on the
# events already being sorted.
self.grouped_by_date = groupby(self.with_alias_dt, attrgetter('dt'))
self.trading_client = tsc(self, environment)
self.trading_client = tsc(self, sim_params)
transact_method = transact_partial(self.slippage, self.commission)
self.set_transact(transact_method)
return self.trading_client.simulate(self.grouped_by_date)
def get_generator(self, environment):
def get_generator(self):
"""
Override this method to add new logic to the construction
of the generator. Overrides can use the _create_generator
method to get a standard construction generator.
"""
return self._create_generator(environment)
return self._create_generator(self.sim_params)
def initialize(self, *args, **kwargs):
pass
@@ -190,6 +192,13 @@ class TradingAlgorithm(object):
else:
self.sources = source
if not self.sim_params:
self.sim_params = create_simulation_parameters(
start=start,
end=end,
capital_base=self.capital_base
)
# Create transforms by wrapping them into StatefulTransforms
self.transforms = []
for namestring, trans_descr in self.registered_transforms.iteritems():
@@ -202,14 +211,8 @@ class TradingAlgorithm(object):
self.transforms.append(sf)
environment = create_trading_environment(
start=start,
end=end,
capital_base=self.capital_base
)
# create transforms and zipline
self.gen = self._create_generator(environment)
self.gen = self._create_generator(self.sim_params)
# loop through simulated_trading, each iteration returns a
# perf ndict
zipline/data/benchmarks.py
+8 -9
View File
@@ -33,7 +33,7 @@ from loader_utils import Mapping
from zipline.finance.risk import DailyReturn
_BENCHMARK_MAPPING = {
# Need to add 'symbol' and GSPC as a constant
# Need to add 'symbol'
'volume': (int, 'Volume'),
'open': (float, 'Open'),
'close': (float, 'Close'),
@@ -50,13 +50,12 @@ def benchmark_mappings():
in _BENCHMARK_MAPPING.iteritems()}
def get_raw_benchmark_data(start_date, end_date):
def get_raw_benchmark_data(start_date, end_date, symbol):
# create benchmark files
# ^GSPC 19500103
params = {
# the s&p 500
's': '^GSPC',
's': symbol,
# end_date month, zero indexed
'd': end_date.month - 1,
# end_date day str(int(todate[6:8])) #day
@@ -79,14 +78,14 @@ def get_raw_benchmark_data(start_date, end_date):
return csv.DictReader(StringIO(res.content))
def get_benchmark_data():
def get_benchmark_data(symbol):
"""
Benchmarks from Yahoo's GSPC source.
Benchmarks from Yahoo.
"""
start_date = datetime(year=1950, month=1, day=3)
end_date = datetime.utcnow()
raw_benchmark_data = get_raw_benchmark_data(start_date, end_date)
raw_benchmark_data = get_raw_benchmark_data(start_date, end_date, symbol)
# Reverse data so we can load it in reverse chron order.
benchmarks_source = reversed(list(raw_benchmark_data))
@@ -95,11 +94,11 @@ def get_benchmark_data():
return source_to_records(mappings, benchmarks_source)
def get_benchmark_returns():
def get_benchmark_returns(symbol):
benchmark_returns = []
for data_point in get_benchmark_data():
for data_point in get_benchmark_data(symbol):
returns = (data_point['close'] - data_point['open']) / \
data_point['open']
daily_return = DailyReturn(date=data_point['date'], returns=returns)
zipline/data/loader.py
+63 -4
View File
@@ -16,12 +16,18 @@
import os
from os.path import expanduser
import msgpack
from collections import OrderedDict
from treasuries import get_treasury_data
from benchmarks import get_benchmark_returns
from zipline.utils.date_utils import tuple_to_date
import zipline.finance.risk as risk
from operator import attrgetter
# TODO: Make this path customizable.
DATA_PATH = os.path.join(
expanduser("~"),
@@ -63,19 +69,72 @@ def dump_treasury_curves():
tr_fp.write(msgpack.dumps(tr_data))
def dump_benchmarks():
def dump_benchmarks(symbol):
"""
Dumps data to be used with zipline.
Puts source treasury and data into zipline.
"""
benchmark_data = []
for daily_return in get_benchmark_returns():
for daily_return in get_benchmark_returns(symbol):
date_as_tuple = daily_return.date.timetuple()[0:6] + \
(daily_return.date.microsecond,)
# Not ideal but massaging data into expected format
benchmark = (date_as_tuple, daily_return.returns)
benchmark_data.append(benchmark)
with get_datafile('benchmark.msgpack', mode='wb') as bmark_fp:
with get_datafile(get_benchmark_filename(symbol), mode='wb') as bmark_fp:
bmark_fp.write(msgpack.dumps(benchmark_data))
def get_benchmark_filename(symbol):
return "%s_benchmark.msgpack" % symbol
def load_market_data(bm_symbol='^GSPC'):
try:
fp_bm = get_datafile(get_benchmark_filename(bm_symbol), "rb")
except IOError:
print """
data msgpacks aren't distribute with source.
Fetching data from Yahoo Finance.
""".strip()
dump_benchmarks(bm_symbol)
fp_bm = get_datafile(get_benchmark_filename(bm_symbol), "rb")
bm_list = msgpack.loads(fp_bm.read())
bm_returns = []
for packed_date, returns in bm_list:
event_dt = tuple_to_date(packed_date)
daily_return = risk.DailyReturn(date=event_dt, returns=returns)
bm_returns.append(daily_return)
fp_bm.close()
bm_returns = sorted(bm_returns, key=attrgetter('date'))
try:
fp_tr = get_datafile('treasury_curves.msgpack', "rb")
except IOError:
print """
data msgpacks aren't distribute with source.
Fetching data from data.treasury.gov
""".strip()
dump_treasury_curves()
fp_tr = get_datafile('treasury_curves.msgpack', "rb")
tr_list = msgpack.loads(fp_tr.read())
tr_curves = {}
for packed_date, curve in tr_list:
tr_dt = tuple_to_date(packed_date)
#tr_dt = tr_dt.replace(hour=0, minute=0, second=0, tzinfo=pytz.utc)
tr_curves[tr_dt] = curve
fp_tr.close()
tr_curves = OrderedDict(sorted(
((dt, c) for dt, c in tr_curves.iteritems()),
key=lambda t: t[0]))
return bm_returns, tr_curves
zipline/finance/performance.py
+20 -32
View File
@@ -141,6 +141,7 @@ import numpy as np
import zipline.protocol as zp
import zipline.finance.risk as risk
import zipline.finance.trading as trading
log = logbook.Logger('Performance')
@@ -157,28 +158,28 @@ class PerformanceTracker(object):
"""
def __init__(self, trading_environment):
def __init__(self, sim_params):
self.trading_environment = trading_environment
self.trading_day = datetime.timedelta(hours=6, minutes=30)
self.sim_params = sim_params
self.started_at = datetime.datetime.utcnow().replace(tzinfo=pytz.utc)
self.period_start = self.trading_environment.period_start
self.period_end = self.trading_environment.period_end
self.last_close = self.trading_environment.last_close
self.market_open = self.trading_environment.first_open
self.market_close = self.market_open + self.trading_day
self.period_start = self.sim_params.period_start
self.period_end = self.sim_params.period_end
self.last_close = self.sim_params.last_close
first_day = self.sim_params.first_open
self.market_open, self.market_close = \
trading.environment.get_open_and_close(first_day)
self.progress = 0.0
self.total_days = self.trading_environment.days_in_period
self.total_days = self.sim_params.days_in_period
# one indexed so that we reach 100%
self.day_count = 0.0
self.capital_base = self.trading_environment.capital_base
self.capital_base = self.sim_params.capital_base
self.returns = []
self.txn_count = 0
self.event_count = 0
self.last_dict = None
self.cumulative_risk_metrics = risk.RiskMetricsIterative(
self.period_start, self.trading_environment)
self.cumulative_risk_metrics = \
risk.RiskMetricsIterative(self.period_start)
# this performance period will span the entire simulation.
self.cumulative_performance = PerformancePeriod(
@@ -203,7 +204,7 @@ class PerformanceTracker(object):
def __repr__(self):
return "%s(%r)" % (
self.__class__.__name__,
{'trading_environment': self.trading_environment})
{'simulation parameters': self.sim_params})
def transform(self, stream_in):
"""
@@ -249,7 +250,7 @@ class PerformanceTracker(object):
if event.type == zp.DATASOURCE_TYPE.TRADE:
messages = []
while event.dt > self.market_close:
while event.dt > self.market_close and event.dt < self.last_close:
messages.append(self.handle_market_close())
if event.TRANSACTION:
@@ -275,7 +276,6 @@ class PerformanceTracker(object):
return messages
def handle_market_close(self):
# add the return results from today to the list of DailyReturn objects.
todays_date = self.market_close.replace(hour=0, minute=0, second=0)
todays_return_obj = risk.DailyReturn(
@@ -305,17 +305,8 @@ class PerformanceTracker(object):
return daily_update
#move the market day markers forward
next_open = self.trading_environment.next_trading_day(self.market_open)
if next_open is None:
raise Exception(
"Attempt to backtest beyond available history. \
Last successful date: %s" % self.market_open)
# next_open is a midnight date, but we want the time too
self.market_open = next_open.replace(hour=self.market_open.hour,
minute=self.market_open.minute,
second=self.market_open.second)
self.market_close = self.market_open + self.trading_day
self.market_open, self.market_close = \
trading.environment.next_open_and_close(self.market_open)
# Roll over positions to current day.
self.todays_performance.rollover()
@@ -350,14 +341,11 @@ Last successful date: %s" % self.market_open)
log_msg = "Simulated {n} trading days out of {m}."
log.info(log_msg.format(n=int(self.day_count), m=self.total_days))
log.info("first open: {d}".format(
d=self.trading_environment.first_open))
d=self.sim_params.first_open))
log.info("last close: {d}".format(
d=self.trading_environment.last_close))
d=self.sim_params.last_close))
self.risk_report = risk.RiskReport(
self.returns,
self.trading_environment
)
self.risk_report = risk.RiskReport(self.returns, self.sim_params)
risk_dict = self.risk_report.to_dict()
return perf_messages, risk_dict
zipline/finance/risk.py
+17 -17
View File
@@ -63,8 +63,11 @@ from collections import OrderedDict
import bisect
import numpy as np
import numpy.linalg as la
import zipline.finance.trading as trading
from zipline.utils.date_utils import epoch_now
log = logbook.Logger('Risk')
@@ -110,20 +113,19 @@ class DailyReturn(object):
class RiskMetricsBase(object):
def __init__(self, start_date, end_date, returns, trading_environment):
def __init__(self, start_date, end_date, returns):
self.treasury_curves = trading_environment.treasury_curves
self.treasury_curves = trading.environment.treasury_curves
assert isinstance(self.treasury_curves, OrderedDict), \
"Treasury curves must be an OrderedDict"
self.start_date = start_date
self.end_date = end_date
self.trading_environment = trading_environment
self.algorithm_period_returns, self.algorithm_returns = \
self.calculate_period_returns(returns)
benchmark_returns = [
x for x in self.trading_environment.benchmark_returns
x for x in trading.environment.benchmark_returns
if x.date >= returns[0].date and x.date <= returns[-1].date
]
@@ -227,7 +229,7 @@ class RiskMetricsBase(object):
x.returns for x in daily_returns
if x.date >= self.start_date and
x.date <= self.end_date and
self.trading_environment.is_trading_day(x.date)
trading.environment.is_trading_day(x.date)
]
period_returns = 1.0
@@ -427,7 +429,7 @@ that date doesn't exceed treasury history range."
raise Exception(message)
def search_day_distance(self, dt):
tdd = self.trading_environment.trading_day_distance(dt, self.end_date)
tdd = trading.environment.trading_day_distance(dt, self.end_date)
if tdd is None:
return None
assert tdd >= 0
@@ -457,11 +459,10 @@ class RiskMetricsIterative(RiskMetricsBase):
Call update() method on each dt to update the metrics.
"""
def __init__(self, start_date, trading_environment):
self.treasury_curves = trading_environment.treasury_curves
def __init__(self, start_date):
self.treasury_curves = trading.environment.treasury_curves
self.start_date = start_date
self.end_date = start_date
self.trading_environment = trading_environment
self.compounded_log_returns = []
self.moving_avg = []
@@ -484,12 +485,12 @@ class RiskMetricsIterative(RiskMetricsBase):
self.trading_days = 0
self.all_benchmark_returns = [
x for x in self.trading_environment.benchmark_returns
x for x in trading.environment.benchmark_returns
if x.date >= self.start_date
]
def update(self, market_close, returns_in_period):
if self.trading_environment.is_trading_day(self.end_date):
if trading.environment.is_trading_day(self.end_date):
self.algorithm_returns.append(returns_in_period)
self.benchmark_returns.append(
self.all_benchmark_returns.pop(0).returns)
@@ -711,7 +712,7 @@ class RiskReport(object):
def __init__(
self,
algorithm_returns,
trading_environment,
sim_params
):
"""
algorithm_returns needs to be a list of daily_return objects
@@ -719,12 +720,12 @@ class RiskReport(object):
"""
self.algorithm_returns = algorithm_returns
self.trading_environment = trading_environment
self.sim_params = sim_params
self.created = epoch_now()
if len(self.algorithm_returns) == 0:
start_date = self.trading_environment.period_start
end_date = self.trading_environment.period_end
start_date = self.sim_params.period_start
end_date = self.sim_params.period_end
else:
start_date = self.algorithm_returns[0].date
end_date = self.algorithm_returns[-1].date
@@ -778,8 +779,7 @@ class RiskReport(object):
cur_period_metrics = RiskMetricsBatch(
start_date=cur_start,
end_date=cur_end,
returns=self.algorithm_returns,
trading_environment=self.trading_environment
returns=self.algorithm_returns
)
ends.append(cur_period_metrics)
zipline/finance/trading.py
+162 -99
View File
@@ -13,22 +13,27 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import bisect
import pytz
import logbook
import datetime
from collections import defaultdict, OrderedDict
import bisect
from delorean import Delorean
from pandas import DatetimeIndex
import zipline.protocol as zp
from zipline.finance.slippage import (
VolumeShareSlippage,
transact_partial
)
from zipline.finance.commission import PerShare
log = logbook.Logger('Transaction Simulator')
environment = None
class TransactionSimulator(object):
@@ -60,107 +65,31 @@ class TradingEnvironment(object):
def __init__(
self,
benchmark_returns,
treasury_curves,
period_start=None,
period_end=None,
capital_base=None
load=None,
bm_symbol='^GSPC',
exchange_tz="US/Eastern"
):
self.trading_day_map = OrderedDict()
self.treasury_curves = treasury_curves
self.benchmark_returns = benchmark_returns
self.period_start = period_start
self.period_end = period_end
self.capital_base = capital_base
self.bm_symbol = bm_symbol
if not load:
from zipline.data.loader import load_market_data
load = load_market_data
self.benchmark_returns, self.treasury_curves = \
load(self.bm_symbol)
self._period_trading_days = None
self._trading_days_series = None
self.full_trading_day = datetime.timedelta(hours=6, minutes=30)
self.exchange_tz = exchange_tz
assert self.period_start <= self.period_end, \
"Period start falls after period end."
for bm in benchmark_returns:
for bm in self.benchmark_returns:
self.trading_day_map[bm.date] = bm
self.first_trading_day = next(self.trading_day_map.iterkeys())
self.last_trading_day = next(reversed(self.trading_day_map))
assert self.period_start <= self.last_trading_day, \
"Period start falls after the last known trading day."
assert self.period_end >= self.first_trading_day, \
"Period end falls before the first known trading day."
self.first_open = self.calculate_first_open()
self.last_close = self.calculate_last_close()
self.prior_day_open = self.calculate_prior_day_open()
def __repr__(self):
return "%s(%r)" % (
self.__class__.__name__,
{'first_open': self.first_open,
'last_close': self.last_close
})
def calculate_first_open(self):
"""
Finds the first trading day on or after self.period_start.
"""
first_open = self.period_start
one_day = datetime.timedelta(days=1)
while not self.is_trading_day(first_open):
first_open = first_open + one_day
first_open = self.set_NYSE_time(first_open, 9, 30)
return first_open
def calculate_prior_day_open(self):
"""
Finds the first trading day open that falls at least a day
before period_start.
"""
one_day = datetime.timedelta(days=1)
first_open = self.period_start - one_day
if first_open <= self.first_trading_day:
log.warn("Cannot calculate prior day open.")
return self.period_start
while not self.is_trading_day(first_open):
first_open = first_open - one_day
first_open = self.set_NYSE_time(first_open, 9, 30)
return first_open
def calculate_last_close(self):
"""
Finds the last trading day on or before self.period_end
"""
last_close = self.period_end
one_day = datetime.timedelta(days=1)
while not self.is_trading_day(last_close):
last_close = last_close - one_day
last_close = self.set_NYSE_time(last_close, 16, 00)
return last_close
#TODO: add other exchanges and timezones...
def set_NYSE_time(self, dt, hour, minute):
naive = datetime.datetime(
year=dt.year,
month=dt.month,
day=dt.day
)
local = pytz.timezone('US/Eastern')
local_dt = naive.replace(tzinfo=local)
# set the clock to the opening bell in NYC time.
local_dt = local_dt.replace(hour=hour, minute=minute)
# convert to UTC
utc_dt = local_dt.astimezone(pytz.utc)
return utc_dt
def normalize_date(self, test_date):
return datetime.datetime(
year=test_date.year,
@@ -181,18 +110,17 @@ class TradingEnvironment(object):
return self._period_trading_days
@property
def days_in_period(self):
"""return the number of trading days within the period [start, end)"""
return len(self.period_trading_days)
def trading_days(self):
if self._trading_days_series is None:
self._trading_days_series = \
DatetimeIndex(self.trading_day_map.iterkeys())
return self._trading_days_series
def is_market_hours(self, test_date):
if not self.is_trading_day(test_date):
return False
mkt_open = self.set_NYSE_time(test_date, 9, 30)
#TODO: half days?
mkt_close = self.set_NYSE_time(test_date, 16, 00)
mkt_open, mkt_close = self.get_open_and_close(test_date)
return test_date >= mkt_open and test_date <= mkt_close
def is_trading_day(self, test_date):
@@ -210,6 +138,46 @@ class TradingEnvironment(object):
return None
def next_open_and_close(self, start_date):
"""
Given the start_date, returns the next open and close of
the market.
"""
next_open = self.next_trading_day(start_date)
if next_open is None:
raise Exception(
"Attempt to backtest beyond available history. \
Last successful date: %s" % self.market_open)
return self.get_open_and_close(next_open)
def get_open_and_close(self, next_open):
# creating a naive datetime with the correct hour,
# minute, and date. this will allow us to use Delorean to
# shift the time between EST and UTC.
next_open = next_open.replace(
hour=9,
minute=30,
second=0,
tzinfo=None
)
# create a new Delorean with the next_open naive date and
# the correct timezone for the exchange.
open_delorean = Delorean(next_open, "US/Eastern")
open_utc = open_delorean.shift("UTC").datetime
market_open = open_utc
market_close = market_open + self.get_trading_day_duration(open_utc)
return market_open, market_close
def get_trading_day_duration(self, trading_day):
# TODO: make a list of half-days and modify the
# calculation of market close to reflect them.
return self.full_trading_day
def trading_day_distance(self, first_date, second_date):
first_date = self.normalize_date(first_date)
second_date = self.normalize_date(second_date)
@@ -224,3 +192,98 @@ class TradingEnvironment(object):
return None
return j - i
def get_index(self, dt):
ndt = self.normalize_date(dt)
return self.trading_days.searchsorted(ndt)
class SimulationParameters(object):
def __init__(self, period_start, period_end,
capital_base=10e3):
# raise and exception if the global environment is not
# set.
global environment
if not environment:
environment = TradingEnvironment()
self.period_start = period_start
self.period_end = period_end
self.capital_base = capital_base
self.first_open = self.calculate_first_open()
self.last_close = self.calculate_last_close()
start_index = \
environment.get_index(self.first_open)
end_index = environment.get_index(self.last_close)
# take an inclusive slice of the environment's
# trading_days.
self.trading_days = \
environment.trading_days[start_index:end_index+1]
self.prior_day_open = self.calculate_prior_day_open()
assert self.period_start <= self.period_end, \
"Period start falls after period end."
assert self.period_start <= environment.last_trading_day, \
"Period start falls after the last known trading day."
assert self.period_end >= environment.first_trading_day, \
"Period end falls before the first known trading day."
def calculate_first_open(self):
"""
Finds the first trading day on or after self.period_start.
"""
first_open = self.period_start
one_day = datetime.timedelta(days=1)
while not environment.is_trading_day(first_open):
first_open = first_open + one_day
mkt_open, _ = environment.get_open_and_close(first_open)
return mkt_open
def calculate_prior_day_open(self):
"""
Finds the first trading day open that falls at least a day
before period_start.
"""
one_day = datetime.timedelta(days=1)
first_open = self.period_start - one_day
if first_open <= environment.first_trading_day:
log.warn("Cannot calculate prior day open.")
return self.period_start
while not environment.is_trading_day(first_open):
first_open = first_open - one_day
mkt_open, _ = environment.get_open_and_close(first_open)
return mkt_open
def calculate_last_close(self):
"""
Finds the last trading day on or before self.period_end
"""
last_close = self.period_end
one_day = datetime.timedelta(days=1)
while not environment.is_trading_day(last_close):
last_close = last_close - one_day
_, mkt_close = environment.get_open_and_close(last_close)
return mkt_close
@property
def days_in_period(self):
"""return the number of trading days within the period [start, end)"""
return len(self.trading_days)
def __repr__(self):
return "%s(%r)" % (
self.__class__.__name__,
{'first_open': self.first_open,
'last_close': self.last_close
})
zipline/transforms/mavg.py
+1
View File
@@ -30,6 +30,7 @@ class MovingAverage(object):
def __init__(self, fields='price',
market_aware=True, window_length=None, delta=None):
if isinstance(fields, basestring):
fields = [fields]
self.fields = fields
zipline/transforms/returns.py
+3 -1
View File
@@ -38,7 +38,9 @@ class Returns(object):
return tracker.returns
def _create(self):
return ReturnsFromPriorClose(self.window_length)
return ReturnsFromPriorClose(
self.window_length
)
class ReturnsFromPriorClose(object):
zipline/transforms/utils.py
+8 -6
View File
@@ -29,8 +29,8 @@ from numbers import Integral
import pandas as pd
from zipline.protocol import Event, DATASOURCE_TYPE
from zipline.utils import tradingcalendar
from zipline.gens.utils import assert_sort_unframe_protocol, hash_args
import zipline.finance.trading as trading
log = logbook.Logger('Transform')
@@ -228,7 +228,8 @@ class EventWindow(object):
# Subclasses should override handle_add to define behavior for
# adding new ticks.
self.handle_add(event)
#if len(self.ticks) > self.window_length:
# import nose.tools; nose.tools.set_trace()
# Clear out any expired events.
#
# oldest newest
@@ -244,8 +245,10 @@ class EventWindow(object):
self.handle_remove(popped)
def out_of_market_window(self, oldest, newest):
oldest_index = tradingcalendar.trading_days.searchsorted(oldest)
newest_index = tradingcalendar.trading_days.searchsorted(newest)
oldest_index = \
trading.environment.trading_days.searchsorted(oldest)
newest_index = \
trading.environment.trading_days.searchsorted(newest)
trading_days_between = newest_index - oldest_index
@@ -350,8 +353,7 @@ class BatchTransform(EventWindow):
full. Returns None if window is not full yet.
"""
super(BatchTransform, self).__init__(True,
window_length=window_length)
super(BatchTransform, self).__init__(True, window_length=window_length)
if func is not None:
self.compute_transform_value = func
zipline/utils/factory.py
+61 -121
View File
@@ -18,9 +18,7 @@
Factory functions to prepare useful data for tests.
"""
import pytz
import msgpack
import random
from operator import attrgetter
from collections import OrderedDict
import pandas as pd
@@ -29,98 +27,37 @@ import numpy as np
from datetime import datetime, timedelta
import zipline.finance.risk as risk
from zipline.utils.date_utils import tuple_to_date
from zipline.protocol import Event, DATASOURCE_TYPE
from zipline.sources import (SpecificEquityTrades,
DataFrameSource,
DataPanelSource)
from zipline.gens.utils import create_trade
from zipline.finance.trading import TradingEnvironment
from zipline.data.loader import (
get_datafile,
dump_benchmarks,
dump_treasury_curves
)
from zipline.finance.trading import SimulationParameters, TradingEnvironment
import zipline.finance.trading as trading
def load_market_data():
try:
fp_bm = get_datafile('benchmark.msgpack', "rb")
except IOError:
print """
data msgpacks aren't distribute with source.
Fetching data from Yahoo Finance.
""".strip()
dump_benchmarks()
fp_bm = get_datafile('benchmark.msgpack', "rb")
bm_list = msgpack.loads(fp_bm.read())
bm_returns = []
for packed_date, returns in bm_list:
event_dt = tuple_to_date(packed_date)
#event_dt = event_dt.replace(
# hour=0,
# minute=0,
# second=0,
# tzinfo=pytz.utc
#)
daily_return = risk.DailyReturn(date=event_dt, returns=returns)
bm_returns.append(daily_return)
fp_bm.close()
bm_returns = sorted(bm_returns, key=attrgetter('date'))
try:
fp_tr = get_datafile('treasury_curves.msgpack', "rb")
except IOError:
print """
data msgpacks aren't distribute with source.
Fetching data from data.treasury.gov
""".strip()
dump_treasury_curves()
fp_tr = get_datafile('treasury_curves.msgpack', "rb")
tr_list = msgpack.loads(fp_tr.read())
tr_curves = {}
for packed_date, curve in tr_list:
tr_dt = tuple_to_date(packed_date)
#tr_dt = tr_dt.replace(hour=0, minute=0, second=0, tzinfo=pytz.utc)
tr_curves[tr_dt] = curve
fp_tr.close()
tr_curves = OrderedDict(sorted(
((dt, c) for dt, c in tr_curves.iteritems()),
key=lambda t: t[0]))
return bm_returns, tr_curves
def create_trading_environment(year=2006, start=None, end=None,
capital_base=float("1.0e5")):
def create_simulation_parameters(year=2006, start=None, end=None,
capital_base=float("1.0e5")
):
"""Construct a complete environment with reasonable defaults"""
benchmark_returns, treasury_curves = load_market_data()
trading.environment = TradingEnvironment()
if start is None:
start = datetime(year, 1, 1, tzinfo=pytz.utc)
if end is None:
end = datetime(year, 12, 31, tzinfo=pytz.utc)
trading_environment = TradingEnvironment(
benchmark_returns,
treasury_curves,
sim_params = SimulationParameters(
period_start=start,
period_end=end,
capital_base=capital_base
capital_base=capital_base,
)
return trading_environment
return sim_params
def create_random_trading_environment():
benchmark_returns, treasury_curves = load_market_data()
def create_random_simulation_parameters():
trading.environment = TradingEnvironment()
treasury_curves = trading.environment.treasury_curves
for n in range(100):
@@ -141,35 +78,33 @@ def create_random_trading_environment():
failed to find a suitable daterange after 100 attempts. please double
check treasury and benchmark data in findb, and re-run the test."""
trading_environment = TradingEnvironment(
benchmark_returns,
treasury_curves,
sim_params = SimulationParameters(
period_start=start_dt,
period_end=end_dt
)
return trading_environment, start_dt, end_dt
return sim_params, start_dt, end_dt
def get_next_trading_dt(current, interval, trading_calendar):
def get_next_trading_dt(current, interval):
next = current
while True:
next = next + interval
if trading_calendar.is_market_hours(next):
if trading.environment.is_market_hours(next):
break
return next
def create_trade_history(sid, prices, amounts, interval, trading_calendar,
def create_trade_history(sid, prices, amounts, interval, sim_params,
source_id="test_factory"):
trades = []
current = trading_calendar.first_open
current = sim_params.first_open
for price, amount in zip(prices, amounts):
trade = create_trade(sid, price, amount, current, source_id)
trades.append(trade)
current = get_next_trading_dt(current, interval, trading_calendar)
current = get_next_trading_dt(current, interval)
assert len(trades) == len(prices)
return trades
@@ -199,115 +134,115 @@ def create_txn(sid, price, amount, datetime):
return txn
def create_txn_history(sid, priceList, amtList, interval, trading_calendar):
def create_txn_history(sid, priceList, amtList, interval, sim_params):
txns = []
current = trading_calendar.first_open
current = sim_params.first_open
for price, amount in zip(priceList, amtList):
current = get_next_trading_dt(current, interval, trading_calendar)
current = get_next_trading_dt(current, interval)
txns.append(create_txn(sid, price, amount, current))
current = current + interval
return txns
def create_returns(daycount, trading_calendar):
def create_returns(daycount, sim_params):
"""
For the given number of calendar (not trading) days return all the trading
days between start and start + daycount.
"""
test_range = []
current = trading_calendar.first_open
current = sim_params.first_open
one_day = timedelta(days=1)
for day in range(daycount):
current = current + one_day
if trading_calendar.is_trading_day(current):
if trading.environment.is_trading_day(current):
r = risk.DailyReturn(current, random.random())
test_range.append(r)
return test_range
def create_returns_from_range(trading_calendar):
current = trading_calendar.first_open
end = trading_calendar.last_close
def create_returns_from_range(sim_params):
current = sim_params.first_open
end = sim_params.last_close
one_day = timedelta(days=1)
test_range = []
while current <= end:
r = risk.DailyReturn(current, random.random())
test_range.append(r)
current = get_next_trading_dt(current, one_day, trading_calendar)
current = get_next_trading_dt(current, one_day)
return test_range
def create_returns_from_list(returns, trading_calendar):
current = trading_calendar.first_open
def create_returns_from_list(returns, sim_params):
current = sim_params.first_open
one_day = timedelta(days=1)
test_range = []
#sometimes the range starts with a non-trading day.
if not trading_calendar.is_trading_day(current):
current = get_next_trading_dt(current, one_day, trading_calendar)
if not trading.environment.is_trading_day(current):
current = get_next_trading_dt(current, one_day)
for return_val in returns:
r = risk.DailyReturn(current, return_val)
test_range.append(r)
current = get_next_trading_dt(current, one_day, trading_calendar)
current = get_next_trading_dt(current, one_day)
return test_range
def create_daily_trade_source(sids, trade_count, trading_environment,
def create_daily_trade_source(sids, trade_count, sim_params,
concurrent=False):
"""
creates trade_count trades for each sid in sids list.
first trade will be on trading_environment.period_start, and daily
first trade will be on sim_params.period_start, and daily
thereafter for each sid. Thus, two sids should result in two trades per
day.
Important side-effect: trading_environment.period_end will be modified
Important side-effect: sim_params.period_end will be modified
to match the day of the final trade.
"""
return create_trade_source(
sids,
trade_count,
timedelta(days=1),
trading_environment,
sim_params,
concurrent=concurrent
)
def create_minutely_trade_source(sids, trade_count, trading_environment,
def create_minutely_trade_source(sids, trade_count, sim_params,
concurrent=False):
"""
creates trade_count trades for each sid in sids list.
first trade will be on trading_environment.period_start, and every minute
first trade will be on sim_params.period_start, and every minute
thereafter for each sid. Thus, two sids should result in two trades per
minute.
Important side-effect: trading_environment.period_end will be modified
Important side-effect: sim_params.period_end will be modified
to match the day of the final trade.
"""
return create_trade_source(
sids,
trade_count,
timedelta(minutes=1),
trading_environment,
sim_params,
concurrent=concurrent
)
def create_trade_source(sids, trade_count,
trade_time_increment, trading_environment,
trade_time_increment, sim_params,
concurrent=False):
args = tuple()
kwargs = {
'count': trade_count,
'sids': sids,
'start': trading_environment.first_open,
'start': sim_params.first_open,
'delta': trade_time_increment,
'filter': sids,
'concurrent': concurrent
@@ -316,19 +251,24 @@ def create_trade_source(sids, trade_count,
# TODO: do we need to set the trading environment's end to same dt as
# the last trade in the history?
#trading_environment.period_end = trade_history[-1].dt
#sim_params.period_end = trade_history[-1].dt
return source
def create_test_df_source(trading_calendar=None):
start = trading_calendar.first_open \
if trading_calendar else pd.datetime(1990, 1, 3, 0, 0, 0, 0, pytz.utc)
def create_test_df_source(sim_params=None):
end = trading_calendar.last_close \
if trading_calendar else pd.datetime(1990, 1, 8, 0, 0, 0, 0, pytz.utc)
if sim_params:
index = sim_params.trading_days
else:
start = pd.datetime(1990, 1, 3, 0, 0, 0, 0, pytz.utc)
end = pd.datetime(1990, 1, 8, 0, 0, 0, 0, pytz.utc)
index = pd.DatetimeIndex(
start=start,
end=end,
freq=pd.datetools.BDay()
)
index = pd.DatetimeIndex(start=start, end=end, freq=pd.datetools.BDay())
x = np.arange(0, len(index))
df = pd.DataFrame(x, index=index, columns=[0])
@@ -336,12 +276,12 @@ def create_test_df_source(trading_calendar=None):
return DataFrameSource(df), df
def create_test_panel_source(trading_calendar=None):
start = trading_calendar.first_open \
if trading_calendar else pd.datetime(1990, 1, 3, 0, 0, 0, 0, pytz.utc)
def create_test_panel_source(sim_params=None):
start = sim_params.first_open \
if sim_params else pd.datetime(1990, 1, 3, 0, 0, 0, 0, pytz.utc)
end = trading_calendar.last_close \
if trading_calendar else pd.datetime(1990, 1, 8, 0, 0, 0, 0, pytz.utc)
end = sim_params.last_close \
if sim_params else pd.datetime(1990, 1, 8, 0, 0, 0, 0, pytz.utc)
index = pd.DatetimeIndex(start=start, end=end, freq=pd.datetools.day)
price = np.arange(0, len(index))
zipline/utils/simfactory.py
+4 -13
View File
@@ -7,8 +7,6 @@ def create_test_zipline(**config):
"""
:param config: A configuration object that is a dict with:
- environment - a \
:py:class:`zipline.finance.trading.TradingEnvironment`
- sid - an integer, which will be used as the security ID.
- order_count - the number of orders the test algo will place,
defaults to 100
@@ -36,14 +34,6 @@ def create_test_zipline(**config):
concurrent_trades = config.get('concurrent_trades', False)
#--------------------
# Trading Environment
#--------------------
if 'environment' in config:
trading_environment = config['environment']
else:
trading_environment = factory.create_trading_environment()
if 'order_count' in config:
order_count = config['order_count']
else:
@@ -70,7 +60,8 @@ def create_test_zipline(**config):
test_algo = TestAlgorithm(
sid,
order_amount,
order_count
order_count,
sim_params=factory.create_simulation_parameters()
)
#-------------------
@@ -82,7 +73,7 @@ def create_test_zipline(**config):
trade_source = factory.create_daily_trade_source(
sid_list,
trade_count,
trading_environment,
test_algo.sim_params,
concurrent=concurrent_trades
)
@@ -105,6 +96,6 @@ def create_test_zipline(**config):
# ------------------
# generator/simulator
sim = test_algo.get_generator(trading_environment)
sim = test_algo.get_generator()
return sim
zipline/utils/test_utils.py
-3
View File
@@ -26,9 +26,6 @@ def check_dict(test, a, b, label):
test.assertTrue(isinstance(a, dict))
test.assertTrue(isinstance(b, dict))
for key in a.keys():
# ignore the extra fields used by dictshield
if key in ['progress']:
continue
test.assertTrue(key in a, "missing key at: " + label + "." + key)
test.assertTrue(key in b, "missing key at: " + label + "." + key)
zipline/utils/tradingcalendar.py
+1 -1
View File
@@ -61,7 +61,7 @@ def get_non_trading_days(start, end):
bymonth=1,
byweekday=(rrule.MO(+3)),
cache=True,
dtstart=start,
dtstart=datetime(1998, 1, 1, tzinfo=pytz.utc),
until=end
)
non_trading_rules.append(mlk_day)