diff --git a/tests/test_algorithm_gen.py b/tests/test_algorithm_gen.py index 6de78158..a2a395c0 100644 --- a/tests/test_algorithm_gen.py +++ b/tests/test_algorithm_gen.py @@ -18,8 +18,9 @@ from unittest import TestCase from nose.tools import timed from datetime import datetime -import pytz +import pytz +import zipline.finance.trading as trading from zipline.algorithm import TradingAlgorithm from zipline.finance import slippage from zipline.utils import factory @@ -78,6 +79,35 @@ class AlgorithmGeneratorTestCase(TestCase): def tearDown(self): teardown_logger(self) + def test_lse_algorithm(self): + + lse = trading.TradingEnvironment( + bm_symbol='^FTSE', + exchange_tz='Europe/London' + ) + + with lse: + + sim_params = factory.create_simulation_parameters( + start=datetime(2012, 5, 1, tzinfo=pytz.utc), + end=datetime(2012, 6, 30, tzinfo=pytz.utc) + ) + algo = TestAlgo(self, sim_params=sim_params) + trade_source = factory.create_daily_trade_source( + [8229], + 200, + sim_params + ) + algo.set_sources([trade_source]) + + gen = algo.get_generator() + results = list(gen) + self.assertEqual(len(results), 42) + # May 7, 2012 was an LSE holiday, confirm the 4th trading + # day was May 8. + self.assertEqual(results[4]['daily_perf']['period_open'], + datetime(2012, 5, 8, 8, 30, tzinfo=pytz.utc)) + @timed(DEFAULT_TIMEOUT) def test_generator_dates(self): """ diff --git a/tests/test_risk_compare_batch_iterative.py b/tests/test_risk_compare_batch_iterative.py index d98fbe94..4d5a7828 100644 --- a/tests/test_risk_compare_batch_iterative.py +++ b/tests/test_risk_compare_batch_iterative.py @@ -22,7 +22,6 @@ import numpy as np import zipline.finance.risk as risk -from zipline.finance.trading import TradingEnvironment import zipline.finance.trading as trading from test_risk import RETURNS @@ -44,8 +43,6 @@ class RiskCompareIterativeToBatch(unittest.TestCase): self.end_date = datetime.datetime( year=2006, month=12, day=31, tzinfo=pytz.utc) - # setup the default trading environment - trading.environment = TradingEnvironment() self.oneday = datetime.timedelta(days=1) def test_risk_metrics_returns(self): @@ -55,7 +52,7 @@ class RiskCompareIterativeToBatch(unittest.TestCase): cur_returns = [] for i, ret in enumerate(RETURNS): - todays_return_obj = risk.DailyReturn( + todays_return_obj = trading.DailyReturn( todays_date, ret ) diff --git a/tests/test_tradingcalendar.py b/tests/test_tradingcalendar.py index 6c29c077..a5416413 100644 --- a/tests/test_tradingcalendar.py +++ b/tests/test_tradingcalendar.py @@ -15,13 +15,20 @@ from unittest import TestCase from zipline.utils import tradingcalendar +from zipline.utils import tradingcalendar_lse import pytz import datetime from zipline.finance.trading import TradingEnvironment +from pandas import DatetimeIndex +from delorean import Delorean class TestTradingCalendar(TestCase): + def setUp(self): + today = Delorean().truncate('day') + self.end = DatetimeIndex([today.datetime]) + def test_calendar_vs_environment(self): """ test_calendar_vs_environment checks whether the @@ -41,6 +48,44 @@ class TestTradingCalendar(TestCase): "{diff} should be empty".format(diff=diff) ) + diff2 = tradingcalendar.trading_days - env_days + # depending on the time of day, data for the current day + # may not be available from yahoo, so don't include end + # of the tradingcalendar + diff2 = diff2 - self.end + self.assertEqual( + len(diff2), + 0, + "{diff} should be empty".format(diff=diff2) + ) + + def test_lse_calendar_vs_environment(self): + env = TradingEnvironment( + bm_symbol='^FTSE', + exchange_tz='Europe/London' + ) + + env_start_index = \ + env.trading_days.searchsorted(tradingcalendar_lse.start) + env_days = env.trading_days[env_start_index:] + diff = env_days - tradingcalendar_lse.trading_days + self.assertEqual( + len(diff), + 0, + "{diff} should be empty".format(diff=diff) + ) + + diff2 = tradingcalendar_lse.trading_days - env_days + # depending on the time of day, data for the current day + # may not be available from yahoo, so don't include end + # of the tradingcalendar + diff2 = diff2 - self.end + self.assertEqual( + len(diff2), + 0, + "{diff} should be empty".format(diff=diff2) + ) + def test_newyears(self): """ Check whether tradingcalendar contains certain dates. diff --git a/zipline/data/benchmarks.py b/zipline/data/benchmarks.py index 919b0f92..8bdaa20a 100644 --- a/zipline/data/benchmarks.py +++ b/zipline/data/benchmarks.py @@ -30,7 +30,6 @@ from loader_utils import ( from loader_utils import Mapping -from zipline.finance.risk import DailyReturn _BENCHMARK_MAPPING = { # Need to add 'symbol' @@ -95,6 +94,7 @@ def get_benchmark_data(symbol): def get_benchmark_returns(symbol): + from zipline.finance.trading import DailyReturn benchmark_returns = [] diff --git a/zipline/data/loader.py b/zipline/data/loader.py index 4ebe4a23..21586da2 100644 --- a/zipline/data/loader.py +++ b/zipline/data/loader.py @@ -24,7 +24,6 @@ from treasuries import get_treasury_data from benchmarks import get_benchmark_returns from zipline.utils.date_utils import tuple_to_date -import zipline.finance.risk as risk from operator import attrgetter @@ -92,6 +91,8 @@ def get_benchmark_filename(symbol): def load_market_data(bm_symbol='^GSPC'): + from zipline.finance.trading import DailyReturn + try: fp_bm = get_datafile(get_benchmark_filename(bm_symbol), "rb") except IOError: @@ -107,7 +108,7 @@ Fetching data from Yahoo Finance. for packed_date, returns in bm_list: event_dt = tuple_to_date(packed_date) - daily_return = risk.DailyReturn(date=event_dt, returns=returns) + daily_return = DailyReturn(date=event_dt, returns=returns) bm_returns.append(daily_return) fp_bm.close() diff --git a/zipline/finance/performance.py b/zipline/finance/performance.py index 10306b68..5c84e8af 100644 --- a/zipline/finance/performance.py +++ b/zipline/finance/performance.py @@ -278,7 +278,10 @@ class PerformanceTracker(object): def handle_market_close(self): # add the return results from today to the list of DailyReturn objects. todays_date = self.market_close.replace(hour=0, minute=0, second=0) - todays_return_obj = risk.DailyReturn( + self.cumulative_performance.update_dividends(todays_date) + self.todays_performance.update_dividends(todays_date) + + todays_return_obj = trading.DailyReturn( todays_date, self.todays_performance.returns ) diff --git a/zipline/finance/risk.py b/zipline/finance/risk.py index 1a6a46e0..b8fb2806 100644 --- a/zipline/finance/risk.py +++ b/zipline/finance/risk.py @@ -94,24 +94,6 @@ def advance_by_months(dt, jump_in_months): return dt.replace(year=dt.year + years, month=month) -class DailyReturn(object): - - def __init__(self, date, returns): - - assert isinstance(date, datetime.datetime) - self.date = date.replace(hour=0, minute=0, second=0) - self.returns = returns - - def to_dict(self): - return { - 'dt': self.date, - 'returns': self.returns - } - - def __repr__(self): - return str(self.date) + " - " + str(self.returns) - - class RiskMetricsBase(object): def __init__(self, start_date, end_date, returns): diff --git a/zipline/finance/trading.py b/zipline/finance/trading.py index 7f1b92af..d01adca1 100644 --- a/zipline/finance/trading.py +++ b/zipline/finance/trading.py @@ -32,6 +32,43 @@ from zipline.finance.commission import PerShare log = logbook.Logger('Transaction Simulator') + +# The financial simulations in zipline depend on information +# about the benchmark index and the risk free rates of return. +# The benchmark index defines the benchmark returns used in +# the calculation of performance metrics such as alpha/beta. Many +# components, including risk, performance, transforms, and +# batch_transforms, need access to a calendar of trading days and +# market hours. The TradingEnvironment maintains two time keeping +# facilities: +# - a DatetimeIndex of trading days for calendar calculations +# - a timezone name, which should be local to the exchange +# hosting the benchmark index. All dates are normalized to UTC +# for serialization and storage, and the timezone is used to +# ensure proper rollover through daylight savings and so on. +# +# This module maintains a global variable, environment, which is +# subsequently referenced directly by zipline financial +# components. To set the environment, you can set the property on +# the module directly: +# import zipline.finance.trading as trading +# trading.environment = TradingEnvironment() +# +# or if you want to switch the environment for a limited context +# you can use a TradingEnvironment in a with clause: +# lse = TradingEnvironment(bm_index="^FTSE", exchange_tz="Europe/London") +# with lse: +# # the code here will have lse as the global trading.environment +# algo.run(start, end) +# +# User code will not normally need to use TradingEnvironment +# directly. If you are extending zipline's core financial +# compponents and need to use the environment, you must import the module +# NOT the variable. If you import the module, you will get a +# reference to the environment at import time, which will prevent +# your code from responding to user code that changes the global +# state. + environment = None @@ -69,7 +106,7 @@ class TradingEnvironment(object): bm_symbol='^GSPC', exchange_tz="US/Eastern" ): - + self.prev_environment = self self.trading_day_map = OrderedDict() self.bm_symbol = bm_symbol if not load: @@ -90,6 +127,21 @@ class TradingEnvironment(object): self.first_trading_day = next(self.trading_day_map.iterkeys()) self.last_trading_day = next(reversed(self.trading_day_map)) + def __enter__(self, *args, **kwargs): + global environment + self.prev_environment = environment + environment = self + # return value here is associated with "as such_and_such" on the + # with clause. + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + global environment + environment = self.prev_environment + # signal that any exceptions need to be propagated up the + # stack. + return False + def normalize_date(self, test_date): return datetime.datetime( year=test_date.year, @@ -165,7 +217,7 @@ Last successful date: %s" % self.market_open) ) # create a new Delorean with the next_open naive date and # the correct timezone for the exchange. - open_delorean = Delorean(next_open, "US/Eastern") + open_delorean = Delorean(next_open, self.exchange_tz) open_utc = open_delorean.shift("UTC").datetime market_open = open_utc @@ -202,10 +254,9 @@ class SimulationParameters(object): def __init__(self, period_start, period_end, capital_base=10e3): - # raise and exception if the global environment is not - # set. global environment if not environment: + # This is the global environment for trading simulation. environment = TradingEnvironment() self.period_start = period_start @@ -287,3 +338,21 @@ class SimulationParameters(object): {'first_open': self.first_open, 'last_close': self.last_close }) + + +class DailyReturn(object): + + def __init__(self, date, returns): + + assert isinstance(date, datetime.datetime) + self.date = date.replace(hour=0, minute=0, second=0) + self.returns = returns + + def to_dict(self): + return { + 'dt': self.date, + 'returns': self.returns + } + + def __repr__(self): + return str(self.date) + " - " + str(self.returns) diff --git a/zipline/utils/factory.py b/zipline/utils/factory.py index 066ba414..34c8b064 100644 --- a/zipline/utils/factory.py +++ b/zipline/utils/factory.py @@ -26,13 +26,12 @@ from pandas.io.data import DataReader import numpy as np from datetime import datetime, timedelta -import zipline.finance.risk as risk from zipline.protocol import Event, DATASOURCE_TYPE from zipline.sources import (SpecificEquityTrades, DataFrameSource, DataPanelSource) from zipline.gens.utils import create_trade -from zipline.finance.trading import SimulationParameters, TradingEnvironment +from zipline.finance.trading import SimulationParameters import zipline.finance.trading as trading @@ -40,7 +39,6 @@ def create_simulation_parameters(year=2006, start=None, end=None, capital_base=float("1.0e5") ): """Construct a complete environment with reasonable defaults""" - trading.environment = TradingEnvironment() if start is None: start = datetime(year, 1, 1, tzinfo=pytz.utc) if end is None: @@ -56,34 +54,33 @@ def create_simulation_parameters(year=2006, start=None, end=None, def create_random_simulation_parameters(): - trading.environment = TradingEnvironment() - treasury_curves = trading.environment.treasury_curves + treasury_curves = trading.environment.treasury_curves - for n in range(100): + for n in range(100): - random_index = random.randint( - 0, - len(treasury_curves) - ) + random_index = random.randint( + 0, + len(treasury_curves) + ) - start_dt = treasury_curves.keys()[random_index] - end_dt = start_dt + timedelta(days=365) + start_dt = treasury_curves.keys()[random_index] + end_dt = start_dt + timedelta(days=365) - now = datetime.utcnow().replace(tzinfo=pytz.utc) + now = datetime.utcnow().replace(tzinfo=pytz.utc) - if end_dt <= now: - break + if end_dt <= now: + break - assert end_dt <= now, """ + assert end_dt <= now, """ failed to find a suitable daterange after 100 attempts. please double check treasury and benchmark data in findb, and re-run the test.""" - sim_params = SimulationParameters( - period_start=start_dt, - period_end=end_dt - ) + sim_params = SimulationParameters( + period_start=start_dt, + period_end=end_dt + ) - return sim_params, start_dt, end_dt + return sim_params, start_dt, end_dt def get_next_trading_dt(current, interval): @@ -158,7 +155,7 @@ def create_returns(daycount, sim_params): for day in range(daycount): current = current + one_day if trading.environment.is_trading_day(current): - r = risk.DailyReturn(current, random.random()) + r = trading.DailyReturn(current, random.random()) test_range.append(r) return test_range @@ -170,7 +167,7 @@ def create_returns_from_range(sim_params): one_day = timedelta(days=1) test_range = [] while current <= end: - r = risk.DailyReturn(current, random.random()) + r = trading.DailyReturn(current, random.random()) test_range.append(r) current = get_next_trading_dt(current, one_day) @@ -187,7 +184,7 @@ def create_returns_from_list(returns, sim_params): current = get_next_trading_dt(current, one_day) for return_val in returns: - r = risk.DailyReturn(current, return_val) + r = trading.DailyReturn(current, return_val) test_range.append(r) current = get_next_trading_dt(current, one_day) diff --git a/zipline/utils/tradingcalendar.py b/zipline/utils/tradingcalendar.py index 08d63fda..36d35398 100644 --- a/zipline/utils/tradingcalendar.py +++ b/zipline/utils/tradingcalendar.py @@ -227,6 +227,8 @@ def get_non_trading_days(start, end): # http://www.nyse.com/pdfs/closings.pdf # # National Days of Mourning + # - President Richard Nixon + non_trading_days.append(datetime(1994, 4, 27, tzinfo=pytz.utc)) # - President Ronald W. Reagan - June 11, 2004 non_trading_days.append(datetime(2004, 6, 11, tzinfo=pytz.utc)) # - President Gerald R. Ford - Jan 2, 2007 diff --git a/zipline/utils/tradingcalendar_lse.py b/zipline/utils/tradingcalendar_lse.py new file mode 100644 index 00000000..60f2d72b --- /dev/null +++ b/zipline/utils/tradingcalendar_lse.py @@ -0,0 +1,187 @@ +# +# Copyright 2012 Quantopian, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# References: +# http://www.londonstockexchange.com +# /about-the-exchange/company-overview/business-days/business-days.htm +# http://en.wikipedia.org/wiki/Bank_holiday +# http://www.adviceguide.org.uk/england/work_e/work_time_off_work_e/ +# bank_and_public_holidays.htm + +import pytz + +import pandas as pd + +from datetime import datetime +from dateutil import rrule +from zipline.utils.date_utils import utcnow + +start = datetime(2002, 1, 1, tzinfo=pytz.utc) +end = utcnow() + +non_trading_rules = [] +# Weekends +weekends = rrule.rrule( + rrule.YEARLY, + byweekday=(rrule.SA, rrule.SU), + cache=True, + dtstart=start, + until=end +) +non_trading_rules.append(weekends) +# New Year's Day +new_year = rrule.rrule( + rrule.MONTHLY, + byyearday=1, + cache=True, + dtstart=start, + until=end +) +# If new years day is on Saturday then Monday 3rd is a holiday +# If new years day is on Sunday then Monday 2nd is a holiday +weekend_new_year = rrule.rrule( + rrule.MONTHLY, + bymonth=1, + bymonthday=[2, 3], + byweekday=(rrule.MO), + cache=True, + dtstart=start, + until=end +) +non_trading_rules.append(new_year) +non_trading_rules.append(weekend_new_year) +# Good Friday +good_friday = rrule.rrule( + rrule.DAILY, + byeaster=-2, + cache=True, + dtstart=start, + until=end +) +non_trading_rules.append(good_friday) +# Easter Monday +easter_monday = rrule.rrule( + rrule.DAILY, + byeaster=1, + cache=True, + dtstart=start, + until=end +) +non_trading_rules.append(easter_monday) +# Early May Bank Holiday (1st Monday in May) +may_bank = rrule.rrule( + rrule.MONTHLY, + bymonth=5, + byweekday=(rrule.MO(1)), + cache=True, + dtstart=start, + until=end +) +non_trading_rules.append(may_bank) +# Spring Bank Holiday (Last Monday in May) +spring_bank = rrule.rrule( + rrule.MONTHLY, + bymonth=5, + byweekday=(rrule.MO(-1)), + cache=True, + dtstart=datetime(2003, 1, 1, tzinfo=pytz.utc), + until=end +) +non_trading_rules.append(spring_bank) +# Summer Bank Holiday (Last Monday in August) +summer_bank = rrule.rrule( + rrule.MONTHLY, + bymonth=8, + byweekday=(rrule.MO(-1)), + cache=True, + dtstart=start, + until=end +) +non_trading_rules.append(summer_bank) +# Christmas Day +christmas = rrule.rrule( + rrule.MONTHLY, + bymonth=12, + bymonthday=25, + cache=True, + dtstart=start, + until=end +) +# If christmas day is Saturday Monday 27th is a holiday +# If christmas day is sunday the Tuesday 27th is a holiday +weekend_christmas = rrule.rrule( + rrule.MONTHLY, + bymonth=12, + bymonthday=27, + byweekday=(rrule.MO, rrule.TU), + cache=True, + dtstart=start, + until=end +) + +non_trading_rules.append(christmas) +non_trading_rules.append(weekend_christmas) +# Boxing Day +boxing_day = rrule.rrule( + rrule.MONTHLY, + bymonth=12, + bymonthday=26, + cache=True, + dtstart=start, + until=end +) +# If boxing day is saturday then Monday 28th is a holiday +# If boxing day is sunday then Tuesday 28th is a holiday +weekend_boxing_day = rrule.rrule( + rrule.MONTHLY, + bymonth=12, + bymonthday=28, + byweekday=(rrule.MO, rrule.TU), + cache=True, + dtstart=start, + until=end +) + +non_trading_rules.append(boxing_day) +non_trading_rules.append(weekend_boxing_day) + +non_trading_ruleset = rrule.rruleset() + +# In 2002 May bank holiday was moved to 4th June to follow the Queens +# Golden Jubilee +non_trading_ruleset.exdate(datetime(2002, 9, 27, tzinfo=pytz.utc)) +non_trading_ruleset.rdate(datetime(2002, 6, 3, tzinfo=pytz.utc)) +non_trading_ruleset.rdate(datetime(2002, 6, 4, tzinfo=pytz.utc)) +# TODO: not sure why Feb 18 2008 is not available in the yahoo data +non_trading_ruleset.rdate(datetime(2008, 2, 18, tzinfo=pytz.utc)) +# In 2011 The Friday before Mayday was the Royal Wedding +non_trading_ruleset.rdate(datetime(2011, 4, 29, tzinfo=pytz.utc)) +# In 2012 May bank holiday was moved to 4th June to preceed the Queens +# Diamond Jubilee +non_trading_ruleset.exdate(datetime(2012, 5, 28, tzinfo=pytz.utc)) +non_trading_ruleset.rdate(datetime(2012, 6, 4, tzinfo=pytz.utc)) +non_trading_ruleset.rdate(datetime(2012, 6, 5, tzinfo=pytz.utc)) + +for rule in non_trading_rules: + non_trading_ruleset.rrule(rule) + +non_trading_days = non_trading_ruleset.between(start, end, inc=True) +non_trading_day_index = pd.DatetimeIndex(sorted(non_trading_days)) + +business_days = pd.DatetimeIndex(start=start, end=end, + freq=pd.datetools.BDay()) + +trading_days = business_days - non_trading_day_index