diff --git a/tests/test_sorting.py b/tests/test_sorting.py index 966dec3f..bec97e31 100644 --- a/tests/test_sorting.py +++ b/tests/test_sorting.py @@ -249,9 +249,11 @@ def compare_by_dt_source_id(x,y): return -1 elif x.source_id > y.source_id: return 1 - else: return 0 #Alias for ease of use comp = compare_by_dt_source_id + +def to_dt(msg): + return ndict({'dt': msg}) diff --git a/tests/test_transforms.py b/tests/test_transforms.py index 1fe1ce3c..b9420633 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -5,9 +5,12 @@ from unittest2 import TestCase from zipline.utils.test_utils import setup_logger, teardown_logger import zipline.utils.factory as factory -from zipline.finance.vwap import DailyVWAP, VWAPTransform + +from zipline.gens.tradegens import SpecificEquityTrades +from zipline.gens.transform import StatefulTransform +from zipline.gens.vwap import VWAP +from zipline.gens.mavg import MovingAverage from zipline.finance.returns import ReturnsFromPriorClose -from zipline.finance.movingaverage import MovingAverage from zipline.lines import SimulatedTrading from zipline.core.devsimulator import AddressAllocator @@ -25,7 +28,7 @@ class ZiplineWithTransformsTestCase(TestCase): 'sid' : 133, 'devel' : True } - setup_logger(self, '/var/log/qexec/qexed.log') + setup_logger(self, '/var/log/qexec/qexec.log') def tearDown(self): teardown_logger(self) @@ -48,25 +51,34 @@ class FinanceTransformsTestCase(TestCase): self.trading_environment = factory.create_trading_environment() setup_logger(self, '/var/log/qexec/qexec.log') - def tearDown(self): - self.log_handler.pop_application() - - def test_vwap(self): - trade_history = factory.create_trade_history( 133, - [10.0, 10.0, 10.0, 11.0], + [10.0, 10.0, 11.0, 11.0], [100, 100, 100, 300], timedelta(days=1), self.trading_environment ) + self.source = SpecificEquityTrades(event_list=trade_history) - vwap = DailyVWAP(days=2) - for trade in trade_history: - vwap.update(trade) + def tearDown(self): + self.log_handler.pop_application() - self.assertEqual(vwap.vwap, 10.75) + def test_vwap(self): + vwap = StatefulTransform(VWAP, timedelta(days = 2)) + transformed = list(vwap.transform(self.source)) + # Output values + tnfm_vals = [message.tnfm_value for message in transformed] + # "Hand calculated" values. + expected = [(10.0 * 100) / 100.0, + ((10.0 * 100) + (10.0 * 100)) / (200.0), + ((10.0 * 100) + (10.0 * 100) + (11.0 * 100)) / (300.0), + # First event should get droppped here. + ((10.0 * 100) + (11.0 * 100) + (11.0 * 300)) / (500.0)] + + # Output should match the expected. + assert tnfm_vals == expected + def test_returns(self): trade_history = factory.create_trade_history( @@ -86,17 +98,29 @@ class FinanceTransformsTestCase(TestCase): def test_moving_average(self): - trade_history = factory.create_trade_history( - 133, - [10.0, 10.0, 10.0, 11.0], - [100, 100, 100, 300], - timedelta(days=1), - self.trading_environment - ) - - ma = MovingAverage(days=2) - for trade in trade_history: - ma.update(trade) - - - self.assertEqual(ma.average, 10.5) + + mavg = StatefulTransform( + MovingAverage, + timedelta(days = 2), + ['price', 'volume'] + ) + + transformed = list(mavg.transform(self.source)) + # Output values. + tnfm_prices = [message.tnfm_value.price for message in transformed] + tnfm_volumes = [message.tnfm_value.volume for message in transformed] + # "Hand-calculated" values + expected_prices = [((10.0) / 1.0), + ((10.0 + 10.0) / 2.0), + ((10.0 + 10.0 + 11.0) / 3.0), + # First event should get dropped here. + ((10.0 + 11.0 + 11.0) / 3.0)] + expected_volumes = [((100.0) / 1.0), + ((100.0 + 100.0) / 2.0), + ((100.0 + 100.0 + 100.0) / 3.0), + # First event should get dropped here. + ((100.0 + 100.0 + 300.0) / 3.0)] + + assert tnfm_prices == expected_prices + assert tnfm_volumes == expected_volumes + diff --git a/zipline/finance/returns.py b/zipline/finance/returns.py index 5585f325..6e390364 100644 --- a/zipline/finance/returns.py +++ b/zipline/finance/returns.py @@ -1,26 +1,24 @@ from collections import defaultdict from zipline.transforms.base import BaseTransform -class ReturnsTransform(BaseTransform): - - def init(self, name): - self.state = {} - self.state['name'] = name - self.by_sid = defaultdict(self._create) - - @property - def get_id(self): - return self.state['name'] - - - def transform(self, event): - cur = self.by_sid[event.sid] - cur.update(event) - self.state['value'] = cur.returns - return self.state +class Returns(object): + """ + Class that maintains a dictionary from sids to the event + representing the most recent closing price. + """ + def __init__(self, days == 1): + self.days = days + self.mapping = defaultdict(self._create) + + def update(self, event): + """ + Update and return the calculated returns for this event's sid. + """ + sid_returns = self.mapping[event.sid].update(event) + return sid_returns def _create(self): - return ReturnsFromPriorClose() + return ReturnsFromPriorClose(days) class ReturnsFromPriorClose(object): """ diff --git a/zipline/gens/tradegens.py b/zipline/gens/tradegens.py index 7b86da05..9a0f7406 100644 --- a/zipline/gens/tradegens.py +++ b/zipline/gens/tradegens.py @@ -3,13 +3,15 @@ Tools to generate trade events without a backing store. Useful for testing and zipline development """ import random +import pytz + from itertools import chain, cycle, ifilter, izip from datetime import datetime, timedelta from zipline.utils.factory import create_trade from zipline.gens.utils import hash_args -def date_gen(start = datetime(2006, 6, 6, 12), +def date_gen(start = datetime(2006, 6, 6, 12, tzinfo=pytz.utc), delta = timedelta(minutes = 1), count = 100): """ @@ -71,7 +73,7 @@ class SpecificEquityTrades(object): # Unpack config dictionary with default values. self.count = kwargs.get('count', 500) self.sids = kwargs.get('sids', [1, 2]) - self.start = kwargs.get('start', datetime(2012, 6, 6, 0)) + self.start = kwargs.get('start', datetime(2008, 6, 6, 15, tzinfo = pytz.utc)) self.delta = kwargs.get('delta', timedelta(minutes = 1)) # Default to None for event_list and filter. diff --git a/zipline/gens/transform.py b/zipline/gens/transform.py index a191e50b..202acc0f 100644 --- a/zipline/gens/transform.py +++ b/zipline/gens/transform.py @@ -10,6 +10,7 @@ from numbers import Number from abc import ABCMeta, abstractmethod from zipline import ndict +from zipline.utils.tradingcalendar import trading_days_between from zipline.gens.utils import assert_sort_unframe_protocol, \ assert_transform_protocol, hash_args diff --git a/zipline/gens/vwap.py b/zipline/gens/vwap.py index 4e8a2f8b..029284c1 100644 --- a/zipline/gens/vwap.py +++ b/zipline/gens/vwap.py @@ -59,7 +59,7 @@ class VWAPEventWindow(EventWindow): Return the calculated vwap for this sid. """ # By convention, vwap is None if we have no events. - if len(self.ticks) == 0 + if len(self.ticks) == 0: return None else: return (self.flux / self.totalvolume) @@ -68,15 +68,3 @@ class VWAPEventWindow(EventWindow): def assert_required_fields(self, event): assert isinstance(event.price, Number) assert isinstance(event.volume, Number) - -if __name__ == "__main__": - - from zipline.gens.tradegens import SpecificEquityTrades - from zipline.gens.transform import StatefulTransform - - source = SpecificEquityTrades() - vwap = StatefulTransform(VWAP, timedelta(minutes = 10)) - - out = vwap.transform(source) - - diff --git a/zipline/utils/date_utils.py b/zipline/utils/date_utils.py index a1fbfad1..2819d4e9 100644 --- a/zipline/utils/date_utils.py +++ b/zipline/utils/date_utils.py @@ -95,7 +95,7 @@ HOLIDAYS = { 'july_4th' : datetime(2008 , 7 , 4 ), 'labor_day' : datetime(2008 , 9 , 1 ), 'tgiving' : datetime(2008 , 11 , 27), - 'christmas' : datetime(2008 , 5 , 25), + 'christmas' : datetime(2008 , 12 , 25), } # Create a rule to recur every weekday starting today diff --git a/zipline/utils/tradingcalendar.py b/zipline/utils/tradingcalendar.py new file mode 100644 index 00000000..c7aa9152 --- /dev/null +++ b/zipline/utils/tradingcalendar.py @@ -0,0 +1,333 @@ +import pytz + +from datetime import datetime, timedelta +from dateutil import rrule +from zipline.utils.date_utils import utcnow + +def market_opens(start, end, inclusive=False): + """ + Returns all market opens between the start date and the end date. + Must use utc-stamped datetimes. + """ + return opens.between(start, end, inc=inclusive) + +def market_closes(start, end, inclusive=False): + """ + Returns all market closes between the start date and the end date. + Must use utc-stamped datetimes. + """ + return closes.between(start, end, inc=inclusive) + +def trading_days_between(start, end): + """ + Calculate the number of "complete" trading days between two + events. We define this as the number of market opens that + occurred between start and end, with the caveat that we subtract 1 + from this total if end falls on the same day as the last market + open and end occurs earlier in its own day than start. This + reflects the fact that we haven't completed a full day + corresponding to the last market open. + + Examples: + + 1.) + start = Tuesday, Aug 7, 2012, 1:00 pm + end = Wednesday, Aug 8, 2012, 1:30 pm + + There is one market open between these dates, on the morning of + Wednesday the 8th. This falls on the same calendar day as end, + but end is later in the day than start, so we count this as a full + day. The correct output is 1. + + 2.) + start = Tuesday, Aug 7, 2012, 1:30 pm + end = Wednesday, Aug 8, 2012, 1:00 pm + + There is one market open between these dayes, on the morning of + Wednesday the 8th. This falls on the same calendar day as end, + and end is earlier in the day than start, so we do not count this + day as completed. The correct output is 0. + + 3.) + start = Tuesday, Aug 7, 2012, 1:00 pm + end = Saturday, Aug 11, 2012, 1:30 pm + + There are 3 market opens between these dates, occurring on + Wednesday, Thursday, and Friday. The last open is not on + the same day as end, so we simply return 3 + + 4.) + start = Tuesday, Aug 7, 2012, 1:30 pm + end = Monday, Aug, 13, 2012, 1:00 pm + + There are 4 market opens between these dates, occurring on + Wednesday, Thursday, Friday, and the following Monday. The + last open occurs on the same calendar day as end, and end + is earlier in the day than start, so we do not count the + last market day as completed. The correct output is 3 days. + """ + # Calculate the number of opens between the events. + opens = (market_opens(start, end)) + days_between = len(opens) + if days_between == 0: + return days_between + + # If end falls on the same day as an open, subtract 1 from the + # total if end is earlier in its respective day than start. + last_open = opens[-1] + if last_open.date() == end.date() and earlier_in_day(end, start): + days_between -=1 + + return days_between + +def earlier_in_day(d1, d2): + """ + Return true if d1 falls earlier in its own day than d2. + """ + d1 = d1.replace(year = d2.year, day = d2.day) + return d1 < d2 + +WEEKDAYS = [rrule.MO, rrule.TU, rrule.WE, rrule.TH, rrule.FR] + +# Recurrence rule that generates all market opens since Jan 1, 1970. +# This does not exclude holidays. +market_opens_with_holidays = rrule.rrule( + rrule.DAILY, + byweekday=WEEKDAYS, + byhour = 14, + byminute = 30, + cache = True, + dtstart=datetime(1970, 1, 1, tzinfo = pytz.utc), +) + +# Recurrence rule that generates all market closes since Jan 1, 1970. +# This does not exclude holidays. +market_closes_with_holidays = rrule.rrule( + rrule.DAILY, + byweekday=WEEKDAYS, + byhour = 21, + byminute = 0, + cache = True, + dtstart=datetime(1970, 1, 1, tzinfo = pytz.utc), +) + +# Recurrence rules for excluding the market open/close on new years. +new_years_opens = rrule.rrule( + rrule.MONTHLY, + byyearday = 1, + byhour = 14, + byminute = 30, + cache = True, + dtstart = datetime(1970, 1,1,tzinfo = pytz.utc) +) +new_years_closes = rrule.rrule( + rrule.MONTHLY, + byyearday = 1, + byhour = 21, + byminute = 0, + cache = True, + dtstart = datetime(1970, 1,1,tzinfo = pytz.utc) +) + +# Recurrence rules for excluding MLK day. It is always the third +# monday in January. +mlk_opens = rrule.rrule( + rrule.MONTHLY, + bymonth = 1, + byweekday = (rrule.MO(3)), + byhour = 14, + byminute = 30, + cache = True, + dtstart = datetime(1970, 1,1,tzinfo = pytz.utc) +) +mlk_closes = rrule.rrule( + rrule.MONTHLY, + bymonth = 1, + byweekday = (rrule.MO(+3)), + byhour = 21, + byminute = 0, + cache = True, + dtstart = datetime(1970, 1,1,tzinfo = pytz.utc) +) + +# Recurrence rules for generating the market open/close for +# presidents' day. Presidents' day always occurs on the third monday +# of February. +presidents_day_opens = rrule.rrule( + rrule.MONTHLY, + bymonth = 2, + byweekday = (rrule.MO(3)), + byhour = 14, + byminute = 30, + cache = True, + dtstart = datetime(1970, 1,1,tzinfo = pytz.utc) +) +presidents_day_closes = rrule.rrule( + rrule.MONTHLY, + bymonth = 2, + byweekday = (rrule.MO(3)), + byhour = 21, + byminute = 0, + cache = True, + dtstart = datetime(1970, 1,1,tzinfo = pytz.utc) +) + +# Recurrence rules for generating the market open/close for good +# friday. Good friday always falls 2 days before easter, which +# thankfully is a built-in refernce in this module. +good_friday_opens = rrule.rrule( + rrule.DAILY, + byeaster = -2, + byhour = 14, + byminute = 30, + cache = True, + dtstart = datetime(1970, 1,1,tzinfo = pytz.utc) +) +good_friday_closes = rrule.rrule( + rrule.DAILY, + byeaster = -2, + byhour = 21, + byminute = 0, + cache = True, + dtstart = datetime(1970, 1,1,tzinfo = pytz.utc) +) + +# Recurrence rules for generating the market open/close for memorial +# day. Memorial day always occurs on the last monday of May. +memorial_day_opens = rrule.rrule( + rrule.MONTHLY, + bymonth = 5, + byweekday = (rrule.MO(-1)), + byhour = 14, + byminute = 30, + cache = True, + dtstart = datetime(1970, 1,1,tzinfo = pytz.utc) +) +memorial_day_closes = rrule.rrule( + rrule.MONTHLY, + bymonth = 5, + byweekday = (rrule.MO(-1)), + byhour = 21, + byminute = 0, + cache = True, + dtstart = datetime(1970, 1,1,tzinfo = pytz.utc) +) + +# Recurrence rules for generating the market open/close for July 4th. +july_4th_opens = rrule.rrule( + rrule.MONTHLY, + bymonth = 6, + bymonthday = 4, + byhour = 14, + byminute = 30, + cache = True, + dtstart = datetime(1970, 1,1,tzinfo = pytz.utc) +) +july_4th_closes = rrule.rrule( + rrule.MONTHLY, + bymonth = 6, + bymonthday = 4, + byhour = 21, + byminute = 0, + cache = True, + dtstart = datetime(1970, 1,1,tzinfo = pytz.utc) +) + +# Recurrence rule for generating the market open/close for labor day. +# Labor day is always the first monday of September. +labor_day_opens = rrule.rrule( + rrule.MONTHLY, + bymonth = 9, + byweekday = (rrule.MO(1)), + byhour = 14, + byminute = 30, + cache = True, + dtstart = datetime(1970, 1,1,tzinfo = pytz.utc) +) +labor_day_closes = rrule.rrule( + rrule.MONTHLY, + bymonth = 9, + byweekday = (rrule.MO(1)), + byhour = 21, + byminute = 0, + cache = True, + dtstart = datetime(1970, 1,1,tzinfo = pytz.utc) +) + +# Recurrence rule for generating the market open/close for +# thanksgiving. Thanksgiving always falls on the fourth thursday in +# November. (Who decides how these holidays work!?!) +thanksgiving_opens = rrule.rrule( + rrule.MONTHLY, + bymonth = 11, + byweekday = (rrule.TH(-1)), + byhour = 14, + byminute = 30, + cache = True, + dtstart = datetime(1970, 1,1,tzinfo = pytz.utc) +) +thanksgiving_closes = rrule.rrule( + rrule.MONTHLY, + bymonth = 11, + byweekday = (rrule.TH(-1)), + byhour = 21, + byminute = 0, + cache = True, + dtstart = datetime(1970, 1,1,tzinfo = pytz.utc) +) + +# Recurrence relation for generating the market open/close for +# christmas. Christmas always occurs on december 25th. + +christmas_opens = rrule.rrule( + rrule.MONTHLY, + bymonth = 12, + bymonthday = 25, + byhour = 14, + byminute = 30, + cache = True, + dtstart = datetime(1970, 1,1,tzinfo = pytz.utc) +) +christmas_closes = rrule.rrule( + rrule.MONTHLY, + bymonth = 12, + bymonthday = 25, + byhour = 21, + byminute = 0, + cache = True, + dtstart = datetime(1970, 1,1,tzinfo = pytz.utc) +) +# All NYSE observed holidays. +holiday_opens = [ + new_years_opens, + mlk_opens, + presidents_day_opens, + good_friday_opens, + memorial_day_opens, + july_4th_opens, + labor_day_opens, + thanksgiving_opens, + christmas_opens +] +holiday_closes = [ + new_years_closes, + mlk_closes, + presidents_day_closes, + good_friday_closes, + memorial_day_closes, + july_4th_closes, + labor_day_closes, + thanksgiving_closes, + christmas_closes +] + +# Valid market opens are given by all market opens minus holidays. +opens = rrule.rruleset() +opens.rrule(market_opens_with_holidays) +for holiday_rule in holiday_opens: + opens.exrule(holiday_rule) + +closes = rrule.rruleset() +closes.rrule(market_closes_with_holidays) +for holiday_rule in holiday_closes: + closes.exrule(holiday_rule)