From 30dfc86ba936440b6fe288bff801509e514d7de0 Mon Sep 17 00:00:00 2001
From: fawce <fawce@quantopian.com>
Date: Fri, 6 Apr 2012 20:49:56 -0400
Subject: [PATCH 1/6] fixed dates front to back to be proper market open/close,
 and to use start/end first_open/last_close from the TradingEnvironment.

---
 zipline/finance/performance.py     |  35 ++++++----
 zipline/finance/risk.py            |  14 ++--
 zipline/finance/trading.py         |  56 ++++++++++++++--
 zipline/protocol.py                |  47 +++++++++++---
 zipline/test/factory.py            |  60 ++++++++---------
 zipline/test/test_finance.py       |  52 +++++++++++++++
 zipline/test/test_perf_tracking.py |  49 ++++++--------
 zipline/test/test_protocol.py      |   1 -
 zipline/test/test_risk.py          | 101 +++++++++++++++++++----------
 9 files changed, 288 insertions(+), 127 deletions(-)

diff --git a/zipline/finance/performance.py b/zipline/finance/performance.py
index e3564c54..7b8211cd 100644
--- a/zipline/finance/performance.py
+++ b/zipline/finance/performance.py
@@ -118,6 +118,7 @@ Performance Period
 
 """
 import datetime
+import pytz
 import msgpack
 import pandas
 import math
@@ -146,11 +147,11 @@ class PerformanceTracker():
         self.trading_environment    = trading_environment
         self.trading_day            = datetime.timedelta(hours = 6, minutes = 30)
         self.calendar_day           = datetime.timedelta(hours = 24)
-        self.started_at             = datetime.datetime.utcnow()
+        self.started_at             = datetime.datetime.utcnow().replace(tzinfo=pytz.utc)
 
         self.period_start            = self.trading_environment.period_start
         self.period_end              = self.trading_environment.period_end
-        self.market_open             = self.period_start
+        self.market_open             = self.trading_environment.first_open
         self.market_close            = self.market_open + self.trading_day
         self.progress                = 0.0
         self.total_days              = self.trading_environment.days_in_period
@@ -266,6 +267,16 @@ class PerformanceTracker():
             returns=self.returns,
             trading_environment=self.trading_environment
         )
+        
+        # increment the day counter before we move markers forward.
+        self.day_count += 1.0
+        # calculate progress of test
+        self.progress = self.day_count / self.total_days
+
+        # Output results
+        if self.result_stream:
+            msg = zp.PERF_FRAME(self.to_dict())
+            self.result_stream.send(msg)
 
         #move the market day markers forward
         self.market_open = self.market_open + self.calendar_day
@@ -276,16 +287,7 @@ class PerformanceTracker():
             self.market_open = self.market_open + self.calendar_day
 
         self.market_close = self.market_open + self.trading_day
-        self.day_count += 1.0
-
-        #calculate progress of test
-        self.progress = self.day_count / self.total_days
-
-        # Output Results
-        if self.result_stream:
-            msg = zp.PERF_FRAME(self.to_dict())
-            self.result_stream.send(msg)
-
+        
         # Roll over positions to current day.
         self.todays_performance.calculate_performance()
         self.todays_performance = PerformancePeriod(
@@ -299,6 +301,15 @@ class PerformanceTracker():
         When the simulation is complete, run the full period risk report
         and send it out on the result_stream.
         """
+        
+        # the stream will end on the last trading day, but will not trigger
+        # an end of day, so we trigger the final market close here.
+        self.handle_market_close()
+        
+        log_msg = "Simulated {n} trading days out of {m}."
+        qutil.LOGGER.info(log_msg.format(n=self.day_count, m=self.total_days))
+        qutil.LOGGER.info("first open: {d}".format(d=self.trading_environment.first_open))
+        
         self.risk_report = risk.RiskReport(
             self.returns,
             self.trading_environment
diff --git a/zipline/finance/risk.py b/zipline/finance/risk.py
index 918ad083..7fa1d103 100644
--- a/zipline/finance/risk.py
+++ b/zipline/finance/risk.py
@@ -64,7 +64,9 @@ def advance_by_months(dt, jump_in_months):
 class DailyReturn():
 
     def __init__(self, date, returns):
-        self.date = date
+        
+        assert isinstance(date, datetime.datetime)
+        self.date = date.replace(hour=0, minute=0, second=0)
         self.returns = returns
 
     def to_dict(self):
@@ -304,9 +306,11 @@ class RiskMetrics():
             if rate != None:
                 return rate * (td.days + 1) / 365
 
-        message = "no rate for end date = {dt} and term = {term}. Using zero."
-        message = message.format(dt=self.end_date,term=self.treasury_duration)
-        raise Exception(message)
+            message = "no rate for end date = {dt} and term = {term}. Check \
+            that date doesn't exceed treasury history range."
+            message = message.format(dt=self.end_date,term=self.treasury_duration)
+            raise Exception(message)
+        
 
 
 class RiskReport():
@@ -326,7 +330,7 @@ class RiskReport():
         else:
             start_date = self.algorithm_returns[0].date
             end_date = self.algorithm_returns[-1].date
-
+            
         self.month_periods = self.periodsInRange(1, start_date, end_date)
         self.three_month_periods = self.periodsInRange(3, start_date, end_date)
         self.six_month_periods = self.periodsInRange(6, start_date, end_date)
diff --git a/zipline/finance/trading.py b/zipline/finance/trading.py
index 13d6d9ec..b60e7fe4 100644
--- a/zipline/finance/trading.py
+++ b/zipline/finance/trading.py
@@ -356,10 +356,10 @@ class TradingEnvironment(object):
         self, 
         benchmark_returns, 
         treasury_curves, 
-        period_start=None, 
-        period_end=None, 
-        capital_base=None,
-        max_drawdown=None
+        period_start    = None, 
+        period_end      = None, 
+        capital_base    = None,
+        max_drawdown    = None
     ):
     
         self.trading_days = []
@@ -372,10 +372,55 @@ class TradingEnvironment(object):
         self.capital_base = capital_base
         self.period_trading_days = None
         self.max_drawdown = max_drawdown
-            
+        
         for bm in benchmark_returns:
             self.trading_days.append(bm.date)
             self.trading_day_map[bm.date] = bm
+        
+        self.first_open = self.calculate_first_open()
+        self.last_close = self.calculate_last_close()
+
+    def calculate_first_open(self):
+        """
+        Finds the first trading day on or after self.period_start.
+        """
+        first_open  = self.period_start
+        one_day      = datetime.timedelta(days=1)
+        
+        while not self.is_trading_day(first_open):
+            first_open = first_open + one_day
+
+        first_open = self.set_NYSE_time(first_open, 9, 30)
+        return first_open
+        
+    def calculate_last_close(self):
+        """
+        Finds the last trading day on or before self.period_end
+        """
+        last_close  = self.period_end
+        one_day     = datetime.timedelta(days=1)
+        
+        while not self.is_trading_day(last_close):
+            last_close = last_close - one_day
+        
+        last_close = self.set_NYSE_time(last_close, 16, 00)
+        
+        return last_close
+
+    #TODO: add other exchanges and timezones...
+    def set_NYSE_time(self, dt, hour, minute):
+        naive = datetime.datetime(
+            year=dt.year,
+            month=dt.month,
+            day=dt.day
+        )
+        local = pytz.timezone ('US/Eastern')
+        local_dt = naive.replace (tzinfo = local)
+        # set the clock to the opening bell in NYC time.
+        local_dt = local_dt.replace(hour=hour, minute=minute)
+        # convert to UTC
+        utc_dt = local_dt.astimezone (pytz.utc)
+        return utc_dt
 
     def normalize_date(self, test_date):
         return datetime.datetime(
@@ -399,6 +444,7 @@ class TradingEnvironment(object):
                 if date >= self.period_start:
                     self.period_trading_days.append(date)
         
+        
         return len(self.period_trading_days)
                 
             
diff --git a/zipline/protocol.py b/zipline/protocol.py
index 5ea0e9c0..9667c7bd 100644
--- a/zipline/protocol.py
+++ b/zipline/protocol.py
@@ -634,6 +634,7 @@ def PERF_FRAME(perf):
 
     daily_perf = {
         'date'            : EPOCH(date),
+        'date_string'     : str(date),     
         'returns'         : tp['returns'],
         'pnl'             : tp['pnl'],
         'portfolio_value' : tp['ending_value']
@@ -675,8 +676,28 @@ def PERF_UNFRAME(msg):
 # Date Helpers
 # -----------------------
 
-def EPOCH(some_date):
-    seconds = time.mktime(some_date.timetuple())
+UNIX_EPOCH = datetime.datetime(1970, 1, 1, 0, 0, tzinfo = pytz.utc)
+def EPOCH(utc_datetime):
+    """
+    The key is to ensure all the dates you are using are in the utc timezone 
+    before you start converting. See http://pytz.sourceforge.net/ to learn how 
+    to do that properly. By normalizing to utc, you eliminate the ambiguity of 
+    daylight savings transitions. Then you can safely use timedelta to calculate 
+    distance from the unix epoch, and then convert to seconds or milliseconds.
+    
+    Note that the resulting unix timestamp is itself in the UTC timezone. If you 
+    wish to see the timestamp in a localized timezone, you will need to make 
+    another conversion.
+    
+    Also note that this will only work for dates after 1970.
+    """
+    assert isinstance(utc_datetime, datetime.datetime)
+    # utc only please
+    assert utc_datetime.tzinfo == pytz.utc
+    
+    # how long since the epoch?
+    delta = utc_datetime - UNIX_EPOCH
+    seconds = delta.total_seconds()
     ms = seconds * 1000
     return ms
     
@@ -694,10 +715,14 @@ def PACK_DATE(event):
     :rtype: None
     """
     assert isinstance(event.dt, datetime.datetime)
-    assert event.dt.tzinfo == pytz.utc #utc only please
-    year, month, day, hour, minute, second =  event.dt.timetuple()[0:6]
-    micros = event.dt.microsecond
-    event['dt'] = tuple([year, month, day, hour, minute, second, micros])
+    # utc only please
+    assert event.dt.tzinfo == pytz.utc 
+    event['dt'] = date_to_tuple(event['dt'])
+
+def date_to_tuple(dt):
+    year, month, day, hour, minute, second =  dt.timetuple()[0:6]
+    micros = dt.microsecond
+    return tuple([year, month, day, hour, minute, second, micros])
 
 def UNPACK_DATE(event):
     """
@@ -720,12 +745,14 @@ def UNPACK_DATE(event):
     assert len(event.dt) == 7
     for item in event.dt:
         assert isinstance(item, numbers.Integral)
-    year, month, day, hour, minute, second, micros = event.dt
+    event.dt = tuple_to_date(event.dt)
+    
+def tuple_to_date(date_tuple):
+    year, month, day, hour, minute, second, micros = date_tuple
     dt = datetime.datetime(year, month, day, hour, minute, second)
     dt = dt.replace(microsecond = micros, tzinfo = pytz.utc)
-    event.dt = dt
-
-
+    return dt
+    
 DATASOURCE_TYPE = Enum(
     'ORDER',
     'TRADE',
diff --git a/zipline/test/factory.py b/zipline/test/factory.py
index 406ae74b..4954cabd 100644
--- a/zipline/test/factory.py
+++ b/zipline/test/factory.py
@@ -14,40 +14,41 @@ from zipline.finance.trading import TradingEnvironment
 
 def load_market_data():
     fp_bm = open("./zipline/test/benchmark.msgpack", "rb")
-    bm_map = msgpack.loads(fp_bm.read())
+    bm_list = msgpack.loads(fp_bm.read())
     bm_returns = []
-    for epoch, returns in bm_map.iteritems():
-        event_dt = datetime.fromtimestamp(epoch)
-        event_dt = event_dt.replace(
-            hour=0, 
-            minute=0, 
-            second=0, 
-            tzinfo=pytz.utc
-        )
+    for packed_date, returns in bm_list:
+        event_dt = zp.tuple_to_date(packed_date)
+        #event_dt = event_dt.replace(
+        #    hour=0, 
+        #    minute=0, 
+        #    second=0, 
+        #    tzinfo=pytz.utc
+        #)
         
         daily_return = risk.DailyReturn(date=event_dt, returns=returns)
         bm_returns.append(daily_return)
     bm_returns = sorted(bm_returns, key=lambda(x): x.date) 
     fp_tr = open("./zipline/test/treasury_curves.msgpack", "rb")
-    tr_map = msgpack.loads(fp_tr.read())
+    tr_list = msgpack.loads(fp_tr.read())
     tr_curves = {}
-    for epoch, curve in tr_map.iteritems():
-        tr_dt = datetime.fromtimestamp(epoch)
-        tr_dt = tr_dt.replace(hour=0, minute=0, second=0, tzinfo=pytz.utc)
+    for packed_date, curve in tr_list:
+        tr_dt = zp.tuple_to_date(packed_date)
+        #tr_dt = tr_dt.replace(hour=0, minute=0, second=0, tzinfo=pytz.utc)
         tr_curves[tr_dt] = curve
-        
+    
     return bm_returns, tr_curves
     
 def create_trading_environment():
     """Construct a complete environment with reasonable defaults"""
     benchmark_returns, treasury_curves = load_market_data()
 
-    start = datetime.strptime("01/01/2006","%m/%d/%Y")
-    start = start.replace(tzinfo=pytz.utc)
+    start = datetime(2006, 1, 1, tzinfo=pytz.utc)
+    end   = datetime(2006, 12, 31, tzinfo=pytz.utc)
     trading_environment = TradingEnvironment(
         benchmark_returns,
         treasury_curves,
         period_start = start,
+        period_end = end,
         capital_base = 100000.0
     )
     
@@ -72,9 +73,9 @@ def get_next_trading_dt(current, interval, trading_calendar):
     
     return next
 
-def create_trade_history(sid, prices, amounts, start_time, interval, trading_calendar):
+def create_trade_history(sid, prices, amounts, interval, trading_calendar):
     trades = []
-    current = start_time.replace(tzinfo = pytz.utc)
+    current = trading_calendar.first_open
 
     for price, amount in zip(prices, amounts):
         
@@ -94,9 +95,9 @@ def create_txn(sid, price, amount, datetime, btrid=None):
     })
     return txn
 
-def create_txn_history(sid, priceList, amtList, startTime, interval, trading_calendar):
+def create_txn_history(sid, priceList, amtList, interval, trading_calendar):
     txns = []
-    current = startTime
+    current = trading_calendar.first_open
 
     for price, amount in zip(priceList, amtList):
         current = get_next_trading_dt(current, interval, trading_calendar)
@@ -106,13 +107,13 @@ def create_txn_history(sid, priceList, amtList, startTime, interval, trading_cal
     return txns
 
 
-def create_returns(daycount, start, trading_calendar):
+def create_returns(daycount, trading_calendar):
     """
     For the given number of calendar (not trading) days return all the trading
     days between start and start + daycount.
     """
     test_range = []
-    current = start.replace(tzinfo=pytz.utc)
+    current = trading_calendar.first_open
     one_day = timedelta(days = 1)
     
     for day in range(daycount): 
@@ -124,9 +125,9 @@ def create_returns(daycount, start, trading_calendar):
     return test_range
     
 
-def create_returns_from_range(start, end, trading_calendar):
-    current = start.replace(tzinfo=pytz.utc)
-    end = end.replace(tzinfo=pytz.utc)
+def create_returns_from_range(trading_calendar):
+    current = trading_calendar.first_open
+    end = trading_calendar.last_close
     one_day = timedelta(days = 1)
     test_range = []
     while current <= end:
@@ -136,8 +137,8 @@ def create_returns_from_range(start, end, trading_calendar):
         
     return test_range
     
-def create_returns_from_list(returns, start, trading_calendar):
-    current = start.replace(tzinfo=pytz.utc)
+def create_returns_from_list(returns, trading_calendar):
+    current = trading_calendar.first_open
     one_day = timedelta(days = 1)
     test_range = []
     
@@ -157,7 +158,7 @@ def create_random_trade_source(sid, trade_count, trading_environment):
     source = RandomEquityTrades(sid, "rand-"+str(sid), trade_count)
     
     # make the period_end of trading_environment match
-    cur = trading_environment.period_start
+    cur = trading_environment.first_open
     one_day = timedelta(days = 1)
     for i in range(trade_count + 2):
        cur = get_next_trading_dt(cur, one_day, trading_environment)
@@ -179,14 +180,13 @@ def create_daily_trade_source(sids, trade_count, trading_environment):
     for sid in sids:
         price = [10.1] * trade_count
         volume = [100] * trade_count
-        start_date = trading_environment.period_start
+        start_date = trading_environment.first_open
         trade_time_increment = timedelta(days=1)
 
         generated_trades = create_trade_history( 
             sid, 
             price, 
             volume, 
-            start_date, 
             trade_time_increment, 
             trading_environment 
         )
diff --git a/zipline/test/test_finance.py b/zipline/test/test_finance.py
index a4d31ecc..52538f6a 100644
--- a/zipline/test/test_finance.py
+++ b/zipline/test/test_finance.py
@@ -50,6 +50,58 @@ class FinanceTestCase(TestCase):
             if prev:
                 self.assertTrue(trade.dt > prev.dt)
             prev = trade
+      
+    @timed(DEFAULT_TIMEOUT)
+    def test_trading_environment(self):
+        benchmark_returns, treasury_curves = \
+        factory.load_market_data()
+        
+        env = TradingEnvironment(
+            benchmark_returns,
+            treasury_curves,
+            period_start = datetime(2008, 1, 1, tzinfo = pytz.utc),
+            period_end = datetime(2008, 12, 31, tzinfo = pytz.utc),
+            capital_base = 100000,
+            max_drawdown = 0.50
+        )      
+        #holidays taken from: http://www.nyse.com/press/1191407641943.html
+        new_years   = datetime(2008, 1, 1, tzinfo = pytz.utc)
+        mlk_day     = datetime(2008, 1, 21, tzinfo = pytz.utc)
+        presidents  = datetime(2008, 2, 18, tzinfo = pytz.utc)
+        good_friday = datetime(2008, 3, 21, tzinfo = pytz.utc)
+        memorial_day= datetime(2008, 5, 26, tzinfo = pytz.utc)
+        july_4th    = datetime(2008, 7, 4, tzinfo = pytz.utc)
+        labor_day   = datetime(2008, 9, 1, tzinfo = pytz.utc)
+        tgiving     = datetime(2008, 11, 27, tzinfo = pytz.utc)
+        christmas   = datetime(2008, 5, 25, tzinfo = pytz.utc)
+        a_saturday  = datetime(2008, 8, 2, tzinfo = pytz.utc)
+        a_sunday    = datetime(2008, 10, 12, tzinfo = pytz.utc)
+        holidays = [
+            new_years, 
+            mlk_day, 
+            presidents, 
+            good_friday, 
+            memorial_day, 
+            july_4th, 
+            labor_day, 
+            tgiving, 
+            christmas,
+            a_saturday,
+            a_sunday
+        ]
+        
+        for holiday in holidays:
+            self.assertTrue(not env.is_trading_day(holiday))
+        
+        first_trading_day = datetime(2008, 1, 2, tzinfo = pytz.utc)
+        last_trading_day  = datetime(2008, 12, 31, tzinfo = pytz.utc)
+        workdays = [first_trading_day, last_trading_day]
+        
+        for workday in workdays:
+            self.assertTrue(env.is_trading_day(workday))
+        
+        self.assertTrue(env.last_close.month == 12)
+        self.assertTrue(env.last_close.day == 31)
         
     @timed(DEFAULT_TIMEOUT)
     def test_orders(self):
diff --git a/zipline/test/test_perf_tracking.py b/zipline/test/test_perf_tracking.py
index 568833bc..50da1475 100644
--- a/zipline/test/test_perf_tracking.py
+++ b/zipline/test/test_perf_tracking.py
@@ -18,18 +18,23 @@ class PerformanceTestCase(unittest.TestCase):
         self.benchmark_returns, self.treasury_curves = \
         factory.load_market_data()
         
+        random_index = random.randint(
+            0,
+            len(self.treasury_curves)
+        )
+        self.dt = self.treasury_curves.keys()[random_index]
+        self.end_dt = self.dt + datetime.timedelta(days=365+2)
         self.trading_environment = TradingEnvironment(
             self.benchmark_returns, 
-            self.treasury_curves
+            self.treasury_curves,
+            period_start = self.dt,
+            period_end = self.end_dt
         )
         
         self.onesec = datetime.timedelta(seconds=1)
         self.oneday = datetime.timedelta(days=1)
         self.tradingday = datetime.timedelta(hours=6, minutes=30)
-        random_index = random.randint(
-            0,
-            len(self.trading_environment.trading_days)
-        )
+        
         
         self.dt = self.trading_environment.trading_days[random_index]
         
@@ -46,7 +51,6 @@ class PerformanceTestCase(unittest.TestCase):
             1,
             [10,10,10,11],
             [100,100,100,100],
-            self.dt, 
             self.onesec,
             self.trading_environment
         )
@@ -110,15 +114,16 @@ class PerformanceTestCase(unittest.TestCase):
     def test_short_position(self):
         """verify that the performance period calculates properly for a \
 single short-sale transaction"""
-        trades_1 = factory.create_trade_history(
+        trades = factory.create_trade_history(
             1,
-            [10,10,10,11],
-            [100,100,100,100],
-            self.dt, 
+            [10,10,10,11,10,9],
+            [100,100,100,100,100,100],
             self.onesec,
             self.trading_environment
         )
         
+        trades_1 = trades[:-2]
+        
         txn = factory.create_txn(1, 10.0, -100, self.dt + self.onesec)
         pp = perf.PerformancePeriod({}, 0.0, 1000.0)
         
@@ -173,16 +178,9 @@ single short-sale transaction"""
         
         self.assertEqual(pp.pnl,-100,"gain of 1 on 100 shares should be 100")
             
-        #simulate additional trades, and ensure that the position value 
-        #reflects the new price
-        trades_2 = factory.create_trade_history(
-            1,
-            [10,9],
-            [100,100],
-            trades_1[-1]['dt'] + self.onesec, 
-            self.onesec,
-            self.trading_environment
-        )
+        # simulate additional trades, and ensure that the position value 
+        # reflects the new price
+        trades_2 = trades[-2:]
         
         #simulate a rollover to a new period
         pp2 = perf.PerformancePeriod(
@@ -314,7 +312,6 @@ trade after cover"""
             1,
             [10,10,10,11,9,8,7,8,9,10],
             [100,100,100,100,100,100,100,100,100,100],
-            self.dt, 
             self.onesec,
             self.trading_environment
         )
@@ -393,8 +390,7 @@ shares in position"
         trades = factory.create_trade_history(
             1, 
             [10,11,11,12], 
-            [100,100,100,100], 
-            self.dt, 
+            [100,100,100,100],  
             self.onesec,
             self.trading_environment
         )
@@ -403,7 +399,6 @@ shares in position"
             1, 
             [10,11,11,12], 
             [100,100,100,100], 
-            self.dt, 
             self.onesec,
             self.trading_environment
         )
@@ -510,14 +505,13 @@ shares in position"
         price = 10.1 
         price_list = [price] * trade_count
         volume = [100] * trade_count
-        start_date = datetime.datetime.strptime("01/01/2011","%m/%d/%Y")
-        start_date = start_date.replace(tzinfo=pytz.utc)
+        #start_date = datetime.datetime.strptime("01/01/2011","%m/%d/%Y")
+        #start_date = start_date.replace(tzinfo=pytz.utc)
         trade_time_increment = datetime.timedelta(days=1)
         trade_history = factory.create_trade_history( 
             sid, 
             price_list, 
             volume, 
-            start_date, 
             trade_time_increment, 
             self.trading_environment 
         )
@@ -529,7 +523,6 @@ shares in position"
             sid2, 
             price2_list, 
             volume, 
-            start_date, 
             trade_time_increment, 
             self.trading_environment 
         )
diff --git a/zipline/test/test_protocol.py b/zipline/test/test_protocol.py
index aaa470f7..c0d4a7c7 100644
--- a/zipline/test/test_protocol.py
+++ b/zipline/test/test_protocol.py
@@ -39,7 +39,6 @@ class ProtocolTestCase(TestCase):
             sid, 
             price, 
             volume, 
-            start_date, 
             one_day_td, 
             self.trading_environment
         )
diff --git a/zipline/test/test_risk.py b/zipline/test/test_risk.py
index 57576223..25685143 100644
--- a/zipline/test/test_risk.py
+++ b/zipline/test/test_risk.py
@@ -27,7 +27,9 @@ class Risk(unittest.TestCase):
         
         self.trading_env = TradingEnvironment(
             self.benchmark_returns, 
-            self.treasury_curves
+            self.treasury_curves,
+            period_start = start_date,
+            period_end = end_date
         )
         
         self.onesec = datetime.timedelta(seconds=1)
@@ -37,7 +39,6 @@ class Risk(unittest.TestCase):
         
         self.algo_returns_06 = factory.create_returns_from_list(
             RETURNS, 
-            start_date, 
             self.trading_env
         )
         
@@ -46,26 +47,43 @@ class Risk(unittest.TestCase):
             self.trading_env
         )
         
+        start_08 = datetime.datetime(
+            year=2008, 
+            month=1, 
+            day=1, 
+            hour=0,
+            minute=0,
+            tzinfo=pytz.utc)
+            
+        end_08 = datetime.datetime(
+            year=2008, 
+            month=12, 
+            day=31, 
+            tzinfo=pytz.utc
+        )
+        self.trading_env08 = TradingEnvironment(
+            self.benchmark_returns, 
+            self.treasury_curves,
+            period_start = start_08,
+            period_end = end_08
+        )
+        
     def tearDown(self):
         return
     
     def test_factory(self):
         returns = [0.1] * 100
-        start_date = datetime.datetime(year=2006, month=1, day=1, tzinfo=pytz.utc) 
-        r_objects = factory.create_returns_from_list(returns, start_date, self.trading_env)
+        r_objects = factory.create_returns_from_list(returns, self.trading_env)
         self.assertTrue(r_objects[-1].date <= datetime.datetime(year=2006, month=12, day=31, tzinfo=pytz.utc))
            
     def test_drawdown(self):
-        start_date = datetime.datetime(year=2006, month=1, day=1)
-        returns = factory.create_returns_from_list([1.0,-0.5,0.8,.17,1.0,-0.1,-0.45], start_date, self.trading_env)
+        returns = factory.create_returns_from_list([1.0,-0.5,0.8,.17,1.0,-0.1,-0.45], self.trading_env)
         #200, 100, 180, 210.6, 421.2, 379.8, 208.494
         metrics = risk.RiskMetrics(returns[0].date, returns[-1].date, returns, self.trading_env)
         self.assertEqual(metrics.max_drawdown, 0.505)
     
     def test_benchmark_returns_06(self):
-        start_date = datetime.datetime(year=2006, month=1, day=1)
-        end_date = datetime.datetime(year=2006, month=12, day=31)
-        returns = factory.create_returns_from_range(start_date, end_date, self.trading_env)
+        returns = factory.create_returns_from_range(self.trading_env)
         metrics = risk.RiskReport(returns, self.trading_env)
         self.assertEqual([round(x.benchmark_period_returns, 4)  for x in metrics.month_periods],    
         [0.0255,0.0005,0.0111,0.0122,-0.0309,0.0001,0.0051,0.0213,0.0246,0.0315,0.0165,0.0126])
@@ -76,17 +94,13 @@ class Risk(unittest.TestCase):
         self.assertEqual([round(x.benchmark_period_returns,4) for x in metrics.year_periods],[0.1362])
 
     def test_trading_days_06(self):
-        start_date = datetime.datetime(year=2006, month=1, day=1)
-        end_date = datetime.datetime(year=2006, month=12, day=31)
-        returns = factory.create_returns_from_range(start_date, end_date, self.trading_env)
+        returns = factory.create_returns_from_range(self.trading_env)
         metrics = risk.RiskReport(returns, self.trading_env)
         self.assertEqual([x.trading_days for x in metrics.year_periods],[251])
         self.assertEqual([x.trading_days for x in metrics.month_periods],[20,19,23,19,22,22,20,23,20,22,21,20])
 
     def test_benchmark_volatility_06(self):
-        start_date = datetime.datetime(year=2006, month=1, day=1)
-        end_date = datetime.datetime(year=2006, month=12, day=31)
-        returns = factory.create_returns_from_range(start_date, end_date, self.trading_env)
+        returns = factory.create_returns_from_range(self.trading_env)
         metrics = risk.RiskReport(returns, self.trading_env)
         self.assertEqual([round(x.benchmark_volatility, 3) for x in metrics.month_periods],
         [0.031,0.026,0.024,0.025,0.037,0.047,0.039,0.022,0.023,0.021,0.025,0.019])
@@ -146,11 +160,13 @@ class Risk(unittest.TestCase):
 
 
     def test_benchmark_returns_08(self):
-        start_date = datetime.datetime(year=2008, month=1, day=1)
-        end_date = datetime.datetime(year=2008, month=12, day=31)
-        returns = factory.create_returns_from_range(start_date, end_date, self.trading_env)
-        metrics = risk.RiskReport(returns, self.trading_env)
-        self.assertEqual([round(x.benchmark_period_returns, 3)  for x in metrics.month_periods],    
+        
+        returns = factory.create_returns_from_range(self.trading_env08)
+        metrics = risk.RiskReport(returns, self.trading_env08)
+
+        monthly = [round(x.benchmark_period_returns, 3)  for x in metrics.month_periods]
+        
+        self.assertEqual( monthly,    
         [-0.061,-0.035,-0.006,0.048,0.011,-0.086,-0.01,0.012,-0.091,-0.169,-0.075,0.008])
         self.assertEqual([round(x.benchmark_period_returns, 3) for x in metrics.three_month_periods],
         [-0.099,0.005,0.052,-0.032,-0.085,-0.084,-0.089,-0.236,-0.301,-0.226])
@@ -159,18 +175,14 @@ class Risk(unittest.TestCase):
         self.assertEqual([round(x.benchmark_period_returns,3) for x in metrics.year_periods],[-0.385])
     
     def test_trading_days_08(self):
-        start_date = datetime.datetime(year=2008, month=1, day=1)
-        end_date = datetime.datetime(year=2008, month=12, day=31)
-        returns = factory.create_returns_from_range(start_date, end_date, self.trading_env)
-        metrics = risk.RiskReport(returns, self.trading_env)
+        returns = factory.create_returns_from_range(self.trading_env08)
+        metrics = risk.RiskReport(returns, self.trading_env08)
         self.assertEqual([x.trading_days for x in metrics.year_periods],[253])
         self.assertEqual([x.trading_days for x in metrics.month_periods],[21,20,20,22,21,21,22,21,21,23,19,22])
         
     def test_benchmark_volatility_08(self):
-        start_date = datetime.datetime(year=2008, month=1, day=1)
-        end_date = datetime.datetime(year=2008, month=12, day=31)
-        returns = factory.create_returns_from_range(start_date, end_date, self.trading_env)
-        metrics = risk.RiskReport(returns, self.trading_env)
+        returns = factory.create_returns_from_range(self.trading_env08)
+        metrics = risk.RiskReport(returns, self.trading_env08)
         self.assertEqual([round(x.benchmark_volatility, 3) for x in metrics.month_periods],
         [0.07,0.058,0.082,0.054,0.041,0.057,0.068,0.06,0.157,0.244,0.195,0.145])
         self.assertEqual([round(x.benchmark_volatility, 3) for x in metrics.three_month_periods],
@@ -181,9 +193,7 @@ class Risk(unittest.TestCase):
         self.assertEqual([round(x.benchmark_volatility, 3) for x in metrics.year_periods],[0.41099999999999998])
         
     def test_treasury_returns_06(self):
-        start_date = datetime.datetime(year=2006, month=1, day=1)
-        end_date = datetime.datetime(year=2006, month=12, day=31)
-        returns = factory.create_returns_from_range(start_date, end_date, self.trading_env)
+        returns = factory.create_returns_from_range(self.trading_env)
         metrics = risk.RiskReport(returns, self.trading_env)
         self.assertEqual([round(x.treasury_period_return, 4) for x in metrics.month_periods],
         [0.0037,0.0034,0.0039,0.0038,0.0040,0.0037,0.0043,0.0043,0.0038,0.0044,0.0043,0.0041])
@@ -198,12 +208,31 @@ class Risk(unittest.TestCase):
         self.check_year_range(datetime.datetime(year=2008,month=1,day=1), 2)
         
     def test_partial_month(self):
-        start_date = datetime.datetime(year=1991, month=1, day=1)
-        returns = factory.create_returns(365 * 5 + 2, start_date, self.trading_env) #1992 and 1996 were leap years
+
+        start = datetime.datetime(
+            year=1991, 
+            month=1, 
+            day=1, 
+            hour=0,
+            minute=0,
+            tzinfo=pytz.utc)
+    
+        #1992 and 1996 were leap years
+        total_days = 365 * 5 + 2
+        end = start + datetime.timedelta(days = total_days)
+        trading_env90s = TradingEnvironment(
+            self.benchmark_returns, 
+            self.treasury_curves,
+            period_start = start,
+            period_end = end
+        )
+        
+        
+        returns = factory.create_returns(total_days, trading_env90s) 
         returns = returns[:-10] #truncate the returns series to end mid-month
-        metrics = risk.RiskReport(returns, self.trading_env)
+        metrics = risk.RiskReport(returns, trading_env90s)
         total_months = 60
-        self.check_metrics(metrics, total_months, start_date)
+        self.check_metrics(metrics, total_months, start)
         
     def check_year_range(self, start_date, years):
         if(start_date.month <= 2):
@@ -211,7 +240,7 @@ class Risk(unittest.TestCase):
         else:
             #because we may catch the leap of the last year, and i think this func is [start,end)
             ld = calendar.leapdays(start_date.year, start_date.year + years + 1) 
-        returns = factory.create_returns(365 * years + ld, start_date, self.trading_env)
+        returns = factory.create_returns(365 * years + ld, self.trading_env08)
         metrics = risk.RiskReport(returns, self.trading_env)
         total_months = years * 12
         self.check_metrics(metrics, total_months, start_date)

From 37a8bda4b249fee1bfec8d439936d8d1b971ccd8 Mon Sep 17 00:00:00 2001
From: fawce <fawce@quantopian.com>
Date: Fri, 6 Apr 2012 22:31:35 -0400
Subject: [PATCH 2/6] fixed bogus initial portfolio value (should be zero).

---
 zipline/finance/performance.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/zipline/finance/performance.py b/zipline/finance/performance.py
index 7b8211cd..56a9ead8 100644
--- a/zipline/finance/performance.py
+++ b/zipline/finance/performance.py
@@ -166,15 +166,23 @@ class PerformanceTracker():
         self.result_stream           = None
         self.last_dict               = None
 
+        # this performance period will span the entire simulation.
         self.cumulative_performance = PerformancePeriod(
+            # initial positions are empty
             {},
-            self.capital_base,
+            # initial portfolio positions have zero value
+            0,
+            # initial cash is your capital base.
             starting_cash = self.capital_base
         )
-
+        
+        # this performance period will span just the current market day
         self.todays_performance = PerformancePeriod(
+            # initial positions are empty
             {},
-            self.capital_base,
+            # initial portfolio positions have zero value
+            0,
+            # initial cash is your capital base.
             starting_cash = self.capital_base
         )
 

From 237b42c11e6e5596b80f4d02d6660e2abba9be0c Mon Sep 17 00:00:00 2001
From: fawce <fawce@quantopian.com>
Date: Sun, 8 Apr 2012 13:53:40 -0400
Subject: [PATCH 3/6] switched reporting to provide cash, equity, and total
 portfolio value

---
 zipline/finance/performance.py |  1 +
 zipline/protocol.py            | 31 ++++++++++++++++++++-----------
 2 files changed, 21 insertions(+), 11 deletions(-)

diff --git a/zipline/finance/performance.py b/zipline/finance/performance.py
index 56a9ead8..4b674e59 100644
--- a/zipline/finance/performance.py
+++ b/zipline/finance/performance.py
@@ -448,6 +448,7 @@ class PerformancePeriod():
             'starting_value' : self.starting_value,
             'starting_cash'  : self.starting_cash,
             'ending_cash'    : self.ending_cash,
+            'portfolio_value': self.ending_cash + self.ending_value,
             'positions'      : positions,
             'timestamp'      : datetime.datetime.now(),
             'pnl'            : self.pnl,
diff --git a/zipline/protocol.py b/zipline/protocol.py
index 9667c7bd..eb6f7271 100644
--- a/zipline/protocol.py
+++ b/zipline/protocol.py
@@ -634,22 +634,31 @@ def PERF_FRAME(perf):
 
     daily_perf = {
         'date'            : EPOCH(date),
-        'date_string'     : str(date),     
         'returns'         : tp['returns'],
         'pnl'             : tp['pnl'],
-        'portfolio_value' : tp['ending_value']
+        'market_value'    : cp['ending_value'],
+        'portfolio_value' : cp['portfolio_value'],
+        'starting_cash'   : tp['starting_cash'],
+        'ending_cash'     : tp['ending_cash'],
+        'capital_used'    : tp['capital_used']           
     }
 
     cumulative_perf = {
-        'alpha'                : risk['alpha'],
-        'beta'                 : risk['beta'],
-        'sharpe'               : risk['sharpe'],
-        'total_returns'        : cp['returns'],
-        'volatility'           : risk['algo_volatility'],
-        'benchmark_volatility' : risk['benchmark_volatility'],
-        'benchmark_returns'    : risk['benchmark_period_return'],
-        'max_drawdown'         : risk['max_drawdown'],
-        'pnl'                  : cp['pnl']
+        'alpha'                 : risk['alpha'],
+        'beta'                  : risk['beta'],
+        'sharpe'                : risk['sharpe'],
+        'volatility'            : risk['algo_volatility'],
+        'benchmark_volatility'  : risk['benchmark_volatility'],
+        'benchmark_returns'     : risk['benchmark_period_return'],
+        'max_drawdown'          : risk['max_drawdown'],
+        'total_returns'         : cp['returns'],
+        'pnl'                   : cp['pnl'],
+        'market_value'          : cp['ending_value'],
+        'portfolio_value'       : cp['portfolio_value'],
+        'starting_cash'         : cp['starting_cash'],
+        'ending_cash'           : cp['ending_cash'],
+        'capital_used'          : cp['capital_used']
+        
     }
     
     # nest the cumulative performance data in the daily.

From aa2bcdf83ef0a828387c32b4f13b403cd6f29a75 Mon Sep 17 00:00:00 2001
From: fawce <fawce@quantopian.com>
Date: Sun, 8 Apr 2012 21:09:42 -0400
Subject: [PATCH 4/6] ending values for portfolio, equity, and cash in
 cumulative were removed, because they are redundant to the same values in
 daily. Also removed starting cash, as it is unchanging and equal to the
 capital base.

---
 zipline/protocol.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/zipline/protocol.py b/zipline/protocol.py
index eb6f7271..f8db09bc 100644
--- a/zipline/protocol.py
+++ b/zipline/protocol.py
@@ -636,8 +636,8 @@ def PERF_FRAME(perf):
         'date'            : EPOCH(date),
         'returns'         : tp['returns'],
         'pnl'             : tp['pnl'],
-        'market_value'    : cp['ending_value'],
-        'portfolio_value' : cp['portfolio_value'],
+        'market_value'    : tp['ending_value'],
+        'portfolio_value' : tp['portfolio_value'],
         'starting_cash'   : tp['starting_cash'],
         'ending_cash'     : tp['ending_cash'],
         'capital_used'    : tp['capital_used']           
@@ -653,10 +653,6 @@ def PERF_FRAME(perf):
         'max_drawdown'          : risk['max_drawdown'],
         'total_returns'         : cp['returns'],
         'pnl'                   : cp['pnl'],
-        'market_value'          : cp['ending_value'],
-        'portfolio_value'       : cp['portfolio_value'],
-        'starting_cash'         : cp['starting_cash'],
-        'ending_cash'           : cp['ending_cash'],
         'capital_used'          : cp['capital_used']
         
     }

From 57c39bf615e31bcbc87785961831ed36ae34e43b Mon Sep 17 00:00:00 2001
From: fawce <fawce@quantopian.com>
Date: Mon, 9 Apr 2012 10:20:08 -0400
Subject: [PATCH 5/6] switching to only calculate the returns and risk on
 market close, rather than per trade.

---
 zipline/finance/performance.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/zipline/finance/performance.py b/zipline/finance/performance.py
index 4b674e59..93d06f2c 100644
--- a/zipline/finance/performance.py
+++ b/zipline/finance/performance.py
@@ -255,12 +255,14 @@ class PerformanceTracker():
         self.cumulative_performance.update_last_sale(event)
         self.todays_performance.update_last_sale(event)
 
+        
+
+    def handle_market_close(self):
         #calculate performance as of last trade
         self.cumulative_performance.calculate_performance()
         self.todays_performance.calculate_performance()
-
-    def handle_market_close(self):
-        #add the return results from today to the list of DailyReturn objects.
+        
+        # add the return results from today to the list of DailyReturn objects.
         todays_date = self.market_close.replace(hour=0, minute=0, second=0)
         todays_return_obj = risk.DailyReturn(
             todays_date,
@@ -297,7 +299,6 @@ class PerformanceTracker():
         self.market_close = self.market_open + self.trading_day
         
         # Roll over positions to current day.
-        self.todays_performance.calculate_performance()
         self.todays_performance = PerformancePeriod(
             self.todays_performance.positions,
             self.todays_performance.ending_value,

From 14166ccc302f2b7c61439939abab34acd75232b9 Mon Sep 17 00:00:00 2001
From: fawce <fawce@quantopian.com>
Date: Mon, 9 Apr 2012 10:31:11 -0400
Subject: [PATCH 6/6] dropped the extra days in trading range...

---
 zipline/test/test_perf_tracking.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/zipline/test/test_perf_tracking.py b/zipline/test/test_perf_tracking.py
index 50da1475..bd6efe7c 100644
--- a/zipline/test/test_perf_tracking.py
+++ b/zipline/test/test_perf_tracking.py
@@ -23,7 +23,7 @@ class PerformanceTestCase(unittest.TestCase):
             len(self.treasury_curves)
         )
         self.dt = self.treasury_curves.keys()[random_index]
-        self.end_dt = self.dt + datetime.timedelta(days=365+2)
+        self.end_dt = self.dt + datetime.timedelta(days=365)
         self.trading_environment = TradingEnvironment(
             self.benchmark_returns, 
             self.treasury_curves,