From cfe755855c21b9e1fa5658ea59e3be13d131a591 Mon Sep 17 00:00:00 2001
From: Nathan Wolfe <nathanwolferl@gmail.com>
Date: Fri, 24 Jun 2016 16:01:42 -0400
Subject: [PATCH 01/10] ENH: Add PanelMinuteBarReader, use it in
 TradingAlgorithm.run.

TradingAlgorithm.run didn't support Panel minute bar data, and assumed
all Panel data was daily.

To rectify this, adding PanelMinuteBarReader class.
TradingAlgorithm.run decides whether to use it or PanelDailyBarReader
by assuming data is daily if and only if the time of day of every
Timestamp is identical.
---
 zipline/algorithm.py        |  45 +++++++++++----
 zipline/data/minute_bars.py | 111 ++++++++++++++++++++++++++++++++++++
 2 files changed, 144 insertions(+), 12 deletions(-)

diff --git a/zipline/algorithm.py b/zipline/algorithm.py
index c8f68078..66468e11 100644
--- a/zipline/algorithm.py
+++ b/zipline/algorithm.py
@@ -38,6 +38,7 @@ from zipline._protocol import handle_non_market_minutes
 from zipline.assets.synthetic import make_simple_equity_info
 from zipline.data.data_portal import DataPortal
 from zipline.data.us_equity_pricing import PanelDailyBarReader
+from zipline.data.minute_bars import PanelMinuteBarReader
 from zipline.errors import (
     AttachPipelineAfterInitialize,
     HistoryInInitialize,
@@ -615,8 +616,8 @@ class TradingAlgorithm(object):
                 # to be inferred.
                 if overwrite_sim_params:
                     self.sim_params = self.sim_params.create_new(
-                        data.major_axis[0],
-                        data.major_axis[-1]
+                        normalize_date(data.major_axis[0]),
+                        normalize_date(data.major_axis[-1])
                     )
 
                 copy_panel = data.rename(
@@ -634,16 +635,36 @@ class TradingAlgorithm(object):
                         copy_panel.items
                     )
                 )
-                equity_daily_reader = PanelDailyBarReader(
-                    self.trading_calendar.all_sessions,
-                    copy_panel,
-                )
-                self.data_portal = DataPortal(
-                    self.asset_finder,
-                    self.trading_calendar,
-                    first_trading_day=equity_daily_reader.first_trading_day,
-                    equity_daily_reader=equity_daily_reader,
-                )
+
+                # Assume data is daily if timestamp times are
+                # standardized, otherwise assume minute bars.
+                times = copy_panel.major_axis.time
+                if np.all(times == times[0]):
+                    equity_daily_reader = PanelDailyBarReader(
+                        self.trading_calendar.all_sessions,
+                        copy_panel,
+                    )
+                    self.data_portal = DataPortal(
+                        self.asset_finder,
+                        self.trading_calendar,
+                        first_trading_day=equity_daily_reader
+                            .first_trading_day,
+                        equity_daily_reader=equity_daily_reader,
+                    )
+                else:
+                    if overwrite_sim_params:
+                        self.sim_params.data_frequency = 'minute'
+                    equity_minute_reader = PanelMinuteBarReader(
+                        self.trading_calendar.all_minutes,
+                        copy_panel,
+                    )
+                    self.data_portal = DataPortal(
+                        self.asset_finder,
+                        self.trading_calendar,
+                        first_trading_day=equity_minute_reader
+                            .first_trading_day,
+                        equity_minute_reader=equity_minute_reader,
+                    )
 
         # Force a reset of the performance tracker, in case
         # this is a repeat run of the algorithm.
diff --git a/zipline/data/minute_bars.py b/zipline/data/minute_bars.py
index df1b4328..0fcb4917 100644
--- a/zipline/data/minute_bars.py
+++ b/zipline/data/minute_bars.py
@@ -21,7 +21,9 @@ import bcolz
 from bcolz import ctable
 from intervaltree import IntervalTree
 import numpy as np
+from numpy import zeros
 import pandas as pd
+from pandas import NaT
 
 from zipline.data._minute_bar_internal import (
     minute_value,
@@ -30,6 +32,11 @@ from zipline.data._minute_bar_internal import (
 )
 
 from zipline.gens.sim_engine import NANOS_IN_MINUTE
+from zipline.utils.preprocess import call
+from zipline.utils.input_validation import (
+    preprocess,
+    verify_indices_all_unique,
+)
 from zipline.utils.cli import maybe_show_progress
 from zipline.utils.memoize import lazyval
 
@@ -979,3 +986,107 @@ class BcolzMinuteBarReader(object):
                 out *= self._ohlc_inverse
             results.append(out)
         return results
+
+
+class PanelMinuteBarReader(object):
+    """
+    Reader for data passed as Panel.
+
+    DataPanel Structure
+    -------
+    items : Int64Index
+        Asset identifiers.  Must be unique.
+    major_axis : DatetimeIndex
+       Datetimes for data provided by the Panel.  Must be unique.
+    minor_axis : ['open', 'high', 'low', 'close', 'volume']
+       Price attributes.  Must be unique.
+
+    Attributes
+    ----------
+    The table with which this loader interacts contains the following
+    attributes:
+
+    panel : pd.Panel
+        The panel from which to read OHLCV data.
+    first_trading_day : pd.Timestamp
+        The first trading day in the dataset.
+    """
+    @preprocess(panel=call(verify_indices_all_unique))
+    def __init__(self, calendar, panel):
+
+        panel = panel.copy()
+        if 'volume' not in panel.minor_axis:
+            # Fake volume if it does not exist.
+            panel.loc[:, :, 'volume'] = int(1e9)
+
+        self.first_trading_day = pd.datetools.normalize_date(
+            panel.major_axis[0]
+        )
+        self._calendar = calendar
+
+        self.panel = panel
+
+        self._ohlc_inverse = 1. / OHLC_RATIO
+
+    @property
+    def last_available_dt(self):
+        return self.panel.major_axis[-1]
+
+    def load_raw_arrays(self, columns, start_dt, end_dt, assets):
+        columns = list(columns)
+        dts = self.panel.major_axis
+        index = dts[dts.slice_indexer(start_dt, end_dt)]
+        shape = (len(index), len(assets))
+        results = []
+        for col in columns:
+            outbuf = zeros(shape=shape)
+            for i, asset in enumerate(assets):
+                data = self.panel.loc[asset, start_dt:end_dt, col]
+                data = data.reindex_axis(index).values
+                outbuf[:, i] = data
+            results.append(outbuf)
+        return results
+
+    def spot_price(self, sid, dt, colname):
+        """
+        Parameters
+        ----------
+        sid : int
+            The asset identifier.
+        dt : datetime64-like
+            Midnight of the day for which data is requested.
+        colname : string
+            The price field. e.g. ('open', 'high', 'low', 'close', 'volume')
+
+        Returns
+        -------
+        float
+            The spot price for colname of the given sid on the given day.
+            Raises a NoDataOnDate exception if the given day and sid is before
+            or after the date range of the equity.
+            Returns -1 if the day is within the date range, but the price is
+            0.
+        """
+        return self.panel.loc[sid, dt, colname]
+
+    get_value = spot_price
+
+    def get_last_traded_dt(self, sid, dt):
+        """
+        Parameters
+        ----------
+        sid : int
+            The asset identifier.
+        dt : datetime64-like
+            Midnight of the day for which data is requested.
+
+        Returns
+        -------
+        pd.Timestamp : The last known dt for the asset and dt;
+                       NaT if no trade is found before the given dt.
+        """
+        for ts in self.panel.major_axis[self.panel.major_axis
+                                        .slice_indexer(end=dt)][::-1]:
+            if not pd.isnull(self.panel.loc[sid, ts, 'close']):
+                return ts
+        return NaT

From 96dc1c37217123213a30290929d178a0982de306 Mon Sep 17 00:00:00 2001
From: Nathan Wolfe <nathanwolferl@gmail.com>
Date: Mon, 27 Jun 2016 16:45:16 -0400
Subject: [PATCH 02/10] BUG: Generate sim_params within run_algorithm, fix it
 for raw data

Previously, run_algorithm caused an error if run on raw (non-bundle)
data, because of uninitialized variables. Initializing those variables
to None to allow run_algorithm to work with Panel data, etc.

Also, run_algorithm did not create sim_params for the TradingAlgorithm
instance it created; this kicked the can to TradingAlgorithm, which
gets default sim_params with data_frequency 'daily'. To support minute
bars, changing run_algorithm to create its own sim_params with the
data_frequency specified in its arguments.
---
 zipline/utils/run_algo.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/zipline/utils/run_algo.py b/zipline/utils/run_algo.py
index 8c590737..d72c3c96 100644
--- a/zipline/utils/run_algo.py
+++ b/zipline/utils/run_algo.py
@@ -21,6 +21,7 @@ from zipline.finance.trading import TradingEnvironment
 from zipline.pipeline.data import USEquityPricing
 from zipline.pipeline.loaders import USEquityPricingLoader
 from zipline.utils.calendars import get_calendar
+from zipline.utils.factory import create_simulation_parameters
 import zipline.utils.paths as pth
 
 
@@ -150,14 +151,21 @@ def _run(handle_data,
             raise ValueError(
                 "No PipelineLoader registered for column %s." % column
             )
+    else:
+        env = None
+        choose_loader = None
 
     perf = TradingAlgorithm(
         namespace=namespace,
         capital_base=capital_base,
-        start=start,
-        end=end,
         env=env,
         get_pipeline_loader=choose_loader,
+        sim_params=create_simulation_parameters(
+            start=start,
+            end=end,
+            capital_base=capital_base,
+            data_frequency=data_frequency,
+        ),
         **{
             'initialize': initialize,
             'handle_data': handle_data,
@@ -314,8 +322,8 @@ def run_algorithm(start,
     load_extensions(default_extension, extensions, strict_extensions, environ)
 
     non_none_data = valfilter(bool, {
-        'data': data,
-        'bundle': bundle,
+        'data': data is not None,
+        'bundle': bundle is not None,
     })
     if not non_none_data:
         # if neither data nor bundle are passed use 'quantopian-quandl'

From 19d493707f19d1c6684788d7aeac8cde92a29a3a Mon Sep 17 00:00:00 2001
From: Nathan Wolfe <nathanwolferl@gmail.com>
Date: Tue, 28 Jun 2016 13:31:50 -0400
Subject: [PATCH 03/10] ENH: Improve TradingAlgorithm.run daily or minute data
 freq assumption

Changing TradingAlgorithm.run not to assume minute data if data freq is
specified as daily and sim params aren't allowed to be overwritten.
---
 zipline/algorithm.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/zipline/algorithm.py b/zipline/algorithm.py
index 66468e11..60ff5621 100644
--- a/zipline/algorithm.py
+++ b/zipline/algorithm.py
@@ -639,7 +639,9 @@ class TradingAlgorithm(object):
                 # Assume data is daily if timestamp times are
                 # standardized, otherwise assume minute bars.
                 times = copy_panel.major_axis.time
-                if np.all(times == times[0]):
+                if (np.all(times == times[0]) or
+                    (self.sim_params.data_frequency == 'daily'
+                     and not overwrite_sim_params)):
                     equity_daily_reader = PanelDailyBarReader(
                         self.trading_calendar.all_sessions,
                         copy_panel,

From 55b79e8f320e4d25ba6cbb4941353a11d3f81aa7 Mon Sep 17 00:00:00 2001
From: Nathan Wolfe <nathanwolferl@gmail.com>
Date: Thu, 30 Jun 2016 15:10:59 -0400
Subject: [PATCH 04/10] TST: Test `TradingAlgorithm.run` and `run_algorithm` on
 raw Panel data

---
 tests/test_algorithm.py | 69 ++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 68 insertions(+), 1 deletion(-)

diff --git a/tests/test_algorithm.py b/tests/test_algorithm.py
index eacff880..fcc67017 100644
--- a/tests/test_algorithm.py
+++ b/tests/test_algorithm.py
@@ -33,7 +33,10 @@ import numpy as np
 import pandas as pd
 import pytz
 
-from zipline import TradingAlgorithm
+from zipline import (
+    run_algorithm,
+    TradingAlgorithm,
+)
 from zipline.api import FixedSlippage
 from zipline.assets import Equity, Future
 from zipline.assets.synthetic import (
@@ -161,6 +164,7 @@ from zipline.test_algorithms import (
     no_handle_data,
 )
 from zipline.utils.api_support import ZiplineAPI, set_algo_instance
+from zipline.utils.calendars import get_calendar
 from zipline.utils.context_tricks import CallbackManager
 from zipline.utils.control_flow import nullctx
 import zipline.utils.events
@@ -4110,3 +4114,66 @@ class AlgoInputValidationTestCase(ZiplineTestCase):
                     script=script,
                     **{method: lambda *args, **kwargs: None}
                 )
+
+
+class TestPanelData(ZiplineTestCase):
+
+    @parameterized.expand([
+        ('daily',
+         pd.Timestamp('2015-12-23', tz='UTC'),
+         pd.Timestamp('2016-01-05', tz='UTC'),),
+        ('minute',
+         pd.Timestamp('2015-12-23', tz='UTC'),
+         pd.Timestamp('2015-12-24', tz='UTC'),),
+    ])
+    def test_panel_data(self, data_frequency, start_dt, end_dt):
+        if data_frequency == 'daily':
+            history_freq = '1d'
+            df = create_daily_df_for_asset(get_calendar('NYSE'),
+                                           start_dt, end_dt)
+        elif data_frequency == 'minute':
+            history_freq = '1m'
+            df = create_minute_df_for_asset(get_calendar('NYSE'),
+                                            start_dt, end_dt)
+
+        panel = pd.Panel({1: df})
+
+        price_record = pd.DataFrame(columns=['current', 'previous'])
+
+        def initialize(algo):
+            algo.first_bar = True
+
+        def handle_data(algo, data):
+            price_record.loc[algo.get_datetime(), 'current'] = (
+                data.current(algo.sid(1), 'price')
+            )
+            if algo.first_bar:
+                algo.first_bar = False
+            else:
+                price_record.loc[algo.get_datetime(), 'previous'] = (
+                    data.history(algo.sid(1), 'price', 2, history_freq)[0]
+                )
+
+        trading_algo = TradingAlgorithm(initialize=initialize,
+                                        handle_data=handle_data)
+        trading_algo.run(data=panel)
+        np.testing.assert_array_equal(
+            np.array(price_record.transpose(), dtype='float64'),
+            np.array([df['close'], df['close'].shift(1)], dtype='float64')
+        )
+
+        price_record.drop(price_record.index)
+
+        run_algorithm(
+            start=start_dt,
+            end=end_dt,
+            capital_base=1,
+            initialize=initialize,
+            handle_data=handle_data,
+            data_frequency=data_frequency,
+            data=panel
+        )
+        np.testing.assert_array_equal(
+            np.array(price_record.transpose(), dtype='float64'),
+            np.array([df['close'], df['close'].shift(1)], dtype='float64')
+        )

From 3efbe6bc171d86dc0f0ed519e9d430a4f21e7881 Mon Sep 17 00:00:00 2001
From: Nathan Wolfe <nathanwolferl@gmail.com>
Date: Thu, 30 Jun 2016 15:58:58 -0400
Subject: [PATCH 05/10] MAINT: Clean up data freq inference in
 `TradingAlgorithm.run`.

---
 zipline/algorithm.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/zipline/algorithm.py b/zipline/algorithm.py
index 60ff5621..6a2e5767 100644
--- a/zipline/algorithm.py
+++ b/zipline/algorithm.py
@@ -620,6 +620,14 @@ class TradingAlgorithm(object):
                         normalize_date(data.major_axis[-1])
                     )
 
+                    # Assume data is daily if timestamp times are
+                    # standardized, otherwise assume minute bars.
+                    times = data.major_axis.time
+                    if np.all(times == times[0]):
+                        self.sim_params.data_frequency = 'daily'
+                    else:
+                        self.sim_params.data_frequency = 'minute'
+
                 copy_panel = data.rename(
                     # These were the old names for the close/open columns.  We
                     # need to make a copy anyway, so swap these for backwards
@@ -636,12 +644,7 @@ class TradingAlgorithm(object):
                     )
                 )
 
-                # Assume data is daily if timestamp times are
-                # standardized, otherwise assume minute bars.
-                times = copy_panel.major_axis.time
-                if (np.all(times == times[0]) or
-                    (self.sim_params.data_frequency == 'daily'
-                     and not overwrite_sim_params)):
+                if self.sim_params.data_frequency == 'daily':
                     equity_daily_reader = PanelDailyBarReader(
                         self.trading_calendar.all_sessions,
                         copy_panel,
@@ -653,9 +656,7 @@ class TradingAlgorithm(object):
                             .first_trading_day,
                         equity_daily_reader=equity_daily_reader,
                     )
-                else:
-                    if overwrite_sim_params:
-                        self.sim_params.data_frequency = 'minute'
+                elif self.sim_params.data_frequency == 'minute':
                     equity_minute_reader = PanelMinuteBarReader(
                         self.trading_calendar.all_minutes,
                         copy_panel,

From bdce4ef25796f11d635b3cd98779cb093a55615b Mon Sep 17 00:00:00 2001
From: Nathan Wolfe <nathanwolferl@gmail.com>
Date: Mon, 11 Jul 2016 16:06:53 -0400
Subject: [PATCH 06/10] TST: Expand Panel data test to test for multiple sids.

---
 tests/test_algorithm.py | 57 ++++++++++++++++++++++++++---------------
 1 file changed, 37 insertions(+), 20 deletions(-)

diff --git a/tests/test_algorithm.py b/tests/test_algorithm.py
index fcc67017..3140caf3 100644
--- a/tests/test_algorithm.py
+++ b/tests/test_algorithm.py
@@ -4127,42 +4127,62 @@ class TestPanelData(ZiplineTestCase):
          pd.Timestamp('2015-12-24', tz='UTC'),),
     ])
     def test_panel_data(self, data_frequency, start_dt, end_dt):
+        trading_calendar = get_calendar('NYSE')
         if data_frequency == 'daily':
             history_freq = '1d'
-            df = create_daily_df_for_asset(get_calendar('NYSE'),
-                                           start_dt, end_dt)
+            create_df_for_asset = create_daily_df_for_asset
+            dt_transform = trading_calendar.minute_to_session_label
         elif data_frequency == 'minute':
             history_freq = '1m'
-            df = create_minute_df_for_asset(get_calendar('NYSE'),
-                                            start_dt, end_dt)
+            create_df_for_asset = create_minute_df_for_asset
 
-        panel = pd.Panel({1: df})
+            def dt_transform(dt):
+                return dt
 
-        price_record = pd.DataFrame(columns=['current', 'previous'])
+        sids = range(1, 3)
+        dfs = {}
+        for sid in sids:
+            dfs[sid] = create_df_for_asset(trading_calendar,
+                                           start_dt, end_dt, interval=sid)
+            dfs[sid]['prev_close'] = dfs[sid]['close'].shift(1)
+        panel = pd.Panel(dfs)
+
+        price_record = pd.Panel(items=sids,
+                                major_axis=panel.major_axis,
+                                minor_axis=['current', 'previous'])
 
         def initialize(algo):
             algo.first_bar = True
+            algo.equities = []
+            for sid in sids:
+                algo.equities.append(algo.sid(sid))
 
         def handle_data(algo, data):
-            price_record.loc[algo.get_datetime(), 'current'] = (
-                data.current(algo.sid(1), 'price')
+            price_record.loc[:, dt_transform(algo.get_datetime()),
+                             'current'] = (
+                data.current(algo.equities, 'price')
             )
             if algo.first_bar:
                 algo.first_bar = False
             else:
-                price_record.loc[algo.get_datetime(), 'previous'] = (
-                    data.history(algo.sid(1), 'price', 2, history_freq)[0]
+                price_record.loc[:, dt_transform(algo.get_datetime()),
+                                 'previous'] = (
+                    data.history(algo.equities, 'price',
+                                 2, history_freq).iloc[0]
                 )
 
+        def check_panels():
+            np.testing.assert_array_equal(
+                price_record.values.astype('float64'),
+                panel.loc[:, :, ['close',
+                                 'prev_close']].values.astype('float64')
+            )
+
         trading_algo = TradingAlgorithm(initialize=initialize,
                                         handle_data=handle_data)
         trading_algo.run(data=panel)
-        np.testing.assert_array_equal(
-            np.array(price_record.transpose(), dtype='float64'),
-            np.array([df['close'], df['close'].shift(1)], dtype='float64')
-        )
-
-        price_record.drop(price_record.index)
+        check_panels()
+        price_record.loc[:] = np.nan
 
         run_algorithm(
             start=start_dt,
@@ -4173,7 +4193,4 @@ class TestPanelData(ZiplineTestCase):
             data_frequency=data_frequency,
             data=panel
         )
-        np.testing.assert_array_equal(
-            np.array(price_record.transpose(), dtype='float64'),
-            np.array([df['close'], df['close'].shift(1)], dtype='float64')
-        )
+        check_panels()

From 69506570dd7fa8dcbb6ea884ba69d882a888d25e Mon Sep 17 00:00:00 2001
From: Nathan Wolfe <nathanwolferl@gmail.com>
Date: Mon, 11 Jul 2016 16:20:34 -0400
Subject: [PATCH 07/10] ENH: Guard against tz-naive index for Panel data.

---
 zipline/algorithm.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/zipline/algorithm.py b/zipline/algorithm.py
index 6a2e5767..1d9e4b75 100644
--- a/zipline/algorithm.py
+++ b/zipline/algorithm.py
@@ -612,6 +612,10 @@ class TradingAlgorithm(object):
                 data = data.swapaxes(0, 2)
 
             if isinstance(data, pd.Panel):
+                # Guard against tz-naive index.
+                if data.major_axis.tz is None:
+                    data.major_axis = data.major_axis.tz_localize('UTC')
+
                 # For compatibility with existing examples allow start/end
                 # to be inferred.
                 if overwrite_sim_params:

From 763f2ab8b4996d1f89b40c0ff4d3cbd55655ec12 Mon Sep 17 00:00:00 2001
From: Nathan Wolfe <nathanwolferl@gmail.com>
Date: Wed, 13 Jul 2016 17:45:07 -0400
Subject: [PATCH 08/10] MAINT: Combine daily and minute into `PanelBarReader`.

Also simplify `load_raw_arrays` and `get_last_traded_dt`.
---
 tests/test_panel_daily_bar_reader.py |   6 +-
 zipline/algorithm.py                 |  37 ++++-----
 zipline/data/data_portal.py          |   2 -
 zipline/data/minute_bars.py          | 111 ---------------------------
 zipline/data/us_equity_pricing.py    |  53 +++++++------
 5 files changed, 42 insertions(+), 167 deletions(-)

diff --git a/tests/test_panel_daily_bar_reader.py b/tests/test_panel_daily_bar_reader.py
index ba30cca9..2e29bd69 100644
--- a/tests/test_panel_daily_bar_reader.py
+++ b/tests/test_panel_daily_bar_reader.py
@@ -18,7 +18,7 @@ from itertools import permutations, product
 import numpy as np
 import pandas as pd
 
-from zipline.data.us_equity_pricing import PanelDailyBarReader
+from zipline.data.us_equity_pricing import PanelBarReader
 from zipline.testing import ExplodingObject
 from zipline.testing.fixtures import (
     WithAssetFinder,
@@ -55,7 +55,7 @@ class TestPanelDailyBarReader(WithAssetFinder,
             minor_axis=minor_axis,
         )
 
-        cls.reader = PanelDailyBarReader(days, cls.panel)
+        cls.reader = PanelBarReader(days, cls.panel)
 
     def test_spot_price(self):
         panel = self.panel
@@ -83,7 +83,7 @@ class TestPanelDailyBarReader(WithAssetFinder,
         for axis_order in permutations((0, 1, 2)):
             transposed = panel.transpose(*axis_order)
             with self.assertRaises(ValueError) as e:
-                PanelDailyBarReader(unused, transposed)
+                PanelBarReader(unused, transposed)
 
             expected = (
                 "Duplicate entries in Panel.{name}: ['a', 'b'].".format(
diff --git a/zipline/algorithm.py b/zipline/algorithm.py
index 1d9e4b75..a84cacb9 100644
--- a/zipline/algorithm.py
+++ b/zipline/algorithm.py
@@ -37,8 +37,7 @@ from six import (
 from zipline._protocol import handle_non_market_minutes
 from zipline.assets.synthetic import make_simple_equity_info
 from zipline.data.data_portal import DataPortal
-from zipline.data.us_equity_pricing import PanelDailyBarReader
-from zipline.data.minute_bars import PanelMinuteBarReader
+from zipline.data.us_equity_pricing import PanelBarReader
 from zipline.errors import (
     AttachPipelineAfterInitialize,
     HistoryInInitialize,
@@ -649,29 +648,19 @@ class TradingAlgorithm(object):
                 )
 
                 if self.sim_params.data_frequency == 'daily':
-                    equity_daily_reader = PanelDailyBarReader(
-                        self.trading_calendar.all_sessions,
-                        copy_panel,
-                    )
-                    self.data_portal = DataPortal(
-                        self.asset_finder,
-                        self.trading_calendar,
-                        first_trading_day=equity_daily_reader
-                            .first_trading_day,
-                        equity_daily_reader=equity_daily_reader,
-                    )
+                    equity_reader_arg = 'equity_daily_reader'
+                    calendar = self.trading_calendar.all_sessions
                 elif self.sim_params.data_frequency == 'minute':
-                    equity_minute_reader = PanelMinuteBarReader(
-                        self.trading_calendar.all_minutes,
-                        copy_panel,
-                    )
-                    self.data_portal = DataPortal(
-                        self.asset_finder,
-                        self.trading_calendar,
-                        first_trading_day=equity_minute_reader
-                            .first_trading_day,
-                        equity_minute_reader=equity_minute_reader,
-                    )
+                    equity_reader_arg = 'equity_minute_reader'
+                    calendar = self.trading_calendar.all_minutes
+                equity_reader = PanelBarReader(calendar, copy_panel)
+
+                self.data_portal = DataPortal(
+                    self.asset_finder,
+                    self.trading_calendar,
+                    first_trading_day=equity_reader.first_trading_day,
+                    **{equity_reader_arg: equity_reader}
+                )
 
         # Force a reset of the performance tracker, in case
         # this is a repeat run of the algorithm.
diff --git a/zipline/data/data_portal.py b/zipline/data/data_portal.py
index dcb3e0b1..294ba86d 100644
--- a/zipline/data/data_portal.py
+++ b/zipline/data/data_portal.py
@@ -554,8 +554,6 @@ class DataPortal(object):
                 self._equity_minute_reader,
                 self._adjustment_reader
             )
-            self.MINUTE_PRICE_ADJUSTMENT_FACTOR = \
-                self._equity_minute_reader._ohlc_inverse
 
         self._first_trading_day = first_trading_day
 
diff --git a/zipline/data/minute_bars.py b/zipline/data/minute_bars.py
index 0fcb4917..df1b4328 100644
--- a/zipline/data/minute_bars.py
+++ b/zipline/data/minute_bars.py
@@ -21,9 +21,7 @@ import bcolz
 from bcolz import ctable
 from intervaltree import IntervalTree
 import numpy as np
-from numpy import zeros
 import pandas as pd
-from pandas import NaT
 
 from zipline.data._minute_bar_internal import (
     minute_value,
@@ -32,11 +30,6 @@ from zipline.data._minute_bar_internal import (
 )
 
 from zipline.gens.sim_engine import NANOS_IN_MINUTE
-from zipline.utils.preprocess import call
-from zipline.utils.input_validation import (
-    preprocess,
-    verify_indices_all_unique,
-)
 from zipline.utils.cli import maybe_show_progress
 from zipline.utils.memoize import lazyval
 
@@ -986,107 +979,3 @@ class BcolzMinuteBarReader(object):
                 out *= self._ohlc_inverse
             results.append(out)
         return results
-
-
-class PanelMinuteBarReader(object):
-    """
-    Reader for data passed as Panel.
-
-    DataPanel Structure
-    -------
-    items : Int64Index
-        Asset identifiers.  Must be unique.
-    major_axis : DatetimeIndex
-       Datetimes for data provided by the Panel.  Must be unique.
-    minor_axis : ['open', 'high', 'low', 'close', 'volume']
-       Price attributes.  Must be unique.
-
-    Attributes
-    ----------
-    The table with which this loader interacts contains the following
-    attributes:
-
-    panel : pd.Panel
-        The panel from which to read OHLCV data.
-    first_trading_day : pd.Timestamp
-        The first trading day in the dataset.
-    """
-    @preprocess(panel=call(verify_indices_all_unique))
-    def __init__(self, calendar, panel):
-
-        panel = panel.copy()
-        if 'volume' not in panel.minor_axis:
-            # Fake volume if it does not exist.
-            panel.loc[:, :, 'volume'] = int(1e9)
-
-        self.first_trading_day = pd.datetools.normalize_date(
-            panel.major_axis[0]
-        )
-        self._calendar = calendar
-
-        self.panel = panel
-
-        self._ohlc_inverse = 1. / OHLC_RATIO
-
-    @property
-    def last_available_dt(self):
-        return self.panel.major_axis[-1]
-
-    def load_raw_arrays(self, columns, start_dt, end_dt, assets):
-        columns = list(columns)
-        dts = self.panel.major_axis
-        index = dts[dts.slice_indexer(start_dt, end_dt)]
-        shape = (len(index), len(assets))
-        results = []
-        for col in columns:
-            outbuf = zeros(shape=shape)
-            for i, asset in enumerate(assets):
-                data = self.panel.loc[asset, start_dt:end_dt, col]
-                data = data.reindex_axis(index).values
-                outbuf[:, i] = data
-            results.append(outbuf)
-        return results
-
-    def spot_price(self, sid, dt, colname):
-        """
-        Parameters
-        ----------
-        sid : int
-            The asset identifier.
-        dt : datetime64-like
-            Midnight of the day for which data is requested.
-        colname : string
-            The price field. e.g. ('open', 'high', 'low', 'close', 'volume')
-
-        Returns
-        -------
-        float
-            The spot price for colname of the given sid on the given day.
-            Raises a NoDataOnDate exception if the given day and sid is before
-            or after the date range of the equity.
-            Returns -1 if the day is within the date range, but the price is
-            0.
-        """
-        return self.panel.loc[sid, dt, colname]
-
-    get_value = spot_price
-
-    def get_last_traded_dt(self, sid, dt):
-        """
-        Parameters
-        ----------
-        sid : int
-            The asset identifier.
-        dt : datetime64-like
-            Midnight of the day for which data is requested.
-
-        Returns
-        -------
-        pd.Timestamp : The last known dt for the asset and dt;
-                       NaT if no trade is found before the given dt.
-        """
-        for ts in self.panel.major_axis[self.panel.major_axis
-                                        .slice_indexer(end=dt)][::-1]:
-            if not pd.isnull(self.panel.loc[sid, ts, 'close']):
-                return ts
-        return NaT
diff --git a/zipline/data/us_equity_pricing.py b/zipline/data/us_equity_pricing.py
index a6d92504..993a20ff 100644
--- a/zipline/data/us_equity_pricing.py
+++ b/zipline/data/us_equity_pricing.py
@@ -35,15 +35,15 @@ from numpy import (
     issubdtype,
     nan,
     uint32,
-    zeros,
 )
 from pandas import (
     DataFrame,
     read_csv,
     Timestamp,
     NaT,
-    isnull,
-    DatetimeIndex)
+    DatetimeIndex
+)
+from pandas.core.datetools import normalize_date
 from pandas.tslib import iNaT
 from six import (
     iteritems,
@@ -746,7 +746,7 @@ class BcolzDailyBarReader(DailyBarReader):
             return price
 
 
-class PanelDailyBarReader(DailyBarReader):
+class PanelBarReader(DailyBarReader):
     """
     Reader for data passed as Panel.
 
@@ -777,7 +777,7 @@ class PanelDailyBarReader(DailyBarReader):
             # Fake volume if it does not exist.
             panel.loc[:, :, 'volume'] = int(1e9)
 
-        self.first_trading_day = panel.major_axis[0]
+        self.first_trading_day = normalize_date(panel.major_axis[0])
         self._calendar = calendar
 
         self.panel = panel
@@ -788,28 +788,28 @@ class PanelDailyBarReader(DailyBarReader):
 
     @property
     def last_available_dt(self):
-        return self._calendar[-1]
+        # Returns the last Panel index that is on the calendar.
+        # The slice end is converted from dt to date string so that
+        # dts on the last day of the calendar get included.
+        return self.panel.major_axis[
+            self.panel.major_axis.slice_indexer(
+                end=self._calendar[-1].strftime('%Y-%m-%d')
+            )
+        ][-1]
 
     @property
     def trading_calendar(self):
         return None
 
-    def load_raw_arrays(self, columns, start_date, end_date, assets):
-        columns = list(columns)
+    def load_raw_arrays(self, columns, start_dt, end_dt, assets):
         cal = self._calendar
-        index = cal[cal.slice_indexer(start_date, end_date)]
-        shape = (len(index), len(assets))
-        results = []
-        for col in columns:
-            outbuf = zeros(shape=shape)
-            for i, asset in enumerate(assets):
-                data = self.panel.loc[asset, start_date:end_date, col]
-                data = data.reindex_axis(index).values
-                outbuf[:, i] = data
-            results.append(outbuf)
-        return results
+        return self.panel.loc[
+            list(assets),
+            start_dt:end_dt,
+            list(columns)
+        ].reindex(major_axis=cal[cal.slice_indexer(start_dt, end_dt)]).values.T
 
-    def spot_price(self, sid, day, colname):
+    def spot_price(self, sid, dt, colname):
         """
         Parameters
         ----------
@@ -829,7 +829,9 @@ class PanelDailyBarReader(DailyBarReader):
             Returns -1 if the day is within the date range, but the price is
             0.
         """
-        return self.panel.loc[sid, day, colname]
+        return self.panel.loc[sid, dt, colname]
+
+    get_value = spot_price
 
     def get_last_traded_dt(self, sid, dt):
         """
@@ -845,12 +847,9 @@ class PanelDailyBarReader(DailyBarReader):
         pd.Timestamp : The last know dt for the asset and dt;
                        NaT if no trade is found before the given dt.
         """
-        while dt in self.panel.major_axis:
-            freq = self.panel.major_axis.freq
-            if not isnull(self.panel.loc[sid, dt, 'close']):
-                return dt
-            dt -= freq
-        else:
+        try:
+            return self.panel.loc[sid, :dt, 'close'].last_valid_index()
+        except IndexError:
             return NaT
 
 

From ab9a899c5bebf889b5672c1922221f71e33453e6 Mon Sep 17 00:00:00 2001
From: Nathan Wolfe <nathanwolferl@gmail.com>
Date: Mon, 18 Jul 2016 18:18:06 -0400
Subject: [PATCH 09/10] MAINT: Switch `PanelBarReader` to take trading calendar
 and freq args

---
 tests/test_panel_daily_bar_reader.py | 14 +++++-----
 zipline/algorithm.py                 | 16 ++++++++----
 zipline/data/us_equity_pricing.py    | 39 +++++++++++++++++-----------
 3 files changed, 43 insertions(+), 26 deletions(-)

diff --git a/tests/test_panel_daily_bar_reader.py b/tests/test_panel_daily_bar_reader.py
index 2e29bd69..39590462 100644
--- a/tests/test_panel_daily_bar_reader.py
+++ b/tests/test_panel_daily_bar_reader.py
@@ -22,13 +22,12 @@ from zipline.data.us_equity_pricing import PanelBarReader
 from zipline.testing import ExplodingObject
 from zipline.testing.fixtures import (
     WithAssetFinder,
-    WithNYSETradingDays,
     ZiplineTestCase,
 )
+from zipline.utils.calendars import get_calendar
 
 
 class TestPanelDailyBarReader(WithAssetFinder,
-                              WithNYSETradingDays,
                               ZiplineTestCase):
 
     START_DATE = pd.Timestamp('2006-01-03', tz='utc')
@@ -39,10 +38,13 @@ class TestPanelDailyBarReader(WithAssetFinder,
         super(TestPanelDailyBarReader, cls).init_class_fixtures()
 
         finder = cls.asset_finder
-        days = cls.trading_days
+        trading_calendar = get_calendar('NYSE')
 
         items = finder.retrieve_all(finder.sids)
-        major_axis = days
+        major_axis = trading_calendar.sessions_in_range(
+            cls.START_DATE,
+            cls.END_DATE
+        )
         minor_axis = ['open', 'high', 'low', 'close', 'volume']
 
         shape = tuple(map(len, [items, major_axis, minor_axis]))
@@ -55,7 +57,7 @@ class TestPanelDailyBarReader(WithAssetFinder,
             minor_axis=minor_axis,
         )
 
-        cls.reader = PanelBarReader(days, cls.panel)
+        cls.reader = PanelBarReader(trading_calendar, cls.panel, 'daily')
 
     def test_spot_price(self):
         panel = self.panel
@@ -83,7 +85,7 @@ class TestPanelDailyBarReader(WithAssetFinder,
         for axis_order in permutations((0, 1, 2)):
             transposed = panel.transpose(*axis_order)
             with self.assertRaises(ValueError) as e:
-                PanelBarReader(unused, transposed)
+                PanelBarReader(unused, transposed, 'daily')
 
             expected = (
                 "Duplicate entries in Panel.{name}: ['a', 'b'].".format(
diff --git a/zipline/algorithm.py b/zipline/algorithm.py
index a84cacb9..8ea302f6 100644
--- a/zipline/algorithm.py
+++ b/zipline/algorithm.py
@@ -619,8 +619,12 @@ class TradingAlgorithm(object):
                 # to be inferred.
                 if overwrite_sim_params:
                     self.sim_params = self.sim_params.create_new(
-                        normalize_date(data.major_axis[0]),
-                        normalize_date(data.major_axis[-1])
+                        self.trading_calendar.minute_to_session_label(
+                            data.major_axis[0]
+                        ),
+                        self.trading_calendar.minute_to_session_label(
+                            data.major_axis[-1]
+                        ),
                     )
 
                     # Assume data is daily if timestamp times are
@@ -649,11 +653,13 @@ class TradingAlgorithm(object):
 
                 if self.sim_params.data_frequency == 'daily':
                     equity_reader_arg = 'equity_daily_reader'
-                    calendar = self.trading_calendar.all_sessions
                 elif self.sim_params.data_frequency == 'minute':
                     equity_reader_arg = 'equity_minute_reader'
-                    calendar = self.trading_calendar.all_minutes
-                equity_reader = PanelBarReader(calendar, copy_panel)
+                equity_reader = PanelBarReader(
+                    self.trading_calendar,
+                    copy_panel,
+                    self.sim_params.data_frequency,
+                )
 
                 self.data_portal = DataPortal(
                     self.asset_finder,
diff --git a/zipline/data/us_equity_pricing.py b/zipline/data/us_equity_pricing.py
index 993a20ff..b1b04680 100644
--- a/zipline/data/us_equity_pricing.py
+++ b/zipline/data/us_equity_pricing.py
@@ -43,7 +43,6 @@ from pandas import (
     NaT,
     DatetimeIndex
 )
-from pandas.core.datetools import normalize_date
 from pandas.tslib import iNaT
 from six import (
     iteritems,
@@ -770,15 +769,34 @@ class PanelBarReader(DailyBarReader):
         The first trading day in the dataset.
     """
     @preprocess(panel=call(verify_indices_all_unique))
-    def __init__(self, calendar, panel):
+    @expect_element(data_frequency={'daily', 'minute'})
+    def __init__(self, trading_calendar, panel, data_frequency):
 
         panel = panel.copy()
         if 'volume' not in panel.minor_axis:
             # Fake volume if it does not exist.
             panel.loc[:, :, 'volume'] = int(1e9)
 
-        self.first_trading_day = normalize_date(panel.major_axis[0])
-        self._calendar = calendar
+        self.trading_calendar = trading_calendar
+        self.first_trading_day = trading_calendar.minute_to_session_label(
+            panel.major_axis[0]
+        )
+        last_trading_day = trading_calendar.minute_to_session_label(
+            panel.major_axis[-1]
+        )
+
+        self._sessions = trading_calendar.sessions_in_range(
+            self.first_trading_day,
+            last_trading_day
+        )
+
+        if data_frequency == 'daily':
+            self._calendar = self._sessions
+        elif data_frequency == 'minute':
+            self._calendar = trading_calendar.minutes_for_sessions_in_range(
+                self.first_trading_day,
+                last_trading_day
+            )
 
         self.panel = panel
 
@@ -788,18 +806,9 @@ class PanelBarReader(DailyBarReader):
 
     @property
     def last_available_dt(self):
-        # Returns the last Panel index that is on the calendar.
-        # The slice end is converted from dt to date string so that
-        # dts on the last day of the calendar get included.
-        return self.panel.major_axis[
-            self.panel.major_axis.slice_indexer(
-                end=self._calendar[-1].strftime('%Y-%m-%d')
-            )
-        ][-1]
+        return self._calendar[-1]
 
-    @property
-    def trading_calendar(self):
-        return None
+    trading_calendar = None
 
     def load_raw_arrays(self, columns, start_dt, end_dt, assets):
         cal = self._calendar

From 0a196c7a693013a581009c3b52ebe74ab211f7fe Mon Sep 17 00:00:00 2001
From: Nathan Wolfe <nathanwolferl@gmail.com>
Date: Mon, 25 Jul 2016 12:53:46 -0400
Subject: [PATCH 10/10] MAINT: Correct PanelBarReader sessions property, expand
 test

`tests/test_panel_daily_bar_reader.py` expanded to cover minute
frequency as well, using the same tests. Renamed to
`test_panel_bar_reader.py`.
---
 ...bar_reader.py => test_panel_bar_reader.py} | 42 +++++++++++++------
 zipline/data/us_equity_pricing.py             |  8 ++--
 2 files changed, 33 insertions(+), 17 deletions(-)
 rename tests/{test_panel_daily_bar_reader.py => test_panel_bar_reader.py} (79%)

diff --git a/tests/test_panel_daily_bar_reader.py b/tests/test_panel_bar_reader.py
similarity index 79%
rename from tests/test_panel_daily_bar_reader.py
rename to tests/test_panel_bar_reader.py
index 39590462..71c4c96b 100644
--- a/tests/test_panel_daily_bar_reader.py
+++ b/tests/test_panel_bar_reader.py
@@ -27,24 +27,20 @@ from zipline.testing.fixtures import (
 from zipline.utils.calendars import get_calendar
 
 
-class TestPanelDailyBarReader(WithAssetFinder,
-                              ZiplineTestCase):
-
-    START_DATE = pd.Timestamp('2006-01-03', tz='utc')
-    END_DATE = pd.Timestamp('2006-02-01', tz='utc')
+class WithPanelBarReader(WithAssetFinder):
 
     @classmethod
     def init_class_fixtures(cls):
-        super(TestPanelDailyBarReader, cls).init_class_fixtures()
+        super(WithPanelBarReader, cls).init_class_fixtures()
 
         finder = cls.asset_finder
         trading_calendar = get_calendar('NYSE')
 
         items = finder.retrieve_all(finder.sids)
-        major_axis = trading_calendar.sessions_in_range(
-            cls.START_DATE,
-            cls.END_DATE
-        )
+        major_axis = (
+            trading_calendar.sessions_in_range if cls.FREQUENCY == 'daily'
+            else trading_calendar.minutes_for_sessions_in_range
+        )(cls.START_DATE, cls.END_DATE)
         minor_axis = ['open', 'high', 'low', 'close', 'volume']
 
         shape = tuple(map(len, [items, major_axis, minor_axis]))
@@ -57,7 +53,7 @@ class TestPanelDailyBarReader(WithAssetFinder,
             minor_axis=minor_axis,
         )
 
-        cls.reader = PanelBarReader(trading_calendar, cls.panel, 'daily')
+        cls.reader = PanelBarReader(trading_calendar, cls.panel, cls.FREQUENCY)
 
     def test_spot_price(self):
         panel = self.panel
@@ -97,6 +93,28 @@ class TestPanelDailyBarReader(WithAssetFinder,
     def test_sessions(self):
         sessions = self.reader.sessions
 
-        self.assertEqual(21, len(sessions))
+        self.assertEqual(self.NUM_SESSIONS, len(sessions))
         self.assertEqual(self.START_DATE, sessions[0])
         self.assertEqual(self.END_DATE, sessions[-1])
+
+
+class TestPanelDailyBarReader(WithPanelBarReader,
+                              ZiplineTestCase):
+
+    FREQUENCY = 'daily'
+
+    START_DATE = pd.Timestamp('2006-01-03', tz='utc')
+    END_DATE = pd.Timestamp('2006-02-01', tz='utc')
+
+    NUM_SESSIONS = 21
+
+
+class TestPanelMinuteBarReader(WithPanelBarReader,
+                               ZiplineTestCase):
+
+    FREQUENCY = 'minute'
+
+    START_DATE = pd.Timestamp('2015-12-23', tz='utc')
+    END_DATE = pd.Timestamp('2015-12-24', tz='utc')
+
+    NUM_SESSIONS = 2
diff --git a/zipline/data/us_equity_pricing.py b/zipline/data/us_equity_pricing.py
index b1b04680..9a05cabd 100644
--- a/zipline/data/us_equity_pricing.py
+++ b/zipline/data/us_equity_pricing.py
@@ -785,13 +785,13 @@ class PanelBarReader(DailyBarReader):
             panel.major_axis[-1]
         )
 
-        self._sessions = trading_calendar.sessions_in_range(
+        self.sessions = trading_calendar.sessions_in_range(
             self.first_trading_day,
             last_trading_day
         )
 
         if data_frequency == 'daily':
-            self._calendar = self._sessions
+            self._calendar = self.sessions
         elif data_frequency == 'minute':
             self._calendar = trading_calendar.minutes_for_sessions_in_range(
                 self.first_trading_day,
@@ -800,9 +800,7 @@ class PanelBarReader(DailyBarReader):
 
         self.panel = panel
 
-    @property
-    def sessions(self):
-        return self._calendar
+    sessions = None
 
     @property
     def last_available_dt(self):