REV: Only use benchmark csv files in source for testing

zipline/data/loader.py

@@ -21,7 +21,6 @@ from pandas_datareader.data import DataReader
 import pytz
 from six import iteritems
 from six.moves.urllib_error import HTTPError
-import zipline
 
 from .benchmarks import get_benchmark_returns
 from . import treasuries, treasuries_can
@@ -44,9 +43,6 @@ INDEX_MAPPING = {
     (treasuries, 'treasury_curves.csv', 'www.federalreserve.gov'),
 }
 
-zipline_dir = os.path.dirname(zipline.__file__)
-MARKET_DATA_DIR = os.path.join(zipline_dir, 'resources', 'market_data')
-
 ONE_HOUR = pd.Timedelta(hours=1)
 
 
@@ -198,9 +194,9 @@ def ensure_benchmark_data(symbol, first_date, last_date, now, trading_day):
         A trading day delta.  Used to find the day before first_date so we can
         get the close of the day prior to first_date.
 
-    We attempt to download data unless we already have data stored in source or
-    in the data cache for `symbol` whose first entry is before or on
-    `first_date` and whose last entry is on or after `last_date`.
+    We attempt to download data unless we already have data stored at the data
+    cache for `symbol` whose first entry is before or on `first_date` and whose
+    last entry is on or after `last_date`.
 
     If we perform a download and the cache criteria are not satisfied, we wait
     at least one hour before attempting a redownload.  This is determined by
@@ -249,9 +245,9 @@ def ensure_treasury_data(symbol, first_date, last_date, now):
         re-download data that isn't available due to scheduling quirks or other
         failures.
 
-    We attempt to download data unless we already have data stored in source or
-    in the cache for `module_name` whose first entry is before or on
-    `first_date` and whose last entry is on or after `last_date`.
+    We attempt to download data unless we already have data stored in the cache
+    for `module_name` whose first entry is before or on `first_date` and whose
+    last entry is on or after `last_date`.
 
     If we perform a download and the cache criteria are not satisfied, we wait
     at least one hour before attempting a redownload.  This is determined by
@@ -287,23 +283,7 @@ def _load_cached_data(filename, first_date, last_date, now, resource_name):
     else:
         from_csv = pd.DataFrame.from_csv
 
-    # First try to retrieve the data from a static csv file in source.
-    source_path = os.path.join(MARKET_DATA_DIR, filename)
-    try:
-        data = from_csv(source_path).tz_localize('UTC')
-        if has_data_for_dates(data, first_date, last_date):
-            return data
-    except (OSError, IOError, ValueError) as e:
-        # These can all be raised by various versions of pandas on various
-        # classes of malformed input.
-        logger.info(
-            'Loading data from source path {path!r} failed with error '
-            '[{error}].',
-            path=source_path,
-            error=e,
-        )
-
-    # If the data in source was missing any dates then check the cache.
+    # Path for the cache.
     path = get_data_filepath(filename)
 
     # If the path does not exist, it means the first download has not happened

zipline/testing/fixtures.py

@@ -12,6 +12,8 @@ import responses
 from .core import (
     create_daily_bar_data,
     create_minute_bar_data,
+    make_simple_equity_info,
+    tmp_asset_finder,
     tmp_dir,
 )
 from ..data.data_portal import (
@@ -22,19 +24,6 @@ from ..data.data_portal import (
 from ..data.loader import (
     get_benchmark_filename,
     INDEX_MAPPING,
-    MARKET_DATA_DIR,
-)
-from ..data.resample import (
-    minute_frame_to_session_frame,
-    MinuteResampleSessionBarReader
-)
-from ..data.us_equity_pricing import (
-    SQLiteAdjustmentReader,
-    SQLiteAdjustmentWriter,
-)
-from ..data.us_equity_pricing import (
-    BcolzDailyBarReader,
-    BcolzDailyBarWriter,
 )
 from ..data.minute_bars import (
     BcolzMinuteBarReader,
@@ -42,12 +31,22 @@ from ..data.minute_bars import (
     US_EQUITIES_MINUTES_PER_DAY,
     FUTURES_MINUTES_PER_DAY,
 )
-
+from ..data.resample import (
+    minute_frame_to_session_frame,
+    MinuteResampleSessionBarReader
+)
+from ..data.us_equity_pricing import (
+    BcolzDailyBarReader,
+    BcolzDailyBarWriter,
+    SQLiteAdjustmentReader,
+    SQLiteAdjustmentWriter,
+)
 from ..finance.trading import TradingEnvironment
 from ..utils import factory
 from ..utils.classproperty import classproperty
 from ..utils.final import FinalMeta, final
-from .core import tmp_asset_finder, make_simple_equity_info
+
+import zipline
 from zipline.assets import Equity, Future
 from zipline.finance.asset_restrictions import NoRestrictions
 from zipline.pipeline import SimplePipelineEngine
@@ -57,6 +56,8 @@ from zipline.utils.calendars import (
     get_calendar,
     register_calendar)
 
+zipline_dir = os.path.dirname(zipline.__file__)
+
 
 class ZiplineTestCase(with_metaclass(FinalMeta, TestCase)):
     """
@@ -487,6 +488,7 @@ class WithTradingEnvironment(WithAssetFinder,
     :class:`zipline.finance.trading.TradingEnvironment`
     """
     TRADING_ENV_FUTURE_CHAIN_PREDICATES = None
+    MARKET_DATA_DIR = os.path.join(zipline_dir, 'resources', 'market_data')
 
     @classmethod
     def make_load_function(cls):
@@ -494,12 +496,12 @@ class WithTradingEnvironment(WithAssetFinder,
             symbol = '^GSPC'
 
             filename = get_benchmark_filename(symbol)
-            source_path = os.path.join(MARKET_DATA_DIR, filename)
+            source_path = os.path.join(cls.MARKET_DATA_DIR, filename)
             benchmark_returns = \
                 pd.Series.from_csv(source_path).tz_localize('UTC')
 
             filename = INDEX_MAPPING[symbol][1]
-            source_path = os.path.join(MARKET_DATA_DIR, filename)
+            source_path = os.path.join(cls.MARKET_DATA_DIR, filename)
             treasury_curves = \
                 pd.DataFrame.from_csv(source_path).tz_localize('UTC')
 
@@ -525,12 +527,12 @@ class WithTradingEnvironment(WithAssetFinder,
                     static_end=static_end_date,
                     given_start=cls.START_DATE.date(),
                     given_end=cls.END_DATE.date(),
-                    resource_dir=MARKET_DATA_DIR,
+                    resource_dir=cls.MARKET_DATA_DIR,
                 )
             )
             if cls.START_DATE.date() < static_start_date or \
                     cls.END_DATE.date() > static_end_date:
-                raise Warning(warning_message)
+                raise AssertionError(warning_message)
 
             return benchmark_returns, treasury_curves
         return load