diff --git a/zipline/data/loader.py b/zipline/data/loader.py index 8e736c15..97b8c825 100644 --- a/zipline/data/loader.py +++ b/zipline/data/loader.py @@ -21,7 +21,6 @@ from pandas_datareader.data import DataReader import pytz from six import iteritems from six.moves.urllib_error import HTTPError -import zipline from .benchmarks import get_benchmark_returns from . import treasuries, treasuries_can @@ -44,9 +43,6 @@ INDEX_MAPPING = { (treasuries, 'treasury_curves.csv', 'www.federalreserve.gov'), } -zipline_dir = os.path.dirname(zipline.__file__) -MARKET_DATA_DIR = os.path.join(zipline_dir, 'resources', 'market_data') - ONE_HOUR = pd.Timedelta(hours=1) @@ -198,9 +194,9 @@ def ensure_benchmark_data(symbol, first_date, last_date, now, trading_day): A trading day delta. Used to find the day before first_date so we can get the close of the day prior to first_date. - We attempt to download data unless we already have data stored in source or - in the data cache for `symbol` whose first entry is before or on - `first_date` and whose last entry is on or after `last_date`. + We attempt to download data unless we already have data stored at the data + cache for `symbol` whose first entry is before or on `first_date` and whose + last entry is on or after `last_date`. If we perform a download and the cache criteria are not satisfied, we wait at least one hour before attempting a redownload. This is determined by @@ -249,9 +245,9 @@ def ensure_treasury_data(symbol, first_date, last_date, now): re-download data that isn't available due to scheduling quirks or other failures. - We attempt to download data unless we already have data stored in source or - in the cache for `module_name` whose first entry is before or on - `first_date` and whose last entry is on or after `last_date`. + We attempt to download data unless we already have data stored in the cache + for `module_name` whose first entry is before or on `first_date` and whose + last entry is on or after `last_date`. If we perform a download and the cache criteria are not satisfied, we wait at least one hour before attempting a redownload. This is determined by @@ -287,23 +283,7 @@ def _load_cached_data(filename, first_date, last_date, now, resource_name): else: from_csv = pd.DataFrame.from_csv - # First try to retrieve the data from a static csv file in source. - source_path = os.path.join(MARKET_DATA_DIR, filename) - try: - data = from_csv(source_path).tz_localize('UTC') - if has_data_for_dates(data, first_date, last_date): - return data - except (OSError, IOError, ValueError) as e: - # These can all be raised by various versions of pandas on various - # classes of malformed input. - logger.info( - 'Loading data from source path {path!r} failed with error ' - '[{error}].', - path=source_path, - error=e, - ) - - # If the data in source was missing any dates then check the cache. + # Path for the cache. path = get_data_filepath(filename) # If the path does not exist, it means the first download has not happened diff --git a/zipline/testing/fixtures.py b/zipline/testing/fixtures.py index febf4e32..48bd4ae8 100644 --- a/zipline/testing/fixtures.py +++ b/zipline/testing/fixtures.py @@ -12,6 +12,8 @@ import responses from .core import ( create_daily_bar_data, create_minute_bar_data, + make_simple_equity_info, + tmp_asset_finder, tmp_dir, ) from ..data.data_portal import ( @@ -22,19 +24,6 @@ from ..data.data_portal import ( from ..data.loader import ( get_benchmark_filename, INDEX_MAPPING, - MARKET_DATA_DIR, -) -from ..data.resample import ( - minute_frame_to_session_frame, - MinuteResampleSessionBarReader -) -from ..data.us_equity_pricing import ( - SQLiteAdjustmentReader, - SQLiteAdjustmentWriter, -) -from ..data.us_equity_pricing import ( - BcolzDailyBarReader, - BcolzDailyBarWriter, ) from ..data.minute_bars import ( BcolzMinuteBarReader, @@ -42,12 +31,22 @@ from ..data.minute_bars import ( US_EQUITIES_MINUTES_PER_DAY, FUTURES_MINUTES_PER_DAY, ) - +from ..data.resample import ( + minute_frame_to_session_frame, + MinuteResampleSessionBarReader +) +from ..data.us_equity_pricing import ( + BcolzDailyBarReader, + BcolzDailyBarWriter, + SQLiteAdjustmentReader, + SQLiteAdjustmentWriter, +) from ..finance.trading import TradingEnvironment from ..utils import factory from ..utils.classproperty import classproperty from ..utils.final import FinalMeta, final -from .core import tmp_asset_finder, make_simple_equity_info + +import zipline from zipline.assets import Equity, Future from zipline.finance.asset_restrictions import NoRestrictions from zipline.pipeline import SimplePipelineEngine @@ -57,6 +56,8 @@ from zipline.utils.calendars import ( get_calendar, register_calendar) +zipline_dir = os.path.dirname(zipline.__file__) + class ZiplineTestCase(with_metaclass(FinalMeta, TestCase)): """ @@ -487,6 +488,7 @@ class WithTradingEnvironment(WithAssetFinder, :class:`zipline.finance.trading.TradingEnvironment` """ TRADING_ENV_FUTURE_CHAIN_PREDICATES = None + MARKET_DATA_DIR = os.path.join(zipline_dir, 'resources', 'market_data') @classmethod def make_load_function(cls): @@ -494,12 +496,12 @@ class WithTradingEnvironment(WithAssetFinder, symbol = '^GSPC' filename = get_benchmark_filename(symbol) - source_path = os.path.join(MARKET_DATA_DIR, filename) + source_path = os.path.join(cls.MARKET_DATA_DIR, filename) benchmark_returns = \ pd.Series.from_csv(source_path).tz_localize('UTC') filename = INDEX_MAPPING[symbol][1] - source_path = os.path.join(MARKET_DATA_DIR, filename) + source_path = os.path.join(cls.MARKET_DATA_DIR, filename) treasury_curves = \ pd.DataFrame.from_csv(source_path).tz_localize('UTC') @@ -525,12 +527,12 @@ class WithTradingEnvironment(WithAssetFinder, static_end=static_end_date, given_start=cls.START_DATE.date(), given_end=cls.END_DATE.date(), - resource_dir=MARKET_DATA_DIR, + resource_dir=cls.MARKET_DATA_DIR, ) ) if cls.START_DATE.date() < static_start_date or \ cls.END_DATE.date() > static_end_date: - raise Warning(warning_message) + raise AssertionError(warning_message) return benchmark_returns, treasury_curves return load