From 1bf33f9ee0677bb52678d3f51a7e790d79cf2668 Mon Sep 17 00:00:00 2001 From: Scott Sanderson Date: Tue, 19 Jan 2016 12:58:51 -0500 Subject: [PATCH] TEST: Add isolated tests for .latest. --- tests/pipeline/test_column.py | 63 ++++++++++++++++++++ zipline/pipeline/loaders/synthetic.py | 84 +++++++++++++++++++++++++++ zipline/pipeline/loaders/testing.py | 21 +++++++ zipline/utils/test_utils.py | 60 ++++++++++++++++++- 4 files changed, 227 insertions(+), 1 deletion(-) create mode 100644 tests/pipeline/test_column.py create mode 100644 zipline/pipeline/loaders/testing.py diff --git a/tests/pipeline/test_column.py b/tests/pipeline/test_column.py new file mode 100644 index 00000000..142e97a6 --- /dev/null +++ b/tests/pipeline/test_column.py @@ -0,0 +1,63 @@ +""" +Tests BoundColumn attributes and methods. +""" +from contextlib2 import ExitStack +from unittest import TestCase + +from pandas import date_range, DataFrame +from pandas.util.testing import assert_frame_equal + +from zipline.pipeline import Pipeline +from zipline.pipeline.data.testing import TestingDataSet as TDS +from zipline.utils.test_utils import chrange, temp_pipeline_engine + + +class LatestTestCase(TestCase): + + @classmethod + def setUpClass(cls): + cls._stack = stack = ExitStack() + cls.calendar = cal = date_range('2014', '2015', freq='D', tz='UTC') + cls.sids = list(range(5)) + cls.engine = stack.enter_context( + temp_pipeline_engine( + cal, + cls.sids, + random_seed=100, + symbols=chrange('A', 'E'), + ), + ) + cls.assets = cls.engine._finder.retrieve_all(cls.sids) + + @classmethod + def tearDownClass(cls): + cls._stack.close() + + def expected_latest(self, column, slice_): + loader = self.engine.get_loader(column) + return DataFrame( + loader.values(column.dtype, self.calendar, self.sids)[slice_], + index=self.calendar[slice_], + columns=self.sids, + ) + + def test_latest(self): + pipe = Pipeline( + columns={ + name: getattr(TDS, name + '_col').latest + # Intentionally not including int and bool because they're not + # yet supported. + for name in ('float', 'datetime') + } + ) + + cal_slice = slice(20, 40) + dates_to_test = self.calendar[cal_slice] + result = self.engine.run_pipeline( + pipe, + dates_to_test[0], + dates_to_test[-1], + ) + float_result = result.float.unstack() + expected_float_result = self.expected_latest(TDS.float_col, cal_slice) + assert_frame_equal(float_result, expected_float_result) diff --git a/zipline/pipeline/loaders/synthetic.py b/zipline/pipeline/loaders/synthetic.py index 8c50d869..1d983a53 100644 --- a/zipline/pipeline/loaders/synthetic.py +++ b/zipline/pipeline/loaders/synthetic.py @@ -12,6 +12,7 @@ from numpy import ( iinfo, uint32, ) +from numpy.random import RandomState from pandas import DataFrame, Timestamp from six import iteritems from sqlite3 import connect as sqlite3_connect @@ -24,6 +25,12 @@ from zipline.data.us_equity_pricing import ( SQLiteAdjustmentWriter, US_EQUITY_PRICING_BCOLZ_COLUMNS, ) +from zipline.utils.numpy_utils import ( + bool_dtype, + datetime64ns_dtype, + float64_dtype, + int64_dtype, +) UINT_32_MAX = iinfo(uint32).max @@ -110,6 +117,83 @@ class EyeLoader(PrecomputedLoader): ) +class SeededRandomLoader(PrecomputedLoader): + """ + A PrecomputedLoader that emits arrays randomly-generated with a given seed. + + Parameters + ---------- + seed : int + Seed for numpy.random.RandomState. + columns : list[BoundColumn] + Columns that this loader should know about. + dates : iterable[datetime-like] + Same as PrecomputedLoader. + sids : iterable[int-like] + Same as PrecomputedLoader + """ + + def __init__(self, seed, columns, dates, sids): + self._seed = seed + super(SeededRandomLoader, self).__init__( + {c: self.values(c.dtype, dates, sids) for c in columns}, + dates, + sids, + ) + + def values(self, dtype, dates, sids): + """ + Make a random array of shape (len(dates), len(sids)) with ``dtype``. + """ + shape = (len(dates), len(sids)) + return { + datetime64ns_dtype: self._datetime_values, + float64_dtype: self._float_values, + int64_dtype: self._int_values, + bool_dtype: self._bool_values, + }[dtype](shape) + + @property + def state(self): + """ + Make a new RandomState from our seed. + + This ensures that every call to _*_values produces the same output + every time for a given SeededRandomLoader instance. + """ + return RandomState(self._seed) + + def _float_values(self, shape): + """ + Return uniformly-distributed floats between -0.0 and 100.0. + """ + return self.state.uniform(low=0.0, high=100.0, size=shape) + + def _int_values(self, shape): + """ + Return uniformly-distributed integers between 0 and 100. + """ + return self.state.random_integers(low=0, high=100, size=shape) + + def _datetime_values(self, shape): + """ + Return uniformly-distributed dates in 2014. + """ + start = Timestamp('2014', tz='UTC').asm8 + offsets = self.state.random_integers( + low=0, + high=364, + size=shape, + ).astype('timedelta64[D]') + return start + offsets + + def _bool_values(self, shape): + """ + Return uniformly-distributed True/False values. + """ + return self.state.randn(*shape) < 0 + + class SyntheticDailyBarWriter(BcolzDailyBarWriter): """ Bcolz writer that creates synthetic data based on asset lifetime metadata. diff --git a/zipline/pipeline/loaders/testing.py b/zipline/pipeline/loaders/testing.py new file mode 100644 index 00000000..cd4cd62f --- /dev/null +++ b/zipline/pipeline/loaders/testing.py @@ -0,0 +1,21 @@ +""" +Loaders for zipline.pipeline.data.testing datasets. +""" +from .synthetic import EyeLoader, SeededRandomLoader +from ..data.testing import TestingDataSet + + +def make_eye_loader(dates, sids): + """ + Make a PipelineLoader that emits np.eye arrays for the columns in + ``TestingDataSet``. + """ + return EyeLoader(TestingDataSet.columns, dates, sids) + + +def make_seeded_random_loader(seed, dates, sids): + """ + Make a PipelineLoader that emits random arrays seeded with `seed` for the + columns in ``TestingDataSet``. + """ + return SeededRandomLoader(seed, TestingDataSet.columns, dates, sids) diff --git a/zipline/utils/test_utils.py b/zipline/utils/test_utils.py index 8ea3aa00..641f2316 100644 --- a/zipline/utils/test_utils.py +++ b/zipline/utils/test_utils.py @@ -25,6 +25,8 @@ from zipline.assets import AssetFinder from zipline.assets.asset_writer import AssetDBWriterFromDataFrame from zipline.assets.futures import CME_CODE_TO_MONTH from zipline.finance.order import ORDER_STATUS +from zipline.pipeline.engine import SimplePipelineEngine +from zipline.pipeline.loaders.testing import make_seeded_random_loader from zipline.utils import security_list from zipline.utils.tradingcalendar import trading_days @@ -238,6 +240,31 @@ def all_subindices(index): ) +def chrange(start, stop): + """ + Construct an iterable of length-1 strings beginning with `start` and ending + with `stop`. + + Parameters + ---------- + start : str + The first character. + stop : str + The last character. + + Returns + ------- + chars: iterable[str] + Iterable of strings beginning with start and ending with stop. + + Example + ------- + >>> list(chrange('A', 'C')) + ['A', 'B', 'C'] + """ + return map(chr, range(ord(start), ord(stop) + 1)) + + def make_rotating_equity_info(num_assets, first_start, frequency, @@ -296,7 +323,7 @@ def make_simple_equity_info(sids, start_date, end_date, symbols=None): sids : array-like of int start_date : pd.Timestamp end_date : pd.Timestamp - symbols : list, optional + symbols : list, optionaln Symbols to use for the assets. If not provided, symbols are generated from the sequence 'A', 'B', ... @@ -664,3 +691,34 @@ def gen_calendars(start, stop, critical_dates): # Also test with the trading calendar. yield (trading_days[trading_days.slice_indexer(start, stop)],) + + +@contextmanager +def temp_pipeline_engine(calendar, sids, random_seed, symbols=None): + """ + A contextManager that yields a SimplePipelineEngine holding a reference to + an AssetFinder generated via tmp_asset_finder. + + Parameters + ---------- + calendar : pd.DatetimeIndex + Calendar to pass to the constructed PipelineEngine. + sids : iterable[int] + Sids to use for the temp asset finder. + random_seed : int + Integer used to seed instances of SeededRandomLoader. + symbols : iterable[str], optional + Symbols for constructed assets. Forwarded to make_simple_equity_info. + """ + equity_info = make_simple_equity_info( + sids=sids, + start_date=calendar[0], + end_date=calendar[-1], + symbols=symbols, + ) + + loader = make_seeded_random_loader(random_seed, calendar, sids) + get_loader = lambda column: loader + + with tmp_asset_finder(equities=equity_info) as finder: + yield SimplePipelineEngine(get_loader, calendar, finder)