From 115f055c83de73828897cc0cf4d22d91e747771c Mon Sep 17 00:00:00 2001 From: Scott Sanderson Date: Wed, 17 Aug 2016 14:30:09 -0400 Subject: [PATCH] MAINT: Clean up downsampling boilerplate. Consolidate docs and mixin applications into one place. --- tests/pipeline/test_downsampling.py | 37 ++++++-- tests/utils/test_preprocess.py | 6 +- zipline/pipeline/classifiers/classifier.py | 17 +--- zipline/pipeline/downsample_helpers.py | 61 ++++++++++++ zipline/pipeline/factors/factor.py | 16 +--- zipline/pipeline/filters/filter.py | 16 +--- zipline/pipeline/mixins.py | 102 +++++++++++---------- zipline/pipeline/term.py | 15 +-- zipline/utils/input_validation.py | 9 +- zipline/utils/sharedoc.py | 90 ++++++++++++++++++ 10 files changed, 263 insertions(+), 106 deletions(-) create mode 100644 zipline/pipeline/downsample_helpers.py create mode 100644 zipline/utils/sharedoc.py diff --git a/tests/pipeline/test_downsampling.py b/tests/pipeline/test_downsampling.py index e57beed6..357c62f0 100644 --- a/tests/pipeline/test_downsampling.py +++ b/tests/pipeline/test_downsampling.py @@ -85,7 +85,9 @@ class ComputeExtraRowsTestcase(WithTradingSessions, ZiplineTestCase): __fail_fast=True ) def test_yearly(self, base_terms, calendar_name): - downsampled_terms = tuple(t.downsample('Y') for t in base_terms) + downsampled_terms = tuple( + t.downsample('year_start') for t in base_terms + ) all_terms = base_terms + downsampled_terms all_sessions = self.trading_sessions[calendar_name] @@ -188,7 +190,9 @@ class ComputeExtraRowsTestcase(WithTradingSessions, ZiplineTestCase): __fail_fast=True ) def test_quarterly(self, calendar_name, base_terms): - downsampled_terms = tuple(t.downsample('Q') for t in base_terms) + downsampled_terms = tuple( + t.downsample('quarter_start') for t in base_terms + ) all_terms = base_terms + downsampled_terms # This region intersects with Q4 2013, Q1 2014, and Q2 2014. @@ -293,7 +297,9 @@ class ComputeExtraRowsTestcase(WithTradingSessions, ZiplineTestCase): __fail_fast=True ) def test_monthly(self, calendar_name, base_terms): - downsampled_terms = tuple(t.downsample('M') for t in base_terms) + downsampled_terms = tuple( + t.downsample('month_start') for t in base_terms + ) all_terms = base_terms + downsampled_terms # This region intersects with Dec 2013, Jan 2014, and Feb 2014. @@ -398,7 +404,9 @@ class ComputeExtraRowsTestcase(WithTradingSessions, ZiplineTestCase): __fail_fast=True ) def test_weekly(self, calendar_name, base_terms): - downsampled_terms = tuple(t.downsample('W') for t in base_terms) + downsampled_terms = tuple( + t.downsample('week_start') for t in base_terms + ) all_terms = base_terms + downsampled_terms # December 2013 @@ -573,10 +581,10 @@ class DownsampledPipelineTestCase(WithSeededRandomPipelineEngine, start_date, end_date = compute_dates[[0, -1]] pipe = Pipeline({ - 'year': term.downsample(frequency='Y'), - 'quarter': term.downsample(frequency='Q'), - 'month': term.downsample(frequency='M'), - 'week': term.downsample(frequency='W'), + 'year': term.downsample(frequency='year_start'), + 'quarter': term.downsample(frequency='quarter_start'), + 'month': term.downsample(frequency='month_start'), + 'week': term.downsample(frequency='week_start'), }) # Raw values for term, computed each day from 2014 to the end of the @@ -662,3 +670,16 @@ class DownsampledPipelineTestCase(WithSeededRandomPipelineEngine, window_length=5, ) self.check_downsampled_term(sma.quantiles(5)) + + def test_errors_on_bad_downsample_frequency(self): + + f = NDaysAgoFactor(window_length=3) + with self.assertRaises(ValueError) as e: + f.downsample('bad') + + expected = ( + "zipline.pipeline.term.downsample() expected a value in " + "('month_start', 'quarter_start', 'week_start', 'year_start') " + "for argument 'frequency', but got 'bad' instead." + ) + self.assertEqual(str(e.exception), expected) diff --git a/tests/utils/test_preprocess.py b/tests/utils/test_preprocess.py index 343966ab..b9e0162a 100644 --- a/tests/utils/test_preprocess.py +++ b/tests/utils/test_preprocess.py @@ -262,7 +262,11 @@ class PreprocessTestCase(TestCase): expected_message = ( "{qualname}() expected a value in {set_!r}" " for argument 'a', but got 'c' instead." - ).format(set_=set_, qualname=qualname(f)) + ).format( + # We special-case set to show a tuple instead of the set repr. + set_=tuple(set_), + qualname=qualname(f), + ) self.assertEqual(e.exception.args[0], expected_message) def test_expect_dtypes(self): diff --git a/zipline/pipeline/classifiers/classifier.py b/zipline/pipeline/classifiers/classifier.py index 12b3df6b..73bf5a31 100644 --- a/zipline/pipeline/classifiers/classifier.py +++ b/zipline/pipeline/classifiers/classifier.py @@ -14,6 +14,7 @@ from zipline.pipeline.sentinels import NotSpecified from zipline.pipeline.term import ComputableTerm from zipline.utils.compat import unicode from zipline.utils.input_validation import expect_types +from zipline.utils.memoize import classlazyval from zipline.utils.numpy_utils import ( categorical_dtype, int64_dtype, @@ -302,9 +303,9 @@ class Classifier(RestrictedDTypeMixin, ComputableTerm): raise AssertionError("Expected a LabelArray, got %s." % type(data)) return data.as_categorical() - @property + @classlazyval def _downsampled_type(self): - return DownsampledClassifier + return DownsampledMixin.make_downsampled_type(Classifier) class Everything(Classifier): @@ -391,18 +392,6 @@ class Latest(LatestMixin, CustomClassifier): pass -class DownsampledClassifier(DownsampledMixin, Classifier): - """ - A Classifier that defers to another Classifier at lower-than-daily - frequency. - - Parameters - ---------- - term : zipline.Classifier - freq : {'Y', 'Q', 'M', 'W'} - """ - - class InvalidClassifierComparison(TypeError): def __init__(self, classifier, compval): super(InvalidClassifierComparison, self).__init__( diff --git a/zipline/pipeline/downsample_helpers.py b/zipline/pipeline/downsample_helpers.py new file mode 100644 index 00000000..d514fd4f --- /dev/null +++ b/zipline/pipeline/downsample_helpers.py @@ -0,0 +1,61 @@ +""" +Helpers for downsampling code. +""" +from operator import attrgetter + +from zipline.utils.input_validation import expect_element +from zipline.utils.numpy_utils import changed_locations +from zipline.utils.sharedoc import ( + templated_docstring, + PIPELINE_DOWNSAMPLING_FREQUENCY_DOC, +) + +_dt_to_period = { + 'year_start': attrgetter('year'), + 'quarter_start': attrgetter('quarter'), + 'month_start': attrgetter('month'), + 'week_start': attrgetter('week'), +} + +SUPPORTED_DOWNSAMPLE_FREQUENCIES = frozenset(_dt_to_period) + + +expect_downsample_frequency = expect_element( + frequency=SUPPORTED_DOWNSAMPLE_FREQUENCIES, +) + + +@expect_downsample_frequency +@templated_docstring(frequency=PIPELINE_DOWNSAMPLING_FREQUENCY_DOC) +def select_sampling_indices(dates, frequency): + """ + Choose entries from ``dates`` to use for downsampling at ``frequency``. + + Parameters + ---------- + dates : pd.DatetimeIndex + Dates from which to select sample choices. + {frequency} + + Returns + ------- + indices : np.array[int64] + An array condtaining indices of dates on which samples should be taken. + + The resulting index will always include 0 as a sample index, and it + will include the first date of each subsequent year/quarter/month/week, + as determined by ``frequency``. + + Notes + ----- + This function assumes that ``dates`` does not have large gaps. + + In particular, it assumes that the maximum distance between any two entries + in ``dates`` is never greater than a year, which we rely on because we use + ``np.diff(dates.)`` to find dates where the sampling + period has changed. + """ + return changed_locations( + _dt_to_period[frequency](dates), + include_first=True + ) diff --git a/zipline/pipeline/factors/factor.py b/zipline/pipeline/factors/factor.py index c78c074c..436ae981 100644 --- a/zipline/pipeline/factors/factor.py +++ b/zipline/pipeline/factors/factor.py @@ -44,6 +44,7 @@ from zipline.pipeline.term import ComputableTerm, Term from zipline.utils.functional import with_doc, with_name from zipline.utils.input_validation import expect_types from zipline.utils.math_utils import nanmean, nanstd +from zipline.utils.memoize import classlazyval from zipline.utils.numpy_utils import ( bool_dtype, categorical_dtype, @@ -1072,9 +1073,9 @@ class Factor(RestrictedDTypeMixin, ComputableTerm): """ return (-inf < self) & (self < inf) - @property + @classlazyval def _downsampled_type(self): - return DownsampledFactor + return DownsampledMixin.make_downsampled_type(Factor) class NumExprFactor(NumericalExpression, Factor): @@ -1515,17 +1516,6 @@ class Latest(LatestMixin, CustomFactor): out[:] = data[-1] -class DownsampledFactor(DownsampledMixin, Factor): - """ - A Factor that defers to another Factor at lower-than-daily frequency. - - Parameters - ---------- - term : zipline.pipeline.Factor - freq : {'Y', 'Q', 'M', 'W'} - """ - - # Functions to be passed to GroupedRowTransform. These aren't defined inline # because the transformation function is part of the instance hash key. def demean(row): diff --git a/zipline/pipeline/filters/filter.py b/zipline/pipeline/filters/filter.py index c600fc15..13380dcd 100644 --- a/zipline/pipeline/filters/filter.py +++ b/zipline/pipeline/filters/filter.py @@ -33,6 +33,7 @@ from zipline.pipeline.mixins import ( ) from zipline.pipeline.term import ComputableTerm, Term from zipline.utils.input_validation import expect_types +from zipline.utils.memoize import classlazyval from zipline.utils.numpy_utils import bool_dtype, repeat_first_axis @@ -202,9 +203,9 @@ class Filter(RestrictedDTypeMixin, ComputableTerm): ) return retval - @property + @classlazyval def _downsampled_type(self): - return DownsampledFilter + return DownsampledMixin.make_downsampled_type(Filter) class NumExprFilter(NumericalExpression, Filter): @@ -463,17 +464,6 @@ class Latest(LatestMixin, CustomFilter): pass -class DownsampledFilter(DownsampledMixin, Filter): - """ - A Filter that defers to another Filter at lower-than-daily frequency. - - Parameters - ---------- - term : zipline.pipeline.Filter - freq : {'Y', 'Q', 'M', 'W'} - """ - - class SingleAsset(Filter): """ A Filter that computes to True only for the given asset. diff --git a/zipline/pipeline/mixins.py b/zipline/pipeline/mixins.py index 9fa0ad6a..7302f7cc 100644 --- a/zipline/pipeline/mixins.py +++ b/zipline/pipeline/mixins.py @@ -1,7 +1,7 @@ """ Mixins classes for use with Filters and Factors. """ -from operator import attrgetter +from textwrap import dedent from numpy import ( array, @@ -17,10 +17,18 @@ from zipline.errors import ( NoFurtherDataError, ) from zipline.utils.control_flow import nullctx -from zipline.utils.input_validation import expect_element, expect_types -from zipline.utils.numpy_utils import changed_locations +from zipline.utils.input_validation import expect_types +from zipline.utils.sharedoc import ( + format_docstring, + PIPELINE_DOWNSAMPLING_FREQUENCY_DOC, +) from zipline.utils.pandas_utils import nearest_unequal_elements + +from .downsample_helpers import ( + select_sampling_indices, + expect_downsample_frequency, +) from .sentinels import NotSpecified from .term import Term @@ -232,49 +240,6 @@ class LatestMixin(SingleInputMixin): ) -_dt_to_period = { - 'Y': attrgetter('year'), - 'Q': attrgetter('quarter'), - 'M': attrgetter('month'), - 'W': attrgetter('week'), -} - - -def select_sampling_indices(dates, frequency): - """ - Choose entries from ``dates`` to use for downsampling at ``frequency``. - - Parameters - ---------- - dates : pd.DatetimeIndex - Dates from which to select sample choices. - frequency : {'Y', 'Q', 'M', 'W'} - Frequency at which samples are to be taken. - - Returns - ------- - indices : np.array[int64] - An array condtaining indices of dates on which samples should be taken. - - The resulting index will always include 0 as a sample index, and it - will include the first date of each subsequent year/quarter/month/week, - as determined by ``frequency``. - - Notes - ----- - This function assumes that ``dates`` does not have large gaps. - - In particular, it assumes that the maximum distance between any two entries - in ``dates`` is never greater than a year, which we rely on because we use - ``np.diff(dates.{quarter,month,week})`` to find dates where the sampling - period has changed. - """ - return changed_locations( - _dt_to_period[frequency](dates), - include_first=True - ) - - class DownsampledMixin(StandardOutputs): """ Mixin for behavior shared by Downsampled{Factor,Filter,Classifier} @@ -291,7 +256,7 @@ class DownsampledMixin(StandardOutputs): window_safe = False @expect_types(term=Term) - @expect_element(frequency=frozenset(_dt_to_period)) + @expect_downsample_frequency def __new__(cls, term, frequency): return super(DownsampledMixin, cls).__new__( cls, @@ -400,6 +365,17 @@ class DownsampledMixin(StandardOutputs): real_compute = self._wrapped_term._compute + # Inputs will contain different kinds of values depending on whether or + # not we're a windowed computation. + + # If we're windowed, then `inputs` is a list of iterators of ndarrays. + # If we're not windowed, then `inputs` is just a list of ndarrays. + # There are two things we care about doing with the input: + # 1. Preparing an input to be passed to our wrapped term. + # 2. Skipping an input if we're going to use an already-computed row. + # We perform these actions differently based on the expected kind of + # input, and we encapsulate these actions with closures so that we + # don't clutter the code below with lots of branching. if self.windowed: # If we're windowed, inputs are stateful AdjustedArrays. We don't # need to do any preparation before forwarding to real_compute, but @@ -412,8 +388,8 @@ class DownsampledMixin(StandardOutputs): next(w) else: # If we're not windowed, inputs are just ndarrays. We need to - # slice off one row when forwarding to real_compute, but we don't - # need to do anything to skip an input. + # slice out a single row when forwarding to real_compute, but we + # don't need to do anything to skip an input. def prepare_inputs(): # i is the loop iteration variable below. return [a[[i]] for a in inputs] @@ -455,3 +431,31 @@ class DownsampledMixin(StandardOutputs): # Concatenate stored results. return vstack(results) + + @classmethod + def make_downsampled_type(cls, other_base): + """ + Factory for making Downsampled{Filter,Factor,Classifier}. + """ + docstring = dedent( + """ + A {t} that defers to another {t} at lower-than-daily frequency. + + Parameters + ---------- + term : {t} + {{frequency}} + """ + ).format(t=other_base.__name__) + + doc = format_docstring( + owner_name=other_base.__name__, + docstring=docstring, + formatters={'frequency': PIPELINE_DOWNSAMPLING_FREQUENCY_DOC}, + ) + + return type( + 'Downsampled' + other_base.__name__, + (cls, other_base,), + {'__doc__': doc}, + ) diff --git a/zipline/pipeline/term.py b/zipline/pipeline/term.py index 98941fe8..6cecd147 100644 --- a/zipline/pipeline/term.py +++ b/zipline/pipeline/term.py @@ -37,7 +37,12 @@ from zipline.utils.numpy_utils import ( datetime64ns_dtype, default_missing_value_for_dtype, ) +from zipline.utils.sharedoc import ( + templated_docstring, + PIPELINE_DOWNSAMPLING_FREQUENCY_DOC, +) +from .downsample_helpers import expect_downsample_frequency from .sentinels import NotSpecified @@ -594,19 +599,15 @@ class ComputableTerm(Term): "for instances of %s." % type(self).__name__ ) + @expect_downsample_frequency + @templated_docstring(frequency=PIPELINE_DOWNSAMPLING_FREQUENCY_DOC) def downsample(self, frequency): """ Make a term that computes from ``self`` at lower-than-daily frequency. Parameters ---------- - frequency : str, {'Y', 'Q', 'M', 'W'} - A string indicating the desired sampling rate. - 'Y' -> sample on the first trading day of each calendar year - 'Q' -> sample on the first trading day of - January, April, July, and October - 'M' -> sample on the first trading day of each month - 'W' -> sample on the first trading day of each week + {frequency} """ return self._downsampled_type(term=self, frequency=frequency) diff --git a/zipline/utils/input_validation.py b/zipline/utils/input_validation.py index 6c5298a4..c8347cd3 100644 --- a/zipline/utils/input_validation.py +++ b/zipline/utils/input_validation.py @@ -483,10 +483,17 @@ def expect_element(*_pos, **named): raise TypeError("expect_element() only takes keyword arguments.") def _expect_element(collection): + if isinstance(collection, (set, frozenset)): + # Special case the error message for set and frozen set to make it + # less verbose. + collection_for_error_message = tuple(sorted(collection)) + else: + collection_for_error_message = collection + template = ( "%(funcname)s() expected a value in {collection} " "for argument '%(argname)s', but got %(actual)s instead." - ).format(collection=collection) + ).format(collection=collection_for_error_message) return make_check( ValueError, template, diff --git a/zipline/utils/sharedoc.py b/zipline/utils/sharedoc.py new file mode 100644 index 00000000..8ba028c6 --- /dev/null +++ b/zipline/utils/sharedoc.py @@ -0,0 +1,90 @@ +""" +Shared docstrings for parameters that should be documented identically +across different functions. +""" +import re +from six import iteritems +from textwrap import dedent + +PIPELINE_DOWNSAMPLING_FREQUENCY_DOC = dedent( + """\ + frequency : {'year_start', 'quarter_start', 'month_start', 'week_start'} + A string indicating desired sampling dates: + + 'year_start' -> first trading day of each year + 'quarter_start' -> first trading day of January, April, July, October + 'month_start' -> first trading day of each month + 'week_start' -> first trading_day of each week + """ +) + + +def pad_lines(prefix, s): + """Apply a prefix to each line in s.""" + return '\n'.join(prefix + line for line in s.splitlines()) + + +def format_docstring(owner_name, docstring, formatters): + """ + Template ``formatters`` into ``docstring``. + + Parameters + ---------- + owner_name : str + The name of the function or class whose docstring is being templated. + Only used for error messages. + docstring : str + The docstring to template. + formatters : dict[str -> str] + Parameters for a a str.format() call on ``docstring``. + + Multi-line values in ``formatters`` will have leading whitespace padded + to match the leading whitespace of the substitution string. + """ + # Build a dict of parameters to a vanilla format() call by searching for + # each entry in **formatters and applying any leading whitespace to each + # line in the desired substitution. + format_params = {} + for target, doc_for_target in iteritems(formatters): + # Search for '{name}', with optional leading whitespace. + regex = re.compile('^(\s*)' + '({' + target + '})$', re.MULTILINE) + matches = regex.findall(docstring) + if not matches: + raise ValueError( + "Couldn't find template for parameter {!r} in docstring " + "for {}." + "\nParameter name must be alone on a line surrounded by " + "braces.".format(target, owner_name), + ) + elif len(matches) > 1: + raise ValueError( + "Couldn't found multiple templates for parameter {!r}" + "in docstring for {}." + "\nParameter should only appear once.".format( + target, owner_name + ) + ) + + (leading_whitespace, _) = matches[0] + format_params[target] = pad_lines(leading_whitespace, doc_for_target) + + return docstring.format(**format_params) + + +def templated_docstring(**docs): + """ + Decorator allowing the use of templated docstrings. + + Usage + ----- + >>> @templated_docstring(foo='bar') + ... def my_func(self, foo): + ... '''{foo}''' + ... + >>> my_func.__doc__ + 'bar' + """ + def decorator(f): + f.__doc__ = format_docstring(f.__name__, f.__doc__, docs) + return f + return decorator