MAINT: Clean up downsampling boilerplate.

Consolidate docs and mixin applications into one place.

tests/pipeline/test_downsampling.py

@@ -85,7 +85,9 @@ class ComputeExtraRowsTestcase(WithTradingSessions, ZiplineTestCase):
         __fail_fast=True
     )
     def test_yearly(self, base_terms, calendar_name):
-        downsampled_terms = tuple(t.downsample('Y') for t in base_terms)
+        downsampled_terms = tuple(
+            t.downsample('year_start') for t in base_terms
+        )
         all_terms = base_terms + downsampled_terms
 
         all_sessions = self.trading_sessions[calendar_name]
@@ -188,7 +190,9 @@ class ComputeExtraRowsTestcase(WithTradingSessions, ZiplineTestCase):
         __fail_fast=True
     )
     def test_quarterly(self, calendar_name, base_terms):
-        downsampled_terms = tuple(t.downsample('Q') for t in base_terms)
+        downsampled_terms = tuple(
+            t.downsample('quarter_start') for t in base_terms
+        )
         all_terms = base_terms + downsampled_terms
 
         # This region intersects with Q4 2013, Q1 2014, and Q2 2014.
@@ -293,7 +297,9 @@ class ComputeExtraRowsTestcase(WithTradingSessions, ZiplineTestCase):
         __fail_fast=True
     )
     def test_monthly(self, calendar_name, base_terms):
-        downsampled_terms = tuple(t.downsample('M') for t in base_terms)
+        downsampled_terms = tuple(
+            t.downsample('month_start') for t in base_terms
+        )
         all_terms = base_terms + downsampled_terms
 
         # This region intersects with Dec 2013, Jan 2014, and Feb 2014.
@@ -398,7 +404,9 @@ class ComputeExtraRowsTestcase(WithTradingSessions, ZiplineTestCase):
         __fail_fast=True
     )
     def test_weekly(self, calendar_name, base_terms):
-        downsampled_terms = tuple(t.downsample('W') for t in base_terms)
+        downsampled_terms = tuple(
+            t.downsample('week_start') for t in base_terms
+        )
         all_terms = base_terms + downsampled_terms
 
         #    December 2013
@@ -573,10 +581,10 @@ class DownsampledPipelineTestCase(WithSeededRandomPipelineEngine,
         start_date, end_date = compute_dates[[0, -1]]
 
         pipe = Pipeline({
-            'year': term.downsample(frequency='Y'),
-            'quarter': term.downsample(frequency='Q'),
-            'month': term.downsample(frequency='M'),
-            'week': term.downsample(frequency='W'),
+            'year': term.downsample(frequency='year_start'),
+            'quarter': term.downsample(frequency='quarter_start'),
+            'month': term.downsample(frequency='month_start'),
+            'week': term.downsample(frequency='week_start'),
         })
 
         # Raw values for term, computed each day from 2014 to the end of the
@@ -662,3 +670,16 @@ class DownsampledPipelineTestCase(WithSeededRandomPipelineEngine,
             window_length=5,
         )
         self.check_downsampled_term(sma.quantiles(5))
+
+    def test_errors_on_bad_downsample_frequency(self):
+
+        f = NDaysAgoFactor(window_length=3)
+        with self.assertRaises(ValueError) as e:
+            f.downsample('bad')
+
+        expected = (
+            "zipline.pipeline.term.downsample() expected a value in "
+            "('month_start', 'quarter_start', 'week_start', 'year_start') "
+            "for argument 'frequency', but got 'bad' instead."
+        )
+        self.assertEqual(str(e.exception), expected)

tests/utils/test_preprocess.py

@@ -262,7 +262,11 @@ class PreprocessTestCase(TestCase):
         expected_message = (
             "{qualname}() expected a value in {set_!r}"
             " for argument 'a', but got 'c' instead."
-        ).format(set_=set_, qualname=qualname(f))
+        ).format(
+            # We special-case set to show a tuple instead of the set repr.
+            set_=tuple(set_),
+            qualname=qualname(f),
+        )
         self.assertEqual(e.exception.args[0], expected_message)
 
     def test_expect_dtypes(self):

zipline/pipeline/classifiers/classifier.py

@@ -14,6 +14,7 @@ from zipline.pipeline.sentinels import NotSpecified
 from zipline.pipeline.term import ComputableTerm
 from zipline.utils.compat import unicode
 from zipline.utils.input_validation import expect_types
+from zipline.utils.memoize import classlazyval
 from zipline.utils.numpy_utils import (
     categorical_dtype,
     int64_dtype,
@@ -302,9 +303,9 @@ class Classifier(RestrictedDTypeMixin, ComputableTerm):
             raise AssertionError("Expected a LabelArray, got %s." % type(data))
         return data.as_categorical()
 
-    @property
+    @classlazyval
     def _downsampled_type(self):
-        return DownsampledClassifier
+        return DownsampledMixin.make_downsampled_type(Classifier)
 
 
 class Everything(Classifier):
@@ -391,18 +392,6 @@ class Latest(LatestMixin, CustomClassifier):
     pass
 
 
-class DownsampledClassifier(DownsampledMixin, Classifier):
-    """
-    A Classifier that defers to another Classifier at lower-than-daily
-    frequency.
-
-    Parameters
-    ----------
-    term : zipline.Classifier
-    freq : {'Y', 'Q', 'M', 'W'}
-    """
-
-
 class InvalidClassifierComparison(TypeError):
     def __init__(self, classifier, compval):
         super(InvalidClassifierComparison, self).__init__(

zipline/pipeline/downsample_helpers.py

@@ -0,0 +1,61 @@
+"""
+Helpers for downsampling code.
+"""
+from operator import attrgetter
+
+from zipline.utils.input_validation import expect_element
+from zipline.utils.numpy_utils import changed_locations
+from zipline.utils.sharedoc import (
+    templated_docstring,
+    PIPELINE_DOWNSAMPLING_FREQUENCY_DOC,
+)
+
+_dt_to_period = {
+    'year_start': attrgetter('year'),
+    'quarter_start': attrgetter('quarter'),
+    'month_start': attrgetter('month'),
+    'week_start': attrgetter('week'),
+}
+
+SUPPORTED_DOWNSAMPLE_FREQUENCIES = frozenset(_dt_to_period)
+
+
+expect_downsample_frequency = expect_element(
+    frequency=SUPPORTED_DOWNSAMPLE_FREQUENCIES,
+)
+
+
+@expect_downsample_frequency
+@templated_docstring(frequency=PIPELINE_DOWNSAMPLING_FREQUENCY_DOC)
+def select_sampling_indices(dates, frequency):
+    """
+    Choose entries from ``dates`` to use for downsampling at ``frequency``.
+
+    Parameters
+    ----------
+    dates : pd.DatetimeIndex
+        Dates from which to select sample choices.
+    {frequency}
+
+    Returns
+    -------
+    indices : np.array[int64]
+        An array condtaining indices of dates on which samples should be taken.
+
+        The resulting index will always include 0 as a sample index, and it
+        will include the first date of each subsequent year/quarter/month/week,
+        as determined by ``frequency``.
+
+    Notes
+    -----
+    This function assumes that ``dates`` does not have large gaps.
+
+    In particular, it assumes that the maximum distance between any two entries
+    in ``dates`` is never greater than a year, which we rely on because we use
+    ``np.diff(dates.<frequency>)`` to find dates where the sampling
+    period has changed.
+    """
+    return changed_locations(
+        _dt_to_period[frequency](dates),
+        include_first=True
+    )

zipline/pipeline/factors/factor.py

@@ -44,6 +44,7 @@ from zipline.pipeline.term import ComputableTerm, Term
 from zipline.utils.functional import with_doc, with_name
 from zipline.utils.input_validation import expect_types
 from zipline.utils.math_utils import nanmean, nanstd
+from zipline.utils.memoize import classlazyval
 from zipline.utils.numpy_utils import (
     bool_dtype,
     categorical_dtype,
@@ -1072,9 +1073,9 @@ class Factor(RestrictedDTypeMixin, ComputableTerm):
         """
         return (-inf < self) & (self < inf)
 
-    @property
+    @classlazyval
     def _downsampled_type(self):
-        return DownsampledFactor
+        return DownsampledMixin.make_downsampled_type(Factor)
 
 
 class NumExprFactor(NumericalExpression, Factor):
@@ -1515,17 +1516,6 @@ class Latest(LatestMixin, CustomFactor):
         out[:] = data[-1]
 
 
-class DownsampledFactor(DownsampledMixin, Factor):
-    """
-    A Factor that defers to another Factor at lower-than-daily frequency.
-
-    Parameters
-    ----------
-    term : zipline.pipeline.Factor
-    freq : {'Y', 'Q', 'M', 'W'}
-    """
-
-
 # Functions to be passed to GroupedRowTransform.  These aren't defined inline
 # because the transformation function is part of the instance hash key.
 def demean(row):

zipline/pipeline/filters/filter.py

@@ -33,6 +33,7 @@ from zipline.pipeline.mixins import (
 )
 from zipline.pipeline.term import ComputableTerm, Term
 from zipline.utils.input_validation import expect_types
+from zipline.utils.memoize import classlazyval
 from zipline.utils.numpy_utils import bool_dtype, repeat_first_axis
 
 
@@ -202,9 +203,9 @@ class Filter(RestrictedDTypeMixin, ComputableTerm):
             )
         return retval
 
-    @property
+    @classlazyval
     def _downsampled_type(self):
-        return DownsampledFilter
+        return DownsampledMixin.make_downsampled_type(Filter)
 
 
 class NumExprFilter(NumericalExpression, Filter):
@@ -463,17 +464,6 @@ class Latest(LatestMixin, CustomFilter):
     pass
 
 
-class DownsampledFilter(DownsampledMixin, Filter):
-    """
-    A Filter that defers to another Filter at lower-than-daily frequency.
-
-    Parameters
-    ----------
-    term : zipline.pipeline.Filter
-    freq : {'Y', 'Q', 'M', 'W'}
-    """
-
-
 class SingleAsset(Filter):
     """
     A Filter that computes to True only for the given asset.

zipline/pipeline/mixins.py

@@ -1,7 +1,7 @@
 """
 Mixins classes for use with Filters and Factors.
 """
-from operator import attrgetter
+from textwrap import dedent
 
 from numpy import (
     array,
@@ -17,10 +17,18 @@ from zipline.errors import (
     NoFurtherDataError,
 )
 from zipline.utils.control_flow import nullctx
-from zipline.utils.input_validation import expect_element, expect_types
-from zipline.utils.numpy_utils import changed_locations
+from zipline.utils.input_validation import expect_types
+from zipline.utils.sharedoc import (
+    format_docstring,
+    PIPELINE_DOWNSAMPLING_FREQUENCY_DOC,
+)
 from zipline.utils.pandas_utils import nearest_unequal_elements
 
+
+from .downsample_helpers import (
+    select_sampling_indices,
+    expect_downsample_frequency,
+)
 from .sentinels import NotSpecified
 from .term import Term
 
@@ -232,49 +240,6 @@ class LatestMixin(SingleInputMixin):
             )
 
 
-_dt_to_period = {
-    'Y': attrgetter('year'),
-    'Q': attrgetter('quarter'),
-    'M': attrgetter('month'),
-    'W': attrgetter('week'),
-}
-
-
-def select_sampling_indices(dates, frequency):
-    """
-    Choose entries from ``dates`` to use for downsampling at ``frequency``.
-
-    Parameters
-    ----------
-    dates : pd.DatetimeIndex
-        Dates from which to select sample choices.
-    frequency : {'Y', 'Q', 'M', 'W'}
-        Frequency at which samples are to be taken.
-
-    Returns
-    -------
-    indices : np.array[int64]
-        An array condtaining indices of dates on which samples should be taken.
-
-        The resulting index will always include 0 as a sample index, and it
-        will include the first date of each subsequent year/quarter/month/week,
-        as determined by ``frequency``.
-
-    Notes
-    -----
-    This function assumes that ``dates`` does not have large gaps.
-
-    In particular, it assumes that the maximum distance between any two entries
-    in ``dates`` is never greater than a year, which we rely on because we use
-    ``np.diff(dates.{quarter,month,week})`` to find dates where the sampling
-    period has changed.
-    """
-    return changed_locations(
-        _dt_to_period[frequency](dates),
-        include_first=True
-    )
-
-
 class DownsampledMixin(StandardOutputs):
     """
     Mixin for behavior shared by Downsampled{Factor,Filter,Classifier}
@@ -291,7 +256,7 @@ class DownsampledMixin(StandardOutputs):
     window_safe = False
 
     @expect_types(term=Term)
-    @expect_element(frequency=frozenset(_dt_to_period))
+    @expect_downsample_frequency
     def __new__(cls, term, frequency):
         return super(DownsampledMixin, cls).__new__(
             cls,
@@ -400,6 +365,17 @@ class DownsampledMixin(StandardOutputs):
 
         real_compute = self._wrapped_term._compute
 
+        # Inputs will contain different kinds of values depending on whether or
+        # not we're a windowed computation.
+
+        # If we're windowed, then `inputs` is a list of iterators of ndarrays.
+        # If we're not windowed, then `inputs` is just a list of ndarrays.
+        # There are two things we care about doing with the input:
+        #   1. Preparing an input to be passed to our wrapped term.
+        #   2. Skipping an input if we're going to use an already-computed row.
+        # We perform these actions differently based on the expected kind of
+        # input, and we encapsulate these actions with closures so that we
+        # don't clutter the code below with lots of branching.
         if self.windowed:
             # If we're windowed, inputs are stateful AdjustedArrays.  We don't
             # need to do any preparation before forwarding to real_compute, but
@@ -412,8 +388,8 @@ class DownsampledMixin(StandardOutputs):
                     next(w)
         else:
             # If we're not windowed, inputs are just ndarrays.  We need to
-            # slice off one row when forwarding to real_compute, but we don't
-            # need to do anything to skip an input.
+            # slice out a single row when forwarding to real_compute, but we
+            # don't need to do anything to skip an input.
             def prepare_inputs():
                 # i is the loop iteration variable below.
                 return [a[[i]] for a in inputs]
@@ -455,3 +431,31 @@ class DownsampledMixin(StandardOutputs):
 
         # Concatenate stored results.
         return vstack(results)
+
+    @classmethod
+    def make_downsampled_type(cls, other_base):
+        """
+        Factory for making Downsampled{Filter,Factor,Classifier}.
+        """
+        docstring = dedent(
+            """
+            A {t} that defers to another {t} at lower-than-daily frequency.
+
+            Parameters
+            ----------
+            term : {t}
+            {{frequency}}
+            """
+        ).format(t=other_base.__name__)
+
+        doc = format_docstring(
+            owner_name=other_base.__name__,
+            docstring=docstring,
+            formatters={'frequency': PIPELINE_DOWNSAMPLING_FREQUENCY_DOC},
+        )
+
+        return type(
+            'Downsampled' + other_base.__name__,
+            (cls, other_base,),
+            {'__doc__': doc},
+        )

zipline/pipeline/term.py

@@ -37,7 +37,12 @@ from zipline.utils.numpy_utils import (
     datetime64ns_dtype,
     default_missing_value_for_dtype,
 )
+from zipline.utils.sharedoc import (
+    templated_docstring,
+    PIPELINE_DOWNSAMPLING_FREQUENCY_DOC,
+)
 
+from .downsample_helpers import expect_downsample_frequency
 from .sentinels import NotSpecified
 
 
@@ -594,19 +599,15 @@ class ComputableTerm(Term):
             "for instances of %s." % type(self).__name__
         )
 
+    @expect_downsample_frequency
+    @templated_docstring(frequency=PIPELINE_DOWNSAMPLING_FREQUENCY_DOC)
     def downsample(self, frequency):
         """
         Make a term that computes from ``self`` at lower-than-daily frequency.
 
         Parameters
         ----------
-        frequency : str, {'Y', 'Q', 'M', 'W'}
-            A string indicating the desired sampling rate.
-            'Y' -> sample on the first trading day of each calendar year
-            'Q' -> sample on the first trading day of
-                   January, April, July, and October
-            'M' -> sample on the first trading day of each month
-            'W' -> sample on the first trading day of each week
+        {frequency}
         """
         return self._downsampled_type(term=self, frequency=frequency)
 

zipline/utils/input_validation.py

@@ -483,10 +483,17 @@ def expect_element(*_pos, **named):
         raise TypeError("expect_element() only takes keyword arguments.")
 
     def _expect_element(collection):
+        if isinstance(collection, (set, frozenset)):
+            # Special case the error message for set and frozen set to make it
+            # less verbose.
+            collection_for_error_message = tuple(sorted(collection))
+        else:
+            collection_for_error_message = collection
+
         template = (
             "%(funcname)s() expected a value in {collection} "
             "for argument '%(argname)s', but got %(actual)s instead."
-        ).format(collection=collection)
+        ).format(collection=collection_for_error_message)
         return make_check(
             ValueError,
             template,

zipline/utils/sharedoc.py

@@ -0,0 +1,90 @@
+"""
+Shared docstrings for parameters that should be documented identically
+across different functions.
+"""
+import re
+from six import iteritems
+from textwrap import dedent
+
+PIPELINE_DOWNSAMPLING_FREQUENCY_DOC = dedent(
+    """\
+    frequency : {'year_start', 'quarter_start', 'month_start', 'week_start'}
+        A string indicating desired sampling dates:
+
+        'year_start'    -> first trading day of each year
+        'quarter_start' -> first trading day of January, April, July, October
+        'month_start'   -> first trading day of each month
+        'week_start'    -> first trading_day of each week
+    """
+)
+
+
+def pad_lines(prefix, s):
+    """Apply a prefix to each line in s."""
+    return '\n'.join(prefix + line for line in s.splitlines())
+
+
+def format_docstring(owner_name, docstring, formatters):
+    """
+    Template ``formatters`` into ``docstring``.
+
+    Parameters
+    ----------
+    owner_name : str
+        The name of the function or class whose docstring is being templated.
+        Only used for error messages.
+    docstring : str
+        The docstring to template.
+    formatters : dict[str -> str]
+        Parameters for a a str.format() call on ``docstring``.
+
+        Multi-line values in ``formatters`` will have leading whitespace padded
+        to match the leading whitespace of the substitution string.
+    """
+    # Build a dict of parameters to a vanilla format() call by searching for
+    # each entry in **formatters and applying any leading whitespace to each
+    # line in the desired substitution.
+    format_params = {}
+    for target, doc_for_target in iteritems(formatters):
+        # Search for '{name}', with optional leading whitespace.
+        regex = re.compile('^(\s*)' + '({' + target + '})$', re.MULTILINE)
+        matches = regex.findall(docstring)
+        if not matches:
+            raise ValueError(
+                "Couldn't find template for parameter {!r} in docstring "
+                "for {}."
+                "\nParameter name must be alone on a line surrounded by "
+                "braces.".format(target, owner_name),
+            )
+        elif len(matches) > 1:
+            raise ValueError(
+                "Couldn't found multiple templates for parameter {!r}"
+                "in docstring for {}."
+                "\nParameter should only appear once.".format(
+                    target, owner_name
+                )
+            )
+
+        (leading_whitespace, _) = matches[0]
+        format_params[target] = pad_lines(leading_whitespace, doc_for_target)
+
+    return docstring.format(**format_params)
+
+
+def templated_docstring(**docs):
+    """
+    Decorator allowing the use of templated docstrings.
+
+    Usage
+    -----
+    >>> @templated_docstring(foo='bar')
+    ... def my_func(self, foo):
+    ...     '''{foo}'''
+    ...
+    >>> my_func.__doc__
+    'bar'
+    """
+    def decorator(f):
+        f.__doc__ = format_docstring(f.__name__, f.__doc__, docs)
+        return f
+    return decorator