Merge pull request #539 from quantopian/pypi_update2

Upgrade PyPI packages, remove hard-coded Cython version in setup.py, Flake8 changes

etc/requirements.txt

@@ -1,30 +1,30 @@
 # Logging
-Logbook==0.6.0
+Logbook==0.9.0
 
 # Scientific Libraries
 
-pytz==2014.4
-numpy==1.8.1
+pytz==2014.10
+numpy==1.9.2
 
 # scipy and pandas are required for statsmodels,
 # statsmodels in turn is required for some pandas packages
-scipy==0.12.0
-pandas==0.12.0
+scipy==0.15.1
+pandas==0.15.2
 # Needed for parts of pandas.stats
-patsy==0.2.1
-statsmodels==0.5.0
+patsy==0.3.0
+statsmodels==0.6.1
 
-python-dateutil==2.2
-six==1.6.1
+# Don't use 2.4.0.
+python-dateutil==2.4.1
+six==1.9.0
 
 # For fetching remote data
-requests==2.5.1
+requests==2.5.3
 
-Cython==0.20.1
+Cython==0.22
 
 # faster OrderedDict
 cyordereddict==0.2.2
 
-# faster array ops. Note once numpy gets to 1.9.1 we
-# we can bump to 1.0.0
-bottleneck==0.8.0
+# faster array ops.
+bottleneck==1.0.0

etc/requirements_dev.txt

@@ -1,15 +1,15 @@
 # Testing
-nose==1.3.3
-nose-parameterized==0.3.3
+nose==1.3.4
+nose-parameterized==0.3.5
 nose-ignore-docstring==0.2
 xlrd==0.9.3
 mock==1.0.1
 
 # Linting
 
-flake8==2.1.0
-mccabe==0.2.1
-pep8==1.5.7
+flake8==2.3.0
+mccabe==0.3
+pep8==1.6.2
 pyflakes==0.8.1
 
 # Documentation Conversion
@@ -21,12 +21,15 @@ mistune==0.5
 
 # Example scripts that are run during unit tests use the following:
 
-matplotlib==1.3.1
+matplotlib==1.4.3
 # tornado and pyparsing are required by matplotlib
-tornado==3.2.1
-pyparsing==2.0.2
+tornado==4.1
+pyparsing==2.0.3
+# Required by tornado
+backports.ssl-match-hostname==3.4.0.2; python_version < '3.0'
+certifi==14.5.14
 
-Markdown==2.4.1
+Markdown==2.6
 
 # This --allow syntax is for compatibility with pip >= 1.5
 # However, this is backwards incompatible change, since previous

scripts/generate_new_sample_saved_state.py

@@ -19,7 +19,7 @@ import pickle
 import pytz
 
 import sys
-sys.path.insert(0, '.')
+sys.path.insert(0, '.')  # noqa
 
 from zipline.finance.blotter import Blotter, Order
 from zipline.finance.commission import PerShare, PerTrade, PerDollar

setup.py

@@ -57,7 +57,7 @@ setup(
         'numpy',
         'pandas',
         'six',
-        'Cython==0.20.1'
+        'Cython',
     ],
     extras_require={
         'talib':  ["talib"],

tests/history_cases.py

@@ -576,10 +576,10 @@ HISTORY_CONTAINER_TEST_CASES = {
                 )
                 for key in [spec.key_str for spec in MIXED_FIELDS_SPECS
                             if spec.field not in HistorySpec.FORWARD_FILLABLE]
-            ]
+            ] +
 
-            +  # Concatenate the expected results for non-ffillable with
-               # expected result for ffillable.
+            # Concatenate the expected results for non-ffillable with
+            # expected result for ffillable.
             [
                 (
                     # Forward-fillable fields

tests/serialization_cases.py

@@ -32,7 +32,8 @@ def stringify_cases(cases, func=None):
     # get better test case names
     results = []
     if func is None:
-        func = lambda case: case[0].__name__
+        def func(case):
+            return case[0].__name__
     for case in cases:
         new_case = list(case)
         key = func(case)

tests/test_algorithm.py

@@ -252,7 +252,8 @@ class TestMiscellaneousAPI(TestCase):
         ('minute'),
     ])
     def test_schedule_funtion_rule_creation(self, mode):
-        nop = lambda *args, **kwargs: None
+        def nop(*args, **kwargs):
+            return None
 
         self.sim_params.data_frequency = mode
         algo = TradingAlgorithm(

tests/test_examples.py

@@ -19,7 +19,6 @@
 # Disable plotting
 #
 import matplotlib
-matplotlib.use('Agg')
 
 import os
 from os import path
@@ -33,6 +32,8 @@ except ImportError:
 import fnmatch
 import imp
 
+matplotlib.use('Agg')
+
 
 def test_examples():
     os.chdir(example_dir())

tests/test_finance.py

@@ -270,8 +270,8 @@ class FinanceTestCase(TestCase):
                    zipline.protocol.DATASOURCE_TYPE.BENCHMARK,
                    'source_id': 'benchmarks'})
             for dt, ret in trading.environment.benchmark_returns.iteritems()
-            if dt.date() >= sim_params.period_start.date()
-            and dt.date() <= sim_params.period_end.date()
+            if dt.date() >= sim_params.period_start.date() and
+            dt.date() <= sim_params.period_end.date()
         ]
 
         generated_events = date_sorted_sources(generated_trades,

tests/test_perf_tracking.py

@@ -135,8 +135,8 @@ def benchmark_events_in_range(sim_params):
                # any other events.
                'source_id': '1Abenchmarks'})
         for dt, ret in trading.environment.benchmark_returns.iteritems()
-        if dt.date() >= sim_params.period_start.date()
-        and dt.date() <= sim_params.period_end.date()
+        if dt.date() >= sim_params.period_start.date() and
+        dt.date() <= sim_params.period_end.date()
     ]
 
 

tests/test_rolling_panel.py

@@ -98,7 +98,8 @@ class TestRollingPanel(unittest.TestCase):
 
         To keep the same api, make sure that the raw option returns a copy too.
         """
-        data_id = lambda values: values.__array_interface__['data']
+        def data_id(values):
+            return values.__array_interface__['data']
 
         items = ('a', 'b')
         sids = (1, 2)

tests/test_transforms_talib.py

@@ -44,8 +44,8 @@ class TestTALIB(TestCase):
         BLACKLIST = ['make_transform', 'BatchTransform',
                      # TODO: Figure out why MAVP generates a KeyError
                      'MAVP']
-        names = [name for name in dir(ta) if name[0].isupper()
-                 and name not in BLACKLIST]
+        names = [name for name in dir(ta)
+                 if name[0].isupper() and name not in BLACKLIST]
 
         for name in names:
             print(name)
@@ -81,8 +81,8 @@ class TestTALIB(TestCase):
                 expected_result = np.array([e[-1] for e in expected_result])
             else:
                 expected_result = np.array(expected_result[-1])
-            if not (np.all(np.isnan(zipline_result))
-                    and np.all(np.isnan(expected_result))):
+            if not (np.all(np.isnan(zipline_result)) and
+                    np.all(np.isnan(expected_result))):
                 self.assertTrue(np.allclose(zipline_result, expected_result))
             else:
                 print('--- NAN')

tests/utils/test_events.py

@@ -222,9 +222,9 @@ class RuleTestCase(TestCase):
 
         dem = {
             k for k, v in iteritems(vars(zipline.utils.events))
-            if isinstance(v, type)
-            and issubclass(v, self.class_)
-            and v is not self.class_
+            if isinstance(v, type) and
+            issubclass(v, self.class_) and
+            v is not self.class_
         }
         ds = {
             k[5:] for k in dir(self)

zipline/algorithm.py

@@ -325,11 +325,13 @@ class TradingAlgorithm(object):
 
         if self.benchmark_return_source is None:
             env = trading.environment
-            if (sim_params.data_frequency == 'minute'
-                    or sim_params.emission_rate == 'minute'):
-                update_time = lambda date: env.get_open_and_close(date)[1]
+            if sim_params.data_frequency == 'minute' or \
+               sim_params.emission_rate == 'minute':
+                def update_time(date):
+                    return env.get_open_and_close(date)[1]
             else:
-                update_time = lambda date: date
+                def update_time(date):
+                    return date
             benchmark_return_source = [
                 Event({'dt': update_time(dt),
                        'returns': ret,
@@ -337,8 +339,8 @@ class TradingAlgorithm(object):
                        'source_id': 'benchmarks'})
                 for dt, ret in
                 trading.environment.benchmark_returns.iteritems()
-                if dt.date() >= sim_params.period_start.date()
-                and dt.date() <= sim_params.period_end.date()
+                if dt.date() >= sim_params.period_start.date() and
+                dt.date() <= sim_params.period_end.date()
             ]
         else:
             benchmark_return_source = self.benchmark_return_source

zipline/data/loader.py

@@ -185,19 +185,13 @@ Fetching data from Yahoo Finance.
     # and including the current day, the difference would still be 1.
     if len(days_up_to_now) - last_bm_date_offset > 2:
         benchmark_returns = update_benchmarks(bm_symbol, last_bm_date)
-        if (
-            benchmark_returns.index.tz is None
-            or
-            benchmark_returns.index.tz.zone != 'UTC'
-        ):
+        if benchmark_returns.index.tz is None or \
+           benchmark_returns.index.tz.zone != 'UTC':
             benchmark_returns = benchmark_returns.tz_localize('UTC')
     else:
         benchmark_returns = saved_benchmarks
-        if (
-            benchmark_returns.index.tz is None
-            or
-            benchmark_returns.index.tz.zone != 'UTC'
-        ):
+        if benchmark_returns.index.tz is None or\
+           benchmark_returns.index.tz.zone != 'UTC':
             benchmark_returns = benchmark_returns.tz_localize('UTC')
 
     # Get treasury curve module, filename & source from mapping.

zipline/finance/blotter.py

@@ -31,14 +31,14 @@ from zipline.finance.slippage import (
     check_order_triggers
 )
 from zipline.finance.commission import PerShare
-
-log = Logger('Blotter')
-
 from zipline.utils.protocol_utils import Enum
+
 from zipline.utils.serialization_utils import (
     VERSION_LABEL
 )
 
+log = Logger('Blotter')
+
 ORDER_STATUS = Enum(
     'OPEN',
     'FILLED',
@@ -217,7 +217,8 @@ class Blotter(object):
 
         # remove closed orders. we should only have to check
         # processed orders
-        not_open = lambda order: not order.open
+        def not_open(order):
+            return not order.open
         closed_orders = filter(not_open, processed_orders)
         for order in closed_orders:
             orders.remove(order)
@@ -241,8 +242,8 @@ class Blotter(object):
 
                 order.filled += txn.amount
                 if txn.commission is not None:
-                    order.commission = ((order.commission or 0.0)
-                                        + txn.commission)
+                    order.commission = ((order.commission or 0.0) +
+                                        txn.commission)
 
             # mark the date of the order to match the transaction
             # that is filling it.

zipline/finance/performance/tracker.py

@@ -317,11 +317,8 @@ class PerformanceTracker(object):
             pass
 
         elif event.type == zp.DATASOURCE_TYPE.BENCHMARK:
-            if (
-                self.sim_params.data_frequency == 'minute'
-                and
-                self.sim_params.emission_rate == 'daily'
-            ):
+            if self.sim_params.data_frequency == 'minute' and \
+               self.sim_params.emission_rate == 'daily':
                 # Minute data benchmarks should have a timestamp of market
                 # close, so that calculations are triggered at the right time.
                 # However, risk module uses midnight as the 'day'

zipline/finance/risk/cumulative.py

@@ -62,17 +62,15 @@ def information_ratio(algo_volatility, algorithm_return, benchmark_return):
     if zp_math.tolerant_equals(algo_volatility, 0):
         return np.nan
 
-    return (
-        (algorithm_return - benchmark_return)
-        # The square of the annualization factor is in the volatility,
-        # because the volatility is also annualized,
-        # i.e. the sqrt(annual factor) is in the volatility's numerator.
-        # So to have the the correct annualization factor for the
-        # Sharpe value's numerator, which should be the sqrt(annual factor).
-        # The square of the sqrt of the annual factor, i.e. the annual factor
-        # itself, is needed in the numerator to factor out the division by
-        # its square root.
-        / algo_volatility)
+    # The square of the annualization factor is in the volatility,
+    # because the volatility is also annualized,
+    # i.e. the sqrt(annual factor) is in the volatility's numerator.
+    # So to have the the correct annualization factor for the
+    # Sharpe value's numerator, which should be the sqrt(annual factor).
+    # The square of the sqrt of the annual factor, i.e. the annual factor
+    # itself, is needed in the numerator to factor out the division by
+    # its square root.
+    return (algorithm_return - benchmark_return) / algo_volatility
 
 
 class RiskMetricsCumulative(object):
@@ -297,8 +295,7 @@ algorithm_returns ({algo_count}) in range {start} : {end} on {dt}"
             self.daily_treasury[treasury_end] = treasury_period_return
         self.treasury_period_return = self.daily_treasury[treasury_end]
         self.excess_returns[self.latest_dt] = (
-            self.algorithm_cumulative_returns[self.latest_dt]
-            -
+            self.algorithm_cumulative_returns[self.latest_dt] -
             self.treasury_period_return)
         self.metrics.beta[dt] = self.calculate_beta()
         self.metrics.alpha[dt] = self.calculate_alpha()
@@ -376,8 +373,7 @@ algorithm_returns ({algo_count}) in range {start} : {end} on {dt}"
         # exceed the previous max_drawdown iff the current return is lower than
         # the previous low in the current drawdown window.
         cur_drawdown = 1.0 - (
-            (1.0 + self.algorithm_cumulative_returns[self.latest_dt])
-            /
+            (1.0 + self.algorithm_cumulative_returns[self.latest_dt]) /
             (1.0 + self.current_max))
 
         self.drawdowns[self.latest_dt] = cur_drawdown

zipline/finance/risk/period.py

@@ -101,10 +101,8 @@ class RiskMetricsPeriod(object):
             index=self.algorithm_returns.index)
         for dt, ret in self.algorithm_returns.iteritems():
             self.mean_algorithm_returns[dt] = (
-                self.algorithm_returns[:dt].sum()
-                /
-                self.trading_day_counts[dt]
-            )
+                self.algorithm_returns[:dt].sum() /
+                self.trading_day_counts[dt])
 
         self.benchmark_volatility = self.calculate_volatility(
             self.benchmark_returns)

zipline/finance/risk/risk.py

@@ -151,11 +151,8 @@ def information_ratio(algorithm_returns, benchmark_returns):
 
     relative_deviation = relative_returns.std(ddof=1)
 
-    if (
-        zp_math.tolerant_equals(relative_deviation, 0)
-        or
-        np.isnan(relative_deviation)
-    ):
+    if zp_math.tolerant_equals(relative_deviation, 0) or \
+       np.isnan(relative_deviation):
         return 0.0
 
     return np.mean(relative_returns) / relative_deviation

zipline/gens/tradesimulation.py

@@ -157,8 +157,8 @@ class AlgorithmSimulator(object):
                                 # If at the end of backtest history,
                                 # skip advancing market close.
                                 pass
-                            if (self.algo.perf_tracker.emission_rate
-                                    == 'minute'):
+                            if self.algo.perf_tracker.emission_rate == \
+                               'minute':
                                 self.algo.perf_tracker\
                                          .handle_intraday_market_close(
                                              mkt_open,
@@ -170,9 +170,8 @@ class AlgorithmSimulator(object):
                     elif data_frequency == 'daily':
                         next_day = trading.environment.next_trading_day(date)
 
-                        if (next_day is not None
-                                and next_day
-                                < self.algo.perf_tracker.last_close):
+                        if next_day is not None and \
+                           next_day < self.algo.perf_tracker.last_close:
                             self._call_before_trading_start(next_day)
 
                     self.algo.portfolio_needs_update = True

zipline/history/history.py

@@ -192,9 +192,9 @@ class Frequency(object):
         """
         if self.unit_str == 'd':
             if self.data_frequency == 'minute':
-                func = lambda dt: env.get_open_and_close(
-                    env.previous_trading_day(dt),
-                )[1]
+                def func(dt):
+                    return env.get_open_and_close(
+                        env.previous_trading_day(dt))[1]
             else:
                 func = env.previous_trading_day
         else:

zipline/history/history_container.py

@@ -179,9 +179,9 @@ class HistoryContainerDelta(HistoryContainerDeltaSuper):
         """
         Checks if the delta is empty.
         """
-        return (self.field is None
-                and self.frequency_delta is None
-                and self.length_delta is None)
+        return (self.field is None and
+                self.frequency_delta is None and
+                self.length_delta is None)
 
 
 def normalize_to_data_freq(data_frequency, dt):

zipline/test_algorithms.py

@@ -80,7 +80,12 @@ from six.moves import range
 from six import itervalues
 
 from zipline.algorithm import TradingAlgorithm
-from zipline.api import FixedSlippage
+from zipline.api import (
+    FixedSlippage,
+    order,
+    set_slippage,
+    record,
+)
 from zipline.errors import UnsupportedOrderParameters
 from zipline.finance.execution import (
     LimitOrder,
@@ -88,6 +93,7 @@ from zipline.finance.execution import (
     StopLimitOrder,
     StopOrder,
 )
+from zipline.transforms import BatchTransform, batch_transform
 
 
 class TestAlgorithm(TradingAlgorithm):
@@ -364,8 +370,8 @@ class TestOrderPercentAlgorithm(TradingAlgorithm):
 
         self.order_percent(0, .001)
         self.target_shares += np.floor((.001 *
-                                        self.portfolio.portfolio_value)
-                                       / data[0].price)
+                                        self.portfolio.portfolio_value) /
+                                       data[0].price)
 
 
 class TestTargetPercentAlgorithm(TradingAlgorithm):
@@ -448,9 +454,6 @@ class SetLongOnlyAlgorithm(TradingAlgorithm):
         self.set_long_only()
 
 
-from zipline.transforms import BatchTransform, batch_transform
-
-
 class TestRegisterTransformAlgorithm(TradingAlgorithm):
     def initialize(self, *args, **kwargs):
         self.set_slippage(FixedSlippage())
@@ -870,10 +873,6 @@ class InvalidOrderAlgorithm(TradingAlgorithm):
 
 ##############################
 # Quantopian style algorithms
-from zipline.api import (order,
-                         set_slippage,
-                         record)
-
 
 # Noop algo
 def initialize_noop(context):

zipline/transforms/ta.py

@@ -95,8 +95,7 @@ def make_transform(talib_fn, name):
     mappings = '\n'.join('        {0} : {1}'.format(k, v)
                          for k, v in talib_fn.input_names.items())
     divider2 = '\n\n#---- Zipline docs\n'
-    help_str = (header + talib_docs + divider1 + mappings
-                + divider2)
+    help_str = header + talib_docs + divider1 + mappings + divider2
 
     class TALibTransform(BatchTransform):
         __doc__ = help_str + """

zipline/transforms/utils.py

@@ -130,10 +130,9 @@ class StatefulTransform(object):
         # messages should only manipulate copies.
         for message in stream_in:
             # we only handle TRADE and CUSTOM events.
-            if (hasattr(message, 'type')
-                    and message.type not in (
-                        DATASOURCE_TYPE.TRADE,
-                        DATASOURCE_TYPE.CUSTOM)):
+            if hasattr(message, 'type') and \
+               message.type not in (DATASOURCE_TYPE.TRADE,
+                                    DATASOURCE_TYPE.CUSTOM):
                 yield message
                 continue
             # allow upstream generators to yield None to avoid

zipline/utils/argcheck.py

@@ -79,10 +79,10 @@ class Argument(namedtuple('Argument', ['name', 'default'])):
 
     def _defaults_match(self, arg):
         return any(map(Argument.ignore_default, [self, arg])) \
-            or (self.default is Argument.any_default
-                and arg.default is not Argument.no_default) \
-            or (arg.default is Argument.any_default
-                and self.default is not Argument.no_default) \
+            or (self.default is Argument.any_default and
+                arg.default is not Argument.no_default) \
+            or (arg.default is Argument.any_default and
+                self.default is not Argument.no_default) \
             or self.default == arg.default
 
     def _names_match(self, arg):

zipline/utils/events.py

@@ -259,8 +259,8 @@ class ComposedRule(StatelessRule):
     expected.
     """
     def __init__(self, first, second, composer):
-        if not (isinstance(first, StatelessRule)
-                and isinstance(second, StatelessRule)):
+        if not (isinstance(first, StatelessRule) and
+                isinstance(second, StatelessRule)):
             raise ValueError('Only two StatelessRules can be composed')
 
         self.first = first

zipline/utils/protocol_utils.py

@@ -24,5 +24,7 @@ def Enum(*options):
     """
     class cstruct(Structure):
         _fields_ = [(o, c_ubyte) for o in options]
-        __iter__ = lambda s: iter(range(len(options)))
+
+        def __iter__(s):
+            return iter(range(len(options)))
     return cstruct(*range(len(options)))