Merge pull request #681 from quantopian/sanest-setup

Sanest setup

.travis.yml

@@ -1,42 +1,26 @@
 language: python
+sudo: false
+cache:
+  directories:
+    - $HOME/.cache/pip
 matrix:
   include:
   - python: 2.7
-    env:
-      - PANDAS_VERSION=0.16.0
-      - NUMPY_VERSION=1.9.2
+    env: PANDAS_VERSION=0.16.1 NUMPY_VERSION=1.9.2 SCIPY_VERSION=0.15.1
   - python: 3.3
-    env:
-      - PANDAS_VERSION=0.16.0
-      - NUMPY_VERSION=1.9.2
+    env: PANDAS_VERSION=0.16.1 NUMPY_VERSION=1.9.2 SCIPY_VERSION=0.15.1
+
 before_install:
   - if [ ${TRAVIS_PYTHON_VERSION:0:1} == "2" ]; then wget http://repo.continuum.io/miniconda/Miniconda-3.7.0-Linux-x86_64.sh -O miniconda.sh; else wget http://repo.continuum.io/miniconda/Miniconda3-3.7.0-Linux-x86_64.sh -O miniconda.sh; fi
   - chmod +x miniconda.sh
   - ./miniconda.sh -b
   - if [ ${TRAVIS_PYTHON_VERSION:0:1} == "2" ]; then export PATH=/home/travis/miniconda/bin:$PATH; else export PATH=/home/travis/miniconda3/bin:$PATH; fi
-  - sudo apt-get install gfortran
 install:
   - conda create -n testenv --yes pip python=$TRAVIS_PYTHON_VERSION
   - source activate testenv
-  - conda install --yes -c https://conda.binstar.org/Quantopian numpy=$NUMPY_VERSION pandas=$PANDAS_VERSION scipy matplotlib Cython patsy statsmodels tornado pyparsing xlrd mock pytz requests six dateutil ta-lib logbook
-  - grep bottleneck== etc/requirements.txt | xargs pip install
-  - grep cyordereddict== etc/requirements.txt | xargs pip install
-  - grep contextlib2== etc/requirements.txt | xargs pip install
-  - grep click== etc/requirements.txt | xargs pip install
-  - grep networkx== etc/requirements.txt | xargs pip install
-  - grep numexpr== etc/requirements.txt | xargs pip install
-  - grep bcolz== etc/requirements.txt | xargs pip install
-  - grep pyflakes== etc/requirements_dev.txt | xargs pip install
-  - grep pep8== etc/requirements_dev.txt | xargs pip install
-  - grep mccabe== etc/requirements_dev.txt | xargs pip install
-  - grep flake8== etc/requirements_dev.txt | xargs pip install
-  - grep nose== etc/requirements_dev.txt | xargs pip install --upgrade --force-reinstall
-  - grep nose-parameterized== etc/requirements_dev.txt | xargs pip install
-  - grep nose-ignore-docstring== etc/requirements_dev.txt | xargs pip install
-  - grep testfixtures== etc/requirements_dev.txt | xargs pip install
-  - pip install coveralls
-  - pip install nose-timer
-  - python setup.py build_ext --inplace
+  - conda install --yes -c https://conda.binstar.org/Quantopian numpy=$NUMPY_VERSION pandas=$PANDAS_VERSION scipy==$SCIPY_VERSION matplotlib Cython patsy statsmodels tornado pyparsing xlrd mock pytz requests six dateutil ta-lib logbook
+  - pip install --upgrade pip nose-timer coverage
+  - pip install -e .[dev]
 before_script:
   - "flake8 zipline tests"
 script:

etc/requirements_dev.txt

@@ -52,11 +52,6 @@ matplotlib==1.4.3
 
 Markdown==2.6.2
 
-# This --allow syntax is for compatibility with pip >= 1.5
-# However, this is backwards incompatible change, since previous
-# versions of pip do not support that flag.
-TA-Lib==0.4.9
-
 # Checking for old PIP packages
 futures==3.0.3
 requests-futures==0.9.5

setup.py

@@ -13,33 +13,177 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from __future__ import print_function
 
-from setuptools import setup, Extension
-from Cython.Build import cythonize
-import numpy as np
+import re
+import sys
+from operator import lt, gt, eq, le, ge
+from os.path import (
+    abspath,
+    dirname,
+    join,
+)
+from distutils.version import StrictVersion
+from setuptools import (
+    Extension,
+    find_packages,
+    setup,
+)
 
-ext_modules = [
-    Extension(
-        'zipline.assets._assets',
-        ['zipline/assets/_assets.pyx'],
-        include_dirs=[np.get_include()],
-    ),
-    Extension(
-        'zipline.lib.adjusted_array',
-        ['zipline/lib/adjusted_array.pyx'],
-        include_dirs=[np.get_include()],
-    ),
-    Extension(
-        'zipline.lib.adjustment',
-        ['zipline/lib/adjustment.pyx'],
-        include_dirs=[np.get_include()],
-    ),
-    Extension(
+
+class LazyCythonizingList(list):
+    cythonized = False
+
+    def lazy_cythonize(self):
+        if self.cythonized:
+            return
+        self.cythonized = True
+
+        from Cython.Build import cythonize
+        from numpy import get_include
+
+        self[:] = cythonize(
+            [
+                Extension(*ext_args, include_dirs=[get_include()])
+                for ext_args in self
+            ]
+        )
+
+    def __iter__(self):
+        self.lazy_cythonize()
+        return super(LazyCythonizingList, self).__iter__()
+
+    def __getitem__(self, num):
+        self.lazy_cythonize()
+        return super(LazyCythonizingList, self).__getitem__(num)
+
+
+ext_modules = LazyCythonizingList([
+    ('zipline.assets._assets', ['zipline/assets/_assets.pyx']),
+    ('zipline.lib.adjusted_array', ['zipline/lib/adjusted_array.pyx']),
+    ('zipline.lib.adjustment', ['zipline/lib/adjustment.pyx']),
+    (
         'zipline.data.ffc.loaders._us_equity_pricing',
-        ['zipline/data/ffc/loaders/_us_equity_pricing.pyx'],
-        include_dirs=[np.get_include()],
+        ['zipline/data/ffc/loaders/_us_equity_pricing.pyx']
     ),
-]
+])
+
+
+STR_TO_CMP = {
+    '<': lt,
+    '<=': le,
+    '=': eq,
+    '==': eq,
+    '>': gt,
+    '>=': ge,
+}
+
+
+def _filter_requirements(lines_iter):
+    for line in lines_iter:
+        line = line.strip()
+        if not line or line.startswith('#'):
+            continue
+
+        # pip install -r understands line with ;python_version<'3.0', but
+        # whatever happens inside extras_requires doesn't.  Parse the line
+        # manually and conditionally add it if needed.
+        if ';' not in line:
+            yield line
+            continue
+
+        requirement, version_spec = line.split(';')
+        try:
+            groups = re.match(
+                "(python_version)([<>=]{1,2})(')([0-9\.]+)(')(.*)",
+                version_spec,
+            ).groups()
+            comp = STR_TO_CMP[groups[1]]
+            version_spec = StrictVersion(groups[3])
+        except Exception as e:
+            # My kingdom for a 'raise from'!
+            raise AssertionError(
+                "Couldn't parse requirement line; '%s'\n"
+                "Error was:\n"
+                "%r" % (line, e)
+            )
+
+        sys_version = '.'.join(list(map(str, sys.version_info[:3])))
+        if comp(sys_version, version_spec):
+            yield requirement
+
+
+def read_requirements(path):
+    """
+    Read a requirements.txt file, expressed as a path relative to Zipline root.
+    """
+    real_path = join(dirname(abspath(__file__)), path)
+    with open(real_path) as f:
+        return list(_filter_requirements(f.readlines()))
+
+
+def install_requires():
+    return read_requirements('etc/requirements.txt')
+
+
+def extras_requires():
+    dev_reqs = read_requirements('etc/requirements_dev.txt')
+    talib_reqs = ['TA-Lib==0.4.9']
+    return {
+        'dev': dev_reqs,
+        'talib': talib_reqs,
+        'all': dev_reqs + talib_reqs,
+    }
+
+
+def module_requirements(requirements_path, module_names):
+    module_names = set(module_names)
+    found = set()
+    module_lines = []
+    parser = re.compile("([^=<>]+)([<=>]{1,2})(.*)")
+    for line in read_requirements(requirements_path):
+        match = parser.match(line)
+        if match is None:
+            raise AssertionError("Could not parse requirement: '%s'" % line)
+
+        groups = match.groups()
+        name = groups[0]
+        if name in module_names:
+            found.add(name)
+            module_lines.append(line)
+
+    if found != module_names:
+        raise AssertionError(
+            "No requirements found for %s." % module_names - found
+        )
+    return module_lines
+
+
+def pre_setup():
+    if not set(sys.argv) & {'install', 'develop', 'egg_info', 'bdist_wheel'}:
+        return
+
+    try:
+        import pip
+        if StrictVersion(pip.__version__) < StrictVersion('7.1.0'):
+            raise AssertionError(
+                "Zipline installation requires pip>=7.1.0, but your pip "
+                "version is {version}. \n"
+                "You can upgrade your pip with "
+                "'pip install --upgrade pip'.".format(
+                    version=pip.__version__,
+                )
+            )
+    except ImportError:
+        raise AssertionError("Zipline installation requires pip")
+
+    required = ('Cython', 'numpy')
+    for line in module_requirements('etc/requirements.txt', required):
+        pip.main(['install', line])
+
+
+pre_setup()
+
 
 setup(
     name='zipline',
@@ -47,8 +191,8 @@ setup(
     description='A backtester for financial algorithms.',
     author='Quantopian Inc.',
     author_email='opensource@quantopian.com',
-    packages=['zipline'],
-    ext_modules=cythonize(ext_modules),
+    packages=find_packages('.', include=['zipline', 'zipline.*']),
+    ext_modules=ext_modules,
     scripts=['scripts/run_algo.py'],
     include_package_data=True,
     license='Apache 2.0',
@@ -65,17 +209,7 @@ setup(
         'Topic :: Scientific/Engineering :: Information Analysis',
         'Topic :: System :: Distributed Computing',
     ],
-    install_requires=[
-        'Logbook',
-        'pytz',
-        'requests',
-        'numpy',
-        'pandas',
-        'six',
-        'Cython',
-    ],
-    extras_require={
-        'talib':  ["talib"],
-    },
+    install_requires=install_requires(),
+    extras_require=extras_requires(),
     url="http://zipline.io"
 )