Changes to build to fix creation of wheels. (#840)

* Pass DPYTHON_EXECUTABLE into cmake for arrow and for ray.

* Add cython to setup.py install_requires.

* Revert custom code for finding python in cmake.

* Correctly find arrow on CentOS.

* In cmake, don't find PythonLibs, just find PYTHON_INCLUDE_DIRS.

* Fix typo.

* Do not use boost shared libraries when building arrow.

* Add six to the setup.py install_requires because it is needed by pyarrow.

* Don't link numbuf against boost_system and boost_filesystem.

* Compile boost when we are on Linux.

* Make numbuf find the correct boost libraries.

* Only use find_package Boost on Linux, suppress output when building boost.

* Changes to wheel building scripts, install cython in mac script.

* Compile flatbuffers ourselves on Linux and pass it in when compiling Arrow.

* Clean up build_flatbuffers.sh and build_boost.sh scripts a little.

* Install cython when building linux wheel.

python/build-wheel-macos.sh

@@ -44,21 +44,20 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do
   curl $MACPYTHON_URL/$PY_VERSION/$PY_INST > $INST_PATH
   sudo installer -pkg $INST_PATH -target /
 
-  # Create a link from "python" to the actual Python executable so that the
-  # Python on the path that Ray finds is the correct version.
-  if [ ! -e $MACPYTHON_PY_PREFIX/$PY_MM/bin/python ]; then
-    ln -s $MACPYTHON_PY_PREFIX/$PY_MM/bin/python$PY_MM $MACPYTHON_PY_PREFIX/$PY_MM/bin/python
-  fi
-  PYTHON_EXE=$MACPYTHON_PY_PREFIX/$PY_MM/bin/python
+  PYTHON_EXE=$MACPYTHON_PY_PREFIX/$PY_MM/bin/python$PY_MM
   PIP_CMD="$(dirname $PYTHON_EXE)/pip$PY_MM"
 
   pushd python
+    # Install setuptools_scm because otherwise when building the wheel for
+    # Python 3.6, we see an error.
+    $PIP_CMD install setuptools_scm
     # Fix the numpy version because this will be the oldest numpy version we can
     # support.
-    $PIP_CMD install numpy==1.10.4
+    $PIP_CMD install numpy==1.10.4 cython
     # Install wheel to avoid the error "invalid command 'bdist_wheel'".
     $PIP_CMD install wheel
-    # Add the correct Python to the path and build the wheel.
+    # Add the correct Python to the path and build the wheel. This is only
+    # needed so that the installation finds the cython executable.
     PATH=$MACPYTHON_PY_PREFIX/$PY_MM/bin:$PATH $PYTHON_EXE setup.py bdist_wheel
     mv dist/*.whl ../.whl/
   popd

python/build-wheel-manylinux1.sh

@@ -2,7 +2,7 @@
 
 cat << EOF > "/usr/bin/nproc"
 #!/bin/bash
-echo 1
+echo 10
 EOF
 chmod +x /usr/bin/nproc
 
@@ -15,7 +15,7 @@ for PYTHON in cp27-cp27mu cp33-cp33m cp34-cp34m cp35-cp35m cp36-cp36m; do
   pushd python
     # Fix the numpy version because this will be the oldest numpy version we can
     # support.
-    /opt/python/${PYTHON}/bin/pip install numpy==1.10.4
+    /opt/python/${PYTHON}/bin/pip install numpy==1.10.4 cython
     PATH=/opt/python/${PYTHON}/bin:$PATH /opt/python/${PYTHON}/bin/python setup.py bdist_wheel
     # In the future, run auditwheel here.
     mv dist/*.whl ../.whl/

python/setup.py

@@ -81,12 +81,15 @@ setup(name="ray",
       # The BinaryDistribution argument triggers build_ext.
       distclass=BinaryDistribution,
       install_requires=["numpy",
+                        "cython",
                         "funcsigs",
                         "click",
                         "colorama",
                         "psutil",
                         "redis",
                         "cloudpickle >= 0.2.2",
+                        # The six module is required by pyarrow.
+                        "six >= 1.0.0",
                         "flatbuffers"],
       entry_points={"console_scripts": ["ray=ray.scripts.scripts:main"]},
       include_package_data=True,